remove readinessProbe, livenessProbe, startupProbe before enter proxy mode, and restore it after exit proxy mode

2025-12-24 11:51:13 +08:00 · 2022-02-01 16:14:44 +08:00
parent 09c26d281f
commit 28dcade734
7 changed files with 155 additions and 42 deletions
--- a/go.mod
+++ b/go.mod
@@ -84,7 +84,6 @@ require (
 	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
-	golang.zx2c4.com/go118/netip v0.0.0-20211111135330-a4a02eeacf9d // indirect
 	golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224 // indirect
 	google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
--- a/go.sum
+++ b/go.sum
@@ -799,8 +799,6 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa h1:idItI2DDfCokpg0N51B2VtiLdJ4vAuXC9fnCb2gACo4=
 golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -876,8 +874,6 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210927181540-4e4d966f7476 h1:s5hu7bTnLKswvidgtqc4GwsW83m9LZu8UAqzmWOZtI4=
-golang.org/x/net v0.0.0-20210927181540-4e4d966f7476/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211111083644-e5c967477495 h1:cjxxlQm6d4kYbhpZ2ghvmI8xnq0AG+jXmzrhzfkyu5A=
 golang.org/x/net v0.0.0-20211111083644-e5c967477495/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -969,8 +965,7 @@ golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo=
-golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 h1:WecRHqgE09JBkh/584XIE6PMz5KKE/vER4izNUi30AQ=
 golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
@@ -1040,13 +1035,10 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.zx2c4.com/go118/netip v0.0.0-20211111135330-a4a02eeacf9d h1:9+v0G0naRhLPOJEeJOL6NuXTtAHHwmkyZlgQJ0XcQ8I=
 golang.zx2c4.com/go118/netip v0.0.0-20211111135330-a4a02eeacf9d/go.mod h1:5yyfuiqVIJ7t+3MqrpTQ+QqRkMWiESiyDvPNvKYCecg=
 golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224 h1:Ug9qvr1myri/zFN6xL17LSCBGFDnphBBhzmILHsM5TY=
 golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
 golang.zx2c4.com/wireguard v0.0.0-20210905140043-2ef39d47540c/go.mod h1:laHzsbfMhGSobUmruXWAyMKKHSqvIcrqZJMyHD+/3O8=
-golang.zx2c4.com/wireguard v0.0.0-20210927201915-bb745b2ea326 h1:4yQQ5d6U5ozGB6n/WSDZa6B0XpPTmoQMtMDMoiZr4n0=
-golang.zx2c4.com/wireguard v0.0.0-20210927201915-bb745b2ea326/go.mod h1:SDoazCvdy7RDjBPNEMBwrXhomlmtG7svs8mgwWEqtVI=
 golang.zx2c4.com/wireguard v0.0.0-20211209221555-9c9e7e272434 h1:3zl8RkJNQ8wfPRomwv/6DBbH2Ut6dgMaWTxM0ZunWnE=
 golang.zx2c4.com/wireguard v0.0.0-20211209221555-9c9e7e272434/go.mod h1:TjUWrnD5ATh7bFvmm/ALEJZQ4ivKbETb6pmyj1vUoNI=
 golang.zx2c4.com/wireguard/windows v0.4.10 h1:HmjzJnb+G4NCdX+sfjsQlsxGPuYaThxRbZUZFLyR0/s=
--- a/pkg/connect.go
+++ b/pkg/connect.go
@@ -94,7 +94,7 @@ func (c *ConnectOptions) DoConnect() (err error) {
 		return
 	}
 	trafficMangerNet := net.IPNet{IP: util.RouterIP, Mask: util.CIDR.Mask}
-	c.routerIP, err = CreateOutboundRouterPod(c.clientset, c.Namespace, trafficMangerNet.String(), c.cidrs)
+	c.routerIP, err = CreateOutboundPod(c.clientset, c.Namespace, trafficMangerNet.String(), c.cidrs)
 	if err != nil {
 		return
 	}
@@ -141,9 +141,21 @@ func (c *ConnectOptions) portForward(ctx context.Context) error {
 					errChan <- err
 				}
 				first = false
+				// exit normal, let context.err to judge to exit or not
+				if err == nil {
+					return
+				}
 				if apierrors.IsNotFound(err) {
-					log.Errorf("can not found port-forward resource, err: %v, exiting", err)
-				} else if err != nil {
+					log.Errorln("can not found outbound pod, try to create one")
+					tm := net.IPNet{IP: util.RouterIP, Mask: util.CIDR.Mask}
+					if _, err = CreateOutboundPod(c.clientset, c.Namespace, tm.String(), c.cidrs); err != nil {
+						log.Errorf("error while create traffic manager, will retry after a snap, err: %v", err)
+					}
+				} else if strings.Contains(err.Error(), "unable to listen on any of the requested ports") ||
+					strings.Contains(err.Error(), "address already in use") {
+					log.Errorf("port 10800 already in use, needs to release it manually")
+					time.Sleep(time.Second * 5)
+				} else {
 					log.Errorf("port-forward occurs error, err: %v, retrying", err)
 				}
 				time.Sleep(time.Second * 2)
--- a/pkg/envoy.go
+++ b/pkg/envoy.go
@@ -35,11 +35,13 @@ func PatchSidecar(factory cmdutil.Factory, clientset *kubernetes.Clientset, name
 	}

 	u := object.Object.(*unstructured.Unstructured)
-	templateSpec, depth, err := util.GetPodTemplateSpecPath(u)
+	templateSpec, path, err := util.GetPodTemplateSpecPath(u)
 	if err != nil {
 		return err
 	}

+	origin := *templateSpec
+
 	port := uint32(templateSpec.Spec.Containers[0].Ports[0].ContainerPort)
 	configMapName := fmt.Sprintf("%s-%s", object.Mapping.Resource.Resource, object.Name)

@@ -58,7 +60,7 @@ func PatchSidecar(factory cmdutil.Factory, clientset *kubernetes.Clientset, name
 		Value interface{} `json:"value"`
 	}{{
 		Op:    "replace",
-		Path:  "/" + strings.Join(append(depth, "spec"), "/"),
+		Path:  "/" + strings.Join(append(path, "spec"), "/"),
 		Value: templateSpec.Spec,
 	}})
 	if err != nil {
@@ -69,6 +71,13 @@ func PatchSidecar(factory cmdutil.Factory, clientset *kubernetes.Clientset, name
 		//Force: &t,
 	})

+	removePatch, restorePatch := patch(origin, path)
+	_, err = helper.Patch(object.Namespace, object.Name, types.JSONPatchType, removePatch, &metav1.PatchOptions{})
+	if err != nil {
+		log.Warnf("error while remove probe of resource: %s %s, ignore, err: %v",
+			object.Mapping.GroupVersionKind.GroupKind().String(), object.Name, err)
+	}
+
 	//_ = util.WaitPod(clientset, namespace, metav1.ListOptions{
 	//	FieldSelector: fields.OneTermEqualSelector("metadata.name", object.Name+"-shadow").String(),
 	//}, func(pod *v1.Pod) bool {
@@ -78,6 +87,10 @@ func PatchSidecar(factory cmdutil.Factory, clientset *kubernetes.Clientset, name
 		if err = UnPatchContainer(factory, clientset, namespace, workloads, headers); err != nil {
 			log.Error(err)
 		}
+		if _, err = helper.Patch(object.Namespace, object.Name, types.JSONPatchType, restorePatch, &metav1.PatchOptions{}); err != nil {
+			log.Warnf("error while restore probe of resource: %s %s, ignore, err: %v",
+				object.Mapping.GroupVersionKind.GroupKind().String(), object.Name, err)
+		}
 	})
 	_ = util.RolloutStatus(factory, namespace, workloads, time.Minute*5)
 	return err
--- a/pkg/exchange/controller.go
+++ b/pkg/exchange/controller.go
@@ -19,15 +19,15 @@ func RemoveContainer(spec *v1.PodSpec) {
 }

 func AddContainer(spec *v1.PodSpec, c util.PodRouteConfig) {
-	var result []v1.Container
-	for _, container := range spec.Containers {
-		if container.Name != VPN {
-			result = append(result, container)
+	// remove vpn container is already exist
+	for i := 0; i < len(spec.Containers); i++ {
+		if spec.Containers[i].Name == VPN {
+			spec.Containers = append(spec.Containers[:i], spec.Containers[i+1:]...)
 		}
 	}
 	t := true
 	zero := int64(0)
-	result = append(spec.Containers, v1.Container{
+	spec.Containers = append(spec.Containers, v1.Container{
 		Name:    VPN,
 		Image:   "naison/kubevpn:v2",
 		Command: []string{"/bin/sh", "-c"},
@@ -64,7 +64,6 @@ func AddContainer(spec *v1.PodSpec, c util.PodRouteConfig) {
 		},
 		ImagePullPolicy: v1.PullAlways,
 	})
-	spec.Containers = result
 	if len(spec.PriorityClassName) == 0 {
 		spec.PriorityClassName = "system-cluster-critical"
 	}
--- a/pkg/remote.go
+++ b/pkg/remote.go
@@ -24,11 +24,12 @@ import (
 	"k8s.io/kubectl/pkg/util/podutils"
 	"net"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 )

-func CreateOutboundRouterPod(clientset *kubernetes.Clientset, namespace string, trafficManagerIP string, nodeCIDR []*net.IPNet) (net.IP, error) {
+func CreateOutboundPod(clientset *kubernetes.Clientset, namespace string, trafficManagerIP string, nodeCIDR []*net.IPNet) (net.IP, error) {
 	manager, _, err := polymorphichelpers.GetFirstPod(clientset.CoreV1(),
 		namespace,
 		fields.OneTermEqualSelector("app", util.TrafficManager).String(),
@@ -39,9 +40,11 @@ func CreateOutboundRouterPod(clientset *kubernetes.Clientset, namespace string,
 	)

 	if err == nil && manager != nil {
+		log.Infoln("traffic manager already exist, reuse it")
 		UpdateRefCount(clientset, namespace, manager.Name, 1)
 		return net.ParseIP(manager.Status.PodIP), nil
 	}
+	log.Infoln("traffic manager not exist, try to create it...")
 	args := []string{
 		"sysctl net.ipv4.ip_forward=1",
 		"iptables -F",
@@ -123,7 +126,6 @@ func CreateOutboundRouterPod(clientset *kubernetes.Clientset, namespace string,
 				phase = podT.Status.Phase
 			}
 		case <-time.Tick(time.Minute * 5):
-			log.Errorf("wait pod %s to be ready timeout", util.TrafficManager)
 			return nil, errors.New(fmt.Sprintf("wait pod %s to be ready timeout", util.TrafficManager))
 		}
 	}
@@ -146,25 +148,16 @@ func CreateInboundPod(factory cmdutil.Factory, namespace, workloads string, conf

 	helper := pkgresource.NewHelper(object.Client, object.Mapping)

-	// how to scale to one
 	exchange.AddContainer(&podTempSpec.Spec, config)

-	bytes, err := json.Marshal([]struct {
-		Op    string      `json:"op"`
-		Path  string      `json:"path"`
-		Value interface{} `json:"value"`
-	}{{
-		Op:    "replace",
-		Path:  "/" + strings.Join(append(path, "spec"), "/"),
-		Value: podTempSpec.Spec,
-	}})
-	if err != nil {
-		return err
-	}
-
-	// pods
+	// pods without controller
 	if len(path) == 0 {
 		podTempSpec.Spec.PriorityClassName = ""
+		for _, c := range podTempSpec.Spec.Containers {
+			c.LivenessProbe = nil
+			c.StartupProbe = nil
+			c.ReadinessProbe = nil
+		}
 		p := &v1.Pod{ObjectMeta: podTempSpec.ObjectMeta, Spec: podTempSpec.Spec}
 		CleanupUselessInfo(p)
 		if err = createAfterDeletePod(factory, p, helper); err != nil {
@@ -178,12 +171,37 @@ func CreateInboundPod(factory cmdutil.Factory, namespace, workloads string, conf
 				log.Error(err)
 			}
 		})
-	} else {
+	} else
+	// controllers
+	{
+		bytes, _ := json.Marshal([]struct {
+			Op    string      `json:"op"`
+			Path  string      `json:"path"`
+			Value interface{} `json:"value"`
+		}{{
+			Op:    "replace",
+			Path:  "/" + strings.Join(append(path, "spec"), "/"),
+			Value: podTempSpec.Spec,
+		}})
 		_, err = helper.Patch(object.Namespace, object.Name, types.JSONPatchType, bytes, &metav1.PatchOptions{})
+		if err != nil {
+			log.Errorf("error while inject proxy container, err: %v, exiting...")
+			return err
+		}
+		removePatch, restorePatch := patch(origin, path)
+		_, err = helper.Patch(object.Namespace, object.Name, types.JSONPatchType, removePatch, &metav1.PatchOptions{})
+		if err != nil {
+			log.Warnf("error while remove probe of resource: %s %s, ignore, err: %v",
+				object.Mapping.GroupVersionKind.GroupKind().String(), object.Name, err)
+		}
 		rollbackFuncList = append(rollbackFuncList, func() {
-			if err = RemoveInboundPod(factory, namespace, workloads); err != nil {
+			if err = removeInboundContainer(factory, namespace, workloads); err != nil {
 				log.Error(err)
 			}
+			if _, err = helper.Patch(object.Namespace, object.Name, types.JSONPatchType, restorePatch, &metav1.PatchOptions{}); err != nil {
+				log.Warnf("error while restore probe of resource: %s %s, ignore, err: %v",
+					object.Mapping.GroupVersionKind.GroupKind().String(), object.Name, err)
+			}
 		})
 	}
 	_ = util.RolloutStatus(factory, namespace, workloads, time.Minute*5)
@@ -226,7 +244,7 @@ func createAfterDeletePod(factory cmdutil.Factory, p *v1.Pod, helper *pkgresourc
 	return nil
 }

-func RemoveInboundPod(factory cmdutil.Factory, namespace, workloads string) error {
+func removeInboundContainer(factory cmdutil.Factory, namespace, workloads string) error {
 	object, err := util.GetUnstructuredObject(factory, namespace, workloads)
 	if err != nil {
 		return err
@@ -283,3 +301,47 @@ func CleanupUselessInfo(pod *v1.Pod) {
 	pod.SetManagedFields(nil)
 	pod.SetOwnerReferences(nil)
 }
+
+func patch(spec v1.PodTemplateSpec, path []string) (removePatch []byte, restorePatch []byte) {
+	type P struct {
+		Op    string      `json:"op,omitempty"`
+		Path  string      `json:"path,omitempty"`
+		Value interface{} `json:"value,omitempty"`
+	}
+	var remove, restore []P
+	for i := range spec.Spec.Containers {
+		index := strconv.Itoa(i)
+		readinessPath := "/" + strings.Join(append(path, "spec", "containers", index, "readinessProbe"), "/")
+		livenessPath := "/" + strings.Join(append(path, "spec", "containers", index, "livenessProbe"), "/")
+		startupPath := "/" + strings.Join(append(path, "spec", "containers", index, "startupProbe"), "/")
+		remove = append(remove, P{
+			Op:    "replace",
+			Path:  readinessPath,
+			Value: nil,
+		}, P{
+			Op:    "replace",
+			Path:  livenessPath,
+			Value: nil,
+		}, P{
+			Op:    "replace",
+			Path:  startupPath,
+			Value: nil,
+		})
+		restore = append(restore, P{
+			Op:    "replace",
+			Path:  readinessPath,
+			Value: spec.Spec.Containers[i].ReadinessProbe,
+		}, P{
+			Op:    "replace",
+			Path:  livenessPath,
+			Value: spec.Spec.Containers[i].LivenessProbe,
+		}, P{
+			Op:    "replace",
+			Path:  startupPath,
+			Value: spec.Spec.Containers[i].StartupProbe,
+		})
+	}
+	removePatch, _ = json.Marshal(remove)
+	restorePatch, _ = json.Marshal(restore)
+	return
+}
--- a/pkg/remote_test.go
+++ b/pkg/remote_test.go
@@ -11,6 +11,8 @@ import (
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	k8sruntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	pkgresource "k8s.io/cli-runtime/pkg/resource"
@@ -49,7 +51,7 @@ import (
 //		Mask: net.IPv4Mask(255, 255, 0, 0),
 //	}
 //
-//	server, err := pkg.CreateOutboundRouterPod(clientset, "test", i, []*net.IPNet{j})
+//	server, err := pkg.CreateOutboundPod(clientset, "test", i, []*net.IPNet{j})
 //	fmt.Println(server)
 //}

@@ -211,3 +213,37 @@ func TestDeleteAndCreate(t *testing.T) {
 		log.Fatal(err)
 	}
 }
+
+func TestReadiness(t *testing.T) {
+	configFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag()
+	configFlags.KubeConfig = &clientcmd.RecommendedHomeFile
+	factory := cmdutil.NewFactory(cmdutil.NewMatchVersionFlags(configFlags))
+	object, err := util.GetUnstructuredObject(factory, "default", "deployment/authors")
+	if err != nil {
+		panic(err)
+	}
+	podTemplateSpec, path, err := util.GetPodTemplateSpecPath(object.Object.(*unstructured.Unstructured))
+	if err != nil {
+		panic(err)
+	}
+	helper := pkgresource.NewHelper(object.Client, object.Mapping)
+	removePatch, restorePatch := patch(*podTemplateSpec, path)
+	_, err = patchs(helper, object.Namespace, object.Name, removePatch)
+	if err != nil {
+		panic(err)
+	}
+	_, err = patchs(helper, object.Namespace, object.Name, restorePatch)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func patchs(helper *pkgresource.Helper, namespace, name string, p []byte) (k8sruntime.Object, error) {
+	return helper.Patch(
+		namespace,
+		name,
+		types.JSONPatchType,
+		p,
+		&metav1.PatchOptions{},
+	)
+}