diff --git a/go.mod b/go.mod index 6247ebbd..6cb0f976 100644 --- a/go.mod +++ b/go.mod @@ -84,7 +84,6 @@ require ( golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect - golang.zx2c4.com/go118/netip v0.0.0-20211111135330-a4a02eeacf9d // indirect golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224 // indirect google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index b837aa3c..180c80e7 100644 --- a/go.sum +++ b/go.sum @@ -799,8 +799,6 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa h1:idItI2DDfCokpg0N51B2VtiLdJ4vAuXC9fnCb2gACo4= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -876,8 +874,6 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210927181540-4e4d966f7476 h1:s5hu7bTnLKswvidgtqc4GwsW83m9LZu8UAqzmWOZtI4= -golang.org/x/net v0.0.0-20210927181540-4e4d966f7476/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211111083644-e5c967477495 h1:cjxxlQm6d4kYbhpZ2ghvmI8xnq0AG+jXmzrhzfkyu5A= golang.org/x/net v0.0.0-20211111083644-e5c967477495/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -969,8 +965,7 @@ golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 h1:WecRHqgE09JBkh/584XIE6PMz5KKE/vER4izNUi30AQ= golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1040,13 +1035,10 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.zx2c4.com/go118/netip v0.0.0-20211111135330-a4a02eeacf9d h1:9+v0G0naRhLPOJEeJOL6NuXTtAHHwmkyZlgQJ0XcQ8I= golang.zx2c4.com/go118/netip v0.0.0-20211111135330-a4a02eeacf9d/go.mod h1:5yyfuiqVIJ7t+3MqrpTQ+QqRkMWiESiyDvPNvKYCecg= golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224 h1:Ug9qvr1myri/zFN6xL17LSCBGFDnphBBhzmILHsM5TY= golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= golang.zx2c4.com/wireguard v0.0.0-20210905140043-2ef39d47540c/go.mod h1:laHzsbfMhGSobUmruXWAyMKKHSqvIcrqZJMyHD+/3O8= -golang.zx2c4.com/wireguard v0.0.0-20210927201915-bb745b2ea326 h1:4yQQ5d6U5ozGB6n/WSDZa6B0XpPTmoQMtMDMoiZr4n0= -golang.zx2c4.com/wireguard v0.0.0-20210927201915-bb745b2ea326/go.mod h1:SDoazCvdy7RDjBPNEMBwrXhomlmtG7svs8mgwWEqtVI= golang.zx2c4.com/wireguard v0.0.0-20211209221555-9c9e7e272434 h1:3zl8RkJNQ8wfPRomwv/6DBbH2Ut6dgMaWTxM0ZunWnE= golang.zx2c4.com/wireguard v0.0.0-20211209221555-9c9e7e272434/go.mod h1:TjUWrnD5ATh7bFvmm/ALEJZQ4ivKbETb6pmyj1vUoNI= golang.zx2c4.com/wireguard/windows v0.4.10 h1:HmjzJnb+G4NCdX+sfjsQlsxGPuYaThxRbZUZFLyR0/s= diff --git a/pkg/connect.go b/pkg/connect.go index d89eb0c9..014f56ce 100644 --- a/pkg/connect.go +++ b/pkg/connect.go @@ -94,7 +94,7 @@ func (c *ConnectOptions) DoConnect() (err error) { return } trafficMangerNet := net.IPNet{IP: util.RouterIP, Mask: util.CIDR.Mask} - c.routerIP, err = CreateOutboundRouterPod(c.clientset, c.Namespace, trafficMangerNet.String(), c.cidrs) + c.routerIP, err = CreateOutboundPod(c.clientset, c.Namespace, trafficMangerNet.String(), c.cidrs) if err != nil { return } @@ -141,9 +141,21 @@ func (c *ConnectOptions) portForward(ctx context.Context) error { errChan <- err } first = false + // exit normal, let context.err to judge to exit or not + if err == nil { + return + } if apierrors.IsNotFound(err) { - log.Errorf("can not found port-forward resource, err: %v, exiting", err) - } else if err != nil { + log.Errorln("can not found outbound pod, try to create one") + tm := net.IPNet{IP: util.RouterIP, Mask: util.CIDR.Mask} + if _, err = CreateOutboundPod(c.clientset, c.Namespace, tm.String(), c.cidrs); err != nil { + log.Errorf("error while create traffic manager, will retry after a snap, err: %v", err) + } + } else if strings.Contains(err.Error(), "unable to listen on any of the requested ports") || + strings.Contains(err.Error(), "address already in use") { + log.Errorf("port 10800 already in use, needs to release it manually") + time.Sleep(time.Second * 5) + } else { log.Errorf("port-forward occurs error, err: %v, retrying", err) } time.Sleep(time.Second * 2) diff --git a/pkg/envoy.go b/pkg/envoy.go index 8ff13df5..52d43a70 100644 --- a/pkg/envoy.go +++ b/pkg/envoy.go @@ -35,11 +35,13 @@ func PatchSidecar(factory cmdutil.Factory, clientset *kubernetes.Clientset, name } u := object.Object.(*unstructured.Unstructured) - templateSpec, depth, err := util.GetPodTemplateSpecPath(u) + templateSpec, path, err := util.GetPodTemplateSpecPath(u) if err != nil { return err } + origin := *templateSpec + port := uint32(templateSpec.Spec.Containers[0].Ports[0].ContainerPort) configMapName := fmt.Sprintf("%s-%s", object.Mapping.Resource.Resource, object.Name) @@ -58,7 +60,7 @@ func PatchSidecar(factory cmdutil.Factory, clientset *kubernetes.Clientset, name Value interface{} `json:"value"` }{{ Op: "replace", - Path: "/" + strings.Join(append(depth, "spec"), "/"), + Path: "/" + strings.Join(append(path, "spec"), "/"), Value: templateSpec.Spec, }}) if err != nil { @@ -69,6 +71,13 @@ func PatchSidecar(factory cmdutil.Factory, clientset *kubernetes.Clientset, name //Force: &t, }) + removePatch, restorePatch := patch(origin, path) + _, err = helper.Patch(object.Namespace, object.Name, types.JSONPatchType, removePatch, &metav1.PatchOptions{}) + if err != nil { + log.Warnf("error while remove probe of resource: %s %s, ignore, err: %v", + object.Mapping.GroupVersionKind.GroupKind().String(), object.Name, err) + } + //_ = util.WaitPod(clientset, namespace, metav1.ListOptions{ // FieldSelector: fields.OneTermEqualSelector("metadata.name", object.Name+"-shadow").String(), //}, func(pod *v1.Pod) bool { @@ -78,6 +87,10 @@ func PatchSidecar(factory cmdutil.Factory, clientset *kubernetes.Clientset, name if err = UnPatchContainer(factory, clientset, namespace, workloads, headers); err != nil { log.Error(err) } + if _, err = helper.Patch(object.Namespace, object.Name, types.JSONPatchType, restorePatch, &metav1.PatchOptions{}); err != nil { + log.Warnf("error while restore probe of resource: %s %s, ignore, err: %v", + object.Mapping.GroupVersionKind.GroupKind().String(), object.Name, err) + } }) _ = util.RolloutStatus(factory, namespace, workloads, time.Minute*5) return err diff --git a/pkg/exchange/controller.go b/pkg/exchange/controller.go index cb1bb753..8d2b8976 100644 --- a/pkg/exchange/controller.go +++ b/pkg/exchange/controller.go @@ -19,15 +19,15 @@ func RemoveContainer(spec *v1.PodSpec) { } func AddContainer(spec *v1.PodSpec, c util.PodRouteConfig) { - var result []v1.Container - for _, container := range spec.Containers { - if container.Name != VPN { - result = append(result, container) + // remove vpn container is already exist + for i := 0; i < len(spec.Containers); i++ { + if spec.Containers[i].Name == VPN { + spec.Containers = append(spec.Containers[:i], spec.Containers[i+1:]...) } } t := true zero := int64(0) - result = append(spec.Containers, v1.Container{ + spec.Containers = append(spec.Containers, v1.Container{ Name: VPN, Image: "naison/kubevpn:v2", Command: []string{"/bin/sh", "-c"}, @@ -64,7 +64,6 @@ func AddContainer(spec *v1.PodSpec, c util.PodRouteConfig) { }, ImagePullPolicy: v1.PullAlways, }) - spec.Containers = result if len(spec.PriorityClassName) == 0 { spec.PriorityClassName = "system-cluster-critical" } diff --git a/pkg/remote.go b/pkg/remote.go index 4d7d4b7a..5e3984ed 100644 --- a/pkg/remote.go +++ b/pkg/remote.go @@ -24,11 +24,12 @@ import ( "k8s.io/kubectl/pkg/util/podutils" "net" "sort" + "strconv" "strings" "time" ) -func CreateOutboundRouterPod(clientset *kubernetes.Clientset, namespace string, trafficManagerIP string, nodeCIDR []*net.IPNet) (net.IP, error) { +func CreateOutboundPod(clientset *kubernetes.Clientset, namespace string, trafficManagerIP string, nodeCIDR []*net.IPNet) (net.IP, error) { manager, _, err := polymorphichelpers.GetFirstPod(clientset.CoreV1(), namespace, fields.OneTermEqualSelector("app", util.TrafficManager).String(), @@ -39,9 +40,11 @@ func CreateOutboundRouterPod(clientset *kubernetes.Clientset, namespace string, ) if err == nil && manager != nil { + log.Infoln("traffic manager already exist, reuse it") UpdateRefCount(clientset, namespace, manager.Name, 1) return net.ParseIP(manager.Status.PodIP), nil } + log.Infoln("traffic manager not exist, try to create it...") args := []string{ "sysctl net.ipv4.ip_forward=1", "iptables -F", @@ -123,7 +126,6 @@ func CreateOutboundRouterPod(clientset *kubernetes.Clientset, namespace string, phase = podT.Status.Phase } case <-time.Tick(time.Minute * 5): - log.Errorf("wait pod %s to be ready timeout", util.TrafficManager) return nil, errors.New(fmt.Sprintf("wait pod %s to be ready timeout", util.TrafficManager)) } } @@ -146,25 +148,16 @@ func CreateInboundPod(factory cmdutil.Factory, namespace, workloads string, conf helper := pkgresource.NewHelper(object.Client, object.Mapping) - // how to scale to one exchange.AddContainer(&podTempSpec.Spec, config) - bytes, err := json.Marshal([]struct { - Op string `json:"op"` - Path string `json:"path"` - Value interface{} `json:"value"` - }{{ - Op: "replace", - Path: "/" + strings.Join(append(path, "spec"), "/"), - Value: podTempSpec.Spec, - }}) - if err != nil { - return err - } - - // pods + // pods without controller if len(path) == 0 { podTempSpec.Spec.PriorityClassName = "" + for _, c := range podTempSpec.Spec.Containers { + c.LivenessProbe = nil + c.StartupProbe = nil + c.ReadinessProbe = nil + } p := &v1.Pod{ObjectMeta: podTempSpec.ObjectMeta, Spec: podTempSpec.Spec} CleanupUselessInfo(p) if err = createAfterDeletePod(factory, p, helper); err != nil { @@ -178,12 +171,37 @@ func CreateInboundPod(factory cmdutil.Factory, namespace, workloads string, conf log.Error(err) } }) - } else { + } else + // controllers + { + bytes, _ := json.Marshal([]struct { + Op string `json:"op"` + Path string `json:"path"` + Value interface{} `json:"value"` + }{{ + Op: "replace", + Path: "/" + strings.Join(append(path, "spec"), "/"), + Value: podTempSpec.Spec, + }}) _, err = helper.Patch(object.Namespace, object.Name, types.JSONPatchType, bytes, &metav1.PatchOptions{}) + if err != nil { + log.Errorf("error while inject proxy container, err: %v, exiting...") + return err + } + removePatch, restorePatch := patch(origin, path) + _, err = helper.Patch(object.Namespace, object.Name, types.JSONPatchType, removePatch, &metav1.PatchOptions{}) + if err != nil { + log.Warnf("error while remove probe of resource: %s %s, ignore, err: %v", + object.Mapping.GroupVersionKind.GroupKind().String(), object.Name, err) + } rollbackFuncList = append(rollbackFuncList, func() { - if err = RemoveInboundPod(factory, namespace, workloads); err != nil { + if err = removeInboundContainer(factory, namespace, workloads); err != nil { log.Error(err) } + if _, err = helper.Patch(object.Namespace, object.Name, types.JSONPatchType, restorePatch, &metav1.PatchOptions{}); err != nil { + log.Warnf("error while restore probe of resource: %s %s, ignore, err: %v", + object.Mapping.GroupVersionKind.GroupKind().String(), object.Name, err) + } }) } _ = util.RolloutStatus(factory, namespace, workloads, time.Minute*5) @@ -226,7 +244,7 @@ func createAfterDeletePod(factory cmdutil.Factory, p *v1.Pod, helper *pkgresourc return nil } -func RemoveInboundPod(factory cmdutil.Factory, namespace, workloads string) error { +func removeInboundContainer(factory cmdutil.Factory, namespace, workloads string) error { object, err := util.GetUnstructuredObject(factory, namespace, workloads) if err != nil { return err @@ -283,3 +301,47 @@ func CleanupUselessInfo(pod *v1.Pod) { pod.SetManagedFields(nil) pod.SetOwnerReferences(nil) } + +func patch(spec v1.PodTemplateSpec, path []string) (removePatch []byte, restorePatch []byte) { + type P struct { + Op string `json:"op,omitempty"` + Path string `json:"path,omitempty"` + Value interface{} `json:"value,omitempty"` + } + var remove, restore []P + for i := range spec.Spec.Containers { + index := strconv.Itoa(i) + readinessPath := "/" + strings.Join(append(path, "spec", "containers", index, "readinessProbe"), "/") + livenessPath := "/" + strings.Join(append(path, "spec", "containers", index, "livenessProbe"), "/") + startupPath := "/" + strings.Join(append(path, "spec", "containers", index, "startupProbe"), "/") + remove = append(remove, P{ + Op: "replace", + Path: readinessPath, + Value: nil, + }, P{ + Op: "replace", + Path: livenessPath, + Value: nil, + }, P{ + Op: "replace", + Path: startupPath, + Value: nil, + }) + restore = append(restore, P{ + Op: "replace", + Path: readinessPath, + Value: spec.Spec.Containers[i].ReadinessProbe, + }, P{ + Op: "replace", + Path: livenessPath, + Value: spec.Spec.Containers[i].LivenessProbe, + }, P{ + Op: "replace", + Path: startupPath, + Value: spec.Spec.Containers[i].StartupProbe, + }) + } + removePatch, _ = json.Marshal(remove) + restorePatch, _ = json.Marshal(restore) + return +} diff --git a/pkg/remote_test.go b/pkg/remote_test.go index 686abcc1..b8c2655e 100644 --- a/pkg/remote_test.go +++ b/pkg/remote_test.go @@ -11,6 +11,8 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/cli-runtime/pkg/genericclioptions" pkgresource "k8s.io/cli-runtime/pkg/resource" @@ -49,7 +51,7 @@ import ( // Mask: net.IPv4Mask(255, 255, 0, 0), // } // -// server, err := pkg.CreateOutboundRouterPod(clientset, "test", i, []*net.IPNet{j}) +// server, err := pkg.CreateOutboundPod(clientset, "test", i, []*net.IPNet{j}) // fmt.Println(server) //} @@ -211,3 +213,37 @@ func TestDeleteAndCreate(t *testing.T) { log.Fatal(err) } } + +func TestReadiness(t *testing.T) { + configFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag() + configFlags.KubeConfig = &clientcmd.RecommendedHomeFile + factory := cmdutil.NewFactory(cmdutil.NewMatchVersionFlags(configFlags)) + object, err := util.GetUnstructuredObject(factory, "default", "deployment/authors") + if err != nil { + panic(err) + } + podTemplateSpec, path, err := util.GetPodTemplateSpecPath(object.Object.(*unstructured.Unstructured)) + if err != nil { + panic(err) + } + helper := pkgresource.NewHelper(object.Client, object.Mapping) + removePatch, restorePatch := patch(*podTemplateSpec, path) + _, err = patchs(helper, object.Namespace, object.Name, removePatch) + if err != nil { + panic(err) + } + _, err = patchs(helper, object.Namespace, object.Name, restorePatch) + if err != nil { + panic(err) + } +} + +func patchs(helper *pkgresource.Helper, namespace, name string, p []byte) (k8sruntime.Object, error) { + return helper.Patch( + namespace, + name, + types.JSONPatchType, + p, + &metav1.PatchOptions{}, + ) +}