hotfix: fix interrupt proxy mode but not rollout deploy

2025-12-24 11:51:13 +08:00 · 2025-10-21 15:28:49 +08:00
parent e830a28581
commit b0bcc16f5f
9 changed files with 145 additions and 174 deletions
--- a/pkg/handler/connect.go
+++ b/pkg/handler/connect.go
@@ -189,21 +189,6 @@ func (c *ConnectOptions) CreateRemoteInboundPod(ctx context.Context, namespace s
 		if err != nil {
 			return
 		}
-		nodeID := fmt.Sprintf("%s.%s", object.Mapping.Resource.GroupResource().String(), object.Name)
-		// todo consider to use ephemeral container
-		// https://kubernetes.io/docs/concepts/workloads/pods/ephemeral-containers/
-		// means mesh mode
-		if util.IsK8sService(object) {
-			err = inject.InjectEnvoyAndSSH(ctx, nodeID, c.factory, c.Namespace, object, controller, headers, portMap, image)
-		} else if len(headers) != 0 || len(portMap) != 0 {
-			err = inject.InjectServiceMesh(ctx, nodeID, c.factory, c.Namespace, controller, configInfo, headers, portMap, tlsSecret, image)
-		} else {
-			err = inject.InjectVPN(ctx, nodeID, c.factory, c.Namespace, controller, configInfo, tlsSecret, image)
-		}
-		if err != nil {
-			plog.G(ctx).Errorf("Injecting inbound sidecar for %s in namespace %s failed: %s", workload, namespace, err.Error())
-			return err
-		}
 		c.proxyWorkloads.Add(c.Namespace, &Proxy{
 			headers:    headers,
 			portMap:    portMap,
@@ -211,6 +196,22 @@ func (c *ConnectOptions) CreateRemoteInboundPod(ctx context.Context, namespace s
 			namespace:  namespace,
 			portMapper: util.If(util.IsK8sService(object), NewMapper(c.clientset, namespace, labels.SelectorFromSet(templateSpec.Labels).String(), headers, workload), nil),
 		})
+
+		nodeID := fmt.Sprintf("%s.%s", object.Mapping.Resource.GroupResource().String(), object.Name)
+		// todo consider to use ephemeral container
+		// https://kubernetes.io/docs/concepts/workloads/pods/ephemeral-containers/
+		// means mesh mode
+		if util.IsK8sService(object) {
+			err = inject.InjectEnvoyAndSSH(ctx, nodeID, c.factory, c.Namespace, object, controller, headers, portMap, image)
+		} else if len(headers) != 0 || len(portMap) != 0 {
+			err = inject.InjectEnvoyAndVPN(ctx, nodeID, c.factory, c.Namespace, controller, configInfo, headers, portMap, tlsSecret, image)
+		} else {
+			err = inject.InjectVPN(ctx, nodeID, c.factory, c.Namespace, controller, configInfo, tlsSecret, image)
+		}
+		if err != nil {
+			plog.G(ctx).Errorf("Injecting inbound sidecar for %s in namespace %s failed: %s", workload, namespace, err.Error())
+			return err
+		}
 	}
 	return
 }
@@ -1177,7 +1178,7 @@ func (c *ConnectOptions) getRolloutFunc() []func() error {
 	return c.rollbackFuncList
 }

-func (c *ConnectOptions) LeavePortMap(ns, workload string) {
+func (c *ConnectOptions) leavePortMap(ns, workload string) {
 	c.proxyWorkloads.Remove(ns, workload)
 }

--- a/pkg/handler/leave.go
+++ b/pkg/handler/leave.go
@@ -69,7 +69,7 @@ func (c *ConnectOptions) LeaveResource(ctx context.Context, resources []Resource
 			err = inject.ModifyServiceTargetPort(ctx, c.clientset, workload.Namespace, object.Name, map[int32]int32{})
 			errs = append(errs, err)
 		}
-		c.LeavePortMap(workload.Namespace, workload.Workload)
+		c.leavePortMap(workload.Namespace, workload.Workload)
 	}
 	return errors.NewAggregate(errs)
 }
--- a/pkg/handler/reset.go
+++ b/pkg/handler/reset.go
@@ -108,7 +108,7 @@ func removeInjectContainer(ctx context.Context, factory cmdutil.Factory, clients

 	plog.G(ctx).Infof("Leaving workload %s", workload)

-	inject.RemoveContainers(templateSpec)
+	inject.RemoveContainers(&templateSpec.Spec)

 	helper := pkgresource.NewHelper(controller.Client, controller.Mapping)
 	plog.G(ctx).Debugf("The %s is under controller management", workload)
--- a/pkg/inject/controller.go
+++ b/pkg/inject/controller.go
@@ -28,19 +28,19 @@ var envoyConfigFargate []byte
 //go:embed fargate_envoy_ipv4.yaml
 var envoyConfigIPv4Fargate []byte

-func RemoveContainers(spec *v1.PodTemplateSpec) {
-	for i := 0; i < len(spec.Spec.Containers); i++ {
-		if sets.New[string](config.ContainerSidecarEnvoyProxy, config.ContainerSidecarVPN).Has(spec.Spec.Containers[i].Name) {
-			spec.Spec.Containers = append(spec.Spec.Containers[:i], spec.Spec.Containers[i+1:]...)
+func RemoveContainers(spec *v1.PodSpec) {
+	for i := 0; i < len(spec.Containers); i++ {
+		if sets.New[string](config.ContainerSidecarEnvoyProxy, config.ContainerSidecarVPN).Has(spec.Containers[i].Name) {
+			spec.Containers = append(spec.Containers[:i], spec.Containers[i+1:]...)
 			i--
 		}
 	}
 }

-// AddMeshContainer todo envoy support ipv6
-func AddMeshContainer(spec *v1.PodTemplateSpec, ns, nodeID string, ipv6 bool, managerNamespace string, secret *v1.Secret, image string) {
+// AddVPNAndEnvoyContainer todo envoy support ipv6
+func AddVPNAndEnvoyContainer(spec *v1.PodTemplateSpec, ns, nodeID string, ipv6 bool, managerNamespace string, secret *v1.Secret, image string) {
 	// remove envoy proxy containers if already exist
-	RemoveContainers(spec)
+	RemoveContainers(&spec.Spec)

 	spec.Spec.Containers = append(spec.Spec.Containers, v1.Container{
 		Name:    config.ContainerSidecarVPN,
@@ -169,9 +169,9 @@ kubevpn server -l "tun:/tcp://${TrafficManagerService}:10801?net=${TunIPv4}&net6
 	})
 }

-func AddEnvoyContainer(spec *v1.PodTemplateSpec, ns, nodeID string, ipv6 bool, managerNamespace string, image string) {
+func AddEnvoyAndSSHContainer(spec *v1.PodTemplateSpec, ns, nodeID string, ipv6 bool, managerNamespace string, image string) {
 	// remove envoy proxy containers if already exist
-	RemoveContainers(spec)
+	RemoveContainers(&spec.Spec)

 	spec.Spec.Containers = append(spec.Spec.Containers, v1.Container{
 		Name:    config.ContainerSidecarVPN,
@@ -243,3 +243,112 @@ func GetEnvoyConfig(tmplStr string, value string) string {
 	}
 	return buf.String()
 }
+
+func AddVPNContainer(spec *v1.PodSpec, c util.PodRouteConfig, managerNamespace string, secret *v1.Secret, image string) {
+	// remove vpn container if already exist
+	RemoveContainers(spec)
+	spec.Containers = append(spec.Containers, v1.Container{
+		Name:  config.ContainerSidecarVPN,
+		Image: image,
+		Env: []v1.EnvVar{
+			{
+				Name:  "LocalTunIPv4",
+				Value: c.LocalTunIPv4,
+			},
+			{
+				Name:  "LocalTunIPv6",
+				Value: c.LocalTunIPv6,
+			},
+			{
+				Name:  config.EnvInboundPodTunIPv4,
+				Value: "",
+			},
+			{
+				Name:  config.EnvInboundPodTunIPv6,
+				Value: "",
+			},
+			{
+				Name:  "CIDR4",
+				Value: config.CIDR.String(),
+			},
+			{
+				Name:  "CIDR6",
+				Value: config.CIDR6.String(),
+			},
+			{
+				Name:  "TrafficManagerService",
+				Value: fmt.Sprintf("%s.%s", config.ConfigMapPodTrafficManager, managerNamespace),
+			},
+			{
+				Name: config.EnvPodNamespace,
+				ValueFrom: &v1.EnvVarSource{
+					FieldRef: &v1.ObjectFieldSelector{
+						FieldPath: "metadata.namespace",
+					},
+				},
+			},
+			{
+				Name: config.EnvPodName,
+				ValueFrom: &v1.EnvVarSource{
+					FieldRef: &v1.ObjectFieldSelector{
+						FieldPath: "metadata.name",
+					},
+				},
+			},
+			{
+				Name:  config.TLSServerName,
+				Value: string(secret.Data[config.TLSServerName]),
+			},
+			{
+				Name:  config.TLSCertKey,
+				Value: string(secret.Data[config.TLSCertKey]),
+			},
+			{
+				Name:  config.TLSPrivateKeyKey,
+				Value: string(secret.Data[config.TLSPrivateKeyKey]),
+			},
+		},
+		Command: []string{"/bin/sh", "-c"},
+		// https://www.netfilter.org/documentation/HOWTO/NAT-HOWTO-6.html#ss6.2
+		// for curl -g -6 [2001:2::999a]:9080/health or curl 127.0.0.1:9080/health hit local PC
+		// output chain
+		// iptables -t nat -A OUTPUT -o lo ! -p icmp -j DNAT --to-destination ${LocalTunIPv4}
+		// ip6tables -t nat -A OUTPUT -o lo ! -p icmp -j DNAT --to-destination ${LocalTunIPv6}
+		Args: []string{`
+echo 1 > /proc/sys/net/ipv4/ip_forward
+echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6
+echo 1 > /proc/sys/net/ipv6/conf/all/forwarding
+echo 1 > /proc/sys/net/ipv4/conf/all/route_localnet
+update-alternatives --set iptables /usr/sbin/iptables-legacy
+iptables -P INPUT ACCEPT
+ip6tables -P INPUT ACCEPT
+iptables -P FORWARD ACCEPT
+ip6tables -P FORWARD ACCEPT
+iptables -t nat -A PREROUTING ! -p icmp -j DNAT --to ${LocalTunIPv4}
+ip6tables -t nat -A PREROUTING ! -p icmp -j DNAT --to ${LocalTunIPv6}
+kubevpn server -l "tun:/tcp://${TrafficManagerService}:10801?net=${TunIPv4}&net6=${TunIPv6}&route=${CIDR4}"`,
+		},
+		SecurityContext: &v1.SecurityContext{
+			Capabilities: &v1.Capabilities{
+				Add: []v1.Capability{
+					"NET_ADMIN",
+					//"SYS_MODULE",
+				},
+			},
+			RunAsUser:  pointer.Int64(0),
+			RunAsGroup: pointer.Int64(0),
+			Privileged: pointer.Bool(true),
+		},
+		Resources: v1.ResourceRequirements{
+			Requests: map[v1.ResourceName]resource.Quantity{
+				v1.ResourceCPU:    resource.MustParse("128m"),
+				v1.ResourceMemory: resource.MustParse("128Mi"),
+			},
+			Limits: map[v1.ResourceName]resource.Quantity{
+				v1.ResourceCPU:    resource.MustParse("256m"),
+				v1.ResourceMemory: resource.MustParse("256Mi"),
+			},
+		},
+		ImagePullPolicy: v1.PullIfNotPresent,
+	})
+}
--- a/pkg/inject/exchange.go
+++ b/pkg/inject/exchange.go
@@ -1,130 +0,0 @@
-package inject
-
-import (
-	"fmt"
-
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/resource"
-	"k8s.io/utils/pointer"
-
-	"github.com/wencaiwulue/kubevpn/v2/pkg/config"
-	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
-)
-
-func RemoveContainer(spec *corev1.PodSpec) {
-	for i := 0; i < len(spec.Containers); i++ {
-		if spec.Containers[i].Name == config.ContainerSidecarVPN {
-			spec.Containers = append(spec.Containers[:i], spec.Containers[i+1:]...)
-			i--
-		}
-	}
-}
-
-func AddContainer(spec *corev1.PodSpec, c util.PodRouteConfig, managerNamespace string, secret *corev1.Secret, image string) {
-	// remove vpn container if already exist
-	RemoveContainer(spec)
-	spec.Containers = append(spec.Containers, corev1.Container{
-		Name:  config.ContainerSidecarVPN,
-		Image: image,
-		Env: []corev1.EnvVar{
-			{
-				Name:  "LocalTunIPv4",
-				Value: c.LocalTunIPv4,
-			},
-			{
-				Name:  "LocalTunIPv6",
-				Value: c.LocalTunIPv6,
-			},
-			{
-				Name:  config.EnvInboundPodTunIPv4,
-				Value: "",
-			},
-			{
-				Name:  config.EnvInboundPodTunIPv6,
-				Value: "",
-			},
-			{
-				Name:  "CIDR4",
-				Value: config.CIDR.String(),
-			},
-			{
-				Name:  "CIDR6",
-				Value: config.CIDR6.String(),
-			},
-			{
-				Name:  "TrafficManagerService",
-				Value: fmt.Sprintf("%s.%s", config.ConfigMapPodTrafficManager, managerNamespace),
-			},
-			{
-				Name: config.EnvPodNamespace,
-				ValueFrom: &corev1.EnvVarSource{
-					FieldRef: &corev1.ObjectFieldSelector{
-						FieldPath: "metadata.namespace",
-					},
-				},
-			},
-			{
-				Name: config.EnvPodName,
-				ValueFrom: &corev1.EnvVarSource{
-					FieldRef: &corev1.ObjectFieldSelector{
-						FieldPath: "metadata.name",
-					},
-				},
-			},
-			{
-				Name:  config.TLSServerName,
-				Value: string(secret.Data[config.TLSServerName]),
-			},
-			{
-				Name:  config.TLSCertKey,
-				Value: string(secret.Data[config.TLSCertKey]),
-			},
-			{
-				Name:  config.TLSPrivateKeyKey,
-				Value: string(secret.Data[config.TLSPrivateKeyKey]),
-			},
-		},
-		Command: []string{"/bin/sh", "-c"},
-		// https://www.netfilter.org/documentation/HOWTO/NAT-HOWTO-6.html#ss6.2
-		// for curl -g -6 [2001:2::999a]:9080/health or curl 127.0.0.1:9080/health hit local PC
-		// output chain
-		// iptables -t nat -A OUTPUT -o lo ! -p icmp -j DNAT --to-destination ${LocalTunIPv4}
-		// ip6tables -t nat -A OUTPUT -o lo ! -p icmp -j DNAT --to-destination ${LocalTunIPv6}
-		Args: []string{`
-echo 1 > /proc/sys/net/ipv4/ip_forward
-echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6
-echo 1 > /proc/sys/net/ipv6/conf/all/forwarding
-echo 1 > /proc/sys/net/ipv4/conf/all/route_localnet
-update-alternatives --set iptables /usr/sbin/iptables-legacy
-iptables -P INPUT ACCEPT
-ip6tables -P INPUT ACCEPT
-iptables -P FORWARD ACCEPT
-ip6tables -P FORWARD ACCEPT
-iptables -t nat -A PREROUTING ! -p icmp -j DNAT --to ${LocalTunIPv4}
-ip6tables -t nat -A PREROUTING ! -p icmp -j DNAT --to ${LocalTunIPv6}
-kubevpn server -l "tun:/tcp://${TrafficManagerService}:10801?net=${TunIPv4}&net6=${TunIPv6}&route=${CIDR4}"`,
-		},
-		SecurityContext: &corev1.SecurityContext{
-			Capabilities: &corev1.Capabilities{
-				Add: []corev1.Capability{
-					"NET_ADMIN",
-					//"SYS_MODULE",
-				},
-			},
-			RunAsUser:  pointer.Int64(0),
-			RunAsGroup: pointer.Int64(0),
-			Privileged: pointer.Bool(true),
-		},
-		Resources: corev1.ResourceRequirements{
-			Requests: map[corev1.ResourceName]resource.Quantity{
-				corev1.ResourceCPU:    resource.MustParse("128m"),
-				corev1.ResourceMemory: resource.MustParse("128Mi"),
-			},
-			Limits: map[corev1.ResourceName]resource.Quantity{
-				corev1.ResourceCPU:    resource.MustParse("256m"),
-				corev1.ResourceMemory: resource.MustParse("256Mi"),
-			},
-		},
-		ImagePullPolicy: corev1.PullIfNotPresent,
-	})
-}
--- a/pkg/inject/fargate.go
+++ b/pkg/inject/fargate.go
@@ -67,7 +67,7 @@ func InjectEnvoyAndSSH(ctx context.Context, nodeID string, f cmdutil.Factory, ma

 	enableIPv6, _ := util.DetectPodSupportIPv6(ctx, f, managerNamespace)
 	// (1) add mesh container
-	AddEnvoyContainer(templateSpec, object.Namespace, nodeID, enableIPv6, managerNamespace, image)
+	AddEnvoyAndSSHContainer(templateSpec, object.Namespace, nodeID, enableIPv6, managerNamespace, image)
 	helper := pkgresource.NewHelper(object.Client, object.Mapping)
 	ps := []P{
 		{
--- a/pkg/inject/mesh.go
+++ b/pkg/inject/mesh.go
@@ -32,8 +32,8 @@ import (

 // https://istio.io/latest/docs/ops/deployment/requirements/#ports-used-by-istio

-// InjectServiceMesh patch a sidecar, using iptables to do port-forward let this pod decide should go to 233.254.254.100 or request to 127.0.0.1
-func InjectServiceMesh(ctx context.Context, nodeID string, f cmdutil.Factory, managerNamespace string, object *runtimeresource.Info, c util.PodRouteConfig, headers map[string]string, portMaps []string, secret *v1.Secret, image string) (err error) {
+// InjectEnvoyAndVPN patch a sidecar, using iptables to do port-forward let this pod decide should go to 233.254.254.100 or request to 127.0.0.1
+func InjectEnvoyAndVPN(ctx context.Context, nodeID string, f cmdutil.Factory, managerNamespace string, object *runtimeresource.Info, c util.PodRouteConfig, headers map[string]string, portMaps []string, secret *v1.Secret, image string) (err error) {
 	var clientset *kubernetes.Clientset
 	clientset, err = f.KubernetesClientSet()
 	if err != nil {
@@ -96,7 +96,7 @@ func InjectServiceMesh(ctx context.Context, nodeID string, f cmdutil.Factory, ma

 	enableIPv6, _ := util.DetectPodSupportIPv6(ctx, f, managerNamespace)
 	// (1) add mesh container
-	AddMeshContainer(templateSpec, object.Namespace, nodeID, enableIPv6, managerNamespace, secret, image)
+	AddVPNAndEnvoyContainer(templateSpec, object.Namespace, nodeID, enableIPv6, managerNamespace, secret, image)
 	helper := pkgresource.NewHelper(object.Client, object.Mapping)
 	ps := []P{
 		{
@@ -142,9 +142,9 @@ func UnPatchContainer(ctx context.Context, nodeID string, factory cmdutil.Factor

 	plog.G(ctx).Infof("Leaving workload %s", workload)

-	RemoveContainers(templateSpec)
-
 	if empty {
+		RemoveContainers(&templateSpec.Spec)
+
 		helper := pkgresource.NewHelper(object.Client, object.Mapping)
 		// pod without controller
 		if len(depth) == 0 {
@@ -331,12 +331,3 @@ func removeEnvoyConfig(mapInterface v12.ConfigMapInterface, namespace string, no
 	_, err = mapInterface.Update(context.Background(), configMap, metav1.UpdateOptions{})
 	return empty, found, err
 }
-
-func contains(a map[string]string, sub map[string]string) bool {
-	for k, v := range sub {
-		if a[k] != v {
-			return false
-		}
-	}
-	return true
-}
--- a/pkg/inject/proxy.go
+++ b/pkg/inject/proxy.go
@@ -50,7 +50,7 @@ func InjectVPN(ctx context.Context, nodeID string, f util.Factory, managerNamesp
 		return err
 	}

-	AddContainer(&podTempSpec.Spec, c, managerNamespace, secret, image)
+	AddVPNContainer(&podTempSpec.Spec, c, managerNamespace, secret, image)

 	workload := fmt.Sprintf("%s/%s", object.Mapping.Resource.Resource, object.Name)
 	helper := resource.NewHelper(object.Client, object.Mapping)
--- a/pkg/run/runconfig.go
+++ b/pkg/run/runconfig.go
@@ -97,7 +97,7 @@ func (option *Options) ConvertPodToContainerConfigList(
 	mountVolume map[string][]mount.Mount,
 	dnsConfig *dns.ClientConfig,
 ) (configList ConfigList, err error) {
-	inject.RemoveContainers(&temp)
+	inject.RemoveContainers(&temp.Spec)
 	// move dev container to location first
 	for index, c := range temp.Spec.Containers {
 		if option.ContainerName == c.Name {