diff --git a/pkg/handler/connect.go b/pkg/handler/connect.go index 2457e7cb..8d61168b 100644 --- a/pkg/handler/connect.go +++ b/pkg/handler/connect.go @@ -189,21 +189,6 @@ func (c *ConnectOptions) CreateRemoteInboundPod(ctx context.Context, namespace s if err != nil { return } - nodeID := fmt.Sprintf("%s.%s", object.Mapping.Resource.GroupResource().String(), object.Name) - // todo consider to use ephemeral container - // https://kubernetes.io/docs/concepts/workloads/pods/ephemeral-containers/ - // means mesh mode - if util.IsK8sService(object) { - err = inject.InjectEnvoyAndSSH(ctx, nodeID, c.factory, c.Namespace, object, controller, headers, portMap, image) - } else if len(headers) != 0 || len(portMap) != 0 { - err = inject.InjectServiceMesh(ctx, nodeID, c.factory, c.Namespace, controller, configInfo, headers, portMap, tlsSecret, image) - } else { - err = inject.InjectVPN(ctx, nodeID, c.factory, c.Namespace, controller, configInfo, tlsSecret, image) - } - if err != nil { - plog.G(ctx).Errorf("Injecting inbound sidecar for %s in namespace %s failed: %s", workload, namespace, err.Error()) - return err - } c.proxyWorkloads.Add(c.Namespace, &Proxy{ headers: headers, portMap: portMap, @@ -211,6 +196,22 @@ func (c *ConnectOptions) CreateRemoteInboundPod(ctx context.Context, namespace s namespace: namespace, portMapper: util.If(util.IsK8sService(object), NewMapper(c.clientset, namespace, labels.SelectorFromSet(templateSpec.Labels).String(), headers, workload), nil), }) + + nodeID := fmt.Sprintf("%s.%s", object.Mapping.Resource.GroupResource().String(), object.Name) + // todo consider to use ephemeral container + // https://kubernetes.io/docs/concepts/workloads/pods/ephemeral-containers/ + // means mesh mode + if util.IsK8sService(object) { + err = inject.InjectEnvoyAndSSH(ctx, nodeID, c.factory, c.Namespace, object, controller, headers, portMap, image) + } else if len(headers) != 0 || len(portMap) != 0 { + err = inject.InjectEnvoyAndVPN(ctx, nodeID, c.factory, c.Namespace, controller, configInfo, headers, portMap, tlsSecret, image) + } else { + err = inject.InjectVPN(ctx, nodeID, c.factory, c.Namespace, controller, configInfo, tlsSecret, image) + } + if err != nil { + plog.G(ctx).Errorf("Injecting inbound sidecar for %s in namespace %s failed: %s", workload, namespace, err.Error()) + return err + } } return } @@ -1177,7 +1178,7 @@ func (c *ConnectOptions) getRolloutFunc() []func() error { return c.rollbackFuncList } -func (c *ConnectOptions) LeavePortMap(ns, workload string) { +func (c *ConnectOptions) leavePortMap(ns, workload string) { c.proxyWorkloads.Remove(ns, workload) } diff --git a/pkg/handler/leave.go b/pkg/handler/leave.go index 0a07263d..e3a3a264 100644 --- a/pkg/handler/leave.go +++ b/pkg/handler/leave.go @@ -69,7 +69,7 @@ func (c *ConnectOptions) LeaveResource(ctx context.Context, resources []Resource err = inject.ModifyServiceTargetPort(ctx, c.clientset, workload.Namespace, object.Name, map[int32]int32{}) errs = append(errs, err) } - c.LeavePortMap(workload.Namespace, workload.Workload) + c.leavePortMap(workload.Namespace, workload.Workload) } return errors.NewAggregate(errs) } diff --git a/pkg/handler/reset.go b/pkg/handler/reset.go index 2d13dc07..1e083c09 100644 --- a/pkg/handler/reset.go +++ b/pkg/handler/reset.go @@ -108,7 +108,7 @@ func removeInjectContainer(ctx context.Context, factory cmdutil.Factory, clients plog.G(ctx).Infof("Leaving workload %s", workload) - inject.RemoveContainers(templateSpec) + inject.RemoveContainers(&templateSpec.Spec) helper := pkgresource.NewHelper(controller.Client, controller.Mapping) plog.G(ctx).Debugf("The %s is under controller management", workload) diff --git a/pkg/inject/controller.go b/pkg/inject/controller.go index 96ad9dcd..474747a1 100644 --- a/pkg/inject/controller.go +++ b/pkg/inject/controller.go @@ -28,19 +28,19 @@ var envoyConfigFargate []byte //go:embed fargate_envoy_ipv4.yaml var envoyConfigIPv4Fargate []byte -func RemoveContainers(spec *v1.PodTemplateSpec) { - for i := 0; i < len(spec.Spec.Containers); i++ { - if sets.New[string](config.ContainerSidecarEnvoyProxy, config.ContainerSidecarVPN).Has(spec.Spec.Containers[i].Name) { - spec.Spec.Containers = append(spec.Spec.Containers[:i], spec.Spec.Containers[i+1:]...) +func RemoveContainers(spec *v1.PodSpec) { + for i := 0; i < len(spec.Containers); i++ { + if sets.New[string](config.ContainerSidecarEnvoyProxy, config.ContainerSidecarVPN).Has(spec.Containers[i].Name) { + spec.Containers = append(spec.Containers[:i], spec.Containers[i+1:]...) i-- } } } -// AddMeshContainer todo envoy support ipv6 -func AddMeshContainer(spec *v1.PodTemplateSpec, ns, nodeID string, ipv6 bool, managerNamespace string, secret *v1.Secret, image string) { +// AddVPNAndEnvoyContainer todo envoy support ipv6 +func AddVPNAndEnvoyContainer(spec *v1.PodTemplateSpec, ns, nodeID string, ipv6 bool, managerNamespace string, secret *v1.Secret, image string) { // remove envoy proxy containers if already exist - RemoveContainers(spec) + RemoveContainers(&spec.Spec) spec.Spec.Containers = append(spec.Spec.Containers, v1.Container{ Name: config.ContainerSidecarVPN, @@ -169,9 +169,9 @@ kubevpn server -l "tun:/tcp://${TrafficManagerService}:10801?net=${TunIPv4}&net6 }) } -func AddEnvoyContainer(spec *v1.PodTemplateSpec, ns, nodeID string, ipv6 bool, managerNamespace string, image string) { +func AddEnvoyAndSSHContainer(spec *v1.PodTemplateSpec, ns, nodeID string, ipv6 bool, managerNamespace string, image string) { // remove envoy proxy containers if already exist - RemoveContainers(spec) + RemoveContainers(&spec.Spec) spec.Spec.Containers = append(spec.Spec.Containers, v1.Container{ Name: config.ContainerSidecarVPN, @@ -243,3 +243,112 @@ func GetEnvoyConfig(tmplStr string, value string) string { } return buf.String() } + +func AddVPNContainer(spec *v1.PodSpec, c util.PodRouteConfig, managerNamespace string, secret *v1.Secret, image string) { + // remove vpn container if already exist + RemoveContainers(spec) + spec.Containers = append(spec.Containers, v1.Container{ + Name: config.ContainerSidecarVPN, + Image: image, + Env: []v1.EnvVar{ + { + Name: "LocalTunIPv4", + Value: c.LocalTunIPv4, + }, + { + Name: "LocalTunIPv6", + Value: c.LocalTunIPv6, + }, + { + Name: config.EnvInboundPodTunIPv4, + Value: "", + }, + { + Name: config.EnvInboundPodTunIPv6, + Value: "", + }, + { + Name: "CIDR4", + Value: config.CIDR.String(), + }, + { + Name: "CIDR6", + Value: config.CIDR6.String(), + }, + { + Name: "TrafficManagerService", + Value: fmt.Sprintf("%s.%s", config.ConfigMapPodTrafficManager, managerNamespace), + }, + { + Name: config.EnvPodNamespace, + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }, + { + Name: config.EnvPodName, + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + { + Name: config.TLSServerName, + Value: string(secret.Data[config.TLSServerName]), + }, + { + Name: config.TLSCertKey, + Value: string(secret.Data[config.TLSCertKey]), + }, + { + Name: config.TLSPrivateKeyKey, + Value: string(secret.Data[config.TLSPrivateKeyKey]), + }, + }, + Command: []string{"/bin/sh", "-c"}, + // https://www.netfilter.org/documentation/HOWTO/NAT-HOWTO-6.html#ss6.2 + // for curl -g -6 [2001:2::999a]:9080/health or curl 127.0.0.1:9080/health hit local PC + // output chain + // iptables -t nat -A OUTPUT -o lo ! -p icmp -j DNAT --to-destination ${LocalTunIPv4} + // ip6tables -t nat -A OUTPUT -o lo ! -p icmp -j DNAT --to-destination ${LocalTunIPv6} + Args: []string{` +echo 1 > /proc/sys/net/ipv4/ip_forward +echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6 +echo 1 > /proc/sys/net/ipv6/conf/all/forwarding +echo 1 > /proc/sys/net/ipv4/conf/all/route_localnet +update-alternatives --set iptables /usr/sbin/iptables-legacy +iptables -P INPUT ACCEPT +ip6tables -P INPUT ACCEPT +iptables -P FORWARD ACCEPT +ip6tables -P FORWARD ACCEPT +iptables -t nat -A PREROUTING ! -p icmp -j DNAT --to ${LocalTunIPv4} +ip6tables -t nat -A PREROUTING ! -p icmp -j DNAT --to ${LocalTunIPv6} +kubevpn server -l "tun:/tcp://${TrafficManagerService}:10801?net=${TunIPv4}&net6=${TunIPv6}&route=${CIDR4}"`, + }, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{ + Add: []v1.Capability{ + "NET_ADMIN", + //"SYS_MODULE", + }, + }, + RunAsUser: pointer.Int64(0), + RunAsGroup: pointer.Int64(0), + Privileged: pointer.Bool(true), + }, + Resources: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("128m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("256m"), + v1.ResourceMemory: resource.MustParse("256Mi"), + }, + }, + ImagePullPolicy: v1.PullIfNotPresent, + }) +} diff --git a/pkg/inject/exchange.go b/pkg/inject/exchange.go deleted file mode 100644 index 1110b455..00000000 --- a/pkg/inject/exchange.go +++ /dev/null @@ -1,130 +0,0 @@ -package inject - -import ( - "fmt" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/utils/pointer" - - "github.com/wencaiwulue/kubevpn/v2/pkg/config" - "github.com/wencaiwulue/kubevpn/v2/pkg/util" -) - -func RemoveContainer(spec *corev1.PodSpec) { - for i := 0; i < len(spec.Containers); i++ { - if spec.Containers[i].Name == config.ContainerSidecarVPN { - spec.Containers = append(spec.Containers[:i], spec.Containers[i+1:]...) - i-- - } - } -} - -func AddContainer(spec *corev1.PodSpec, c util.PodRouteConfig, managerNamespace string, secret *corev1.Secret, image string) { - // remove vpn container if already exist - RemoveContainer(spec) - spec.Containers = append(spec.Containers, corev1.Container{ - Name: config.ContainerSidecarVPN, - Image: image, - Env: []corev1.EnvVar{ - { - Name: "LocalTunIPv4", - Value: c.LocalTunIPv4, - }, - { - Name: "LocalTunIPv6", - Value: c.LocalTunIPv6, - }, - { - Name: config.EnvInboundPodTunIPv4, - Value: "", - }, - { - Name: config.EnvInboundPodTunIPv6, - Value: "", - }, - { - Name: "CIDR4", - Value: config.CIDR.String(), - }, - { - Name: "CIDR6", - Value: config.CIDR6.String(), - }, - { - Name: "TrafficManagerService", - Value: fmt.Sprintf("%s.%s", config.ConfigMapPodTrafficManager, managerNamespace), - }, - { - Name: config.EnvPodNamespace, - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.namespace", - }, - }, - }, - { - Name: config.EnvPodName, - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.name", - }, - }, - }, - { - Name: config.TLSServerName, - Value: string(secret.Data[config.TLSServerName]), - }, - { - Name: config.TLSCertKey, - Value: string(secret.Data[config.TLSCertKey]), - }, - { - Name: config.TLSPrivateKeyKey, - Value: string(secret.Data[config.TLSPrivateKeyKey]), - }, - }, - Command: []string{"/bin/sh", "-c"}, - // https://www.netfilter.org/documentation/HOWTO/NAT-HOWTO-6.html#ss6.2 - // for curl -g -6 [2001:2::999a]:9080/health or curl 127.0.0.1:9080/health hit local PC - // output chain - // iptables -t nat -A OUTPUT -o lo ! -p icmp -j DNAT --to-destination ${LocalTunIPv4} - // ip6tables -t nat -A OUTPUT -o lo ! -p icmp -j DNAT --to-destination ${LocalTunIPv6} - Args: []string{` -echo 1 > /proc/sys/net/ipv4/ip_forward -echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6 -echo 1 > /proc/sys/net/ipv6/conf/all/forwarding -echo 1 > /proc/sys/net/ipv4/conf/all/route_localnet -update-alternatives --set iptables /usr/sbin/iptables-legacy -iptables -P INPUT ACCEPT -ip6tables -P INPUT ACCEPT -iptables -P FORWARD ACCEPT -ip6tables -P FORWARD ACCEPT -iptables -t nat -A PREROUTING ! -p icmp -j DNAT --to ${LocalTunIPv4} -ip6tables -t nat -A PREROUTING ! -p icmp -j DNAT --to ${LocalTunIPv6} -kubevpn server -l "tun:/tcp://${TrafficManagerService}:10801?net=${TunIPv4}&net6=${TunIPv6}&route=${CIDR4}"`, - }, - SecurityContext: &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{ - "NET_ADMIN", - //"SYS_MODULE", - }, - }, - RunAsUser: pointer.Int64(0), - RunAsGroup: pointer.Int64(0), - Privileged: pointer.Bool(true), - }, - Resources: corev1.ResourceRequirements{ - Requests: map[corev1.ResourceName]resource.Quantity{ - corev1.ResourceCPU: resource.MustParse("128m"), - corev1.ResourceMemory: resource.MustParse("128Mi"), - }, - Limits: map[corev1.ResourceName]resource.Quantity{ - corev1.ResourceCPU: resource.MustParse("256m"), - corev1.ResourceMemory: resource.MustParse("256Mi"), - }, - }, - ImagePullPolicy: corev1.PullIfNotPresent, - }) -} diff --git a/pkg/inject/fargate.go b/pkg/inject/fargate.go index 95ab158d..b656dc17 100644 --- a/pkg/inject/fargate.go +++ b/pkg/inject/fargate.go @@ -67,7 +67,7 @@ func InjectEnvoyAndSSH(ctx context.Context, nodeID string, f cmdutil.Factory, ma enableIPv6, _ := util.DetectPodSupportIPv6(ctx, f, managerNamespace) // (1) add mesh container - AddEnvoyContainer(templateSpec, object.Namespace, nodeID, enableIPv6, managerNamespace, image) + AddEnvoyAndSSHContainer(templateSpec, object.Namespace, nodeID, enableIPv6, managerNamespace, image) helper := pkgresource.NewHelper(object.Client, object.Mapping) ps := []P{ { diff --git a/pkg/inject/mesh.go b/pkg/inject/mesh.go index 5bcc736b..865b2d36 100644 --- a/pkg/inject/mesh.go +++ b/pkg/inject/mesh.go @@ -32,8 +32,8 @@ import ( // https://istio.io/latest/docs/ops/deployment/requirements/#ports-used-by-istio -// InjectServiceMesh patch a sidecar, using iptables to do port-forward let this pod decide should go to 233.254.254.100 or request to 127.0.0.1 -func InjectServiceMesh(ctx context.Context, nodeID string, f cmdutil.Factory, managerNamespace string, object *runtimeresource.Info, c util.PodRouteConfig, headers map[string]string, portMaps []string, secret *v1.Secret, image string) (err error) { +// InjectEnvoyAndVPN patch a sidecar, using iptables to do port-forward let this pod decide should go to 233.254.254.100 or request to 127.0.0.1 +func InjectEnvoyAndVPN(ctx context.Context, nodeID string, f cmdutil.Factory, managerNamespace string, object *runtimeresource.Info, c util.PodRouteConfig, headers map[string]string, portMaps []string, secret *v1.Secret, image string) (err error) { var clientset *kubernetes.Clientset clientset, err = f.KubernetesClientSet() if err != nil { @@ -96,7 +96,7 @@ func InjectServiceMesh(ctx context.Context, nodeID string, f cmdutil.Factory, ma enableIPv6, _ := util.DetectPodSupportIPv6(ctx, f, managerNamespace) // (1) add mesh container - AddMeshContainer(templateSpec, object.Namespace, nodeID, enableIPv6, managerNamespace, secret, image) + AddVPNAndEnvoyContainer(templateSpec, object.Namespace, nodeID, enableIPv6, managerNamespace, secret, image) helper := pkgresource.NewHelper(object.Client, object.Mapping) ps := []P{ { @@ -142,9 +142,9 @@ func UnPatchContainer(ctx context.Context, nodeID string, factory cmdutil.Factor plog.G(ctx).Infof("Leaving workload %s", workload) - RemoveContainers(templateSpec) - if empty { + RemoveContainers(&templateSpec.Spec) + helper := pkgresource.NewHelper(object.Client, object.Mapping) // pod without controller if len(depth) == 0 { @@ -331,12 +331,3 @@ func removeEnvoyConfig(mapInterface v12.ConfigMapInterface, namespace string, no _, err = mapInterface.Update(context.Background(), configMap, metav1.UpdateOptions{}) return empty, found, err } - -func contains(a map[string]string, sub map[string]string) bool { - for k, v := range sub { - if a[k] != v { - return false - } - } - return true -} diff --git a/pkg/inject/proxy.go b/pkg/inject/proxy.go index adda27ea..8f5cf7ad 100644 --- a/pkg/inject/proxy.go +++ b/pkg/inject/proxy.go @@ -50,7 +50,7 @@ func InjectVPN(ctx context.Context, nodeID string, f util.Factory, managerNamesp return err } - AddContainer(&podTempSpec.Spec, c, managerNamespace, secret, image) + AddVPNContainer(&podTempSpec.Spec, c, managerNamespace, secret, image) workload := fmt.Sprintf("%s/%s", object.Mapping.Resource.Resource, object.Name) helper := resource.NewHelper(object.Client, object.Mapping) diff --git a/pkg/run/runconfig.go b/pkg/run/runconfig.go index eea4deec..71bf4733 100644 --- a/pkg/run/runconfig.go +++ b/pkg/run/runconfig.go @@ -97,7 +97,7 @@ func (option *Options) ConvertPodToContainerConfigList( mountVolume map[string][]mount.Mount, dnsConfig *dns.ClientConfig, ) (configList ConfigList, err error) { - inject.RemoveContainers(&temp) + inject.RemoveContainers(&temp.Spec) // move dev container to location first for index, c := range temp.Spec.Containers { if option.ContainerName == c.Name {