hotfix: gen envoy rule id by ns and resource uid (#500)

* hotfix: gen envoy rule id by ns and uid

.github/workflows/coverage.yml

@@ -56,7 +56,7 @@ jobs:
 
       - name: Wait for pods reviews to be ready
         run: |
-          kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=3600s
+          kubectl wait --for=condition=Ready pods --all --timeout=3600s
           kubectl get svc -A -o wide
           kubectl get pod -A -o wide
           kubectl get all -o wide

.github/workflows/test.yml

@@ -74,7 +74,7 @@ jobs:
 
       - name: Wait for pods reviews to be ready
         run: |
-          kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=3600s
+          kubectl wait --for=condition=Ready pods --all --timeout=3600s
           kubectl get svc -A -o wide
           kubectl get pod -A -o wide
           kubectl get all -o wide
@@ -113,12 +113,12 @@ jobs:
               }
             }
       - uses: azure/setup-kubectl@v4
-      - name: Install kind
+      - name: Install minikube
         run: |
           set -x
           docker version
-          brew install kind
-          kind create cluster
+          brew install minikube
+          minikube start --driver=docker --memory=max --cpus=max --wait=all --wait-timeout=60m
           kubectl cluster-info
           kubectl config view --flatten --raw
           kubectl get pod -A -o wide
@@ -146,7 +146,7 @@ jobs:
 
       - name: Wait for pods reviews to be ready
         run: |
-          kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=3600s
+          kubectl wait --for=condition=Ready pods --all --timeout=3600s
           kubectl get svc -A -o wide || true
           kubectl get pod -A -o wide || true
           kubectl get all -o wide || true

cmd/kubevpn/cmds/connect.go

@@ -81,10 +81,6 @@ func CmdConnect(f cmdutil.Factory) *cobra.Command {
 			if err != nil {
 				return err
 			}
-			helmNs, _ := util.GetHelmInstalledNamespace(cmd.Context(), f)
-			if helmNs != "" {
-				ns = helmNs
-			}
 			if !sshConf.IsEmpty() {
 				if ip := util.GetAPIServerFromKubeConfigBytes(bytes); ip != nil {
 					extraRoute.ExtraCIDR = append(extraRoute.ExtraCIDR, ip.String())

pkg/config/config.go

@@ -14,6 +14,9 @@ const (
 	// configmap name
 	ConfigMapPodTrafficManager = "kubevpn-traffic-manager"
 
+	// helm app name kubevpn
+	HelmAppNameKubevpn = "kubevpn"
+
 	// config map keys
 	KeyDHCP             = "DHCP"
 	KeyDHCP6            = "DHCP6"

pkg/controlplane/processor.go

@@ -57,7 +57,7 @@ func (p *Processor) ProcessFile(file NotifyMessage) error {
 		if len(config.Uid) == 0 {
 			continue
 		}
-		uid := fmt.Sprintf("%s_%s", config.Namespace, config.Uid)
+		uid := util.GenEnvoyUID(config.Namespace, config.Uid)
 		lastConfig, ok := p.expireCache.Get(uid)
 		if ok && reflect.DeepEqual(lastConfig.(*Virtual), config) {
 			marshal, _ := json.Marshal(config)

pkg/daemon/action/connect-fork.go

@@ -88,12 +88,6 @@ func (svr *Server) redirectConnectForkToSudoDaemon(req *rpc.ConnectRequest, resp
 	if cli == nil {
 		return fmt.Errorf("sudo daemon not start")
 	}
-	connect := &handler.ConnectOptions{
-		Namespace:            req.Namespace,
-		ExtraRouteInfo:       *handler.ParseExtraRouteFromRPC(req.ExtraRoute),
-		Engine:               config.Engine(req.Engine),
-		OriginKubeconfigPath: req.OriginKubeconfigPath,
-	}
 	var sshConf = ssh.ParseSshFromRPC(req.SshJump)
 	file, err := util.ConvertToTempKubeconfigFile([]byte(req.KubeconfigBytes))
 	if err != nil {
@@ -105,6 +99,12 @@ func (svr *Server) redirectConnectForkToSudoDaemon(req *rpc.ConnectRequest, resp
 		DefValue: file,
 	})
 	sshCtx, sshCancel := context.WithCancel(context.Background())
+	connect := &handler.ConnectOptions{
+		Namespace:            req.Namespace,
+		ExtraRouteInfo:       *handler.ParseExtraRouteFromRPC(req.ExtraRoute),
+		Engine:               config.Engine(req.Engine),
+		OriginKubeconfigPath: req.OriginKubeconfigPath,
+	}
 	connect.AddRolloutFunc(func() error {
 		sshCancel()
 		return nil
@@ -125,6 +125,14 @@ func (svr *Server) redirectConnectForkToSudoDaemon(req *rpc.ConnectRequest, resp
 		return err
 	}
 
+	helmNs, _ := util.GetHelmInstalledNamespace(sshCtx, connect.GetFactory())
+	if helmNs != "" {
+		logger.Infof("Using helm namespace: %s", helmNs)
+		connect.Namespace = helmNs
+	} else {
+		logger.Infof("Use namespace: %s", req.Namespace)
+	}
+
 	for _, options := range svr.secondaryConnect {
 		isSameCluster, _ := util.IsSameCluster(
 			sshCtx,

pkg/daemon/action/connect.go

@@ -103,12 +103,6 @@ func (svr *Server) redirectToSudoDaemon(req *rpc.ConnectRequest, resp rpc.Daemon
 	if cli == nil {
 		return fmt.Errorf("sudo daemon not start")
 	}
-	connect := &handler.ConnectOptions{
-		Namespace:            req.Namespace,
-		ExtraRouteInfo:       *handler.ParseExtraRouteFromRPC(req.ExtraRoute),
-		Engine:               config.Engine(req.Engine),
-		OriginKubeconfigPath: req.OriginKubeconfigPath,
-	}
 	var sshConf = ssh.ParseSshFromRPC(req.SshJump)
 	file, err := util.ConvertToTempKubeconfigFile([]byte(req.KubeconfigBytes))
 	if err != nil {
@@ -120,6 +114,12 @@ func (svr *Server) redirectToSudoDaemon(req *rpc.ConnectRequest, resp rpc.Daemon
 		DefValue: file,
 	})
 	sshCtx, sshCancel := context.WithCancel(context.Background())
+	connect := &handler.ConnectOptions{
+		Namespace:            req.Namespace,
+		ExtraRouteInfo:       *handler.ParseExtraRouteFromRPC(req.ExtraRoute),
+		Engine:               config.Engine(req.Engine),
+		OriginKubeconfigPath: req.OriginKubeconfigPath,
+	}
 	connect.AddRolloutFunc(func() error {
 		sshCancel()
 		return nil
@@ -140,6 +140,14 @@ func (svr *Server) redirectToSudoDaemon(req *rpc.ConnectRequest, resp rpc.Daemon
 		return err
 	}
 
+	helmNs, _ := util.GetHelmInstalledNamespace(sshCtx, connect.GetFactory())
+	if helmNs != "" {
+		logger.Infof("Using helm namespace: %s", helmNs)
+		connect.Namespace = helmNs
+	} else {
+		logger.Infof("Use namespace: %s", req.Namespace)
+	}
+
 	if svr.connect != nil {
 		isSameCluster, _ := util.IsSameCluster(
 			sshCtx,

pkg/daemon/action/proxy.go

@@ -73,7 +73,10 @@ func (svr *Server) Proxy(req *rpc.ProxyRequest, resp rpc.Daemon_ProxyServer) (e
 	}
 	helmNs, _ := util.GetHelmInstalledNamespace(ctx, connect.GetFactory())
 	if helmNs != "" {
+		logger.Infof("Using helm namespace: %s", helmNs)
 		connect.Namespace = helmNs
+	} else {
+		logger.Infof("Use namespace: %s", req.Namespace)
 	}
 
 	if svr.connect != nil {

pkg/handler/function_test.go

@@ -42,30 +42,30 @@ const (
 func TestFunctions(t *testing.T) {
 	// 1) test connect
 	Init()
-	kubevpnConnect(t)
-	commonTest(t)
+	t.Run("kubevpnConnect", kubevpnConnect)
+	t.Run("commonTest", commonTest)
 
 	// 2) test proxy mode
-	kubevpnProxy(t)
-	commonTest(t)
+	t.Run("kubevpnProxy", kubevpnProxy)
+	t.Run("commonTest", commonTest)
 	t.Run("testUDP", testUDP)
 	t.Run("proxyServiceReviewsServiceIP", proxyServiceReviewsServiceIP)
 	t.Run("proxyServiceReviewsPodIP", proxyServiceReviewsPodIP)
 
 	// 3) test proxy mode with service mesh
-	kubevpnLeave(t)
-	kubevpnProxyWithServiceMesh(t)
-	commonTest(t)
+	t.Run("kubevpnLeave", kubevpnLeave)
+	t.Run("kubevpnProxyWithServiceMesh", kubevpnProxyWithServiceMesh)
+	t.Run("commonTest", commonTest)
 	t.Run("serviceMeshReviewsServiceIP", serviceMeshReviewsServiceIP)
 	t.Run("serviceMeshReviewsPodIP", serviceMeshReviewsPodIP)
 
 	// 4) test proxy mode with service mesh and gvisor
-	kubevpnLeave(t)
-	kubevpnUninstall(t)
-	kubevpnProxyWithServiceMeshAndGvisorMode(t)
-	commonTest(t)
+	t.Run("kubevpnLeave", kubevpnLeave)
+	t.Run("kubevpnUninstall", kubevpnUninstall)
+	t.Run("kubevpnProxyWithServiceMeshAndGvisorMode", kubevpnProxyWithServiceMeshAndGvisorMode)
+	t.Run("commonTest", commonTest)
 	t.Run("serviceMeshReviewsServiceIP", serviceMeshReviewsServiceIP)
-	kubevpnQuit(t)
+	t.Run("kubevpnQuit", kubevpnQuit)
 }
 
 func commonTest(t *testing.T) {

pkg/handler/proxy.go

@@ -181,7 +181,6 @@ func (m *Mapper) Run(connectNamespace string) {
 }
 
 func (m *Mapper) getLocalPort2EnvoyRulePort(connectNamespace string) (map[int32]int32, error) {
-	// todo get kubevpn-system configmap
 	configMap, err := m.clientset.CoreV1().ConfigMaps(connectNamespace).Get(m.ctx, config.ConfigMapPodTrafficManager, v1.GetOptions{})
 	if err != nil {
 		return nil, err

pkg/handler/uninstall.go

@@ -110,7 +110,7 @@ func (c *ConnectOptions) LeaveAllProxyResources(ctx context.Context) (err error)
 			return rule.LocalTunIPv4 == v4
 		})
 		if err != nil {
-			plog.G(ctx).Errorf("Failed to leave workload %s: %v", workload, err)
+			plog.G(ctx).Errorf("Failed to leave workload %s in namespace %s: %v", workload.workload, workload.namespace, err)
 			continue
 		}
 		if empty {

pkg/inject/controller.go

@@ -35,7 +35,7 @@ func RemoveContainers(spec *v1.PodTemplateSpec) {
 }
 
 // AddMeshContainer todo envoy support ipv6
-func AddMeshContainer(spec *v1.PodTemplateSpec, nodeId string, c util.PodRouteConfig, ipv6 bool) {
+func AddMeshContainer(spec *v1.PodTemplateSpec, ns, nodeId string, c util.PodRouteConfig, ipv6 bool) {
 	// remove envoy proxy containers if already exist
 	RemoveContainers(spec)
 
@@ -131,9 +131,9 @@ kubevpn serve -L "tun:/localhost:8422?net=${TunIPv4}&route=${CIDR4}" -F "tcp://$
 			"--base-id",
 			"1",
 			"--service-node",
-			nodeId,
+			util.GenEnvoyUID(ns, nodeId),
 			"--service-cluster",
-			nodeId,
+			util.GenEnvoyUID(ns, nodeId),
 			"--config-yaml",
 		},
 		Args: []string{
@@ -158,7 +158,7 @@ kubevpn serve -L "tun:/localhost:8422?net=${TunIPv4}&route=${CIDR4}" -F "tcp://$
 	})
 }
 
-func AddEnvoyContainer(spec *v1.PodTemplateSpec, nodeId string, ipv6 bool) {
+func AddEnvoyContainer(spec *v1.PodTemplateSpec, ns, nodeId string, ipv6 bool) {
 	// remove envoy proxy containers if already exist
 	RemoveContainers(spec)
 
@@ -195,9 +195,9 @@ kubevpn serve -L "ssh://:2222"`,
 			"--base-id",
 			"1",
 			"--service-node",
-			nodeId,
+			util.GenEnvoyUID(ns, nodeId),
 			"--service-cluster",
-			nodeId,
+			util.GenEnvoyUID(ns, nodeId),
 			"--config-yaml",
 		},
 		Args: []string{

pkg/inject/fargate.go

@@ -66,7 +66,7 @@ func InjectEnvoySidecar(ctx context.Context, f cmdutil.Factory, clientset *kuber
 
 	enableIPv6, _ := util.DetectPodSupportIPv6(ctx, f, connectNamespace)
 	// (1) add mesh container
-	AddEnvoyContainer(templateSpec, nodeID, enableIPv6)
+	AddEnvoyContainer(templateSpec, object.Namespace, nodeID, enableIPv6)
 	helper := pkgresource.NewHelper(object.Client, object.Mapping)
 	ps := []P{
 		{

pkg/inject/mesh.go

@@ -90,7 +90,7 @@ func InjectVPNAndEnvoySidecar(ctx context.Context, f cmdutil.Factory, mapInterfa
 
 	enableIPv6, _ := util.DetectPodSupportIPv6(ctx, f, connectNamespace)
 	// (1) add mesh container
-	AddMeshContainer(templateSpec, nodeID, c, enableIPv6)
+	AddMeshContainer(templateSpec, object.Namespace, nodeID, c, enableIPv6)
 	helper := pkgresource.NewHelper(object.Client, object.Mapping)
 	ps := []P{
 		{

pkg/util/helm.go

@@ -2,12 +2,15 @@ package util
 
 import (
 	"context"
+    "fmt"
 	"os"
 
 	"github.com/pkg/errors"
 	"helm.sh/helm/v4/pkg/action"
 	"helm.sh/helm/v4/pkg/release/v1"
 	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/config"
 )
 
 // GetHelmInstalledNamespace
@@ -28,10 +31,10 @@ func GetHelmInstalledNamespace(ctx context.Context, f cmdutil.Factory) (string,
 		return "", err
 	}
 	for _, app := range releases {
-		if app.Name == "kubevpn" &&
+		if app.Name == config.HelmAppNameKubevpn &&
 			app.Info != nil && app.Info.Status == v1.StatusDeployed {
 			return app.Namespace, nil
 		}
 	}
-	return "", errors.New("app kubevpn not found")
+	return "", errors.New(fmt.Sprintf("app %s not found", config.HelmAppNameKubevpn))
 }

pkg/util/name.go

@@ -12,3 +12,7 @@ func Join(names ...string) string {
 func ContainerNet(name string) string {
 	return fmt.Sprintf("container:%s", name)
 }
+
+func GenEnvoyUID(ns, uid string) string {
+	return fmt.Sprintf("%s.%s", ns, uid)
+}