diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aa8d069e..903e9db4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -157,7 +157,7 @@ jobs: - name: Test # for docker mount env: - TMPDIR: /tmp + TMPDIR: . run: make ut windows: diff --git a/cmd/kubevpn/cmds/proxy.go b/cmd/kubevpn/cmds/proxy.go index 90fe836b..fa977f05 100644 --- a/cmd/kubevpn/cmds/proxy.go +++ b/cmd/kubevpn/cmds/proxy.go @@ -156,6 +156,11 @@ func CmdProxy(f cmdutil.Factory) *cobra.Command { if err != nil { return err } + err = disconnect(cli, bytes, ns, sshConf) + if err != nil { + return err + } + _, _ = fmt.Fprint(os.Stdout, "Disconnect completed") } return nil }, diff --git a/pkg/handler/function_test.go b/pkg/handler/function_test.go index ff46fad5..af79be72 100644 --- a/pkg/handler/function_test.go +++ b/pkg/handler/function_test.go @@ -11,6 +11,7 @@ import ( "os/exec" "reflect" "runtime" + "strings" "sync" "testing" "time" @@ -91,15 +92,10 @@ func TestFunctions(t *testing.T) { t.Run("kubevpnUnSync", u.kubevpnUnSync) // 6) test mode run - // because of: - // Run container with cmd: [docker run --env-file /tmp/623917040.env --domainname --workdir --cap-add SYS_PTRACE --cap-add SYS_ADMIN --cap-add SYS_PTRACE --cap-add SYS_ADMIN --security-opt apparmor=unconfined --security-opt seccomp=unconfined --pull missing --name default_authors_716db --user root --env LC_ALL=C.UTF-8 --label app=authors --volume /tmp/329021857635767916:/var/run/secrets/kubernetes.io/serviceaccount --network container:default_nginx_45ee1 --pid container:default_nginx_45ee1 --pull missing --attach STDIN --attach STDOUT --attach STDERR --interactive --privileged --volume /tmp/TestFunctionskubevpnRunWithFullProxy2095435677/001:/app/test --rm --entrypoint go ghcr.io/kubenetworks/authors:latest run /app/test/main.go] - // Error: stat /app/test/main.go: no such file or directory - if runtime.GOOS != "darwin" { - t.Run("resetDeployAuthors", u.resetDeployAuthors) - t.Run("kubevpnRunWithFullProxy", u.kubevpnRunWithFullProxy) - t.Run("kubevpnRunWithServiceMesh", u.kubevpnRunWithServiceMesh) - t.Run("kubevpnQuit", u.kubevpnQuit) - } + t.Run("resetDeployAuthors", u.resetDeployAuthors) + t.Run("kubevpnRunWithFullProxy", u.kubevpnRunWithFullProxy) + t.Run("kubevpnRunWithServiceMesh", u.kubevpnRunWithServiceMesh) + t.Run("kubevpnQuit", u.kubevpnQuit) // 7) install centrally in ns test -- connect mode t.Run("centerKubevpnUninstall", u.kubevpnUninstall) @@ -152,15 +148,10 @@ func TestFunctions(t *testing.T) { t.Run("kubevpnQuit", u.kubevpnQuit) // 12) test mode run - // because of: - // Run container with cmd: [docker run --env-file /tmp/623917040.env --domainname --workdir --cap-add SYS_PTRACE --cap-add SYS_ADMIN --cap-add SYS_PTRACE --cap-add SYS_ADMIN --security-opt apparmor=unconfined --security-opt seccomp=unconfined --pull missing --name default_authors_716db --user root --env LC_ALL=C.UTF-8 --label app=authors --volume /tmp/329021857635767916:/var/run/secrets/kubernetes.io/serviceaccount --network container:default_nginx_45ee1 --pid container:default_nginx_45ee1 --pull missing --attach STDIN --attach STDOUT --attach STDERR --interactive --privileged --volume /tmp/TestFunctionskubevpnRunWithFullProxy2095435677/001:/app/test --rm --entrypoint go ghcr.io/kubenetworks/authors:latest run /app/test/main.go] - // Error: stat /app/test/main.go: no such file or directory - if runtime.GOOS != "darwin" { - t.Run("resetDeployAuthors", u.resetDeployAuthors) - t.Run("kubevpnRunWithFullProxy", u.kubevpnRunWithFullProxy) - t.Run("kubevpnRunWithServiceMesh", u.kubevpnRunWithServiceMesh) - t.Run("kubevpnQuit", u.kubevpnQuit) - } + t.Run("resetDeployAuthors", u.resetDeployAuthors) + t.Run("kubevpnRunWithFullProxy", u.kubevpnRunWithFullProxy) + t.Run("kubevpnRunWithServiceMesh", u.kubevpnRunWithServiceMesh) + t.Run("kubevpnQuit", u.kubevpnQuit) } func (u *ut) commonTest(t *testing.T) { @@ -216,6 +207,20 @@ func (u *ut) healthCheckPodDetails(t *testing.T) { } func (u *ut) healthChecker(t *testing.T, endpoint string, header map[string]string, keyword string) { + // 0 = this frame. + _, file, line, ok := runtime.Caller(1) + if ok { + // Trim any directory path from the file. + slash := strings.LastIndexByte(file, byte('/')) + if slash >= 0 { + file = file[slash+1:] + } + } else { + // We don't have a filename. + file = "???" + line = 0 + } + req, err := http.NewRequest("GET", endpoint, nil) if err != nil { t.Fatal(err) @@ -226,7 +231,7 @@ func (u *ut) healthChecker(t *testing.T, endpoint string, header map[string]stri client := &http.Client{Timeout: time.Second * 1} err = retry.OnError( - wait.Backoff{Duration: time.Second, Factor: 1, Jitter: 0, Steps: 240}, + wait.Backoff{Duration: time.Second, Factor: 1, Jitter: 0, Steps: 120}, func(err error) bool { return err != nil }, func() error { var resp *http.Response @@ -259,7 +264,7 @@ func (u *ut) healthChecker(t *testing.T, endpoint string, header map[string]stri ) if err != nil { u.kubectl(t) - t.Fatal(err) + t.Fatal(fmt.Sprintf("%s:%d", file, line), err) } } @@ -873,10 +878,10 @@ func (u *ut) checkServiceShouldNotInNsDefault(t *testing.T) { } func (u *ut) kubectl(t *testing.T) { - cmdGetPod := exec.Command("kubectl", "get", "pods", "-o", "wide") - cmdDescribePod := exec.Command("kubectl", "describe", "pods") - cmdGetSvc := exec.Command("kubectl", "get", "services", "-o", "wide") - cmdDescribeSvc := exec.Command("kubectl", "describe", "services") + cmdGetPod := exec.Command("kubectl", "get", "pods", "-o", "wide", "-A") + cmdGetSvc := exec.Command("kubectl", "get", "services", "-o", "wide", "-A") + cmdDescribePod := exec.Command("kubectl", "describe", "pods", "-A") + cmdDescribeSvc := exec.Command("kubectl", "describe", "services", "-A") for _, cmd := range []*exec.Cmd{cmdGetPod, cmdDescribePod, cmdGetSvc, cmdDescribeSvc} { t.Logf("exec: %v", cmd.Args) cmd.Stdout = os.Stdout diff --git a/pkg/handler/mode_sync_test.go b/pkg/handler/mode_sync_test.go index 850aea59..1884367a 100644 --- a/pkg/handler/mode_sync_test.go +++ b/pkg/handler/mode_sync_test.go @@ -164,7 +164,8 @@ func (u *ut) checkContent(ctx context.Context, t *testing.T, podName string, con } } -func (u *ut) TestCompile(t *testing.T) { +func TestCompile(t *testing.T) { + u := &ut{} u.writeTempFile(t) } @@ -187,7 +188,11 @@ func (u *ut) writeTempFile(t *testing.T) string { if err != nil { t.Fatal(err) } - return temp.Name() + absPath, err := filepath.Abs(temp.Name()) + if err != nil { + t.Fatal(err) + } + return absPath } func (u *ut) checkSyncWithFullProxyStatus(t *testing.T) { diff --git a/pkg/handler/sync.go b/pkg/handler/sync.go index 777a4b90..4b739c46 100644 --- a/pkg/handler/sync.go +++ b/pkg/handler/sync.go @@ -122,6 +122,8 @@ func (d *SyncOptions) DoSync(ctx context.Context, kubeconfigJsonBytes []byte, im "origin-workload": originName, } u.SetLabels(labels.Merge(u.GetLabels(), labelsMap)) + // for leave resource and disconnect from cluster + u.SetDeletionGracePeriodSeconds(ptr.To[int64](60)) var path []string var spec *v1.PodTemplateSpec spec, path, err = util.GetPodTemplateSpecPath(u) @@ -489,6 +491,13 @@ func (d *SyncOptions) Cleanup(ctx context.Context, workloads ...string) error { } } } + for _, workload := range d.Workloads { + plog.G(ctx).Infof("Wait workload %s", workload) + err := util.RolloutStatus(ctx, d.factory, d.Namespace, workload, time.Minute*60) + if err != nil { + plog.G(ctx).Warnf("Failed to rollback workload %s: %v", workload, err) + } + } return nil } diff --git a/pkg/inject/mesh.go b/pkg/inject/mesh.go index ef24d60d..39dafad0 100644 --- a/pkg/inject/mesh.go +++ b/pkg/inject/mesh.go @@ -21,6 +21,7 @@ import ( runtimeresource "k8s.io/cli-runtime/pkg/resource" "k8s.io/client-go/kubernetes" v12 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/util/retry" cmdutil "k8s.io/kubectl/pkg/cmd/util" "sigs.k8s.io/yaml" @@ -179,25 +180,27 @@ func UnPatchContainer(ctx context.Context, nodeID string, factory cmdutil.Factor } func addEnvoyConfig(mapInterface v12.ConfigMapInterface, ns, nodeID string, tunIP util.PodRouteConfig, headers map[string]string, port []controlplane.ContainerPort, portmap map[int32]string) error { - configMap, err := mapInterface.Get(context.Background(), config.ConfigMapPodTrafficManager, metav1.GetOptions{}) - if err != nil { - return err - } - var v = make([]*controlplane.Virtual, 0) - if str, ok := configMap.Data[config.KeyEnvoy]; ok { - if err = yaml.Unmarshal([]byte(str), &v); err != nil { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + configMap, err := mapInterface.Get(context.Background(), config.ConfigMapPodTrafficManager, metav1.GetOptions{}) + if err != nil { return err } - } + var v = make([]*controlplane.Virtual, 0) + if str, ok := configMap.Data[config.KeyEnvoy]; ok { + if err = yaml.Unmarshal([]byte(str), &v); err != nil { + return err + } + } - v = addVirtualRule(v, ns, nodeID, port, headers, tunIP, portmap) - marshal, err := yaml.Marshal(v) - if err != nil { + v = addVirtualRule(v, ns, nodeID, port, headers, tunIP, portmap) + marshal, err := yaml.Marshal(v) + if err != nil { + return err + } + configMap.Data[config.KeyEnvoy] = string(marshal) + _, err = mapInterface.Update(context.Background(), configMap, metav1.UpdateOptions{}) return err - } - configMap.Data[config.KeyEnvoy] = string(marshal) - _, err = mapInterface.Update(context.Background(), configMap, metav1.UpdateOptions{}) - return err + }) } func addVirtualRule(v []*controlplane.Virtual, ns, nodeID string, port []controlplane.ContainerPort, headers map[string]string, tunIP util.PodRouteConfig, portmap map[int32]string) []*controlplane.Virtual { diff --git a/pkg/run/runconfig.go b/pkg/run/runconfig.go index debea579..eea4deec 100644 --- a/pkg/run/runconfig.go +++ b/pkg/run/runconfig.go @@ -4,7 +4,7 @@ import ( "context" "fmt" "net" - "os" + "path/filepath" "strconv" "strings" @@ -12,6 +12,7 @@ import ( "github.com/docker/docker/api/types/mount" "github.com/docker/go-connections/nat" "github.com/google/uuid" + "github.com/joho/godotenv" "github.com/miekg/dns" corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" @@ -120,8 +121,12 @@ func (option *Options) ConvertPodToContainerConfigList( for index, container := range temp.Spec.Containers { name := util.Join(option.Namespace, container.Name) randomName := util.Join(name, strings.ReplaceAll(uuid.New().String(), "-", "")[:5]) + localEnv, err := filepath.Abs(envMap[name]) + if err != nil { + return nil, err + } var options = []string{ - "--env-file", envMap[name], + "--env-file", localEnv, "--domainname", temp.Spec.Subdomain, "--workdir", container.WorkingDir, "--cap-add", "SYS_PTRACE", @@ -145,7 +150,11 @@ func (option *Options) ConvertPodToContainerConfigList( options = append(options, "--privileged") } for _, m := range mountVolume[name] { - options = append(options, "--volume", fmt.Sprintf("%s:%s", m.Source, m.Target)) + localDir, err := filepath.Abs(m.Source) + if err != nil { + return nil, err + } + options = append(options, "--volume", fmt.Sprintf("%s:%s", localDir, m.Target)) } for _, port := range container.Ports { @@ -269,16 +278,14 @@ func MergeDockerOptions(conf *RunConfig, options *Options, config *Config, hostC } func GetEnvByKey(filepath string, key string, defaultValue string) string { - content, err := os.ReadFile(filepath) + parse, err := godotenv.Read(filepath) if err != nil { return defaultValue } - for _, kv := range strings.Split(string(content), "\n") { - env := strings.Split(kv, "=") - if len(env) == 2 && env[0] == key { - return env[1] - } + value, ok := parse[key] + if !ok { + return defaultValue } - return defaultValue + return value }