feat: optimize code

cmd/kubevpn/cmds/cp.go

@@ -65,7 +65,7 @@ func CmdCp(f cmdutil.Factory) *cobra.Command {
 		Use:                   "cp <file-spec-src> <file-spec-dest>",
 		DisableFlagsInUseLine: true,
 		Short:                 i18n.T("Copy files and directories to and from containers"),
-		Long:                  i18n.T("Copy files and directories to and from containers."),
+		Long:                  i18n.T("Copy files and directories to and from containers. Different between kubectl cp is it will de-reference symbol link."),
 		Example:               cpExample,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			cmdutil.CheckErr(handler.SshJump(sshConf, cmd.Flags()))

cmd/kubevpn/cmds/dev.go

@@ -32,15 +32,22 @@ func CmdDev(f cmdutil.Factory) *cobra.Command {
 		Env:        opts.NewListOpts(nil),
 		Volumes:    opts.NewListOpts(nil),
 		ExtraHosts: opts.NewListOpts(nil),
-		Aliases:    opts.NewListOpts(nil),
+		//Aliases:    opts.NewListOpts(nil),
-		NoProxy:    false,
+		NoProxy:   false,
-		ExtraCIDR:  []string{},
+		ExtraCIDR: []string{},
 	}
 	var sshConf = &util.SshConfig{}
 	cmd := &cobra.Command{
 		Use:   "dev",
-		Short: i18n.T("Startup your workloads in local Docker container use same volume、env、and network with cluster"),
+		Short: i18n.T("Startup your kubernetes workloads in local Docker container with same volume、env、and network"),
-		Long:  templates.LongDesc(i18n.T(`Startup your workloads in local Docker container use same volume、env、and network with cluster`)),
+		Long: templates.LongDesc(i18n.T(`
+Startup your kubernetes workloads in local Docker container with same volume、env、and network
+
+## What did i do:
+- Download volume which MountPath point to, mount to docker container
+- Connect to cluster network, set network to docker container
+- Get all environment with command (env), set env to docker container
+`)),
 		Example: templates.Examples(i18n.T(`
         # Develop workloads
 		- develop deployment
@@ -150,15 +157,14 @@ func CmdDev(f cmdutil.Factory) *cobra.Command {
 
 	// docker options
 	cmd.Flags().Var(&devOptions.ExtraHosts, "add-host", "Add a custom host-to-IP mapping (host:ip)")
-	//cmd.Flags().StringVar(&devOptions.ParentContainer, "parent-container", "", "Parent container name if running in Docker (Docker in Docker)")
 	// We allow for both "--net" and "--network", although the latter is the recommended way.
-	cmd.Flags().Var(&devOptions.NetMode, "net", "Connect a container to a network")
+	cmd.Flags().Var(&devOptions.NetMode, "net", "Connect a container to a network, eg: [default|bridge|host|none|container:$CONTAINER_ID]")
 	cmd.Flags().Var(&devOptions.NetMode, "network", "Connect a container to a network")
 	cmd.Flags().MarkHidden("net")
 	// We allow for both "--net-alias" and "--network-alias", although the latter is the recommended way.
-	cmd.Flags().Var(&devOptions.Aliases, "net-alias", "Add network-scoped alias for the container")
+	//cmd.Flags().Var(&devOptions.Aliases, "net-alias", "Add network-scoped alias for the container")
-	cmd.Flags().Var(&devOptions.Aliases, "network-alias", "Add network-scoped alias for the container")
+	//cmd.Flags().Var(&devOptions.Aliases, "network-alias", "Add network-scoped alias for the container")
-	cmd.Flags().MarkHidden("net-alias")
+	//cmd.Flags().MarkHidden("net-alias")
 	cmd.Flags().VarP(&devOptions.Volumes, "volume", "v", "Bind mount a volume")
 	cmd.Flags().Var(&devOptions.Mounts, "mount", "Attach a filesystem mount to the container")
 	cmd.Flags().Var(&devOptions.Expose, "expose", "Expose a port or a range of ports")

pkg/dev/convert.go

@@ -3,6 +3,9 @@ package dev
 import (
 	"context"
 	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"unsafe"
@@ -14,12 +17,18 @@ import (
 	"github.com/docker/go-connections/nat"
 	"github.com/google/uuid"
 	miekgdns "github.com/miekg/dns"
+	"github.com/moby/term"
 	v12 "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/spf13/cobra"
 	"k8s.io/api/core/v1"
 	v13 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/kubectl/pkg/cmd/util"
 
+	"github.com/wencaiwulue/kubevpn/pkg/config"
+	"github.com/wencaiwulue/kubevpn/pkg/cp"
 	"github.com/wencaiwulue/kubevpn/pkg/dns"
+	"github.com/wencaiwulue/kubevpn/pkg/handler"
 )
 
 type RunConfig struct {
@@ -143,7 +152,7 @@ func ConvertKubeResourceToContainer(namespace string, temp v1.PodTemplateSpec, e
 		r.k8sContainerName = c.Name
 		r.config = config
 		r.hostConfig = hostConfig
-		r.networkingConfig = nil
+		r.networkingConfig = &network.NetworkingConfig{EndpointsConfig: make(map[string]*network.EndpointSettings)}
 		r.platform = /*&v12.Platform{Architecture: "amd64", OS: "linux"}*/ nil
 
 		runConfigList = append(runConfigList, &r)
@@ -176,3 +185,63 @@ func GetDNS(ctx context.Context, f util.Factory, ns, pod string) (*miekgdns.Clie
 	}
 	return fromPod, nil
 }
+
+// GetVolume key format: [container name]-[volume mount name]
+func GetVolume(ctx context.Context, f util.Factory, ns, pod string) (map[string][]mount.Mount, error) {
+	clientSet, err := f.KubernetesClientSet()
+	if err != nil {
+		return nil, err
+	}
+	var get *v1.Pod
+	get, err = clientSet.CoreV1().Pods(ns).Get(ctx, pod, v13.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+	result := map[string][]mount.Mount{}
+	for _, c := range get.Spec.Containers {
+		// if container name is vpn or envoy-proxy, not need to download volume
+		if c.Name == config.ContainerSidecarVPN || c.Name == config.ContainerSidecarEnvoyProxy {
+			continue
+		}
+		var m []mount.Mount
+		for _, volumeMount := range c.VolumeMounts {
+			if volumeMount.MountPath == "/tmp" {
+				continue
+			}
+			join := filepath.Join(os.TempDir(), strconv.Itoa(rand.Int()))
+			err = os.MkdirAll(join, 0755)
+			if err != nil {
+				return nil, err
+			}
+			if volumeMount.SubPath != "" {
+				join = filepath.Join(join, volumeMount.SubPath)
+			}
+			handler.RollbackFuncList = append(handler.RollbackFuncList, func() {
+				_ = os.RemoveAll(join)
+			})
+			// pod-namespace/pod-name:path
+			remotePath := fmt.Sprintf("%s/%s:%s", ns, pod, volumeMount.MountPath)
+			stdIn, stdOut, stdErr := term.StdStreams()
+			copyOptions := cp.NewCopyOptions(genericclioptions.IOStreams{In: stdIn, Out: stdOut, ErrOut: stdErr})
+			copyOptions.Container = c.Name
+			copyOptions.MaxTries = 10
+			err = copyOptions.Complete(f, &cobra.Command{}, []string{remotePath, join})
+			if err != nil {
+				return nil, err
+			}
+			err = copyOptions.Run()
+			if err != nil {
+				_, _ = fmt.Fprintf(os.Stderr, "failed to download volume %s path %s to %s, err: %v, ignore...\n", volumeMount.Name, remotePath, join, err)
+				continue
+			}
+			m = append(m, mount.Mount{
+				Type:   mount.TypeBind,
+				Source: join,
+				Target: volumeMount.MountPath,
+			})
+			fmt.Printf("%s:%s\n", join, volumeMount.MountPath)
+		}
+		result[c.Name] = m
+	}
+	return result, nil
+}

pkg/dev/main.go

@@ -44,14 +44,14 @@ type Options struct {
 	// docker options
 	Platform string
 	//Pull         string // always, missing, never
-	PublishAll   bool
+	PublishAll  bool
-	Entrypoint   string
+	Entrypoint  string
-	DockerImage  string
+	DockerImage string
-	Publish      opts.ListOpts
+	Publish     opts.ListOpts
-	Expose       opts.ListOpts
+	Expose      opts.ListOpts
-	ExtraHosts   opts.ListOpts
+	ExtraHosts  opts.ListOpts
-	NetMode      opts.NetworkOpt
+	NetMode     opts.NetworkOpt
-	Aliases      opts.ListOpts
+	//Aliases      opts.ListOpts
 	Env          opts.ListOpts
 	Mounts       opts.MountOpt
 	Volumes      opts.ListOpts
@@ -99,7 +99,7 @@ func (d Options) Main(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	volume, err := util.GetVolume(ctx, d.Factory, d.Namespace, pod)
+	volume, err := GetVolume(ctx, d.Factory, d.Namespace, pod)
 	if err != nil {
 		return err
 	}
@@ -232,7 +232,7 @@ func (r Run) Run(ctx context.Context, volume map[string][]mount.Mount) error {
 		id, err = run(ctx, config, cli)
 		if err != nil {
 			// try another way to startup container
-			log.Info("try another way to startup container")
+			log.Infof("occur err: %v, try another way to startup container...", err)
 			config.hostConfig.Mounts = nil
 			id, err = run(ctx, config, cli)
 			if err != nil {

pkg/dev/option.go

@@ -148,11 +148,6 @@ func fillOptions(r Run, copts Options) error {
 
 	config.hostConfig.Binds = binds
 
-	// todo
-	if copts.Aliases.Len() != 0 {
-		//config.networkingConfig.EndpointsConfig
-	}
-
 	return nil
 }
 

pkg/dev/run.go

@@ -70,6 +70,11 @@ func run(ctx context.Context, runConfig *RunConfig, cli *client.Client) (id stri
 	}
 	id = create.ID
 	log.Infof("Created container: %s", name)
+	defer func() {
+		if err != nil {
+			_ = cli.ContainerRemove(ctx, id, types.ContainerRemoveOptions{Force: true})
+		}
+	}()
 
 	err = cli.ContainerStart(ctx, create.ID, types.ContainerStartOptions{})
 	if err != nil {

pkg/dns/dns_windows.go

@@ -53,7 +53,7 @@ func SetupDNS(clientConfig *miekgdns.ClientConfig, _ []string) error {
 
 func CancelDNS() {
 	updateHosts("")
-	getenv := os.Getenv("luid")
+	getenv := os.Getenv(config.EnvTunNameOrLUID)
 	parseUint, err := strconv.ParseUint(getenv, 10, 64)
 	if err != nil {
 		log.Warningln(err)
@@ -61,6 +61,7 @@ func CancelDNS() {
 	}
 	luid := winipcfg.LUID(parseUint)
 	_ = luid.FlushDNS(windows.AF_INET)
+	_ = luid.FlushRoutes(windows.AF_INET)
 }
 
 func updateNicMetric(name string) error {

pkg/driver/driver.go

@@ -33,11 +33,11 @@ func InstallWireGuardTunDriver() {
 }
 
 func UninstallWireGuardTunDriver() error {
-	wd, err := os.Getwd()
+	executable, err := os.Executable()
 	if err != nil {
 		return err
 	}
-	filename := filepath.Join(wd, "wintun.dll")
+	filename := filepath.Join(filepath.Dir(executable), "wintun.dll")
 	return os.Remove(filename)
 }
 

pkg/handler/cleaner.go

@@ -27,34 +27,34 @@ var stopChan = make(chan os.Signal)
 var RollbackFuncList = make([]func(), 2)
 var ctx, cancel = context.WithCancel(context.Background())
 
-func (c *ConnectOptions) addCleanUpResourceHandler(clientset *kubernetes.Clientset, namespace string) {
+func (c *ConnectOptions) addCleanUpResourceHandler() {
-	signal.Notify(stopChan, os.Interrupt, os.Kill, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGKILL /*, syscall.SIGSTOP*/)
+	signal.Notify(stopChan, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGKILL)
 	go func() {
 		<-stopChan
 		log.Info("prepare to exit, cleaning up")
-		dns.CancelDNS()
 		err := c.dhcp.ReleaseIpToDHCP(c.usedIPs...)
 		if err != nil {
 			log.Errorf("failed to release ip to dhcp, err: %v", err)
 		}
-		cancel()
 		for _, function := range RollbackFuncList {
 			if function != nil {
 				function()
 			}
 		}
-		_ = clientset.CoreV1().Pods(namespace).Delete(context.Background(), config.CniNetName, v1.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)})
+		_ = c.clientset.CoreV1().Pods(c.Namespace).Delete(context.Background(), config.CniNetName, v1.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)})
 		var count int
-		count, err = updateRefCount(clientset.CoreV1().ConfigMaps(namespace), config.ConfigMapPodTrafficManager, -1)
+		count, err = updateRefCount(c.clientset.CoreV1().ConfigMaps(c.Namespace), config.ConfigMapPodTrafficManager, -1)
 		if err == nil {
 			// if ref-count is less than zero or equals to zero, means nobody is using this traffic pod, so clean it
 			if count <= 0 {
 				log.Info("ref-count is zero, prepare to clean up resource")
-				cleanup(clientset, namespace, config.ConfigMapPodTrafficManager, true)
+				cleanup(c.clientset, c.Namespace, config.ConfigMapPodTrafficManager, true)
 			}
 		} else {
 			log.Error(err)
 		}
+		dns.CancelDNS()
+		cancel()
 		log.Info("clean up successful")
 		util.CleanExtensionLib()
 		os.Exit(0)

pkg/handler/connect.go

@@ -71,20 +71,18 @@ func (c *ConnectOptions) createRemoteInboundPod(ctx1 context.Context) (err error
 	}
 
 	for _, workload := range c.Workloads {
-		if len(workload) > 0 {
+		configInfo := util.PodRouteConfig{
-			configInfo := util.PodRouteConfig{
+			LocalTunIP:           c.localTunIP.IP.String(),
-				LocalTunIP:           c.localTunIP.IP.String(),
+			TrafficManagerRealIP: c.routerIP.String(),
-				TrafficManagerRealIP: c.routerIP.String(),
+		}
-			}
+		// means mesh mode
-			// means mesh mode
+		if len(c.Headers) != 0 {
-			if len(c.Headers) != 0 {
+			err = InjectVPNAndEnvoySidecar(ctx1, c.factory, c.clientset.CoreV1().ConfigMaps(c.Namespace), c.Namespace, workload, configInfo, c.Headers)
-				err = InjectVPNAndEnvoySidecar(ctx1, c.factory, c.clientset.CoreV1().ConfigMaps(c.Namespace), c.Namespace, workload, configInfo, c.Headers)
+		} else {
-			} else {
+			err = InjectVPNSidecar(ctx1, c.factory, c.Namespace, workload, configInfo)
-				err = InjectVPNSidecar(ctx1, c.factory, c.Namespace, workload, configInfo)
+		}
-			}
+		if err != nil {
-			if err != nil {
+			return err
-				return err
-			}
 		}
 	}
 	return
@@ -115,7 +113,7 @@ func Rollback(f cmdutil.Factory, ns, workload string) {
 }
 
 func (c *ConnectOptions) DoConnect() (err error) {
-	c.addCleanUpResourceHandler(c.clientset, c.Namespace)
+	c.addCleanUpResourceHandler()
 	trafficMangerNet := net.IPNet{IP: config.RouterIP, Mask: config.CIDR.Mask}
 	c.dhcp = NewDHCPManager(c.clientset.CoreV1().ConfigMaps(c.Namespace), c.Namespace, &trafficMangerNet)
 	if err = c.dhcp.InitDHCP(ctx); err != nil {

pkg/util/pod.go

@@ -5,24 +5,13 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"math/rand"
-	"os"
-	"path/filepath"
-	"strconv"
 	"strings"
 	"text/tabwriter"
 
-	"github.com/docker/docker/api/types/mount"
-	"github.com/moby/term"
-	"github.com/spf13/cobra"
 	"golang.org/x/exp/constraints"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/kubectl/pkg/cmd/util"
-
-	"github.com/wencaiwulue/kubevpn/pkg/config"
-	"github.com/wencaiwulue/kubevpn/pkg/cp"
 )
 
 func PrintStatus(pod *corev1.Pod, writer io.Writer) {
@@ -103,60 +92,3 @@ func GetEnv(ctx context.Context, f util.Factory, ns, pod string) (map[string][]s
 	}
 	return result, nil
 }
-
-// GetVolume key format: [container name]-[volume mount name]
-func GetVolume(ctx context.Context, f util.Factory, ns, pod string) (map[string][]mount.Mount, error) {
-	clientSet, err := f.KubernetesClientSet()
-	if err != nil {
-		return nil, err
-	}
-	var get *corev1.Pod
-	get, err = clientSet.CoreV1().Pods(ns).Get(ctx, pod, v1.GetOptions{})
-	if err != nil {
-		return nil, err
-	}
-	result := map[string][]mount.Mount{}
-	for _, c := range get.Spec.Containers {
-		// if container name is vpn or envoy-proxy, not need to download volume
-		if c.Name == config.ContainerSidecarVPN || c.Name == config.ContainerSidecarEnvoyProxy {
-			continue
-		}
-		var m []mount.Mount
-		for _, volumeMount := range c.VolumeMounts {
-			if volumeMount.MountPath == "/tmp" {
-				continue
-			}
-			join := filepath.Join(os.TempDir(), strconv.Itoa(rand.Int()))
-			err = os.MkdirAll(join, 0755)
-			if err != nil {
-				return nil, err
-			}
-			if volumeMount.SubPath != "" {
-				join = filepath.Join(join, volumeMount.SubPath)
-			}
-			// pod-namespace/pod-name:path
-			remotePath := fmt.Sprintf("%s/%s:%s", ns, pod, volumeMount.MountPath)
-			stdIn, stdOut, stdErr := term.StdStreams()
-			copyOptions := cp.NewCopyOptions(genericclioptions.IOStreams{In: stdIn, Out: stdOut, ErrOut: stdErr})
-			copyOptions.Container = c.Name
-			copyOptions.MaxTries = 10
-			err = copyOptions.Complete(f, &cobra.Command{}, []string{remotePath, join})
-			if err != nil {
-				return nil, err
-			}
-			err = copyOptions.Run()
-			if err != nil {
-				_, _ = fmt.Fprintf(os.Stderr, "failed to download volume %s path %s to %s, err: %v, ignore...\n", volumeMount.Name, remotePath, join, err)
-				continue
-			}
-			m = append(m, mount.Mount{
-				Type:   mount.TypeBind,
-				Source: join,
-				Target: volumeMount.MountPath,
-			})
-			fmt.Printf("%s:%s\n", join, volumeMount.MountPath)
-		}
-		result[c.Name] = m
-	}
-	return result, nil
-}

pkg/util/util.go

@@ -31,6 +31,7 @@ import (
 	k8sruntime "k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	runtimeresource "k8s.io/cli-runtime/pkg/resource"
@@ -631,30 +632,48 @@ func AllContainerIsRunning(pod *v1.Pod) bool {
 }
 
 func CleanExtensionLib() {
-	if IsWindows() {
+	if !IsWindows() {
-		err := retry.OnError(retry.DefaultRetry, func(err error) bool {
+		return
-			return err != nil
+	}
-		}, func() error {
+	path, err := os.Executable()
-			return driver.UninstallWireGuardTunDriver()
+	if err != nil {
-		})
+		return
-		if err != nil {
+	}
-			var wd string
+	filename := filepath.Join(filepath.Dir(path), "wintun.dll")
-			wd, err = os.Getwd()
+	_ = retry.OnError(
-			if err != nil {
+		// step : 0 13 34 55 100 194 433 661 1384 2689 (ms)
-				return
+		// total: 5.57s
-			}
+		wait.Backoff{
-			filename := filepath.Join(wd, "wintun.dll")
+			Steps:    10,
-			var temp *os.File
+			Duration: 10 * time.Millisecond,
-			if temp, err = os.CreateTemp("", ""); err != nil {
+			Factor:   2.0,
-				return
+			Jitter:   0.5,
-			}
+		},
-			if err = temp.Close(); err != nil {
+		func(error) bool {
-				return
+			_, err = os.Lstat(filename)
-			}
+			return !errors.Is(err, os.ErrNotExist)
-			if err = os.Rename(filename, temp.Name()); err != nil {
+		},
-				log.Debugln(err)
+		func() error {
-			}
+			err = driver.UninstallWireGuardTunDriver()
-		}
+			return fmt.Errorf("%v", err)
+		},
+	)
+	_, err = os.Lstat(filename)
+	if errors.Is(err, os.ErrNotExist) {
+		return
+	}
+	var temp *os.File
+	if temp, err = os.CreateTemp("", ""); err != nil {
+		return
+	}
+	if err = temp.Close(); err != nil {
+		return
+	}
+	if err = os.Remove(temp.Name()); err != nil {
+		return
+	}
+	if err = os.Rename(filename, temp.Name()); err != nil {
+		log.Debugln(err)
 	}
 }
 

pkg/util/util_test.go

@@ -3,6 +3,7 @@ package util
 import (
 	"encoding/json"
 	"net"
+	"strings"
 	"testing"
 
 	"github.com/containernetworking/cni/libcni"
@@ -94,6 +95,9 @@ func TestPing(t *testing.T) {
 	}
 	ipConn, err := net.ListenPacket("ip4:icmp", "0.0.0.0")
 	if err != nil {
+		if strings.Contains(err.Error(), "operation not permitted") {
+			return
+		}
 		t.Error(err)
 	}
 	bytes := buf.Bytes()