diff --git a/cmd/kubevpn/cmds/cp.go b/cmd/kubevpn/cmds/cp.go index 9def66f8..5592b77c 100644 --- a/cmd/kubevpn/cmds/cp.go +++ b/cmd/kubevpn/cmds/cp.go @@ -65,7 +65,7 @@ func CmdCp(f cmdutil.Factory) *cobra.Command { Use: "cp ", DisableFlagsInUseLine: true, Short: i18n.T("Copy files and directories to and from containers"), - Long: i18n.T("Copy files and directories to and from containers."), + Long: i18n.T("Copy files and directories to and from containers. Different between kubectl cp is it will de-reference symbol link."), Example: cpExample, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { cmdutil.CheckErr(handler.SshJump(sshConf, cmd.Flags())) diff --git a/cmd/kubevpn/cmds/dev.go b/cmd/kubevpn/cmds/dev.go index 863c6f1a..e1dfabc3 100644 --- a/cmd/kubevpn/cmds/dev.go +++ b/cmd/kubevpn/cmds/dev.go @@ -32,15 +32,22 @@ func CmdDev(f cmdutil.Factory) *cobra.Command { Env: opts.NewListOpts(nil), Volumes: opts.NewListOpts(nil), ExtraHosts: opts.NewListOpts(nil), - Aliases: opts.NewListOpts(nil), - NoProxy: false, - ExtraCIDR: []string{}, + //Aliases: opts.NewListOpts(nil), + NoProxy: false, + ExtraCIDR: []string{}, } var sshConf = &util.SshConfig{} cmd := &cobra.Command{ Use: "dev", - Short: i18n.T("Startup your workloads in local Docker container use same volume、env、and network with cluster"), - Long: templates.LongDesc(i18n.T(`Startup your workloads in local Docker container use same volume、env、and network with cluster`)), + Short: i18n.T("Startup your kubernetes workloads in local Docker container with same volume、env、and network"), + Long: templates.LongDesc(i18n.T(` +Startup your kubernetes workloads in local Docker container with same volume、env、and network + +## What did i do: +- Download volume which MountPath point to, mount to docker container +- Connect to cluster network, set network to docker container +- Get all environment with command (env), set env to docker container +`)), Example: templates.Examples(i18n.T(` # Develop workloads - develop deployment @@ -150,15 +157,14 @@ func CmdDev(f cmdutil.Factory) *cobra.Command { // docker options cmd.Flags().Var(&devOptions.ExtraHosts, "add-host", "Add a custom host-to-IP mapping (host:ip)") - //cmd.Flags().StringVar(&devOptions.ParentContainer, "parent-container", "", "Parent container name if running in Docker (Docker in Docker)") // We allow for both "--net" and "--network", although the latter is the recommended way. - cmd.Flags().Var(&devOptions.NetMode, "net", "Connect a container to a network") + cmd.Flags().Var(&devOptions.NetMode, "net", "Connect a container to a network, eg: [default|bridge|host|none|container:$CONTAINER_ID]") cmd.Flags().Var(&devOptions.NetMode, "network", "Connect a container to a network") cmd.Flags().MarkHidden("net") // We allow for both "--net-alias" and "--network-alias", although the latter is the recommended way. - cmd.Flags().Var(&devOptions.Aliases, "net-alias", "Add network-scoped alias for the container") - cmd.Flags().Var(&devOptions.Aliases, "network-alias", "Add network-scoped alias for the container") - cmd.Flags().MarkHidden("net-alias") + //cmd.Flags().Var(&devOptions.Aliases, "net-alias", "Add network-scoped alias for the container") + //cmd.Flags().Var(&devOptions.Aliases, "network-alias", "Add network-scoped alias for the container") + //cmd.Flags().MarkHidden("net-alias") cmd.Flags().VarP(&devOptions.Volumes, "volume", "v", "Bind mount a volume") cmd.Flags().Var(&devOptions.Mounts, "mount", "Attach a filesystem mount to the container") cmd.Flags().Var(&devOptions.Expose, "expose", "Expose a port or a range of ports") diff --git a/pkg/dev/convert.go b/pkg/dev/convert.go index 198d8dda..88a01cd3 100644 --- a/pkg/dev/convert.go +++ b/pkg/dev/convert.go @@ -3,6 +3,9 @@ package dev import ( "context" "fmt" + "math/rand" + "os" + "path/filepath" "strconv" "strings" "unsafe" @@ -14,12 +17,18 @@ import ( "github.com/docker/go-connections/nat" "github.com/google/uuid" miekgdns "github.com/miekg/dns" + "github.com/moby/term" v12 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/spf13/cobra" "k8s.io/api/core/v1" v13 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/kubectl/pkg/cmd/util" + "github.com/wencaiwulue/kubevpn/pkg/config" + "github.com/wencaiwulue/kubevpn/pkg/cp" "github.com/wencaiwulue/kubevpn/pkg/dns" + "github.com/wencaiwulue/kubevpn/pkg/handler" ) type RunConfig struct { @@ -143,7 +152,7 @@ func ConvertKubeResourceToContainer(namespace string, temp v1.PodTemplateSpec, e r.k8sContainerName = c.Name r.config = config r.hostConfig = hostConfig - r.networkingConfig = nil + r.networkingConfig = &network.NetworkingConfig{EndpointsConfig: make(map[string]*network.EndpointSettings)} r.platform = /*&v12.Platform{Architecture: "amd64", OS: "linux"}*/ nil runConfigList = append(runConfigList, &r) @@ -176,3 +185,63 @@ func GetDNS(ctx context.Context, f util.Factory, ns, pod string) (*miekgdns.Clie } return fromPod, nil } + +// GetVolume key format: [container name]-[volume mount name] +func GetVolume(ctx context.Context, f util.Factory, ns, pod string) (map[string][]mount.Mount, error) { + clientSet, err := f.KubernetesClientSet() + if err != nil { + return nil, err + } + var get *v1.Pod + get, err = clientSet.CoreV1().Pods(ns).Get(ctx, pod, v13.GetOptions{}) + if err != nil { + return nil, err + } + result := map[string][]mount.Mount{} + for _, c := range get.Spec.Containers { + // if container name is vpn or envoy-proxy, not need to download volume + if c.Name == config.ContainerSidecarVPN || c.Name == config.ContainerSidecarEnvoyProxy { + continue + } + var m []mount.Mount + for _, volumeMount := range c.VolumeMounts { + if volumeMount.MountPath == "/tmp" { + continue + } + join := filepath.Join(os.TempDir(), strconv.Itoa(rand.Int())) + err = os.MkdirAll(join, 0755) + if err != nil { + return nil, err + } + if volumeMount.SubPath != "" { + join = filepath.Join(join, volumeMount.SubPath) + } + handler.RollbackFuncList = append(handler.RollbackFuncList, func() { + _ = os.RemoveAll(join) + }) + // pod-namespace/pod-name:path + remotePath := fmt.Sprintf("%s/%s:%s", ns, pod, volumeMount.MountPath) + stdIn, stdOut, stdErr := term.StdStreams() + copyOptions := cp.NewCopyOptions(genericclioptions.IOStreams{In: stdIn, Out: stdOut, ErrOut: stdErr}) + copyOptions.Container = c.Name + copyOptions.MaxTries = 10 + err = copyOptions.Complete(f, &cobra.Command{}, []string{remotePath, join}) + if err != nil { + return nil, err + } + err = copyOptions.Run() + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "failed to download volume %s path %s to %s, err: %v, ignore...\n", volumeMount.Name, remotePath, join, err) + continue + } + m = append(m, mount.Mount{ + Type: mount.TypeBind, + Source: join, + Target: volumeMount.MountPath, + }) + fmt.Printf("%s:%s\n", join, volumeMount.MountPath) + } + result[c.Name] = m + } + return result, nil +} diff --git a/pkg/dev/main.go b/pkg/dev/main.go index 1206d05f..f0e44296 100644 --- a/pkg/dev/main.go +++ b/pkg/dev/main.go @@ -44,14 +44,14 @@ type Options struct { // docker options Platform string //Pull string // always, missing, never - PublishAll bool - Entrypoint string - DockerImage string - Publish opts.ListOpts - Expose opts.ListOpts - ExtraHosts opts.ListOpts - NetMode opts.NetworkOpt - Aliases opts.ListOpts + PublishAll bool + Entrypoint string + DockerImage string + Publish opts.ListOpts + Expose opts.ListOpts + ExtraHosts opts.ListOpts + NetMode opts.NetworkOpt + //Aliases opts.ListOpts Env opts.ListOpts Mounts opts.MountOpt Volumes opts.ListOpts @@ -99,7 +99,7 @@ func (d Options) Main(ctx context.Context) error { if err != nil { return err } - volume, err := util.GetVolume(ctx, d.Factory, d.Namespace, pod) + volume, err := GetVolume(ctx, d.Factory, d.Namespace, pod) if err != nil { return err } @@ -232,7 +232,7 @@ func (r Run) Run(ctx context.Context, volume map[string][]mount.Mount) error { id, err = run(ctx, config, cli) if err != nil { // try another way to startup container - log.Info("try another way to startup container") + log.Infof("occur err: %v, try another way to startup container...", err) config.hostConfig.Mounts = nil id, err = run(ctx, config, cli) if err != nil { diff --git a/pkg/dev/option.go b/pkg/dev/option.go index e4b8b525..30ffcc3b 100644 --- a/pkg/dev/option.go +++ b/pkg/dev/option.go @@ -148,11 +148,6 @@ func fillOptions(r Run, copts Options) error { config.hostConfig.Binds = binds - // todo - if copts.Aliases.Len() != 0 { - //config.networkingConfig.EndpointsConfig - } - return nil } diff --git a/pkg/dev/run.go b/pkg/dev/run.go index a77d9a46..e6e893a8 100644 --- a/pkg/dev/run.go +++ b/pkg/dev/run.go @@ -70,6 +70,11 @@ func run(ctx context.Context, runConfig *RunConfig, cli *client.Client) (id stri } id = create.ID log.Infof("Created container: %s", name) + defer func() { + if err != nil { + _ = cli.ContainerRemove(ctx, id, types.ContainerRemoveOptions{Force: true}) + } + }() err = cli.ContainerStart(ctx, create.ID, types.ContainerStartOptions{}) if err != nil { diff --git a/pkg/dns/dns_windows.go b/pkg/dns/dns_windows.go index 214e5b5e..a65ff8fd 100644 --- a/pkg/dns/dns_windows.go +++ b/pkg/dns/dns_windows.go @@ -53,7 +53,7 @@ func SetupDNS(clientConfig *miekgdns.ClientConfig, _ []string) error { func CancelDNS() { updateHosts("") - getenv := os.Getenv("luid") + getenv := os.Getenv(config.EnvTunNameOrLUID) parseUint, err := strconv.ParseUint(getenv, 10, 64) if err != nil { log.Warningln(err) @@ -61,6 +61,7 @@ func CancelDNS() { } luid := winipcfg.LUID(parseUint) _ = luid.FlushDNS(windows.AF_INET) + _ = luid.FlushRoutes(windows.AF_INET) } func updateNicMetric(name string) error { diff --git a/pkg/driver/driver.go b/pkg/driver/driver.go index 68f7387a..68c716e5 100644 --- a/pkg/driver/driver.go +++ b/pkg/driver/driver.go @@ -33,11 +33,11 @@ func InstallWireGuardTunDriver() { } func UninstallWireGuardTunDriver() error { - wd, err := os.Getwd() + executable, err := os.Executable() if err != nil { return err } - filename := filepath.Join(wd, "wintun.dll") + filename := filepath.Join(filepath.Dir(executable), "wintun.dll") return os.Remove(filename) } diff --git a/pkg/handler/cleaner.go b/pkg/handler/cleaner.go index ee663d49..7088e9fd 100644 --- a/pkg/handler/cleaner.go +++ b/pkg/handler/cleaner.go @@ -27,34 +27,34 @@ var stopChan = make(chan os.Signal) var RollbackFuncList = make([]func(), 2) var ctx, cancel = context.WithCancel(context.Background()) -func (c *ConnectOptions) addCleanUpResourceHandler(clientset *kubernetes.Clientset, namespace string) { - signal.Notify(stopChan, os.Interrupt, os.Kill, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGKILL /*, syscall.SIGSTOP*/) +func (c *ConnectOptions) addCleanUpResourceHandler() { + signal.Notify(stopChan, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGKILL) go func() { <-stopChan log.Info("prepare to exit, cleaning up") - dns.CancelDNS() err := c.dhcp.ReleaseIpToDHCP(c.usedIPs...) if err != nil { log.Errorf("failed to release ip to dhcp, err: %v", err) } - cancel() for _, function := range RollbackFuncList { if function != nil { function() } } - _ = clientset.CoreV1().Pods(namespace).Delete(context.Background(), config.CniNetName, v1.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)}) + _ = c.clientset.CoreV1().Pods(c.Namespace).Delete(context.Background(), config.CniNetName, v1.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)}) var count int - count, err = updateRefCount(clientset.CoreV1().ConfigMaps(namespace), config.ConfigMapPodTrafficManager, -1) + count, err = updateRefCount(c.clientset.CoreV1().ConfigMaps(c.Namespace), config.ConfigMapPodTrafficManager, -1) if err == nil { // if ref-count is less than zero or equals to zero, means nobody is using this traffic pod, so clean it if count <= 0 { log.Info("ref-count is zero, prepare to clean up resource") - cleanup(clientset, namespace, config.ConfigMapPodTrafficManager, true) + cleanup(c.clientset, c.Namespace, config.ConfigMapPodTrafficManager, true) } } else { log.Error(err) } + dns.CancelDNS() + cancel() log.Info("clean up successful") util.CleanExtensionLib() os.Exit(0) diff --git a/pkg/handler/connect.go b/pkg/handler/connect.go index 622463f2..c1d609f9 100644 --- a/pkg/handler/connect.go +++ b/pkg/handler/connect.go @@ -71,20 +71,18 @@ func (c *ConnectOptions) createRemoteInboundPod(ctx1 context.Context) (err error } for _, workload := range c.Workloads { - if len(workload) > 0 { - configInfo := util.PodRouteConfig{ - LocalTunIP: c.localTunIP.IP.String(), - TrafficManagerRealIP: c.routerIP.String(), - } - // means mesh mode - if len(c.Headers) != 0 { - err = InjectVPNAndEnvoySidecar(ctx1, c.factory, c.clientset.CoreV1().ConfigMaps(c.Namespace), c.Namespace, workload, configInfo, c.Headers) - } else { - err = InjectVPNSidecar(ctx1, c.factory, c.Namespace, workload, configInfo) - } - if err != nil { - return err - } + configInfo := util.PodRouteConfig{ + LocalTunIP: c.localTunIP.IP.String(), + TrafficManagerRealIP: c.routerIP.String(), + } + // means mesh mode + if len(c.Headers) != 0 { + err = InjectVPNAndEnvoySidecar(ctx1, c.factory, c.clientset.CoreV1().ConfigMaps(c.Namespace), c.Namespace, workload, configInfo, c.Headers) + } else { + err = InjectVPNSidecar(ctx1, c.factory, c.Namespace, workload, configInfo) + } + if err != nil { + return err } } return @@ -115,7 +113,7 @@ func Rollback(f cmdutil.Factory, ns, workload string) { } func (c *ConnectOptions) DoConnect() (err error) { - c.addCleanUpResourceHandler(c.clientset, c.Namespace) + c.addCleanUpResourceHandler() trafficMangerNet := net.IPNet{IP: config.RouterIP, Mask: config.CIDR.Mask} c.dhcp = NewDHCPManager(c.clientset.CoreV1().ConfigMaps(c.Namespace), c.Namespace, &trafficMangerNet) if err = c.dhcp.InitDHCP(ctx); err != nil { diff --git a/pkg/util/pod.go b/pkg/util/pod.go index ecf98912..bb854076 100644 --- a/pkg/util/pod.go +++ b/pkg/util/pod.go @@ -5,24 +5,13 @@ import ( "context" "fmt" "io" - "math/rand" - "os" - "path/filepath" - "strconv" "strings" "text/tabwriter" - "github.com/docker/docker/api/types/mount" - "github.com/moby/term" - "github.com/spf13/cobra" "golang.org/x/exp/constraints" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/kubectl/pkg/cmd/util" - - "github.com/wencaiwulue/kubevpn/pkg/config" - "github.com/wencaiwulue/kubevpn/pkg/cp" ) func PrintStatus(pod *corev1.Pod, writer io.Writer) { @@ -103,60 +92,3 @@ func GetEnv(ctx context.Context, f util.Factory, ns, pod string) (map[string][]s } return result, nil } - -// GetVolume key format: [container name]-[volume mount name] -func GetVolume(ctx context.Context, f util.Factory, ns, pod string) (map[string][]mount.Mount, error) { - clientSet, err := f.KubernetesClientSet() - if err != nil { - return nil, err - } - var get *corev1.Pod - get, err = clientSet.CoreV1().Pods(ns).Get(ctx, pod, v1.GetOptions{}) - if err != nil { - return nil, err - } - result := map[string][]mount.Mount{} - for _, c := range get.Spec.Containers { - // if container name is vpn or envoy-proxy, not need to download volume - if c.Name == config.ContainerSidecarVPN || c.Name == config.ContainerSidecarEnvoyProxy { - continue - } - var m []mount.Mount - for _, volumeMount := range c.VolumeMounts { - if volumeMount.MountPath == "/tmp" { - continue - } - join := filepath.Join(os.TempDir(), strconv.Itoa(rand.Int())) - err = os.MkdirAll(join, 0755) - if err != nil { - return nil, err - } - if volumeMount.SubPath != "" { - join = filepath.Join(join, volumeMount.SubPath) - } - // pod-namespace/pod-name:path - remotePath := fmt.Sprintf("%s/%s:%s", ns, pod, volumeMount.MountPath) - stdIn, stdOut, stdErr := term.StdStreams() - copyOptions := cp.NewCopyOptions(genericclioptions.IOStreams{In: stdIn, Out: stdOut, ErrOut: stdErr}) - copyOptions.Container = c.Name - copyOptions.MaxTries = 10 - err = copyOptions.Complete(f, &cobra.Command{}, []string{remotePath, join}) - if err != nil { - return nil, err - } - err = copyOptions.Run() - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "failed to download volume %s path %s to %s, err: %v, ignore...\n", volumeMount.Name, remotePath, join, err) - continue - } - m = append(m, mount.Mount{ - Type: mount.TypeBind, - Source: join, - Target: volumeMount.MountPath, - }) - fmt.Printf("%s:%s\n", join, volumeMount.MountPath) - } - result[c.Name] = m - } - return result, nil -} diff --git a/pkg/util/util.go b/pkg/util/util.go index 34a23a33..209b51db 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -31,6 +31,7 @@ import ( k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" "k8s.io/cli-runtime/pkg/genericclioptions" runtimeresource "k8s.io/cli-runtime/pkg/resource" @@ -631,30 +632,48 @@ func AllContainerIsRunning(pod *v1.Pod) bool { } func CleanExtensionLib() { - if IsWindows() { - err := retry.OnError(retry.DefaultRetry, func(err error) bool { - return err != nil - }, func() error { - return driver.UninstallWireGuardTunDriver() - }) - if err != nil { - var wd string - wd, err = os.Getwd() - if err != nil { - return - } - filename := filepath.Join(wd, "wintun.dll") - var temp *os.File - if temp, err = os.CreateTemp("", ""); err != nil { - return - } - if err = temp.Close(); err != nil { - return - } - if err = os.Rename(filename, temp.Name()); err != nil { - log.Debugln(err) - } - } + if !IsWindows() { + return + } + path, err := os.Executable() + if err != nil { + return + } + filename := filepath.Join(filepath.Dir(path), "wintun.dll") + _ = retry.OnError( + // step : 0 13 34 55 100 194 433 661 1384 2689 (ms) + // total: 5.57s + wait.Backoff{ + Steps: 10, + Duration: 10 * time.Millisecond, + Factor: 2.0, + Jitter: 0.5, + }, + func(error) bool { + _, err = os.Lstat(filename) + return !errors.Is(err, os.ErrNotExist) + }, + func() error { + err = driver.UninstallWireGuardTunDriver() + return fmt.Errorf("%v", err) + }, + ) + _, err = os.Lstat(filename) + if errors.Is(err, os.ErrNotExist) { + return + } + var temp *os.File + if temp, err = os.CreateTemp("", ""); err != nil { + return + } + if err = temp.Close(); err != nil { + return + } + if err = os.Remove(temp.Name()); err != nil { + return + } + if err = os.Rename(filename, temp.Name()); err != nil { + log.Debugln(err) } } diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index c2533e20..c51b3434 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -3,6 +3,7 @@ package util import ( "encoding/json" "net" + "strings" "testing" "github.com/containernetworking/cni/libcni" @@ -94,6 +95,9 @@ func TestPing(t *testing.T) { } ipConn, err := net.ListenPacket("ip4:icmp", "0.0.0.0") if err != nil { + if strings.Contains(err.Error(), "operation not permitted") { + return + } t.Error(err) } bytes := buf.Bytes()