refactor: dev mode use exec command instead of library

pkg/dev/options.go

@@ -5,21 +5,14 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"os/exec"
 	"strconv"
 	"strings"
 
-	"github.com/containerd/containerd/platforms"
-	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/api/types/container"
 	typescontainer "github.com/docker/docker/api/types/container"
-	"github.com/docker/docker/api/types/network"
-	"github.com/docker/docker/api/types/strslice"
-	"github.com/docker/docker/client"
 	"github.com/docker/go-connections/nat"
 	"github.com/google/uuid"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	log "github.com/sirupsen/logrus"
-	"github.com/spf13/pflag"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/labels"
@@ -32,7 +25,6 @@ import (
 	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon"
 	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon/rpc"
 	"github.com/wencaiwulue/kubevpn/v2/pkg/handler"
-	"github.com/wencaiwulue/kubevpn/v2/pkg/inject"
 	pkgssh "github.com/wencaiwulue/kubevpn/v2/pkg/ssh"
 	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
 )
@@ -60,10 +52,6 @@ type Options struct {
 	RunOptions       RunOptions
 	ContainerOptions *ContainerOptions
 
-	// inner
-	cli       *client.Client
-	dockerCli *command.DockerCli
-
 	factory    cmdutil.Factory
 	clientset  *kubernetes.Clientset
 	restclient *rest.RESTClient
@@ -73,22 +61,10 @@ type Options struct {
 	rollbackFuncList []func() error
 }
 
-func (option *Options) Main(ctx context.Context, sshConfig *pkgssh.SshConfig, flags *pflag.FlagSet, transferImage bool, imagePullSecretName string) error {
+func (option *Options) Main(ctx context.Context, sshConfig *pkgssh.SshConfig, config *Config, hostConfig *HostConfig, imagePullSecretName string) error {
 	mode := typescontainer.NetworkMode(option.ContainerOptions.netMode.NetworkMode())
 	if mode.IsContainer() {
 		log.Infof("Network mode container is %s", mode.ConnectedContainer())
-		inspect, err := option.cli.ContainerInspect(ctx, mode.ConnectedContainer())
-		if err != nil {
-			log.Errorf("Failed to inspect container %s, err: %v", mode.ConnectedContainer(), err)
-			return err
-		}
-		if inspect.State == nil {
-			return fmt.Errorf("can not get container status, please make container name is valid")
-		}
-		if !inspect.State.Running {
-			return fmt.Errorf("container %s status is %s, expect is running, please make sure your outer docker name is correct", mode.ConnectedContainer(), inspect.State.Status)
-		}
-		log.Infof("Container %s is running", mode.ConnectedContainer())
 	} else if mode.IsDefault() && util.RunningInContainer() {
 		hostname, err := os.Hostname()
 		if err != nil {
@@ -101,14 +77,8 @@ func (option *Options) Main(ctx context.Context, sshConfig *pkgssh.SshConfig, fl
 		}
 	}
 
-	config, hostConfig, err := Parse(flags, option.ContainerOptions)
-	// just in case the Parse does not exit
-	if err != nil {
-		return err
-	}
-
 	// Connect to cluster, in container or host
-	err = option.Connect(ctx, sshConfig, transferImage, imagePullSecretName, hostConfig.PortBindings)
+	err := option.Connect(ctx, sshConfig, imagePullSecretName, hostConfig.PortBindings)
 	if err != nil {
 		log.Errorf("Connect to cluster failed, err: %v", err)
 		return err
@@ -118,9 +88,8 @@ func (option *Options) Main(ctx context.Context, sshConfig *pkgssh.SshConfig, fl
 }
 
 // Connect to cluster network on docker container or host
-func (option *Options) Connect(ctx context.Context, sshConfig *pkgssh.SshConfig, transferImage bool, imagePullSecretName string, portBindings nat.PortMap) error {
-	switch option.ConnectMode {
-	case ConnectModeHost:
+func (option *Options) Connect(ctx context.Context, sshConfig *pkgssh.SshConfig, imagePullSecretName string, portBindings nat.PortMap) error {
+	if option.ConnectMode == ConnectModeHost {
 		daemonCli := daemon.GetClient(false)
 		if daemonCli == nil {
 			return fmt.Errorf("get nil daemon client")
@@ -144,19 +113,15 @@ func (option *Options) Connect(ctx context.Context, sshConfig *pkgssh.SshConfig,
 			KubeconfigBytes:      string(kubeConfigBytes),
 			Namespace:            ns,
 			Headers:              option.Headers,
-			Workloads:            []string{option.Workload},
+			Workloads:            util.If(option.NoProxy, nil, []string{option.Workload}),
 			ExtraRoute:           option.ExtraRouteInfo.ToRPC(),
 			Engine:               string(option.Engine),
 			OriginKubeconfigPath: util.GetKubeConfigPath(option.factory),
-			TransferImage:        transferImage,
 			Image:                config.Image,
 			ImagePullSecretName:  imagePullSecretName,
 			Level:                int32(logLevel),
 			SshJump:              sshConfig.ToRPC(),
 		}
-		if option.NoProxy {
-			req.Workloads = nil
-		}
 		option.AddRollbackFunc(func() error {
 			resp, err := daemonCli.Disconnect(ctx, &rpc.DisconnectRequest{
 				KubeconfigBytes: ptr.To(string(kubeConfigBytes)),
@@ -177,24 +142,25 @@ func (option *Options) Connect(ctx context.Context, sshConfig *pkgssh.SshConfig,
 		}
 		err = util.PrintGRPCStream[rpc.CloneResponse](resp)
 		return err
+	}
 
-	case ConnectModeContainer:
-		runConfig, err := option.CreateConnectContainer(portBindings)
+	if option.ConnectMode == ConnectModeContainer {
+		name, err := option.CreateConnectContainer(ctx, portBindings)
 		if err != nil {
 			return err
 		}
-		var id string
 		log.Infof("Starting connect to cluster in container")
-		id, err = run(ctx, option.cli, option.dockerCli, runConfig)
+		err = WaitDockerContainerRunning(ctx, *name)
 		if err != nil {
 			return err
 		}
 		option.AddRollbackFunc(func() error {
-			_ = option.cli.ContainerKill(context.Background(), id, "SIGTERM")
-			_ = runLogsSinceNow(option.dockerCli, id, true)
+			// docker kill --signal
+			_, _ = ContainerKill(context.Background(), name)
+			_ = RunLogsSinceNow(*name, true)
 			return nil
 		})
-		err = runLogsWaitRunning(ctx, option.dockerCli, id)
+		err = RunLogsWaitRunning(ctx, *name)
 		if err != nil {
 			// interrupt by signal KILL
 			if errors.Is(err, context.Canceled) {
@@ -203,16 +169,17 @@ func (option *Options) Connect(ctx context.Context, sshConfig *pkgssh.SshConfig,
 			return err
 		}
 		log.Infof("Connected to cluster in container")
-		err = option.ContainerOptions.netMode.Set(fmt.Sprintf("container:%s", id))
+		err = option.ContainerOptions.netMode.Set(fmt.Sprintf("container:%s", *name))
 		return err
-	default:
-		return fmt.Errorf("unsupport connect mode: %s", option.ConnectMode)
 	}
+
+	return fmt.Errorf("unsupport connect mode: %s", option.ConnectMode)
 }
 
-func (option *Options) Dev(ctx context.Context, cConfig *Config, hostConfig *HostConfig) error {
+func (option *Options) Dev(ctx context.Context, config *Config, hostConfig *HostConfig) error {
 	templateSpec, err := option.GetPodTemplateSpec()
 	if err != nil {
+		log.Errorf("Failed to get unstructured object error: %v", err)
 		return err
 	}
 
@@ -229,7 +196,13 @@ func (option *Options) Dev(ctx context.Context, cConfig *Config, hostConfig *Hos
 		log.Errorf("Failed to get env from k8s: %v", err)
 		return err
 	}
-	volume, err := util.GetVolume(ctx, option.factory, option.Namespace, list[0].Name)
+	option.AddRollbackFunc(func() error {
+		for _, s := range env {
+			_ = os.RemoveAll(s)
+		}
+		return nil
+	})
+	volume, err := util.GetVolume(ctx, option.clientset, option.factory, option.Namespace, list[0].Name)
 	if err != nil {
 		log.Errorf("Failed to get volume from k8s: %v", err)
 		return err
@@ -242,96 +215,20 @@ func (option *Options) Dev(ctx context.Context, cConfig *Config, hostConfig *Hos
 		log.Errorf("Failed to get DNS from k8s: %v", err)
 		return err
 	}
-
-	inject.RemoveContainers(templateSpec)
-	if option.ContainerName != "" {
-		var index = -1
-		for i, c := range templateSpec.Spec.Containers {
-			if option.ContainerName == c.Name {
-				index = i
-				break
-			}
-		}
-		if index != -1 {
-			templateSpec.Spec.Containers[0], templateSpec.Spec.Containers[index] = templateSpec.Spec.Containers[index], templateSpec.Spec.Containers[0]
-		}
+	configList, err := option.ConvertPodToContainerConfigList(ctx, *templateSpec, config, hostConfig, env, volume, dns)
+	if err != nil {
+		return err
 	}
-	configList := ConvertPodToContainer(option.Namespace, *templateSpec, env, volume, dns)
-	MergeDockerOptions(configList, option, cConfig, hostConfig)
-
-	mode := container.NetworkMode(option.ContainerOptions.netMode.NetworkMode())
-	if len(option.ContainerOptions.netMode.Value()) != 0 {
-		log.Infof("Network mode is %s", option.ContainerOptions.netMode.NetworkMode())
-		for _, runConfig := range configList[:] {
-			// remove expose port
-			runConfig.config.ExposedPorts = nil
-			runConfig.hostConfig.NetworkMode = mode
-			if mode.IsContainer() {
-				runConfig.hostConfig.PidMode = typescontainer.PidMode(option.ContainerOptions.netMode.NetworkMode())
-			}
-			runConfig.hostConfig.PortBindings = nil
-
-			// remove dns
-			runConfig.hostConfig.DNS = nil
-			runConfig.hostConfig.DNSOptions = nil
-			runConfig.hostConfig.DNSSearch = nil
-			runConfig.hostConfig.PublishAllPorts = false
-			runConfig.config.Hostname = ""
-		}
-	} else {
-		var networkID string
-		networkID, err = createNetwork(ctx, option.cli)
-		if err != nil {
-			log.Errorf("Failed to create network for %s: %v", option.Workload, err)
-			return err
-		}
-		log.Infof("Create docker network %s", networkID)
-
-		configList[len(configList)-1].networkingConfig.EndpointsConfig = map[string]*network.EndpointSettings{
-			configList[len(configList)-1].name: {NetworkID: networkID},
-		}
-		var portMap = nat.PortMap{}
-		var portSet = nat.PortSet{}
-		for _, runConfig := range configList {
-			for k, v := range runConfig.hostConfig.PortBindings {
-				if oldValue, ok := portMap[k]; ok {
-					portMap[k] = append(oldValue, v...)
-				} else {
-					portMap[k] = v
-				}
-			}
-			for k, v := range runConfig.config.ExposedPorts {
-				portSet[k] = v
-			}
-		}
-		configList[len(configList)-1].hostConfig.PortBindings = portMap
-		configList[len(configList)-1].config.ExposedPorts = portSet
-
-		// skip last, use last container network
-		for _, runConfig := range configList[:len(configList)-1] {
-			// remove expose port
-			runConfig.config.ExposedPorts = nil
-			runConfig.hostConfig.NetworkMode = typescontainer.NetworkMode("container:" + configList[len(configList)-1].name)
-			runConfig.hostConfig.PidMode = typescontainer.PidMode("container:" + configList[len(configList)-1].name)
-			runConfig.hostConfig.PortBindings = nil
-
-			// remove dns
-			runConfig.hostConfig.DNS = nil
-			runConfig.hostConfig.DNSOptions = nil
-			runConfig.hostConfig.DNSSearch = nil
-			runConfig.hostConfig.PublishAllPorts = false
-			runConfig.config.Hostname = ""
-		}
-	}
-
 	option.AddRollbackFunc(func() error {
-		_ = configList.Remove(ctx, option.cli)
+		if hostConfig.AutoRemove {
+			_ = configList.Remove(context.Background(), len(option.ContainerOptions.netMode.Value()) != 0)
+		}
 		return nil
 	})
-	return configList.Run(ctx, volume, option.cli, option.dockerCli)
+	return configList.Run(ctx)
 }
 
-func (option *Options) CreateConnectContainer(portBindings nat.PortMap) (*RunConfig, error) {
+func (option *Options) CreateConnectContainer(ctx context.Context, portBindings nat.PortMap) (*string, error) {
 	portMap, portSet, err := option.GetExposePort(portBindings)
 	if err != nil {
 		return nil, err
@@ -366,54 +263,49 @@ func (option *Options) CreateConnectContainer(portBindings nat.PortMap) (*RunCon
 		entrypoint = append(entrypoint, "--extra-node-ip")
 	}
 
-	runConfig := &container.Config{
-		User:         "root",
-		ExposedPorts: portSet,
-		Env:          []string{},
-		Cmd:          []string{},
-		Healthcheck:  nil,
-		Image:        config.Image,
-		Entrypoint:   entrypoint,
-	}
-	hostConfig := &container.HostConfig{
-		Binds:         []string{fmt.Sprintf("%s:%s", kubeconfigPath, "/root/.kube/config")},
-		LogConfig:     container.LogConfig{},
-		PortBindings:  portMap,
-		AutoRemove:    true,
-		Privileged:    true,
-		RestartPolicy: container.RestartPolicy{},
-		CapAdd:        strslice.StrSlice{"SYS_PTRACE", "SYS_ADMIN"}, // for dlv
-		// https://stackoverflow.com/questions/24319662/from-inside-of-a-docker-container-how-do-i-connect-to-the-localhost-of-the-mach
-		// couldn't get current server API group list: Get "https://host.docker.internal:62844/api?timeout=32s": tls: failed to verify certificate: x509: certificate is valid for kubernetes.default.svc.cluster.local, kubernetes.default.svc, kubernetes.default, kubernetes, istio-sidecar-injector.istio-system.svc, proxy-exporter.kube-system.svc, not host.docker.internal
-		ExtraHosts:  []string{"host.docker.internal:host-gateway", "kubernetes:host-gateway"},
-		SecurityOpt: []string{"apparmor=unconfined", "seccomp=unconfined"},
-		Sysctls:     map[string]string{"net.ipv6.conf.all.disable_ipv6": strconv.Itoa(0)},
-		Resources:   container.Resources{},
-	}
-	newUUID, err := uuid.NewUUID()
-	if err != nil {
-		return nil, err
-	}
-	suffix := strings.ReplaceAll(newUUID.String(), "-", "")[:5]
+	suffix := strings.ReplaceAll(uuid.New().String(), "-", "")[:5]
 	name := util.Join(option.Namespace, "kubevpn", suffix)
-	networkID, err := createNetwork(context.Background(), option.cli)
+	_, err = CreateNetwork(ctx, config.ConfigMapPodTrafficManager)
 	if err != nil {
 		return nil, err
 	}
-	var platform *specs.Platform
-	if option.RunOptions.Platform != "" {
-		plat, _ := platforms.Parse(option.RunOptions.Platform)
-		platform = &plat
+	args := []string{
+		"run",
+		"--detach",
+		"--volume", fmt.Sprintf("%s:%s", kubeconfigPath, "/root/.kube/config"),
+		"--privileged",
+		"--rm",
+		"--cap-add", "SYS_PTRACE",
+		"--cap-add", "SYS_ADMIN",
+		"--security-opt", "apparmor=unconfined",
+		"--security-opt", "seccomp=unconfined",
+		"--sysctl", "net.ipv6.conf.all.disable_ipv6=0",
+		"--add-host", "host.docker.internal:host-gateway",
+		"--add-host", "kubernetes:host-gateway",
+		"--network", config.ConfigMapPodTrafficManager,
+		"--name", name,
 	}
-	c := &RunConfig{
-		config:           runConfig,
-		hostConfig:       hostConfig,
-		networkingConfig: &network.NetworkingConfig{EndpointsConfig: map[string]*network.EndpointSettings{name: {NetworkID: networkID}}},
-		platform:         platform,
-		name:             name,
-		Options:          RunOptions{Pull: PullImageMissing},
+	for port := range portSet {
+		args = append(args, "--expose", port.Port())
 	}
-	return c, nil
+	for port, bindings := range portMap {
+		args = append(args, "--publish", fmt.Sprintf("%s:%s", port.Port(), bindings[0].HostPort))
+	}
+
+	var result []string
+	result = append(result, args...)
+	result = append(result, config.Image)
+	result = append(result, entrypoint...)
+	err = ContainerRun(ctx, result...)
+	if err != nil {
+		return nil, err
+	}
+	return &name, nil
+}
+
+func ContainerRun(ctx context.Context, args ...string) error {
+	err := exec.CommandContext(ctx, "docker", args...).Run()
+	return err
 }
 
 func (option *Options) AddRollbackFunc(f func() error) {
@@ -424,24 +316,10 @@ func (option *Options) GetRollbackFuncList() []func() error {
 	return option.rollbackFuncList
 }
 
-func AddDockerFlags(options *Options, p *pflag.FlagSet) {
-	p.SetInterspersed(false)
-
-	// These are flags not stored in Config/HostConfig
-	p.StringVar(&options.RunOptions.Pull, "pull", PullImageMissing, `Pull image before running ("`+PullImageAlways+`"|"`+PullImageMissing+`"|"`+PullImageNever+`")`)
-	p.BoolVar(&options.RunOptions.SigProxy, "sig-proxy", true, "Proxy received signals to the process")
-
-	// Add an explicit help that doesn't have a `-h` to prevent the conflict
-	// with hostname
-	p.Bool("help", false, "Print usage")
-
-	command.AddPlatformFlag(p, &options.RunOptions.Platform)
-	options.ContainerOptions = addFlags(p)
-}
-
 func (option *Options) GetExposePort(portBinds nat.PortMap) (nat.PortMap, nat.PortSet, error) {
 	templateSpec, err := option.GetPodTemplateSpec()
 	if err != nil {
+		log.Errorf("Failed to get unstructured object error: %v", err)
 		return nil, nil, err
 	}
 
@@ -483,16 +361,12 @@ func (option *Options) InitClient(f cmdutil.Factory) (err error) {
 	if option.Namespace, _, err = option.factory.ToRawKubeConfigLoader().Namespace(); err != nil {
 		return
 	}
-	if option.cli, option.dockerCli, err = pkgssh.GetClient(); err != nil {
-		return err
-	}
 	return
 }
 
 func (option *Options) GetPodTemplateSpec() (*v1.PodTemplateSpec, error) {
 	object, err := util.GetUnstructuredObject(option.factory, option.Namespace, option.Workload)
 	if err != nil {
-		log.Errorf("Failed to get unstructured object error: %v", err)
 		return nil, err
 	}