feat: support more docker options

cmd/kubevpn/cmds/dev.go

@@ -4,7 +4,8 @@ import (
 	"os"
 
 	"github.com/docker/cli/cli"
-	"github.com/docker/cli/opts"
+	"github.com/docker/cli/cli/command"
+	dockercomp "github.com/docker/cli/cli/command/completion"
 	"github.com/spf13/cobra"
 	cmdutil "k8s.io/kubectl/pkg/cmd/util"
 	"k8s.io/kubectl/pkg/util/completion"
@@ -18,21 +19,19 @@ import (
 )
 
 func CmdDev(f cmdutil.Factory) *cobra.Command {
-	var devOptions = dev.Options{
+	var devOptions = &dev.Options{
 		Factory:   f,
-		Entrypoint: "",
-		Publish:    opts.NewListOpts(nil),
-		Expose:     opts.NewListOpts(nil),
-		Env:        opts.NewListOpts(nil),
-		Volumes:    opts.NewListOpts(nil),
-		ExtraHosts: opts.NewListOpts(nil),
 		NoProxy:   false,
 		ExtraCIDR: []string{},
 	}
+	_, dockerCli, err := dev.GetClient()
+	if err != nil {
+		panic(err)
+	}
 	var sshConf = &util.SshConfig{}
 	var transferImage bool
 	cmd := &cobra.Command{
-		Use:   "dev",
+		Use:   "dev [OPTIONS] RESOURCE [COMMAND] [ARG...]",
 		Short: i18n.T("Startup your kubernetes workloads in local Docker container with same volume、env、and network"),
 		Long: templates.LongDesc(i18n.T(`
 Startup your kubernetes workloads in local Docker container with same volume、env、and network
@@ -51,22 +50,22 @@ Startup your kubernetes workloads in local Docker container with same volume、e
 		  kubevpn dev service/productpage
 
 		# Develop workloads with mesh, traffic with header a=1, will hit local PC, otherwise no effect
-		kubevpn dev service/productpage --headers a=1
+		kubevpn dev --headers a=1 service/productpage
 
         # Develop workloads without proxy traffic
-		kubevpn dev service/productpage --no-proxy
+		kubevpn dev --no-proxy service/productpage
 
 		# Develop workloads which api-server behind of bastion host or ssh jump host
-		kubevpn dev deployment/productpage --ssh-addr 192.168.1.100:22 --ssh-username root --ssh-keyfile /Users/naison/.ssh/ssh.pem
+		kubevpn dev --ssh-addr 192.168.1.100:22 --ssh-username root --ssh-keyfile /Users/naison/.ssh/ssh.pem deployment/productpage
 
 		# it also support ProxyJump, like
 		┌──────┐     ┌──────┐     ┌──────┐     ┌──────┐                 ┌────────────┐
 		│  pc  ├────►│ ssh1 ├────►│ ssh2 ├────►│ ssh3 ├─────►... ─────► │ api-server │
 		└──────┘     └──────┘     └──────┘     └──────┘                 └────────────┘
-		kubevpn dev deployment/productpage --ssh-alias <alias>
+		kubevpn dev --ssh-alias <alias> deployment/productpage
 
 `)),
-		Args: cli.ExactArgs(1),
+		Args: cli.RequiresMinArgs(1),
 		PreRunE: func(cmd *cobra.Command, args []string) error {
 			if !util.IsAdmin() {
 				util.RunWithElevated()
@@ -80,9 +79,14 @@ Startup your kubernetes workloads in local Docker container with same volume、e
 				}
 			}
 			return handler.SshJump(sshConf, cmd.Flags())
+			return nil
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return dev.DoDev(devOptions, args, f)
+			devOptions.Workload = args[0]
+			if len(args) > 1 {
+				devOptions.Copts.Args = args[1:]
+			}
+			return dev.DoDev(devOptions, cmd.Flags(), f)
 		},
 	}
 	cmd.Flags().StringToStringVarP(&devOptions.Headers, "headers", "H", map[string]string{}, "Traffic with special headers with reverse it to local PC, you should startup your service after reverse workloads successfully, If not special, redirect all traffic to local PC, format is k=v, like: k1=v1,k2=v2")
@@ -96,24 +100,42 @@ Startup your kubernetes workloads in local Docker container with same volume、e
 	cmd.Flags().StringVar((*string)(&devOptions.ConnectMode), "connect-mode", string(dev.ConnectModeHost), "Connect to kubernetes network in container or in host, eg: ["+string(dev.ConnectModeContainer)+"|"+string(dev.ConnectModeHost)+"]")
 	cmd.Flags().BoolVar(&transferImage, "transfer-image", false, "transfer image to remote registry, it will transfer image "+config.OriginImage+" to flags `--image` special image, default: "+config.Image)
 
-	// docker options
-	cmd.Flags().Var(&devOptions.ExtraHosts, "add-host", "Add a custom host-to-IP mapping (host:ip)")
-	// We allow for both "--net" and "--network", although the latter is the recommended way.
-	cmd.Flags().Var(&devOptions.NetMode, "net", "Connect a container to a network, eg: [default|bridge|host|none|container:$CONTAINER_ID]")
-	cmd.Flags().Var(&devOptions.NetMode, "network", "Connect a container to a network")
-	cmd.Flags().MarkHidden("net")
-	cmd.Flags().VarP(&devOptions.Volumes, "volume", "v", "Bind mount a volume")
-	cmd.Flags().Var(&devOptions.Mounts, "mount", "Attach a filesystem mount to the container")
-	cmd.Flags().Var(&devOptions.Expose, "expose", "Expose a port or a range of ports")
-	cmd.Flags().VarP(&devOptions.Publish, "publish", "p", "Publish a container's port(s) to the host")
-	cmd.Flags().BoolVarP(&devOptions.PublishAll, "publish-all", "P", false, "Publish all exposed ports to random ports")
-	cmd.Flags().VarP(&devOptions.Env, "env", "e", "Set environment variables")
-	cmd.Flags().StringVar(&devOptions.Entrypoint, "entrypoint", "", "Overwrite the default ENTRYPOINT of the image")
+	// diy docker options
 	cmd.Flags().StringVar(&devOptions.DockerImage, "docker-image", "", "Overwrite the default K8s pod of the image")
-	//cmd.Flags().StringVar(&devOptions.Pull, "pull", container.PullImageMissing, `Pull image before creating ("`+container.PullImageAlways+`"|"`+container.PullImageMissing+`"|"`+container.PullImageNever+`")`)
-	cmd.Flags().StringVar(&devOptions.Platform, "platform", os.Getenv("DOCKER_DEFAULT_PLATFORM"), "Set platform if server is multi-platform capable")
-	cmd.Flags().StringVar(&devOptions.VolumeDriver, "volume-driver", "", "Optional volume driver for the container")
-	_ = cmd.Flags().SetAnnotation("platform", "version", []string{"1.32"})
+	// origin docker options
+	flags := cmd.Flags()
+	flags.SetInterspersed(false)
+
+	// These are flags not stored in Config/HostConfig
+	flags.BoolVarP(&devOptions.Options.Detach, "detach", "d", false, "Run container in background and print container ID")
+	flags.StringVar(&devOptions.Options.Name, "name", "", "Assign a name to the container")
+	flags.StringVar(&devOptions.Options.Pull, "pull", dev.PullImageMissing, `Pull image before running ("`+dev.PullImageAlways+`"|"`+dev.PullImageMissing+`"|"`+dev.PullImageNever+`")`)
+	flags.BoolVarP(&devOptions.Options.Quiet, "quiet", "q", false, "Suppress the pull output")
+
+	// Add an explicit help that doesn't have a `-h` to prevent the conflict
+	// with hostname
+	flags.Bool("help", false, "Print usage")
+
+	command.AddPlatformFlag(flags, &devOptions.Options.Platform)
+	command.AddTrustVerificationFlags(flags, &devOptions.Options.Untrusted, dockerCli.ContentTrustEnabled())
+	devOptions.Copts = dev.AddFlags(flags)
+
+	_ = cmd.RegisterFlagCompletionFunc(
+		"env",
+		func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return os.Environ(), cobra.ShellCompDirectiveNoFileComp
+		},
+	)
+	_ = cmd.RegisterFlagCompletionFunc(
+		"env-file",
+		func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return nil, cobra.ShellCompDirectiveDefault
+		},
+	)
+	_ = cmd.RegisterFlagCompletionFunc(
+		"network",
+		dockercomp.NetworkNames(nil),
+	)
 
 	addSshFlags(cmd, sshConf)
 	return cmd

pkg/dev/LICENSE

@@ -0,0 +1,191 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        https://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright 2013-2017 Docker, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       https://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

pkg/dev/convert.go

@@ -32,19 +32,23 @@ import (
 )
 
 type RunConfig struct {
+	containerName    string
+	k8sContainerName string
+
 	config           *container.Config
 	hostConfig       *container.HostConfig
 	networkingConfig *network.NetworkingConfig
 	platform         *v12.Platform
-	containerName    string
-	k8sContainerName string
+
+	Options RunOptions
+	Copts   *ContainerOptions
 }
 
-func ConvertKubeResourceToContainer(namespace string, temp v1.PodTemplateSpec, envMap map[string][]string, mountVolume map[string][]mount.Mount, dnsConfig *miekgdns.ClientConfig) (runConfigList Run) {
+func ConvertKubeResourceToContainer(namespace string, temp v1.PodTemplateSpec, envMap map[string][]string, mountVolume map[string][]mount.Mount, dnsConfig *miekgdns.ClientConfig) (runConfigList ConfigList) {
 	spec := temp.Spec
 	for _, c := range spec.Containers {
 		var r RunConfig
-		config := &container.Config{
+		tmpConfig := &container.Config{
 			Hostname: func() string {
 				var hostname = spec.Hostname
 				if hostname == "" {
@@ -84,7 +88,7 @@ func ConvertKubeResourceToContainer(namespace string, temp v1.PodTemplateSpec, e
 			Shell:           nil,
 		}
 		if temp.DeletionGracePeriodSeconds != nil {
-			config.StopTimeout = (*int)(unsafe.Pointer(temp.DeletionGracePeriodSeconds))
+			tmpConfig.StopTimeout = (*int)(unsafe.Pointer(temp.DeletionGracePeriodSeconds))
 		}
 		hostConfig := &container.HostConfig{
 			Binds:           []string{},
@@ -138,7 +142,7 @@ func ConvertKubeResourceToContainer(namespace string, temp v1.PodTemplateSpec, e
 			portset[port1] = struct{}{}
 		}
 		hostConfig.PortBindings = portmap
-		config.ExposedPorts = portset
+		tmpConfig.ExposedPorts = portset
 		if c.SecurityContext != nil && c.SecurityContext.Capabilities != nil {
 			hostConfig.CapAdd = append(hostConfig.CapAdd, *(*strslice.StrSlice)(unsafe.Pointer(&c.SecurityContext.Capabilities.Add))...)
 			hostConfig.CapDrop = *(*strslice.StrSlice)(unsafe.Pointer(&c.SecurityContext.Capabilities.Drop))
@@ -150,10 +154,10 @@ func ConvertKubeResourceToContainer(namespace string, temp v1.PodTemplateSpec, e
 		}
 		r.containerName = fmt.Sprintf("%s_%s_%s_%s", c.Name, namespace, "kubevpn", suffix)
 		r.k8sContainerName = c.Name
-		r.config = config
+		r.config = tmpConfig
 		r.hostConfig = hostConfig
 		r.networkingConfig = &network.NetworkingConfig{EndpointsConfig: make(map[string]*network.EndpointSettings)}
-		r.platform = /*&v12.Platform{Architecture: "amd64", OS: "linux"}*/ nil
+		r.platform = nil
 
 		runConfigList = append(runConfigList, &r)
 	}
@@ -179,11 +183,11 @@ func GetDNS(ctx context.Context, f util.Factory, ns, pod string) (*miekgdns.Clie
 		return nil, err
 	}
 
-	fromPod, err := dns.GetDNSServiceIPFromPod(clientSet, client, config, pod, ns)
+	clientConfig, err := dns.GetDNSServiceIPFromPod(clientSet, client, config, pod, ns)
 	if err != nil {
 		return nil, err
 	}
-	return fromPod, nil
+	return clientConfig, nil
 }
 
 // GetVolume key format: [container name]-[volume mount name]

pkg/dev/dockercreate.go

@@ -0,0 +1,271 @@
+package dev
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"regexp"
+
+	"github.com/containerd/containerd/platforms"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/cli/cli/command/image"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/versions"
+	apiclient "github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/jsonmessage"
+	"github.com/docker/docker/registry"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+// Pull constants
+const (
+	PullImageAlways  = "always"
+	PullImageMissing = "missing" // Default (matches previous behavior)
+	PullImageNever   = "never"
+)
+
+type createOptions struct {
+	Name      string
+	Platform  string
+	Untrusted bool
+	Pull      string // always, missing, never
+	Quiet     bool
+}
+
+func pullImage(ctx context.Context, dockerCli command.Cli, image string, platform string, out io.Writer) error {
+	ref, err := reference.ParseNormalizedNamed(image)
+	if err != nil {
+		return err
+	}
+
+	// Resolve the Repository name from fqn to RepositoryInfo
+	repoInfo, err := registry.ParseRepositoryInfo(ref)
+	if err != nil {
+		return err
+	}
+
+	authConfig := command.ResolveAuthConfig(ctx, dockerCli, repoInfo.Index)
+	encodedAuth, err := command.EncodeAuthToBase64(authConfig)
+	if err != nil {
+		return err
+	}
+
+	options := types.ImageCreateOptions{
+		RegistryAuth: encodedAuth,
+		Platform:     platform,
+	}
+
+	responseBody, err := dockerCli.Client().ImageCreate(ctx, image, options)
+	if err != nil {
+		return err
+	}
+	defer responseBody.Close()
+
+	return jsonmessage.DisplayJSONMessagesStream(
+		responseBody,
+		out,
+		dockerCli.Out().FD(),
+		dockerCli.Out().IsTerminal(),
+		nil)
+}
+
+type cidFile struct {
+	path    string
+	file    *os.File
+	written bool
+}
+
+func (cid *cidFile) Close() error {
+	if cid.file == nil {
+		return nil
+	}
+	cid.file.Close()
+
+	if cid.written {
+		return nil
+	}
+	if err := os.Remove(cid.path); err != nil {
+		return errors.Wrapf(err, "failed to remove the CID file '%s'", cid.path)
+	}
+
+	return nil
+}
+
+func (cid *cidFile) Write(id string) error {
+	if cid.file == nil {
+		return nil
+	}
+	if _, err := cid.file.Write([]byte(id)); err != nil {
+		return errors.Wrap(err, "failed to write the container ID to the file")
+	}
+	cid.written = true
+	return nil
+}
+
+func newCIDFile(path string) (*cidFile, error) {
+	if path == "" {
+		return &cidFile{}, nil
+	}
+	if _, err := os.Stat(path); err == nil {
+		return nil, errors.Errorf("container ID file found, make sure the other container isn't running or delete %s", path)
+	}
+
+	f, err := os.Create(path)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create the container ID file")
+	}
+
+	return &cidFile{path: path, file: f}, nil
+}
+
+//nolint:gocyclo
+func createContainer(ctx context.Context, dockerCli command.Cli, containerConfig *containerConfig, opts *createOptions) (*container.CreateResponse, error) {
+	config := containerConfig.Config
+	hostConfig := containerConfig.HostConfig
+	networkingConfig := containerConfig.NetworkingConfig
+	stderr := dockerCli.Err()
+
+	warnOnOomKillDisable(*hostConfig, stderr)
+	warnOnLocalhostDNS(*hostConfig, stderr)
+
+	var (
+		trustedRef reference.Canonical
+		namedRef   reference.Named
+	)
+
+	containerIDFile, err := newCIDFile(hostConfig.ContainerIDFile)
+	if err != nil {
+		return nil, err
+	}
+	defer containerIDFile.Close()
+
+	ref, err := reference.ParseAnyReference(config.Image)
+	if err != nil {
+		return nil, err
+	}
+	if named, ok := ref.(reference.Named); ok {
+		namedRef = reference.TagNameOnly(named)
+
+		if taggedRef, ok := namedRef.(reference.NamedTagged); ok && !opts.Untrusted {
+			var err error
+			trustedRef, err = image.TrustedReference(ctx, dockerCli, taggedRef, nil)
+			if err != nil {
+				return nil, err
+			}
+			config.Image = reference.FamiliarString(trustedRef)
+		}
+	}
+
+	pullAndTagImage := func() error {
+		pullOut := stderr
+		if opts.Quiet {
+			pullOut = io.Discard
+		}
+		if err := pullImage(ctx, dockerCli, config.Image, opts.Platform, pullOut); err != nil {
+			return err
+		}
+		if taggedRef, ok := namedRef.(reference.NamedTagged); ok && trustedRef != nil {
+			return image.TagTrusted(ctx, dockerCli, trustedRef, taggedRef)
+		}
+		return nil
+	}
+
+	var platform *specs.Platform
+	// Engine API version 1.41 first introduced the option to specify platform on
+	// create. It will produce an error if you try to set a platform on older API
+	// versions, so check the API version here to maintain backwards
+	// compatibility for CLI users.
+	if opts.Platform != "" && versions.GreaterThanOrEqualTo(dockerCli.Client().ClientVersion(), "1.41") {
+		p, err := platforms.Parse(opts.Platform)
+		if err != nil {
+			return nil, errors.Wrap(err, "error parsing specified platform")
+		}
+		platform = &p
+	}
+
+	if opts.Pull == PullImageAlways {
+		if err := pullAndTagImage(); err != nil {
+			return nil, err
+		}
+	}
+
+	hostConfig.ConsoleSize[0], hostConfig.ConsoleSize[1] = dockerCli.Out().GetTtySize()
+
+	response, err := dockerCli.Client().ContainerCreate(ctx, config, hostConfig, networkingConfig, platform, opts.Name)
+	if err != nil {
+		// Pull image if it does not exist locally and we have the PullImageMissing option. Default behavior.
+		if apiclient.IsErrNotFound(err) && namedRef != nil && opts.Pull == PullImageMissing {
+			if !opts.Quiet {
+				// we don't want to write to stdout anything apart from container.ID
+				fmt.Fprintf(stderr, "Unable to find image '%s' locally\n", reference.FamiliarString(namedRef))
+			}
+
+			if err := pullAndTagImage(); err != nil {
+				return nil, err
+			}
+
+			var retryErr error
+			response, retryErr = dockerCli.Client().ContainerCreate(ctx, config, hostConfig, networkingConfig, platform, opts.Name)
+			if retryErr != nil {
+				return nil, retryErr
+			}
+		} else {
+			return nil, err
+		}
+	}
+
+	for _, warning := range response.Warnings {
+		fmt.Fprintf(stderr, "WARNING: %s\n", warning)
+	}
+	err = containerIDFile.Write(response.ID)
+	return &response, err
+}
+
+func warnOnOomKillDisable(hostConfig container.HostConfig, stderr io.Writer) {
+	if hostConfig.OomKillDisable != nil && *hostConfig.OomKillDisable && hostConfig.Memory == 0 {
+		fmt.Fprintln(stderr, "WARNING: Disabling the OOM killer on containers without setting a '-m/--memory' limit may be dangerous.")
+	}
+}
+
+// check the DNS settings passed via --dns against localhost regexp to warn if
+// they are trying to set a DNS to a localhost address
+func warnOnLocalhostDNS(hostConfig container.HostConfig, stderr io.Writer) {
+	for _, dnsIP := range hostConfig.DNS {
+		if isLocalhost(dnsIP) {
+			fmt.Fprintf(stderr, "WARNING: Localhost DNS setting (--dns=%s) may fail in containers.\n", dnsIP)
+			return
+		}
+	}
+}
+
+// IPLocalhost is a regex pattern for IPv4 or IPv6 loopback range.
+const ipLocalhost = `((127\.([0-9]{1,3}\.){2}[0-9]{1,3})|(::1)$)`
+
+var localhostIPRegexp = regexp.MustCompile(ipLocalhost)
+
+// IsLocalhost returns true if ip matches the localhost IP regular expression.
+// Used for determining if nameserver settings are being passed which are
+// localhost addresses
+func isLocalhost(ip string) bool {
+	return localhostIPRegexp.MatchString(ip)
+}
+
+func validatePullOpt(val string) error {
+	switch val {
+	case PullImageAlways, PullImageMissing, PullImageNever, "":
+		// valid option, but nothing to do yet
+		return nil
+	default:
+		return fmt.Errorf(
+			"invalid pull option: '%s': must be one of %q, %q or %q",
+			val,
+			PullImageAlways,
+			PullImageMissing,
+			PullImageNever,
+		)
+	}
+}

pkg/dev/dockerrun.go

@@ -0,0 +1,108 @@
+package dev
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+	"syscall"
+
+	"github.com/docker/cli/cli"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+)
+
+type RunOptions struct {
+	createOptions
+	Detach     bool
+	sigProxy   bool
+	detachKeys string
+}
+
+func attachContainer(ctx context.Context, dockerCli command.Cli, errCh *chan error, config *container.Config, containerID string) (func(), error) {
+	options := types.ContainerAttachOptions{
+		Stream:     true,
+		Stdin:      config.AttachStdin,
+		Stdout:     config.AttachStdout,
+		Stderr:     config.AttachStderr,
+		DetachKeys: dockerCli.ConfigFile().DetachKeys,
+	}
+
+	resp, errAttach := dockerCli.Client().ContainerAttach(ctx, containerID, options)
+	if errAttach != nil {
+		return nil, errAttach
+	}
+
+	var (
+		out, cerr io.Writer
+		in        io.ReadCloser
+	)
+	if config.AttachStdin {
+		in = dockerCli.In()
+	}
+	if config.AttachStdout {
+		out = dockerCli.Out()
+	}
+	if config.AttachStderr {
+		if config.Tty {
+			cerr = dockerCli.Out()
+		} else {
+			cerr = dockerCli.Err()
+		}
+	}
+
+	ch := make(chan error, 1)
+	*errCh = ch
+
+	if in != nil && out != nil && cerr != nil {
+
+	}
+
+	go func() {
+		ch <- func() error {
+			streamer := hijackedIOStreamer{
+				streams:      dockerCli,
+				inputStream:  in,
+				outputStream: out,
+				errorStream:  cerr,
+				resp:         resp,
+				tty:          config.Tty,
+				detachKeys:   options.DetachKeys,
+			}
+
+			if errHijack := streamer.stream(ctx); errHijack != nil {
+				return errHijack
+			}
+			return errAttach
+		}()
+	}()
+	return resp.Close, nil
+}
+
+// reportError is a utility method that prints a user-friendly message
+// containing the error that occurred during parsing and a suggestion to get help
+func reportError(stderr io.Writer, name string, str string, withHelp bool) {
+	str = strings.TrimSuffix(str, ".") + "."
+	if withHelp {
+		str += "\nSee 'docker " + name + " --help'."
+	}
+	_, _ = fmt.Fprintln(stderr, "docker:", str)
+}
+
+// if container start fails with 'not found'/'no such' error, return 127
+// if container start fails with 'permission denied' error, return 126
+// return 125 for generic docker daemon failures
+func runStartContainerErr(err error) error {
+	trimmedErr := strings.TrimPrefix(err.Error(), "Error response from daemon: ")
+	statusError := cli.StatusError{StatusCode: 125}
+	if strings.Contains(trimmedErr, "executable file not found") ||
+		strings.Contains(trimmedErr, "no such file or directory") ||
+		strings.Contains(trimmedErr, "system cannot find the file specified") {
+		statusError = cli.StatusError{StatusCode: 127}
+	} else if strings.Contains(trimmedErr, syscall.EACCES.Error()) {
+		statusError = cli.StatusError{StatusCode: 126}
+	}
+
+	return statusError
+}

pkg/dev/hijack.go

@@ -0,0 +1,207 @@
+package dev
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"runtime"
+	"sync"
+
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/pkg/ioutils"
+	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/moby/term"
+	"github.com/sirupsen/logrus"
+)
+
+// The default escape key sequence: ctrl-p, ctrl-q
+// TODO: This could be moved to `pkg/term`.
+var defaultEscapeKeys = []byte{16, 17}
+
+// A hijackedIOStreamer handles copying input to and output from streams to the
+// connection.
+type hijackedIOStreamer struct {
+	streams      command.Streams
+	inputStream  io.ReadCloser
+	outputStream io.Writer
+	errorStream  io.Writer
+
+	resp types.HijackedResponse
+
+	tty        bool
+	detachKeys string
+}
+
+// stream handles setting up the IO and then begins streaming stdin/stdout
+// to/from the hijacked connection, blocking until it is either done reading
+// output, the user inputs the detach key sequence when in TTY mode, or when
+// the given context is cancelled.
+func (h *hijackedIOStreamer) stream(ctx context.Context) error {
+	restoreInput, err := h.setupInput()
+	if err != nil {
+		return fmt.Errorf("unable to setup input stream: %s", err)
+	}
+
+	defer restoreInput()
+
+	outputDone := h.beginOutputStream(restoreInput)
+	inputDone, detached := h.beginInputStream(restoreInput)
+
+	select {
+	case err := <-outputDone:
+		return err
+	case <-inputDone:
+		// Input stream has closed.
+		if h.outputStream != nil || h.errorStream != nil {
+			// Wait for output to complete streaming.
+			select {
+			case err := <-outputDone:
+				return err
+			case <-ctx.Done():
+				return ctx.Err()
+			}
+		}
+		return nil
+	case err := <-detached:
+		// Got a detach key sequence.
+		return err
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+func (h *hijackedIOStreamer) setupInput() (restore func(), err error) {
+	if h.inputStream == nil || !h.tty {
+		// No need to setup input TTY.
+		// The restore func is a nop.
+		return func() {}, nil
+	}
+
+	if err := setRawTerminal(h.streams); err != nil {
+		return nil, fmt.Errorf("unable to set IO streams as raw terminal: %s", err)
+	}
+
+	// Use sync.Once so we may call restore multiple times but ensure we
+	// only restore the terminal once.
+	var restoreOnce sync.Once
+	restore = func() {
+		restoreOnce.Do(func() {
+			restoreTerminal(h.streams, h.inputStream)
+		})
+	}
+
+	// Wrap the input to detect detach escape sequence.
+	// Use default escape keys if an invalid sequence is given.
+	escapeKeys := defaultEscapeKeys
+	if h.detachKeys != "" {
+		customEscapeKeys, err := term.ToBytes(h.detachKeys)
+		if err != nil {
+			logrus.Warnf("invalid detach escape keys, using default: %s", err)
+		} else {
+			escapeKeys = customEscapeKeys
+		}
+	}
+
+	h.inputStream = ioutils.NewReadCloserWrapper(term.NewEscapeProxy(h.inputStream, escapeKeys), h.inputStream.Close)
+
+	return restore, nil
+}
+
+func (h *hijackedIOStreamer) beginOutputStream(restoreInput func()) <-chan error {
+	if h.outputStream == nil && h.errorStream == nil {
+		// There is no need to copy output.
+		return nil
+	}
+
+	outputDone := make(chan error)
+	go func() {
+		var err error
+
+		// When TTY is ON, use regular copy
+		if h.outputStream != nil && h.tty {
+			_, err = io.Copy(h.outputStream, h.resp.Reader)
+			// We should restore the terminal as soon as possible
+			// once the connection ends so any following print
+			// messages will be in normal type.
+			restoreInput()
+		} else {
+			_, err = stdcopy.StdCopy(h.outputStream, h.errorStream, h.resp.Reader)
+		}
+
+		logrus.Debug("[hijack] End of stdout")
+
+		if err != nil {
+			logrus.Debugf("Error receiveStdout: %s", err)
+		}
+
+		outputDone <- err
+	}()
+
+	return outputDone
+}
+
+func (h *hijackedIOStreamer) beginInputStream(restoreInput func()) (doneC <-chan struct{}, detachedC <-chan error) {
+	inputDone := make(chan struct{})
+	detached := make(chan error)
+
+	go func() {
+		if h.inputStream != nil {
+			_, err := io.Copy(h.resp.Conn, h.inputStream)
+			// We should restore the terminal as soon as possible
+			// once the connection ends so any following print
+			// messages will be in normal type.
+			restoreInput()
+
+			logrus.Debug("[hijack] End of stdin")
+
+			if _, ok := err.(term.EscapeError); ok {
+				detached <- err
+				return
+			}
+
+			if err != nil {
+				// This error will also occur on the receive
+				// side (from stdout) where it will be
+				// propagated back to the caller.
+				logrus.Debugf("Error sendStdin: %s", err)
+			}
+		}
+
+		if err := h.resp.CloseWrite(); err != nil {
+			logrus.Debugf("Couldn't send EOF: %s", err)
+		}
+
+		close(inputDone)
+	}()
+
+	return inputDone, detached
+}
+
+func setRawTerminal(streams command.Streams) error {
+	if err := streams.In().SetRawTerminal(); err != nil {
+		return err
+	}
+	return streams.Out().SetRawTerminal()
+}
+
+func restoreTerminal(streams command.Streams, in io.Closer) error {
+	streams.In().RestoreTerminal()
+	streams.Out().RestoreTerminal()
+	// WARNING: DO NOT REMOVE THE OS CHECKS !!!
+	// For some reason this Close call blocks on darwin..
+	// As the client exits right after, simply discard the close
+	// until we find a better solution.
+	//
+	// This can also cause the client on Windows to get stuck in Win32 CloseHandle()
+	// in some cases. See https://github.com/docker/docker/issues/28267#issuecomment-288237442
+	// Tracked internally at Microsoft by VSO #11352156. In the
+	// Windows case, you hit this if you are using the native/v2 console,
+	// not the "legacy" console, and you start the client in a new window. eg
+	// `start docker run --rm -it microsoft/nanoserver cmd /s /c echo foobar`
+	// will hang. Remove start, and it won't repro.
+	if in != nil && runtime.GOOS != "darwin" && runtime.GOOS != "windows" {
+		return in.Close()
+	}
+	return nil
+}

pkg/dev/main.go

@@ -29,10 +29,12 @@ import (
 	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/docker/go-connections/nat"
 	"github.com/google/uuid"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	pkgerr "github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
+	"github.com/spf13/pflag"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/labels"
@@ -67,22 +69,12 @@ type Options struct {
 	ConnectMode   ConnectMode
 
 	// docker options
-	Platform string
-	//Pull         string // always, missing, never
-	PublishAll   bool
-	Entrypoint   string
 	DockerImage string
-	Publish      opts.ListOpts
-	Expose       opts.ListOpts
-	ExtraHosts   opts.ListOpts
-	NetMode      opts.NetworkOpt
-	Env          opts.ListOpts
-	Mounts       opts.MountOpt
-	Volumes      opts.ListOpts
-	VolumeDriver string
+	Options     RunOptions
+	Copts       *ContainerOptions
 }
 
-func (d Options) Main(ctx context.Context) error {
+func (d *Options) Main(ctx context.Context, cli *client.Client, dockerCli *command.DockerCli, tempContainerConfig *containerConfig) error {
 	rand.Seed(time.Now().UnixNano())
 	object, err := util.GetUnstructuredObject(d.Factory, d.Namespace, d.Workload)
 	if err != nil {
@@ -133,31 +125,25 @@ func (d Options) Main(ctx context.Context) error {
 	}
 
 	mesh.RemoveContainers(templateSpec)
-	list := ConvertKubeResourceToContainer(d.Namespace, *templateSpec, env, volume, dns)
-	err = fillOptions(list, d)
+	runConfigList := ConvertKubeResourceToContainer(d.Namespace, *templateSpec, env, volume, dns)
+	err = mergeDockerOptions(runConfigList, d, tempContainerConfig)
 	if err != nil {
 		return fmt.Errorf("can not fill docker options, err: %v", err)
 	}
-	var dockerCli *command.DockerCli
-	var cli *client.Client
-	cli, dockerCli, err = GetClient()
-	if err != nil {
-		return err
-	}
 	// check resource
 	var outOfMemory bool
 	outOfMemory, _ = checkOutOfMemory(templateSpec, cli)
 	if outOfMemory {
 		return fmt.Errorf("your pod resource request is bigger than docker-desktop resource, please adjust your docker-desktop resource")
 	}
-	mode := container.NetworkMode(d.NetMode.NetworkMode())
-	if len(d.NetMode.Value()) != 0 {
-		for _, runConfig := range list[:] {
+	mode := container.NetworkMode(d.Copts.netMode.NetworkMode())
+	if len(d.Copts.netMode.Value()) != 0 {
+		for _, runConfig := range runConfigList[:] {
 			// remove expose port
 			runConfig.config.ExposedPorts = nil
 			runConfig.hostConfig.NetworkMode = mode
 			if mode.IsContainer() {
-				runConfig.hostConfig.PidMode = containertypes.PidMode(d.NetMode.NetworkMode())
+				runConfig.hostConfig.PidMode = containertypes.PidMode(d.Copts.netMode.NetworkMode())
 			}
 			runConfig.hostConfig.PortBindings = nil
 
@@ -175,15 +161,32 @@ func (d Options) Main(ctx context.Context) error {
 			return err
 		}
 
-		list[0].networkingConfig.EndpointsConfig[list[0].containerName] = &network.EndpointSettings{
+		runConfigList[len(runConfigList)-1].networkingConfig.EndpointsConfig[runConfigList[len(runConfigList)-1].containerName] = &network.EndpointSettings{
 			NetworkID: networkID,
 		}
-		// skip first
-		for _, runConfig := range list[1:] {
+		var portmap = nat.PortMap{}
+		var portset = nat.PortSet{}
+		for _, runConfig := range runConfigList {
+			for k, v := range runConfig.hostConfig.PortBindings {
+				if oldValue, ok := portmap[k]; ok {
+					portmap[k] = append(oldValue, v...)
+				} else {
+					portmap[k] = v
+				}
+			}
+			for k, v := range runConfig.config.ExposedPorts {
+				portset[k] = v
+			}
+		}
+		runConfigList[len(runConfigList)-1].hostConfig.PortBindings = portmap
+		runConfigList[len(runConfigList)-1].config.ExposedPorts = portset
+
+		// skip last, use last container network
+		for _, runConfig := range runConfigList[:len(runConfigList)-1] {
 			// remove expose port
 			runConfig.config.ExposedPorts = nil
-			runConfig.hostConfig.NetworkMode = containertypes.NetworkMode("container:" + list[0].containerName)
-			runConfig.hostConfig.PidMode = containertypes.PidMode("container:" + list[0].containerName)
+			runConfig.hostConfig.NetworkMode = containertypes.NetworkMode("container:" + runConfigList[len(runConfigList)-1].containerName)
+			runConfig.hostConfig.PidMode = containertypes.PidMode("container:" + runConfigList[len(runConfigList)-1].containerName)
 			runConfig.hostConfig.PortBindings = nil
 
 			// remove dns
@@ -196,25 +199,20 @@ func (d Options) Main(ctx context.Context) error {
 	}
 
 	handler.RollbackFuncList = append(handler.RollbackFuncList, func() {
-		_ = list.Remove(ctx)
+		_ = runConfigList.Remove(ctx, cli)
 	})
-	err = list.Run(ctx, volume)
+	err = runConfigList.Run(ctx, volume, cli, dockerCli)
 	if err != nil {
 		return err
 	}
-	return terminal(list[0].containerName, dockerCli)
+	return terminal(runConfigList[0].containerName, dockerCli)
 }
 
-type Run []*RunConfig
+type ConfigList []*RunConfig
 
-func (r Run) Remove(ctx context.Context) error {
-	cli, _, err := GetClient()
-	if err != nil {
-		return err
-	}
-
-	for _, runConfig := range r {
-		err = cli.NetworkDisconnect(ctx, runConfig.containerName, runConfig.containerName, true)
+func (l ConfigList) Remove(ctx context.Context, cli *client.Client) error {
+	for _, runConfig := range l {
+		err := cli.NetworkDisconnect(ctx, runConfig.containerName, runConfig.containerName, true)
 		if err != nil {
 			log.Debug(err)
 		}
@@ -223,8 +221,7 @@ func (r Run) Remove(ctx context.Context) error {
 			log.Debug(err)
 		}
 	}
-	var i types.NetworkResource
-	i, err = cli.NetworkInspect(ctx, config.ConfigMapPodTrafficManager, types.NetworkInspectOptions{})
+	i, err := cli.NetworkInspect(ctx, config.ConfigMapPodTrafficManager, types.NetworkInspectOptions{})
 	if err != nil {
 		return err
 	}
@@ -254,32 +251,35 @@ func GetClient() (*client.Client, *command.DockerCli, error) {
 	return cli, dockerCli, nil
 }
 
-func (r Run) Run(ctx context.Context, volume map[string][]mount.Mount) error {
-	cli, c, err := GetClient()
+func (l ConfigList) Run(ctx context.Context, volume map[string][]mount.Mount, cli *client.Client, dockerCli *command.DockerCli) error {
+	for index := len(l) - 1; index >= 0; index-- {
+		runConfig := l[index]
+		if index == 0 {
+			_, err := runFirst(ctx, runConfig, cli, dockerCli)
 			if err != nil {
 				return err
 			}
-	for _, runConfig := range r {
-		var id string
-		id, err = run(ctx, runConfig, cli, c)
+		} else {
+			id, err := run(ctx, runConfig, cli, dockerCli)
 			if err != nil {
 				// try another way to startup container
 				log.Infof("occur err: %v, try another way to startup container...", err)
 				runConfig.hostConfig.Mounts = nil
-			id, err = run(ctx, runConfig, cli, c)
+				id, err = run(ctx, runConfig, cli, dockerCli)
 				if err != nil {
 					return err
 				}
-			err = r.copyToContainer(ctx, volume[runConfig.k8sContainerName], cli, id)
+				err = l.copyToContainer(ctx, volume[runConfig.k8sContainerName], cli, id)
 				if err != nil {
 					return err
 				}
 			}
 		}
+	}
 	return nil
 }
 
-func (r Run) copyToContainer(ctx context.Context, volume []mount.Mount, cli *client.Client, id string) error {
+func (l ConfigList) copyToContainer(ctx context.Context, volume []mount.Mount, cli *client.Client, id string) error {
 	// copy volume into container
 	for _, v := range volume {
 		target, err := createFolder(ctx, cli, id, v.Source, v.Target)
@@ -367,22 +367,22 @@ func checkOutOfMemory(spec *v1.PodTemplateSpec, cli *client.Client) (outOfMemory
 	return
 }
 
-func DoDev(devOptions Options, args []string, f cmdutil.Factory) error {
+func DoDev(devOptions *Options, flags *pflag.FlagSet, f cmdutil.Factory) error {
 	connect := handler.ConnectOptions{
 		Headers:     devOptions.Headers,
-		Workloads:   args,
+		Workloads:   []string{devOptions.Workload},
 		ExtraCIDR:   devOptions.ExtraCIDR,
 		ExtraDomain: devOptions.ExtraDomain,
 	}
-
-	mode := container.NetworkMode(devOptions.NetMode.NetworkMode())
-	if mode.IsContainer() {
-		client, _, err := GetClient()
+	cli, dockerCli, err := GetClient()
 	if err != nil {
 		return err
 	}
+
+	mode := container.NetworkMode(devOptions.Copts.netMode.NetworkMode())
+	if mode.IsContainer() {
 		var inspect types.ContainerJSON
-		inspect, err = client.ContainerInspect(context.Background(), mode.ConnectedContainer())
+		inspect, err = cli.ContainerInspect(context.Background(), mode.ConnectedContainer())
 		if err != nil {
 			return err
 		}
@@ -397,8 +397,7 @@ func DoDev(devOptions Options, args []string, f cmdutil.Factory) error {
 	if err := connect.InitClient(f); err != nil {
 		return err
 	}
-	err := connect.PreCheckResource()
-	if err != nil {
+	if err = connect.PreCheckResource(); err != nil {
 		return err
 	}
 
@@ -410,8 +409,8 @@ func DoDev(devOptions Options, args []string, f cmdutil.Factory) error {
 	}
 
 	var platform *specs.Platform
-	if devOptions.Platform != "" {
-		p, err := platforms.Parse(devOptions.Platform)
+	if devOptions.Options.Platform != "" {
+		p, err := platforms.Parse(devOptions.Options.Platform)
 		if err != nil {
 			return pkgerr.Wrap(err, "error parsing specified platform")
 		}
@@ -441,13 +440,74 @@ func DoDev(devOptions Options, args []string, f cmdutil.Factory) error {
 			return err
 		}
 	case ConnectModeContainer:
-		var dockerCli *command.DockerCli
-		var cli *client.Client
-		cli, dockerCli, err = GetClient()
+		var connectContainer *RunConfig
+		connectContainer, err = createConnectContainer(*devOptions, connect, path, err, cli, platform)
 		if err != nil {
 			return err
 		}
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		var id string
+		if id, err = run(ctx, connectContainer, cli, dockerCli); err != nil {
+			return err
+		}
+		h := interrupt.New(func(signal os.Signal) {
+			os.Exit(0)
+		}, func() {
+			cancel()
+			_ = cli.ContainerKill(context.Background(), id, "SIGTERM")
+			_ = runLogsSinceNow(dockerCli, id)
+		})
+		go h.Run(func() error { select {} })
+		defer h.Close()
+		if err = runLogsWaitRunning(ctx, dockerCli, id); err != nil {
+			// interrupt by signal KILL
+			if ctx.Err() == context.Canceled {
+				return nil
+			}
+			return err
+		}
+		if err = devOptions.Copts.netMode.Set("container:" + id); err != nil {
+			return err
+		}
+	default:
+		return fmt.Errorf("unsupport connect mode: %s", devOptions.ConnectMode)
+	}
 
+	var tempContainerConfig *containerConfig
+	{
+		if err := validatePullOpt(devOptions.Options.Pull); err != nil {
+			return err
+		}
+		proxyConfig := dockerCli.ConfigFile().ParseProxyConfig(dockerCli.Client().DaemonHost(), opts.ConvertKVStringsToMapWithNil(devOptions.Copts.env.GetAll()))
+		newEnv := []string{}
+		for k, v := range proxyConfig {
+			if v == nil {
+				newEnv = append(newEnv, k)
+			} else {
+				newEnv = append(newEnv, fmt.Sprintf("%s=%s", k, *v))
+			}
+		}
+		devOptions.Copts.env = *opts.NewListOptsRef(&newEnv, nil)
+		tempContainerConfig, err = parse(flags, devOptions.Copts, dockerCli.ServerInfo().OSType)
+		// just in case the parse does not exit
+		if err != nil {
+			return err
+		}
+		if err = validateAPIVersion(tempContainerConfig, dockerCli.Client().ClientVersion()); err != nil {
+			return err
+		}
+	}
+
+	devOptions.Namespace = connect.Namespace
+	err = devOptions.Main(context.Background(), cli, dockerCli, tempContainerConfig)
+	if err != nil {
+		log.Errorln(err)
+	}
+	return err
+}
+
+func createConnectContainer(devOptions Options, connect handler.ConnectOptions, path string, err error, cli *client.Client, platform *specs.Platform) (*RunConfig, error) {
 	var entrypoint []string
 	if devOptions.NoProxy {
 		entrypoint = []string{"kubevpn", "connect", "-n", connect.Namespace, "--kubeconfig", "/root/.kube/config", "--image", config.Image}
@@ -533,7 +593,7 @@ func DoDev(devOptions Options, args []string, f cmdutil.Factory) error {
 	var kubevpnNetwork string
 	kubevpnNetwork, err = createKubevpnNetwork(context.Background(), cli)
 	if err != nil {
-			return err
+		return nil, err
 	}
 	name := fmt.Sprintf("%s_%s_%s", "kubevpn", "local", suffix)
 	c := &RunConfig{
@@ -548,41 +608,7 @@ func DoDev(devOptions Options, args []string, f cmdutil.Factory) error {
 		containerName:    name,
 		k8sContainerName: name,
 	}
-		ctx, cancel := context.WithCancel(context.Background())
-		defer cancel()
-		var id string
-		if id, err = run(ctx, c, cli, dockerCli); err != nil {
-			return err
-		}
-		h := interrupt.New(func(signal os.Signal) {
-			os.Exit(0)
-		}, func() {
-			cancel()
-			_ = cli.ContainerKill(context.Background(), id, "SIGTERM")
-			_ = runLogsSinceNow(dockerCli, id)
-		})
-		go h.Run(func() error { select {} })
-		defer h.Close()
-		if err = runLogsWaitRunning(ctx, dockerCli, id); err != nil {
-			// interrupt by signal KILL
-			if ctx.Err() == context.Canceled {
-				return nil
-			}
-			return err
-		}
-		if err = devOptions.NetMode.Set("container:" + id); err != nil {
-			return err
-		}
-	default:
-		return fmt.Errorf("unsupport connect mode: %s", devOptions.ConnectMode)
-	}
-
-	devOptions.Namespace = connect.Namespace
-	err = devOptions.Main(context.Background())
-	if err != nil {
-		log.Errorln(err)
-	}
-	return err
+	return c, nil
 }
 
 func runLogsWaitRunning(ctx context.Context, dockerCli command.Cli, container string) error {
@@ -685,37 +711,6 @@ func runKill(dockerCli command.Cli, containers ...string) error {
 	}
 	return nil
 }
-func parallelOperation(ctx context.Context, containers []string, op func(ctx context.Context, container string) error) chan error {
-	if len(containers) == 0 {
-		return nil
-	}
-	const defaultParallel int = 50
-	sem := make(chan struct{}, defaultParallel)
-	errChan := make(chan error)
-
-	// make sure result is printed in correct order
-	output := map[string]chan error{}
-	for _, c := range containers {
-		output[c] = make(chan error, 1)
-	}
-	go func() {
-		for _, c := range containers {
-			err := <-output[c]
-			errChan <- err
-		}
-	}()
-
-	go func() {
-		for _, c := range containers {
-			sem <- struct{}{} // Wait for active queue sem to drain.
-			go func(container string) {
-				output[container] <- op(ctx, container)
-				<-sem
-			}(c)
-		}
-	}()
-	return errChan
-}
 
 func createKubevpnNetwork(ctx context.Context, cli *client.Client) (string, error) {
 	by := map[string]string{"owner": config.ConfigMapPodTrafficManager}

pkg/dev/option.go

@@ -1,26 +1,13 @@
 package dev
 
 import (
-	"fmt"
-	"path/filepath"
-	"reflect"
-	"strconv"
-	"strings"
-
-	"github.com/docker/cli/cli/compose/loader"
-	"github.com/docker/cli/opts"
-	"github.com/docker/docker/api/types/container"
-	mounttypes "github.com/docker/docker/api/types/mount"
+	"github.com/containerd/containerd/platforms"
 	"github.com/docker/docker/api/types/network"
-	"github.com/docker/docker/api/types/strslice"
-	"github.com/docker/docker/errdefs"
-	"github.com/docker/go-connections/nat"
-	v12 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
+	"github.com/wencaiwulue/kubevpn/pkg/util"
 )
 
-func fillOptions(r Run, copts Options) error {
+func mergeDockerOptions(r ConfigList, copts *Options, tempContainerConfig *containerConfig) error {
 	if copts.ContainerName != "" {
 		var index = -1
 		for i, config := range r {
@@ -35,226 +22,49 @@ func fillOptions(r Run, copts Options) error {
 	}
 
 	config := r[0]
-	config.hostConfig.PublishAllPorts = copts.PublishAll
+	config.Options = copts.Options
+	config.Copts = copts.Copts
 
 	if copts.DockerImage != "" {
 		config.config.Image = copts.DockerImage
 	}
-
-	if copts.Entrypoint != "" {
-		if strings.Count(copts.Entrypoint, " ") != 0 {
-			split := strings.Split(copts.Entrypoint, " ")
-			config.config.Entrypoint = split
+	if copts.Options.Name != "" {
+		config.containerName = copts.Options.Name
 	} else {
-			config.config.Entrypoint = strslice.StrSlice{copts.Entrypoint}
+		config.Options.Name = config.containerName
 	}
-		config.config.Cmd = []string{}
-	}
-	if copts.Platform != "" {
-		split := strings.Split(copts.Platform, "/")
-		if len(split) != 2 {
-			return errors.Errorf("invalid port format for --platform: %s", copts.Platform)
-		}
-		config.platform = &v12.Platform{
-			OS:           split[0],
-			Architecture: split[1],
-		}
-	}
-
-	// collect all the environment variables for the container
-	envVariables, err := opts.ReadKVEnvStrings([]string{}, copts.Env.GetAll())
+	if copts.Options.Platform != "" {
+		p, err := platforms.Parse(copts.Options.Platform)
 		if err != nil {
-		return err
+			return errors.Wrap(err, "error parsing specified platform")
 		}
-	config.config.Env = append(config.config.Env, envVariables...)
-
-	publishOpts := copts.Publish.GetAll()
-	var (
-		ports         map[nat.Port]struct{}
-		portBindings  map[nat.Port][]nat.PortBinding
-		convertedOpts []string
-	)
-
-	convertedOpts, err = convertToStandardNotation(publishOpts)
-	if err != nil {
-		return err
+		config.platform = &p
 	}
 
-	ports, portBindings, err = nat.ParsePortSpecs(convertedOpts)
-	if err != nil {
-		return err
-	}
+	config.hostConfig = tempContainerConfig.HostConfig
+	config.networkingConfig.EndpointsConfig = util.Merge[string, *network.EndpointSettings](tempContainerConfig.NetworkingConfig.EndpointsConfig, config.networkingConfig.EndpointsConfig)
 
-	// Merge in exposed ports to the map of published ports
-	for _, e := range copts.Expose.GetAll() {
-		if strings.Contains(e, ":") {
-			return errors.Errorf("invalid port format for --expose: %s", e)
+	c := tempContainerConfig.Config
+	var entrypoint = config.config.Entrypoint
+	var args = config.config.Cmd
+	// if special --entrypoint, then use it
+	if len(c.Entrypoint) != 0 {
+		entrypoint = c.Entrypoint
+		args = c.Cmd
 	}
-		// support two formats for expose, original format <portnum>/[<proto>]
-		// or <startport-endport>/[<proto>]
-		proto, port := nat.SplitProtoPort(e)
-		// parse the start and end port and create a sequence of ports to expose
-		// if expose a port, the start and end port are the same
-		start, end, err := nat.ParsePortRange(port)
-		if err != nil {
-			return errors.Errorf("invalid range format for --expose: %s, error: %s", e, err)
+	if len(c.Cmd) != 0 {
+		args = c.Cmd
 	}
-		for i := start; i <= end; i++ {
-			p, err := nat.NewPort(proto, strconv.FormatUint(i, 10))
-			if err != nil {
-				return err
-			}
-			if _, exists := ports[p]; !exists {
-				ports[p] = struct{}{}
-			}
-		}
-	}
-	for port, bindings := range portBindings {
-		config.hostConfig.PortBindings[port] = bindings
-	}
-	for port, s := range ports {
-		config.config.ExposedPorts[port] = s
+	c.Entrypoint = entrypoint
+	c.Cmd = args
+	c.Env = append(config.config.Env, c.Env...)
+	c.Image = config.config.Image
+	if c.User == "" {
+		c.User = config.config.User
 	}
+	c.Volumes = util.Merge[string, struct{}](c.Volumes, config.config.Volumes)
 
-	mounts := copts.Mounts.Value()
-	if len(mounts) > 0 && copts.VolumeDriver != "" {
-		logrus.Warn("`--volume-driver` is ignored for volumes specified via `--mount`. Use `--mount type=volume,volume-driver=...` instead.")
-	}
-	var binds []string
-	volumes := copts.Volumes.GetMap()
-	// add any bind targets to the list of container volumes
-	for bind := range copts.Volumes.GetMap() {
-		parsed, _ := loader.ParseVolume(bind)
-
-		if parsed.Source != "" {
-			toBind := bind
-
-			if parsed.Type == string(mounttypes.TypeBind) {
-				if arr := strings.SplitN(bind, ":", 2); len(arr) == 2 {
-					hostPart := arr[0]
-					if strings.HasPrefix(hostPart, "."+string(filepath.Separator)) || hostPart == "." {
-						if absHostPart, err := filepath.Abs(hostPart); err == nil {
-							hostPart = absHostPart
-						}
-					}
-					toBind = hostPart + ":" + arr[1]
-				}
-			}
-
-			// after creating the bind mount we want to delete it from the copts.volumes values because
-			// we do not want bind mounts being committed to image configs
-			binds = append(binds, toBind)
-			// We should delete from the map (`volumes`) here, as deleting from copts.volumes will not work if
-			// there are duplicates entries.
-			delete(volumes, bind)
-		}
-	}
-
-	config.hostConfig.Binds = binds
-	networkOpts, err := parseNetworkOpts(copts)
-	if err != nil {
-		return err
-	}
-	config.networkingConfig = &network.NetworkingConfig{EndpointsConfig: networkOpts}
+	config.config = c
 
 	return nil
 }
-
-func convertToStandardNotation(ports []string) ([]string, error) {
-	optsList := []string{}
-	for _, publish := range ports {
-		if strings.Contains(publish, "=") {
-			params := map[string]string{"protocol": "tcp"}
-			for _, param := range strings.Split(publish, ",") {
-				opt := strings.Split(param, "=")
-				if len(opt) < 2 {
-					return optsList, errors.Errorf("invalid publish opts format (should be name=value but got '%s')", param)
-				}
-
-				params[opt[0]] = opt[1]
-			}
-			optsList = append(optsList, fmt.Sprintf("%s:%s/%s", params["published"], params["target"], params["protocol"]))
-		} else {
-			optsList = append(optsList, publish)
-		}
-	}
-	return optsList, nil
-}
-
-// parseNetworkOpts converts --network advanced options to endpoint-specs, and combines
-// them with the old --network-alias and --links. If returns an error if conflicting options
-// are found.
-//
-// this function may return _multiple_ endpoints, which is not currently supported
-// by the daemon, but may be in future; it's up to the daemon to produce an error
-// in case that is not supported.
-func parseNetworkOpts(copts Options) (map[string]*network.EndpointSettings, error) {
-	var (
-		endpoints                         = make(map[string]*network.EndpointSettings, len(copts.NetMode.Value()))
-		hasUserDefined, hasNonUserDefined bool
-	)
-
-	for i, n := range copts.NetMode.Value() {
-		n := n
-		if container.NetworkMode(n.Target).IsUserDefined() {
-			hasUserDefined = true
-		} else {
-			hasNonUserDefined = true
-		}
-		ep, err := parseNetworkAttachmentOpt(n)
-		if err != nil {
-			return nil, err
-		}
-		if _, ok := endpoints[n.Target]; ok {
-			return nil, errdefs.InvalidParameter(errors.Errorf("network %q is specified multiple times", n.Target))
-		}
-
-		// For backward compatibility: if no custom options are provided for the network,
-		// and only a single network is specified, omit the endpoint-configuration
-		// on the client (the daemon will still create it when creating the container)
-		if i == 0 && len(copts.NetMode.Value()) == 1 {
-			if ep == nil || reflect.DeepEqual(*ep, network.EndpointSettings{}) {
-				continue
-			}
-		}
-		endpoints[n.Target] = ep
-	}
-	if hasUserDefined && hasNonUserDefined {
-		return nil, errdefs.InvalidParameter(errors.New("conflicting options: cannot attach both user-defined and non-user-defined network-modes"))
-	}
-	return endpoints, nil
-}
-
-func parseNetworkAttachmentOpt(ep opts.NetworkAttachmentOpts) (*network.EndpointSettings, error) {
-	if strings.TrimSpace(ep.Target) == "" {
-		return nil, errors.New("no name set for network")
-	}
-	if !container.NetworkMode(ep.Target).IsUserDefined() {
-		if len(ep.Aliases) > 0 {
-			return nil, errors.New("network-scoped aliases are only supported for user-defined networks")
-		}
-		if len(ep.Links) > 0 {
-			return nil, errors.New("links are only supported for user-defined networks")
-		}
-	}
-
-	epConfig := &network.EndpointSettings{
-		NetworkID: ep.Target,
-	}
-	epConfig.Aliases = append(epConfig.Aliases, ep.Aliases...)
-	if len(ep.DriverOpts) > 0 {
-		epConfig.DriverOpts = make(map[string]string)
-		epConfig.DriverOpts = ep.DriverOpts
-	}
-	if len(ep.Links) > 0 {
-		epConfig.Links = ep.Links
-	}
-	if ep.IPv4Address != "" || ep.IPv6Address != "" || len(ep.LinkLocalIPs) > 0 {
-		epConfig.IPAMConfig = &network.EndpointIPAMConfig{
-			IPv4Address:  ep.IPv4Address,
-			IPv6Address:  ep.IPv6Address,
-			LinkLocalIPs: ep.LinkLocalIPs,
-		}
-	}
-	return epConfig, nil
-}

pkg/dev/run.go

@@ -2,11 +2,13 @@ package dev
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"math/rand"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -20,10 +22,13 @@ import (
 	"github.com/docker/docker/api/types"
 	typescommand "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/client"
+	apiclient "github.com/docker/docker/client"
 	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/pkg/jsonmessage"
+	"github.com/moby/term"
 	dockerterm "github.com/moby/term"
 	v12 "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/sirupsen/logrus"
 	log "github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/wait"
 
@@ -90,6 +95,11 @@ func run(ctx context.Context, runConfig *RunConfig, cli *client.Client, c *comma
 		if err != nil {
 			return
 		}
+		if inspect.State != nil && (inspect.State.Status == "exited" || inspect.State.Status == "dead" || inspect.State.Dead) {
+			once.Do(func() { close(chanStop) })
+			err = errors.New(fmt.Sprintf("container status: %s", inspect.State.Status))
+			return
+		}
 		if inspect.State != nil && inspect.State.Running {
 			once.Do(func() { close(chanStop) })
 			return
@@ -125,6 +135,175 @@ func run(ctx context.Context, runConfig *RunConfig, cli *client.Client, c *comma
 	return
 }
 
+func runFirst(ctx context.Context, runConfig *RunConfig, cli *apiclient.Client, dockerCli *command.DockerCli) (id string, err error) {
+	rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	defer func() {
+		if err != nil {
+			_ = cli.ContainerRemove(ctx, id, types.ContainerRemoveOptions{Force: true})
+		}
+	}()
+
+	stdout, stderr := dockerCli.Out(), dockerCli.Err()
+	client := dockerCli.Client()
+
+	runConfig.config.ArgsEscaped = false
+
+	if err := dockerCli.In().CheckTty(runConfig.config.AttachStdin, runConfig.config.Tty); err != nil {
+		return id, err
+	}
+
+	if !runConfig.Options.Detach {
+		if err := dockerCli.In().CheckTty(runConfig.config.AttachStdin, runConfig.config.Tty); err != nil {
+			return id, err
+		}
+	} else {
+		if runConfig.Copts.attach.Len() != 0 {
+			return id, errors.New("Conflicting options: -a and -d")
+		}
+
+		runConfig.config.AttachStdin = false
+		runConfig.config.AttachStdout = false
+		runConfig.config.AttachStderr = false
+		runConfig.config.StdinOnce = false
+	}
+
+	ctx, cancelFun := context.WithCancel(context.Background())
+	defer cancelFun()
+
+	createResponse, err := createContainer(ctx, dockerCli, &containerConfig{
+		Config:           runConfig.config,
+		HostConfig:       runConfig.hostConfig,
+		NetworkingConfig: runConfig.networkingConfig,
+	}, &runConfig.Options.createOptions)
+	if err != nil {
+		return "", err
+	}
+	log.Infof("Created container: %s", runConfig.containerName)
+
+	var (
+		waitDisplayID chan struct{}
+		errCh         chan error
+	)
+	if !runConfig.config.AttachStdout && !runConfig.config.AttachStderr {
+		// Make this asynchronous to allow the client to write to stdin before having to read the ID
+		waitDisplayID = make(chan struct{})
+		go func() {
+			defer close(waitDisplayID)
+			fmt.Fprintln(stdout, createResponse.ID)
+		}()
+	}
+	attach := runConfig.config.AttachStdin || runConfig.config.AttachStdout || runConfig.config.AttachStderr
+	if attach {
+		close, err := attachContainer(ctx, dockerCli, &errCh, runConfig.config, createResponse.ID)
+
+		if err != nil {
+			return id, err
+		}
+		defer close()
+	}
+	statusChan := waitExitOrRemoved(ctx, dockerCli, createResponse.ID, runConfig.Copts.autoRemove)
+	// start the container
+	if err := client.ContainerStart(ctx, createResponse.ID, types.ContainerStartOptions{}); err != nil {
+		// If we have hijackedIOStreamer, we should notify
+		// hijackedIOStreamer we are going to exit and wait
+		// to avoid the terminal are not restored.
+		if attach {
+			cancelFun()
+			<-errCh
+		}
+
+		reportError(stderr, "run", err.Error(), false)
+		if runConfig.Copts.autoRemove {
+			// wait container to be removed
+			<-statusChan
+		}
+		return id, runStartContainerErr(err)
+	}
+
+	if (runConfig.config.AttachStdin || runConfig.config.AttachStdout || runConfig.config.AttachStderr) && runConfig.config.Tty && dockerCli.Out().IsTerminal() {
+		if err := container.MonitorTtySize(ctx, dockerCli, createResponse.ID, false); err != nil {
+			fmt.Fprintln(stderr, "Error monitoring TTY size:", err)
+		}
+	}
+
+	if errCh != nil {
+		if err := <-errCh; err != nil {
+			if _, ok := err.(term.EscapeError); ok {
+				// The user entered the detach escape sequence.
+				return id, nil
+			}
+
+			logrus.Debugf("Error hijack: %s", err)
+			return id, err
+		}
+	}
+
+	// Detached mode: wait for the id to be displayed and return.
+	if !runConfig.config.AttachStdout && !runConfig.config.AttachStderr {
+		// Detached mode
+		<-waitDisplayID
+		return id, nil
+	}
+
+	status := <-statusChan
+	if status != 0 {
+		return id, errors.New(strconv.Itoa(status))
+	}
+	log.Infof("Wait container %s to be running...", runConfig.containerName)
+	chanStop := make(chan struct{})
+	var inspect types.ContainerJSON
+	var once = &sync.Once{}
+	wait.Until(func() {
+		inspect, err = cli.ContainerInspect(ctx, createResponse.ID)
+		if err != nil && errdefs.IsNotFound(err) {
+			once.Do(func() { close(chanStop) })
+			return
+		}
+		if err != nil {
+			return
+		}
+		if inspect.State != nil && (inspect.State.Status == "exited" || inspect.State.Status == "dead" || inspect.State.Dead) {
+			once.Do(func() { close(chanStop) })
+			err = errors.New(fmt.Sprintf("container status: %s", inspect.State.Status))
+			return
+		}
+		if inspect.State != nil && inspect.State.Running {
+			once.Do(func() { close(chanStop) })
+			return
+		}
+	}, time.Second, chanStop)
+	if err != nil {
+		err = fmt.Errorf("failed to wait container to be ready: %v", err)
+		return
+	}
+
+	// print port mapping to host
+	var empty = true
+	var str string
+	if inspect.NetworkSettings != nil && inspect.NetworkSettings.Ports != nil {
+		var list []string
+		for port, bindings := range inspect.NetworkSettings.Ports {
+			var p []string
+			for _, binding := range bindings {
+				if binding.HostPort != "" {
+					p = append(p, binding.HostPort)
+					empty = false
+				}
+			}
+			list = append(list, fmt.Sprintf("%s:%s", port, strings.Join(p, ",")))
+		}
+		str = fmt.Sprintf("Container %s is running on port %s now", runConfig.containerName, strings.Join(list, " "))
+	}
+	if !empty {
+		log.Infoln(str)
+	} else {
+		log.Infof("Container %s is running now", runConfig.containerName)
+	}
+
+	return
+}
+
 func PullImage(ctx context.Context, platform *v12.Platform, cli *client.Client, c *command.DockerCli, img string) error {
 	var readCloser io.ReadCloser
 	var plat string

pkg/dev/utils.go

@@ -0,0 +1,162 @@
+package dev
+
+import (
+	"context"
+	"strconv"
+
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/events"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/versions"
+	"github.com/sirupsen/logrus"
+)
+
+func waitExitOrRemoved(ctx context.Context, dockerCli command.Cli, containerID string, waitRemove bool) <-chan int {
+	if len(containerID) == 0 {
+		// containerID can never be empty
+		panic("Internal Error: waitExitOrRemoved needs a containerID as parameter")
+	}
+
+	// Older versions used the Events API, and even older versions did not
+	// support server-side removal. This legacyWaitExitOrRemoved method
+	// preserves that old behavior and any issues it may have.
+	if versions.LessThan(dockerCli.Client().ClientVersion(), "1.30") {
+		return legacyWaitExitOrRemoved(ctx, dockerCli, containerID, waitRemove)
+	}
+
+	condition := container.WaitConditionNextExit
+	if waitRemove {
+		condition = container.WaitConditionRemoved
+	}
+
+	resultC, errC := dockerCli.Client().ContainerWait(ctx, containerID, condition)
+
+	statusC := make(chan int)
+	go func() {
+		select {
+		case result := <-resultC:
+			if result.Error != nil {
+				logrus.Errorf("Error waiting for container: %v", result.Error.Message)
+				statusC <- 125
+			} else {
+				statusC <- int(result.StatusCode)
+			}
+		case err := <-errC:
+			logrus.Errorf("error waiting for container: %v", err)
+			statusC <- 125
+		}
+	}()
+
+	return statusC
+}
+
+func legacyWaitExitOrRemoved(ctx context.Context, dockerCli command.Cli, containerID string, waitRemove bool) <-chan int {
+	var removeErr error
+	statusChan := make(chan int)
+	exitCode := 125
+
+	// Get events via Events API
+	f := filters.NewArgs()
+	f.Add("type", "container")
+	f.Add("container", containerID)
+	options := types.EventsOptions{
+		Filters: f,
+	}
+	eventCtx, cancel := context.WithCancel(ctx)
+	eventq, errq := dockerCli.Client().Events(eventCtx, options)
+
+	eventProcessor := func(e events.Message) bool {
+		stopProcessing := false
+		switch e.Status {
+		case "die":
+			if v, ok := e.Actor.Attributes["exitCode"]; ok {
+				code, cerr := strconv.Atoi(v)
+				if cerr != nil {
+					logrus.Errorf("failed to convert exitcode '%q' to int: %v", v, cerr)
+				} else {
+					exitCode = code
+				}
+			}
+			if !waitRemove {
+				stopProcessing = true
+			} else {
+				// If we are talking to an older daemon, `AutoRemove` is not supported.
+				// We need to fall back to the old behavior, which is client-side removal
+				if versions.LessThan(dockerCli.Client().ClientVersion(), "1.25") {
+					go func() {
+						removeErr = dockerCli.Client().ContainerRemove(ctx, containerID, types.ContainerRemoveOptions{RemoveVolumes: true})
+						if removeErr != nil {
+							logrus.Errorf("error removing container: %v", removeErr)
+							cancel() // cancel the event Q
+						}
+					}()
+				}
+			}
+		case "detach":
+			exitCode = 0
+			stopProcessing = true
+		case "destroy":
+			stopProcessing = true
+		}
+		return stopProcessing
+	}
+
+	go func() {
+		defer func() {
+			statusChan <- exitCode // must always send an exit code or the caller will block
+			cancel()
+		}()
+
+		for {
+			select {
+			case <-eventCtx.Done():
+				if removeErr != nil {
+					return
+				}
+			case evt := <-eventq:
+				if eventProcessor(evt) {
+					return
+				}
+			case err := <-errq:
+				logrus.Errorf("error getting events from daemon: %v", err)
+				return
+			}
+		}
+	}()
+
+	return statusChan
+}
+
+func parallelOperation(ctx context.Context, containers []string, op func(ctx context.Context, container string) error) chan error {
+	if len(containers) == 0 {
+		return nil
+	}
+	const defaultParallel int = 50
+	sem := make(chan struct{}, defaultParallel)
+	errChan := make(chan error)
+
+	// make sure result is printed in correct order
+	output := map[string]chan error{}
+	for _, c := range containers {
+		output[c] = make(chan error, 1)
+	}
+	go func() {
+		for _, c := range containers {
+			err := <-output[c]
+			errChan <- err
+		}
+	}()
+
+	go func() {
+		for _, c := range containers {
+			sem <- struct{}{} // Wait for active queue sem to drain.
+			go func(container string) {
+				output[container] <- op(ctx, container)
+				<-sem
+			}(c)
+		}
+	}()
+	return errChan
+}

pkg/test/2pod.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test
+  labels:
+    app: test
+spec:
+  terminationGracePeriodSeconds: 0
+  containers:
+    - name: nginx
+      image: nginx
+      imagePullPolicy: IfNotPresent
+    - name: tomcat
+      image: tomcat
+      imagePullPolicy: IfNotPresent
+  restartPolicy: Always

pkg/util/util.go

@@ -749,3 +749,15 @@ func MoveToTemp() {
 		log.Debugln(err)
 	}
 }
+
+func Merge[K comparable, V any](fromMap, ToMap map[K]V) map[K]V {
+	for keyToMap, valueToMap := range ToMap {
+		fromMap[keyToMap] = valueToMap
+	}
+	if fromMap == nil {
+		// merge(nil, map[string]interface{...}) -> map[string]interface{...}
+		return ToMap
+	}
+
+	return fromMap
+}