diff --git a/cmd/kubevpn/cmds/dev.go b/cmd/kubevpn/cmds/dev.go index 90a3586a..eaa1e4db 100644 --- a/cmd/kubevpn/cmds/dev.go +++ b/cmd/kubevpn/cmds/dev.go @@ -4,7 +4,8 @@ import ( "os" "github.com/docker/cli/cli" - "github.com/docker/cli/opts" + "github.com/docker/cli/cli/command" + dockercomp "github.com/docker/cli/cli/command/completion" "github.com/spf13/cobra" cmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/completion" @@ -18,21 +19,19 @@ import ( ) func CmdDev(f cmdutil.Factory) *cobra.Command { - var devOptions = dev.Options{ - Factory: f, - Entrypoint: "", - Publish: opts.NewListOpts(nil), - Expose: opts.NewListOpts(nil), - Env: opts.NewListOpts(nil), - Volumes: opts.NewListOpts(nil), - ExtraHosts: opts.NewListOpts(nil), - NoProxy: false, - ExtraCIDR: []string{}, + var devOptions = &dev.Options{ + Factory: f, + NoProxy: false, + ExtraCIDR: []string{}, + } + _, dockerCli, err := dev.GetClient() + if err != nil { + panic(err) } var sshConf = &util.SshConfig{} var transferImage bool cmd := &cobra.Command{ - Use: "dev", + Use: "dev [OPTIONS] RESOURCE [COMMAND] [ARG...]", Short: i18n.T("Startup your kubernetes workloads in local Docker container with same volume、env、and network"), Long: templates.LongDesc(i18n.T(` Startup your kubernetes workloads in local Docker container with same volume、env、and network @@ -51,22 +50,22 @@ Startup your kubernetes workloads in local Docker container with same volume、e kubevpn dev service/productpage # Develop workloads with mesh, traffic with header a=1, will hit local PC, otherwise no effect - kubevpn dev service/productpage --headers a=1 + kubevpn dev --headers a=1 service/productpage # Develop workloads without proxy traffic - kubevpn dev service/productpage --no-proxy + kubevpn dev --no-proxy service/productpage # Develop workloads which api-server behind of bastion host or ssh jump host - kubevpn dev deployment/productpage --ssh-addr 192.168.1.100:22 --ssh-username root --ssh-keyfile /Users/naison/.ssh/ssh.pem + kubevpn dev --ssh-addr 192.168.1.100:22 --ssh-username root --ssh-keyfile /Users/naison/.ssh/ssh.pem deployment/productpage # it also support ProxyJump, like ┌──────┐ ┌──────┐ ┌──────┐ ┌──────┐ ┌────────────┐ │ pc ├────►│ ssh1 ├────►│ ssh2 ├────►│ ssh3 ├─────►... ─────► │ api-server │ └──────┘ └──────┘ └──────┘ └──────┘ └────────────┘ - kubevpn dev deployment/productpage --ssh-alias + kubevpn dev --ssh-alias deployment/productpage `)), - Args: cli.ExactArgs(1), + Args: cli.RequiresMinArgs(1), PreRunE: func(cmd *cobra.Command, args []string) error { if !util.IsAdmin() { util.RunWithElevated() @@ -80,9 +79,14 @@ Startup your kubernetes workloads in local Docker container with same volume、e } } return handler.SshJump(sshConf, cmd.Flags()) + return nil }, RunE: func(cmd *cobra.Command, args []string) error { - return dev.DoDev(devOptions, args, f) + devOptions.Workload = args[0] + if len(args) > 1 { + devOptions.Copts.Args = args[1:] + } + return dev.DoDev(devOptions, cmd.Flags(), f) }, } cmd.Flags().StringToStringVarP(&devOptions.Headers, "headers", "H", map[string]string{}, "Traffic with special headers with reverse it to local PC, you should startup your service after reverse workloads successfully, If not special, redirect all traffic to local PC, format is k=v, like: k1=v1,k2=v2") @@ -96,24 +100,42 @@ Startup your kubernetes workloads in local Docker container with same volume、e cmd.Flags().StringVar((*string)(&devOptions.ConnectMode), "connect-mode", string(dev.ConnectModeHost), "Connect to kubernetes network in container or in host, eg: ["+string(dev.ConnectModeContainer)+"|"+string(dev.ConnectModeHost)+"]") cmd.Flags().BoolVar(&transferImage, "transfer-image", false, "transfer image to remote registry, it will transfer image "+config.OriginImage+" to flags `--image` special image, default: "+config.Image) - // docker options - cmd.Flags().Var(&devOptions.ExtraHosts, "add-host", "Add a custom host-to-IP mapping (host:ip)") - // We allow for both "--net" and "--network", although the latter is the recommended way. - cmd.Flags().Var(&devOptions.NetMode, "net", "Connect a container to a network, eg: [default|bridge|host|none|container:$CONTAINER_ID]") - cmd.Flags().Var(&devOptions.NetMode, "network", "Connect a container to a network") - cmd.Flags().MarkHidden("net") - cmd.Flags().VarP(&devOptions.Volumes, "volume", "v", "Bind mount a volume") - cmd.Flags().Var(&devOptions.Mounts, "mount", "Attach a filesystem mount to the container") - cmd.Flags().Var(&devOptions.Expose, "expose", "Expose a port or a range of ports") - cmd.Flags().VarP(&devOptions.Publish, "publish", "p", "Publish a container's port(s) to the host") - cmd.Flags().BoolVarP(&devOptions.PublishAll, "publish-all", "P", false, "Publish all exposed ports to random ports") - cmd.Flags().VarP(&devOptions.Env, "env", "e", "Set environment variables") - cmd.Flags().StringVar(&devOptions.Entrypoint, "entrypoint", "", "Overwrite the default ENTRYPOINT of the image") + // diy docker options cmd.Flags().StringVar(&devOptions.DockerImage, "docker-image", "", "Overwrite the default K8s pod of the image") - //cmd.Flags().StringVar(&devOptions.Pull, "pull", container.PullImageMissing, `Pull image before creating ("`+container.PullImageAlways+`"|"`+container.PullImageMissing+`"|"`+container.PullImageNever+`")`) - cmd.Flags().StringVar(&devOptions.Platform, "platform", os.Getenv("DOCKER_DEFAULT_PLATFORM"), "Set platform if server is multi-platform capable") - cmd.Flags().StringVar(&devOptions.VolumeDriver, "volume-driver", "", "Optional volume driver for the container") - _ = cmd.Flags().SetAnnotation("platform", "version", []string{"1.32"}) + // origin docker options + flags := cmd.Flags() + flags.SetInterspersed(false) + + // These are flags not stored in Config/HostConfig + flags.BoolVarP(&devOptions.Options.Detach, "detach", "d", false, "Run container in background and print container ID") + flags.StringVar(&devOptions.Options.Name, "name", "", "Assign a name to the container") + flags.StringVar(&devOptions.Options.Pull, "pull", dev.PullImageMissing, `Pull image before running ("`+dev.PullImageAlways+`"|"`+dev.PullImageMissing+`"|"`+dev.PullImageNever+`")`) + flags.BoolVarP(&devOptions.Options.Quiet, "quiet", "q", false, "Suppress the pull output") + + // Add an explicit help that doesn't have a `-h` to prevent the conflict + // with hostname + flags.Bool("help", false, "Print usage") + + command.AddPlatformFlag(flags, &devOptions.Options.Platform) + command.AddTrustVerificationFlags(flags, &devOptions.Options.Untrusted, dockerCli.ContentTrustEnabled()) + devOptions.Copts = dev.AddFlags(flags) + + _ = cmd.RegisterFlagCompletionFunc( + "env", + func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return os.Environ(), cobra.ShellCompDirectiveNoFileComp + }, + ) + _ = cmd.RegisterFlagCompletionFunc( + "env-file", + func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return nil, cobra.ShellCompDirectiveDefault + }, + ) + _ = cmd.RegisterFlagCompletionFunc( + "network", + dockercomp.NetworkNames(nil), + ) addSshFlags(cmd, sshConf) return cmd diff --git a/pkg/dev/LICENSE b/pkg/dev/LICENSE new file mode 100644 index 00000000..9c8e20ab --- /dev/null +++ b/pkg/dev/LICENSE @@ -0,0 +1,191 @@ + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2013-2017 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/pkg/dev/convert.go b/pkg/dev/convert.go index d5151996..a59052aa 100644 --- a/pkg/dev/convert.go +++ b/pkg/dev/convert.go @@ -32,19 +32,23 @@ import ( ) type RunConfig struct { + containerName string + k8sContainerName string + config *container.Config hostConfig *container.HostConfig networkingConfig *network.NetworkingConfig platform *v12.Platform - containerName string - k8sContainerName string + + Options RunOptions + Copts *ContainerOptions } -func ConvertKubeResourceToContainer(namespace string, temp v1.PodTemplateSpec, envMap map[string][]string, mountVolume map[string][]mount.Mount, dnsConfig *miekgdns.ClientConfig) (runConfigList Run) { +func ConvertKubeResourceToContainer(namespace string, temp v1.PodTemplateSpec, envMap map[string][]string, mountVolume map[string][]mount.Mount, dnsConfig *miekgdns.ClientConfig) (runConfigList ConfigList) { spec := temp.Spec for _, c := range spec.Containers { var r RunConfig - config := &container.Config{ + tmpConfig := &container.Config{ Hostname: func() string { var hostname = spec.Hostname if hostname == "" { @@ -84,7 +88,7 @@ func ConvertKubeResourceToContainer(namespace string, temp v1.PodTemplateSpec, e Shell: nil, } if temp.DeletionGracePeriodSeconds != nil { - config.StopTimeout = (*int)(unsafe.Pointer(temp.DeletionGracePeriodSeconds)) + tmpConfig.StopTimeout = (*int)(unsafe.Pointer(temp.DeletionGracePeriodSeconds)) } hostConfig := &container.HostConfig{ Binds: []string{}, @@ -138,7 +142,7 @@ func ConvertKubeResourceToContainer(namespace string, temp v1.PodTemplateSpec, e portset[port1] = struct{}{} } hostConfig.PortBindings = portmap - config.ExposedPorts = portset + tmpConfig.ExposedPorts = portset if c.SecurityContext != nil && c.SecurityContext.Capabilities != nil { hostConfig.CapAdd = append(hostConfig.CapAdd, *(*strslice.StrSlice)(unsafe.Pointer(&c.SecurityContext.Capabilities.Add))...) hostConfig.CapDrop = *(*strslice.StrSlice)(unsafe.Pointer(&c.SecurityContext.Capabilities.Drop)) @@ -150,10 +154,10 @@ func ConvertKubeResourceToContainer(namespace string, temp v1.PodTemplateSpec, e } r.containerName = fmt.Sprintf("%s_%s_%s_%s", c.Name, namespace, "kubevpn", suffix) r.k8sContainerName = c.Name - r.config = config + r.config = tmpConfig r.hostConfig = hostConfig r.networkingConfig = &network.NetworkingConfig{EndpointsConfig: make(map[string]*network.EndpointSettings)} - r.platform = /*&v12.Platform{Architecture: "amd64", OS: "linux"}*/ nil + r.platform = nil runConfigList = append(runConfigList, &r) } @@ -179,11 +183,11 @@ func GetDNS(ctx context.Context, f util.Factory, ns, pod string) (*miekgdns.Clie return nil, err } - fromPod, err := dns.GetDNSServiceIPFromPod(clientSet, client, config, pod, ns) + clientConfig, err := dns.GetDNSServiceIPFromPod(clientSet, client, config, pod, ns) if err != nil { return nil, err } - return fromPod, nil + return clientConfig, nil } // GetVolume key format: [container name]-[volume mount name] diff --git a/pkg/dev/dockercreate.go b/pkg/dev/dockercreate.go new file mode 100644 index 00000000..c08a10a1 --- /dev/null +++ b/pkg/dev/dockercreate.go @@ -0,0 +1,271 @@ +package dev + +import ( + "context" + "fmt" + "io" + "os" + "regexp" + + "github.com/containerd/containerd/platforms" + "github.com/docker/cli/cli/command" + "github.com/docker/cli/cli/command/image" + "github.com/docker/distribution/reference" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/versions" + apiclient "github.com/docker/docker/client" + "github.com/docker/docker/pkg/jsonmessage" + "github.com/docker/docker/registry" + specs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// Pull constants +const ( + PullImageAlways = "always" + PullImageMissing = "missing" // Default (matches previous behavior) + PullImageNever = "never" +) + +type createOptions struct { + Name string + Platform string + Untrusted bool + Pull string // always, missing, never + Quiet bool +} + +func pullImage(ctx context.Context, dockerCli command.Cli, image string, platform string, out io.Writer) error { + ref, err := reference.ParseNormalizedNamed(image) + if err != nil { + return err + } + + // Resolve the Repository name from fqn to RepositoryInfo + repoInfo, err := registry.ParseRepositoryInfo(ref) + if err != nil { + return err + } + + authConfig := command.ResolveAuthConfig(ctx, dockerCli, repoInfo.Index) + encodedAuth, err := command.EncodeAuthToBase64(authConfig) + if err != nil { + return err + } + + options := types.ImageCreateOptions{ + RegistryAuth: encodedAuth, + Platform: platform, + } + + responseBody, err := dockerCli.Client().ImageCreate(ctx, image, options) + if err != nil { + return err + } + defer responseBody.Close() + + return jsonmessage.DisplayJSONMessagesStream( + responseBody, + out, + dockerCli.Out().FD(), + dockerCli.Out().IsTerminal(), + nil) +} + +type cidFile struct { + path string + file *os.File + written bool +} + +func (cid *cidFile) Close() error { + if cid.file == nil { + return nil + } + cid.file.Close() + + if cid.written { + return nil + } + if err := os.Remove(cid.path); err != nil { + return errors.Wrapf(err, "failed to remove the CID file '%s'", cid.path) + } + + return nil +} + +func (cid *cidFile) Write(id string) error { + if cid.file == nil { + return nil + } + if _, err := cid.file.Write([]byte(id)); err != nil { + return errors.Wrap(err, "failed to write the container ID to the file") + } + cid.written = true + return nil +} + +func newCIDFile(path string) (*cidFile, error) { + if path == "" { + return &cidFile{}, nil + } + if _, err := os.Stat(path); err == nil { + return nil, errors.Errorf("container ID file found, make sure the other container isn't running or delete %s", path) + } + + f, err := os.Create(path) + if err != nil { + return nil, errors.Wrap(err, "failed to create the container ID file") + } + + return &cidFile{path: path, file: f}, nil +} + +//nolint:gocyclo +func createContainer(ctx context.Context, dockerCli command.Cli, containerConfig *containerConfig, opts *createOptions) (*container.CreateResponse, error) { + config := containerConfig.Config + hostConfig := containerConfig.HostConfig + networkingConfig := containerConfig.NetworkingConfig + stderr := dockerCli.Err() + + warnOnOomKillDisable(*hostConfig, stderr) + warnOnLocalhostDNS(*hostConfig, stderr) + + var ( + trustedRef reference.Canonical + namedRef reference.Named + ) + + containerIDFile, err := newCIDFile(hostConfig.ContainerIDFile) + if err != nil { + return nil, err + } + defer containerIDFile.Close() + + ref, err := reference.ParseAnyReference(config.Image) + if err != nil { + return nil, err + } + if named, ok := ref.(reference.Named); ok { + namedRef = reference.TagNameOnly(named) + + if taggedRef, ok := namedRef.(reference.NamedTagged); ok && !opts.Untrusted { + var err error + trustedRef, err = image.TrustedReference(ctx, dockerCli, taggedRef, nil) + if err != nil { + return nil, err + } + config.Image = reference.FamiliarString(trustedRef) + } + } + + pullAndTagImage := func() error { + pullOut := stderr + if opts.Quiet { + pullOut = io.Discard + } + if err := pullImage(ctx, dockerCli, config.Image, opts.Platform, pullOut); err != nil { + return err + } + if taggedRef, ok := namedRef.(reference.NamedTagged); ok && trustedRef != nil { + return image.TagTrusted(ctx, dockerCli, trustedRef, taggedRef) + } + return nil + } + + var platform *specs.Platform + // Engine API version 1.41 first introduced the option to specify platform on + // create. It will produce an error if you try to set a platform on older API + // versions, so check the API version here to maintain backwards + // compatibility for CLI users. + if opts.Platform != "" && versions.GreaterThanOrEqualTo(dockerCli.Client().ClientVersion(), "1.41") { + p, err := platforms.Parse(opts.Platform) + if err != nil { + return nil, errors.Wrap(err, "error parsing specified platform") + } + platform = &p + } + + if opts.Pull == PullImageAlways { + if err := pullAndTagImage(); err != nil { + return nil, err + } + } + + hostConfig.ConsoleSize[0], hostConfig.ConsoleSize[1] = dockerCli.Out().GetTtySize() + + response, err := dockerCli.Client().ContainerCreate(ctx, config, hostConfig, networkingConfig, platform, opts.Name) + if err != nil { + // Pull image if it does not exist locally and we have the PullImageMissing option. Default behavior. + if apiclient.IsErrNotFound(err) && namedRef != nil && opts.Pull == PullImageMissing { + if !opts.Quiet { + // we don't want to write to stdout anything apart from container.ID + fmt.Fprintf(stderr, "Unable to find image '%s' locally\n", reference.FamiliarString(namedRef)) + } + + if err := pullAndTagImage(); err != nil { + return nil, err + } + + var retryErr error + response, retryErr = dockerCli.Client().ContainerCreate(ctx, config, hostConfig, networkingConfig, platform, opts.Name) + if retryErr != nil { + return nil, retryErr + } + } else { + return nil, err + } + } + + for _, warning := range response.Warnings { + fmt.Fprintf(stderr, "WARNING: %s\n", warning) + } + err = containerIDFile.Write(response.ID) + return &response, err +} + +func warnOnOomKillDisable(hostConfig container.HostConfig, stderr io.Writer) { + if hostConfig.OomKillDisable != nil && *hostConfig.OomKillDisable && hostConfig.Memory == 0 { + fmt.Fprintln(stderr, "WARNING: Disabling the OOM killer on containers without setting a '-m/--memory' limit may be dangerous.") + } +} + +// check the DNS settings passed via --dns against localhost regexp to warn if +// they are trying to set a DNS to a localhost address +func warnOnLocalhostDNS(hostConfig container.HostConfig, stderr io.Writer) { + for _, dnsIP := range hostConfig.DNS { + if isLocalhost(dnsIP) { + fmt.Fprintf(stderr, "WARNING: Localhost DNS setting (--dns=%s) may fail in containers.\n", dnsIP) + return + } + } +} + +// IPLocalhost is a regex pattern for IPv4 or IPv6 loopback range. +const ipLocalhost = `((127\.([0-9]{1,3}\.){2}[0-9]{1,3})|(::1)$)` + +var localhostIPRegexp = regexp.MustCompile(ipLocalhost) + +// IsLocalhost returns true if ip matches the localhost IP regular expression. +// Used for determining if nameserver settings are being passed which are +// localhost addresses +func isLocalhost(ip string) bool { + return localhostIPRegexp.MatchString(ip) +} + +func validatePullOpt(val string) error { + switch val { + case PullImageAlways, PullImageMissing, PullImageNever, "": + // valid option, but nothing to do yet + return nil + default: + return fmt.Errorf( + "invalid pull option: '%s': must be one of %q, %q or %q", + val, + PullImageAlways, + PullImageMissing, + PullImageNever, + ) + } +} diff --git a/pkg/dev/dockeropts.go b/pkg/dev/dockeropts.go new file mode 100644 index 00000000..550ac7e5 --- /dev/null +++ b/pkg/dev/dockeropts.go @@ -0,0 +1,1076 @@ +package dev + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path" + "path/filepath" + "reflect" + "regexp" + "strconv" + "strings" + "time" + + "github.com/docker/cli/cli/compose/loader" + "github.com/docker/cli/opts" + "github.com/docker/docker/api/types/container" + mounttypes "github.com/docker/docker/api/types/mount" + networktypes "github.com/docker/docker/api/types/network" + "github.com/docker/docker/api/types/strslice" + "github.com/docker/docker/api/types/versions" + "github.com/docker/docker/errdefs" + "github.com/docker/go-connections/nat" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/spf13/pflag" +) + +var ( + deviceCgroupRuleRegexp = regexp.MustCompile(`^[acb] ([0-9]+|\*):([0-9]+|\*) [rwm]{1,3}$`) +) + +// ContainerOptions is a data object with all the options for creating a container +type ContainerOptions struct { + attach opts.ListOpts + volumes opts.ListOpts + tmpfs opts.ListOpts + mounts opts.MountOpt + blkioWeightDevice opts.WeightdeviceOpt + deviceReadBps opts.ThrottledeviceOpt + deviceWriteBps opts.ThrottledeviceOpt + links opts.ListOpts + aliases opts.ListOpts + linkLocalIPs opts.ListOpts + deviceReadIOps opts.ThrottledeviceOpt + deviceWriteIOps opts.ThrottledeviceOpt + env opts.ListOpts + labels opts.ListOpts + deviceCgroupRules opts.ListOpts + devices opts.ListOpts + gpus opts.GpuOpts + ulimits *opts.UlimitOpt + sysctls *opts.MapOpts + publish opts.ListOpts + expose opts.ListOpts + dns opts.ListOpts + dnsSearch opts.ListOpts + dnsOptions opts.ListOpts + extraHosts opts.ListOpts + volumesFrom opts.ListOpts + envFile opts.ListOpts + capAdd opts.ListOpts + capDrop opts.ListOpts + groupAdd opts.ListOpts + securityOpt opts.ListOpts + storageOpt opts.ListOpts + labelsFile opts.ListOpts + loggingOpts opts.ListOpts + privileged bool + pidMode string + utsMode string + usernsMode string + cgroupnsMode string + publishAll bool + stdin bool + tty bool + oomKillDisable bool + oomScoreAdj int + containerIDFile string + entrypoint string + hostname string + domainname string + memory opts.MemBytes + memoryReservation opts.MemBytes + memorySwap opts.MemSwapBytes + kernelMemory opts.MemBytes + user string + workingDir string + cpuCount int64 + cpuShares int64 + cpuPercent int64 + cpuPeriod int64 + cpuRealtimePeriod int64 + cpuRealtimeRuntime int64 + cpuQuota int64 + cpus opts.NanoCPUs + cpusetCpus string + cpusetMems string + blkioWeight uint16 + ioMaxBandwidth opts.MemBytes + ioMaxIOps uint64 + swappiness int64 + netMode opts.NetworkOpt + macAddress string + ipv4Address string + ipv6Address string + ipcMode string + pidsLimit int64 + restartPolicy string + readonlyRootfs bool + loggingDriver string + cgroupParent string + volumeDriver string + stopSignal string + stopTimeout int + isolation string + shmSize opts.MemBytes + noHealthcheck bool + healthCmd string + healthInterval time.Duration + healthTimeout time.Duration + healthStartPeriod time.Duration + healthRetries int + runtime string + autoRemove bool + init bool + + Image string + Args []string +} + +// AddFlags adds all command line flags that will be used by parse to the FlagSet +func AddFlags(flags *pflag.FlagSet) *ContainerOptions { + copts := &ContainerOptions{ + aliases: opts.NewListOpts(nil), + attach: opts.NewListOpts(validateAttach), + blkioWeightDevice: opts.NewWeightdeviceOpt(opts.ValidateWeightDevice), + capAdd: opts.NewListOpts(nil), + capDrop: opts.NewListOpts(nil), + dns: opts.NewListOpts(opts.ValidateIPAddress), + dnsOptions: opts.NewListOpts(nil), + dnsSearch: opts.NewListOpts(opts.ValidateDNSSearch), + deviceCgroupRules: opts.NewListOpts(validateDeviceCgroupRule), + deviceReadBps: opts.NewThrottledeviceOpt(opts.ValidateThrottleBpsDevice), + deviceReadIOps: opts.NewThrottledeviceOpt(opts.ValidateThrottleIOpsDevice), + deviceWriteBps: opts.NewThrottledeviceOpt(opts.ValidateThrottleBpsDevice), + deviceWriteIOps: opts.NewThrottledeviceOpt(opts.ValidateThrottleIOpsDevice), + devices: opts.NewListOpts(nil), // devices can only be validated after we know the server OS + env: opts.NewListOpts(opts.ValidateEnv), + envFile: opts.NewListOpts(nil), + expose: opts.NewListOpts(nil), + extraHosts: opts.NewListOpts(opts.ValidateExtraHost), + groupAdd: opts.NewListOpts(nil), + labels: opts.NewListOpts(opts.ValidateLabel), + labelsFile: opts.NewListOpts(nil), + linkLocalIPs: opts.NewListOpts(nil), + links: opts.NewListOpts(opts.ValidateLink), + loggingOpts: opts.NewListOpts(nil), + publish: opts.NewListOpts(nil), + securityOpt: opts.NewListOpts(nil), + storageOpt: opts.NewListOpts(nil), + sysctls: opts.NewMapOpts(nil, opts.ValidateSysctl), + tmpfs: opts.NewListOpts(nil), + ulimits: opts.NewUlimitOpt(nil), + volumes: opts.NewListOpts(nil), + volumesFrom: opts.NewListOpts(nil), + } + + // General purpose flags + flags.VarP(&copts.attach, "attach", "a", "Attach to STDIN, STDOUT or STDERR") + flags.Var(&copts.deviceCgroupRules, "device-cgroup-rule", "Add a rule to the cgroup allowed devices list") + flags.Var(&copts.devices, "device", "Add a host device to the container") + flags.Var(&copts.gpus, "gpus", "GPU devices to add to the container ('all' to pass all GPUs)") + flags.SetAnnotation("gpus", "version", []string{"1.40"}) + flags.VarP(&copts.env, "env", "e", "Set environment variables") + flags.Var(&copts.envFile, "env-file", "Read in a file of environment variables") + flags.StringVar(&copts.entrypoint, "entrypoint", "", "Overwrite the default ENTRYPOINT of the image") + flags.Var(&copts.groupAdd, "group-add", "Add additional groups to join") + flags.StringVarP(&copts.hostname, "hostname", "h", "", "Container host name") + flags.StringVar(&copts.domainname, "domainname", "", "Container NIS domain name") + flags.BoolVarP(&copts.stdin, "interactive", "i", false, "Keep STDIN open even if not attached") + flags.VarP(&copts.labels, "label", "l", "Set meta data on a container") + flags.Var(&copts.labelsFile, "label-file", "Read in a line delimited file of labels") + flags.BoolVar(&copts.readonlyRootfs, "read-only", false, "Mount the container's root filesystem as read only") + flags.StringVar(&copts.restartPolicy, "restart", "no", "Restart policy to apply when a container exits") + flags.StringVar(&copts.stopSignal, "stop-signal", "", "Signal to stop the container") + flags.IntVar(&copts.stopTimeout, "stop-timeout", 0, "Timeout (in seconds) to stop a container") + flags.SetAnnotation("stop-timeout", "version", []string{"1.25"}) + flags.Var(copts.sysctls, "sysctl", "Sysctl options") + flags.BoolVarP(&copts.tty, "tty", "t", false, "Allocate a pseudo-TTY") + flags.Var(copts.ulimits, "ulimit", "Ulimit options") + flags.StringVarP(&copts.user, "user", "u", "", "Username or UID (format: [:])") + flags.StringVarP(&copts.workingDir, "workdir", "w", "", "Working directory inside the container") + flags.BoolVar(&copts.autoRemove, "rm", false, "Automatically remove the container when it exits") + + // Security + flags.Var(&copts.capAdd, "cap-add", "Add Linux capabilities") + flags.Var(&copts.capDrop, "cap-drop", "Drop Linux capabilities") + flags.BoolVar(&copts.privileged, "privileged", false, "Give extended privileges to this container") + flags.Var(&copts.securityOpt, "security-opt", "Security Options") + flags.StringVar(&copts.usernsMode, "userns", "", "User namespace to use") + flags.StringVar(&copts.cgroupnsMode, "cgroupns", "", `Cgroup namespace to use (host|private) +'host': Run the container in the Docker host's cgroup namespace +'private': Run the container in its own private cgroup namespace +'': Use the cgroup namespace as configured by the + default-cgroupns-mode option on the daemon (default)`) + flags.SetAnnotation("cgroupns", "version", []string{"1.41"}) + + // Network and port publishing flag + flags.Var(&copts.extraHosts, "add-host", "Add a custom host-to-IP mapping (host:ip)") + flags.Var(&copts.dns, "dns", "Set custom DNS servers") + // We allow for both "--dns-opt" and "--dns-option", although the latter is the recommended way. + // This is to be consistent with service create/update + flags.Var(&copts.dnsOptions, "dns-opt", "Set DNS options") + flags.Var(&copts.dnsOptions, "dns-option", "Set DNS options") + flags.MarkHidden("dns-opt") + flags.Var(&copts.dnsSearch, "dns-search", "Set custom DNS search domains") + flags.Var(&copts.expose, "expose", "Expose a port or a range of ports") + flags.StringVar(&copts.ipv4Address, "ip", "", "IPv4 address (e.g., 172.30.100.104)") + flags.StringVar(&copts.ipv6Address, "ip6", "", "IPv6 address (e.g., 2001:db8::33)") + flags.Var(&copts.links, "link", "Add link to another container") + flags.Var(&copts.linkLocalIPs, "link-local-ip", "Container IPv4/IPv6 link-local addresses") + flags.StringVar(&copts.macAddress, "mac-address", "", "Container MAC address (e.g., 92:d0:c6:0a:29:33)") + flags.VarP(&copts.publish, "publish", "p", "Publish a container's port(s) to the host") + flags.BoolVarP(&copts.publishAll, "publish-all", "P", false, "Publish all exposed ports to random ports") + // We allow for both "--net" and "--network", although the latter is the recommended way. + flags.Var(&copts.netMode, "net", "Connect a container to a network") + flags.Var(&copts.netMode, "network", "Connect a container to a network") + flags.MarkHidden("net") + // We allow for both "--net-alias" and "--network-alias", although the latter is the recommended way. + flags.Var(&copts.aliases, "net-alias", "Add network-scoped alias for the container") + flags.Var(&copts.aliases, "network-alias", "Add network-scoped alias for the container") + flags.MarkHidden("net-alias") + + // Logging and storage + flags.StringVar(&copts.loggingDriver, "log-driver", "", "Logging driver for the container") + flags.StringVar(&copts.volumeDriver, "volume-driver", "", "Optional volume driver for the container") + flags.Var(&copts.loggingOpts, "log-opt", "Log driver options") + flags.Var(&copts.storageOpt, "storage-opt", "Storage driver options for the container") + flags.Var(&copts.tmpfs, "tmpfs", "Mount a tmpfs directory") + flags.Var(&copts.volumesFrom, "volumes-from", "Mount volumes from the specified container(s)") + flags.VarP(&copts.volumes, "volume", "v", "Bind mount a volume") + flags.Var(&copts.mounts, "mount", "Attach a filesystem mount to the container") + + // Health-checking + flags.StringVar(&copts.healthCmd, "health-cmd", "", "Command to run to check health") + flags.DurationVar(&copts.healthInterval, "health-interval", 0, "Time between running the check (ms|s|m|h) (default 0s)") + flags.IntVar(&copts.healthRetries, "health-retries", 0, "Consecutive failures needed to report unhealthy") + flags.DurationVar(&copts.healthTimeout, "health-timeout", 0, "Maximum time to allow one check to run (ms|s|m|h) (default 0s)") + flags.DurationVar(&copts.healthStartPeriod, "health-start-period", 0, "Start period for the container to initialize before starting health-retries countdown (ms|s|m|h) (default 0s)") + flags.SetAnnotation("health-start-period", "version", []string{"1.29"}) + flags.BoolVar(&copts.noHealthcheck, "no-healthcheck", false, "Disable any container-specified HEALTHCHECK") + + // Resource management + flags.Uint16Var(&copts.blkioWeight, "blkio-weight", 0, "Block IO (relative weight), between 10 and 1000, or 0 to disable (default 0)") + flags.Var(&copts.blkioWeightDevice, "blkio-weight-device", "Block IO weight (relative device weight)") + flags.StringVar(&copts.containerIDFile, "cidfile", "", "Write the container ID to the file") + flags.StringVar(&copts.cpusetCpus, "cpuset-cpus", "", "CPUs in which to allow execution (0-3, 0,1)") + flags.StringVar(&copts.cpusetMems, "cpuset-mems", "", "MEMs in which to allow execution (0-3, 0,1)") + flags.Int64Var(&copts.cpuCount, "cpu-count", 0, "CPU count (Windows only)") + flags.SetAnnotation("cpu-count", "ostype", []string{"windows"}) + flags.Int64Var(&copts.cpuPercent, "cpu-percent", 0, "CPU percent (Windows only)") + flags.SetAnnotation("cpu-percent", "ostype", []string{"windows"}) + flags.Int64Var(&copts.cpuPeriod, "cpu-period", 0, "Limit CPU CFS (Completely Fair Scheduler) period") + flags.Int64Var(&copts.cpuQuota, "cpu-quota", 0, "Limit CPU CFS (Completely Fair Scheduler) quota") + flags.Int64Var(&copts.cpuRealtimePeriod, "cpu-rt-period", 0, "Limit CPU real-time period in microseconds") + flags.SetAnnotation("cpu-rt-period", "version", []string{"1.25"}) + flags.Int64Var(&copts.cpuRealtimeRuntime, "cpu-rt-runtime", 0, "Limit CPU real-time runtime in microseconds") + flags.SetAnnotation("cpu-rt-runtime", "version", []string{"1.25"}) + flags.Int64Var(&copts.cpuShares, "cpu-shares", 0, "CPU shares (relative weight)") + flags.Var(&copts.cpus, "cpus", "Number of CPUs") + flags.SetAnnotation("cpus", "version", []string{"1.25"}) + flags.Var(&copts.deviceReadBps, "device-read-bps", "Limit read rate (bytes per second) from a device") + flags.Var(&copts.deviceReadIOps, "device-read-iops", "Limit read rate (IO per second) from a device") + flags.Var(&copts.deviceWriteBps, "device-write-bps", "Limit write rate (bytes per second) to a device") + flags.Var(&copts.deviceWriteIOps, "device-write-iops", "Limit write rate (IO per second) to a device") + flags.Var(&copts.ioMaxBandwidth, "io-maxbandwidth", "Maximum IO bandwidth limit for the system drive (Windows only)") + flags.SetAnnotation("io-maxbandwidth", "ostype", []string{"windows"}) + flags.Uint64Var(&copts.ioMaxIOps, "io-maxiops", 0, "Maximum IOps limit for the system drive (Windows only)") + flags.SetAnnotation("io-maxiops", "ostype", []string{"windows"}) + flags.Var(&copts.kernelMemory, "kernel-memory", "Kernel memory limit") + flags.VarP(&copts.memory, "memory", "m", "Memory limit") + flags.Var(&copts.memoryReservation, "memory-reservation", "Memory soft limit") + flags.Var(&copts.memorySwap, "memory-swap", "Swap limit equal to memory plus swap: '-1' to enable unlimited swap") + flags.Int64Var(&copts.swappiness, "memory-swappiness", -1, "Tune container memory swappiness (0 to 100)") + flags.BoolVar(&copts.oomKillDisable, "oom-kill-disable", false, "Disable OOM Killer") + flags.IntVar(&copts.oomScoreAdj, "oom-score-adj", 0, "Tune host's OOM preferences (-1000 to 1000)") + flags.Int64Var(&copts.pidsLimit, "pids-limit", 0, "Tune container pids limit (set -1 for unlimited)") + + // Low-level execution (cgroups, namespaces, ...) + flags.StringVar(&copts.cgroupParent, "cgroup-parent", "", "Optional parent cgroup for the container") + flags.StringVar(&copts.ipcMode, "ipc", "", "IPC mode to use") + flags.StringVar(&copts.isolation, "isolation", "", "Container isolation technology") + flags.StringVar(&copts.pidMode, "pid", "", "PID namespace to use") + flags.Var(&copts.shmSize, "shm-size", "Size of /dev/shm") + flags.StringVar(&copts.utsMode, "uts", "", "UTS namespace to use") + flags.StringVar(&copts.runtime, "runtime", "", "Runtime to use for this container") + + flags.BoolVar(&copts.init, "init", false, "Run an init inside the container that forwards signals and reaps processes") + flags.SetAnnotation("init", "version", []string{"1.25"}) + return copts +} + +type containerConfig struct { + Config *container.Config + HostConfig *container.HostConfig + NetworkingConfig *networktypes.NetworkingConfig +} + +// parse parses the args for the specified command and generates a Config, +// a HostConfig and returns them with the specified command. +// If the specified args are not valid, it will return an error. +// +//nolint:gocyclo +func parse(flags *pflag.FlagSet, copts *ContainerOptions, serverOS string) (*containerConfig, error) { + var ( + attachStdin = copts.attach.Get("stdin") + attachStdout = copts.attach.Get("stdout") + attachStderr = copts.attach.Get("stderr") + ) + + // Validate the input mac address + if copts.macAddress != "" { + if _, err := opts.ValidateMACAddress(copts.macAddress); err != nil { + return nil, errors.Errorf("%s is not a valid mac address", copts.macAddress) + } + } + if copts.stdin { + attachStdin = true + } + // If -a is not set, attach to stdout and stderr + if copts.attach.Len() == 0 { + attachStdout = true + attachStderr = true + } + + var err error + + swappiness := copts.swappiness + if swappiness != -1 && (swappiness < 0 || swappiness > 100) { + return nil, errors.Errorf("invalid value: %d. Valid memory swappiness range is 0-100", swappiness) + } + + mounts := copts.mounts.Value() + if len(mounts) > 0 && copts.volumeDriver != "" { + logrus.Warn("`--volume-driver` is ignored for volumes specified via `--mount`. Use `--mount type=volume,volume-driver=...` instead.") + } + var binds []string + volumes := copts.volumes.GetMap() + // add any bind targets to the list of container volumes + for bind := range copts.volumes.GetMap() { + parsed, _ := loader.ParseVolume(bind) + + if parsed.Source != "" { + toBind := bind + + if parsed.Type == string(mounttypes.TypeBind) { + if arr := strings.SplitN(bind, ":", 2); len(arr) == 2 { + hostPart := arr[0] + if strings.HasPrefix(hostPart, "."+string(filepath.Separator)) || hostPart == "." { + if absHostPart, err := filepath.Abs(hostPart); err == nil { + hostPart = absHostPart + } + } + toBind = hostPart + ":" + arr[1] + } + } + + // after creating the bind mount we want to delete it from the copts.volumes values because + // we do not want bind mounts being committed to image configs + binds = append(binds, toBind) + // We should delete from the map (`volumes`) here, as deleting from copts.volumes will not work if + // there are duplicates entries. + delete(volumes, bind) + } + } + + // Can't evaluate options passed into --tmpfs until we actually mount + tmpfs := make(map[string]string) + for _, t := range copts.tmpfs.GetAll() { + if arr := strings.SplitN(t, ":", 2); len(arr) > 1 { + tmpfs[arr[0]] = arr[1] + } else { + tmpfs[arr[0]] = "" + } + } + + var ( + runCmd strslice.StrSlice + entrypoint strslice.StrSlice + ) + + if len(copts.Args) > 0 { + runCmd = strslice.StrSlice(copts.Args) + } + + if copts.entrypoint != "" { + entrypoint = strslice.StrSlice{copts.entrypoint} + } else if flags.Changed("entrypoint") { + // if `--entrypoint=` is parsed then Entrypoint is reset + entrypoint = []string{""} + } + + publishOpts := copts.publish.GetAll() + var ( + ports map[nat.Port]struct{} + portBindings map[nat.Port][]nat.PortBinding + convertedOpts []string + ) + + convertedOpts, err = convertToStandardNotation(publishOpts) + if err != nil { + return nil, err + } + + ports, portBindings, err = nat.ParsePortSpecs(convertedOpts) + if err != nil { + return nil, err + } + + // Merge in exposed ports to the map of published ports + for _, e := range copts.expose.GetAll() { + if strings.Contains(e, ":") { + return nil, errors.Errorf("invalid port format for --expose: %s", e) + } + // support two formats for expose, original format /[] + // or /[] + proto, port := nat.SplitProtoPort(e) + // parse the start and end port and create a sequence of ports to expose + // if expose a port, the start and end port are the same + start, end, err := nat.ParsePortRange(port) + if err != nil { + return nil, errors.Errorf("invalid range format for --expose: %s, error: %s", e, err) + } + for i := start; i <= end; i++ { + p, err := nat.NewPort(proto, strconv.FormatUint(i, 10)) + if err != nil { + return nil, err + } + if _, exists := ports[p]; !exists { + ports[p] = struct{}{} + } + } + } + + // validate and parse device mappings. Note we do late validation of the + // device path (as opposed to during flag parsing), as at the time we are + // parsing flags, we haven't yet sent a _ping to the daemon to determine + // what operating system it is. + deviceMappings := []container.DeviceMapping{} + for _, device := range copts.devices.GetAll() { + var ( + validated string + deviceMapping container.DeviceMapping + err error + ) + validated, err = validateDevice(device, serverOS) + if err != nil { + return nil, err + } + deviceMapping, err = parseDevice(validated, serverOS) + if err != nil { + return nil, err + } + deviceMappings = append(deviceMappings, deviceMapping) + } + + // collect all the environment variables for the container + envVariables, err := opts.ReadKVEnvStrings(copts.envFile.GetAll(), copts.env.GetAll()) + if err != nil { + return nil, err + } + + // collect all the labels for the container + labels, err := opts.ReadKVStrings(copts.labelsFile.GetAll(), copts.labels.GetAll()) + if err != nil { + return nil, err + } + + pidMode := container.PidMode(copts.pidMode) + if !pidMode.Valid() { + return nil, errors.Errorf("--pid: invalid PID mode") + } + + utsMode := container.UTSMode(copts.utsMode) + if !utsMode.Valid() { + return nil, errors.Errorf("--uts: invalid UTS mode") + } + + usernsMode := container.UsernsMode(copts.usernsMode) + if !usernsMode.Valid() { + return nil, errors.Errorf("--userns: invalid USER mode") + } + + cgroupnsMode := container.CgroupnsMode(copts.cgroupnsMode) + if !cgroupnsMode.Valid() { + return nil, errors.Errorf("--cgroupns: invalid CGROUP mode") + } + + restartPolicy, err := opts.ParseRestartPolicy(copts.restartPolicy) + if err != nil { + return nil, err + } + + loggingOpts, err := parseLoggingOpts(copts.loggingDriver, copts.loggingOpts.GetAll()) + if err != nil { + return nil, err + } + + securityOpts, err := parseSecurityOpts(copts.securityOpt.GetAll()) + if err != nil { + return nil, err + } + + securityOpts, maskedPaths, readonlyPaths := parseSystemPaths(securityOpts) + + storageOpts, err := parseStorageOpts(copts.storageOpt.GetAll()) + if err != nil { + return nil, err + } + + // Healthcheck + var healthConfig *container.HealthConfig + haveHealthSettings := copts.healthCmd != "" || + copts.healthInterval != 0 || + copts.healthTimeout != 0 || + copts.healthStartPeriod != 0 || + copts.healthRetries != 0 + if copts.noHealthcheck { + if haveHealthSettings { + return nil, errors.Errorf("--no-healthcheck conflicts with --health-* options") + } + test := strslice.StrSlice{"NONE"} + healthConfig = &container.HealthConfig{Test: test} + } else if haveHealthSettings { + var probe strslice.StrSlice + if copts.healthCmd != "" { + args := []string{"CMD-SHELL", copts.healthCmd} + probe = strslice.StrSlice(args) + } + if copts.healthInterval < 0 { + return nil, errors.Errorf("--health-interval cannot be negative") + } + if copts.healthTimeout < 0 { + return nil, errors.Errorf("--health-timeout cannot be negative") + } + if copts.healthRetries < 0 { + return nil, errors.Errorf("--health-retries cannot be negative") + } + if copts.healthStartPeriod < 0 { + return nil, fmt.Errorf("--health-start-period cannot be negative") + } + + healthConfig = &container.HealthConfig{ + Test: probe, + Interval: copts.healthInterval, + Timeout: copts.healthTimeout, + StartPeriod: copts.healthStartPeriod, + Retries: copts.healthRetries, + } + } + + resources := container.Resources{ + CgroupParent: copts.cgroupParent, + Memory: copts.memory.Value(), + MemoryReservation: copts.memoryReservation.Value(), + MemorySwap: copts.memorySwap.Value(), + MemorySwappiness: &copts.swappiness, + KernelMemory: copts.kernelMemory.Value(), + OomKillDisable: &copts.oomKillDisable, + NanoCPUs: copts.cpus.Value(), + CPUCount: copts.cpuCount, + CPUPercent: copts.cpuPercent, + CPUShares: copts.cpuShares, + CPUPeriod: copts.cpuPeriod, + CpusetCpus: copts.cpusetCpus, + CpusetMems: copts.cpusetMems, + CPUQuota: copts.cpuQuota, + CPURealtimePeriod: copts.cpuRealtimePeriod, + CPURealtimeRuntime: copts.cpuRealtimeRuntime, + PidsLimit: &copts.pidsLimit, + BlkioWeight: copts.blkioWeight, + BlkioWeightDevice: copts.blkioWeightDevice.GetList(), + BlkioDeviceReadBps: copts.deviceReadBps.GetList(), + BlkioDeviceWriteBps: copts.deviceWriteBps.GetList(), + BlkioDeviceReadIOps: copts.deviceReadIOps.GetList(), + BlkioDeviceWriteIOps: copts.deviceWriteIOps.GetList(), + IOMaximumIOps: copts.ioMaxIOps, + IOMaximumBandwidth: uint64(copts.ioMaxBandwidth), + Ulimits: copts.ulimits.GetList(), + DeviceCgroupRules: copts.deviceCgroupRules.GetAll(), + Devices: deviceMappings, + DeviceRequests: copts.gpus.Value(), + } + + config := &container.Config{ + Hostname: copts.hostname, + Domainname: copts.domainname, + ExposedPorts: ports, + User: copts.user, + Tty: copts.tty, + // TODO: deprecated, it comes from -n, --networking + // it's still needed internally to set the network to disabled + // if e.g. bridge is none in daemon opts, and in inspect + NetworkDisabled: false, + OpenStdin: copts.stdin, + AttachStdin: attachStdin, + AttachStdout: attachStdout, + AttachStderr: attachStderr, + Env: envVariables, + Cmd: runCmd, + Image: copts.Image, + Volumes: volumes, + MacAddress: copts.macAddress, + Entrypoint: entrypoint, + WorkingDir: copts.workingDir, + Labels: opts.ConvertKVStringsToMap(labels), + StopSignal: copts.stopSignal, + Healthcheck: healthConfig, + } + if flags.Changed("stop-timeout") { + config.StopTimeout = &copts.stopTimeout + } + + hostConfig := &container.HostConfig{ + Binds: binds, + ContainerIDFile: copts.containerIDFile, + OomScoreAdj: copts.oomScoreAdj, + AutoRemove: copts.autoRemove, + Privileged: copts.privileged, + PortBindings: portBindings, + Links: copts.links.GetAll(), + PublishAllPorts: copts.publishAll, + // Make sure the dns fields are never nil. + // New containers don't ever have those fields nil, + // but pre created containers can still have those nil values. + // See https://github.com/docker/docker/pull/17779 + // for a more detailed explanation on why we don't want that. + DNS: copts.dns.GetAllOrEmpty(), + DNSSearch: copts.dnsSearch.GetAllOrEmpty(), + DNSOptions: copts.dnsOptions.GetAllOrEmpty(), + ExtraHosts: copts.extraHosts.GetAll(), + VolumesFrom: copts.volumesFrom.GetAll(), + IpcMode: container.IpcMode(copts.ipcMode), + NetworkMode: container.NetworkMode(copts.netMode.NetworkMode()), + PidMode: pidMode, + UTSMode: utsMode, + UsernsMode: usernsMode, + CgroupnsMode: cgroupnsMode, + CapAdd: strslice.StrSlice(copts.capAdd.GetAll()), + CapDrop: strslice.StrSlice(copts.capDrop.GetAll()), + GroupAdd: copts.groupAdd.GetAll(), + RestartPolicy: restartPolicy, + SecurityOpt: securityOpts, + StorageOpt: storageOpts, + ReadonlyRootfs: copts.readonlyRootfs, + LogConfig: container.LogConfig{Type: copts.loggingDriver, Config: loggingOpts}, + VolumeDriver: copts.volumeDriver, + Isolation: container.Isolation(copts.isolation), + ShmSize: copts.shmSize.Value(), + Resources: resources, + Tmpfs: tmpfs, + Sysctls: copts.sysctls.GetAll(), + Runtime: copts.runtime, + Mounts: mounts, + MaskedPaths: maskedPaths, + ReadonlyPaths: readonlyPaths, + } + + if copts.autoRemove && !hostConfig.RestartPolicy.IsNone() { + return nil, errors.Errorf("Conflicting options: --restart and --rm") + } + + // only set this value if the user provided the flag, else it should default to nil + if flags.Changed("init") { + hostConfig.Init = &copts.init + } + + // When allocating stdin in attached mode, close stdin at client disconnect + if config.OpenStdin && config.AttachStdin { + config.StdinOnce = true + } + + networkingConfig := &networktypes.NetworkingConfig{ + EndpointsConfig: make(map[string]*networktypes.EndpointSettings), + } + + networkingConfig.EndpointsConfig, err = parseNetworkOpts(copts) + if err != nil { + return nil, err + } + + return &containerConfig{ + Config: config, + HostConfig: hostConfig, + NetworkingConfig: networkingConfig, + }, nil +} + +// parseNetworkOpts converts --network advanced options to endpoint-specs, and combines +// them with the old --network-alias and --links. If returns an error if conflicting options +// are found. +// +// this function may return _multiple_ endpoints, which is not currently supported +// by the daemon, but may be in future; it's up to the daemon to produce an error +// in case that is not supported. +func parseNetworkOpts(copts *ContainerOptions) (map[string]*networktypes.EndpointSettings, error) { + var ( + endpoints = make(map[string]*networktypes.EndpointSettings, len(copts.netMode.Value())) + hasUserDefined, hasNonUserDefined bool + ) + + for i, n := range copts.netMode.Value() { + n := n + if container.NetworkMode(n.Target).IsUserDefined() { + hasUserDefined = true + } else { + hasNonUserDefined = true + } + if i == 0 { + // The first network corresponds with what was previously the "only" + // network, and what would be used when using the non-advanced syntax + // `--network-alias`, `--link`, `--ip`, `--ip6`, and `--link-local-ip` + // are set on this network, to preserve backward compatibility with + // the non-advanced notation + if err := applyContainerOptions(&n, copts); err != nil { + return nil, err + } + } + ep, err := parseNetworkAttachmentOpt(n) + if err != nil { + return nil, err + } + if _, ok := endpoints[n.Target]; ok { + return nil, errdefs.InvalidParameter(errors.Errorf("network %q is specified multiple times", n.Target)) + } + + // For backward compatibility: if no custom options are provided for the network, + // and only a single network is specified, omit the endpoint-configuration + // on the client (the daemon will still create it when creating the container) + if i == 0 && len(copts.netMode.Value()) == 1 { + if ep == nil || reflect.DeepEqual(*ep, networktypes.EndpointSettings{}) { + continue + } + } + endpoints[n.Target] = ep + } + if hasUserDefined && hasNonUserDefined { + return nil, errdefs.InvalidParameter(errors.New("conflicting options: cannot attach both user-defined and non-user-defined network-modes")) + } + return endpoints, nil +} + +func applyContainerOptions(n *opts.NetworkAttachmentOpts, copts *ContainerOptions) error { + // TODO should copts.MacAddress actually be set on the first network? (currently it's not) + // TODO should we error if _any_ advanced option is used? (i.e. forbid to combine advanced notation with the "old" flags (`--network-alias`, `--link`, `--ip`, `--ip6`)? + if len(n.Aliases) > 0 && copts.aliases.Len() > 0 { + return errdefs.InvalidParameter(errors.New("conflicting options: cannot specify both --network-alias and per-network alias")) + } + if len(n.Links) > 0 && copts.links.Len() > 0 { + return errdefs.InvalidParameter(errors.New("conflicting options: cannot specify both --link and per-network links")) + } + if n.IPv4Address != "" && copts.ipv4Address != "" { + return errdefs.InvalidParameter(errors.New("conflicting options: cannot specify both --ip and per-network IPv4 address")) + } + if n.IPv6Address != "" && copts.ipv6Address != "" { + return errdefs.InvalidParameter(errors.New("conflicting options: cannot specify both --ip6 and per-network IPv6 address")) + } + if copts.aliases.Len() > 0 { + n.Aliases = make([]string, copts.aliases.Len()) + copy(n.Aliases, copts.aliases.GetAll()) + } + if copts.links.Len() > 0 { + n.Links = make([]string, copts.links.Len()) + copy(n.Links, copts.links.GetAll()) + } + if copts.ipv4Address != "" { + n.IPv4Address = copts.ipv4Address + } + if copts.ipv6Address != "" { + n.IPv6Address = copts.ipv6Address + } + + // TODO should linkLocalIPs be added to the _first_ network only, or to _all_ networks? (should this be a per-network option as well?) + if copts.linkLocalIPs.Len() > 0 { + n.LinkLocalIPs = make([]string, copts.linkLocalIPs.Len()) + copy(n.LinkLocalIPs, copts.linkLocalIPs.GetAll()) + } + return nil +} + +func parseNetworkAttachmentOpt(ep opts.NetworkAttachmentOpts) (*networktypes.EndpointSettings, error) { + if strings.TrimSpace(ep.Target) == "" { + return nil, errors.New("no name set for network") + } + if !container.NetworkMode(ep.Target).IsUserDefined() { + if len(ep.Aliases) > 0 { + return nil, errors.New("network-scoped aliases are only supported for user-defined networks") + } + if len(ep.Links) > 0 { + return nil, errors.New("links are only supported for user-defined networks") + } + } + + epConfig := &networktypes.EndpointSettings{} + epConfig.Aliases = append(epConfig.Aliases, ep.Aliases...) + if len(ep.DriverOpts) > 0 { + epConfig.DriverOpts = make(map[string]string) + epConfig.DriverOpts = ep.DriverOpts + } + if len(ep.Links) > 0 { + epConfig.Links = ep.Links + } + if ep.IPv4Address != "" || ep.IPv6Address != "" || len(ep.LinkLocalIPs) > 0 { + epConfig.IPAMConfig = &networktypes.EndpointIPAMConfig{ + IPv4Address: ep.IPv4Address, + IPv6Address: ep.IPv6Address, + LinkLocalIPs: ep.LinkLocalIPs, + } + } + return epConfig, nil +} + +func convertToStandardNotation(ports []string) ([]string, error) { + optsList := []string{} + for _, publish := range ports { + if strings.Contains(publish, "=") { + params := map[string]string{"protocol": "tcp"} + for _, param := range strings.Split(publish, ",") { + opt := strings.Split(param, "=") + if len(opt) < 2 { + return optsList, errors.Errorf("invalid publish opts format (should be name=value but got '%s')", param) + } + + params[opt[0]] = opt[1] + } + optsList = append(optsList, fmt.Sprintf("%s:%s/%s", params["published"], params["target"], params["protocol"])) + } else { + optsList = append(optsList, publish) + } + } + return optsList, nil +} + +func parseLoggingOpts(loggingDriver string, loggingOpts []string) (map[string]string, error) { + loggingOptsMap := opts.ConvertKVStringsToMap(loggingOpts) + if loggingDriver == "none" && len(loggingOpts) > 0 { + return map[string]string{}, errors.Errorf("invalid logging opts for driver %s", loggingDriver) + } + return loggingOptsMap, nil +} + +// takes a local seccomp daemon, reads the file contents for sending to the daemon +func parseSecurityOpts(securityOpts []string) ([]string, error) { + for key, opt := range securityOpts { + con := strings.SplitN(opt, "=", 2) + if len(con) == 1 && con[0] != "no-new-privileges" { + if strings.Contains(opt, ":") { + con = strings.SplitN(opt, ":", 2) + } else { + return securityOpts, errors.Errorf("Invalid --security-opt: %q", opt) + } + } + if con[0] == "seccomp" && con[1] != "unconfined" { + f, err := os.ReadFile(con[1]) + if err != nil { + return securityOpts, errors.Errorf("opening seccomp profile (%s) failed: %v", con[1], err) + } + b := bytes.NewBuffer(nil) + if err := json.Compact(b, f); err != nil { + return securityOpts, errors.Errorf("compacting json for seccomp profile (%s) failed: %v", con[1], err) + } + securityOpts[key] = fmt.Sprintf("seccomp=%s", b.Bytes()) + } + } + + return securityOpts, nil +} + +// parseSystemPaths checks if `systempaths=unconfined` security option is set, +// and returns the `MaskedPaths` and `ReadonlyPaths` accordingly. An updated +// list of security options is returned with this option removed, because the +// `unconfined` option is handled client-side, and should not be sent to the +// daemon. +func parseSystemPaths(securityOpts []string) (filtered, maskedPaths, readonlyPaths []string) { + filtered = securityOpts[:0] + for _, opt := range securityOpts { + if opt == "systempaths=unconfined" { + maskedPaths = []string{} + readonlyPaths = []string{} + } else { + filtered = append(filtered, opt) + } + } + + return filtered, maskedPaths, readonlyPaths +} + +// parses storage options per container into a map +func parseStorageOpts(storageOpts []string) (map[string]string, error) { + m := make(map[string]string) + for _, option := range storageOpts { + if strings.Contains(option, "=") { + opt := strings.SplitN(option, "=", 2) + m[opt[0]] = opt[1] + } else { + return nil, errors.Errorf("invalid storage option") + } + } + return m, nil +} + +// parseDevice parses a device mapping string to a container.DeviceMapping struct +func parseDevice(device, serverOS string) (container.DeviceMapping, error) { + switch serverOS { + case "linux": + return parseLinuxDevice(device) + case "windows": + return parseWindowsDevice(device) + } + return container.DeviceMapping{}, errors.Errorf("unknown server OS: %s", serverOS) +} + +// parseLinuxDevice parses a device mapping string to a container.DeviceMapping struct +// knowing that the target is a Linux daemon +func parseLinuxDevice(device string) (container.DeviceMapping, error) { + src := "" + dst := "" + permissions := "rwm" + arr := strings.Split(device, ":") + switch len(arr) { + case 3: + permissions = arr[2] + fallthrough + case 2: + if validDeviceMode(arr[1]) { + permissions = arr[1] + } else { + dst = arr[1] + } + fallthrough + case 1: + src = arr[0] + default: + return container.DeviceMapping{}, errors.Errorf("invalid device specification: %s", device) + } + + if dst == "" { + dst = src + } + + deviceMapping := container.DeviceMapping{ + PathOnHost: src, + PathInContainer: dst, + CgroupPermissions: permissions, + } + return deviceMapping, nil +} + +// parseWindowsDevice parses a device mapping string to a container.DeviceMapping struct +// knowing that the target is a Windows daemon +func parseWindowsDevice(device string) (container.DeviceMapping, error) { + return container.DeviceMapping{PathOnHost: device}, nil +} + +// validateDeviceCgroupRule validates a device cgroup rule string format +// It will make sure 'val' is in the form: +// +// 'type major:minor mode' +func validateDeviceCgroupRule(val string) (string, error) { + if deviceCgroupRuleRegexp.MatchString(val) { + return val, nil + } + + return val, errors.Errorf("invalid device cgroup format '%s'", val) +} + +// validDeviceMode checks if the mode for device is valid or not. +// Valid mode is a composition of r (read), w (write), and m (mknod). +func validDeviceMode(mode string) bool { + var legalDeviceMode = map[rune]bool{ + 'r': true, + 'w': true, + 'm': true, + } + if mode == "" { + return false + } + for _, c := range mode { + if !legalDeviceMode[c] { + return false + } + legalDeviceMode[c] = false + } + return true +} + +// validateDevice validates a path for devices +func validateDevice(val string, serverOS string) (string, error) { + switch serverOS { + case "linux": + return validateLinuxPath(val, validDeviceMode) + case "windows": + // Windows does validation entirely server-side + return val, nil + } + return "", errors.Errorf("unknown server OS: %s", serverOS) +} + +// validateLinuxPath is the implementation of validateDevice knowing that the +// target server operating system is a Linux daemon. +// It will make sure 'val' is in the form: +// +// [host-dir:]container-path[:mode] +// +// It also validates the device mode. +func validateLinuxPath(val string, validator func(string) bool) (string, error) { + var containerPath string + var mode string + + if strings.Count(val, ":") > 2 { + return val, errors.Errorf("bad format for path: %s", val) + } + + split := strings.SplitN(val, ":", 3) + if split[0] == "" { + return val, errors.Errorf("bad format for path: %s", val) + } + switch len(split) { + case 1: + containerPath = split[0] + val = path.Clean(containerPath) + case 2: + if isValid := validator(split[1]); isValid { + containerPath = split[0] + mode = split[1] + val = fmt.Sprintf("%s:%s", path.Clean(containerPath), mode) + } else { + containerPath = split[1] + val = fmt.Sprintf("%s:%s", split[0], path.Clean(containerPath)) + } + case 3: + containerPath = split[1] + mode = split[2] + if isValid := validator(split[2]); !isValid { + return val, errors.Errorf("bad mode specified: %s", mode) + } + val = fmt.Sprintf("%s:%s:%s", split[0], containerPath, mode) + } + + if !path.IsAbs(containerPath) { + return val, errors.Errorf("%s is not an absolute path", containerPath) + } + return val, nil +} + +// validateAttach validates that the specified string is a valid attach option. +func validateAttach(val string) (string, error) { + s := strings.ToLower(val) + for _, str := range []string{"stdin", "stdout", "stderr"} { + if s == str { + return s, nil + } + } + return val, errors.Errorf("valid streams are STDIN, STDOUT and STDERR") +} + +func validateAPIVersion(c *containerConfig, serverAPIVersion string) error { + for _, m := range c.HostConfig.Mounts { + if m.BindOptions != nil && m.BindOptions.NonRecursive && versions.LessThan(serverAPIVersion, "1.40") { + return errors.Errorf("bind-nonrecursive requires API v1.40 or later") + } + } + return nil +} diff --git a/pkg/dev/dockerrun.go b/pkg/dev/dockerrun.go new file mode 100644 index 00000000..e17287ef --- /dev/null +++ b/pkg/dev/dockerrun.go @@ -0,0 +1,108 @@ +package dev + +import ( + "context" + "fmt" + "io" + "strings" + "syscall" + + "github.com/docker/cli/cli" + "github.com/docker/cli/cli/command" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" +) + +type RunOptions struct { + createOptions + Detach bool + sigProxy bool + detachKeys string +} + +func attachContainer(ctx context.Context, dockerCli command.Cli, errCh *chan error, config *container.Config, containerID string) (func(), error) { + options := types.ContainerAttachOptions{ + Stream: true, + Stdin: config.AttachStdin, + Stdout: config.AttachStdout, + Stderr: config.AttachStderr, + DetachKeys: dockerCli.ConfigFile().DetachKeys, + } + + resp, errAttach := dockerCli.Client().ContainerAttach(ctx, containerID, options) + if errAttach != nil { + return nil, errAttach + } + + var ( + out, cerr io.Writer + in io.ReadCloser + ) + if config.AttachStdin { + in = dockerCli.In() + } + if config.AttachStdout { + out = dockerCli.Out() + } + if config.AttachStderr { + if config.Tty { + cerr = dockerCli.Out() + } else { + cerr = dockerCli.Err() + } + } + + ch := make(chan error, 1) + *errCh = ch + + if in != nil && out != nil && cerr != nil { + + } + + go func() { + ch <- func() error { + streamer := hijackedIOStreamer{ + streams: dockerCli, + inputStream: in, + outputStream: out, + errorStream: cerr, + resp: resp, + tty: config.Tty, + detachKeys: options.DetachKeys, + } + + if errHijack := streamer.stream(ctx); errHijack != nil { + return errHijack + } + return errAttach + }() + }() + return resp.Close, nil +} + +// reportError is a utility method that prints a user-friendly message +// containing the error that occurred during parsing and a suggestion to get help +func reportError(stderr io.Writer, name string, str string, withHelp bool) { + str = strings.TrimSuffix(str, ".") + "." + if withHelp { + str += "\nSee 'docker " + name + " --help'." + } + _, _ = fmt.Fprintln(stderr, "docker:", str) +} + +// if container start fails with 'not found'/'no such' error, return 127 +// if container start fails with 'permission denied' error, return 126 +// return 125 for generic docker daemon failures +func runStartContainerErr(err error) error { + trimmedErr := strings.TrimPrefix(err.Error(), "Error response from daemon: ") + statusError := cli.StatusError{StatusCode: 125} + if strings.Contains(trimmedErr, "executable file not found") || + strings.Contains(trimmedErr, "no such file or directory") || + strings.Contains(trimmedErr, "system cannot find the file specified") { + statusError = cli.StatusError{StatusCode: 127} + } else if strings.Contains(trimmedErr, syscall.EACCES.Error()) { + statusError = cli.StatusError{StatusCode: 126} + } + + return statusError +} diff --git a/pkg/dev/hijack.go b/pkg/dev/hijack.go new file mode 100644 index 00000000..48079c4e --- /dev/null +++ b/pkg/dev/hijack.go @@ -0,0 +1,207 @@ +package dev + +import ( + "context" + "fmt" + "io" + "runtime" + "sync" + + "github.com/docker/cli/cli/command" + "github.com/docker/docker/api/types" + "github.com/docker/docker/pkg/ioutils" + "github.com/docker/docker/pkg/stdcopy" + "github.com/moby/term" + "github.com/sirupsen/logrus" +) + +// The default escape key sequence: ctrl-p, ctrl-q +// TODO: This could be moved to `pkg/term`. +var defaultEscapeKeys = []byte{16, 17} + +// A hijackedIOStreamer handles copying input to and output from streams to the +// connection. +type hijackedIOStreamer struct { + streams command.Streams + inputStream io.ReadCloser + outputStream io.Writer + errorStream io.Writer + + resp types.HijackedResponse + + tty bool + detachKeys string +} + +// stream handles setting up the IO and then begins streaming stdin/stdout +// to/from the hijacked connection, blocking until it is either done reading +// output, the user inputs the detach key sequence when in TTY mode, or when +// the given context is cancelled. +func (h *hijackedIOStreamer) stream(ctx context.Context) error { + restoreInput, err := h.setupInput() + if err != nil { + return fmt.Errorf("unable to setup input stream: %s", err) + } + + defer restoreInput() + + outputDone := h.beginOutputStream(restoreInput) + inputDone, detached := h.beginInputStream(restoreInput) + + select { + case err := <-outputDone: + return err + case <-inputDone: + // Input stream has closed. + if h.outputStream != nil || h.errorStream != nil { + // Wait for output to complete streaming. + select { + case err := <-outputDone: + return err + case <-ctx.Done(): + return ctx.Err() + } + } + return nil + case err := <-detached: + // Got a detach key sequence. + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +func (h *hijackedIOStreamer) setupInput() (restore func(), err error) { + if h.inputStream == nil || !h.tty { + // No need to setup input TTY. + // The restore func is a nop. + return func() {}, nil + } + + if err := setRawTerminal(h.streams); err != nil { + return nil, fmt.Errorf("unable to set IO streams as raw terminal: %s", err) + } + + // Use sync.Once so we may call restore multiple times but ensure we + // only restore the terminal once. + var restoreOnce sync.Once + restore = func() { + restoreOnce.Do(func() { + restoreTerminal(h.streams, h.inputStream) + }) + } + + // Wrap the input to detect detach escape sequence. + // Use default escape keys if an invalid sequence is given. + escapeKeys := defaultEscapeKeys + if h.detachKeys != "" { + customEscapeKeys, err := term.ToBytes(h.detachKeys) + if err != nil { + logrus.Warnf("invalid detach escape keys, using default: %s", err) + } else { + escapeKeys = customEscapeKeys + } + } + + h.inputStream = ioutils.NewReadCloserWrapper(term.NewEscapeProxy(h.inputStream, escapeKeys), h.inputStream.Close) + + return restore, nil +} + +func (h *hijackedIOStreamer) beginOutputStream(restoreInput func()) <-chan error { + if h.outputStream == nil && h.errorStream == nil { + // There is no need to copy output. + return nil + } + + outputDone := make(chan error) + go func() { + var err error + + // When TTY is ON, use regular copy + if h.outputStream != nil && h.tty { + _, err = io.Copy(h.outputStream, h.resp.Reader) + // We should restore the terminal as soon as possible + // once the connection ends so any following print + // messages will be in normal type. + restoreInput() + } else { + _, err = stdcopy.StdCopy(h.outputStream, h.errorStream, h.resp.Reader) + } + + logrus.Debug("[hijack] End of stdout") + + if err != nil { + logrus.Debugf("Error receiveStdout: %s", err) + } + + outputDone <- err + }() + + return outputDone +} + +func (h *hijackedIOStreamer) beginInputStream(restoreInput func()) (doneC <-chan struct{}, detachedC <-chan error) { + inputDone := make(chan struct{}) + detached := make(chan error) + + go func() { + if h.inputStream != nil { + _, err := io.Copy(h.resp.Conn, h.inputStream) + // We should restore the terminal as soon as possible + // once the connection ends so any following print + // messages will be in normal type. + restoreInput() + + logrus.Debug("[hijack] End of stdin") + + if _, ok := err.(term.EscapeError); ok { + detached <- err + return + } + + if err != nil { + // This error will also occur on the receive + // side (from stdout) where it will be + // propagated back to the caller. + logrus.Debugf("Error sendStdin: %s", err) + } + } + + if err := h.resp.CloseWrite(); err != nil { + logrus.Debugf("Couldn't send EOF: %s", err) + } + + close(inputDone) + }() + + return inputDone, detached +} + +func setRawTerminal(streams command.Streams) error { + if err := streams.In().SetRawTerminal(); err != nil { + return err + } + return streams.Out().SetRawTerminal() +} + +func restoreTerminal(streams command.Streams, in io.Closer) error { + streams.In().RestoreTerminal() + streams.Out().RestoreTerminal() + // WARNING: DO NOT REMOVE THE OS CHECKS !!! + // For some reason this Close call blocks on darwin.. + // As the client exits right after, simply discard the close + // until we find a better solution. + // + // This can also cause the client on Windows to get stuck in Win32 CloseHandle() + // in some cases. See https://github.com/docker/docker/issues/28267#issuecomment-288237442 + // Tracked internally at Microsoft by VSO #11352156. In the + // Windows case, you hit this if you are using the native/v2 console, + // not the "legacy" console, and you start the client in a new window. eg + // `start docker run --rm -it microsoft/nanoserver cmd /s /c echo foobar` + // will hang. Remove start, and it won't repro. + if in != nil && runtime.GOOS != "darwin" && runtime.GOOS != "windows" { + return in.Close() + } + return nil +} diff --git a/pkg/dev/main.go b/pkg/dev/main.go index d607d468..d4f8f4c2 100644 --- a/pkg/dev/main.go +++ b/pkg/dev/main.go @@ -29,10 +29,12 @@ import ( "github.com/docker/docker/errdefs" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/stdcopy" + "github.com/docker/go-connections/nat" "github.com/google/uuid" specs "github.com/opencontainers/image-spec/specs-go/v1" pkgerr "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "github.com/spf13/pflag" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" @@ -67,22 +69,12 @@ type Options struct { ConnectMode ConnectMode // docker options - Platform string - //Pull string // always, missing, never - PublishAll bool - Entrypoint string - DockerImage string - Publish opts.ListOpts - Expose opts.ListOpts - ExtraHosts opts.ListOpts - NetMode opts.NetworkOpt - Env opts.ListOpts - Mounts opts.MountOpt - Volumes opts.ListOpts - VolumeDriver string + DockerImage string + Options RunOptions + Copts *ContainerOptions } -func (d Options) Main(ctx context.Context) error { +func (d *Options) Main(ctx context.Context, cli *client.Client, dockerCli *command.DockerCli, tempContainerConfig *containerConfig) error { rand.Seed(time.Now().UnixNano()) object, err := util.GetUnstructuredObject(d.Factory, d.Namespace, d.Workload) if err != nil { @@ -133,31 +125,25 @@ func (d Options) Main(ctx context.Context) error { } mesh.RemoveContainers(templateSpec) - list := ConvertKubeResourceToContainer(d.Namespace, *templateSpec, env, volume, dns) - err = fillOptions(list, d) + runConfigList := ConvertKubeResourceToContainer(d.Namespace, *templateSpec, env, volume, dns) + err = mergeDockerOptions(runConfigList, d, tempContainerConfig) if err != nil { return fmt.Errorf("can not fill docker options, err: %v", err) } - var dockerCli *command.DockerCli - var cli *client.Client - cli, dockerCli, err = GetClient() - if err != nil { - return err - } // check resource var outOfMemory bool outOfMemory, _ = checkOutOfMemory(templateSpec, cli) if outOfMemory { return fmt.Errorf("your pod resource request is bigger than docker-desktop resource, please adjust your docker-desktop resource") } - mode := container.NetworkMode(d.NetMode.NetworkMode()) - if len(d.NetMode.Value()) != 0 { - for _, runConfig := range list[:] { + mode := container.NetworkMode(d.Copts.netMode.NetworkMode()) + if len(d.Copts.netMode.Value()) != 0 { + for _, runConfig := range runConfigList[:] { // remove expose port runConfig.config.ExposedPorts = nil runConfig.hostConfig.NetworkMode = mode if mode.IsContainer() { - runConfig.hostConfig.PidMode = containertypes.PidMode(d.NetMode.NetworkMode()) + runConfig.hostConfig.PidMode = containertypes.PidMode(d.Copts.netMode.NetworkMode()) } runConfig.hostConfig.PortBindings = nil @@ -175,15 +161,32 @@ func (d Options) Main(ctx context.Context) error { return err } - list[0].networkingConfig.EndpointsConfig[list[0].containerName] = &network.EndpointSettings{ + runConfigList[len(runConfigList)-1].networkingConfig.EndpointsConfig[runConfigList[len(runConfigList)-1].containerName] = &network.EndpointSettings{ NetworkID: networkID, } - // skip first - for _, runConfig := range list[1:] { + var portmap = nat.PortMap{} + var portset = nat.PortSet{} + for _, runConfig := range runConfigList { + for k, v := range runConfig.hostConfig.PortBindings { + if oldValue, ok := portmap[k]; ok { + portmap[k] = append(oldValue, v...) + } else { + portmap[k] = v + } + } + for k, v := range runConfig.config.ExposedPorts { + portset[k] = v + } + } + runConfigList[len(runConfigList)-1].hostConfig.PortBindings = portmap + runConfigList[len(runConfigList)-1].config.ExposedPorts = portset + + // skip last, use last container network + for _, runConfig := range runConfigList[:len(runConfigList)-1] { // remove expose port runConfig.config.ExposedPorts = nil - runConfig.hostConfig.NetworkMode = containertypes.NetworkMode("container:" + list[0].containerName) - runConfig.hostConfig.PidMode = containertypes.PidMode("container:" + list[0].containerName) + runConfig.hostConfig.NetworkMode = containertypes.NetworkMode("container:" + runConfigList[len(runConfigList)-1].containerName) + runConfig.hostConfig.PidMode = containertypes.PidMode("container:" + runConfigList[len(runConfigList)-1].containerName) runConfig.hostConfig.PortBindings = nil // remove dns @@ -196,25 +199,20 @@ func (d Options) Main(ctx context.Context) error { } handler.RollbackFuncList = append(handler.RollbackFuncList, func() { - _ = list.Remove(ctx) + _ = runConfigList.Remove(ctx, cli) }) - err = list.Run(ctx, volume) + err = runConfigList.Run(ctx, volume, cli, dockerCli) if err != nil { return err } - return terminal(list[0].containerName, dockerCli) + return terminal(runConfigList[0].containerName, dockerCli) } -type Run []*RunConfig +type ConfigList []*RunConfig -func (r Run) Remove(ctx context.Context) error { - cli, _, err := GetClient() - if err != nil { - return err - } - - for _, runConfig := range r { - err = cli.NetworkDisconnect(ctx, runConfig.containerName, runConfig.containerName, true) +func (l ConfigList) Remove(ctx context.Context, cli *client.Client) error { + for _, runConfig := range l { + err := cli.NetworkDisconnect(ctx, runConfig.containerName, runConfig.containerName, true) if err != nil { log.Debug(err) } @@ -223,8 +221,7 @@ func (r Run) Remove(ctx context.Context) error { log.Debug(err) } } - var i types.NetworkResource - i, err = cli.NetworkInspect(ctx, config.ConfigMapPodTrafficManager, types.NetworkInspectOptions{}) + i, err := cli.NetworkInspect(ctx, config.ConfigMapPodTrafficManager, types.NetworkInspectOptions{}) if err != nil { return err } @@ -254,32 +251,35 @@ func GetClient() (*client.Client, *command.DockerCli, error) { return cli, dockerCli, nil } -func (r Run) Run(ctx context.Context, volume map[string][]mount.Mount) error { - cli, c, err := GetClient() - if err != nil { - return err - } - for _, runConfig := range r { - var id string - id, err = run(ctx, runConfig, cli, c) - if err != nil { - // try another way to startup container - log.Infof("occur err: %v, try another way to startup container...", err) - runConfig.hostConfig.Mounts = nil - id, err = run(ctx, runConfig, cli, c) +func (l ConfigList) Run(ctx context.Context, volume map[string][]mount.Mount, cli *client.Client, dockerCli *command.DockerCli) error { + for index := len(l) - 1; index >= 0; index-- { + runConfig := l[index] + if index == 0 { + _, err := runFirst(ctx, runConfig, cli, dockerCli) if err != nil { return err } - err = r.copyToContainer(ctx, volume[runConfig.k8sContainerName], cli, id) + } else { + id, err := run(ctx, runConfig, cli, dockerCli) if err != nil { - return err + // try another way to startup container + log.Infof("occur err: %v, try another way to startup container...", err) + runConfig.hostConfig.Mounts = nil + id, err = run(ctx, runConfig, cli, dockerCli) + if err != nil { + return err + } + err = l.copyToContainer(ctx, volume[runConfig.k8sContainerName], cli, id) + if err != nil { + return err + } } } } return nil } -func (r Run) copyToContainer(ctx context.Context, volume []mount.Mount, cli *client.Client, id string) error { +func (l ConfigList) copyToContainer(ctx context.Context, volume []mount.Mount, cli *client.Client, id string) error { // copy volume into container for _, v := range volume { target, err := createFolder(ctx, cli, id, v.Source, v.Target) @@ -367,22 +367,22 @@ func checkOutOfMemory(spec *v1.PodTemplateSpec, cli *client.Client) (outOfMemory return } -func DoDev(devOptions Options, args []string, f cmdutil.Factory) error { +func DoDev(devOptions *Options, flags *pflag.FlagSet, f cmdutil.Factory) error { connect := handler.ConnectOptions{ Headers: devOptions.Headers, - Workloads: args, + Workloads: []string{devOptions.Workload}, ExtraCIDR: devOptions.ExtraCIDR, ExtraDomain: devOptions.ExtraDomain, } + cli, dockerCli, err := GetClient() + if err != nil { + return err + } - mode := container.NetworkMode(devOptions.NetMode.NetworkMode()) + mode := container.NetworkMode(devOptions.Copts.netMode.NetworkMode()) if mode.IsContainer() { - client, _, err := GetClient() - if err != nil { - return err - } var inspect types.ContainerJSON - inspect, err = client.ContainerInspect(context.Background(), mode.ConnectedContainer()) + inspect, err = cli.ContainerInspect(context.Background(), mode.ConnectedContainer()) if err != nil { return err } @@ -397,8 +397,7 @@ func DoDev(devOptions Options, args []string, f cmdutil.Factory) error { if err := connect.InitClient(f); err != nil { return err } - err := connect.PreCheckResource() - if err != nil { + if err = connect.PreCheckResource(); err != nil { return err } @@ -410,8 +409,8 @@ func DoDev(devOptions Options, args []string, f cmdutil.Factory) error { } var platform *specs.Platform - if devOptions.Platform != "" { - p, err := platforms.Parse(devOptions.Platform) + if devOptions.Options.Platform != "" { + p, err := platforms.Parse(devOptions.Options.Platform) if err != nil { return pkgerr.Wrap(err, "error parsing specified platform") } @@ -441,117 +440,15 @@ func DoDev(devOptions Options, args []string, f cmdutil.Factory) error { return err } case ConnectModeContainer: - var dockerCli *command.DockerCli - var cli *client.Client - cli, dockerCli, err = GetClient() + var connectContainer *RunConfig + connectContainer, err = createConnectContainer(*devOptions, connect, path, err, cli, platform) if err != nil { return err } - - var entrypoint []string - if devOptions.NoProxy { - entrypoint = []string{"kubevpn", "connect", "-n", connect.Namespace, "--kubeconfig", "/root/.kube/config", "--image", config.Image} - for _, v := range connect.ExtraCIDR { - entrypoint = append(entrypoint, "--extra-cidr", v) - } - for _, v := range connect.ExtraDomain { - entrypoint = append(entrypoint, "--extra-domain", v) - } - } else { - entrypoint = []string{"kubevpn", "proxy", connect.Workloads[0], "-n", connect.Namespace, "--kubeconfig", "/root/.kube/config", "--image", config.Image} - for k, v := range connect.Headers { - entrypoint = append(entrypoint, "--headers", fmt.Sprintf("%s=%s", k, v)) - } - for _, v := range connect.ExtraCIDR { - entrypoint = append(entrypoint, "--extra-cidr", v) - } - for _, v := range connect.ExtraDomain { - entrypoint = append(entrypoint, "--extra-domain", v) - } - } - - runConfig := &container.Config{ - User: "root", - AttachStdin: false, - AttachStdout: false, - AttachStderr: false, - ExposedPorts: nil, - StdinOnce: false, - Env: []string{fmt.Sprintf("%s=1", config.EnvStartSudoKubeVPNByKubeVPN)}, - Cmd: []string{}, - Healthcheck: nil, - ArgsEscaped: false, - Image: config.Image, - Volumes: nil, - Entrypoint: entrypoint, - NetworkDisabled: false, - MacAddress: "", - OnBuild: nil, - StopSignal: "", - StopTimeout: nil, - Shell: nil, - } - hostConfig := &container.HostConfig{ - Binds: []string{fmt.Sprintf("%s:%s", path, "/root/.kube/config")}, - LogConfig: container.LogConfig{}, - PortBindings: nil, - RestartPolicy: container.RestartPolicy{}, - AutoRemove: true, - VolumeDriver: "", - VolumesFrom: nil, - ConsoleSize: [2]uint{}, - CapAdd: strslice.StrSlice{"SYS_PTRACE", "SYS_ADMIN"}, // for dlv - CgroupnsMode: "", - ExtraHosts: nil, - GroupAdd: nil, - IpcMode: "", - Cgroup: "", - Links: nil, - OomScoreAdj: 0, - PidMode: "", - Privileged: true, - PublishAllPorts: false, - ReadonlyRootfs: false, - SecurityOpt: []string{"apparmor=unconfined", "seccomp=unconfined"}, - StorageOpt: nil, - Tmpfs: nil, - UTSMode: "", - UsernsMode: "", - ShmSize: 0, - Sysctls: nil, - Runtime: "", - Isolation: "", - Resources: container.Resources{}, - MaskedPaths: nil, - ReadonlyPaths: nil, - Init: nil, - } - var suffix string - if newUUID, err := uuid.NewUUID(); err == nil { - suffix = strings.ReplaceAll(newUUID.String(), "-", "")[:5] - } - var kubevpnNetwork string - kubevpnNetwork, err = createKubevpnNetwork(context.Background(), cli) - if err != nil { - return err - } - name := fmt.Sprintf("%s_%s_%s", "kubevpn", "local", suffix) - c := &RunConfig{ - config: runConfig, - hostConfig: hostConfig, - networkingConfig: &network.NetworkingConfig{ - EndpointsConfig: map[string]*network.EndpointSettings{name: { - NetworkID: kubevpnNetwork, - }}, - }, - platform: platform, - containerName: name, - k8sContainerName: name, - } ctx, cancel := context.WithCancel(context.Background()) defer cancel() var id string - if id, err = run(ctx, c, cli, dockerCli); err != nil { + if id, err = run(ctx, connectContainer, cli, dockerCli); err != nil { return err } h := interrupt.New(func(signal os.Signal) { @@ -570,21 +467,150 @@ func DoDev(devOptions Options, args []string, f cmdutil.Factory) error { } return err } - if err = devOptions.NetMode.Set("container:" + id); err != nil { + if err = devOptions.Copts.netMode.Set("container:" + id); err != nil { return err } default: return fmt.Errorf("unsupport connect mode: %s", devOptions.ConnectMode) } + var tempContainerConfig *containerConfig + { + if err := validatePullOpt(devOptions.Options.Pull); err != nil { + return err + } + proxyConfig := dockerCli.ConfigFile().ParseProxyConfig(dockerCli.Client().DaemonHost(), opts.ConvertKVStringsToMapWithNil(devOptions.Copts.env.GetAll())) + newEnv := []string{} + for k, v := range proxyConfig { + if v == nil { + newEnv = append(newEnv, k) + } else { + newEnv = append(newEnv, fmt.Sprintf("%s=%s", k, *v)) + } + } + devOptions.Copts.env = *opts.NewListOptsRef(&newEnv, nil) + tempContainerConfig, err = parse(flags, devOptions.Copts, dockerCli.ServerInfo().OSType) + // just in case the parse does not exit + if err != nil { + return err + } + if err = validateAPIVersion(tempContainerConfig, dockerCli.Client().ClientVersion()); err != nil { + return err + } + } + devOptions.Namespace = connect.Namespace - err = devOptions.Main(context.Background()) + err = devOptions.Main(context.Background(), cli, dockerCli, tempContainerConfig) if err != nil { log.Errorln(err) } return err } +func createConnectContainer(devOptions Options, connect handler.ConnectOptions, path string, err error, cli *client.Client, platform *specs.Platform) (*RunConfig, error) { + var entrypoint []string + if devOptions.NoProxy { + entrypoint = []string{"kubevpn", "connect", "-n", connect.Namespace, "--kubeconfig", "/root/.kube/config", "--image", config.Image} + for _, v := range connect.ExtraCIDR { + entrypoint = append(entrypoint, "--extra-cidr", v) + } + for _, v := range connect.ExtraDomain { + entrypoint = append(entrypoint, "--extra-domain", v) + } + } else { + entrypoint = []string{"kubevpn", "proxy", connect.Workloads[0], "-n", connect.Namespace, "--kubeconfig", "/root/.kube/config", "--image", config.Image} + for k, v := range connect.Headers { + entrypoint = append(entrypoint, "--headers", fmt.Sprintf("%s=%s", k, v)) + } + for _, v := range connect.ExtraCIDR { + entrypoint = append(entrypoint, "--extra-cidr", v) + } + for _, v := range connect.ExtraDomain { + entrypoint = append(entrypoint, "--extra-domain", v) + } + } + + runConfig := &container.Config{ + User: "root", + AttachStdin: false, + AttachStdout: false, + AttachStderr: false, + ExposedPorts: nil, + StdinOnce: false, + Env: []string{fmt.Sprintf("%s=1", config.EnvStartSudoKubeVPNByKubeVPN)}, + Cmd: []string{}, + Healthcheck: nil, + ArgsEscaped: false, + Image: config.Image, + Volumes: nil, + Entrypoint: entrypoint, + NetworkDisabled: false, + MacAddress: "", + OnBuild: nil, + StopSignal: "", + StopTimeout: nil, + Shell: nil, + } + hostConfig := &container.HostConfig{ + Binds: []string{fmt.Sprintf("%s:%s", path, "/root/.kube/config")}, + LogConfig: container.LogConfig{}, + PortBindings: nil, + RestartPolicy: container.RestartPolicy{}, + AutoRemove: true, + VolumeDriver: "", + VolumesFrom: nil, + ConsoleSize: [2]uint{}, + CapAdd: strslice.StrSlice{"SYS_PTRACE", "SYS_ADMIN"}, // for dlv + CgroupnsMode: "", + ExtraHosts: nil, + GroupAdd: nil, + IpcMode: "", + Cgroup: "", + Links: nil, + OomScoreAdj: 0, + PidMode: "", + Privileged: true, + PublishAllPorts: false, + ReadonlyRootfs: false, + SecurityOpt: []string{"apparmor=unconfined", "seccomp=unconfined"}, + StorageOpt: nil, + Tmpfs: nil, + UTSMode: "", + UsernsMode: "", + ShmSize: 0, + Sysctls: nil, + Runtime: "", + Isolation: "", + Resources: container.Resources{}, + MaskedPaths: nil, + ReadonlyPaths: nil, + Init: nil, + } + var suffix string + if newUUID, err := uuid.NewUUID(); err == nil { + suffix = strings.ReplaceAll(newUUID.String(), "-", "")[:5] + } + var kubevpnNetwork string + kubevpnNetwork, err = createKubevpnNetwork(context.Background(), cli) + if err != nil { + return nil, err + } + name := fmt.Sprintf("%s_%s_%s", "kubevpn", "local", suffix) + c := &RunConfig{ + config: runConfig, + hostConfig: hostConfig, + networkingConfig: &network.NetworkingConfig{ + EndpointsConfig: map[string]*network.EndpointSettings{name: { + NetworkID: kubevpnNetwork, + }}, + }, + platform: platform, + containerName: name, + k8sContainerName: name, + } + return c, nil +} + func runLogsWaitRunning(ctx context.Context, dockerCli command.Cli, container string) error { c, err := dockerCli.Client().ContainerInspect(ctx, container) if err != nil { @@ -685,37 +711,6 @@ func runKill(dockerCli command.Cli, containers ...string) error { } return nil } -func parallelOperation(ctx context.Context, containers []string, op func(ctx context.Context, container string) error) chan error { - if len(containers) == 0 { - return nil - } - const defaultParallel int = 50 - sem := make(chan struct{}, defaultParallel) - errChan := make(chan error) - - // make sure result is printed in correct order - output := map[string]chan error{} - for _, c := range containers { - output[c] = make(chan error, 1) - } - go func() { - for _, c := range containers { - err := <-output[c] - errChan <- err - } - }() - - go func() { - for _, c := range containers { - sem <- struct{}{} // Wait for active queue sem to drain. - go func(container string) { - output[container] <- op(ctx, container) - <-sem - }(c) - } - }() - return errChan -} func createKubevpnNetwork(ctx context.Context, cli *client.Client) (string, error) { by := map[string]string{"owner": config.ConfigMapPodTrafficManager} diff --git a/pkg/dev/option.go b/pkg/dev/option.go index 87ae094c..fc4655a2 100644 --- a/pkg/dev/option.go +++ b/pkg/dev/option.go @@ -1,26 +1,13 @@ package dev import ( - "fmt" - "path/filepath" - "reflect" - "strconv" - "strings" - - "github.com/docker/cli/cli/compose/loader" - "github.com/docker/cli/opts" - "github.com/docker/docker/api/types/container" - mounttypes "github.com/docker/docker/api/types/mount" + "github.com/containerd/containerd/platforms" "github.com/docker/docker/api/types/network" - "github.com/docker/docker/api/types/strslice" - "github.com/docker/docker/errdefs" - "github.com/docker/go-connections/nat" - v12 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" - "github.com/sirupsen/logrus" + "github.com/wencaiwulue/kubevpn/pkg/util" ) -func fillOptions(r Run, copts Options) error { +func mergeDockerOptions(r ConfigList, copts *Options, tempContainerConfig *containerConfig) error { if copts.ContainerName != "" { var index = -1 for i, config := range r { @@ -35,226 +22,49 @@ func fillOptions(r Run, copts Options) error { } config := r[0] - config.hostConfig.PublishAllPorts = copts.PublishAll + config.Options = copts.Options + config.Copts = copts.Copts if copts.DockerImage != "" { config.config.Image = copts.DockerImage } - - if copts.Entrypoint != "" { - if strings.Count(copts.Entrypoint, " ") != 0 { - split := strings.Split(copts.Entrypoint, " ") - config.config.Entrypoint = split - } else { - config.config.Entrypoint = strslice.StrSlice{copts.Entrypoint} - } - config.config.Cmd = []string{} + if copts.Options.Name != "" { + config.containerName = copts.Options.Name + } else { + config.Options.Name = config.containerName } - if copts.Platform != "" { - split := strings.Split(copts.Platform, "/") - if len(split) != 2 { - return errors.Errorf("invalid port format for --platform: %s", copts.Platform) - } - config.platform = &v12.Platform{ - OS: split[0], - Architecture: split[1], - } - } - - // collect all the environment variables for the container - envVariables, err := opts.ReadKVEnvStrings([]string{}, copts.Env.GetAll()) - if err != nil { - return err - } - config.config.Env = append(config.config.Env, envVariables...) - - publishOpts := copts.Publish.GetAll() - var ( - ports map[nat.Port]struct{} - portBindings map[nat.Port][]nat.PortBinding - convertedOpts []string - ) - - convertedOpts, err = convertToStandardNotation(publishOpts) - if err != nil { - return err - } - - ports, portBindings, err = nat.ParsePortSpecs(convertedOpts) - if err != nil { - return err - } - - // Merge in exposed ports to the map of published ports - for _, e := range copts.Expose.GetAll() { - if strings.Contains(e, ":") { - return errors.Errorf("invalid port format for --expose: %s", e) - } - // support two formats for expose, original format /[] - // or /[] - proto, port := nat.SplitProtoPort(e) - // parse the start and end port and create a sequence of ports to expose - // if expose a port, the start and end port are the same - start, end, err := nat.ParsePortRange(port) + if copts.Options.Platform != "" { + p, err := platforms.Parse(copts.Options.Platform) if err != nil { - return errors.Errorf("invalid range format for --expose: %s, error: %s", e, err) + return errors.Wrap(err, "error parsing specified platform") } - for i := start; i <= end; i++ { - p, err := nat.NewPort(proto, strconv.FormatUint(i, 10)) - if err != nil { - return err - } - if _, exists := ports[p]; !exists { - ports[p] = struct{}{} - } - } - } - for port, bindings := range portBindings { - config.hostConfig.PortBindings[port] = bindings - } - for port, s := range ports { - config.config.ExposedPorts[port] = s + config.platform = &p } - mounts := copts.Mounts.Value() - if len(mounts) > 0 && copts.VolumeDriver != "" { - logrus.Warn("`--volume-driver` is ignored for volumes specified via `--mount`. Use `--mount type=volume,volume-driver=...` instead.") + config.hostConfig = tempContainerConfig.HostConfig + config.networkingConfig.EndpointsConfig = util.Merge[string, *network.EndpointSettings](tempContainerConfig.NetworkingConfig.EndpointsConfig, config.networkingConfig.EndpointsConfig) + + c := tempContainerConfig.Config + var entrypoint = config.config.Entrypoint + var args = config.config.Cmd + // if special --entrypoint, then use it + if len(c.Entrypoint) != 0 { + entrypoint = c.Entrypoint + args = c.Cmd } - var binds []string - volumes := copts.Volumes.GetMap() - // add any bind targets to the list of container volumes - for bind := range copts.Volumes.GetMap() { - parsed, _ := loader.ParseVolume(bind) - - if parsed.Source != "" { - toBind := bind - - if parsed.Type == string(mounttypes.TypeBind) { - if arr := strings.SplitN(bind, ":", 2); len(arr) == 2 { - hostPart := arr[0] - if strings.HasPrefix(hostPart, "."+string(filepath.Separator)) || hostPart == "." { - if absHostPart, err := filepath.Abs(hostPart); err == nil { - hostPart = absHostPart - } - } - toBind = hostPart + ":" + arr[1] - } - } - - // after creating the bind mount we want to delete it from the copts.volumes values because - // we do not want bind mounts being committed to image configs - binds = append(binds, toBind) - // We should delete from the map (`volumes`) here, as deleting from copts.volumes will not work if - // there are duplicates entries. - delete(volumes, bind) - } + if len(c.Cmd) != 0 { + args = c.Cmd } - - config.hostConfig.Binds = binds - networkOpts, err := parseNetworkOpts(copts) - if err != nil { - return err + c.Entrypoint = entrypoint + c.Cmd = args + c.Env = append(config.config.Env, c.Env...) + c.Image = config.config.Image + if c.User == "" { + c.User = config.config.User } - config.networkingConfig = &network.NetworkingConfig{EndpointsConfig: networkOpts} + c.Volumes = util.Merge[string, struct{}](c.Volumes, config.config.Volumes) + + config.config = c return nil } - -func convertToStandardNotation(ports []string) ([]string, error) { - optsList := []string{} - for _, publish := range ports { - if strings.Contains(publish, "=") { - params := map[string]string{"protocol": "tcp"} - for _, param := range strings.Split(publish, ",") { - opt := strings.Split(param, "=") - if len(opt) < 2 { - return optsList, errors.Errorf("invalid publish opts format (should be name=value but got '%s')", param) - } - - params[opt[0]] = opt[1] - } - optsList = append(optsList, fmt.Sprintf("%s:%s/%s", params["published"], params["target"], params["protocol"])) - } else { - optsList = append(optsList, publish) - } - } - return optsList, nil -} - -// parseNetworkOpts converts --network advanced options to endpoint-specs, and combines -// them with the old --network-alias and --links. If returns an error if conflicting options -// are found. -// -// this function may return _multiple_ endpoints, which is not currently supported -// by the daemon, but may be in future; it's up to the daemon to produce an error -// in case that is not supported. -func parseNetworkOpts(copts Options) (map[string]*network.EndpointSettings, error) { - var ( - endpoints = make(map[string]*network.EndpointSettings, len(copts.NetMode.Value())) - hasUserDefined, hasNonUserDefined bool - ) - - for i, n := range copts.NetMode.Value() { - n := n - if container.NetworkMode(n.Target).IsUserDefined() { - hasUserDefined = true - } else { - hasNonUserDefined = true - } - ep, err := parseNetworkAttachmentOpt(n) - if err != nil { - return nil, err - } - if _, ok := endpoints[n.Target]; ok { - return nil, errdefs.InvalidParameter(errors.Errorf("network %q is specified multiple times", n.Target)) - } - - // For backward compatibility: if no custom options are provided for the network, - // and only a single network is specified, omit the endpoint-configuration - // on the client (the daemon will still create it when creating the container) - if i == 0 && len(copts.NetMode.Value()) == 1 { - if ep == nil || reflect.DeepEqual(*ep, network.EndpointSettings{}) { - continue - } - } - endpoints[n.Target] = ep - } - if hasUserDefined && hasNonUserDefined { - return nil, errdefs.InvalidParameter(errors.New("conflicting options: cannot attach both user-defined and non-user-defined network-modes")) - } - return endpoints, nil -} - -func parseNetworkAttachmentOpt(ep opts.NetworkAttachmentOpts) (*network.EndpointSettings, error) { - if strings.TrimSpace(ep.Target) == "" { - return nil, errors.New("no name set for network") - } - if !container.NetworkMode(ep.Target).IsUserDefined() { - if len(ep.Aliases) > 0 { - return nil, errors.New("network-scoped aliases are only supported for user-defined networks") - } - if len(ep.Links) > 0 { - return nil, errors.New("links are only supported for user-defined networks") - } - } - - epConfig := &network.EndpointSettings{ - NetworkID: ep.Target, - } - epConfig.Aliases = append(epConfig.Aliases, ep.Aliases...) - if len(ep.DriverOpts) > 0 { - epConfig.DriverOpts = make(map[string]string) - epConfig.DriverOpts = ep.DriverOpts - } - if len(ep.Links) > 0 { - epConfig.Links = ep.Links - } - if ep.IPv4Address != "" || ep.IPv6Address != "" || len(ep.LinkLocalIPs) > 0 { - epConfig.IPAMConfig = &network.EndpointIPAMConfig{ - IPv4Address: ep.IPv4Address, - IPv6Address: ep.IPv6Address, - LinkLocalIPs: ep.LinkLocalIPs, - } - } - return epConfig, nil -} diff --git a/pkg/dev/run.go b/pkg/dev/run.go index 9ee19ef0..11d93992 100644 --- a/pkg/dev/run.go +++ b/pkg/dev/run.go @@ -2,11 +2,13 @@ package dev import ( "context" + "errors" "fmt" "io" "math/rand" "os" "path/filepath" + "strconv" "strings" "sync" "time" @@ -20,10 +22,13 @@ import ( "github.com/docker/docker/api/types" typescommand "github.com/docker/docker/api/types/container" "github.com/docker/docker/client" + apiclient "github.com/docker/docker/client" "github.com/docker/docker/errdefs" "github.com/docker/docker/pkg/jsonmessage" + "github.com/moby/term" dockerterm "github.com/moby/term" v12 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/wait" @@ -90,6 +95,11 @@ func run(ctx context.Context, runConfig *RunConfig, cli *client.Client, c *comma if err != nil { return } + if inspect.State != nil && (inspect.State.Status == "exited" || inspect.State.Status == "dead" || inspect.State.Dead) { + once.Do(func() { close(chanStop) }) + err = errors.New(fmt.Sprintf("container status: %s", inspect.State.Status)) + return + } if inspect.State != nil && inspect.State.Running { once.Do(func() { close(chanStop) }) return @@ -125,6 +135,175 @@ func run(ctx context.Context, runConfig *RunConfig, cli *client.Client, c *comma return } +func runFirst(ctx context.Context, runConfig *RunConfig, cli *apiclient.Client, dockerCli *command.DockerCli) (id string, err error) { + rand.New(rand.NewSource(time.Now().UnixNano())) + + defer func() { + if err != nil { + _ = cli.ContainerRemove(ctx, id, types.ContainerRemoveOptions{Force: true}) + } + }() + + stdout, stderr := dockerCli.Out(), dockerCli.Err() + client := dockerCli.Client() + + runConfig.config.ArgsEscaped = false + + if err := dockerCli.In().CheckTty(runConfig.config.AttachStdin, runConfig.config.Tty); err != nil { + return id, err + } + + if !runConfig.Options.Detach { + if err := dockerCli.In().CheckTty(runConfig.config.AttachStdin, runConfig.config.Tty); err != nil { + return id, err + } + } else { + if runConfig.Copts.attach.Len() != 0 { + return id, errors.New("Conflicting options: -a and -d") + } + + runConfig.config.AttachStdin = false + runConfig.config.AttachStdout = false + runConfig.config.AttachStderr = false + runConfig.config.StdinOnce = false + } + + ctx, cancelFun := context.WithCancel(context.Background()) + defer cancelFun() + + createResponse, err := createContainer(ctx, dockerCli, &containerConfig{ + Config: runConfig.config, + HostConfig: runConfig.hostConfig, + NetworkingConfig: runConfig.networkingConfig, + }, &runConfig.Options.createOptions) + if err != nil { + return "", err + } + log.Infof("Created container: %s", runConfig.containerName) + + var ( + waitDisplayID chan struct{} + errCh chan error + ) + if !runConfig.config.AttachStdout && !runConfig.config.AttachStderr { + // Make this asynchronous to allow the client to write to stdin before having to read the ID + waitDisplayID = make(chan struct{}) + go func() { + defer close(waitDisplayID) + fmt.Fprintln(stdout, createResponse.ID) + }() + } + attach := runConfig.config.AttachStdin || runConfig.config.AttachStdout || runConfig.config.AttachStderr + if attach { + close, err := attachContainer(ctx, dockerCli, &errCh, runConfig.config, createResponse.ID) + + if err != nil { + return id, err + } + defer close() + } + statusChan := waitExitOrRemoved(ctx, dockerCli, createResponse.ID, runConfig.Copts.autoRemove) + // start the container + if err := client.ContainerStart(ctx, createResponse.ID, types.ContainerStartOptions{}); err != nil { + // If we have hijackedIOStreamer, we should notify + // hijackedIOStreamer we are going to exit and wait + // to avoid the terminal are not restored. + if attach { + cancelFun() + <-errCh + } + + reportError(stderr, "run", err.Error(), false) + if runConfig.Copts.autoRemove { + // wait container to be removed + <-statusChan + } + return id, runStartContainerErr(err) + } + + if (runConfig.config.AttachStdin || runConfig.config.AttachStdout || runConfig.config.AttachStderr) && runConfig.config.Tty && dockerCli.Out().IsTerminal() { + if err := container.MonitorTtySize(ctx, dockerCli, createResponse.ID, false); err != nil { + fmt.Fprintln(stderr, "Error monitoring TTY size:", err) + } + } + + if errCh != nil { + if err := <-errCh; err != nil { + if _, ok := err.(term.EscapeError); ok { + // The user entered the detach escape sequence. + return id, nil + } + + logrus.Debugf("Error hijack: %s", err) + return id, err + } + } + + // Detached mode: wait for the id to be displayed and return. + if !runConfig.config.AttachStdout && !runConfig.config.AttachStderr { + // Detached mode + <-waitDisplayID + return id, nil + } + + status := <-statusChan + if status != 0 { + return id, errors.New(strconv.Itoa(status)) + } + log.Infof("Wait container %s to be running...", runConfig.containerName) + chanStop := make(chan struct{}) + var inspect types.ContainerJSON + var once = &sync.Once{} + wait.Until(func() { + inspect, err = cli.ContainerInspect(ctx, createResponse.ID) + if err != nil && errdefs.IsNotFound(err) { + once.Do(func() { close(chanStop) }) + return + } + if err != nil { + return + } + if inspect.State != nil && (inspect.State.Status == "exited" || inspect.State.Status == "dead" || inspect.State.Dead) { + once.Do(func() { close(chanStop) }) + err = errors.New(fmt.Sprintf("container status: %s", inspect.State.Status)) + return + } + if inspect.State != nil && inspect.State.Running { + once.Do(func() { close(chanStop) }) + return + } + }, time.Second, chanStop) + if err != nil { + err = fmt.Errorf("failed to wait container to be ready: %v", err) + return + } + + // print port mapping to host + var empty = true + var str string + if inspect.NetworkSettings != nil && inspect.NetworkSettings.Ports != nil { + var list []string + for port, bindings := range inspect.NetworkSettings.Ports { + var p []string + for _, binding := range bindings { + if binding.HostPort != "" { + p = append(p, binding.HostPort) + empty = false + } + } + list = append(list, fmt.Sprintf("%s:%s", port, strings.Join(p, ","))) + } + str = fmt.Sprintf("Container %s is running on port %s now", runConfig.containerName, strings.Join(list, " ")) + } + if !empty { + log.Infoln(str) + } else { + log.Infof("Container %s is running now", runConfig.containerName) + } + + return +} + func PullImage(ctx context.Context, platform *v12.Platform, cli *client.Client, c *command.DockerCli, img string) error { var readCloser io.ReadCloser var plat string diff --git a/pkg/dev/utils.go b/pkg/dev/utils.go new file mode 100644 index 00000000..51049b18 --- /dev/null +++ b/pkg/dev/utils.go @@ -0,0 +1,162 @@ +package dev + +import ( + "context" + "strconv" + + "github.com/docker/cli/cli/command" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/events" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/api/types/versions" + "github.com/sirupsen/logrus" +) + +func waitExitOrRemoved(ctx context.Context, dockerCli command.Cli, containerID string, waitRemove bool) <-chan int { + if len(containerID) == 0 { + // containerID can never be empty + panic("Internal Error: waitExitOrRemoved needs a containerID as parameter") + } + + // Older versions used the Events API, and even older versions did not + // support server-side removal. This legacyWaitExitOrRemoved method + // preserves that old behavior and any issues it may have. + if versions.LessThan(dockerCli.Client().ClientVersion(), "1.30") { + return legacyWaitExitOrRemoved(ctx, dockerCli, containerID, waitRemove) + } + + condition := container.WaitConditionNextExit + if waitRemove { + condition = container.WaitConditionRemoved + } + + resultC, errC := dockerCli.Client().ContainerWait(ctx, containerID, condition) + + statusC := make(chan int) + go func() { + select { + case result := <-resultC: + if result.Error != nil { + logrus.Errorf("Error waiting for container: %v", result.Error.Message) + statusC <- 125 + } else { + statusC <- int(result.StatusCode) + } + case err := <-errC: + logrus.Errorf("error waiting for container: %v", err) + statusC <- 125 + } + }() + + return statusC +} + +func legacyWaitExitOrRemoved(ctx context.Context, dockerCli command.Cli, containerID string, waitRemove bool) <-chan int { + var removeErr error + statusChan := make(chan int) + exitCode := 125 + + // Get events via Events API + f := filters.NewArgs() + f.Add("type", "container") + f.Add("container", containerID) + options := types.EventsOptions{ + Filters: f, + } + eventCtx, cancel := context.WithCancel(ctx) + eventq, errq := dockerCli.Client().Events(eventCtx, options) + + eventProcessor := func(e events.Message) bool { + stopProcessing := false + switch e.Status { + case "die": + if v, ok := e.Actor.Attributes["exitCode"]; ok { + code, cerr := strconv.Atoi(v) + if cerr != nil { + logrus.Errorf("failed to convert exitcode '%q' to int: %v", v, cerr) + } else { + exitCode = code + } + } + if !waitRemove { + stopProcessing = true + } else { + // If we are talking to an older daemon, `AutoRemove` is not supported. + // We need to fall back to the old behavior, which is client-side removal + if versions.LessThan(dockerCli.Client().ClientVersion(), "1.25") { + go func() { + removeErr = dockerCli.Client().ContainerRemove(ctx, containerID, types.ContainerRemoveOptions{RemoveVolumes: true}) + if removeErr != nil { + logrus.Errorf("error removing container: %v", removeErr) + cancel() // cancel the event Q + } + }() + } + } + case "detach": + exitCode = 0 + stopProcessing = true + case "destroy": + stopProcessing = true + } + return stopProcessing + } + + go func() { + defer func() { + statusChan <- exitCode // must always send an exit code or the caller will block + cancel() + }() + + for { + select { + case <-eventCtx.Done(): + if removeErr != nil { + return + } + case evt := <-eventq: + if eventProcessor(evt) { + return + } + case err := <-errq: + logrus.Errorf("error getting events from daemon: %v", err) + return + } + } + }() + + return statusChan +} + +func parallelOperation(ctx context.Context, containers []string, op func(ctx context.Context, container string) error) chan error { + if len(containers) == 0 { + return nil + } + const defaultParallel int = 50 + sem := make(chan struct{}, defaultParallel) + errChan := make(chan error) + + // make sure result is printed in correct order + output := map[string]chan error{} + for _, c := range containers { + output[c] = make(chan error, 1) + } + go func() { + for _, c := range containers { + err := <-output[c] + errChan <- err + } + }() + + go func() { + for _, c := range containers { + sem <- struct{}{} // Wait for active queue sem to drain. + go func(container string) { + output[container] <- op(ctx, container) + <-sem + }(c) + } + }() + return errChan +} diff --git a/pkg/test/2pod.yaml b/pkg/test/2pod.yaml new file mode 100644 index 00000000..4e9005e1 --- /dev/null +++ b/pkg/test/2pod.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test + labels: + app: test +spec: + terminationGracePeriodSeconds: 0 + containers: + - name: nginx + image: nginx + imagePullPolicy: IfNotPresent + - name: tomcat + image: tomcat + imagePullPolicy: IfNotPresent + restartPolicy: Always diff --git a/pkg/util/util.go b/pkg/util/util.go index 7fc68817..46307d43 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -749,3 +749,15 @@ func MoveToTemp() { log.Debugln(err) } } + +func Merge[K comparable, V any](fromMap, ToMap map[K]V) map[K]V { + for keyToMap, valueToMap := range ToMap { + fromMap[keyToMap] = valueToMap + } + if fromMap == nil { + // merge(nil, map[string]interface{...}) -> map[string]interface{...} + return ToMap + } + + return fromMap +}