apps/kubevpn
1
0
Fork 0
mirror of https://github.com/kubenetworks/kubevpn.git synced 2025-10-06 15:57:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: support dev docker in docker (dind)

Browse Source
This commit is contained in:
fengcaiwen
2023-02-23 17:08:51 +08:00
parent c56e0c0baf
commit c883398f37
8 changed files with 385 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
README.md
View File

@@ -251,9 +251,10 @@ dns service ok
Hello world!%
```
### 本地进入开发模式
### Dev mode in local
将 Kubernetes pod 运行在本地的 Docker 容器中，同时配合 service mesh, 拦截带有制定 header 的流量到本地，或者所有的流量到本地
Run the Kubernetes pod in the local Docker container, and cooperate with the service mesh to intercept the traffic with
the specified header to the local, or all the traffic to the local.
```shell
➜ ~ kubevpn dev deployment/authors -n kube-system --headers a=1 -p 9080:9080 -p 80:80
@@ -323,6 +324,131 @@ de9e2f8ab57d nginx:latest "/docker-entrypoint.…" 5 seconds
➜ ~
```
If you want to specify the image to start the container locally, you can use the parameter `--docker-image`. When the
image does not exist locally, it will be pulled from the corresponding mirror warehouse. If you want to specify startup
parameters, you can use `--entrypoint` parameter, replace it with the command you want to execute, such
as `--entrypoint "tail -f /dev/null"`, for more parameters, see `kubevpn dev --help`.
Notice:
***If you want to start the development mode locally using Docker in Docker (DinD), since the program will read and
write the `/tmp` directory, you need to manually add the parameter `-v /tmp:/tmp` (outer docker) and other thing is you
need to special
parameter `--parent-container` (inner docker) for sharing network and pid ***
Example:
```shell
docker run -it --privileged -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp -v /Users/naison/.kube/config:/root/.kube/config naison/kubevpn:v1.1.21
```
```shell
➜ ~ docker run -it --privileged -c authors -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp -v /Users/naison/.kube/vke:/root/.kube/config -v /Users/naison/Desktop/kubevpn/bin:/app naison/kubevpn:v1.1.21
root@4d0c3c4eae2b:/# hostname
4d0c3c4eae2b
root@4d0c3c4eae2b:/# kubevpn dev deployment/authors -n kube-system --image naison/kubevpn:v1.1.21 --headers user=naison --parent-container 4d0c3c4eae2b --entrypoint "tail -f /dev/null"
----------------------------------------------------------------------------------
Warn: Use sudo to execute command kubevpn can not use user env KUBECONFIG.
Because of sudo user env and user env are different.
Current env KUBECONFIG value:
----------------------------------------------------------------------------------
got cidr from cache
traffic manager not exist, try to create it...
pod [kubevpn-traffic-manager] status is Pending
Container Reason Message
pod [kubevpn-traffic-manager] status is Pending
Container Reason Message
control-plane ContainerCreating
vpn ContainerCreating
webhook ContainerCreating
pod [kubevpn-traffic-manager] status is Running
Container Reason Message
control-plane ContainerRunning
vpn ContainerRunning
webhook ContainerRunning
update ref count successfully
Waiting for deployment "authors" rollout to finish: 1 old replicas are pending termination...
Waiting for deployment "authors" rollout to finish: 1 old replicas are pending termination...
deployment "authors" successfully rolled out
port forward ready
tunnel connected
dns service ok
tar: removing leading '/' from member names
/tmp/3122262358661539581:/var/run/secrets/kubernetes.io/serviceaccount
tar: Removing leading `/' from member names
tar: Removing leading `/' from hard link targets
/tmp/7677066538742627822:/var/run/secrets/kubernetes.io/serviceaccount
latest: Pulling from naison/authors
Digest: sha256:2e7b2d6a4c6143cde888fcdb70ba091d533e11de70e13e151adff7510a5d52d4
Status: Downloaded newer image for naison/authors:latest
Created container: authors_kube-system_kubevpn_c68e4
Wait container authors_kube-system_kubevpn_c68e4 to be running...
Container authors_kube-system_kubevpn_c68e4 is running now
Created container: nginx_kube-system_kubevpn_c68e7
Wait container nginx_kube-system_kubevpn_c68e7 to be running...
Container nginx_kube-system_kubevpn_c68e7 is running now
/opt/microservices # ps -ef
PID USER TIME COMMAND
1 root 0:00 {bash} /usr/bin/qemu-x86_64 /bin/bash /bin/bash
60 root 0:07 {kubevpn} /usr/bin/qemu-x86_64 kubevpn kubevpn dev deployment/authors -n kube-system --image naison/kubevpn:v1.1.21 --headers user=naison --parent
73 root 0:00 {tail} /usr/bin/qemu-x86_64 /usr/bin/tail tail -f /dev/null
80 root 0:00 {nginx} /usr/bin/qemu-x86_64 /usr/sbin/nginx nginx -g daemon off;
92 root 0:00 {sh} /usr/bin/qemu-x86_64 /bin/sh /bin/sh
156 101 0:00 {nginx} /usr/bin/qemu-x86_64 /usr/sbin/nginx nginx -g daemon off;
158 101 0:00 {nginx} /usr/bin/qemu-x86_64 /usr/sbin/nginx nginx -g daemon off;
160 101 0:00 {nginx} /usr/bin/qemu-x86_64 /usr/sbin/nginx nginx -g daemon off;
162 101 0:00 {nginx} /usr/bin/qemu-x86_64 /usr/sbin/nginx nginx -g daemon off;
164 root 0:00 ps -ef
/opt/microservices # ls
app
/opt/microservices # apk add curl
fetch https://dl-cdn.alpinelinux.org/alpine/v3.14/main/x86_64/APKINDEX.tar.gz
fetch https://dl-cdn.alpinelinux.org/alpine/v3.14/community/x86_64/APKINDEX.tar.gz
(1/4) Installing brotli-libs (1.0.9-r5)
(2/4) Installing nghttp2-libs (1.43.0-r0)
(3/4) Installing libcurl (7.79.1-r5)
(4/4) Installing curl (7.79.1-r5)
Executing busybox-1.33.1-r3.trigger
OK: 8 MiB in 19 packages
/opt/microservices # curl localhost:80
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
html { color-scheme: light dark; }
body { width: 35em; margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif; }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
/opt/microservices # ls
app
/opt/microservices # exit
prepare to exit, cleaning up
update ref count successfully
ref-count is zero, prepare to clean up resource
clean up successful
root@4d0c3c4eae2b:/# exit
exit
```
### Multiple Protocol
- TCP

123
README_ZH.md
View File

@@ -251,7 +251,7 @@ Hello world!%
### 本地进入开发模式
将 Kubernetes pod 运行在本地的 Docker 容器中，同时配合 service mesh, 拦截带有制定 header 的流量到本地，或者所有的流量到本地
将 Kubernetes pod 运行在本地的 Docker 容器中，同时配合 service mesh, 拦截带有制定 header 的流量到本地，或者所有的流量到本地。这个开发模式依赖于本地 Docker .
```shell
➜ ~ kubevpn dev deployment/authors -n kube-system --headers a=1 -p 9080:9080 -p 80:80
@@ -319,6 +319,127 @@ de9e2f8ab57d nginx:latest "/docker-entrypoint.…" 5 seconds
➜ ~
```
如果你想指定在本地启动容器的镜像, 可以使用参数 `--docker-image`, 当本地不存在该镜像时, 会从对应的镜像仓库拉取。如果你想指定启动参数，可以使用 `--entrypoint`
参数，替换为你想要执行的命令，比如 `--entrypoint "tail -f /dev/null"`, 更多使用参数，请参见 `kubevpn dev --help`.
注意：
***如果你想在本地使用 Docker in Docker (DinD) 的方式启动开发模式, 由于程序会读写 `/tmp` 目录，您需要手动添加参数 `-v /tmp:/tmp`, 还有一点需要注意, 如果使用 DinD
模式，为了共享容器网络和 pid, 还需要指定参数 `--parent-container` ***
例如:
```shell
docker run -it --privileged -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp -v /Users/naison/.kube/config:/root/.kube/config naison/kubevpn:v1.1.21
```
```shell
➜ ~ docker run -it --privileged -c authors -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp -v /Users/naison/.kube/config:/root/.kube/config naison/kubevpn:v1.1.21
root@4d0c3c4eae2b:/# hostname
4d0c3c4eae2b
root@4d0c3c4eae2b:/# kubevpn dev deployment/authors -n kube-system --image naison/kubevpn:v1.1.21 --headers user=naison --parent-container 4d0c3c4eae2b --entrypoint "tail -f /dev/null"
----------------------------------------------------------------------------------
Warn: Use sudo to execute command kubevpn can not use user env KUBECONFIG.
Because of sudo user env and user env are different.
Current env KUBECONFIG value:
----------------------------------------------------------------------------------
got cidr from cache
traffic manager not exist, try to create it...
pod [kubevpn-traffic-manager] status is Pending
Container Reason Message
pod [kubevpn-traffic-manager] status is Pending
Container Reason Message
control-plane ContainerCreating
vpn ContainerCreating
webhook ContainerCreating
pod [kubevpn-traffic-manager] status is Running
Container Reason Message
control-plane ContainerRunning
vpn ContainerRunning
webhook ContainerRunning
update ref count successfully
Waiting for deployment "authors" rollout to finish: 1 old replicas are pending termination...
Waiting for deployment "authors" rollout to finish: 1 old replicas are pending termination...
deployment "authors" successfully rolled out
port forward ready
tunnel connected
dns service ok
tar: removing leading '/' from member names
/tmp/3122262358661539581:/var/run/secrets/kubernetes.io/serviceaccount
tar: Removing leading `/' from member names
tar: Removing leading `/' from hard link targets
/tmp/7677066538742627822:/var/run/secrets/kubernetes.io/serviceaccount
latest: Pulling from naison/authors
Digest: sha256:2e7b2d6a4c6143cde888fcdb70ba091d533e11de70e13e151adff7510a5d52d4
Status: Downloaded newer image for naison/authors:latest
Created container: authors_kube-system_kubevpn_c68e4
Wait container authors_kube-system_kubevpn_c68e4 to be running...
Container authors_kube-system_kubevpn_c68e4 is running now
Created container: nginx_kube-system_kubevpn_c68e7
Wait container nginx_kube-system_kubevpn_c68e7 to be running...
Container nginx_kube-system_kubevpn_c68e7 is running now
/opt/microservices # ps -ef
PID USER TIME COMMAND
1 root 0:00 {bash} /usr/bin/qemu-x86_64 /bin/bash /bin/bash
60 root 0:07 {kubevpn} /usr/bin/qemu-x86_64 kubevpn kubevpn dev deployment/authors -n kube-system --image naison/kubevpn:v1.1.21 --headers user=naison --parent
73 root 0:00 {tail} /usr/bin/qemu-x86_64 /usr/bin/tail tail -f /dev/null
80 root 0:00 {nginx} /usr/bin/qemu-x86_64 /usr/sbin/nginx nginx -g daemon off;
92 root 0:00 {sh} /usr/bin/qemu-x86_64 /bin/sh /bin/sh
156 101 0:00 {nginx} /usr/bin/qemu-x86_64 /usr/sbin/nginx nginx -g daemon off;
158 101 0:00 {nginx} /usr/bin/qemu-x86_64 /usr/sbin/nginx nginx -g daemon off;
160 101 0:00 {nginx} /usr/bin/qemu-x86_64 /usr/sbin/nginx nginx -g daemon off;
162 101 0:00 {nginx} /usr/bin/qemu-x86_64 /usr/sbin/nginx nginx -g daemon off;
164 root 0:00 ps -ef
/opt/microservices # ls
app
/opt/microservices # apk add curl
fetch https://dl-cdn.alpinelinux.org/alpine/v3.14/main/x86_64/APKINDEX.tar.gz
fetch https://dl-cdn.alpinelinux.org/alpine/v3.14/community/x86_64/APKINDEX.tar.gz
(1/4) Installing brotli-libs (1.0.9-r5)
(2/4) Installing nghttp2-libs (1.43.0-r0)
(3/4) Installing libcurl (7.79.1-r5)
(4/4) Installing curl (7.79.1-r5)
Executing busybox-1.33.1-r3.trigger
OK: 8 MiB in 19 packages
/opt/microservices # curl localhost:80
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
html { color-scheme: light dark; }
body { width: 35em; margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif; }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
/opt/microservices # ls
app
/opt/microservices # exit
prepare to exit, cleaning up
update ref count successfully
ref-count is zero, prepare to clean up resource
clean up successful
root@4d0c3c4eae2b:/# exit
exit
```
### 支持多种协议
- TCP

21
cmd/kubevpn/cmds/dev.go
View File

@@ -2,6 +2,7 @@ package cmds
import (
"context"
"fmt"
"net/http"
"os"
"path/filepath"
@@ -9,6 +10,7 @@ import (
"github.com/docker/cli/cli"
"github.com/docker/cli/opts"
"github.com/docker/docker/api/types"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"k8s.io/client-go/util/retry"
@@ -68,6 +70,24 @@ func CmdDev(f cmdutil.Factory) *cobra.Command {
Workloads: []string{devOptions.Workload},
}
if devOptions.ParentContainer != "" {
client, _, err := dev.GetClient()
if err != nil {
return err
}
var inspect types.ContainerJSON
inspect, err = client.ContainerInspect(context.Background(), devOptions.ParentContainer)
if err != nil {
return err
}
if inspect.State == nil {
return fmt.Errorf("can not get container status, please make contianer name is valid")
}
if !inspect.State.Running {
return fmt.Errorf("container %s status is %s, expect is running, please make sure your outer docker name is correct", devOptions.ParentContainer, inspect.State.Status)
}
}
if err := connect.InitClient(f); err != nil {
return err
}
@@ -123,6 +143,7 @@ func CmdDev(f cmdutil.Factory) *cobra.Command {
// docker options
cmd.Flags().Var(&devOptions.ExtraHosts, "add-host", "Add a custom host-to-IP mapping (host:ip)")
cmd.Flags().StringVar(&devOptions.ParentContainer, "parent-container", "", "Parent container name if running in Docker (Docker in Docker)")
cmd.Flags().VarP(&devOptions.Volumes, "volume", "v", "Bind mount a volume")
cmd.Flags().Var(&devOptions.Mounts, "mount", "Attach a filesystem mount to the container")
cmd.Flags().Var(&devOptions.Expose, "expose", "Expose a port or a range of ports")

7
pkg/dev/convert.go
View File

@@ -211,24 +211,27 @@ func GetVolume(ctx context.Context, f util.Factory, ns, pod string) (map[string]
if volumeMount.MountPath == "/tmp" {
continue
}
join := filepath.Join(os.TempDir(), strconv.Itoa(rand.Int()))
err = os.MkdirAll(join, 0755)
if err != nil {
return nil, err
}
if volumeMount.SubPath != "" {
join = filepath.Join(join, volumeMount.SubPath)
}
// pod-namespace/pod-name:path
remotePath := fmt.Sprintf("%s/%s:%s", ns, pod, volumeMount.MountPath)
stdIn, stdOut, stdErr := dockerterm.StdStreams()
copyOptions := cp.NewCopyOptions(genericclioptions.IOStreams{In: stdIn, Out: stdOut, ErrOut: stdErr})
copyOptions.Container = c.Name
copyOptions.MaxTries = 10
err = copyOptions.Complete(f, &cobra.Command{}, []string{remotePath, join})
if err != nil {
return nil, err
}
err = copyOptions.Run()
if err != nil {
_, _ = fmt.Fprintf(os.Stderr, "Can not download volume %s path %s, ignore...", volumeMount.Name, volumeMount.MountPath)
_, _ = fmt.Fprintf(os.Stderr, "Can not download volume %s path %s, err: %v, ignore...\n", volumeMount.Name, volumeMount.MountPath, err)
continue
}
m = append(m, mount.Mount{

74
pkg/dev/main.go
View File

@@ -35,16 +35,17 @@ type Options struct {
// docker options
Platform string
//Pull string // always, missing, never
PublishAll bool
Entrypoint string
DockerImage string
Publish opts.ListOpts
Expose opts.ListOpts
ExtraHosts opts.ListOpts
Env opts.ListOpts
Mounts opts.MountOpt
Volumes opts.ListOpts
VolumeDriver string
PublishAll bool
Entrypoint string
DockerImage string
Publish opts.ListOpts
Expose opts.ListOpts
ExtraHosts opts.ListOpts
ParentContainer string
Env opts.ListOpts
Mounts opts.MountOpt
Volumes opts.ListOpts
VolumeDriver string
}
func (d Options) Main(ctx context.Context) error {
@@ -103,21 +104,42 @@ func (d Options) Main(ctx context.Context) error {
if err != nil {
return err
}
var dockerCli *command.DockerCli
_, dockerCli, err = GetClient()
if err != nil {
return err
}
if d.ParentContainer != "" {
for _, config := range list[:] {
// remove expose port
config.config.ExposedPorts = nil
config.hostConfig.NetworkMode = containertypes.NetworkMode("container:" + d.ParentContainer)
config.hostConfig.PidMode = containertypes.PidMode("container:" + d.ParentContainer)
config.hostConfig.PortBindings = nil
// skip first
for _, config := range list[1:] {
// remove expose port
config.config.ExposedPorts = nil
config.hostConfig.NetworkMode = containertypes.NetworkMode("container:" + list[0].containerName)
config.hostConfig.PidMode = containertypes.PidMode("container:" + list[0].containerName)
config.hostConfig.PortBindings = nil
// remove dns
config.hostConfig.DNS = nil
config.hostConfig.DNSOptions = nil
config.hostConfig.DNSSearch = nil
config.hostConfig.PublishAllPorts = false
config.config.Hostname = ""
}
} else {
// skip first
for _, config := range list[1:] {
// remove expose port
config.config.ExposedPorts = nil
config.hostConfig.NetworkMode = containertypes.NetworkMode("container:" + list[0].containerName)
config.hostConfig.PidMode = containertypes.PidMode("container:" + list[0].containerName)
config.hostConfig.PortBindings = nil
// remove dns
config.hostConfig.DNS = nil
config.hostConfig.DNSOptions = nil
config.hostConfig.DNSSearch = nil
config.hostConfig.PublishAllPorts = false
config.config.Hostname = ""
// remove dns
config.hostConfig.DNS = nil
config.hostConfig.DNSOptions = nil
config.hostConfig.DNSSearch = nil
config.hostConfig.PublishAllPorts = false
config.config.Hostname = ""
}
}
handler.RollbackFuncList = append(handler.RollbackFuncList, func() {
@@ -127,11 +149,7 @@ func (d Options) Main(ctx context.Context) error {
if err != nil {
return err
}
_, cli, err := GetClient()
if err != nil {
return err
}
return terminal(list[0].containerName, cli)
return terminal(list[0].containerName, dockerCli)
}
type Run []*RunConfig

4
pkg/dev/run.go
View File

@@ -50,7 +50,7 @@ func run(ctx context.Context, runConfig *RunConfig, cli *client.Client) (err err
}
readCloser, err = cli.ImagePull(ctx, config.Image, types.ImagePullOptions{Platform: plat})
if err != nil {
return err
return fmt.Errorf("can not pull image %s, err: %s, please make sure image is exist and can be pulled from local", config.Image, err)
}
defer readCloser.Close()
_, stdout, _ := dockerterm.StdStreams()
@@ -64,7 +64,7 @@ func run(ctx context.Context, runConfig *RunConfig, cli *client.Client) (err err
var create typescommand.CreateResponse
create, err = cli.ContainerCreate(ctx, config, hostConfig, networkConfig, platform, name)
if err != nil {
return err
return fmt.Errorf("failed to create container %s, err: %s", name, err)
}
log.Infof("Created container: %s", name)

48
pkg/dns/dns_linux.go
View File

@@ -4,9 +4,14 @@
package dns
import (
"bytes"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/docker/docker/libnetwork/resolvconf"
miekgdns "github.com/miekg/dns"
log "github.com/sirupsen/logrus"
@@ -42,16 +47,55 @@ func SetupDNS(clientConfig *miekgdns.ClientConfig, _ []string) error {
}...)
output, err := cmd.CombinedOutput()
if err != nil {
log.Warnf("cmd: %s, output: %s, error: %v\n", cmd.Args, string(output), err)
log.Debugf("failed to exec cmd: %s, message: %s, ignore", strings.Join(cmd.Args, " "), string(output))
}
return nil
filename := filepath.Join("/", "etc", "resolv.conf")
readFile, err := os.ReadFile(filename)
if err == nil {
resolvConf, err := miekgdns.ClientConfigFromReader(bytes.NewBufferString(string(readFile)))
if err == nil {
if len(resolvConf.Servers) != 0 {
clientConfig.Servers = append(clientConfig.Servers, resolvConf.Servers...)
}
if len(resolvConf.Search) != 0 {
clientConfig.Search = append(clientConfig.Search, resolvConf.Search...)
}
}
}
return WriteResolvConf(*clientConfig)
}
func CancelDNS() {
updateHosts("")
filename := filepath.Join("/", "etc", "resolv.conf")
_ = os.Rename(getBackupFilename(filename), filename)
}
func GetHostFile() string {
return "/etc/hosts"
}
func WriteResolvConf(config miekgdns.ClientConfig) error {
var options []string
if config.Ndots != 0 {
options = append(options, fmt.Sprintf("ndots:%d", config.Ndots))
}
if config.Attempts != 0 {
options = append(options, fmt.Sprintf("attempts:%d", config.Attempts))
}
if config.Timeout != 0 {
options = append(options, fmt.Sprintf("timeout:%d", config.Timeout))
}
filename := filepath.Join("/", "etc", "resolv.conf")
_ = os.Rename(filename, getBackupFilename(filename))
_, err := resolvconf.Build(filename, config.Servers, config.Search, options)
return err
}
func getBackupFilename(filename string) string {
return filename + ".kubevpn_backup"
}

15
pkg/dns/dns_server_test.go
View File

@@ -11,6 +11,7 @@ import (
"testing"
"time"
"github.com/docker/docker/libnetwork/resolvconf"
miekgdns "github.com/miekg/dns"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
@@ -178,3 +179,17 @@ func TestFix(t *testing.T) {
println(s)
}
}
func TestName(t *testing.T) {
temp, _ := os.CreateTemp("", "")
temp.Close()
println(temp.Name())
_, err := resolvconf.Build(temp.Name(), []string{"10.233.0.3", "10.233.97.159", "10.233.122.162"}, []string{
"vke-system.svc.cluster.local",
"svc.cluster.local",
"cluster.local",
}, []string{"ndots:5", "timeout:5"})
if err != nil {
panic(err)
}
}