mirror of
https://github.com/kubenetworks/kubevpn.git
synced 2025-10-05 07:16:54 +08:00
332 lines
8.8 KiB
Go
332 lines
8.8 KiB
Go
package dev
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"math/rand"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"time"
|
|
|
|
"github.com/docker/cli/cli/command"
|
|
"github.com/docker/cli/cli/flags"
|
|
"github.com/docker/cli/opts"
|
|
"github.com/docker/docker/api/types"
|
|
containertypes "github.com/docker/docker/api/types/container"
|
|
"github.com/docker/docker/api/types/mount"
|
|
"github.com/docker/docker/client"
|
|
"github.com/docker/docker/pkg/archive"
|
|
log "github.com/sirupsen/logrus"
|
|
v1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
|
"k8s.io/apimachinery/pkg/labels"
|
|
"k8s.io/apimachinery/pkg/util/wait"
|
|
cmdutil "k8s.io/kubectl/pkg/cmd/util"
|
|
"k8s.io/kubectl/pkg/polymorphichelpers"
|
|
"k8s.io/kubectl/pkg/util/podutils"
|
|
|
|
"github.com/wencaiwulue/kubevpn/pkg/handler"
|
|
"github.com/wencaiwulue/kubevpn/pkg/mesh"
|
|
"github.com/wencaiwulue/kubevpn/pkg/util"
|
|
)
|
|
|
|
type Options struct {
|
|
Headers map[string]string
|
|
Namespace string
|
|
Workload string
|
|
Factory cmdutil.Factory
|
|
ContainerName string
|
|
NoProxy bool
|
|
ExtraCIDR []string
|
|
|
|
// docker options
|
|
Platform string
|
|
//Pull string // always, missing, never
|
|
PublishAll bool
|
|
Entrypoint string
|
|
DockerImage string
|
|
Publish opts.ListOpts
|
|
Expose opts.ListOpts
|
|
ExtraHosts opts.ListOpts
|
|
ParentContainer string
|
|
Env opts.ListOpts
|
|
Mounts opts.MountOpt
|
|
Volumes opts.ListOpts
|
|
VolumeDriver string
|
|
}
|
|
|
|
func (d Options) Main(ctx context.Context) error {
|
|
rand.Seed(time.Now().UnixNano())
|
|
object, err := util.GetUnstructuredObject(d.Factory, d.Namespace, d.Workload)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
u := object.Object.(*unstructured.Unstructured)
|
|
var templateSpec *v1.PodTemplateSpec
|
|
//var path []string
|
|
templateSpec, _, err = util.GetPodTemplateSpecPath(u)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
set, err := d.Factory.KubernetesClientSet()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
sortBy := func(pods []*v1.Pod) sort.Interface {
|
|
for i := 0; i < len(pods); i++ {
|
|
if pods[i].DeletionTimestamp != nil {
|
|
pods = append(pods[:i], pods[i+1:]...)
|
|
i--
|
|
}
|
|
}
|
|
return sort.Reverse(podutils.ActivePods(pods))
|
|
}
|
|
lab := labels.SelectorFromSet(templateSpec.Labels).String()
|
|
firstPod, _, err := polymorphichelpers.GetFirstPod(set.CoreV1(), d.Namespace, lab, time.Second*60, sortBy)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
pod := firstPod.Name
|
|
|
|
env, err := util.GetEnv(ctx, d.Factory, d.Namespace, pod)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
volume, err := util.GetVolume(ctx, d.Factory, d.Namespace, pod)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
dns, err := GetDNS(ctx, d.Factory, d.Namespace, pod)
|
|
if err != nil {
|
|
return fmt.Errorf("can not get dns conf from pod: %s, err: %v", pod, err)
|
|
}
|
|
|
|
mesh.RemoveContainers(templateSpec)
|
|
list := ConvertKubeResourceToContainer(d.Namespace, *templateSpec, env, volume, dns)
|
|
err = fillOptions(list, d)
|
|
if err != nil {
|
|
return fmt.Errorf("can not fill docker options, err: %v", err)
|
|
}
|
|
var dockerCli *command.DockerCli
|
|
var cli *client.Client
|
|
cli, dockerCli, err = GetClient()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// check resource
|
|
var outOfMemory bool
|
|
outOfMemory, _ = checkOutOfMemory(templateSpec, cli)
|
|
if outOfMemory {
|
|
return fmt.Errorf("your pod resource request is bigger than docker-desktop resource, please adjust your docker-desktop resource")
|
|
}
|
|
if d.ParentContainer != "" {
|
|
for _, config := range list[:] {
|
|
// remove expose port
|
|
config.config.ExposedPorts = nil
|
|
config.hostConfig.NetworkMode = containertypes.NetworkMode("container:" + d.ParentContainer)
|
|
config.hostConfig.PidMode = containertypes.PidMode("container:" + d.ParentContainer)
|
|
config.hostConfig.PortBindings = nil
|
|
|
|
// remove dns
|
|
config.hostConfig.DNS = nil
|
|
config.hostConfig.DNSOptions = nil
|
|
config.hostConfig.DNSSearch = nil
|
|
config.hostConfig.PublishAllPorts = false
|
|
config.config.Hostname = ""
|
|
}
|
|
} else {
|
|
// skip first
|
|
for _, config := range list[1:] {
|
|
// remove expose port
|
|
config.config.ExposedPorts = nil
|
|
config.hostConfig.NetworkMode = containertypes.NetworkMode("container:" + list[0].containerName)
|
|
config.hostConfig.PidMode = containertypes.PidMode("container:" + list[0].containerName)
|
|
config.hostConfig.PortBindings = nil
|
|
|
|
// remove dns
|
|
config.hostConfig.DNS = nil
|
|
config.hostConfig.DNSOptions = nil
|
|
config.hostConfig.DNSSearch = nil
|
|
config.hostConfig.PublishAllPorts = false
|
|
config.config.Hostname = ""
|
|
}
|
|
}
|
|
|
|
handler.RollbackFuncList = append(handler.RollbackFuncList, func() {
|
|
_ = list.Remove(ctx)
|
|
})
|
|
err = list.Run(ctx, volume)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return terminal(list[0].containerName, dockerCli)
|
|
}
|
|
|
|
type Run []*RunConfig
|
|
|
|
func (r Run) Remove(ctx context.Context) error {
|
|
cli, _, err := GetClient()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, config := range r {
|
|
//err = cli.NetworkDisconnect(ctx, config.networkName, config.containerName, true)
|
|
//if err != nil {
|
|
// log.Errorln(err)
|
|
//}
|
|
err = cli.ContainerRemove(ctx, config.containerName, types.ContainerRemoveOptions{Force: true})
|
|
if err != nil {
|
|
log.Errorln(err)
|
|
}
|
|
}
|
|
//for _, config := range r {
|
|
// _, err = cli.NetworkInspect(ctx, config.networkName, types.NetworkInspectOptions{})
|
|
// if err == nil {
|
|
// err = cli.NetworkRemove(ctx, config.networkName)
|
|
// if err != nil {
|
|
// log.Errorln(err)
|
|
// }
|
|
// }
|
|
//}
|
|
return nil
|
|
}
|
|
|
|
func GetClient() (*client.Client, *command.DockerCli, error) {
|
|
cli, err := client.NewClientWithOpts(
|
|
client.FromEnv,
|
|
client.WithAPIVersionNegotiation(),
|
|
)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("can not create docker client from env, err: %v", err)
|
|
}
|
|
var dockerCli *command.DockerCli
|
|
dockerCli, err = command.NewDockerCli(command.WithAPIClient(cli))
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("can not create docker client from env, err: %v", err)
|
|
}
|
|
err = dockerCli.Initialize(flags.NewClientOptions())
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("can not init docker client, err: %v", err)
|
|
}
|
|
return cli, dockerCli, nil
|
|
}
|
|
|
|
func (r Run) Run(ctx context.Context, volume map[string][]mount.Mount) error {
|
|
cli, _, err := GetClient()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, config := range r {
|
|
var id string
|
|
id, err = run(ctx, config, cli)
|
|
if err != nil {
|
|
// try another way to startup container
|
|
log.Info("try another way to startup container")
|
|
config.hostConfig.Mounts = nil
|
|
id, err = run(ctx, config, cli)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = r.copyToContainer(ctx, volume[config.k8sContainerName], cli, id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r Run) copyToContainer(ctx context.Context, volume []mount.Mount, cli *client.Client, id string) error {
|
|
// copy volume into container
|
|
for _, v := range volume {
|
|
target, err := createFolder(ctx, cli, id, v.Source, v.Target)
|
|
if err != nil {
|
|
log.Debugf("create folder %s previoully faied, err: %v", target, err)
|
|
}
|
|
log.Debugf("from %s to %s", v.Source, v.Target)
|
|
srcInfo, err := archive.CopyInfoSourcePath(v.Source, true)
|
|
if err != nil {
|
|
return fmt.Errorf("copy info source path, err: %v", err)
|
|
}
|
|
srcArchive, err := archive.TarResource(srcInfo)
|
|
if err != nil {
|
|
return fmt.Errorf("tar resource failed, err: %v", err)
|
|
}
|
|
dstDir, preparedArchive, err := archive.PrepareArchiveCopy(srcArchive, srcInfo, archive.CopyInfo{Path: v.Target})
|
|
if err != nil {
|
|
return fmt.Errorf("can not prepare archive copy, err: %v", err)
|
|
}
|
|
|
|
err = cli.CopyToContainer(ctx, id, dstDir, preparedArchive, types.CopyToContainerOptions{
|
|
AllowOverwriteDirWithFile: true,
|
|
CopyUIDGID: true,
|
|
})
|
|
if err != nil {
|
|
log.Info(fmt.Errorf("can not copy %s to container %s:%s, err: %v", v.Source, id, v.Target, err))
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func createFolder(ctx context.Context, cli *client.Client, id string, src string, target string) (string, error) {
|
|
lstat, err := os.Lstat(src)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if !lstat.IsDir() {
|
|
target = filepath.Dir(target)
|
|
}
|
|
var create types.IDResponse
|
|
create, err = cli.ContainerExecCreate(ctx, id, types.ExecConfig{
|
|
AttachStdin: true,
|
|
AttachStderr: true,
|
|
AttachStdout: true,
|
|
Cmd: []string{"mkdir", "-p", target},
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
err = cli.ContainerExecStart(ctx, create.ID, types.ExecStartCheck{})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
chanStop := make(chan struct{})
|
|
wait.Until(func() {
|
|
inspect, err := cli.ContainerExecInspect(ctx, create.ID)
|
|
if err != nil {
|
|
return
|
|
}
|
|
if !inspect.Running {
|
|
close(chanStop)
|
|
}
|
|
}, time.Second, chanStop)
|
|
return target, nil
|
|
}
|
|
|
|
func checkOutOfMemory(spec *v1.PodTemplateSpec, cli *client.Client) (outOfMemory bool, err error) {
|
|
var info types.Info
|
|
info, err = cli.Info(context.Background())
|
|
if err != nil {
|
|
return
|
|
}
|
|
total := info.MemTotal
|
|
var req int64
|
|
for _, container := range spec.Spec.Containers {
|
|
memory := container.Resources.Requests.Memory()
|
|
if memory != nil {
|
|
req += memory.Value()
|
|
}
|
|
}
|
|
if req > total {
|
|
outOfMemory = true
|
|
return
|
|
}
|
|
return
|
|
}
|