diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 45e3d4d3..ceaaa204 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,7 +22,7 @@ jobs: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USER }} --password-stdin docker buildx create --use export VERSION=test - make container + # make container linux: runs-on: ubuntu-latest needs: [ "image" ] @@ -36,8 +36,11 @@ jobs: check-latest: true cache: true - name: Setup Minikube + id: minikube timeout-minutes: 30 uses: medyagh/setup-minikube@master + with: + cache: true - name: Kubernetes info run: | @@ -69,6 +72,8 @@ jobs: run: | kubectl wait pods -l app=reviews --for=condition=Ready --timeout=3600s kubectl wait pods -l app=productpage --for=condition=Ready --timeout=3600s + kubectl get svc -A -o wide + kubectl get pod -A -o wide kubectl get all -o wide kubectl get nodes -o yaml ifconfig @@ -76,10 +81,10 @@ jobs: sudo ln /usr/bin/resolvectl /usr/bin/systemd-resolve - name: Test - run: go test -v ./... -timeout=60m + run: go test -v -failfast ./... -timeout=60m macos: - runs-on: macos-10.15 + runs-on: macos-latest needs: [ "image" ] steps: - uses: actions/checkout@v2 @@ -90,19 +95,17 @@ jobs: go-version: 1.19 check-latest: true cache: true - - uses: docker-practice/actions-setup-docker@master - - name: Pull image in advance - run: | - rm '/usr/local/bin/kubectl' - set -x - docker version + - name: Setup Docker on macOS + uses: douglascamata/setup-docker-macos-action@v1-alpha - name: Install minikube run: | + set -x + docker version brew install minikube minikube start --driver=docker - kubectl get po -A - minikube kubectl -- get po -A + kubectl get pod -A -o wide + minikube kubectl -- get pod -A -o wide - name: Kubernetes info run: | @@ -135,13 +138,15 @@ jobs: run: | kubectl wait pods -l app=reviews --for=condition=Ready --timeout=3600s kubectl wait pods -l app=productpage --for=condition=Ready --timeout=3600s - kubectl get all -o wide - kubectl get nodes -o yaml + kubectl get svc -A -o wide || true + kubectl get pod -A -o wide || true + kubectl get all -o wide || true + kubectl get nodes -o yaml || true ifconfig netstat -anr - name: Test - run: go test -v ./... -timeout=60m + run: go test -v -failfast ./... -timeout=60m # windows: # runs-on: windows-latest diff --git a/README.md b/README.md index f59d84a1..e7352e4b 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,27 @@ +![kubevpn](samples/flat_log.png) + +[![GitHub Workflow][1]](https://github.com/wencaiwulue/kubevpn/actions) +[![Go Version][2]](https://github.com/wencaiwulue/kubevpn/blob/master/go.mod) +[![Go Report][3]](https://goreportcard.com/badge/github.com/wencaiwulue/kubevpn) +[![Maintainability][4]](https://codeclimate.com/github/wencaiwulue/kubevpn/maintainability) +[![GitHub License][5]](https://github.com/wencaiwulue/kubevpn/blob/main/LICENSE) +[![Docker Pulls][6]](https://hub.docker.com/r/naison/kubevpn) +[![Releases][7]](https://github.com/wencaiwulue/kubevpn/releases) + +[1]: https://img.shields.io/github/actions/workflow/status/wencaiwulue/kubevpn/release.yml?logo=github + +[2]: https://img.shields.io/github/go-mod/go-version/wencaiwulue/kubevpn?logo=go + +[3]: https://goreportcard.com/badge/github.com/wencaiwulue/kubevpn + +[4]: https://api.codeclimate.com/v1/badges/b5b30239174fc6603aca/maintainability + +[5]: https://img.shields.io/github/license/wencaiwulue/kubevpn + +[6]: https://img.shields.io/docker/pulls/naison/kubevpn?logo=docker + +[7]: https://img.shields.io/github/v/release/wencaiwulue/kubevpn?logo=smartthings + # KubeVPN [中文](README_ZH.md) | [English](README.md) | [Wiki](https://github.com/wencaiwulue/kubevpn/wiki/Architecture) diff --git a/README_ZH.md b/README_ZH.md index 36e0181a..939ea02e 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -1,3 +1,27 @@ +![kubevpn](samples/flat_log.png) + +[![GitHub Workflow][1]](https://github.com/wencaiwulue/kubevpn/actions) +[![Go Version][2]](https://github.com/wencaiwulue/kubevpn/blob/master/go.mod) +[![Go Report][3]](https://goreportcard.com/badge/github.com/wencaiwulue/kubevpn) +[![Maintainability][4]](https://codeclimate.com/github/wencaiwulue/kubevpn/maintainability) +[![GitHub License][5]](https://github.com/wencaiwulue/kubevpn/blob/main/LICENSE) +[![Docker Pulls][6]](https://hub.docker.com/r/naison/kubevpn) +[![Releases][7]](https://github.com/wencaiwulue/kubevpn/releases) + +[1]: https://img.shields.io/github/actions/workflow/status/wencaiwulue/kubevpn/release.yml?logo=github + +[2]: https://img.shields.io/github/go-mod/go-version/wencaiwulue/kubevpn?logo=go + +[3]: https://goreportcard.com/badge/github.com/wencaiwulue/kubevpn + +[4]: https://api.codeclimate.com/v1/badges/b5b30239174fc6603aca/maintainability + +[5]: https://img.shields.io/github/license/wencaiwulue/kubevpn + +[6]: https://img.shields.io/docker/pulls/naison/kubevpn?logo=docker + +[7]: https://img.shields.io/github/v/release/wencaiwulue/kubevpn?logo=smartthings + # KubeVPN [English](README.md) | [中文](README_ZH.md) | [维基](https://github.com/wencaiwulue/kubevpn/wiki/%E6%9E%B6%E6%9E%84) diff --git a/cmd/kubevpn/cmds/version.go b/cmd/kubevpn/cmds/version.go index 767027b2..98ff97e6 100644 --- a/cmd/kubevpn/cmds/version.go +++ b/cmd/kubevpn/cmds/version.go @@ -31,8 +31,8 @@ func reformatDate(buildTime string) string { func CmdVersion(cmdutil.Factory) *cobra.Command { cmd := &cobra.Command{ Use: "version", - Short: "Print the version number of KubeVPN", - Long: `This is the version of KubeVPN`, + Short: "Print the client version information", + Long: `Print the client version information`, Run: func(cmd *cobra.Command, args []string) { fmt.Printf("KubeVPN: CLI\n") fmt.Printf(" Version: %s\n", config.Version) diff --git a/pkg/handler/function_test.go b/pkg/handler/function_test.go index 8f38dc73..294f12eb 100644 --- a/pkg/handler/function_test.go +++ b/pkg/handler/function_test.go @@ -13,10 +13,9 @@ import ( "testing" "time" - "github.com/docker/distribution/reference" log "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/cli-runtime/pkg/genericclioptions" @@ -33,7 +32,7 @@ var ( namespace string clientset *kubernetes.Clientset restclient *rest.RESTClient - c *rest.Config + restconfig *rest.Config ) func TestFunctions(t *testing.T) { @@ -47,9 +46,7 @@ func TestFunctions(t *testing.T) { } func pingPodIP(t *testing.T) { - ctx, f := context.WithTimeout(context.Background(), time.Second*60) - defer f() - list, err := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) + list, err := clientset.CoreV1().Pods(namespace).List(context.Background(), v1.ListOptions{}) if err != nil { t.Error(err) } @@ -72,44 +69,69 @@ func pingPodIP(t *testing.T) { } func healthCheckPod(t *testing.T) { - podList, err := clientset.CoreV1().Pods(namespace).List(context.Background(), metav1.ListOptions{ - LabelSelector: fields.OneTermEqualSelector("app", "productpage").String(), + var app = "authors" + podList, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), v1.ListOptions{ + LabelSelector: fields.OneTermEqualSelector("app", app).String(), }) if err != nil { t.Error(err) } if len(podList.Items) == 0 { - t.Error("can not found pods of product page") + t.Error("can not found pods of authors") } - endpoint := fmt.Sprintf("http://%s:%v/health", podList.Items[0].Status.PodIP, podList.Items[0].Spec.Containers[0].Ports[0].ContainerPort) - req, _ := http.NewRequest("GET", endpoint, nil) - res, err := http.DefaultClient.Do(req) - if err != nil { - t.Error(err) - return - } - if res == nil || res.StatusCode != 200 { - t.Errorf("health check not pass") - return + for _, pod := range podList.Items { + pod := pod + if pod.Status.Phase != corev1.PodRunning { + continue + } + endpoint := fmt.Sprintf("http://%s:%v/health", pod.Status.PodIP, pod.Spec.Containers[0].Ports[0].ContainerPort) + req, _ := http.NewRequest("GET", endpoint, nil) + var res *http.Response + err = retry.OnError( + wait.Backoff{Duration: time.Second, Factor: 2, Jitter: 0.2, Steps: 5}, + func(err error) bool { + return err != nil + }, + func() error { + res, err = http.DefaultClient.Do(req) + return err + }, + ) + if err != nil { + t.Error(err) + } + if res == nil || res.StatusCode != 200 { + t.Errorf("health check not pass") + } } } func healthCheckService(t *testing.T) { - serviceList, err := clientset.CoreV1().Services(namespace).List(context.Background(), metav1.ListOptions{ - LabelSelector: fields.OneTermEqualSelector("app", "productpage").String(), + var app = "authors" + serviceList, err := clientset.CoreV1().Services(namespace).List(context.TODO(), v1.ListOptions{ + LabelSelector: fields.OneTermEqualSelector("app", app).String(), }) if err != nil { t.Error(err) } if len(serviceList.Items) == 0 { - t.Error("can not found pods of product page") + t.Error("can not found pods of authors") } endpoint := fmt.Sprintf("http://%s:%v/health", serviceList.Items[0].Spec.ClusterIP, serviceList.Items[0].Spec.Ports[0].Port) req, _ := http.NewRequest("GET", endpoint, nil) - res, err := http.DefaultClient.Do(req) + var res *http.Response + err = retry.OnError( + wait.Backoff{Duration: time.Second, Factor: 2, Jitter: 0.2, Steps: 5}, + func(err error) bool { + return err != nil + }, + func() error { + res, err = http.DefaultClient.Do(req) + return err + }, + ) if err != nil { t.Error(err) - return } if res == nil || res.StatusCode != 200 { t.Errorf("health check not pass") @@ -118,8 +140,8 @@ func healthCheckService(t *testing.T) { } func shortDomain(t *testing.T) { - var app = "productpage" - serviceList, err := clientset.CoreV1().Services(namespace).List(context.Background(), metav1.ListOptions{ + var app = "authors" + serviceList, err := clientset.CoreV1().Services(namespace).List(context.TODO(), v1.ListOptions{ LabelSelector: fields.OneTermEqualSelector("app", app).String(), }) if err != nil { @@ -130,20 +152,28 @@ func shortDomain(t *testing.T) { } endpoint := fmt.Sprintf("http://%s:%v/health", app, serviceList.Items[0].Spec.Ports[0].Port) req, _ := http.NewRequest("GET", endpoint, nil) - res, err := http.DefaultClient.Do(req) + var res *http.Response + err = retry.OnError( + wait.Backoff{Duration: time.Second, Factor: 2, Jitter: 0.2, Steps: 5}, + func(err error) bool { + return err != nil + }, + func() error { + res, err = http.DefaultClient.Do(req) + return err + }, + ) if err != nil { t.Error(err) - return } if res == nil || res.StatusCode != 200 { t.Errorf("health check not pass") - return } } func fullDomain(t *testing.T) { - var app = "productpage" - serviceList, err := clientset.CoreV1().Services(namespace).List(context.Background(), metav1.ListOptions{ + var app = "authors" + serviceList, err := clientset.CoreV1().Services(namespace).List(context.TODO(), v1.ListOptions{ LabelSelector: fields.OneTermEqualSelector("app", app).String(), }) if err != nil { @@ -154,10 +184,19 @@ func fullDomain(t *testing.T) { } endpoint := fmt.Sprintf("http://%s:%v/health", fmt.Sprintf("%s.%s.svc.cluster.local", app, namespace), serviceList.Items[0].Spec.Ports[0].Port) req, _ := http.NewRequest("GET", endpoint, nil) - res, err := http.DefaultClient.Do(req) + var res *http.Response + err = retry.OnError( + wait.Backoff{Duration: time.Second, Factor: 2, Jitter: 0.2, Steps: 5}, + func(err error) bool { + return err != nil + }, + func() error { + res, err = http.DefaultClient.Do(req) + return err + }, + ) if err != nil { t.Error(err) - return } if res == nil || res.StatusCode != 200 { t.Errorf("health check not pass") @@ -167,9 +206,9 @@ func fullDomain(t *testing.T) { func dialUDP(t *testing.T) { port := util.GetAvailableUDPPortOrDie() - go UDPServer(port) + go server(port) - list, err := clientset.CoreV1().Pods(namespace).List(context.Background(), metav1.ListOptions{ + list, err := clientset.CoreV1().Pods(namespace).List(context.Background(), v1.ListOptions{ LabelSelector: fields.OneTermEqualSelector("app", "reviews").String(), }) if err != nil { @@ -184,6 +223,7 @@ func dialUDP(t *testing.T) { } if len(ip) == 0 { t.Errorf("can not found pods for service reviews") + return } log.Printf("dail udp to ip: %s", ip) if err = retry.OnError( @@ -191,13 +231,13 @@ func dialUDP(t *testing.T) { func(err error) bool { return err != nil }, func() error { - return UDPClient(ip, port) + return udpclient(ip, port) }); err != nil { t.Errorf("can not access pod ip: %s, port: %v", ip, port) } } -func UDPClient(ip string, port int) error { +func udpclient(ip string, port int) error { udpConn, err := net.DialUDP("udp4", nil, &net.UDPAddr{ IP: net.ParseIP(ip), Port: port, @@ -217,7 +257,7 @@ func UDPClient(ip string, port int) error { sendData := []byte("hello server!") _, err = udpConn.Write(sendData) if err != nil { - fmt.Println("[client] 发送数据失败!", err) + fmt.Println("发送数据失败!", err) return err } @@ -225,7 +265,7 @@ func UDPClient(ip string, port int) error { data := make([]byte, 4096) read, remoteAddr, err := udpConn.ReadFromUDP(data) if err != nil { - fmt.Println("[client] 读取数据失败!", err) + fmt.Println("读取数据失败!", err) return err } fmt.Println(read, remoteAddr) @@ -233,7 +273,7 @@ func UDPClient(ip string, port int) error { return nil } -func UDPServer(port int) { +func server(port int) { // 创建监听 udpConn, err := net.ListenUDP("udp4", &net.UDPAddr{ IP: net.IPv4(0, 0, 0, 0), @@ -248,7 +288,7 @@ func UDPServer(port int) { data := make([]byte, 4096) read, remoteAddr, err := udpConn.ReadFromUDP(data) if err != nil { - fmt.Println("[server] 读取数据失败!", err) + fmt.Println("读取数据失败!", err) continue } fmt.Println(read, remoteAddr) @@ -257,27 +297,32 @@ func UDPServer(port int) { sendData := []byte("hello client!") _, err = udpConn.WriteToUDP(sendData, remoteAddr) if err != nil { - fmt.Println("[server] 发送数据失败!", err) + fmt.Println("发送数据失败!", err) return } } } func kubevpnConnect(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Hour) - cmd := exec.CommandContext(context.Background(), "kubevpn", "proxy", "deployments/reviews", "--debug") + ctx2, timeoutFunc := context.WithTimeout(context.Background(), 2*time.Hour) + + cmd := exec.Command("kubevpn", "proxy", "--debug", "deployments/reviews") go func() { - var checker = func(log string) { - if strings.Contains(log, "dns service ok") { - cancel() + stdout, stderr, err := util.RunWithRollingOutWithChecker(cmd, func(log string) { + ok := strings.Contains(log, "dns service ok") + if ok { + timeoutFunc() } - } - _, _, err := util.RunWithRollingOutWithChecker(cmd, checker) + }) + defer timeoutFunc() if err != nil { - t.Log(err) + t.Log(stdout, stderr) + t.Error(err) + t.Fail() + return } }() - <-ctx.Done() + <-ctx2.Done() } func init() { @@ -287,27 +332,16 @@ func init() { configFlags.KubeConfig = &clientcmd.RecommendedHomeFile f := cmdutil.NewFactory(cmdutil.NewMatchVersionFlags(configFlags)) - if c, err = f.ToRESTConfig(); err != nil { + if restconfig, err = f.ToRESTConfig(); err != nil { log.Fatal(err) } - if restclient, err = rest.RESTClientFor(c); err != nil { + if restclient, err = rest.RESTClientFor(restconfig); err != nil { log.Fatal(err) } - if clientset, err = kubernetes.NewForConfig(c); err != nil { + if clientset, err = kubernetes.NewForConfig(restconfig); err != nil { log.Fatal(err) } if namespace, _, err = f.ToRawKubeConfigLoader().Namespace(); err != nil { log.Fatal(err) } } - -func TestName(t *testing.T) { - name := "docker.io/naison/alpine@sha256:b733d4a32c4da6a00a84df2ca32791bb03df95400243648d8c539e7b4cce329c" - named, err := reference.ParseNormalizedNamed(name) - if err != nil { - t.Error(err) - } - domain := reference.Domain(named) - path := reference.Path(named) - fmt.Println(domain, path) -} diff --git a/pkg/handler/remote.go b/pkg/handler/remote.go index b57c4550..6c18eedb 100644 --- a/pkg/handler/remote.go +++ b/pkg/handler/remote.go @@ -177,11 +177,11 @@ func createOutboundPod(ctx context.Context, factory cmdutil.Factory, clientset * var Resources = v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("500m"), + v1.ResourceCPU: resource.MustParse("250m"), v1.ResourceMemory: resource.MustParse("512Mi"), }, Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("2000m"), + v1.ResourceCPU: resource.MustParse("1000m"), v1.ResourceMemory: resource.MustParse("2048Mi"), }, } @@ -371,41 +371,50 @@ kubevpn serve -L "tcp://:10800" -L "tun://:8422?net=${TunIPv4}" --debug=true`, if _, err = clientset.AppsV1().Deployments(namespace).Create(ctx, deployment, metav1.CreateOptions{}); err != nil { return err } - var last string -out: - for { - select { - case e, ok := <-watchStream.ResultChan(): - if !ok { - return fmt.Errorf("can not wait pod to be ready because of watch chan has closed") - } - if podT, ok := e.Object.(*v1.Pod); ok { - if podT.DeletionTimestamp != nil { - continue - } - var sb = bytes.NewBuffer(nil) - sb.WriteString(fmt.Sprintf("pod [%s] status is %s\n", config.ConfigMapPodTrafficManager, podT.Status.Phase)) - util.PrintStatus(podT, sb) - - if last != sb.String() { - log.Infof(sb.String()) - } - if podutils.IsPodReady(podT) && func() bool { - for _, status := range podT.Status.ContainerStatuses { - if !status.Ready { - return false - } - } - return true - }() { - break out - } - last = sb.String() - } - case <-time.Tick(time.Minute * 60): - return errors.New(fmt.Sprintf("wait pod %s to be ready timeout", config.ConfigMapPodTrafficManager)) + var ok bool + ctx2, cancelFunc := context.WithTimeout(ctx, time.Minute*60) + defer cancelFunc() + wait.UntilWithContext(ctx2, func(ctx context.Context) { + podList, err := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fields.OneTermEqualSelector("app", config.ConfigMapPodTrafficManager).String(), + }) + if err != nil { + return } + + for _, podT := range podList.Items { + podT := &podT + if podT.DeletionTimestamp != nil { + continue + } + var sb = bytes.NewBuffer(nil) + sb.WriteString(fmt.Sprintf("pod %s is %s\n", podT.Name, podT.Status.Phase)) + if podT.Status.Reason != "" { + sb.WriteString(fmt.Sprintf(" reason %s", podT.Status.Reason)) + } + if podT.Status.Message != "" { + sb.WriteString(fmt.Sprintf(" message %s", podT.Status.Message)) + } + util.PrintStatus(podT, sb) + log.Infof(sb.String()) + + if podutils.IsPodReady(podT) && func() bool { + for _, status := range podT.Status.ContainerStatuses { + if !status.Ready { + return false + } + } + return true + }() { + cancelFunc() + ok = true + } + } + }, time.Second*3) + if !ok { + return errors.New(fmt.Sprintf("wait pod %s to be ready timeout", config.ConfigMapPodTrafficManager)) } + _, err = clientset.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(ctx, &admissionv1.MutatingWebhookConfiguration{ ObjectMeta: metav1.ObjectMeta{ Name: config.ConfigMapPodTrafficManager + "." + namespace, diff --git a/pkg/util/pod.go b/pkg/util/pod.go index 20679c52..d961fb96 100644 --- a/pkg/util/pod.go +++ b/pkg/util/pod.go @@ -31,6 +31,15 @@ func PrintStatus(pod *corev1.Pod, writer io.Writer) { _, _ = fmt.Fprintf(w, "%s\t%v\t%v\n", name, v1, v2) } + if len(pod.Status.ContainerStatuses) == 0 { + show("Type", "Reason", "Message") + for _, condition := range pod.Status.Conditions { + if condition.Status == corev1.ConditionFalse { + show(string(condition.Type), condition.Reason, condition.Message) + } + } + return + } show("Container", "Reason", "Message") for _, status := range pod.Status.ContainerStatuses { if status.State.Waiting != nil { diff --git a/pkg/util/util.go b/pkg/util/util.go index 46307d43..3abf00f2 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -460,7 +460,9 @@ func RunWithRollingOutWithChecker(cmd *osexec.Cmd, checker func(log string)) (st } return stdoutBuf.String(), stderrBuf.String(), err } - _ = cmd.Wait() + if err := cmd.Wait(); err != nil { + return "", "", err + } var err error if !cmd.ProcessState.Success() { err = errors.New("exit code is not 0") diff --git a/samples/flat_log.png b/samples/flat_log.png new file mode 100644 index 00000000..057a61bf Binary files /dev/null and b/samples/flat_log.png differ