feat: use dns query as port-forward health check (#570)

refactor: refactor code

.github/krew.yaml

@@ -0,0 +1,102 @@
+apiVersion: krew.googlecontainertools.github.com/v1alpha2
+kind: Plugin
+metadata:
+  name: kubevpn
+spec:
+  version: {{ .TagName }}
+  homepage: https://github.com/kubenetworks/kubevpn
+  shortDescription: "KubeVPN offers a Cloud Native Dev Environment that connects to kubernetes cluster network"
+  description: |
+    KubeVPN offers a Cloud-Native Dev Environment that seamlessly connects to your Kubernetes cluster network.
+    Gain access to the Kubernetes cluster network effortlessly using service names or Pod IP/Service IP. Facilitate the interception of inbound traffic from remote Kubernetes cluster services to your local PC through a service mesh and more.
+    For instance, you have the flexibility to run your Kubernetes pod within a local Docker container, ensuring an identical environment, volume, and network setup. With KubeVPN, empower yourself to develop applications entirely on your local PC!
+
+  platforms:
+  - selector:
+      matchLabels:
+        os: windows
+        arch: amd64
+    {{addURIAndSha "https://github.com/kubenetworks/kubevpn/releases/download/{{ .TagName }}/kubevpn_{{ .TagName }}_windows_amd64.zip" .TagName }}
+    files:
+      - from: ./bin/kubevpn.exe
+        to: .
+      - from: LICENSE
+        to: .
+    bin: kubevpn.exe
+  - selector:
+      matchLabels:
+        os: windows
+        arch: arm64
+    {{addURIAndSha "https://github.com/kubenetworks/kubevpn/releases/download/{{ .TagName }}/kubevpn_{{ .TagName }}_windows_arm64.zip" .TagName }}
+    files:
+      - from: ./bin/kubevpn.exe
+        to: .
+      - from: LICENSE
+        to: .
+    bin: kubevpn.exe
+  - selector:
+      matchLabels:
+        os: windows
+        arch: 386
+    {{addURIAndSha "https://github.com/kubenetworks/kubevpn/releases/download/{{ .TagName }}/kubevpn_{{ .TagName }}_windows_386.zip" .TagName }}
+    files:
+      - from: ./bin/kubevpn.exe
+        to: .
+      - from: LICENSE
+        to: .
+    bin: kubevpn.exe
+  - selector:
+      matchLabels:
+        os: linux
+        arch: amd64
+    {{addURIAndSha "https://github.com/kubenetworks/kubevpn/releases/download/{{ .TagName }}/kubevpn_{{ .TagName }}_linux_amd64.zip" .TagName }}
+    files:
+      - from: ./bin/kubevpn
+        to: .
+      - from: LICENSE
+        to: .
+    bin: kubevpn
+  - selector:
+      matchLabels:
+        os: linux
+        arch: arm64
+    {{addURIAndSha "https://github.com/kubenetworks/kubevpn/releases/download/{{ .TagName }}/kubevpn_{{ .TagName }}_linux_arm64.zip" .TagName }}
+    files:
+      - from: ./bin/kubevpn
+        to: .
+      - from: LICENSE
+        to: .
+    bin: kubevpn
+  - selector:
+      matchLabels:
+        os: linux
+        arch: 386
+    {{addURIAndSha "https://github.com/kubenetworks/kubevpn/releases/download/{{ .TagName }}/kubevpn_{{ .TagName }}_linux_386.zip" .TagName }}
+    files:
+      - from: ./bin/kubevpn
+        to: .
+      - from: LICENSE
+        to: .
+    bin: kubevpn
+  - selector:
+      matchLabels:
+        os: darwin
+        arch: amd64
+    {{addURIAndSha "https://github.com/kubenetworks/kubevpn/releases/download/{{ .TagName }}/kubevpn_{{ .TagName }}_darwin_amd64.zip" .TagName }}
+    files:
+      - from: ./bin/kubevpn
+        to: .
+      - from: LICENSE
+        to: .
+    bin: kubevpn
+  - selector:
+      matchLabels:
+        os: darwin
+        arch: arm64
+    {{addURIAndSha "https://github.com/kubenetworks/kubevpn/releases/download/{{ .TagName }}/kubevpn_{{ .TagName }}_darwin_arm64.zip" .TagName }}
+    files:
+      - from: ./bin/kubevpn
+        to: .
+      - from: LICENSE
+        to: .
+    bin: kubevpn

.github/release-note.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+RELEASE=${RELEASE:-$2}
+PREVIOUS_RELEASE=${PREVIOUS_RELEASE:-$1}
+
+# ref https://stackoverflow.com/questions/1441010/the-shortest-possible-output-from-git-log-containing-author-and-date
+CHANGELOG=$(git log --no-merges --date=short --pretty=format:'- %h %an %ad %s' "${PREVIOUS_RELEASE}".."${RELEASE}")
+
+cat <<EOF
+# KubeVPN release ${RELEASE}
+
+KubeVPN ${RELEASE} is available now ! 🎉
+
+## Download KubeVPN for your platform
+
+**Mac** (x86-64/Intel)
+
+\`\`\`
+curl -Lo kubevpn.zip https://github.com/kubenetworks/kubevpn/releases/download/${RELEASE}/kubevpn_${RELEASE}_darwin_amd64.zip && unzip -d kubevpn kubevpn.zip
+\`\`\`
+
+**Mac** (AArch64/Apple M1 silicon)
+
+\`\`\`
+curl -Lo kubevpn.zip https://github.com/kubenetworks/kubevpn/releases/download/${RELEASE}/kubevpn_${RELEASE}_darwin_arm64.zip && unzip -d kubevpn kubevpn.zip
+\`\`\`
+
+**Linux** (x86-64)
+
+\`\`\`
+curl -Lo kubevpn.zip https://github.com/kubenetworks/kubevpn/releases/download/${RELEASE}/kubevpn_${RELEASE}_linux_amd64.zip && unzip -d kubevpn kubevpn.zip
+\`\`\`
+
+**Linux** (AArch64)
+
+\`\`\`
+curl -Lo kubevpn.zip https://github.com/kubenetworks/kubevpn/releases/download/${RELEASE}/kubevpn_${RELEASE}_linux_arm64.zip && unzip -d kubevpn kubevpn.zip
+\`\`\`
+
+**Linux** (i386)
+
+\`\`\`
+curl -Lo kubevpn.zip https://github.com/kubenetworks/kubevpn/releases/download/${RELEASE}/kubevpn_${RELEASE}_linux_386.zip && unzip -d kubevpn kubevpn.zip
+\`\`\`
+
+**Windows** (x86-64)
+
+\`\`\`
+curl -LO https://github.com/kubenetworks/kubevpn/releases/download/${RELEASE}/kubevpn_${RELEASE}_windows_amd64.zip
+\`\`\`
+
+**Windows** (AArch64)
+
+\`\`\`
+curl -LO https://github.com/kubenetworks/kubevpn/releases/download/${RELEASE}/kubevpn_${RELEASE}_windows_arm64.zip
+\`\`\`
+
+**Windows** (i386)
+
+\`\`\`
+curl -LO https://github.com/kubenetworks/kubevpn/releases/download/${RELEASE}/kubevpn_${RELEASE}_windows_386.zip
+\`\`\`
+
+## Checksums
+
+SHA256 checksums available for compiled binaries.
+Run \`shasum -a 256 -c checksums.txt\` to verify.
+
+## Upgrading
+
+Run \`kubevpn upgrade\` to upgrade from a previous version.
+
+## Changelog
+
+${CHANGELOG}
+EOF

.github/workflows/coverage.yml

@@ -0,0 +1,77 @@
+name: Coverage
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  linux:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.23'
+          check-latest: true
+          
+      - name: Setup Minikube
+        id: minikube
+        timeout-minutes: 30
+        uses: medyagh/setup-minikube@latest
+        with:
+          cache: true
+
+      - name: Kubernetes info
+        run: |
+          kubectl cluster-info
+          cat ~/.kube/config
+          kubectl get pods -n kube-system -o wide
+
+      - name: Install demo bookinfo
+        run: |
+          minikube image load --remote ghcr.io/kubenetworks/examples-bookinfo-details-v1:1.20.2
+          minikube image load --remote ghcr.io/kubenetworks/examples-bookinfo-ratings-v1:1.20.2
+          minikube image load --remote ghcr.io/kubenetworks/examples-bookinfo-reviews-v1:1.20.2
+          minikube image load --remote ghcr.io/kubenetworks/examples-bookinfo-productpage-v1:1.20.2
+          minikube image load --remote ghcr.io/kubenetworks/authors:latest
+          minikube image load --remote ghcr.io/kubenetworks/nginx:latest
+          minikube image ls
+          eval $(minikube docker-env)
+          kubectl apply -f https://raw.githubusercontent.com/kubenetworks/kubevpn/master/samples/bookinfo.yaml
+
+      - name: Build
+        run: |
+          export VERSION=${{github.event.pull_request.head.sha}}
+          if [[ -z "$VERSION" ]]; then
+            export VERSION=${{ github.sha }}
+          fi
+          make kubevpn-linux-amd64
+          chmod +x ./bin/kubevpn
+          cp ./bin/kubevpn /usr/local/bin/kubevpn
+          kubevpn version
+
+      - name: Wait for pods reviews to be ready
+        run: |
+          kubectl wait --for=condition=Ready pods --all --timeout=3600s
+          kubectl get svc -A -o wide
+          kubectl get pod -A -o wide
+          kubectl get all -o wide
+          kubectl get nodes -o yaml
+          ifconfig
+          route -n
+          sudo ln /usr/bin/resolvectl /usr/bin/systemd-resolve
+
+      - name: Test
+        run: make ut
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@0cfda1dd0a4ad9efc75517f399d859cd1ea4ced1 # v4.0.2
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        with:
+          verbose: true
+          slug: wencaiwulue/kubevpn

.github/workflows/release.yml

@@ -11,11 +11,29 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.17
+          go-version: '1.23'
+          check-latest: true
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Push image to docker hub
+        run: |
+          echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USER }} --password-stdin
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+          docker buildx create --use
+          make container
+
+      - name: Release Note
+        run: |
+          RELEASE_VERSION=${GITHUB_REF#refs/*/}
+          PREVERSION=$(git for-each-ref --sort='-creatordate' --format='%(refname:lstrip=2)' --count=50 'refs/tags/*' | grep -v 'rc' | awk 'NR==2')
+          echo ${PREVERSION}
+          echo ${PREVERSION}
+          echo "$(./.github/release-note.sh ${PREVERSION} ${RELEASE_VERSION})" > release_note.md
       - name: Create Release
         id: create_release
         uses: actions/create-release@v1
@@ -24,12 +42,9 @@ jobs:
         with:
           tag_name: ${{ github.ref }}
           release_name: Release ${{ github.ref }}
-          body: |
-            Changes in this Release
-            - Fix known bugs
-            - Optimize code
           draft: false
           prerelease: false
+          body_path: release_note.md
 
       - name: Collect Release Info
         run: |
@@ -38,25 +53,211 @@ jobs:
           git reset --hard
 
       - name: Upload RELEASE_VERSION
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: RELEASE_VERSION
           path: RELEASE_VERSION
 
       - name: Upload UPLOAD_URL
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: UPLOAD_URL
           path: UPLOAD_URL
 
-      - name: Push image to docker hub
-        run: |
-          echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USER }} --password-stdin
-          make all-image
-
       - name: Repository Dispatch
-        uses: peter-evans/repository-dispatch@v1
+        uses: aurelien-baudet/workflow-dispatch@v2
         with:
+          workflow: Upload_release
           token: ${{ secrets.REPOSITORYDISPATCH }}
-          event-type: release-event
-          client-payload: '{"url": "${{ steps.create_release.outputs.upload_url }}", "tag": "${{ github.ref }}"}'
+          inputs: '{"url": "${{ steps.create_release.outputs.upload_url }}", "tag": "${{ github.ref_name }}"}'
+
+      - name: Make changes to pull request
+        run: make version && echo ${GITHUB_REF#refs/*/} > plugins/stable.txt
+      - name: Create Pull Request
+        id: cpr
+        uses: peter-evans/create-pull-request@v4
+        with:
+          add-paths: |
+            *.yaml
+            plugins/stable.txt
+          token: ${{ secrets.REPOSITORYDISPATCH }}
+          commit-message: "feat: update krew index version to ${{ github.ref }}"
+          committer: GitHub <noreply@github.com>
+          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+          signoff: false
+          branch: feat/update-krew-index-version
+          base: master
+          delete-branch: true
+          title: 'feat: update krew index version to ${{ github.ref }}'
+          body: |
+            update report
+            - update with *today's* date
+            - update krew index version to ${{ github.ref }}
+          labels: |
+            report
+            automated pr
+          draft: false
+
+  release-helm-chart:
+    name: Release KubeVPN Helm Chart
+    needs: [ build ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.23'
+          check-latest: true
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Helm tool installer
+        uses: azure/setup-helm@v4
+        with:
+          version: "v3.6.3"
+      - name: Change chart version
+        run: |
+          VERSION=${GITHUB_REF#refs/*/}
+          CHART_VERSION=${VERSION/#v/}
+          sed -i "s/^appVersion:.*$/appVersion: \"${VERSION}\"/;s/^version:.*$/version: ${CHART_VERSION}/" charts/kubevpn/Chart.yaml
+          sed -i "s/tag:.*$/tag: \"${VERSION}\"/" charts/kubevpn/values.yaml
+      - name: Tar chart
+        run: |
+          VERSION=${GITHUB_REF#refs/*/}
+          CHART_VERSION=${VERSION/#v/}
+          tar --transform 's/^charts\/kubevpn/kubevpn/' -zcf kubevpn-${CHART_VERSION}.tgz charts/kubevpn
+          shasum -a 256 kubevpn-${CHART_VERSION}.tgz | awk '{print $1}' > kubevpn-${CHART_VERSION}.tgz-SHA256
+      - name: Download UPLOAD_URL
+        uses: actions/download-artifact@v4
+        with:
+          name: UPLOAD_URL
+      - name: Get Release UPLOAD_URL
+        id: get_release_info
+        run: |
+          UploadUrl=$(cat ./UPLOAD_URL)
+          echo "::set-output name=upload_url::$UploadUrl"
+      - name: Get assert name
+        id: get_assert_info
+        run: |
+          VERSION=${GITHUB_REF#refs/*/}
+          CHART_VERSION=${VERSION/#v/}
+          AssertName=kubevpn-${CHART_VERSION}.tgz
+          echo "::set-output name=assert_name::$AssertName"
+      - name: Get assert SHA256 name
+        id: get_assert_info_sha256
+        run: |
+          VERSION=${GITHUB_REF#refs/*/}
+          CHART_VERSION=${VERSION/#v/}
+          AssertName=kubevpn-${CHART_VERSION}.tgz-SHA256
+          echo "::set-output name=assert_name::$AssertName"
+      - name: Upload Release Asset KubeVPN Server Chart
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.get_release_info.outputs.upload_url }}
+          asset_path: ${{ steps.get_assert_info.outputs.assert_name }}
+          asset_name: ${{ steps.get_assert_info.outputs.assert_name }}
+          asset_content_type: application/octet-stream
+      - name: Upload Release Asset KubeVPN Chart SHA256
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.get_release_info.outputs.upload_url }}
+          asset_path: ${{ steps.get_assert_info_sha256.outputs.assert_name }}
+          asset_name: ${{ steps.get_assert_info_sha256.outputs.assert_name }}
+          asset_content_type: application/octet-stream
+
+  github-pages-deploy:
+    name: Release Helm Chart To branch master
+    permissions:
+      contents: write
+    runs-on: ubuntu-latest
+    needs: release-helm-chart
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.23'
+          check-latest: true
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Configure Git
+        run: |
+          git config user.name "$GITHUB_ACTOR"
+          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+      - name: Change chart version
+        run: |
+          VERSION=${GITHUB_REF#refs/*/}
+          CHART_VERSION=${VERSION/#v/}
+          sed -i "s/^appVersion:.*$/appVersion: \"${VERSION}\"/;s/^version:.*$/version: ${CHART_VERSION}/" charts/kubevpn/Chart.yaml
+          sed -i "s/tag:.*$/tag: \"${VERSION}\"/" charts/kubevpn/values.yaml
+      - name: Package and upload helm chart
+        run: |
+          # download helm chart releaser
+          curl -sSLo cr.tar.gz "https://github.com/helm/chart-releaser/releases/download/v1.6.1/chart-releaser_1.6.1_linux_amd64.tar.gz"
+          tar -xzf cr.tar.gz
+          rm -f cr.tar.gz
+          owner=$(cut -d '/' -f 1 <<< "$GITHUB_REPOSITORY")
+          repo=$(cut -d '/' -f 2 <<< "$GITHUB_REPOSITORY")
+          # package chart
+          ./cr package charts/$repo
+          # update index and push to github pages
+          git config user.email "$owner@users.noreply.github.com"
+          git config user.name "$owner"
+          ./cr index \
+              --owner "$owner" \
+              --git-repo "$repo" \
+              --token "${{ secrets.CREATE_HELM_PR }}" \
+              --release-name-template "v{{ .Version }}" \
+              --index-path ./index.yaml \
+              --charts-repo https://github.com/$owner/$repo \
+              --pages-branch master \
+              --pages-index-path charts/index.yaml \
+              --pr
+  snapcraft:
+    runs-on: ubuntu-24.04
+    env:
+      SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.SNAPCRAFT_TOKEN }}
+    steps:
+      - name: Check out Git repository
+        uses: actions/checkout@v3
+
+      - name: Install Snapcraft
+        uses: samuelmeuli/action-snapcraft@v3
+
+      - name: Setup LXD
+        uses: canonical/setup-lxd@main
+
+      - name: Use Snapcraft
+        run: |
+          RELEASE_VERSION=${GITHUB_REF#refs/*/}
+          sed -i s#CRAFT_ARCH_BUILD_VERSION#$RELEASE_VERSION#g snap/snapcraft.yaml
+          snapcraft
+          snapcraft upload --release=stable kubevpn_${RELEASE_VERSION}_amd64.snap
+  snapcraft-arm:
+    runs-on: ubuntu-24.04-arm
+    env:
+      SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.SNAPCRAFT_TOKEN }}
+    steps:
+      - name: Check out Git repository
+        uses: actions/checkout@v3
+
+      - name: Install Snapcraft
+        uses: samuelmeuli/action-snapcraft@v3
+
+      - name: Setup LXD
+        uses: canonical/setup-lxd@main
+
+      - name: Use Snapcraft
+        run: |
+          RELEASE_VERSION=${GITHUB_REF#refs/*/}
+          sed -i s#CRAFT_ARCH_BUILD_VERSION#$RELEASE_VERSION#g snap/snapcraft.yaml
+          snapcraft
+          snapcraft upload --release=stable kubevpn_${RELEASE_VERSION}_arm64.snap

.github/workflows/test.yml

@@ -7,20 +7,40 @@ on:
     branches: [ master ]
 
 jobs:
-
-  linux:
+  image:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.23'
+          check-latest: true
+      - name: Push image to docker hub
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+          export VERSION=${{github.event.pull_request.head.sha}}
+          if [[ -z "$VERSION" ]]; then
+            export VERSION=${{ github.sha }}
+          fi
+          make container-test
+  linux:
+    runs-on: ubuntu-latest
+    needs: [ "image" ]
+    steps:
+      - uses: actions/checkout@v4
 
       - name: Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.17
-
+          go-version: '1.23'
+          check-latest: true
       - name: Setup Minikube
+        id: minikube
         timeout-minutes: 30
-        uses: medyagh/setup-minikube@master
+        uses: medyagh/setup-minikube@latest
+        with:
+          cache: true
 
       - name: Kubernetes info
         run: |
@@ -28,47 +48,78 @@ jobs:
           cat ~/.kube/config
           kubectl get pods -n kube-system -o wide
       - name: Install demo bookinfo
-        run: kubectl apply -f https://raw.githubusercontent.com/wencaiwulue/kubevpn/master/samples/bookinfo.yaml
+        run: |
+          minikube image load --remote ghcr.io/kubenetworks/examples-bookinfo-details-v1:1.20.2
+          minikube image load --remote ghcr.io/kubenetworks/examples-bookinfo-ratings-v1:1.20.2
+          minikube image load --remote ghcr.io/kubenetworks/examples-bookinfo-reviews-v1:1.20.2
+          minikube image load --remote ghcr.io/kubenetworks/examples-bookinfo-productpage-v1:1.20.2
+          minikube image load --remote ghcr.io/kubenetworks/authors:latest
+          minikube image load --remote ghcr.io/kubenetworks/nginx:latest
+          minikube image ls
+          eval $(minikube docker-env)
+          kubectl apply -f https://raw.githubusercontent.com/kubenetworks/kubevpn/master/samples/bookinfo.yaml
 
       - name: Build
-        run: make kubevpn-linux-amd64
+        run: |
+          export VERSION=${{github.event.pull_request.head.sha}}
+          if [[ -z "$VERSION" ]]; then
+            export VERSION=${{ github.sha }}
+          fi
+          make kubevpn-linux-amd64
+          chmod +x ./bin/kubevpn
+          cp ./bin/kubevpn /usr/local/bin/kubevpn
+          kubevpn version
 
       - name: Wait for pods reviews to be ready
         run: |
-          kubectl wait pods -l app=reviews --for=condition=Ready --timeout=600s
+          kubectl wait --for=condition=Ready pods --all --timeout=3600s
+          kubectl get svc -A -o wide
+          kubectl get pod -A -o wide
           kubectl get all -o wide
           kubectl get nodes -o yaml
           ifconfig
           route -n
+          sudo ln /usr/bin/resolvectl /usr/bin/systemd-resolve
 
       - name: Test
-        run: go test -v ./test/
+        run: make ut
 
   macos:
-    runs-on: macos-latest
+    runs-on: macos-13
+    needs: [ "image" ]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.17
-      - uses: docker-practice/actions-setup-docker@master
-      - name: Pull image in advance
-        run: |
-          rm '/usr/local/bin/kubectl'
-          set -x
-          docker version
-          docker pull naison/kubevpn:v2
-          docker pull naison/kubevpnmesh:v2
-          docker run --rm hello-world
+          go-version: '1.23'
+          check-latest: true
 
+      # https://github.com/crazy-max/ghaction-setup-docker/issues/108
+      - name: Set up QEMU
+        uses: docker/actions-toolkit/.github/actions/macos-setup-qemu@19ca9ade20f5da695f76a10988d6532058575f82
+
+      - name: Set up Docker
+        uses: docker/setup-docker-action@v4
+        with:
+          daemon-config: |
+            {
+              "debug": true,
+              "features": {
+                "containerd-snapshotter": true
+              }
+            }
+      - uses: azure/setup-kubectl@v4
       - name: Install minikube
         run: |
+          set -x
+          docker version
           brew install minikube
-          minikube start --driver=docker
-          kubectl get po -A
-          minikube kubectl -- get po -A
+          minikube start --driver=docker --memory=max --cpus=max --wait=all --wait-timeout=60m
+          kubectl cluster-info
+          kubectl config view --flatten --raw
+          kubectl get pod -A -o wide
 
       - name: Kubernetes info
         run: |
@@ -77,58 +128,70 @@ jobs:
           kubectl get pods -n kube-system -o wide
 
       - name: Install demo bookinfo
-        run: kubectl apply -f https://raw.githubusercontent.com/wencaiwulue/kubevpn/master/samples/bookinfo.yaml
+        run: |
+          kubectl apply -f https://raw.githubusercontent.com/kubenetworks/kubevpn/master/samples/bookinfo.yaml
 
       - name: Build
-        run: make kubevpn-darwin-amd64
+        run: |
+          export VERSION=${{github.event.pull_request.head.sha}}
+          if [[ -z "$VERSION" ]]; then
+            export VERSION=${{ github.sha }}
+          fi
+          make kubevpn-darwin-amd64
+          chmod +x ./bin/kubevpn
+          cp ./bin/kubevpn /usr/local/bin/kubevpn
+          kubevpn version
 
       - name: Wait for pods reviews to be ready
         run: |
-          kubectl wait pods -l app=reviews --for=condition=Ready --timeout=600s
-          kubectl get all -o wide
-          kubectl get nodes -o yaml
+          kubectl wait --for=condition=Ready pods --all --timeout=3600s
+          kubectl get svc -A -o wide || true
+          kubectl get pod -A -o wide || true
+          kubectl get all -o wide || true
+          kubectl get nodes -o yaml || true
           ifconfig
           netstat -anr
 
       - name: Test
-        run: go test -v ./test/
+        run: make ut
 
-#  windows:
-#    runs-on: windows-latest
-#    steps:
-#      - uses: actions/checkout@v2
-#
-#      - name: Set up Go
-#        uses: actions/setup-go@v2
-#        with:
-#          go-version: 1.17
-#      #      - run: |
-#      #          choco install docker-desktop
-#      #          docker version
-#      #          docker run --rm hello-world
-#      - run: |
-#          choco install virtualbox
-#          choco install minikube
-#          minikube start --driver=virtualbox
-#          minikube kubectl -- get po -A
-#          choco install make
-#      - name: Kubernetes info
-#        run: |
-#          kubectl cluster-info dump
-#          kubectl get pods -n kube-system -o wide
-#      - name: Install demo bookinfo
-#        run: kubectl apply -f https://raw.githubusercontent.com/wencaiwulue/kubevpn/master/samples/bookinfo.yaml
-#
-#      - name: Build
-#        run: make kubevpn-windows
-#
-#      - name: Wait for pods reviews to be ready
-#        run: |
-#          kubectl wait pods -l app=reviews --for=condition=Ready --timeout=600s
-#          kubectl get all -o wide
-#          kubectl get nodes -o yaml
-#          ipconfig
-#
-#      - name: Test
-#        run: go test -v ./test/
+  windows:
+    runs-on: windows-latest
+    env:
+      VERSION: ${{ github.event.pull_request.head.sha || github.sha }}
+    needs: [ "image" ]
+    steps:
+      - uses: actions/checkout@v4
 
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.23'
+
+      - name: Set up Docker
+        uses: docker/setup-docker-action@v4
+        with:
+          daemon-config: |
+            {
+              "debug": true,
+              "features": {
+                "containerd-snapshotter": true
+              }
+            }
+      - run: |
+          docker info --format '{{.OSType}}'
+          choco install kind
+          kind create cluster
+          kubectl cluster-info
+          kubectl config view --flatten --raw
+
+      - run: |
+          choco install minikube
+          minikube start --driver=docker
+          choco install make
+
+      - name: Build
+        run: |
+          make kubevpn-windows-amd64
+          ./bin/kubevpn.exe version
+          ./bin/kubevpn.exe status

.github/workflows/upload_release.yml

@@ -1,8 +1,14 @@
-name: Upload release
+name: Upload_release
 
 on:
-  repository_dispatch:
-    types: [ release-event ]
+  workflow_dispatch:
+    inputs:
+      url:
+        description: 'github release url'
+        required: true
+      tag:
+        description: 'latest tag'
+        required: true
 
 jobs:
   build:
@@ -10,35 +16,40 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        os-arch: [
-          kubevpn-darwin-amd64,
-          kubevpn-darwin-arm64,
-          kubevpn-windows-amd64.exe,
-          kubevpn-windows-arm64.exe,
-          kubevpn-windows-386.exe,
-          kubevpn-linux-amd64,
-          kubevpn-linux-arm64,
-          kubevpn-linux-386,
-        ]
+        os: [ darwin, windows, linux ]
+        arch: [ amd64, arm64, 386 ]
+        exclude:
+          - os: darwin
+            arch: 386
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.17
+          go-version: '1.23'
+          check-latest: true
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
-      - name: Build kubevpn-all-arch
+      - name: Build kubevpn
         run: |
-          git tag ${{ github.event.client_payload.tag }} || true
-          make all-kubevpn
+          git tag ${{ github.event.inputs.tag }} || true
+          export GitHubOAuthToken=${{ secrets.KUBEVPN_UPGRADE_OAUTH }}
+          make kubevpn-${{ matrix.os }}-${{ matrix.arch }}
+          SUFFIX=""
+          if [ "${{ matrix.os }}" = "windows" ]; then
+            SUFFIX=".exe"
+          fi
+          
+          shasum -a 256 ./bin/kubevpn${SUFFIX} | awk '{print $1}' > checksums.txt
+          zip -r kubevpn_${{ github.event.inputs.tag }}_${{ matrix.os }}_${{ matrix.arch }}.zip ./bin/kubevpn${SUFFIX} LICENSE README.md README_ZH.md checksums.txt
 
       - name: Upload Release Asset
         uses: actions/upload-release-asset@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          upload_url: ${{ github.event.client_payload.url }}
-          asset_path: ${{ matrix.os-arch }}
-          asset_name: ${{ matrix.os-arch }}
-          asset_content_type: application/octet-stream
+          upload_url: ${{ github.event.inputs.url }}
+          asset_path: ./kubevpn_${{ github.event.inputs.tag }}_${{ matrix.os }}_${{ matrix.arch }}.zip
+          asset_name: kubevpn_${{ github.event.inputs.tag }}_${{ matrix.os }}_${{ matrix.arch }}.zip
+          asset_content_type: application/zip
+

.gitignore

@@ -1,8 +1,6 @@
 # Binaries for programs and plugins
 *.exe
 *.exe~
-*.dll
-*.so
 *.dylib
 
 # Test binary, built with `go test -c`
@@ -19,4 +17,6 @@
 .vscode/
 
 # Build artifacts
-kubevpn
+bin
+
+*.DS_Store

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 wencaiwulue
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -1,97 +1,114 @@
-# These are the values we want to pass for VERSION and BUILD
-VERSION := $(shell git tag -l --sort=v:refname | tail -1)
-GIT_COMMIT := $(shell git describe --match=NeVeRmAtCh --always --abbrev=40)
-BUILD_TIME := $(shell date +"%Y-%m-%dT%H:%M:%SZ")
-BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
+VERSION ?= $(shell git tag -l --sort=v:refname | tail -1)
+GIT_COMMIT ?= $(shell git describe --match=NeVeRmAtCh --always --abbrev=7)
+BUILD_TIME ?= $(shell date +"%Y-%m-%dT%H:%M:%SZ")
+BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
 
 GOOS := $(shell go env GOHOSTOS)
 GOARCH := $(shell go env GOHOSTARCH)
 TARGET := kubevpn-${GOOS}-${GOARCH}
 OS_ARCH := ${GOOS}/${GOARCH}
 
-FOLDER := github.com/wencaiwulue/kubevpn/cmd/kubevpn
-CONTROL_PLANE_FOLDER := github.com/wencaiwulue/kubevpn/pkg/controlplane/cmd/server
+BASE := github.com/wencaiwulue/kubevpn/v2
+FOLDER := ${BASE}/cmd/kubevpn
+BUILD_DIR ?= ./build
+OUTPUT_DIR ?= ./bin
+REGISTRY ?= docker.io
+NAMESPACE ?= naison
+REPOSITORY ?= kubevpn
+IMAGE ?= $(REGISTRY)/$(NAMESPACE)/$(REPOSITORY):$(VERSION)
+IMAGE_LATEST ?= docker.io/naison/kubevpn:latest
+IMAGE_GH ?= ghcr.io/kubenetworks/kubevpn:$(VERSION)
+IMAGE_GH_LATEST ?= ghcr.io/kubenetworks/kubevpn:latest
 
 # Setup the -ldflags option for go build here, interpolate the variable values
-LDFLAGS=--ldflags "-w -s \
- -X ${FOLDER}/cmds.Version=${VERSION} \
+LDFLAGS=--ldflags "-s -w\
+ -X ${BASE}/pkg/config.Image=${IMAGE_GH} \
+ -X ${BASE}/pkg/config.Version=${VERSION} \
+ -X ${BASE}/pkg/config.GitCommit=${GIT_COMMIT} \
+ -X ${BASE}/pkg/config.GitHubOAuthToken=${GitHubOAuthToken} \
  -X ${FOLDER}/cmds.BuildTime=${BUILD_TIME} \
- -X ${FOLDER}/cmds.GitCommit=${GIT_COMMIT} \
  -X ${FOLDER}/cmds.Branch=${BRANCH} \
  -X ${FOLDER}/cmds.OsArch=${OS_ARCH} \
 "
 
-.PHONY: all
-all: all-kubevpn all-image
+GO111MODULE=on
+GOPROXY=https://goproxy.cn,direct
 
-.PHONY: all-kubevpn
-all-kubevpn: kubevpn-darwin-amd64 kubevpn-darwin-arm64 \
+.PHONY: all
+all: kubevpn-all container
+
+.PHONY: kubevpn-all
+kubevpn-all: kubevpn-darwin-amd64 kubevpn-darwin-arm64 \
 kubevpn-windows-amd64 kubevpn-windows-386 kubevpn-windows-arm64 \
 kubevpn-linux-amd64 kubevpn-linux-386 kubevpn-linux-arm64
 
-.PHONY: all-image
-all-image: image image-mesh image-control-plane
+.PHONY: kubevpn
+kubevpn:
+	make $(TARGET)
 
 # ---------darwin-----------
 .PHONY: kubevpn-darwin-amd64
 kubevpn-darwin-amd64:
-	CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build ${LDFLAGS} -o kubevpn-darwin-amd64 ${FOLDER}
-	chmod +x kubevpn-darwin-amd64
-	cp kubevpn-darwin-amd64 /usr/local/bin/kubevpn
+	CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build ${LDFLAGS} -o $(OUTPUT_DIR)/kubevpn ${FOLDER}
+	chmod +x $(OUTPUT_DIR)/kubevpn
 .PHONY: kubevpn-darwin-arm64
 kubevpn-darwin-arm64:
-	CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build ${LDFLAGS} -o kubevpn-darwin-arm64 ${FOLDER}
-	chmod +x kubevpn-darwin-arm64
-	cp kubevpn-darwin-arm64 /usr/local/bin/kubevpn
+	CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build ${LDFLAGS} -o $(OUTPUT_DIR)/kubevpn ${FOLDER}
+	chmod +x $(OUTPUT_DIR)/kubevpn
 # ---------darwin-----------
 
 # ---------windows-----------
 .PHONY: kubevpn-windows-amd64
 kubevpn-windows-amd64:
-	CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build ${LDFLAGS} -o kubevpn-windows-amd64.exe ${FOLDER}
+	CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build ${LDFLAGS} -o $(OUTPUT_DIR)/kubevpn.exe ${FOLDER}
 .PHONY: kubevpn-windows-arm64
 kubevpn-windows-arm64:
-	CGO_ENABLED=0 GOOS=windows GOARCH=arm64 go build ${LDFLAGS} -o kubevpn-windows-arm64.exe ${FOLDER}
+	CGO_ENABLED=0 GOOS=windows GOARCH=arm64 go build ${LDFLAGS} -o $(OUTPUT_DIR)/kubevpn.exe ${FOLDER}
 .PHONY: kubevpn-windows-386
 kubevpn-windows-386:
-	CGO_ENABLED=0 GOOS=windows GOARCH=386 go build ${LDFLAGS} -o kubevpn-windows-386.exe ${FOLDER}
+	CGO_ENABLED=0 GOOS=windows GOARCH=386 go build ${LDFLAGS} -o $(OUTPUT_DIR)/kubevpn.exe ${FOLDER}
 # ---------windows-----------
 
 # ---------linux-----------
 .PHONY: kubevpn-linux-amd64
 kubevpn-linux-amd64:
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ${LDFLAGS} -o kubevpn-linux-amd64 ${FOLDER}
-	chmod +x kubevpn-linux-amd64
-	cp kubevpn-linux-amd64 /usr/local/bin/kubevpn
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ${LDFLAGS} -o $(OUTPUT_DIR)/kubevpn ${FOLDER}
+	chmod +x $(OUTPUT_DIR)/kubevpn
 .PHONY: kubevpn-linux-arm64
 kubevpn-linux-arm64:
-	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build ${LDFLAGS} -o kubevpn-linux-arm64 ${FOLDER}
-	chmod +x kubevpn-linux-arm64
-	cp kubevpn-linux-arm64 /usr/local/bin/kubevpn
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build ${LDFLAGS} -o $(OUTPUT_DIR)/kubevpn ${FOLDER}
+	chmod +x $(OUTPUT_DIR)/kubevpn
 .PHONY: kubevpn-linux-386
 kubevpn-linux-386:
-	CGO_ENABLED=0 GOOS=linux GOARCH=386 go build ${LDFLAGS} -o kubevpn-linux-386 ${FOLDER}
-	chmod +x kubevpn-linux-386
-	cp kubevpn-linux-386 /usr/local/bin/kubevpn
+	CGO_ENABLED=0 GOOS=linux GOARCH=386 go build ${LDFLAGS} -o $(OUTPUT_DIR)/kubevpn ${FOLDER}
+	chmod +x $(OUTPUT_DIR)/kubevpn
 # ---------linux-----------
 
-.PHONY: image
-image: kubevpn-linux-amd64
-	mv kubevpn-linux-amd64 kubevpn
-	docker build -t naison/kubevpn:v2 -f ./dockerfile/server/Dockerfile .
-	rm -fr kubevpn
-	docker push naison/kubevpn:v2
+.PHONY: container
+container:
+	docker buildx build --platform linux/amd64,linux/arm64 -t ${IMAGE} -t ${IMAGE_LATEST} -t ${IMAGE_GH} -t ${IMAGE_GH_LATEST} -f $(BUILD_DIR)/Dockerfile --push .
 
-.PHONY: image-mesh
-image-mesh:
-	docker build -t naison/kubevpnmesh:v2 -f ./dockerfile/mesh/Dockerfile .
-	docker push naison/kubevpnmesh:v2
+############################ build local
+.PHONY: container-local
+container-local: kubevpn-linux-amd64
+	docker buildx build --platform linux/amd64,linux/arm64 -t ${IMAGE_LATEST} -t ${IMAGE_GH_LATEST} -f $(BUILD_DIR)/local.Dockerfile --push .
 
-.PHONY: image-control-plane
-image-control-plane:
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o envoy-xds-server ${CONTROL_PLANE_FOLDER}
-	chmod +x envoy-xds-server
-	docker build -t naison/envoy-xds-server:latest -f ./dockerfile/controlplane/Dockerfile .
-	rm -fr envoy-xds-server
-	docker push naison/envoy-xds-server:latest
+.PHONY: container-test
+container-test: kubevpn-linux-amd64
+	docker build -t ${IMAGE_GH} -f $(BUILD_DIR)/test.Dockerfile --push .
 
+.PHONY: version
+version:
+	go run ${BASE}/pkg/util/krew
+
+.PHONY: gen
+gen:
+	go generate ./...
+
+.PHONY: ut
+ut:
+	go test -p=1 -v -timeout=60m -coverprofile=coverage.txt -coverpkg=./... ./...
+
+.PHONY: cover
+cover: ut
+	go tool cover -html=coverage.txt

README.md

@@ -1,93 +1,204 @@
+![kubevpn](https://raw.githubusercontent.com/wencaiwulue/kubevpn/master/samples/flat_log.png)
+
+[![GitHub Workflow][1]](https://github.com/kubenetworks/kubevpn/actions)
+[![Go Version][2]](https://github.com/kubenetworks/kubevpn/blob/master/go.mod)
+[![Go Report][3]](https://goreportcard.com/report/github.com/wencaiwulue/kubevpn)
+[![Maintainability][4]](https://codeclimate.com/github/kubenetworks/kubevpn/maintainability)
+[![GitHub License][5]](https://github.com/kubenetworks/kubevpn/blob/main/LICENSE)
+[![Docker Pulls][6]](https://hub.docker.com/r/naison/kubevpn)
+[![Releases][7]](https://github.com/kubenetworks/kubevpn/releases)
+[![GoDoc](https://godoc.org/github.com/kubenetworks/kubevpn?status.png)](https://pkg.go.dev/github.com/wencaiwulue/kubevpn/v2)
+[![codecov](https://codecov.io/gh/wencaiwulue/kubevpn/graph/badge.svg?token=KMDSINSDEP)](https://codecov.io/gh/wencaiwulue/kubevpn)
+[![Snapcraft](https://snapcraft.io/kubevpn/badge.svg)](https://snapcraft.io/kubevpn)
+
+[1]: https://img.shields.io/github/actions/workflow/status/kubenetworks/kubevpn/release.yml?logo=github
+
+[2]: https://img.shields.io/github/go-mod/go-version/kubenetworks/kubevpn?logo=go
+
+[3]: https://goreportcard.com/badge/github.com/wencaiwulue/kubevpn?style=flat
+
+[4]: https://api.codeclimate.com/v1/badges/b5b30239174fc6603aca/maintainability
+
+[5]: https://img.shields.io/github/license/kubenetworks/kubevpn
+
+[6]: https://img.shields.io/docker/pulls/naison/kubevpn?logo=docker
+
+[7]: https://img.shields.io/github/v/release/kubenetworks/kubevpn?logo=smartthings
+
 # KubeVPN
 
-[中文](README_ZH.md) | [English](README.md) | [Wiki](https://github.com/wencaiwulue/kubevpn/wiki/Architecture)
+[中文](README_ZH.md) | [English](README.md) | [Wiki](https://github.com/kubenetworks/kubevpn/wiki/Architecture)
 
-A tools which can connect to kubernetes cluster network, you can access remote kubernetes cluster network, remote
-kubernetes cluster service can also access your local service
+KubeVPN offers a Cloud-Native Dev Environment that seamlessly connects to your Kubernetes cluster network.
+
+Gain access to the Kubernetes cluster network effortlessly using service names or Pod IP/Service IP. Facilitate the
+interception of inbound traffic from remote Kubernetes cluster services to your local PC through a service mesh and
+more.
+
+For instance, you have the flexibility to run your Kubernetes pod within a local Docker container, ensuring an identical
+environment, volume, and network setup.
+With KubeVPN, empower yourself to develop applications entirely on your local PC!
+
+![arch](docs/en/images/kubevpn-proxy-tun-arch.svg)
+
+## Content
+
+1. [QuickStart](./README.md#quickstart)
+2. [Functions](./README.md#functions)
+3. [Architecture](./README.md#architecture)
+4. [Contributions](./README.md#Contributions)
 
 ## QuickStart
 
-```shell
-git clone https://github.com/wencaiwulue/kubevpn.git
-cd kubevpn
-make kubevpn-linux-amd64
-make kubevpn-darwin-amd64
-make kubevpn-windows-amd64
-```
-
-if you are using windows, you can build by this command:
+### Install from script ( macOS / Linux)
 
 ```shell
-go build github.com/wencaiwulue/kubevpn/cmd/kubevpn -o kubevpn.exe
+curl -fsSL https://kubevpn.dev/install.sh | sh
 ```
 
-if you installed Go 1.16+, you can use install it by this command directly:
+
+### Install from [brew](https://brew.sh/) (macOS / Linux)
 
 ```shell
-go install github.com/wencaiwulue/kubevpn/cmd/kubevpn@latest
+brew install kubevpn
 ```
 
+### Install from [snap](https://snapcraft.io/kubevpn) (Linux)
+
+```shell
+sudo snap install kubevpn
+```
+
+### Install from [scoop](https://scoop.sh/) (Windows)
+
+```shell
+scoop bucket add extras
+scoop install kubevpn
+```
+
+### Install from [krew](https://krew.sigs.k8s.io/) (Windows / macOS / Linux)
+
+```shell
+kubectl krew index add kubevpn https://github.com/kubenetworks/kubevpn.git
+kubectl krew install kubevpn/kubevpn 
+kubectl kubevpn 
+```
+
+### Install from GitHub release (Windows / macOS / Linux)
+
+[https://github.com/kubenetworks/kubevpn/releases/latest](https://github.com/kubenetworks/kubevpn/releases/latest)
+
 ### Install bookinfo as demo application
 
 ```shell
-kubectl apply -f https://raw.githubusercontent.com/wencaiwulue/kubevpn/master/samples/bookinfo.yaml
+kubectl apply -f https://raw.githubusercontent.com/kubenetworks/kubevpn/master/samples/bookinfo.yaml
+```
+
+For clean up after test
+
+```shell
+kubectl delete -f https://raw.githubusercontent.com/kubenetworks/kubevpn/master/samples/bookinfo.yaml
 ```
 
 ## Functions
 
 ### Connect to k8s cluster network
 
+use command `kubevpn connect` connect to k8s cluster network, prompt `Password:` need to input computer
+password. to enable root operation (create a tun device).
+
 ```shell
 ➜  ~ kubevpn connect
-INFO[0000] [sudo kubevpn connect]
 Password:
-2022/02/05 12:09:22 connect.go:303: kubeconfig path: /Users/naison/.kube/config, namespace: default, services: []
-2022/02/05 12:09:28 remote.go:47: traffic manager not exist, try to create it...
-2022/02/05 12:09:28 remote.go:121: pod kubevpn.traffic.manager status is Pending
-2022/02/05 12:09:29 remote.go:121: pod kubevpn.traffic.manager status is Running
-Forwarding from 0.0.0.0:10800 -> 10800
-2022/02/05 12:09:31 connect.go:171: port forward ready
-2022/02/05 12:09:31 connect.go:193: your ip is 223.254.254.176
-2022/02/05 12:09:31 connect.go:197: tunnel connected
-Handling connection for 10800
-2022/02/05 12:09:31 connect.go:211: dns service ok
+Starting connect
+Getting network CIDR from cluster info...
+Getting network CIDR from CNI...
+Getting network CIDR from services...
+Labeling Namespace default
+Creating ServiceAccount kubevpn-traffic-manager
+Creating Roles kubevpn-traffic-manager
+Creating RoleBinding kubevpn-traffic-manager
+Creating Service kubevpn-traffic-manager
+Creating MutatingWebhookConfiguration kubevpn-traffic-manager
+Creating Deployment kubevpn-traffic-manager
+
+Pod kubevpn-traffic-manager-66d969fd45-9zlbp is Pending
+Container     Reason            Message
+control-plane ContainerCreating
+vpn           ContainerCreating
+webhook       ContainerCreating
+
+Pod kubevpn-traffic-manager-66d969fd45-9zlbp is Running
+Container     Reason           Message
+control-plane ContainerRunning
+vpn           ContainerRunning
+webhook       ContainerRunning
+
+Forwarding port...
+Connected tunnel
+Adding route...
+Configured DNS service
++----------------------------------------------------------+
+| Now you can access resources in the kubernetes cluster ! |
++----------------------------------------------------------+
+➜  ~
 ```
 
+already connected to cluster network, use command `kubevpn status` to check status
+
+```shell
+➜  ~ kubevpn status
+ID Mode Cluster               Kubeconfig                 Namespace Status
+0  full ccijorbccotmqodvr189g /Users/naison/.kube/config default   Connected
+➜  ~
+```
+
+use pod `productpage-788df7ff7f-jpkcs` IP `172.29.2.134`
+
 ```shell
 ➜  ~ kubectl get pods -o wide
-NAME                          READY   STATUS      RESTARTS   AGE     IP             NODE          NOMINATED NODE   READINESS GATES
-details-7db5668668-mq9qr      1/1     Running     0          7m      172.27.0.199   172.30.0.14   <none>           <none>
-kubevpn.traffic.manager       1/1     Running     0          74s     172.27.0.207   172.30.0.14   <none>           <none>
-productpage-8f9d86644-z8snh   1/1     Running     0          6m59s   172.27.0.206   172.30.0.14   <none>           <none>
-ratings-859b96848d-68d7n      1/1     Running     0          6m59s   172.27.0.201   172.30.0.14   <none>           <none>
-reviews-dcf754f9d-46l4j       1/1     Running     0          6m59s   172.27.0.202   172.30.0.14   <none>           <none>
+NAME                                       READY   STATUS             RESTARTS   AGE     IP                NODE              NOMINATED NODE   READINESS GATES
+authors-dbb57d856-mbgqk                    3/3     Running            0          7d23h   172.29.2.132      192.168.0.5       <none>           <none>
+details-7d8b5f6bcf-hcl4t                   1/1     Running            0          61d     172.29.0.77       192.168.104.255   <none>           <none>
+kubevpn-traffic-manager-66d969fd45-9zlbp   3/3     Running            0          74s     172.29.2.136      192.168.0.5       <none>           <none>
+productpage-788df7ff7f-jpkcs               1/1     Running            0          61d     172.29.2.134      192.168.0.5       <none>           <none>
+ratings-77b6cd4499-zvl6c                   1/1     Running            0          61d     172.29.0.86       192.168.104.255   <none>           <none>
+reviews-85c88894d9-vgkxd                   1/1     Running            0          24d     172.29.2.249      192.168.0.5       <none>           <none>
 ```
 
+use `ping` to test connection, seems good
+
 ```shell
-➜  ~ ping 172.27.0.206
-PING 172.27.0.206 (172.27.0.206): 56 data bytes
-64 bytes from 172.27.0.206: icmp_seq=0 ttl=63 time=49.563 ms
-64 bytes from 172.27.0.206: icmp_seq=1 ttl=63 time=43.014 ms
-64 bytes from 172.27.0.206: icmp_seq=2 ttl=63 time=43.841 ms
-64 bytes from 172.27.0.206: icmp_seq=3 ttl=63 time=44.004 ms
-64 bytes from 172.27.0.206: icmp_seq=4 ttl=63 time=43.484 ms
+➜  ~ ping 172.29.2.134
+PING 172.29.2.134 (172.29.2.134): 56 data bytes
+64 bytes from 172.29.2.134: icmp_seq=0 ttl=63 time=55.727 ms
+64 bytes from 172.29.2.134: icmp_seq=1 ttl=63 time=56.270 ms
+64 bytes from 172.29.2.134: icmp_seq=2 ttl=63 time=55.228 ms
+64 bytes from 172.29.2.134: icmp_seq=3 ttl=63 time=54.293 ms
 ^C
---- 172.27.0.206 ping statistics ---
-5 packets transmitted, 5 packets received, 0.0% packet loss
-round-trip min/avg/max/stddev = 43.014/44.781/49.563/2.415 ms
+--- 172.29.2.134 ping statistics ---
+4 packets transmitted, 4 packets received, 0.0% packet loss
+round-trip min/avg/max/stddev = 54.293/55.380/56.270/0.728 ms
 ```
 
+use service `productpage` IP `172.21.10.49`
+
 ```shell
 ➜  ~ kubectl get services -o wide
-NAME          TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGE     SELECTOR
-details       ClusterIP   172.27.255.92    <none>        9080/TCP   9m7s    app=details
-productpage   ClusterIP   172.27.255.48    <none>        9080/TCP   9m6s    app=productpage
-ratings       ClusterIP   172.27.255.154   <none>        9080/TCP   9m7s    app=ratings
-reviews       ClusterIP   172.27.255.155   <none>        9080/TCP   9m6s    app=reviews
+NAME                      TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                              AGE     SELECTOR
+authors                   ClusterIP   172.21.5.160    <none>        9080/TCP                             114d    app=authors
+details                   ClusterIP   172.21.6.183    <none>        9080/TCP                             114d    app=details
+kubernetes                ClusterIP   172.21.0.1      <none>        443/TCP                              319d    <none>
+kubevpn-traffic-manager   ClusterIP   172.21.2.86     <none>        8422/UDP,10800/TCP,9002/TCP,80/TCP   2m28s   app=kubevpn-traffic-manager
+productpage               ClusterIP   172.21.10.49    <none>        9080/TCP                             114d    app=productpage
+ratings                   ClusterIP   172.21.3.247    <none>        9080/TCP                             114d    app=ratings
+reviews                   ClusterIP   172.21.8.24     <none>        9080/TCP                             114d    app=reviews
 ```
 
+use command `curl` to test service connection
+
 ```shell
-➜  ~ curl 172.27.255.48:9080
+➜  ~ curl 172.21.10.49:9080
 <!DOCTYPE html>
 <html>
   <head>
@@ -97,8 +208,18 @@ reviews       ClusterIP   172.27.255.155   <none>        9080/TCP   9m6s    app=
 <meta name="viewport" content="width=device-width, initial-scale=1">
 ```
 
+seems good too~
+
 ### Domain resolve
 
+support k8s dns name resolve.
+
+a Pod/Service named `productpage` in the `default` namespace can successfully resolve by following name:
+
+- `productpage`
+- `productpage.default`
+- `productpage.default.svc.cluster.local`
+
 ```shell
 ➜  ~ curl productpage.default.svc.cluster.local:9080
 <!DOCTYPE html>
@@ -112,6 +233,9 @@ reviews       ClusterIP   172.27.255.155   <none>        9080/TCP   9m6s    app=
 
 ### Short domain resolve
 
+To access the service in the cluster, service name or you can use the short domain name, such
+as `productpage`
+
 ```shell
 ➜  ~ curl productpage:9080
 <!DOCTYPE html>
@@ -123,31 +247,73 @@ reviews       ClusterIP   172.27.255.155   <none>        9080/TCP   9m6s    app=
 ...
 ```
 
-### Reverse proxy
+***Disclaimer:*** This only works on the namespace where kubevpn-traffic-manager is deployed. Otherwise,
+use [Domain resolve](./README.md#domain-resolve)
+
+### Connect to multiple kubernetes cluster network
+
+- Mode `lite`: can connect to multiple cluster network, design for only connecting to multiple cluster network.
+- Mode `Full`: not only connect to cluster network, it also supports proxy workloads inbound traffic to local PC.
+
+already connected cluster `ccijorbccotmqodvr189g` with mode `full`
 
 ```shell
-➜  ~ kubevpn connect --workloads=service/productpage
-INFO[0000] [sudo kubevpn connect --workloads=service/productpage]
-Password:
-2022/02/05 12:18:22 connect.go:303: kubeconfig path: /Users/naison/.kube/config, namespace: default, services: [service/productpage]
-2022/02/05 12:18:28 remote.go:47: traffic manager not exist, try to create it...
-2022/02/05 12:18:28 remote.go:121: pod kubevpn.traffic.manager status is Pending
-2022/02/05 12:18:29 remote.go:121: pod kubevpn.traffic.manager status is Running
-Waiting for deployment "productpage" rollout to finish: 1 old replicas are pending termination...
-Waiting for deployment "productpage" rollout to finish: 1 old replicas are pending termination...
-deployment "productpage" successfully rolled out
-Forwarding from 0.0.0.0:10800 -> 10800
-2022/02/05 12:18:34 connect.go:171: port forward ready
-2022/02/05 12:18:34 connect.go:193: your ip is 223.254.254.176
-2022/02/05 12:18:34 connect.go:197: tunnel connected
-Handling connection for 10800
-2022/02/05 12:18:35 connect.go:211: dns service ok
+➜  ~ kubevpn status
+ID Mode Cluster               Kubeconfig                 Namespace Status
+0  full ccijorbccotmqodvr189g /Users/naison/.kube/config default   Connected
 ```
 
+then connect to another cluster `ccidd77aam2dtnc3qnddg` with mode `lite`
+
+```shell
+➜  ~ kubevpn connect -n default --kubeconfig ~/.kube/dev_config --lite
+Starting connect
+Got network CIDR from cache
+Use exist traffic manager
+Forwarding port...
+Connected tunnel
+Adding route...
+Configured DNS service
++----------------------------------------------------------+
+| Now you can access resources in the kubernetes cluster ! |
++----------------------------------------------------------+
+```
+
+use command `kubevpn status` to check connection status
+
+```shell
+➜  ~ kubevpn status
+ID Mode Cluster               Kubeconfig                     Namespace Status
+0  full ccijorbccotmqodvr189g /Users/naison/.kube/config     default   Connected
+1  lite ccidd77aam2dtnc3qnddg /Users/naison/.kube/dev_config default   Connected
+➜  ~
+```
+
+### Reverse proxy
+
+use command `kubevpn proxy` to proxy all inbound traffic to local computer.
+
+```shell
+➜  ~ kubevpn proxy deployment/productpage
+Connected to cluster
+Injecting inbound sidecar for deployment/productpage
+Checking rollout status for deployment/productpage
+Waiting for deployment "productpage" rollout to finish: 1 old replicas are pending termination...
+Waiting for deployment "productpage" rollout to finish: 1 old replicas are pending termination...
+Rollout successfully for deployment/productpage
++----------------------------------------------------------+
+| Now you can access resources in the kubernetes cluster ! |
++----------------------------------------------------------+
+➜  ~
+```
+
+For local testing, save the following code as `hello.go`
+
 ```go
 package main
 
 import (
+	"fmt"
 	"io"
 	"net/http"
 )
@@ -155,11 +321,56 @@ import (
 func main() {
 	http.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
 		_, _ = io.WriteString(writer, "Hello world!")
+		fmt.Printf(">>Received request: %s %s from %s\n", request.Method, request.RequestURI, request.RemoteAddr)
 	})
 	_ = http.ListenAndServe(":9080", nil)
 }
 ```
 
+and compile it
+
+```
+go build hello.go
+```
+
+then run it
+
+```
+./hello &
+```
+
+```shell
+export selector=productpage
+export pod=`kubectl get pods -l app=${selector} -n default -o jsonpath='{.items[0].metadata.name}'`
+export pod_ip=`kubectl get pod $pod -n default -o jsonpath='{.status.podIP}'`
+curl -v -H "foo: bar" http://$pod_ip:9080/health
+```
+
+response would like below
+
+```
+❯ curl -v -H "foo: bar" http://$pod_ip:9080/health
+*   Trying 192.168.72.77:9080...
+* Connected to 192.168.72.77 (192.168.72.77) port 9080 (#0)
+> GET /health HTTP/1.1
+> Host: 192.168.72.77:9080
+> User-Agent: curl/7.87.0
+> Accept: */*
+> foo: bar
+> 
+>>Received request: GET /health from xxx.xxx.xxx.xxx:52974
+* Mark bundle as not supporting multiuse
+< HTTP/1.1 200 OK
+< Date: Sat, 04 Nov 2023 10:19:50 GMT
+< Content-Length: 12
+< Content-Type: text/plain; charset=utf-8
+< 
+* Connection #0 to host 192.168.72.77 left intact
+Hello world!
+```
+
+also you can access via service name
+
 ```shell
 ➜  ~ curl productpage:9080
 Hello world!%
@@ -169,26 +380,24 @@ Hello world!%
 
 ### Reverse proxy with mesh
 
-Only support HTTP and GRPC, with specific header `"a: 1"` will route to your local machine
+Support HTTP, GRPC and WebSocket etc. with specific header `"foo: bar"` will route to your local machine
 
 ```shell
-➜  ~ kubevpn connect --workloads=service/productpage --mode=mesh --headers a=1
-INFO[0000] [sudo kubevpn connect --workloads=service/productpage --mode=mesh --headers a=1]
-2022/02/05 12:22:28 connect.go:303: kubeconfig path: /Users/naison/.kube/config, namespace: default, services: [service/productpage]
-2022/02/05 12:22:34 remote.go:47: traffic manager not exist, try to create it...
-2022/02/05 12:22:34 remote.go:121: pod kubevpn.traffic.manager status is Pending
-2022/02/05 12:22:36 remote.go:121: pod kubevpn.traffic.manager status is Running
+➜  ~ kubevpn proxy deployment/productpage --headers foo=bar
+Connected to cluster
+Injecting inbound sidecar for deployment/productpage
+Checking rollout status for deployment/productpage
 Waiting for deployment "productpage" rollout to finish: 1 old replicas are pending termination...
 Waiting for deployment "productpage" rollout to finish: 1 old replicas are pending termination...
-deployment "productpage" successfully rolled out
-Forwarding from 0.0.0.0:10800 -> 10800
-2022/02/05 12:22:43 connect.go:171: port forward ready
-2022/02/05 12:22:43 connect.go:193: your ip is 223.254.254.176
-2022/02/05 12:22:43 connect.go:197: tunnel connected
-Handling connection for 10800
-2022/02/05 12:22:43 connect.go:211: dns service ok
+Rollout successfully for deployment/productpage
++----------------------------------------------------------+
+| Now you can access resources in the kubernetes cluster ! |
++----------------------------------------------------------+
+➜  ~
 ```
 
+first access without header "foo: bar", it will access existing pod on kubernetes cluster.
+
 ```shell
 ➜  ~ curl productpage:9080
 <!DOCTYPE html>
@@ -201,17 +410,313 @@ Handling connection for 10800
 ...
 ```
 
+Now let's access local service with header `"foo: bar"`
+
 ```shell
-➜  ~ curl productpage:9080 -H "a: 1"
-Hello world!%
+➜  ~ curl productpage:9080 -H "foo: bar"
+>>Received request: GET / from xxx.xxx.xxx.xxx:51296
+Hello world!  
+```
+
+If you want to cancel proxy, just run command:
+
+```shell
+➜  ~ kubevpn leave deployments/productpage
+Leaving workload deployments/productpage
+Checking rollout status for deployments/productpage
+Waiting for deployment "productpage" rollout to finish: 0 out of 1 new replicas have been updated...
+Waiting for deployment "productpage" rollout to finish: 1 old replicas are pending termination...
+Waiting for deployment "productpage" rollout to finish: 1 old replicas are pending termination...
+Rollout successfully for deployments/productpage
+```
+
+### Dev mode in local Docker 🐳
+
+Run the Kubernetes pod in the local Docker container, and cooperate with the service mesh to intercept the traffic with
+the specified header to the local, or all the traffic to the local.
+
+```shell
+➜  ~ kubevpn dev deployment/authors --headers foo=bar --entrypoint sh
+Starting connect
+Got network CIDR from cache
+Use exist traffic manager
+Forwarding port...
+Connected tunnel
+Adding route...
+Configured DNS service
+Injecting inbound sidecar for deployment/authors
+Patching workload deployment/authors
+Checking rollout status for deployment/authors
+Waiting for deployment "authors" rollout to finish: 0 out of 1 new replicas have been updated...
+Waiting for deployment "authors" rollout to finish: 1 old replicas are pending termination...
+deployment "authors" successfully rolled out
+Rollout successfully for Deployment.apps/authors 
+tar: removing leading '/' from member names
+/var/folders/30/cmv9c_5j3mq_kthx63sb1t5c0000gn/T/4563987760170736212:/var/run/secrets/kubernetes.io/serviceaccount
+tar: Removing leading `/' from member names
+tar: Removing leading `/' from hard link targets
+/var/folders/30/cmv9c_5j3mq_kthx63sb1t5c0000gn/T/4044542168121221027:/var/run/secrets/kubernetes.io/serviceaccount
+create docker network 56c25058d4b7498d02c2c2386ccd1b2b127cb02e8a1918d6d24bffd18570200e
+Created container: nginx_default_kubevpn_a9a22
+Wait container nginx_default_kubevpn_a9a22 to be running...
+Container nginx_default_kubevpn_a9a22 is running on port 80/tcp:80 8888/tcp:8888 9080/tcp:9080 now
+WARNING: The requested image's platform (linux/amd64) does not match the detected host platform (linux/arm64/v8) and no specific platform was requested
+Created main container: authors_default_kubevpn_a9a22
+/opt/microservices # ls
+app
+/opt/microservices # ps -ef
+PID   USER     TIME  COMMAND
+    1 root      0:00 nginx: master process nginx -g daemon off;
+   29 101       0:00 nginx: worker process
+   30 101       0:00 nginx: worker process
+   31 101       0:00 nginx: worker process
+   32 101       0:00 nginx: worker process
+   33 101       0:00 nginx: worker process
+   34 root      0:00 {sh} /usr/bin/qemu-x86_64 /bin/sh sh
+   44 root      0:00 ps -ef
+/opt/microservices # apk add curl
+fetch https://dl-cdn.alpinelinux.org/alpine/v3.14/main/x86_64/APKINDEX.tar.gz
+fetch https://dl-cdn.alpinelinux.org/alpine/v3.14/community/x86_64/APKINDEX.tar.gz
+(1/4) Installing brotli-libs (1.0.9-r5)
+(2/4) Installing nghttp2-libs (1.43.0-r0)
+(3/4) Installing libcurl (8.0.1-r0)
+(4/4) Installing curl (8.0.1-r0)
+Executing busybox-1.33.1-r3.trigger
+OK: 8 MiB in 19 packages
+/opt/microservices # ./app &
+/opt/microservices # 2023/09/30 13:41:58 Start listening http port 9080 ...
+
+/opt/microservices # curl localhost:9080/health
+{"status":"Authors is healthy"} /opt/microservices # echo "continue testing pod access..."
+continue testing pod access...
+/opt/microservices # exit
+Created container: default_authors
+Wait container default_authors to be running...
+Container default_authors is running now
+Disconnecting from the cluster...
+Leaving workload deployments.apps/authors
+Disconnecting from the cluster...
+Performing cleanup operations
+Clearing DNS settings
+➜  ~
+```
+
+You can see that it will start up two containers with docker, mapping to pod two container, and share port with same
+network, you can use `localhost:port`
+to access another container. And more, all environment、volume and network are the same as remote kubernetes pod, it is
+truly consistent with the kubernetes runtime. Makes develop on local PC come true.
+
+```shell
+➜  ~ docker ps
+CONTAINER ID   IMAGE                           COMMAND                  CREATED          STATUS          PORTS                                                                NAMES
+afdecf41c08d   naison/authors:latest           "sh"                     37 seconds ago   Up 36 seconds                                                                        authors_default_kubevpn_a9a22
+fc04e42799a5   nginx:latest                    "/docker-entrypoint.…"   37 seconds ago   Up 37 seconds   0.0.0.0:80->80/tcp, 0.0.0.0:8888->8888/tcp, 0.0.0.0:9080->9080/tcp   nginx_default_kubevpn_a9a22
+➜  ~
+```
+
+Here is how to access pod in local docker container
+
+```shell
+export authors_pod=`kubectl get pods -l app=authors -n default -o jsonpath='{.items[0].metadata.name}'`
+export authors_pod_ip=`kubectl get pod $authors_pod -n default -o jsonpath='{.status.podIP}'`
+curl -kv -H "foo: bar" http://$authors_pod_ip:80/health
+```
+
+Verify logs of nginx container
+
+```shell
+docker logs $(docker ps --format '{{.Names}}' | grep nginx_default_kubevpn)
+```
+
+If you just want to start up a docker image, you can use a simple way like this:
+
+```shell
+kubevpn dev deployment/authors --no-proxy
+```
+
+Example：
+
+```shell
+➜  ~ kubevpn dev deployment/authors --no-proxy
+Starting connect
+Got network CIDR from cache
+Use exist traffic manager
+Forwarding port...
+Connected tunnel
+Adding route...
+Configured DNS service
+tar: removing leading '/' from member names
+/var/folders/30/cmv9c_5j3mq_kthx63sb1t5c0000gn/T/5631078868924498209:/var/run/secrets/kubernetes.io/serviceaccount
+tar: Removing leading `/' from member names
+tar: Removing leading `/' from hard link targets
+/var/folders/30/cmv9c_5j3mq_kthx63sb1t5c0000gn/T/1548572512863475037:/var/run/secrets/kubernetes.io/serviceaccount
+create docker network 56c25058d4b7498d02c2c2386ccd1b2b127cb02e8a1918d6d24bffd18570200e
+Created container: nginx_default_kubevpn_ff34b
+Wait container nginx_default_kubevpn_ff34b to be running...
+Container nginx_default_kubevpn_ff34b is running on port 80/tcp:80 8888/tcp:8888 9080/tcp:9080 now
+WARNING: The requested image's platform (linux/amd64) does not match the detected host platform (linux/arm64/v8) and no specific platform was requested
+Created main container: authors_default_kubevpn_ff34b
+2023/09/30 14:02:31 Start listening http port 9080 ...
+
+```
+
+Now the main process will hang up to show you log.
+
+If you want to specify the image to start the container locally, you can use the parameter `--dev-image`. When the
+image does not exist locally, it will be pulled from the corresponding mirror warehouse. If you want to specify startup
+parameters, you can use `--entrypoint` parameter, replace it with the command you want to execute, such
+as `--entrypoint /bin/bash`, for more parameters, see `kubevpn dev --help`.
+
+### DinD ( Docker in Docker ) use kubevpn in Docker
+
+If you want to start the development mode locally using Docker in Docker (DinD), because the program will read and
+write the `/tmp` directory, you need to manually add the parameter `-v /tmp:/tmp` (outer docker) and another thing is
+you
+need to special parameter `--network` (inner docker) for sharing network and pid
+
+Example:
+
+```shell
+docker run -it --privileged --sysctl net.ipv6.conf.all.disable_ipv6=0 -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp -v ~/.kube/config:/root/.kube/config --platform linux/amd64 ghcr.io/kubenetworks/kubevpn:latest
+```
+
+```shell
+➜  ~ docker run -it --privileged --sysctl net.ipv6.conf.all.disable_ipv6=0 -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp -v ~/.kube/vke:/root/.kube/config --platform linux/amd64 ghcr.io/kubenetworks/kubevpn:latest
+Unable to find image 'ghcr.io/kubenetworks/kubevpn:latest' locally
+latest: Pulling from ghcr.io/kubenetworks/kubevpn
+9c704ecd0c69: Already exists
+4987d0a976b5: Pull complete
+8aa94c4fc048: Pull complete
+526fee014382: Pull complete
+6c1c2bedceb6: Pull complete
+97ac845120c5: Pull complete
+ca82aef6a9eb: Pull complete
+1fd9534c7596: Pull complete
+588bd802eb9c: Pull complete
+Digest: sha256:368db2e0d98f6866dcefd60512960ce1310e85c24a398fea2a347905ced9507d
+Status: Downloaded newer image for ghcr.io/kubenetworks/kubevpn:latest
+WARNING: image with reference ghcr.io/kubenetworks/kubevpn was found but does not match the specified platform: wanted linux/amd64, actual: linux/arm64
+root@5732124e6447:/app# kubevpn dev deployment/authors --headers user=naison --entrypoint sh
+hostname is 5732124e6447
+Starting connect
+Got network CIDR from cache
+Use exist traffic manager
+Forwarding port...
+Connected tunnel
+Adding route...
+Configured DNS service
+Injecting inbound sidecar for deployment/authors
+Patching workload deployment/authors
+Checking rollout status for deployment/authors
+Waiting for deployment "authors" rollout to finish: 1 old replicas are pending termination...
+deployment "authors" successfully rolled out
+Rollout successfully for Deployment.apps/authors
+tar: removing leading '/' from member names
+/tmp/6460902982794789917:/var/run/secrets/kubernetes.io/serviceaccount
+tar: Removing leading `/' from member names
+tar: Removing leading `/' from hard link targets
+/tmp/5028895788722532426:/var/run/secrets/kubernetes.io/serviceaccount
+Network mode is container:d0b3dab8912a
+Created container: nginx_default_kubevpn_6df63
+Wait container nginx_default_kubevpn_6df63 to be running...
+Container nginx_default_kubevpn_6df63 is running now
+WARNING: The requested image's platform (linux/amd64) does not match the detected host platform (linux/arm64/v8) and no specific platform was requested
+Created main container: authors_default_kubevpn_6df5f
+/opt/microservices # ps -ef
+PID   USER     TIME  COMMAND
+    1 root      0:00 {bash} /usr/bin/qemu-x86_64 /bin/bash /bin/bash
+   14 root      0:02 {kubevpn} /usr/bin/qemu-x86_64 /usr/local/bin/kubevpn kubevpn dev deployment/authors --headers
+   25 root      0:01 {kubevpn} /usr/bin/qemu-x86_64 /usr/local/bin/kubevpn /usr/local/bin/kubevpn daemon
+   37 root      0:04 {kubevpn} /usr/bin/qemu-x86_64 /usr/local/bin/kubevpn /usr/local/bin/kubevpn daemon --sudo
+   53 root      0:00 nginx: master process nginx -g daemon off;
+(4/4) Installing curl (8.0.1-r0)
+Executing busybox-1.33.1-r3.trigger
+OK: 8 MiB in 19 packagesnx: worker process
+/opt/microservices #
+
+/opt/microservices # cat > hello.go <<EOF
+package main
+
+import (
+    "fmt"
+    "io"
+    "net/http"
+)
+
+func main() {
+    http.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
+        _, _ = io.WriteString(writer, "Hello world!")
+        fmt.Println(">> Container Received request: %s %s from %s\n", request.Method, request.RequestURI, request.RemoteAddr)
+    })
+    fmt.Println("Start listening http port 9080 ...")
+    _ = http.ListenAndServe(":9080", nil)
+}
+EOF
+/opt/microservices # go build hello.go
+/opt/microservices # 
+//opt/microservices # ls -alh
+total 12M    
+drwxr-xr-x    1 root     root          26 Nov  4 10:29 .
+drwxr-xr-x    1 root     root          26 Oct 18  2021 ..
+-rwxr-xr-x    1 root     root        6.3M Oct 18  2021 app
+-rwxr-xr-x    1 root     root        5.8M Nov  4 10:29 hello
+-rw-r--r--    1 root     root         387 Nov  4 10:28 hello.go
+/opt/microservices # 
+/opt/microservices # apk add curl
+OK: 8 MiB in 19 packages
+/opt/microservices # ./hello &
+/opt/microservices # Start listening http port 9080 ...
+[2]+  Done                       ./hello
+/opt/microservices # curl localhost:9080
+>> Container Received request: GET / from 127.0.0.1:41230
+Hello world!/opt/microservices # 
+
+/opt/microservices # curl authors:9080/health -H "foo: bar"
+>>Received request: GET /health from 198.19.0.109:57930
+                                                        Hello world!/opt/microservices # 
+/opt/microservices # curl localhost:9080/health
+{"status":"Authors is healthy"}/opt/microservices # exit
+Created container: default_authors
+Wait container default_authors to be running...
+Container default_authors is running now
+Disconnecting from the cluster...
+Leaving workload deployments.apps/authors
+Disconnecting from the cluster...
+Performing cleanup operations
+Clearing DNS settings
+root@d0b3dab8912a:/app# exit
+exit
+➜  ~
+```
+
+during test, check what container is running
+
+```text
+➜  ~ docker ps
+CONTAINER ID   IMAGE                                  COMMAND                  CREATED         STATUS         PORTS     NAMES
+1cd576b51b66   naison/authors:latest                  "sh"                     4 minutes ago   Up 4 minutes             authors_default_kubevpn_6df5f
+56a6793df82d   nginx:latest                           "/docker-entrypoint.…"   4 minutes ago   Up 4 minutes             nginx_default_kubevpn_6df63
+d0b3dab8912a   ghcr.io/kubenetworks/kubevpn:v2.0.0    "/bin/bash"              5 minutes ago   Up 5 minutes             upbeat_noyce
+➜  ~
+```
+
+* For clean up after test
+
+```shell
+kubectl delete -f https://raw.githubusercontent.com/kubenetworks/kubevpn/master/samples/bookinfo.yaml
 ```
 
 ### Multiple Protocol
 
+support OSI model layers 3 and above, protocols like `ICMP`, `TCP`, and `UDP`...
+
 - TCP
 - UDP
-- HTTP
 - ICMP
+- gRPC
+- Thrift
+- WebSocket
+- HTTP
 - ...
 
 ### Cross-platform
@@ -220,6 +725,21 @@ Hello world!%
 - Linux
 - Windows
 
-on Windows platform, you need to
-install [PowerShell](https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-on-windows?view=powershell-7.2)
-in advance
+## Architecture
+
+[architecture](https://kubevpn.dev/docs/architecture/connect).
+
+## Contributions
+
+Always welcome. Just opening an issue should be also grateful.
+
+If you want to debug this project on local PC. Please follow the steps bellow:
+
+- Startup daemon and sudo daemon process with IDE debug mode. (Essentially two GRPC server)
+- Add breakpoint to file `pkg/daemon/action/connect.go:21`.
+- Open another terminal run `make kubevpn`.
+- Then run `./bin/kubevpn connect` and it will hit breakpoint.
+
+### Supported by
+
+[![JetBrains logo.](https://resources.jetbrains.com/storage/products/company/brand/logos/jetbrains.svg)](https://jb.gg/OpenSourceSupport)

README_ZH.md

@@ -1,92 +1,182 @@
+![kubevpn](https://raw.githubusercontent.com/wencaiwulue/kubevpn/master/samples/flat_log.png)
+
+[![GitHub Workflow][1]](https://github.com/kubenetworks/kubevpn/actions)
+[![Go Version][2]](https://github.com/kubenetworks/kubevpn/blob/master/go.mod)
+[![Go Report][3]](https://goreportcard.com/report/github.com/wencaiwulue/kubevpn)
+[![Maintainability][4]](https://codeclimate.com/github/kubenetworks/kubevpn/maintainability)
+[![GitHub License][5]](https://github.com/kubenetworks/kubevpn/blob/main/LICENSE)
+[![Docker Pulls][6]](https://hub.docker.com/r/naison/kubevpn)
+[![Releases][7]](https://github.com/kubenetworks/kubevpn/releases)
+[![GoDoc](https://godoc.org/github.com/kubenetworks/kubevpn?status.png)](https://pkg.go.dev/github.com/wencaiwulue/kubevpn/v2)
+[![codecov](https://codecov.io/gh/wencaiwulue/kubevpn/graph/badge.svg?token=KMDSINSDEP)](https://codecov.io/gh/wencaiwulue/kubevpn)
+[![Snapcraft](https://snapcraft.io/kubevpn/badge.svg)](https://snapcraft.io/kubevpn)
+
+[1]: https://img.shields.io/github/actions/workflow/status/kubenetworks/kubevpn/release.yml?logo=github
+
+[2]: https://img.shields.io/github/go-mod/go-version/kubenetworks/kubevpn?logo=go
+
+[3]: https://goreportcard.com/badge/github.com/wencaiwulue/kubevpn?style=flat
+
+[4]: https://api.codeclimate.com/v1/badges/b5b30239174fc6603aca/maintainability
+
+[5]: https://img.shields.io/github/license/kubenetworks/kubevpn
+
+[6]: https://img.shields.io/docker/pulls/naison/kubevpn?logo=docker
+
+[7]: https://img.shields.io/github/v/release/kubenetworks/kubevpn?logo=smartthings
+
 # KubeVPN
 
-[English](README.md) | [中文](README_ZH.md) | [维基](https://github.com/wencaiwulue/kubevpn/wiki/%E6%9E%B6%E6%9E%84)
+[English](README.md) | [中文](README_ZH.md) | [维基](https://github.com/kubenetworks/kubevpn/wiki/%E6%9E%B6%E6%9E%84)
 
-一个本地连接云端 kubernetes 网络的工具，可以在本地直接访问远端集群的服务。也可以在远端集群访问到本地服务，便于调试及开发
+KubeVPN 提供一个云原生开发环境。通过连接云端 kubernetes 网络，可以在本地使用 k8s dns 或者 Pod IP / Service IP
+直接访问远端集群中的服务。拦截远端集群中的工作负载的入流量到本地电脑，配合服务网格便于调试及开发。同时还可以使用开发模式，直接在本地使用
+Docker
+模拟 k8s pod runtime 将容器运行在本地 (具有相同的环境变量，磁盘和网络)。
+
+![架构](docs/en/images/kubevpn-proxy-tun-arch.svg)
+
+## 内容
+
+1. [快速开始](./README_ZH.md#快速开始)
+2. [功能](./README_ZH.md#功能)
+3. [架构](./README_ZH.md#架构)
+4. [贡献代码](./README_ZH.md#贡献代码)
 
 ## 快速开始
 
-```shell
-git clone https://github.com/wencaiwulue/kubevpn.git
-cd kubevpn
-make kubevpn-linux-amd64
-make kubevpn-darwin-amd64
-make kubevpn-windows-amd64
-```
-
-如果你在使用 Windows 系统，可以使用下面这条命令构建：
+### 使用脚本安装 ( macOS / Linux)
 
 ```shell
-go build github.com/wencaiwulue/kubevpn/cmd/kubevpn -o kubevpn.exe
+curl -fsSL https://kubevpn.dev/install.sh | sh
 ```
 
-如果安装了 Go 1.16 及以上版本，可以使用如下命令安装：
+### 使用 [brew](https://brew.sh/) 安装 (macOS / Linux)
 
 ```shell
-go install github.com/wencaiwulue/kubevpn/cmd/kubevpn@latest
+brew install kubevpn
 ```
 
+### 使用 [snap](https://snapcraft.io/kubevpn) 安装 (Linux)
+
+```shell
+sudo snap install kubevpn
+```
+
+### 使用 [scoop](https://scoop.sh/) (Windows)
+
+```shell
+scoop bucket add extras
+scoop install kubevpn
+```
+
+### 使用 [krew](https://krew.sigs.k8s.io/) (Windows / macOS / Linux)
+
+```shell
+kubectl krew index add kubevpn https://github.com/kubenetworks/kubevpn.git
+kubectl krew install kubevpn/kubevpn
+kubectl kubevpn 
+```
+
+### 从 Github release 下载 (Windows / macOS / Linux)
+
+[https://github.com/kubenetworks/kubevpn/releases/latest](https://github.com/kubenetworks/kubevpn/releases/latest)
+
 ### 安装 bookinfo 作为 demo 应用
 
 ```shell
-kubectl apply -f https://raw.githubusercontent.com/wencaiwulue/kubevpn/master/samples/bookinfo.yaml
+kubectl apply -f https://raw.githubusercontent.com/kubenetworks/kubevpn/master/samples/bookinfo.yaml
 ```
 
 ## 功能
 
 ### 链接到集群网络
 
+使用命令 `kubevpn connect` 链接到集群，请注意这里需要输入电脑密码。因为需要 `root` 权限。(创建虚拟网卡)
+
 ```shell
 ➜  ~ kubevpn connect
-INFO[0000] [sudo kubevpn connect]
 Password:
-2022/02/05 12:09:22 connect.go:303: kubeconfig path: /Users/naison/.kube/config, namespace: default, services: []
-2022/02/05 12:09:28 remote.go:47: traffic manager not exist, try to create it...
-2022/02/05 12:09:28 remote.go:121: pod kubevpn.traffic.manager status is Pending
-2022/02/05 12:09:29 remote.go:121: pod kubevpn.traffic.manager status is Running
-Forwarding from 0.0.0.0:10800 -> 10800
-2022/02/05 12:09:31 connect.go:171: port forward ready
-2022/02/05 12:09:31 connect.go:193: your ip is 223.254.254.176
-2022/02/05 12:09:31 connect.go:197: tunnel connected
-Handling connection for 10800
-2022/02/05 12:09:31 connect.go:211: dns service ok
+Starting connect
+Getting network CIDR from cluster info...
+Getting network CIDR from CNI...
+Getting network CIDR from services...
+Labeling Namespace default
+Creating ServiceAccount kubevpn-traffic-manager
+Creating Roles kubevpn-traffic-manager
+Creating RoleBinding kubevpn-traffic-manager
+Creating Service kubevpn-traffic-manager
+Creating MutatingWebhookConfiguration kubevpn-traffic-manager
+Creating Deployment kubevpn-traffic-manager
+
+Pod kubevpn-traffic-manager-66d969fd45-9zlbp is Pending
+Container     Reason            Message
+control-plane ContainerCreating
+vpn           ContainerCreating
+webhook       ContainerCreating
+
+Pod kubevpn-traffic-manager-66d969fd45-9zlbp is Running
+Container     Reason           Message
+control-plane ContainerRunning
+vpn           ContainerRunning
+webhook       ContainerRunning
+
+Forwarding port...
+Connected tunnel
+Adding route...
+Configured DNS service
++----------------------------------------------------------+
+| Now you can access resources in the kubernetes cluster ! |
++----------------------------------------------------------+
+➜  ~
 ```
 
+提示已经链接到集群了。使用命令 `kubevpn status` 检查一下状态。
+
 ```shell
 ➜  ~ kubectl get pods -o wide
-NAME                          READY   STATUS      RESTARTS   AGE     IP             NODE          NOMINATED NODE   READINESS GATES
-details-7db5668668-mq9qr      1/1     Running     0          7m      172.27.0.199   172.30.0.14   <none>           <none>
-kubevpn.traffic.manager       1/1     Running     0          74s     172.27.0.207   172.30.0.14   <none>           <none>
-productpage-8f9d86644-z8snh   1/1     Running     0          6m59s   172.27.0.206   172.30.0.14   <none>           <none>
-ratings-859b96848d-68d7n      1/1     Running     0          6m59s   172.27.0.201   172.30.0.14   <none>           <none>
-reviews-dcf754f9d-46l4j       1/1     Running     0          6m59s   172.27.0.202   172.30.0.14   <none>           <none>
+NAME                                       READY   STATUS             RESTARTS   AGE     IP                NODE              NOMINATED NODE   READINESS GATES
+authors-dbb57d856-mbgqk                    3/3     Running            0          7d23h   172.29.2.132      192.168.0.5       <none>           <none>
+details-7d8b5f6bcf-hcl4t                   1/1     Running            0          61d     172.29.0.77       192.168.104.255   <none>           <none>
+kubevpn-traffic-manager-66d969fd45-9zlbp   3/3     Running            0          74s     172.29.2.136      192.168.0.5       <none>           <none>
+productpage-788df7ff7f-jpkcs               1/1     Running            0          61d     172.29.2.134      192.168.0.5       <none>           <none>
+ratings-77b6cd4499-zvl6c                   1/1     Running            0          61d     172.29.0.86       192.168.104.255   <none>           <none>
+reviews-85c88894d9-vgkxd                   1/1     Running            0          24d     172.29.2.249      192.168.0.5       <none>           <none>
 ```
 
+找一个 pod 的 IP，比如 `productpage-788df7ff7f-jpkcs` 的 IP `172.29.2.134`
+
 ```shell
-➜  ~ ping 172.27.0.206
-PING 172.27.0.206 (172.27.0.206): 56 data bytes
-64 bytes from 172.27.0.206: icmp_seq=0 ttl=63 time=49.563 ms
-64 bytes from 172.27.0.206: icmp_seq=1 ttl=63 time=43.014 ms
-64 bytes from 172.27.0.206: icmp_seq=2 ttl=63 time=43.841 ms
-64 bytes from 172.27.0.206: icmp_seq=3 ttl=63 time=44.004 ms
-64 bytes from 172.27.0.206: icmp_seq=4 ttl=63 time=43.484 ms
+➜  ~ ping 172.29.2.134
+PING 172.29.2.134 (172.29.2.134): 56 data bytes
+64 bytes from 172.29.2.134: icmp_seq=0 ttl=63 time=55.727 ms
+64 bytes from 172.29.2.134: icmp_seq=1 ttl=63 time=56.270 ms
+64 bytes from 172.29.2.134: icmp_seq=2 ttl=63 time=55.228 ms
+64 bytes from 172.29.2.134: icmp_seq=3 ttl=63 time=54.293 ms
 ^C
---- 172.27.0.206 ping statistics ---
-5 packets transmitted, 5 packets received, 0.0% packet loss
-round-trip min/avg/max/stddev = 43.014/44.781/49.563/2.415 ms
+--- 172.29.2.134 ping statistics ---
+4 packets transmitted, 4 packets received, 0.0% packet loss
+round-trip min/avg/max/stddev = 54.293/55.380/56.270/0.728 ms
 ```
 
+测试应该可以直接 Ping 通，说明本地可以正常访问到集群网络了。
+
 ```shell
 ➜  ~ kubectl get services -o wide
-NAME          TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGE     SELECTOR
-details       ClusterIP   172.27.255.92    <none>        9080/TCP   9m7s    app=details
-productpage   ClusterIP   172.27.255.48    <none>        9080/TCP   9m6s    app=productpage
-ratings       ClusterIP   172.27.255.154   <none>        9080/TCP   9m7s    app=ratings
-reviews       ClusterIP   172.27.255.155   <none>        9080/TCP   9m6s    app=reviews
+NAME                      TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                              AGE     SELECTOR
+authors                   ClusterIP   172.21.5.160    <none>        9080/TCP                             114d    app=authors
+details                   ClusterIP   172.21.6.183    <none>        9080/TCP                             114d    app=details
+kubernetes                ClusterIP   172.21.0.1      <none>        443/TCP                              319d    <none>
+kubevpn-traffic-manager   ClusterIP   172.21.2.86     <none>        8422/UDP,10800/TCP,9002/TCP,80/TCP   2m28s   app=kubevpn-traffic-manager
+productpage               ClusterIP   172.21.10.49    <none>        9080/TCP                             114d    app=productpage
+ratings                   ClusterIP   172.21.3.247    <none>        9080/TCP                             114d    app=ratings
+reviews                   ClusterIP   172.21.8.24     <none>        9080/TCP                             114d    app=reviews
 ```
 
+找一个 service 的 IP，比如 `productpage` 的 IP `172.21.10.49`，试着访问一下服务 `productpage`
+
 ```shell
-➜  ~ curl 172.27.255.48:9080
+➜  ~ curl 172.21.10.49:9080
 <!DOCTYPE html>
 <html>
   <head>
@@ -96,8 +186,16 @@ reviews       ClusterIP   172.27.255.155   <none>        9080/TCP   9m6s    app=
 <meta name="viewport" content="width=device-width, initial-scale=1">
 ```
 
+可以看到也可以正常访问，也就是可以在本地访问到集群的 pod 和 service 了～
+
 ### 域名解析功能
 
+支持 k8s dns 解析。比如一个名为 `productpage` 的 Pod 或者 Service 处于 `default` 命名空间下可以被如下域名正常解析到：
+
+- `productpage`
+- `productpage.default`
+- `productpage.default.svc.cluster.local`
+
 ```shell
 ➜  ~ curl productpage.default.svc.cluster.local:9080
 <!DOCTYPE html>
@@ -109,8 +207,15 @@ reviews       ClusterIP   172.27.255.155   <none>        9080/TCP   9m6s    app=
 <meta name="viewport" content="width=device-width, initial-scale=1">
 ```
 
+可以看到能够被正常解析，并且返回相应内容。
+
 ### 短域名解析功能
 
+连接到此命名空间下，可以直接使用 `service` name 的方式访问，否则访问其它命令空间下的服务，需要带上命令空间作为域名的一部分，使用如下的域名即可。
+
+- `productpage.default`
+- `productpage.default.svc.cluster.local`
+
 ```shell
 ➜  ~ curl productpage:9080
 <!DOCTYPE html>
@@ -122,27 +227,71 @@ reviews       ClusterIP   172.27.255.155   <none>        9080/TCP   9m6s    app=
 ...
 ```
 
-### 反向代理
+可以看到直接使用 service name 的方式，可以正常访问到集群资源。
+
+### 链接到多集群网络
+
+有个两个模式
+
+- 模式 `lite`: 可以链接到多个集群网络，但是仅支持链接到多集群。
+- 模式 `full`: 不仅支持链接到单个集群网络，还可以拦截工作负载流量到本地电脑。
+
+可以看到已经链接到了一个集群 `ccijorbccotmqodvr189g`，是 `full` 模式
 
 ```shell
-➜  ~ kubevpn connect --workloads=service/productpage
-INFO[0000] [sudo kubevpn connect --workloads=service/productpage]
-Password:
-2022/02/05 12:18:22 connect.go:303: kubeconfig path: /Users/naison/.kube/config, namespace: default, services: [service/productpage]
-2022/02/05 12:18:28 remote.go:47: traffic manager not exist, try to create it...
-2022/02/05 12:18:28 remote.go:121: pod kubevpn.traffic.manager status is Pending
-2022/02/05 12:18:29 remote.go:121: pod kubevpn.traffic.manager status is Running
-Waiting for deployment "productpage" rollout to finish: 1 old replicas are pending termination...
-Waiting for deployment "productpage" rollout to finish: 1 old replicas are pending termination...
-deployment "productpage" successfully rolled out
-Forwarding from 0.0.0.0:10800 -> 10800
-2022/02/05 12:18:34 connect.go:171: port forward ready
-2022/02/05 12:18:34 connect.go:193: your ip is 223.254.254.176
-2022/02/05 12:18:34 connect.go:197: tunnel connected
-Handling connection for 10800
-2022/02/05 12:18:35 connect.go:211: dns service ok
+➜  ~ kubevpn status
+ID Mode Cluster               Kubeconfig                 Namespace Status
+0  full ccijorbccotmqodvr189g /Users/naison/.kube/config default   Connected
 ```
 
+此时还可以使用 `lite` 模式链接到其它集群
+
+```shell
+➜  ~ kubevpn connect -n default --kubeconfig ~/.kube/dev_config --lite
+Starting connect
+Got network CIDR from cache
+Use exist traffic manager
+Forwarding port...
+Connected tunnel
+Adding route...
+Configured DNS service
++----------------------------------------------------------+
+| Now you can access resources in the kubernetes cluster ! |
++----------------------------------------------------------+
+```
+
+使用命令 `kubevpn status` 查看当前链接状态。
+
+```shell
+➜  ~ kubevpn status
+ID Mode Cluster               Kubeconfig                     Namespace Status
+0  full ccijorbccotmqodvr189g /Users/naison/.kube/config     default   Connected
+1  lite ccidd77aam2dtnc3qnddg /Users/naison/.kube/dev_config default   Connected
+➜  ~
+```
+
+可以看到连接到了多个集群。
+
+### 反向代理
+
+使用命令 `kubevpn proxy` 代理所有的入站流量到本地电脑。
+
+```shell
+➜  ~ kubevpn proxy deployment/productpage
+Connected to cluster
+Injecting inbound sidecar for deployment/productpage
+Checking rollout status for deployment/productpage
+Waiting for deployment "productpage" rollout to finish: 1 old replicas are pending termination...
+Waiting for deployment "productpage" rollout to finish: 1 old replicas are pending termination...
+Rollout successfully for deployment/productpage
++----------------------------------------------------------+
+| Now you can access resources in the kubernetes cluster ! |
++----------------------------------------------------------+
+➜  ~
+```
+
+此时在本地使用 `go` 启动一个服务，用于承接流量。
+
 ```go
 package main
 
@@ -159,6 +308,8 @@ func main() {
 }
 ```
 
+使用 `service` name 的方式，直接访问集群中的 `productpage` 服务。
+
 ```shell
 ➜  ~ curl productpage:9080
 Hello world!%
@@ -166,28 +317,28 @@ Hello world!%
 Hello world!%
 ```
 
+可以看到直接击中了本地电脑的服务。
+
 ### 反向代理支持 service mesh
 
-只支持 HTTP 和 GRPC, 携带了指定 header `"a: 1"` 的流量，将会路由到本地
+支持 HTTP, GRPC 和 WebSocket 等, 携带了指定 header `"foo: bar"` 的流量，将会路由到本地
 
 ```shell
-➜  ~ kubevpn connect --workloads=service/productpage --mode=mesh --headers a=1
-INFO[0000] [sudo kubevpn connect --workloads=service/productpage --mode=mesh --headers a=1]
-2022/02/05 12:22:28 connect.go:303: kubeconfig path: /Users/naison/.kube/config, namespace: default, services: [service/productpage]
-2022/02/05 12:22:34 remote.go:47: traffic manager not exist, try to create it...
-2022/02/05 12:22:34 remote.go:121: pod kubevpn.traffic.manager status is Pending
-2022/02/05 12:22:36 remote.go:121: pod kubevpn.traffic.manager status is Running
+➜  ~ kubevpn proxy deployment/productpage --headers foo=bar
+Connected to cluster
+Injecting inbound sidecar for deployment/productpage
+Checking rollout status for deployment/productpage
 Waiting for deployment "productpage" rollout to finish: 1 old replicas are pending termination...
 Waiting for deployment "productpage" rollout to finish: 1 old replicas are pending termination...
-deployment "productpage" successfully rolled out
-Forwarding from 0.0.0.0:10800 -> 10800
-2022/02/05 12:22:43 connect.go:171: port forward ready
-2022/02/05 12:22:43 connect.go:193: your ip is 223.254.254.176
-2022/02/05 12:22:43 connect.go:197: tunnel connected
-Handling connection for 10800
-2022/02/05 12:22:43 connect.go:211: dns service ok
+Rollout successfully for deployment/productpage
++----------------------------------------------------------+
+| Now you can access resources in the kubernetes cluster ! |
++----------------------------------------------------------+
+➜  ~
 ```
 
+不带 header 直接访问集群资源，可以看到返回的是集群中的服务内容。
+
 ```shell
 ➜  ~ curl productpage:9080
 <!DOCTYPE html>
@@ -200,17 +351,289 @@ Handling connection for 10800
 ...
 ```
 
+带上特定 header 访问集群资源，可以看到返回了本地服务的内容。
+
 ```shell
-➜  ~ curl productpage:9080 -H "a: 1"
+➜  ~ curl productpage:9080 -H "foo: bar"
 Hello world!%
 ```
 
+如果你需要取消代理流量，可以执行如下命令：
+
+```shell
+➜  ~ kubevpn leave deployments/productpage
+Leaving workload deployments/productpage
+Checking rollout status for deployments/productpage
+Waiting for deployment "productpage" rollout to finish: 0 out of 1 new replicas have been updated...
+Waiting for deployment "productpage" rollout to finish: 1 old replicas are pending termination...
+Waiting for deployment "productpage" rollout to finish: 1 old replicas are pending termination...
+Rollout successfully for deployments/productpage
+```
+
+### 本地进入开发模式 🐳
+
+将 Kubernetes pod 运行在本地的 Docker 容器中，同时配合 service mesh, 拦截带有指定 header 的流量到本地，或者所有的流量到本地。这个开发模式依赖于本地
+Docker。
+
+```shell
+➜  ~ kubevpn dev deployment/authors --headers foo=bar --entrypoint sh
+Starting connect
+Got network CIDR from cache
+Use exist traffic manager
+Forwarding port...
+Connected tunnel
+Adding route...
+Configured DNS service
+Injecting inbound sidecar for deployment/authors
+Patching workload deployment/authors
+Checking rollout status for deployment/authors
+Waiting for deployment "authors" rollout to finish: 0 out of 1 new replicas have been updated...
+Waiting for deployment "authors" rollout to finish: 1 old replicas are pending termination...
+deployment "authors" successfully rolled out
+Rollout successfully for Deployment.apps/authors 
+tar: removing leading '/' from member names
+/var/folders/30/cmv9c_5j3mq_kthx63sb1t5c0000gn/T/4563987760170736212:/var/run/secrets/kubernetes.io/serviceaccount
+tar: Removing leading `/' from member names
+tar: Removing leading `/' from hard link targets
+/var/folders/30/cmv9c_5j3mq_kthx63sb1t5c0000gn/T/4044542168121221027:/var/run/secrets/kubernetes.io/serviceaccount
+create docker network 56c25058d4b7498d02c2c2386ccd1b2b127cb02e8a1918d6d24bffd18570200e
+Created container: nginx_default_kubevpn_a9a22
+Wait container nginx_default_kubevpn_a9a22 to be running...
+Container nginx_default_kubevpn_a9a22 is running on port 80/tcp:80 8888/tcp:8888 9080/tcp:9080 now
+WARNING: The requested image's platform (linux/amd64) does not match the detected host platform (linux/arm64/v8) and no specific platform was requested
+Created main container: authors_default_kubevpn_a9a22
+/opt/microservices # ls
+app
+/opt/microservices # ps -ef
+PID   USER     TIME  COMMAND
+    1 root      0:00 nginx: master process nginx -g daemon off;
+   29 101       0:00 nginx: worker process
+   30 101       0:00 nginx: worker process
+   31 101       0:00 nginx: worker process
+   32 101       0:00 nginx: worker process
+   33 101       0:00 nginx: worker process
+   34 root      0:00 {sh} /usr/bin/qemu-x86_64 /bin/sh sh
+   44 root      0:00 ps -ef
+/opt/microservices # apk add curl
+fetch https://dl-cdn.alpinelinux.org/alpine/v3.14/main/x86_64/APKINDEX.tar.gz
+fetch https://dl-cdn.alpinelinux.org/alpine/v3.14/community/x86_64/APKINDEX.tar.gz
+(1/4) Installing brotli-libs (1.0.9-r5)
+(2/4) Installing nghttp2-libs (1.43.0-r0)
+(3/4) Installing libcurl (8.0.1-r0)
+(4/4) Installing curl (8.0.1-r0)
+Executing busybox-1.33.1-r3.trigger
+OK: 8 MiB in 19 packages
+/opt/microservices # ./app &
+/opt/microservices # 2023/09/30 13:41:58 Start listening http port 9080 ...
+
+/opt/microservices # curl localhost:9080/health
+{"status":"Authors is healthy"} /opt/microservices # echo "continue testing pod access..."
+continue testing pod access...
+/opt/microservices # exit
+Created container: default_authors
+Wait container default_authors to be running...
+Container default_authors is running now
+Disconnecting from the cluster...
+Leaving workload deployments.apps/authors
+Disconnecting from the cluster...
+Performing cleanup operations
+Clearing DNS settings
+➜  ~
+```
+
+此时本地会启动两个 container, 对应 pod 容器中的两个 container, 并且共享端口, 可以直接使用 localhost:port 的形式直接访问另一个
+container,
+并且, 所有的环境变量、挂载卷、网络条件都和 pod 一样, 真正做到与 kubernetes 运行环境一致。
+
+```shell
+➜  ~ docker ps
+CONTAINER ID   IMAGE                           COMMAND                  CREATED          STATUS          PORTS                                                                NAMES
+afdecf41c08d   naison/authors:latest           "sh"                     37 seconds ago   Up 36 seconds                                                                        authors_default_kubevpn_a9a22
+fc04e42799a5   nginx:latest                    "/docker-entrypoint.…"   37 seconds ago   Up 37 seconds   0.0.0.0:80->80/tcp, 0.0.0.0:8888->8888/tcp, 0.0.0.0:9080->9080/tcp   nginx_default_kubevpn_a9a22
+➜  ~
+```
+
+如果你只是想在本地启动镜像，可以用一种简单的方式：
+
+```shell
+kubevpn dev deployment/authors --no-proxy
+```
+
+例如：
+
+```shell
+➜  ~ kubevpn dev deployment/authors --no-proxy
+Starting connect
+Got network CIDR from cache
+Use exist traffic manager
+Forwarding port...
+Connected tunnel
+Adding route...
+Configured DNS service
+tar: removing leading '/' from member names
+/var/folders/30/cmv9c_5j3mq_kthx63sb1t5c0000gn/T/5631078868924498209:/var/run/secrets/kubernetes.io/serviceaccount
+tar: Removing leading `/' from member names
+tar: Removing leading `/' from hard link targets
+/var/folders/30/cmv9c_5j3mq_kthx63sb1t5c0000gn/T/1548572512863475037:/var/run/secrets/kubernetes.io/serviceaccount
+create docker network 56c25058d4b7498d02c2c2386ccd1b2b127cb02e8a1918d6d24bffd18570200e
+Created container: nginx_default_kubevpn_ff34b
+Wait container nginx_default_kubevpn_ff34b to be running...
+Container nginx_default_kubevpn_ff34b is running on port 80/tcp:80 8888/tcp:8888 9080/tcp:9080 now
+WARNING: The requested image's platform (linux/amd64) does not match the detected host platform (linux/arm64/v8) and no specific platform was requested
+Created main container: authors_default_kubevpn_ff34b
+2023/09/30 14:02:31 Start listening http port 9080 ...
+
+```
+
+此时程序会挂起，默认为显示日志
+
+如果你想指定在本地启动容器的镜像, 可以使用参数 `--dev-image`, 当本地不存在该镜像时,
+会从对应的镜像仓库拉取。如果你想指定启动参数，可以使用 `--entrypoint`
+参数，替换为你想要执行的命令，比如 `--entrypoint /bin/bash`, 更多使用参数，请参见 `kubevpn dev --help`.
+
+### DinD ( Docker in Docker ) 在 Docker 中使用 kubevpn
+
+如果你想在本地使用 Docker in Docker (DinD) 的方式启动开发模式, 由于程序会读写 `/tmp`
+目录，您需要手动添加参数 `-v /tmp:/tmp`, 还有一点需要注意, 如果使用 DinD
+模式，为了共享容器网络和 pid, 还需要指定参数 `--network`
+
+例如:
+
+```shell
+docker run -it --privileged --sysctl net.ipv6.conf.all.disable_ipv6=0 -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp -v ~/.kube/config:/root/.kube/config --platform linux/amd64 ghcr.io/kubenetworks/kubevpn:latest
+```
+
+```shell
+➜  ~ docker run -it --privileged --sysctl net.ipv6.conf.all.disable_ipv6=0 -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp -v ~/.kube/vke:/root/.kube/config --platform linux/amd64 ghcr.io/kubenetworks/kubevpn:latest
+Unable to find image 'ghcr.io/kubenetworks/kubevpn:latest' locally
+latest: Pulling from ghcr.io/kubenetworks/kubevpn
+9c704ecd0c69: Already exists
+4987d0a976b5: Pull complete
+8aa94c4fc048: Pull complete
+526fee014382: Pull complete
+6c1c2bedceb6: Pull complete
+97ac845120c5: Pull complete
+ca82aef6a9eb: Pull complete
+1fd9534c7596: Pull complete
+588bd802eb9c: Pull complete
+Digest: sha256:368db2e0d98f6866dcefd60512960ce1310e85c24a398fea2a347905ced9507d
+Status: Downloaded newer image for ghcr.io/kubenetworks/kubevpn:latest
+WARNING: image with reference ghcr.io/kubenetworks/kubevpn was found but does not match the specified platform: wanted linux/amd64, actual: linux/arm64
+root@5732124e6447:/app# kubevpn dev deployment/authors --headers user=naison --entrypoint sh
+hostname is 5732124e6447
+Starting connect
+Got network CIDR from cache
+Use exist traffic manager
+Forwarding port...
+Connected tunnel
+Adding route...
+Configured DNS service
+Injecting inbound sidecar for deployment/authors
+Patching workload deployment/authors
+Checking rollout status for deployment/authors
+Waiting for deployment "authors" rollout to finish: 1 old replicas are pending termination...
+deployment "authors" successfully rolled out
+Rollout successfully for Deployment.apps/authors
+tar: removing leading '/' from member names
+/tmp/6460902982794789917:/var/run/secrets/kubernetes.io/serviceaccount
+tar: Removing leading `/' from member names
+tar: Removing leading `/' from hard link targets
+/tmp/5028895788722532426:/var/run/secrets/kubernetes.io/serviceaccount
+Network mode is container:d0b3dab8912a
+Created container: nginx_default_kubevpn_6df63
+Wait container nginx_default_kubevpn_6df63 to be running...
+Container nginx_default_kubevpn_6df63 is running now
+WARNING: The requested image's platform (linux/amd64) does not match the detected host platform (linux/arm64/v8) and no specific platform was requested
+Created main container: authors_default_kubevpn_6df5f
+/opt/microservices # ps -ef
+PID   USER     TIME  COMMAND
+    1 root      0:00 {bash} /usr/bin/qemu-x86_64 /bin/bash /bin/bash
+   14 root      0:02 {kubevpn} /usr/bin/qemu-x86_64 /usr/local/bin/kubevpn kubevpn dev deployment/authors --headers
+   25 root      0:01 {kubevpn} /usr/bin/qemu-x86_64 /usr/local/bin/kubevpn /usr/local/bin/kubevpn daemon
+   37 root      0:04 {kubevpn} /usr/bin/qemu-x86_64 /usr/local/bin/kubevpn /usr/local/bin/kubevpn daemon --sudo
+   53 root      0:00 nginx: master process nginx -g daemon off;
+(4/4) Installing curl (8.0.1-r0)
+Executing busybox-1.33.1-r3.trigger
+OK: 8 MiB in 19 packagesnx: worker process
+/opt/microservices #
+
+/opt/microservices # cat > hello.go <<EOF
+package main
+
+import (
+    "fmt"
+    "io"
+    "net/http"
+)
+
+func main() {
+    http.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
+        _, _ = io.WriteString(writer, "Hello world!")
+        fmt.Println(">> Container Received request: %s %s from %s\n", request.Method, request.RequestURI, request.RemoteAddr)
+    })
+    fmt.Println("Start listening http port 9080 ...")
+    _ = http.ListenAndServe(":9080", nil)
+}
+EOF
+/opt/microservices # go build hello.go
+/opt/microservices # 
+//opt/microservices # ls -alh
+total 12M    
+drwxr-xr-x    1 root     root          26 Nov  4 10:29 .
+drwxr-xr-x    1 root     root          26 Oct 18  2021 ..
+-rwxr-xr-x    1 root     root        6.3M Oct 18  2021 app
+-rwxr-xr-x    1 root     root        5.8M Nov  4 10:29 hello
+-rw-r--r--    1 root     root         387 Nov  4 10:28 hello.go
+/opt/microservices # 
+/opt/microservices # apk add curl
+OK: 8 MiB in 19 packages
+/opt/microservices # ./hello &
+/opt/microservices # Start listening http port 9080 ...
+[2]+  Done                       ./hello
+/opt/microservices # curl localhost:9080
+>> Container Received request: GET / from 127.0.0.1:41230
+Hello world!/opt/microservices # 
+
+/opt/microservices # curl authors:9080/health -H "foo: bar"
+>>Received request: GET /health from 198.19.0.109:57930
+                                                        Hello world!/opt/microservices # 
+/opt/microservices # curl localhost:9080/health
+{"status":"Authors is healthy"}/opt/microservices # exit
+Created container: default_authors
+Wait container default_authors to be running...
+Container default_authors is running now
+Disconnecting from the cluster...
+Leaving workload deployments.apps/authors
+Disconnecting from the cluster...
+Performing cleanup operations
+Clearing DNS settings
+root@d0b3dab8912a:/app# exit
+exit
+➜  ~
+```
+
+可以看到实际上是在本地使用 `Docker` 启动了三个容器。
+
+```text
+➜  ~ docker ps
+CONTAINER ID   IMAGE                                  COMMAND                  CREATED         STATUS         PORTS     NAMES
+1cd576b51b66   naison/authors:latest                  "sh"                     4 minutes ago   Up 4 minutes             authors_default_kubevpn_6df5f
+56a6793df82d   nginx:latest                           "/docker-entrypoint.…"   4 minutes ago   Up 4 minutes             nginx_default_kubevpn_6df63
+d0b3dab8912a   ghcr.io/kubenetworks/kubevpn:v2.0.0    "/bin/bash"              5 minutes ago   Up 5 minutes             upbeat_noyce
+➜  ~
+```
+
 ### 支持多种协议
 
+支持 OSI 模型三层及三层以上的协议，例如：
+
 - TCP
 - UDP
-- HTTP
 - ICMP
+- gRPC
+- Thrift
+- WebSocket
+- HTTP
 - ...
 
 ### 支持三大平台
@@ -219,5 +642,21 @@ Hello world!%
 - Linux
 - Windows
 
-Windows
-下需要安装 [PowerShell](https://docs.microsoft.com/zh-cn/powershell/scripting/install/installing-powershell-on-windows?view=powershell-7.2)
+## 架构
+
+[架构](https://kubevpn.dev/docs/architecture/connect)
+
+## 贡献代码
+
+所有都是欢迎的，只是打开一个问题也是受欢迎的～
+
+如果你想在本地电脑上调试项目，可以按照这样的步骤：
+
+- 使用喜欢的 IDE Debug 启动 daemon 和 sudo daemon 两个后台进程。（本质上是两个 GRPC server）
+- 添加断点给文件 `pkg/daemon/action/connect.go:21`
+- 新开个终端，执行命令 `make kubevpn`
+- 然后运行命令 `./bin/kubevpn connect` 这样将会击中断点
+
+### 支持者
+
+[![JetBrains logo.](https://resources.jetbrains.com/storage/products/company/brand/logos/jetbrains.svg)](https://jb.gg/OpenSourceSupport)

TODO.MD

@@ -1,24 +0,0 @@
-## TODO
-
-- [x] 访问集群网络
-- [x] 域名解析功能
-- [x] 支持多个 service 反向代理
-- [x] 短域名解析
-- [x] 优化 DHCP 功能
-- [x] 支持多种类型，例如 statefulset, replicaset...
-- [ ] 支持 ipv6
-- [x] 自己实现 socks5 协议
-- [ ] 考虑是否需要把 openvpn tap/tun 驱动作为后备方案
-- [x] 加入 TLS 以提高安全性
-- [ ] 写个 CNI 网络插件，直接提供 VPN 功能
-- [x] 优化重连逻辑
-- [x] 支持 service mesh
-- [ ] service mesh 支持多端口
-- [x] 使用自己写的 proxy 替换 envoy
-- [ ] 优化性能，Windows 上考虑使用 IPC 通信
-- [x] 自己写个 control plane
-- [ ] 考虑是否将 control plane 和服务分开
-- [x] 写单元测试，优化 GitHub action
-- [ ] Linux 和 macOS 也改用 WireGuard library
-- [x] 探测是否有重复路由的 utun设备，禁用 `sudo ifconfig utun1 down`
-

build/Dockerfile

@@ -0,0 +1,34 @@
+FROM envoyproxy/envoy:v1.25.0 AS envoy
+FROM golang:1.23 AS builder
+ARG BASE=github.com/wencaiwulue/kubevpn
+
+COPY . /go/src/$BASE
+
+WORKDIR /go/src/$BASE
+
+RUN make kubevpn
+
+FROM debian:bookworm-slim
+ARG BASE=github.com/wencaiwulue/kubevpn
+
+RUN apt-get update && apt-get install -y openssl iptables curl dnsutils \
+    && if [ $(uname -m) = "x86_64" ]; then \
+         echo "The architecture is AMD64"; \
+         curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"; \
+       elif [ $(uname -m) = "aarch64" ]; then \
+         echo "The architecture is ARM64"; \
+         curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/arm64/kubectl"; \
+       else \
+         echo "Unsupported architecture."; \
+         exit 1; \
+       fi \
+    && chmod +x kubectl && mv kubectl /usr/local/bin \
+    && apt-get remove -y curl \
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY --from=builder /go/src/$BASE/bin/kubevpn /usr/local/bin/kubevpn
+COPY --from=envoy /usr/local/bin/envoy /usr/local/bin/envoy

build/dlv.Dockerfile

@@ -0,0 +1,6 @@
+FROM golang:1.23 as delve
+RUN curl --location --output delve-1.23.1.tar.gz https://github.com/go-delve/delve/archive/v1.23.1.tar.gz \
+  && tar xzf delve-1.23.1.tar.gz
+RUN cd delve-1.23.1 && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /go/dlv -ldflags '-extldflags "-static"' ./cmd/dlv/
+FROM busybox
+COPY --from=delve /go/dlv /bin/dlv

build/local.Dockerfile

@@ -0,0 +1,37 @@
+FROM golang:1.23 AS builder
+RUN go env -w GO111MODULE=on && go env -w GOPROXY=https://goproxy.cn,direct
+RUN go install github.com/go-delve/delve/cmd/dlv@latest
+
+FROM envoyproxy/envoy:v1.25.0 AS envoy
+FROM ubuntu:latest
+
+RUN sed -i s@/security.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list \
+    && sed -i s@/archive.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list
+RUN apt-get clean && apt-get update && apt-get install -y wget dnsutils vim curl  \
+    net-tools iptables iputils-ping lsof iproute2 tcpdump binutils traceroute conntrack socat iperf3 \
+    apt-transport-https ca-certificates curl
+
+RUN if [ $(uname -m) = "x86_64" ]; then \
+      echo "The architecture is AMD64"; \
+      curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && chmod +x kubectl && mv kubectl /usr/local/bin; \
+    elif [ $(uname -m) = "aarch64" ]; then \
+      echo "The architecture is ARM64"; \
+      curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/arm64/kubectl" && chmod +x kubectl && mv kubectl /usr/local/bin; \
+    else \
+      echo "Unsupported architecture."; \
+    fi
+
+ENV TZ=Asia/Shanghai \
+    DEBIAN_FRONTEND=noninteractive
+RUN apt update \
+    && apt install -y tzdata \
+    && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \
+    && echo ${TZ} > /etc/timezone \
+    && dpkg-reconfigure --frontend noninteractive tzdata \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY bin/kubevpn /usr/local/bin/kubevpn
+COPY --from=envoy /usr/local/bin/envoy /usr/local/bin/envoy
+COPY --from=builder /go/bin/dlv /usr/local/bin/dlv

build/test.Dockerfile

@@ -0,0 +1,34 @@
+FROM envoyproxy/envoy:v1.25.0 AS envoy
+FROM golang:1.23 AS builder
+ARG BASE=github.com/wencaiwulue/kubevpn
+
+COPY . /go/src/$BASE
+
+WORKDIR /go/src/$BASE
+
+RUN make kubevpn
+
+FROM debian:bookworm-slim
+ARG BASE=github.com/wencaiwulue/kubevpn
+
+RUN apt-get update && apt-get install -y openssl iptables curl dnsutils \
+    && if [ $(uname -m) = "x86_64" ]; then \
+         echo "The architecture is AMD64"; \
+         curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"; \
+       elif [ $(uname -m) = "aarch64" ]; then \
+         echo "The architecture is ARM64"; \
+         curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/arm64/kubectl"; \
+       else \
+         echo "Unsupported architecture."; \
+         exit 1; \
+       fi \
+    && chmod +x kubectl && mv kubectl /usr/local/bin \
+    && apt-get remove -y curl \
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY --from=builder /go/src/$BASE/bin/kubevpn /usr/local/bin/kubevpn
+COPY --from=envoy /usr/local/bin/envoy /usr/local/bin/envoy

charts/index.yaml

@@ -0,0 +1,458 @@
+apiVersion: v1
+entries:
+  kubevpn:
+  - annotations:
+      app: kubevpn
+    apiVersion: v2
+    appVersion: v2.7.2
+    created: "2025-04-25T15:40:08.296727519Z"
+    description: A Helm chart for KubeVPN
+    digest: 8711dae30f4ff9bc9cea018fa16ae70087a17af42262f7f31c43950a34fffa08
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.7.2/kubevpn-2.7.2.tgz
+    version: 2.7.2
+  - annotations:
+      app: kubevpn
+    apiVersion: v2
+    appVersion: v2.7.1
+    created: "2025-04-15T15:18:20.818055207Z"
+    description: A Helm chart for KubeVPN
+    digest: 79c40c942fd2cfcca63dd82921e04871680838f01717c6fcb3ee06bfb7f59535
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.7.1/kubevpn-2.7.1.tgz
+    version: 2.7.1
+  - annotations:
+      app: kubevpn
+    apiVersion: v2
+    appVersion: v2.7.0
+    created: "2025-04-12T05:37:01.063235951Z"
+    description: A Helm chart for KubeVPN
+    digest: a4b4de15f474fba43367fc7239c31e2020a6a1e0e3b29e02eb653cb9922b02e8
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.7.0/kubevpn-2.7.0.tgz
+    version: 2.7.0
+  - annotations:
+      app: kubevpn
+    apiVersion: v2
+    appVersion: v2.6.0
+    created: "2025-04-06T12:54:49.852649414Z"
+    description: A Helm chart for KubeVPN
+    digest: 58d930de19ac808e9f0ee501fe6f74b6f38376692708fc94fe7200496d9c5ca2
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.6.0/kubevpn-2.6.0.tgz
+    version: 2.6.0
+  - annotations:
+      app: kubevpn
+    apiVersion: v2
+    appVersion: v2.5.1
+    created: "2025-04-03T15:46:28.062220333Z"
+    description: A Helm chart for KubeVPN
+    digest: 6daf003256c42bb0db414eb17eb06294e46d33bc6c63f01419012a37318d0d2f
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.5.1/kubevpn-2.5.1.tgz
+    version: 2.5.1
+  - annotations:
+      app: kubevpn
+    apiVersion: v2
+    appVersion: v2.5.0
+    created: "2025-03-31T05:36:16.050204161Z"
+    description: A Helm chart for KubeVPN
+    digest: 301137b1599c232efd61ce9360e0a60da89e0a5c2eb076750bf461b38d26cfaf
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.5.0/kubevpn-2.5.0.tgz
+    version: 2.5.0
+  - annotations:
+      app: kubevpn
+    apiVersion: v2
+    appVersion: v2.4.3
+    created: "2025-03-30T13:48:42.333380676Z"
+    description: A Helm chart for KubeVPN
+    digest: 8ef28a43cb3d04f071445cf7d1199aba7392d78e1941707bab82853c5541c93c
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.4.3/kubevpn-2.4.3.tgz
+    version: 2.4.3
+  - apiVersion: v2
+    appVersion: v2.4.2
+    created: "2025-03-23T12:53:35.793492243Z"
+    description: A Helm chart for KubeVPN
+    digest: c627f69ac904ddb41c396909873425d85264fb3393d550fa1b0e8d2abfc402e9
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.4.2/kubevpn-2.4.2.tgz
+    version: 2.4.2
+  - apiVersion: v2
+    appVersion: v2.4.1
+    created: "2025-03-16T09:48:30.691242519Z"
+    description: A Helm chart for KubeVPN
+    digest: 1766431ce46b43758353928188cc993832e41cd0e352c9bc7991390bbbf41b04
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.4.1/kubevpn-2.4.1.tgz
+    version: 2.4.1
+  - apiVersion: v2
+    appVersion: v2.4.0
+    created: "2025-03-14T14:16:56.392516206Z"
+    description: A Helm chart for KubeVPN
+    digest: ffece68d3234ba629e02456fd3b0d31b5d2d1330c4c7f5d82ac2e0e1e97d82f3
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.4.0/kubevpn-2.4.0.tgz
+    version: 2.4.0
+  - apiVersion: v2
+    appVersion: v2.3.13
+    created: "2025-02-23T14:30:35.221348419Z"
+    description: A Helm chart for KubeVPN
+    digest: e79cdd07eae2ba3f36997debf898b091e1e68412fde7a34e823bad902e803105
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.3.13/kubevpn-2.3.13.tgz
+    version: 2.3.13
+  - apiVersion: v2
+    appVersion: v2.3.12
+    created: "2025-02-13T07:46:06.029130129Z"
+    description: A Helm chart for KubeVPN
+    digest: 0b7d9f8b4cd306377e4452a9d86530387afcae379e11665909b90e15f2d82a04
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.3.12/kubevpn-2.3.12.tgz
+    version: 2.3.12
+  - apiVersion: v2
+    appVersion: v2.3.11
+    created: "2025-02-03T09:24:54.033585049Z"
+    description: A Helm chart for KubeVPN
+    digest: a54a2ed19e6f4aa5c274186d6b188c0230244582055905155c4620ebe8864838
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.3.11/kubevpn-2.3.11.tgz
+    version: 2.3.11
+  - apiVersion: v2
+    appVersion: v2.3.10
+    created: "2025-01-24T13:36:34.489289734Z"
+    description: A Helm chart for KubeVPN
+    digest: 987b73399637eee01570492115114696fdb054074507f0d16e47d077e4ea770c
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.3.10/kubevpn-2.3.10.tgz
+    version: 2.3.10
+  - apiVersion: v2
+    appVersion: v2.3.9
+    created: "2024-12-21T15:29:42.173109915Z"
+    description: A Helm chart for KubeVPN
+    digest: 0f9dd91504c1d1c3149cca785f0a9d72ef860d002ee73590f41e3d8decc99365
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.3.9/kubevpn-2.3.9.tgz
+    version: 2.3.9
+  - apiVersion: v2
+    appVersion: v2.3.8
+    created: "2024-12-19T14:19:38.126241384Z"
+    description: A Helm chart for KubeVPN
+    digest: 84239f1bce053eaa9314e53b820ad0ba32bbc51c37dcac6ae8abd03bef6f7fd2
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.3.8/kubevpn-2.3.8.tgz
+    version: 2.3.8
+  - apiVersion: v2
+    appVersion: v2.3.7
+    created: "2024-12-14T17:25:08.398840622Z"
+    description: A Helm chart for KubeVPN
+    digest: 437faa6cd98e81c4ad2c1b48c9ef7a33e7d435cf6343c5cc2c88ea251b2a545b
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.3.7/kubevpn-2.3.7.tgz
+    version: 2.3.7
+  - apiVersion: v2
+    appVersion: v2.3.6
+    created: "2024-12-09T11:52:04.779835011Z"
+    description: A Helm chart for KubeVPN
+    digest: 7b23d14f6aea4410d68911d202199f15c88cb96cef8edbd94d4a95e9b9254bf7
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.3.6/kubevpn-2.3.6.tgz
+    version: 2.3.6
+  - apiVersion: v2
+    appVersion: v2.3.5
+    created: "2024-12-06T14:40:11.685095653Z"
+    description: A Helm chart for KubeVPN
+    digest: c2a85f446af834b60308b1384e6cae5662229c34370053319c0f759f650a1cb5
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.3.5/kubevpn-2.3.5.tgz
+    version: 2.3.5
+  - apiVersion: v2
+    appVersion: v2.3.4
+    created: "2024-11-29T13:03:24.255324387Z"
+    description: A Helm chart for KubeVPN
+    digest: 2804aa624f6139695f3fb723bdc6ba087492bcd8810baf7196a1ae88bd2a62b5
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.3.4/kubevpn-2.3.4.tgz
+    version: 2.3.4
+  - apiVersion: v2
+    appVersion: v2.3.3
+    created: "2024-11-22T14:54:13.795282085Z"
+    description: A Helm chart for KubeVPN
+    digest: 33cbbc9312e7b7e415fb14f80f17df50d305194617bcf75d1501227cb90b8f32
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.3.3/kubevpn-2.3.3.tgz
+    version: 2.3.3
+  - apiVersion: v2
+    appVersion: v2.3.2
+    created: "2024-11-18T11:52:12.076510627Z"
+    description: A Helm chart for KubeVPN
+    digest: cdb38ab84bf1649ac4280f6996060c49a095f9c056044cd5f691e7bf4f259dad
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.3.2/kubevpn-2.3.2.tgz
+    version: 2.3.2
+  - apiVersion: v2
+    appVersion: v2.3.1
+    created: "2024-11-15T13:36:37.056311943Z"
+    description: A Helm chart for KubeVPN
+    digest: 10c1200241309be4ec2eb88e9689ebbf96704c8fad270e6fda30047135aeccf2
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.3.1/kubevpn-2.3.1.tgz
+    version: 2.3.1
+  - apiVersion: v2
+    appVersion: v2.2.22
+    created: "2024-10-30T08:46:08.845218523Z"
+    description: A Helm chart for KubeVPN
+    digest: c2dc336383d7de2fb97cfd40a15e9f6c29a9a598484b88515a98bcaeb4925eda
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.22/kubevpn-2.2.22.tgz
+    version: 2.2.22
+  - apiVersion: v2
+    appVersion: v2.2.21
+    created: "2024-10-25T14:10:25.545716679Z"
+    description: A Helm chart for KubeVPN
+    digest: 98ae51247535525ff6a10b5f493d8bfc573af62759432f7aa54dd7eb6edeffd5
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.21/kubevpn-2.2.21.tgz
+    version: 2.2.21
+  - apiVersion: v2
+    appVersion: v2.2.20
+    created: "2024-10-20T04:00:07.263734809Z"
+    description: A Helm chart for KubeVPN
+    digest: 7863701dff5b3fce0795ee8e0b73044b7c88f8777c86a65adc1f5563123565dc
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.20/kubevpn-2.2.20.tgz
+    version: 2.2.20
+  - apiVersion: v2
+    appVersion: v2.2.19
+    created: "2024-10-10T00:47:08.858011096Z"
+    description: A Helm chart for KubeVPN
+    digest: be2c672081307c03b7fe6b635d524c8f3f73d70ae3316efa85e781a62c25a46d
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.19/kubevpn-2.2.19.tgz
+    version: 2.2.19
+  - apiVersion: v2
+    appVersion: v2.2.18
+    created: "2024-09-10T09:39:11.71407425Z"
+    description: A Helm chart for KubeVPN
+    digest: 2d953103425ca2a087a2d521c9297662f97b72e78cf831e947942f292bbcc643
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.18/kubevpn-2.2.18.tgz
+    version: 2.2.18
+  - apiVersion: v2
+    appVersion: v2.2.17
+    created: "2024-08-03T07:45:55.228743946Z"
+    description: A Helm chart for KubeVPN
+    digest: 476317ad82b2c59a623e1fca968c09a28554ebcabec337c1c363e7296bb27514
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.17/kubevpn-2.2.17.tgz
+    version: 2.2.17
+  - apiVersion: v2
+    appVersion: v2.2.16
+    created: "2024-07-26T13:43:50.473565863Z"
+    description: A Helm chart for KubeVPN
+    digest: 6cdb809d04687197a8defbf4349871c505ac699924833fecc210d8a6d82a9f20
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.16/kubevpn-2.2.16.tgz
+    version: 2.2.16
+  - apiVersion: v2
+    appVersion: v2.2.15
+    created: "2024-07-19T15:03:13.558586823Z"
+    description: A Helm chart for KubeVPN
+    digest: 279b24976cef25e1dd8a4cd612a7c6a5767cecd4ba386ccab80fc00db76117e7
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.15/kubevpn-2.2.15.tgz
+    version: 2.2.15
+  - apiVersion: v2
+    appVersion: v2.2.14
+    created: "2024-07-12T15:24:27.825047662Z"
+    description: A Helm chart for KubeVPN
+    digest: 52ab9b89ea3773792bf3839e4a7c23a9ea60a6c72547024dc0907c973a8d34b3
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.14/kubevpn-2.2.14.tgz
+    version: 2.2.14
+  - apiVersion: v2
+    appVersion: v2.2.13
+    created: "2024-07-05T15:08:40.140645659Z"
+    description: A Helm chart for KubeVPN
+    digest: 610c5528952826839d5636b8bd940ac907ab0e70377e37538063cb53a5f75443
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.13/kubevpn-2.2.13.tgz
+    version: 2.2.13
+  - apiVersion: v2
+    appVersion: v2.2.12
+    created: "2024-06-29T15:36:12.429229459Z"
+    description: A Helm chart for KubeVPN
+    digest: a129ac0efda2e2967937407b904d59122e7b9725fb225c0bcbfdf2260337c032
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.12/kubevpn-2.2.12.tgz
+    version: 2.2.12
+  - apiVersion: v2
+    appVersion: v2.2.11
+    created: "2024-06-21T14:13:53.982206886Z"
+    description: A Helm chart for KubeVPN
+    digest: 3a7fa4cb3e1785da68e422ef151a3c7f621fbe76862b557ae2750af70d34e1ad
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.11/kubevpn-2.2.11.tgz
+    version: 2.2.11
+  - apiVersion: v2
+    appVersion: v2.2.10
+    created: "2024-05-21T06:46:20.368800554Z"
+    description: A Helm chart for KubeVPN
+    digest: 89be252c9eedb13560224550f06270f8be88049edfb0a46ca170ab5c8c493a6c
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.10/kubevpn-2.2.10.tgz
+    version: 2.2.10
+  - apiVersion: v2
+    appVersion: v2.2.9
+    created: "2024-05-14T11:50:54.700148975Z"
+    description: A Helm chart for KubeVPN
+    digest: e94debe7c904e21f791c1e3bb877ca8132888a3bb3c53beaa74e2ff1e7dd8769
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.9/kubevpn-2.2.9.tgz
+    version: 2.2.9
+  - apiVersion: v2
+    appVersion: v2.2.8
+    created: "2024-05-03T15:50:13.647253665Z"
+    description: A Helm chart for KubeVPN
+    digest: 9e18d0d02f123e5d8f096362daa5e6893d5db1e8447a632585ae23d6ce755489
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.8/kubevpn-2.2.8.tgz
+    version: 2.2.8
+  - apiVersion: v2
+    appVersion: v2.2.7
+    created: "2024-04-27T12:11:35.594701859Z"
+    description: A Helm chart for KubeVPN
+    digest: 3828f5b20d6bf4c0c7d94654cc33fd8d7b4c5f2aa20a3cc18d18b9298f459456
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.7/kubevpn-2.2.7.tgz
+    version: 2.2.7
+  - apiVersion: v2
+    appVersion: v2.2.6
+    created: "2024-04-16T05:44:31.777079658Z"
+    description: A Helm chart for KubeVPN
+    digest: 63668930b99e6c18f6dd77a25e5ce2d21579d52a83451f58be3bc0ca32678829
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.6/kubevpn-2.2.6.tgz
+    version: 2.2.6
+  - apiVersion: v2
+    appVersion: v2.2.5
+    created: "2024-04-14T08:46:13.877936123Z"
+    description: A Helm chart for KubeVPN
+    digest: 8509aeec7584935344bdf465efd8f0d5efb58ef1b7a31fd2738e5c2790f680c4
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.5/kubevpn-2.2.5.tgz
+    version: 2.2.5
+  - apiVersion: v2
+    appVersion: v2.2.4
+    created: "2024-04-02T05:15:00.372823536Z"
+    description: A Helm chart for KubeVPN
+    digest: 07e87e648b7ad5688146a356c93c1771e94485c2fd9d5441553d94ce6371c19f
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.4/kubevpn-2.2.4.tgz
+    version: 2.2.4
+  - apiVersion: v2
+    appVersion: v2.2.3
+    created: "2024-03-03T11:52:37.856463964Z"
+    description: A Helm chart for KubeVPN
+    digest: cb1b8c210259292488548853bdeb2eb9ef4c60d1643e0d6537174349514dc8e9
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.3/kubevpn-2.2.3.tgz
+    version: 2.2.3
+  - apiVersion: v2
+    appVersion: v2.2.2
+    created: "2024-02-15T13:35:35.121411893Z"
+    description: A Helm chart for KubeVPN
+    digest: b7589312eab83e50db9ae5703a30e76f0b40fd280c81d102a823aeeb61e14c1c
+    name: kubevpn
+    type: application
+    urls:
+    - https://github.com/kubenetworks/kubevpn/releases/download/v2.2.2/kubevpn-2.2.2.tgz
+    version: 2.2.2
+generated: "2025-04-25T15:40:08.297053107Z"

charts/kubevpn/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/kubevpn/Chart.yaml

@@ -0,0 +1,8 @@
+apiVersion: v2
+name: kubevpn
+description: A Helm chart for KubeVPN
+type: application
+version: 0.1.0
+appVersion: "1.16.0"
+annotations:
+  app: kubevpn

charts/kubevpn/README.md

@@ -0,0 +1,24 @@
+# Helm charts for KubeVPN server
+
+Use helm to install kubevpn server means use cluster mode. All user will use this instance.
+
+- Please make sure users should have permission to namespace `kubevpn`.
+- Otherwise, will fall back to create `kubevpn` deployment in own namespace.
+
+## Install with default mode
+
+```shell
+helm install kubevpn kubevpn/kubevpn -n kubevpn --create-namespace
+```
+
+in China, you can use tencent image registry
+
+```shell
+helm install kubevpn kubevpn/kubevpn --set image.repository=ccr.ccs.tencentyun.com/kubevpn/kubevpn -n kubevpn --create-namespace
+```
+
+## AWS Fargate cluster
+
+```shell
+helm install kubevpn kubevpn/kubevpn --set netstack=gvisor -n kubevpn --create-namespace
+```

charts/kubevpn/templates/NOTES.txt

@@ -0,0 +1,4 @@
+1. Connect to cluster network by running these commands:
+  kubevpn connect --namespace {{ include "kubevpn.namespace" . }}
+  export POD_IP=$(kubectl get pods --namespace {{ include "kubevpn.namespace" . }} -l "app.kubernetes.io/name={{ include "kubevpn.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].status.podIP}")
+  ping $POD_IP

charts/kubevpn/templates/_helpers.tpl

@@ -0,0 +1,82 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "kubevpn.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "kubevpn.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "kubevpn.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "kubevpn.labels" -}}
+helm.sh/chart: {{ include "kubevpn.chart" . }}
+app: kubevpn-traffic-manager
+{{ include "kubevpn.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "kubevpn.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "kubevpn.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "kubevpn.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "kubevpn.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Namespace
+1. special by -n
+2. use default namespace kubevpn
+*/}}
+{{- define "kubevpn.namespace" -}}
+{{- if .Release.Namespace }}
+  {{- if eq .Release.Namespace "default" }}
+{{- .Values.namespace }}
+  {{- else }}
+{{- .Release.Namespace }}
+  {{- end }}
+{{- else if .Values.namespace }}
+{{- .Values.namespace }}
+{{- else }}
+{{- .Values.namespace }}
+{{- end }}
+{{- end }}

charts/kubevpn/templates/configmap.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "kubevpn.fullname" . }}
+  namespace: {{ include "kubevpn.namespace" . }}
+data:
+  DHCP: ""
+  DHCP6: ""
+  ENVOY_CONFIG: ""
+  IPv4_POOLS: "{{ .Values.cidr.pod }} {{ .Values.cidr.service }}"
+  REF_COUNT: "0"

charts/kubevpn/templates/deployment.yaml

@@ -0,0 +1,138 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "kubevpn.fullname" . }}
+  namespace: {{ include "kubevpn.namespace" . }}
+  labels:
+    {{- include "kubevpn.labels" . | nindent 4 }}
+spec:
+  {{- if not .Values.autoscaling.enabled }}
+  replicas: {{ .Values.replicaCount }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "kubevpn.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "kubevpn.labels" . | nindent 8 }}
+        {{- with .Values.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "kubevpn.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - args:
+            {{- if eq .Values.netstack "system" }}
+            - |
+              echo 1 > /proc/sys/net/ipv4/ip_forward
+              echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6
+              echo 1 > /proc/sys/net/ipv6/conf/all/forwarding
+              update-alternatives --set iptables /usr/sbin/iptables-legacy
+              iptables -P INPUT ACCEPT
+              ip6tables -P INPUT ACCEPT
+              iptables -P FORWARD ACCEPT
+              ip6tables -P FORWARD ACCEPT
+              iptables -t nat -A POSTROUTING -s ${CIDR4} -o eth0 -j MASQUERADE
+              ip6tables -t nat -A POSTROUTING -s ${CIDR6} -o eth0 -j MASQUERADE
+              kubevpn server -l "tcp://:10800" -l "tun://:8422?net=${TunIPv4}&net6=${TunIPv6}" -l "gtcp://:10801" -l "gudp://:10802"
+            {{- else }}
+            - kubevpn server -l "tcp://:10800" -l "gtcp://:10801" -l "gudp://:10802"
+            {{- end }}
+          command:
+            - /bin/sh
+            - -c
+          env:
+            - name: CIDR4
+              value: 198.19.0.0/16
+            - name: CIDR6
+              value: 2001:2::/64
+            - name: TunIPv4
+              value: 198.19.0.100/16
+            - name: TunIPv6
+              value: 2001:2::9999/64
+          envFrom:
+            - secretRef:
+                name: {{ include "kubevpn.fullname" . }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          name: vpn
+          ports:
+            - containerPort: {{ .Values.service.port8422 }}
+              name: 8422-for-udp
+              protocol: UDP
+            - containerPort: {{ .Values.service.port10800 }}
+              name: 10800-for-tcp
+              protocol: TCP
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          {{- if eq .Values.netstack "system" }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          {{- end }}
+        - args:
+            - control-plane
+            - --watchDirectoryFilename
+            - /etc/envoy/envoy-config.yaml
+          command:
+            - kubevpn
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          name: control-plane
+          ports:
+            - containerPort: {{ .Values.service.port9002 }}
+              name: 9002-for-envoy
+              protocol: TCP
+          resources:
+            {{- toYaml .Values.resourcesSmall | nindent 12 }}
+          volumeMounts:
+            - mountPath: /etc/envoy
+              name: envoy-config
+              readOnly: true
+        - args:
+            - webhook
+          command:
+            - kubevpn
+          envFrom:
+            - secretRef:
+                name: {{ include "kubevpn.fullname" . }}
+          env:
+            - name: "POD_NAMESPACE"
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          name: webhook
+          ports:
+            - containerPort: 80
+              name: 80-for-webhook
+              protocol: TCP
+          resources:
+            {{- toYaml .Values.resourcesSmall | nindent 12 }}
+      {{- with .Values.volumes }}
+      volumes:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

charts/kubevpn/templates/hpa.yaml

@@ -0,0 +1,33 @@
+{{- if .Values.autoscaling.enabled }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "kubevpn.fullname" . }}
+  namespace: {{ include "kubevpn.namespace" . }}
+  labels:
+    {{- include "kubevpn.labels" . | nindent 4 }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "kubevpn.fullname" . }}
+  minReplicas: {{ .Values.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.autoscaling.maxReplicas }}
+  metrics:
+    {{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
+    {{- end }}
+    {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
+    {{- end }}
+{{- end }}

charts/kubevpn/templates/job.yaml

@@ -0,0 +1,73 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "kubevpn.fullname" . }}
+  namespace: {{ include "kubevpn.namespace" . }}
+  labels:
+    {{- include "kubevpn.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": post-install
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "kubevpn.labels" . | nindent 8 }}
+        {{- with .Values.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      restartPolicy: Never
+      serviceAccountName: {{ include "kubevpn.serviceAccountName" . }}
+      containers:
+        - name: label-ns
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command:
+            - /bin/bash
+            - -c
+          args:
+            - |2-
+
+              echo "Label namespace {{ include "kubevpn.namespace" . }}"
+              kubectl label ns {{ include "kubevpn.namespace" . }} ns={{ include "kubevpn.namespace" . }}
+
+              echo "Generating https certificate"
+              openssl req -x509 -nodes -days 36500 -newkey rsa:2048 -subj "/CN={{ include "kubevpn.fullname" . }}.{{ include "kubevpn.namespace" . }}" -addext "subjectAltName=DNS:{{ include "kubevpn.fullname" . }}.{{ include "kubevpn.namespace" . }}.svc.cluster.local,DNS:{{ include "kubevpn.fullname" . }}.{{ include "kubevpn.namespace" . }}.svc,DNS:{{ include "kubevpn.fullname" . }}.{{ include "kubevpn.namespace" . }},DNS:localhost,IP:127.0.0.1" -keyout server.key -out server.crt
+
+              export TLS_CRT=$(cat server.crt | base64 | tr -d '\n')
+              echo "Patch mutatingwebhookconfigurations {{ include "kubevpn.fullname" . }}.{{ include "kubevpn.namespace" . }}"
+              kubectl patch mutatingwebhookconfigurations {{ include "kubevpn.fullname" . }}.{{ include "kubevpn.namespace" . }} -p "{\"webhooks\":[{\"name\":\"{{ include "kubevpn.fullname" . }}.naison.io\",\"sideEffects\":\"None\",\"admissionReviewVersions\":[\"v1\", \"v1beta1\"],\"clientConfig\":{\"service\":{\"namespace\":\"{{ include "kubevpn.namespace" . }}\",\"name\":\"{{ include "kubevpn.fullname" . }}\"},\"caBundle\":\"$TLS_CRT\"}}]}"
+
+              export TLS_KEY=$(cat server.key | base64 | tr -d '\n')
+              echo "Patch secret {{ include "kubevpn.fullname" . }}"
+              kubectl patch secret {{ include "kubevpn.fullname" . }} -n {{ include "kubevpn.namespace" . }} -p "{\"data\":{\"tls_key\":\"$TLS_KEY\",\"tls_crt\":\"$TLS_CRT\"}}"
+
+              echo "Restart the pods..."
+              kubectl scale -n {{ include "kubevpn.namespace" . }} --replicas=0 deployment/{{ include "kubevpn.fullname" . }}
+              kubectl scale -n {{ include "kubevpn.namespace" . }} --replicas=1 deployment/{{ include "kubevpn.fullname" . }}
+
+              export POOLS=$(kubectl get cm {{ include "kubevpn.fullname" . }} -n {{ include "kubevpn.namespace" . }} -o jsonpath='{.data.IPv4_POOLS}')
+              if [[ -z "${POOLS// }" ]];then
+                echo "Cidr is empty"
+                echo "Get pod cidr..."
+                export POD_CIDR=$(kubectl get nodes -o jsonpath='{.items[*].spec.podCIDR}' | tr -s '\n' ' ')
+                echo "Get service cidr..."
+                export SVC_CIDR=$(echo '{"apiVersion":"v1","kind":"Service","metadata":{"name":"kubevpn-get-svc-cidr-{{ include "kubevpn.namespace" . }}", "namespace": "{{ include "kubevpn.namespace" . }}"},"spec":{"clusterIP":"1.1.1.1","ports":[{"port":443}]}}' | kubectl apply -f - 2>&1 | sed 's/.*valid IPs is //')
+                echo "Pod cidr: $POD_CIDR, service cidr: $SVC_CIDR"
+                echo "Patch configmap {{ include "kubevpn.fullname" . }}"
+                kubectl patch configmap {{ include "kubevpn.fullname" . }} -n {{ include "kubevpn.namespace" . }} -p "{\"data\":{\"IPv4_POOLS\":\"$POD_CIDR $SVC_CIDR\"}}"
+              else
+                echo "Cidr is NOT empty"
+              fi
+
+              echo "Done~"
+              exit 0

charts/kubevpn/templates/mutatingwebhookconfiguration.yaml

@@ -0,0 +1,35 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: {{ include "kubevpn.fullname" . }}.{{ include "kubevpn.namespace" . }}
+  namespace: {{ include "kubevpn.namespace" . }}
+webhooks:
+  - admissionReviewVersions:
+      - v1
+      - v1beta1
+    clientConfig:
+      caBundle: {{ .Values.tls.crt }}
+      service:
+        name: {{ include "kubevpn.fullname" . }}
+        namespace: {{ include "kubevpn.namespace" . }}
+        path: /pods
+        port: 80
+    failurePolicy: Ignore
+    matchPolicy: Equivalent
+    name: {{ include "kubevpn.fullname" . }}.naison.io
+    namespaceSelector: { }
+    objectSelector: { }
+    reinvocationPolicy: Never
+    rules:
+      - apiGroups:
+          - ""
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+          - DELETE
+        resources:
+          - pods
+        scope: Namespaced
+    sideEffects: None
+    timeoutSeconds: 15

charts/kubevpn/templates/role.yaml

@@ -0,0 +1,70 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "kubevpn.fullname" . }}
+  namespace: {{ include "kubevpn.namespace" . }}
+rules:
+  - apiGroups:
+      - ""
+    resourceNames:
+      - {{ include "kubevpn.fullname" . }}
+    resources:
+      - configmaps
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - patch
+      - delete
+  - apiGroups: [ "" ]
+    resources: [ "namespaces" ]
+    resourceNames: ["{{ include "kubevpn.namespace" . }}"]
+    verbs:
+      - get
+      - patch
+  - apiGroups: [ "apps" ]
+    resources: [ "deployments/scale", "deployments" ]
+    resourceNames:
+      - {{ include "kubevpn.fullname" . }}
+    verbs:
+      - get
+      - update
+      - patch
+      - list
+  - apiGroups:
+      - ""
+    resources:
+      - services
+    verbs:
+      - create
+      - get
+      - update
+      - patch
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "kubevpn.fullname" . }}.{{ include "kubevpn.namespace" . }}
+rules:
+  - apiGroups:
+      - admissionregistration.k8s.io
+    resources:
+      - mutatingwebhookconfigurations
+    resourceNames:
+      - {{ include "kubevpn.fullname" . }}.{{ include "kubevpn.namespace" . }}
+    verbs:
+      - get
+      - list
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - get
+      - list
+      - watch

charts/kubevpn/templates/rolebinding.yaml

@@ -0,0 +1,27 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "kubevpn.fullname" . }}
+  namespace: {{ include "kubevpn.namespace" . }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "kubevpn.fullname" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "kubevpn.fullname" . }}
+    namespace: {{ include "kubevpn.namespace" . }}
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "kubevpn.fullname" . }}.{{ include "kubevpn.namespace" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "kubevpn.fullname" . }}
+    namespace: {{ include "kubevpn.namespace" . }}
+roleRef:
+  kind: ClusterRole
+  name: {{ include "kubevpn.fullname" . }}.{{ include "kubevpn.namespace" . }}
+  apiGroup: rbac.authorization.k8s.io

charts/kubevpn/templates/secret.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "kubevpn.fullname" . }}
+  namespace: {{ include "kubevpn.namespace" . }}
+type: Opaque
+stringData:
+  tls_crt: {{ .Values.tls.crt }}
+  tls_key: {{ .Values.tls.key }}
+  tls_server_name: {{ include "kubevpn.fullname" . }}.{{ include "kubevpn.namespace" . }}

charts/kubevpn/templates/service.yaml

@@ -0,0 +1,32 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "kubevpn.fullname" . }}
+  namespace: {{ include "kubevpn.namespace" . }}
+  labels:
+    {{- include "kubevpn.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - name: 8422-for-udp
+      port: {{ .Values.service.port8422 }}
+      protocol: UDP
+      targetPort: 8422
+    - name: 10800-for-tcp
+      port: {{ .Values.service.port10800 }}
+      protocol: TCP
+      targetPort: 10800
+    - name: 9002-for-envoy
+      port: {{ .Values.service.port9002 }}
+      protocol: TCP
+      targetPort: 9002
+    - name: 80-for-webhook
+      port: {{ .Values.service.port80 }}
+      protocol: TCP
+      targetPort: 80
+    - name: 53-for-dns
+      port: {{ .Values.service.port53 }}
+      protocol: UDP
+      targetPort: 53
+  selector:
+    {{- include "kubevpn.selectorLabels" . | nindent 4 }}

charts/kubevpn/templates/serviceaccount.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "kubevpn.serviceAccountName" . }}
+  namespace: {{ include "kubevpn.namespace" . }}
+  labels:
+    {{- include "kubevpn.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
+{{- end }}

charts/kubevpn/values.yaml

@@ -0,0 +1,120 @@
+# Default values for kubevpn.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# default namespace
+namespace: kubevpn
+# default is system mode, available ["system", "gvisor"]
+# system: needs privilege permission and cap NET_ADMIN (Best experience)
+# gvisor: no needs any additional permission (Best compatibility)
+netstack: system
+
+replicaCount: 1
+
+image:
+  repository: ghcr.io/kubenetworks/kubevpn
+  pullPolicy: IfNotPresent
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: ""
+
+imagePullSecrets: [ ]
+nameOverride: ""
+fullnameOverride: "kubevpn-traffic-manager"
+
+# this filed is import if configured this value
+# if not configured, it will get this value from cluster automatically
+cidr:
+  pod: ""
+  service: ""
+
+tls:
+  # will auto generate in job
+  crt: ''''''
+  key: ''''''
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Automatically mount a ServiceAccount's API credentials?
+  automount: true
+  # Annotations to add to the service account
+  annotations: { }
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+podAnnotations: { }
+podLabels:
+
+podSecurityContext: { }
+# fsGroup: 2000
+securityContext:
+  capabilities:
+    add:
+      - NET_ADMIN
+  privileged: true
+  runAsUser: 0
+  runAsGroup: 0
+
+service:
+  type: ClusterIP
+  port8422: 8422
+  port9002: 9002
+  port10800: 10800
+  port80: 80
+  port53: 53
+
+resources:
+  limits:
+    cpu: "2"
+    memory: 2Gi
+  requests:
+    cpu: 500m
+    memory: 512Mi
+resourcesSmall:
+  limits:
+    cpu: 100m
+    memory: 128Mi
+  requests:
+    cpu: 100m
+    memory: 128Mi
+
+livenessProbe:
+  httpGet:
+    path: /
+    port: http
+readinessProbe:
+  httpGet:
+    path: /
+    port: http
+
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 1
+  targetCPUUtilizationPercentage: 80
+  # targetMemoryUtilizationPercentage: 80
+
+# Additional volumes on the output Deployment definition.
+volumes:
+  - configMap:
+      defaultMode: 420
+      items:
+        - key: ENVOY_CONFIG
+          path: envoy-config.yaml
+      name: kubevpn-traffic-manager
+      optional: false
+    name: envoy-config
+
+
+# Additional volumeMounts on the output Deployment definition.
+volumeMounts: [ ]
+# - name: foo
+#   mountPath: "/etc/foo"
+#   readOnly: true
+
+nodeSelector: { }
+
+tolerations: [ ]
+
+affinity: { }

cmd/kubevpn/cmds/alias.go

@@ -0,0 +1,210 @@
+package cmds
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"k8s.io/apimachinery/pkg/util/sets"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+	yaml "sigs.k8s.io/yaml/goyaml.v3"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/config"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
+)
+
+// CmdAlias
+/**
+Name: test
+Description: this is a test environment
+Needs: test1
+Flags:
+  - connect
+  - --kubeconfig=~/.kube/config
+  - --namespace=test
+  - --lite
+---
+
+Name: test1
+Description: this is another test environment
+Flags:
+  - connect
+  - --kubeconfig=~/.kube/jumper_config
+  - --namespace=test
+  - --extra-hosts=xxx.com
+*/
+func CmdAlias(f cmdutil.Factory) *cobra.Command {
+	var localFile, remoteAddr string
+	cmd := &cobra.Command{
+		Use:   "alias",
+		Short: i18n.T("Config file alias to execute command simply"),
+		Long: templates.LongDesc(i18n.T(`
+		Config file alias to execute command simply, just like ssh alias config
+
+		Please point to an existing, complete config file:
+
+		1. Via the command-line flag --kubevpnconfig
+		2. Via the KUBEVPNCONFIG environment variable
+		3. In your home directory as ~/.kubevpn/config.yaml
+
+		It will read ~/.kubevpn/config.yaml file as config, also support special file path
+		by flag -f. It also supports depends relationship, like one cluster api server needs to 
+		access via another cluster, you can use syntax needs. it will do action to needs cluster first
+ 		and then do action to target cluster
+		`)),
+		Example: templates.Examples(i18n.T(`
+		If you have following config in your ~/.kubevpn/config.yaml
+
+		Name: dev
+		Needs: jumper
+		Flags:
+		  - connect
+		  - --kubeconfig=~/.kube/config
+		  - --namespace=default
+		  - --lite
+		---
+		
+		Name: jumper
+		Flags:
+		  - connect
+		  - --kubeconfig=~/.kube/jumper_config
+		  - --namespace=test
+		  - --extra-hosts=xxx.com
+		
+		Config file support three field: Name,Needs,Flags
+
+		# Use kubevpn alias config to simply execute command, connect to cluster network by order: jumper --> dev
+		kubevpn alias dev
+		
+		# kubevpn alias jumper, just connect to cluster jumper
+		kubevpn alias jumper
+		`)),
+		PreRunE: func(cmd *cobra.Command, args []string) (err error) {
+			if localFile != "" {
+				_, err = os.Stat(localFile)
+			}
+			return err
+		},
+		Args: cobra.MatchAll(cobra.ExactArgs(1)),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			configs, err := ParseAndGet(localFile, remoteAddr, args[0])
+			if err != nil {
+				return err
+			}
+			name, err := os.Executable()
+			if err != nil {
+				return err
+			}
+			for _, conf := range configs {
+				c := exec.Command(name, conf.Flags...)
+				c.Stdout = os.Stdout
+				c.Stdin = os.Stdin
+				c.Stderr = os.Stderr
+				fmt.Println(fmt.Sprintf("Name: %s", conf.Name))
+				if conf.Description != "" {
+					fmt.Println(fmt.Sprintf("Description: %s", conf.Description))
+				}
+				fmt.Println(fmt.Sprintf("Command: %v", c.Args))
+				err = c.Run()
+				if err != nil {
+					return err
+				}
+			}
+			return nil
+		},
+	}
+	cmd.Flags().StringVarP(&localFile, "kubevpnconfig", "f", util.If(os.Getenv("KUBEVPNCONFIG") != "", os.Getenv("KUBEVPNCONFIG"), config.GetConfigFilePath()), "Path to the kubevpnconfig file to use for CLI requests.")
+	cmd.Flags().StringVarP(&remoteAddr, "remote", "r", "", "Remote config file, eg: https://raw.githubusercontent.com/kubenetworks/kubevpn/master/pkg/config/config.yaml")
+	return cmd
+}
+
+func ParseAndGet(localFile, remoteAddr string, aliasName string) ([]Config, error) {
+	var content []byte
+	var err error
+	var path string
+	if localFile != "" {
+		path = localFile
+		content, err = os.ReadFile(path)
+	} else if remoteAddr != "" {
+		path = remoteAddr
+		content, err = util.DownloadFileStream(path)
+	} else {
+		path = config.GetConfigFilePath()
+		content, err = os.ReadFile(path)
+	}
+	if err != nil {
+		return nil, err
+	}
+	list, err := ParseConfig(content)
+	if err != nil {
+		return nil, err
+	}
+	configs, err := GetConfigs(list, aliasName)
+	if err != nil {
+		return nil, err
+	}
+	if len(configs) == 0 {
+		var names []string
+		for _, c := range list {
+			if c.Name != "" {
+				names = append(names, c.Name)
+			}
+		}
+		err = errors.New(fmt.Sprintf("Can't find any alias for the name: '%s', avaliable: \n[\"%s\"]\nPlease check config file: %s", aliasName, strings.Join(names, "\", \""), path))
+		return nil, err
+	}
+	return configs, nil
+}
+
+func ParseConfig(file []byte) ([]Config, error) {
+	decoder := yaml.NewDecoder(strings.NewReader(string(file)))
+	var configs []Config
+	for {
+		var cfg Config
+		err := decoder.Decode(&cfg)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+		configs = append(configs, cfg)
+	}
+	return configs, nil
+}
+
+func GetConfigs(configs []Config, name string) ([]Config, error) {
+	m := make(map[string]Config)
+	for _, config := range configs {
+		m[config.Name] = config
+	}
+	var result []Config
+	var set []string
+	for !sets.New[string](set...).Has(name) {
+		config, ok := m[name]
+		if ok {
+			result = append([]Config{config}, result...)
+			set = append(set, name)
+			name = config.Needs
+			if name == "" {
+				return result, nil
+			}
+		} else {
+			return result, nil
+		}
+	}
+	return nil, fmt.Errorf("loop jump detected: %s. verify your configuration", strings.Join(append(set, name), " -> "))
+}
+
+type Config struct {
+	Name        string   `yaml:"Name"`
+	Description string   `yaml:"Description"`
+	Needs       string   `yaml:"Needs,omitempty"`
+	Flags       []string `yaml:"Flags,omitempty"`
+}

cmd/kubevpn/cmds/alias_test.go

@@ -0,0 +1,229 @@
+package cmds
+
+import (
+	"context"
+	"reflect"
+	"testing"
+
+	plog "github.com/wencaiwulue/kubevpn/v2/pkg/log"
+)
+
+func TestAlias(t *testing.T) {
+	str := `Name: test
+Needs: test1
+Flags:
+  - --kubeconfig=~/.kube/config
+  - --namespace=test
+
+---
+
+Name: test1
+Flags:
+  - --kubeconfig=~/.kube/config
+  - --namespace=test
+  - --extra-hosts=xxx.com`
+	_, err := ParseConfig([]byte(str))
+	if err != nil {
+		plog.G(context.Background()).Fatal(err)
+	}
+}
+
+func TestCheckLoop(t *testing.T) {
+	str := `Name: test
+Needs: test1
+Flags:
+  - --kubeconfig=~/.kube/config
+  - --namespace=test
+
+---
+
+Name: test1
+Flags:
+  - --kubeconfig=~/.kube/config
+  - --namespace=test
+  - --extra-hosts=xxx.com`
+	_, err := ParseConfig([]byte(str))
+	if err != nil {
+		plog.G(context.Background()).Fatal(err)
+	}
+}
+
+func TestLoop(t *testing.T) {
+	data := []struct {
+		Config      string
+		Run         string
+		ExpectError bool
+		ExpectOrder []string
+	}{
+		{
+			Config: `
+Name: test
+Needs: test1
+Flags:
+  - --kubeconfig=~/.kube/config
+  - --namespace=test
+
+---
+
+Name: test1
+Needs: test
+Flags:
+  - --kubeconfig=~/.kube/config
+  - --namespace=test
+  - --extra-hosts=xxx.com
+`,
+			Run:         "test",
+			ExpectError: true,
+			ExpectOrder: nil,
+		},
+		{
+			Config: `
+Name: test
+Needs: test1
+Flags:
+  - --kubeconfig=~/.kube/config
+  - --namespace=test
+
+---
+
+Name: test1
+Flags:
+  - --kubeconfig=~/.kube/config
+  - --namespace=test
+  - --extra-hosts=xxx.com
+`,
+			Run:         "test",
+			ExpectError: false,
+			ExpectOrder: []string{"test1", "test"},
+		},
+		{
+			Config: `
+Name: test
+Needs: test1
+Flags:
+  - --kubeconfig=~/.kube/config
+  - --namespace=test
+
+---
+
+Name: test1
+Needs: test2
+Flags:
+  - --kubeconfig=~/.kube/config
+  - --namespace=test
+  - --extra-hosts=xxx.com
+`,
+			Run:         "test",
+			ExpectError: false,
+			ExpectOrder: []string{"test1", "test"},
+		},
+		{
+			Config: `
+Name: test
+Needs: test1
+Flags:
+  - --kubeconfig=~/.kube/config
+  - --namespace=test
+
+---
+
+Name: test1
+Needs: test2
+Flags:
+  - --kubeconfig=~/.kube/config
+  - --namespace=test
+  - --extra-hosts=xxx.com
+
+---
+
+Name: test2
+Flags:
+  - --kubeconfig=~/.kube/config
+  - --namespace=test
+  - --extra-hosts=xxx.com
+`,
+			Run:         "test",
+			ExpectError: false,
+			ExpectOrder: []string{"test2", "test1", "test"},
+		},
+		{
+			Config: `
+Name: test
+Needs: test1
+Flags:
+  - --kubeconfig=~/.kube/config
+  - --namespace=test
+
+---
+
+Name: test1
+Needs: test2
+Flags:
+  - --kubeconfig=~/.kube/config
+  - --namespace=test
+  - --extra-hosts=xxx.com
+
+---
+
+Name: test2
+Flags:
+  - --kubeconfig=~/.kube/config
+  - --namespace=test
+  - --extra-hosts=xxx.com
+`,
+			Run:         "test2",
+			ExpectError: false,
+			ExpectOrder: []string{"test2"},
+		},
+		{
+			Config: `
+Name: test
+Needs: test1
+Flags:
+  - --kubeconfig=~/.kube/config
+  - --namespace=test
+
+---
+
+Name: test1
+Needs: test2
+Flags:
+  - --kubeconfig=~/.kube/config
+  - --namespace=test
+  - --extra-hosts=xxx.com
+
+---
+
+Name: test2
+Flags:
+  - --kubeconfig=~/.kube/config
+  - --namespace=test
+  - --extra-hosts=xxx.com
+`,
+			Run:         "test1",
+			ExpectError: false,
+			ExpectOrder: []string{"test2", "test1"},
+		},
+	}
+	for _, datum := range data {
+		configs, err := ParseConfig([]byte(datum.Config))
+		if err != nil {
+			plog.G(context.Background()).Fatal(err)
+		}
+		getConfigs, err := GetConfigs(configs, datum.Run)
+		if err != nil && !datum.ExpectError {
+			plog.G(context.Background()).Fatal(err)
+		} else if err != nil {
+		}
+		if datum.ExpectError {
+			continue
+		}
+		var c []string
+		for _, config := range getConfigs {
+			c = append(c, config.Name)
+		}
+		if !reflect.DeepEqual(c, datum.ExpectOrder) {
+			plog.G(context.Background()).Fatalf("Not match, expect: %v, real: %v", datum.ExpectOrder, c)
+		}
+	}
+}

cmd/kubevpn/cmds/clone.go

@@ -0,0 +1,197 @@
+package cmds
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	pkgerr "github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	utilcomp "k8s.io/kubectl/pkg/util/completion"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/config"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon/rpc"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/handler"
+	plog "github.com/wencaiwulue/kubevpn/v2/pkg/log"
+	pkgssh "github.com/wencaiwulue/kubevpn/v2/pkg/ssh"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util/regctl"
+)
+
+// CmdClone multiple cluster operate, can start up one deployment to another cluster
+// kubectl exec POD_NAME -c CONTAINER_NAME /sbin/killall5 or ephemeralcontainers
+func CmdClone(f cmdutil.Factory) *cobra.Command {
+	var options = handler.CloneOptions{}
+	var sshConf = &pkgssh.SshConfig{}
+	var extraRoute = &handler.ExtraRouteInfo{}
+	var transferImage bool
+	var syncDir string
+	var imagePullSecretName string
+	cmd := &cobra.Command{
+		Use:   "clone",
+		Short: i18n.T("Clone workloads to run in target-kubeconfig cluster with same volume、env、and network"),
+		Long: templates.LongDesc(i18n.T(`
+		Clone workloads to run into target-kubeconfig cluster with same volume、env、and network
+
+		In this way, you can startup another deployment in same cluster or not, but with different image version,
+		it also supports service mesh proxy. only traffic with special header will hit to cloned_resource.
+		`)),
+		Example: templates.Examples(i18n.T(`
+		# clone
+		- clone deployment run into current cluster and current namespace
+		  kubevpn clone deployment/productpage
+
+		- clone deployment run into current cluster with different namespace
+		  kubevpn clone deployment/productpage -n test
+        
+		- clone deployment run into another cluster
+		  kubevpn clone deployment/productpage --target-kubeconfig ~/.kube/other-kubeconfig
+
+        - clone multiple workloads run into current cluster and current namespace
+          kubevpn clone deployment/authors deployment/productpage
+          or 
+          kubevpn clone deployment authors productpage
+
+		# clone with mesh, traffic with header foo=bar, will hit cloned workloads, otherwise hit origin workloads
+		kubevpn clone deployment/productpage --headers foo=bar
+
+		# clone workloads which api-server behind of bastion host or ssh jump host
+		kubevpn clone deployment/productpage --ssh-addr 192.168.1.100:22 --ssh-username root --ssh-keyfile ~/.ssh/ssh.pem --headers foo=bar
+
+		# It also supports ProxyJump, like
+		┌──────┐     ┌──────┐     ┌──────┐     ┌──────┐                 ┌────────────┐
+		│  pc  ├────►│ ssh1 ├────►│ ssh2 ├────►│ ssh3 ├─────►... ─────► │ api-server │
+		└──────┘     └──────┘     └──────┘     └──────┘                 └────────────┘
+		kubevpn clone service/productpage --ssh-alias <alias> --headers foo=bar
+
+		# Support ssh auth GSSAPI
+        kubevpn clone service/productpage --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-keytab /path/to/keytab
+        kubevpn clone service/productpage --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-cache /path/to/cache
+        kubevpn clone service/productpage --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-password <PASSWORD>
+		`)),
+		PreRunE: func(cmd *cobra.Command, args []string) (err error) {
+			plog.InitLoggerForClient()
+			// startup daemon process and sudo process
+			err = daemon.StartupDaemon(cmd.Context())
+			if err != nil {
+				return err
+			}
+			if transferImage {
+				err = regctl.TransferImageWithRegctl(cmd.Context(), config.OriginImage, config.Image)
+			}
+			return err
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				_, _ = fmt.Fprintf(os.Stdout, "You must specify the type of resource to proxy. %s\n\n", cmdutil.SuggestAPIResources("kubevpn"))
+				fullCmdName := cmd.Parent().CommandPath()
+				usageString := "Required resource not specified."
+				if len(fullCmdName) > 0 && cmdutil.IsSiblingCommandExists(cmd, "explain") {
+					usageString = fmt.Sprintf("%s\nUse \"%s explain <resource>\" for a detailed description of that resource (e.g. %[2]s explain pods).", usageString, fullCmdName)
+				}
+				return cmdutil.UsageErrorf(cmd, usageString)
+			}
+			// special empty string, eg: --target-registry ""
+			options.IsChangeTargetRegistry = cmd.Flags().Changed("target-registry")
+
+			if syncDir != "" {
+				local, remote, err := util.ParseDirMapping(syncDir)
+				if err != nil {
+					return pkgerr.Wrapf(err, "options 'sync' is invalid, %s", syncDir)
+				}
+				options.LocalDir = local
+				options.RemoteDir = remote
+			} else {
+				options.RemoteDir = config.DefaultRemoteDir
+			}
+
+			bytes, ns, err := util.ConvertToKubeConfigBytes(f)
+			if err != nil {
+				return err
+			}
+			if !sshConf.IsEmpty() {
+				if ip := util.GetAPIServerFromKubeConfigBytes(bytes); ip != nil {
+					extraRoute.ExtraCIDR = append(extraRoute.ExtraCIDR, ip.String())
+				}
+			}
+			req := &rpc.CloneRequest{
+				KubeconfigBytes:        string(bytes),
+				Namespace:              ns,
+				Headers:                options.Headers,
+				Workloads:              args,
+				ExtraRoute:             extraRoute.ToRPC(),
+				OriginKubeconfigPath:   util.GetKubeConfigPath(f),
+				Engine:                 string(options.Engine),
+				SshJump:                sshConf.ToRPC(),
+				TargetKubeconfig:       options.TargetKubeconfig,
+				TargetNamespace:        options.TargetNamespace,
+				TargetContainer:        options.TargetContainer,
+				TargetImage:            options.TargetImage,
+				TargetRegistry:         options.TargetRegistry,
+				IsChangeTargetRegistry: options.IsChangeTargetRegistry,
+				TransferImage:          transferImage,
+				Image:                  config.Image,
+				ImagePullSecretName:    imagePullSecretName,
+				Level:                  int32(util.If(config.Debug, log.DebugLevel, log.InfoLevel)),
+				LocalDir:               options.LocalDir,
+				RemoteDir:              options.RemoteDir,
+			}
+			cli, err := daemon.GetClient(false)
+			if err != nil {
+				return err
+			}
+			resp, err := cli.Clone(cmd.Context(), req)
+			if err != nil {
+				return err
+			}
+			err = util.PrintGRPCStream[rpc.CloneResponse](resp)
+			if err != nil {
+				if status.Code(err) == codes.Canceled {
+					err = remove(cli, args)
+					return err
+				}
+				return err
+			}
+			util.Print(os.Stdout, config.Slogan)
+			return nil
+		},
+	}
+	cmd.Flags().StringToStringVarP(&options.Headers, "headers", "H", map[string]string{}, "Traffic with special headers (use `and` to match all headers) with reverse it to target cluster cloned workloads, If not special, redirect all traffic to target cluster cloned workloads. eg: --headers foo=bar --headers env=dev")
+	handler.AddCommonFlags(cmd.Flags(), &transferImage, &imagePullSecretName, &options.Engine)
+
+	cmdutil.AddContainerVarFlags(cmd, &options.TargetContainer, options.TargetContainer)
+	cmd.Flags().StringVar(&options.TargetImage, "target-image", "", "Clone container use this image to startup container, if not special, use origin image")
+	cmd.Flags().StringVar(&options.TargetNamespace, "target-namespace", "", "Clone workloads in this namespace, if not special, use origin namespace")
+	cmd.Flags().StringVar(&options.TargetKubeconfig, "target-kubeconfig", "", "Clone workloads will create in this cluster, if not special, use origin cluster")
+	cmd.Flags().StringVar(&options.TargetRegistry, "target-registry", "", "Clone workloads will create this registry domain to replace origin registry, if not special, use origin registry")
+	cmd.Flags().StringVar(&syncDir, "sync", "", "Sync local dir to remote pod dir. format: LOCAL_DIR:REMOTE_DIR, eg: ~/code:/app/code")
+
+	handler.AddExtraRoute(cmd.Flags(), extraRoute)
+	pkgssh.AddSshFlags(cmd.Flags(), sshConf)
+	cmd.ValidArgsFunction = utilcomp.ResourceTypeAndNameCompletionFunc(f)
+	return cmd
+}
+
+func remove(cli rpc.DaemonClient, args []string) error {
+	resp, err := cli.Remove(context.Background(), &rpc.RemoveRequest{
+		Workloads: args,
+	})
+	if err != nil {
+		return err
+	}
+	err = util.PrintGRPCStream[rpc.DisconnectResponse](resp)
+	if err != nil {
+		if status.Code(err) == codes.Canceled {
+			return nil
+		}
+		return err
+	}
+	return nil
+}

cmd/kubevpn/cmds/config.go

@@ -0,0 +1,106 @@
+package cmds
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon/rpc"
+	pkgssh "github.com/wencaiwulue/kubevpn/v2/pkg/ssh"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
+)
+
+func CmdConfig(f cmdutil.Factory) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "config",
+		Short: "Proxy kubeconfig which behind of ssh jump server",
+	}
+	cmd.AddCommand(cmdConfigAdd(f))
+	cmd.AddCommand(cmdConfigRemove(f))
+	return cmd
+}
+
+func cmdConfigAdd(f cmdutil.Factory) *cobra.Command {
+	var sshConf = &pkgssh.SshConfig{}
+	cmd := &cobra.Command{
+		Use:   "add",
+		Short: i18n.T("Proxy kubeconfig"),
+		Long:  templates.LongDesc(i18n.T(`proxy kubeconfig which behind of ssh jump server`)),
+		Example: templates.Examples(i18n.T(`
+		# proxy api-server which api-server behind of bastion host or ssh jump host
+		kubevpn config add --ssh-addr 192.168.1.100:22 --ssh-username root --ssh-keyfile ~/.ssh/ssh.pem
+
+		# It also supports ProxyJump, like
+		┌──────┐     ┌──────┐     ┌──────┐     ┌──────┐                 ┌────────────┐
+		│  pc  ├────►│ ssh1 ├────►│ ssh2 ├────►│ ssh3 ├─────►... ─────► │ api-server │
+		└──────┘     └──────┘     └──────┘     └──────┘                 └────────────┘
+		kubevpn config add --ssh-alias <alias>
+		`)),
+		PreRunE: func(cmd *cobra.Command, args []string) (err error) {
+			// startup daemon process and sudo process
+			return daemon.StartupDaemon(cmd.Context())
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			bytes, ns, err := util.ConvertToKubeConfigBytes(f)
+			if err != nil {
+				return err
+			}
+			req := &rpc.ConfigAddRequest{
+				KubeconfigBytes: string(bytes),
+				Namespace:       ns,
+				SshJump:         sshConf.ToRPC(),
+			}
+			cli, err := daemon.GetClient(false)
+			if err != nil {
+				return err
+			}
+			resp, err := cli.ConfigAdd(cmd.Context(), req)
+			if err != nil {
+				return err
+			}
+			_, _ = fmt.Fprint(os.Stdout, resp.ClusterID)
+			return nil
+		},
+	}
+	pkgssh.AddSshFlags(cmd.Flags(), sshConf)
+	return cmd
+}
+
+func cmdConfigRemove(f cmdutil.Factory) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "remove",
+		Short: i18n.T("Remove proxy kubeconfig"),
+		Long: templates.LongDesc(i18n.T(`
+		Remove proxy kubeconfig which behind of ssh jump server
+		`)),
+		Example: templates.Examples(i18n.T(`
+		# remove proxy api-server which api-server behind of bastion host or ssh jump host
+		kubevpn config remove --kubeconfig /var/folders/30/cmv9c_5j3mq_kthx63sb1t5c0000gn/T/947048961.kubeconfig
+		`)),
+		PreRunE: func(cmd *cobra.Command, args []string) (err error) {
+			// startup daemon process and sudo process
+			return daemon.StartupDaemon(cmd.Context())
+		},
+		Args: cobra.MatchAll(cobra.OnlyValidArgs, cobra.ExactArgs(1)),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			req := &rpc.ConfigRemoveRequest{
+				ClusterID: args[0],
+			}
+			cli, err := daemon.GetClient(false)
+			if err != nil {
+				return err
+			}
+			_, err = cli.ConfigRemove(cmd.Context(), req)
+			if err != nil {
+				return err
+			}
+			return nil
+		},
+	}
+	return cmd
+}

cmd/kubevpn/cmds/connect.go

@@ -1,70 +1,164 @@
 package cmds
 
 import (
+	"context"
+	"fmt"
+	"os"
+
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"github.com/wencaiwulue/kubevpn/driver"
-	"github.com/wencaiwulue/kubevpn/pkg"
-	"github.com/wencaiwulue/kubevpn/util"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/util/retry"
-	"os"
-	"path/filepath"
-	"syscall"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+	"k8s.io/utils/ptr"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/config"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon/rpc"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/handler"
+	plog "github.com/wencaiwulue/kubevpn/v2/pkg/log"
+	pkgssh "github.com/wencaiwulue/kubevpn/v2/pkg/ssh"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util/regctl"
 )
 
-var connect = pkg.ConnectOptions{}
+func CmdConnect(f cmdutil.Factory) *cobra.Command {
+	var connect = &handler.ConnectOptions{}
+	var extraRoute = &handler.ExtraRouteInfo{}
+	var sshConf = &pkgssh.SshConfig{}
+	var transferImage, foreground, lite bool
+	var imagePullSecretName string
+	cmd := &cobra.Command{
+		Use:   "connect",
+		Short: i18n.T("Connect to kubernetes cluster network"),
+		Long: templates.LongDesc(i18n.T(`
+		Connect to kubernetes cluster network
+		
+		After connect to kubernetes cluster network, you can ping PodIP or
+		curl ServiceIP in local PC, it also supports k8s DNS resolve. 
+		Like: curl authors/authors.default/authors.default.svc/authors.default.svc.cluster.local.
+		So you can start up your application in local PC. depends on anything in
+		k8s cluster is ok, connect to them just like in k8s cluster.
+		`)),
+		Example: templates.Examples(i18n.T(`
+		# Connect to k8s cluster network
+		kubevpn connect
 
-func init() {
-	connectCmd.Flags().StringVar(&connect.KubeconfigPath, "kubeconfig", clientcmd.RecommendedHomeFile, "kubeconfig")
-	connectCmd.Flags().StringVarP(&connect.Namespace, "namespace", "n", "", "namespace")
-	connectCmd.PersistentFlags().StringArrayVar(&connect.Workloads, "workloads", []string{}, "workloads, like: pods/tomcat, deployment/nginx, replicaset/tomcat...")
-	connectCmd.Flags().StringVar((*string)(&connect.Mode), "mode", string(pkg.Reverse), "default mode is reverse")
-	connectCmd.Flags().StringToStringVarP(&connect.Headers, "headers", "H", map[string]string{}, "headers, format is k=v, like: k1=v1,k2=v2")
-	connectCmd.Flags().BoolVar(&util.Debug, "debug", false, "true/false")
-	RootCmd.AddCommand(connectCmd)
-}
+		# Connect to api-server behind of bastion host or ssh jump host
+		kubevpn connect --ssh-addr 192.168.1.100:22 --ssh-username root --ssh-keyfile ~/.ssh/ssh.pem
 
-var connectCmd = &cobra.Command{
-	Use:   "connect",
-	Short: "connect",
-	Long:  `connect`,
-	PersistentPreRun: func(cmd *cobra.Command, args []string) {
-		if !util.IsAdmin() {
-			util.RunWithElevated()
-			os.Exit(0)
-		}
-	},
-	PreRun: func(*cobra.Command, []string) {
-		util.InitLogger(util.Debug)
-		if util.IsWindows() {
-			driver.InstallWireGuardTunDriver()
-		}
-	},
-	Run: func(cmd *cobra.Command, args []string) {
-		if err := connect.InitClient(); err != nil {
-			log.Fatal(err)
-		}
-		connect.PreCheckResource()
-		if err := connect.DoConnect(); err != nil {
-			log.Errorln(err)
-			pkg.Cleanup(syscall.SIGQUIT)
-		}
-		select {}
-	},
-	PostRun: func(_ *cobra.Command, _ []string) {
-		if util.IsWindows() {
-			if err := retry.OnError(retry.DefaultRetry, func(err error) bool {
-				return err != nil
-			}, func() error {
-				return driver.UninstallWireGuardTunDriver()
-			}); err != nil {
-				wd, _ := os.Getwd()
-				filename := filepath.Join(wd, "wintun.dll")
-				if err = os.Rename(filename, filepath.Join(os.TempDir(), "wintun.dll")); err != nil {
-					log.Warn(err)
+		# It also supports ProxyJump, like
+		┌──────┐     ┌──────┐     ┌──────┐     ┌──────┐                 ┌────────────┐
+		│  pc  ├────►│ ssh1 ├────►│ ssh2 ├────►│ ssh3 ├─────►... ─────► │ api-server │
+		└──────┘     └──────┘     └──────┘     └──────┘                 └────────────┘
+		kubevpn connect --ssh-alias <alias>
+
+		# Support ssh auth GSSAPI
+        kubevpn connect --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-keytab /path/to/keytab
+        kubevpn connect --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-cache /path/to/cache
+        kubevpn connect --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-password <PASSWORD>
+
+		# Support ssh jump inline
+		kubevpn connect --ssh-jump "--ssh-addr jump.naison.org --ssh-username naison --gssapi-password xxx" --ssh-username root --ssh-addr 127.0.0.1:22 --ssh-keyfile ~/.ssh/dst.pem
+		`)),
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			plog.InitLoggerForClient()
+			// startup daemon process and sudo process
+			err := daemon.StartupDaemon(cmd.Context())
+			if err != nil {
+				return err
+			}
+			if transferImage {
+				err = regctl.TransferImageWithRegctl(cmd.Context(), config.OriginImage, config.Image)
+			}
+			return err
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			bytes, ns, err := util.ConvertToKubeConfigBytes(f)
+			if err != nil {
+				return err
+			}
+			if !sshConf.IsEmpty() {
+				if ip := util.GetAPIServerFromKubeConfigBytes(bytes); ip != nil {
+					extraRoute.ExtraCIDR = append(extraRoute.ExtraCIDR, ip.String())
 				}
 			}
-		}
-	},
+			req := &rpc.ConnectRequest{
+				KubeconfigBytes:      string(bytes),
+				Namespace:            ns,
+				ExtraRoute:           extraRoute.ToRPC(),
+				Engine:               string(connect.Engine),
+				OriginKubeconfigPath: util.GetKubeConfigPath(f),
+
+				SshJump:             sshConf.ToRPC(),
+				TransferImage:       transferImage,
+				Image:               config.Image,
+				ImagePullSecretName: imagePullSecretName,
+				Level:               int32(util.If(config.Debug, log.DebugLevel, log.InfoLevel)),
+			}
+			// if is foreground, send to sudo daemon server
+			cli, err := daemon.GetClient(false)
+			if err != nil {
+				return err
+			}
+			var resp grpc.ClientStream
+			if lite {
+				resp, err = cli.ConnectFork(cmd.Context(), req)
+			} else {
+				resp, err = cli.Connect(cmd.Context(), req)
+			}
+			if err != nil {
+				return err
+			}
+			err = util.PrintGRPCStream[rpc.ConnectResponse](resp)
+			if err != nil {
+				if status.Code(err) == codes.Canceled {
+					err = disconnect(cli, bytes, ns, sshConf)
+					return err
+				}
+				return err
+			}
+			if !foreground {
+				util.Print(os.Stdout, config.Slogan)
+			} else {
+				<-cmd.Context().Done()
+				err = disconnect(cli, bytes, ns, sshConf)
+				if err != nil {
+					return err
+				}
+				_, _ = fmt.Fprint(os.Stdout, "Disconnect completed")
+			}
+			return nil
+		},
+	}
+	handler.AddCommonFlags(cmd.Flags(), &transferImage, &imagePullSecretName, &connect.Engine)
+	cmd.Flags().BoolVar(&foreground, "foreground", false, "Hang up")
+	cmd.Flags().BoolVar(&lite, "lite", false, "connect to multiple cluster in lite mode. mode \"lite\": design for only connecting to multiple cluster network. mode \"full\": not only connect to cluster network, it also supports proxy workloads inbound traffic to local PC.")
+
+	handler.AddExtraRoute(cmd.Flags(), extraRoute)
+	pkgssh.AddSshFlags(cmd.Flags(), sshConf)
+	return cmd
+}
+
+func disconnect(cli rpc.DaemonClient, bytes []byte, ns string, sshConf *pkgssh.SshConfig) error {
+	resp, err := cli.Disconnect(context.Background(), &rpc.DisconnectRequest{
+		KubeconfigBytes: ptr.To(string(bytes)),
+		Namespace:       ptr.To(ns),
+		SshJump:         sshConf.ToRPC(),
+	})
+	if err != nil {
+		plog.G(context.Background()).Errorf("Disconnect error: %v", err)
+		return err
+	}
+	err = util.PrintGRPCStream[rpc.DisconnectResponse](resp)
+	if err != nil {
+		if status.Code(err) == codes.Canceled {
+			return nil
+		}
+		return err
+	}
+	return nil
 }

cmd/kubevpn/cmds/controlplane.go

@@ -0,0 +1,48 @@
+package cmds
+
+import (
+	"context"
+
+	"github.com/docker/docker/libnetwork/resolvconf"
+	miekgdns "github.com/miekg/dns"
+	"github.com/spf13/cobra"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/config"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/controlplane"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/dns"
+	plog "github.com/wencaiwulue/kubevpn/v2/pkg/log"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
+)
+
+func CmdControlPlane(cmdutil.Factory) *cobra.Command {
+	var (
+		watchDirectoryFilename string
+		port                   uint = 9002
+	)
+	cmd := &cobra.Command{
+		Use:    "control-plane",
+		Hidden: true,
+		Short:  i18n.T("Control-plane is a envoy xds server"),
+		Long: templates.LongDesc(i18n.T(`
+		Control-plane is a envoy xds server, distribute envoy route configuration
+		`)),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			go util.StartupPProfForServer(0)
+			go func() {
+				conf, err := miekgdns.ClientConfigFromFile(resolvconf.Path())
+				if err != nil {
+					plog.G(context.Background()).Fatal(err)
+				}
+				plog.G(context.Background()).Fatal(dns.ListenAndServe("udp", ":53", conf))
+			}()
+			err := controlplane.Main(cmd.Context(), watchDirectoryFilename, port, plog.G(context.Background()))
+			return err
+		},
+	}
+	cmd.Flags().StringVarP(&watchDirectoryFilename, "watchDirectoryFilename", "w", "/etc/envoy/envoy-config.yaml", "full path to directory to watch for files")
+	cmd.Flags().BoolVar(&config.Debug, "debug", false, "true/false")
+	return cmd
+}

cmd/kubevpn/cmds/cp.go

@@ -0,0 +1,140 @@
+package cmds
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"k8s.io/cli-runtime/pkg/genericiooptions"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/completion"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/cp"
+	pkgssh "github.com/wencaiwulue/kubevpn/v2/pkg/ssh"
+)
+
+var cpExample = templates.Examples(i18n.T(`
+		# !!!Important Note!!!
+		# Requires that the 'tar' binary is present in your container
+		# image.  If 'tar' is not present, 'kubectl cp' will fail.
+		#
+		# For advanced use cases, such as symlinks, wildcard expansion or
+		# file mode preservation, consider using 'kubectl exec'.
+
+		# Copy /tmp/foo local file to /tmp/bar in a remote pod in namespace <some-namespace>
+		tar cf - /tmp/foo | kubectl exec -i -n <some-namespace> <some-pod> -- tar xf - -C /tmp/bar
+
+		# Copy /tmp/foo from a remote pod to /tmp/bar locally
+		kubectl exec -n <some-namespace> <some-pod> -- tar cf - /tmp/foo | tar xf - -C /tmp/bar
+
+		# Copy /tmp/foo_dir local directory to /tmp/bar_dir in a remote pod in the default namespace
+		kubectl cp /tmp/foo_dir <some-pod>:/tmp/bar_dir
+
+		# Copy /tmp/foo local file to /tmp/bar in a remote pod in a specific container
+		kubectl cp /tmp/foo <some-pod>:/tmp/bar -c <specific-container>
+
+		# Copy /tmp/foo local file to /tmp/bar in a remote pod in namespace <some-namespace>
+		kubectl cp /tmp/foo <some-namespace>/<some-pod>:/tmp/bar
+
+		# Copy /tmp/foo from a remote pod to /tmp/bar locally
+		kubectl cp <some-namespace>/<some-pod>:/tmp/foo /tmp/bar
+
+		# copy reverse proxy api-server behind of bastion host or ssh jump host
+		kubevpn cp deployment/productpage --ssh-addr 192.168.1.100:22 --ssh-username root --ssh-keyfile ~/.ssh/ssh.pem
+
+		# It also supports ProxyJump, like
+		┌──────┐     ┌──────┐     ┌──────┐     ┌──────┐                 ┌────────────┐
+		│  pc  ├────►│ ssh1 ├────►│ ssh2 ├────►│ ssh3 ├─────►... ─────► │ api-server │
+		└──────┘     └──────┘     └──────┘     └──────┘                 └────────────┘
+		kubevpn cp deployment/productpage:/tmp/foo /tmp/bar --ssh-alias <alias>
+
+		# Support ssh auth GSSAPI
+        kubevpn cp deployment/productpage:/tmp/foo /tmp/bar --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-keytab /path/to/keytab
+        kubevpn cp deployment/productpage:/tmp/foo /tmp/bar --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-cache /path/to/cache
+        kubevpn cp deployment/productpage:/tmp/foo /tmp/bar --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-password <PASSWORD>
+`,
+))
+
+func CmdCp(f cmdutil.Factory) *cobra.Command {
+	o := cp.NewCopyOptions(genericiooptions.IOStreams{
+		In:     os.Stdin,
+		Out:    os.Stdout,
+		ErrOut: os.Stderr,
+	})
+	var sshConf = &pkgssh.SshConfig{}
+	cmd := &cobra.Command{
+		Use:                   "cp <file-spec-src> <file-spec-dest>",
+		DisableFlagsInUseLine: true,
+		Hidden:                true,
+		Short:                 i18n.T("Copy files and directories to and from containers"),
+		Long:                  i18n.T("Copy files and directories to and from containers. Different between kubectl cp is it will de-reference symbol link."),
+		Example:               cpExample,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			cmdutil.CheckErr(pkgssh.SshJumpAndSetEnv(cmd.Context(), sshConf, cmd.Flags(), false))
+
+			var comps []string
+			if len(args) == 0 {
+				if strings.IndexAny(toComplete, "/.~") == 0 {
+					// Looks like a path, do nothing
+				} else if strings.Contains(toComplete, ":") {
+					// TODO: complete remote files in the pod
+				} else if idx := strings.Index(toComplete, "/"); idx > 0 {
+					// complete <namespace>/<pod>
+					namespace := toComplete[:idx]
+					template := "{{ range .items }}{{ .metadata.namespace }}/{{ .metadata.name }}: {{ end }}"
+					comps = completion.CompGetFromTemplate(&template, f, namespace, []string{"pod"}, toComplete)
+				} else {
+					// Complete namespaces followed by a /
+					for _, ns := range completion.CompGetResource(f, "namespace", toComplete) {
+						comps = append(comps, fmt.Sprintf("%s/", ns))
+					}
+					// Complete pod names followed by a :
+					for _, pod := range completion.CompGetResource(f, "pod", toComplete) {
+						comps = append(comps, fmt.Sprintf("%s:", pod))
+					}
+
+					// Finally, provide file completion if we need to.
+					// We only do this if:
+					// 1- There are other completions found (if there are no completions,
+					//    the shell will do file completion itself)
+					// 2- If there is some input from the user (or else we will end up
+					//    listing the entire content of the current directory which could
+					//    be too many choices for the user)
+					if len(comps) > 0 && len(toComplete) > 0 {
+						if files, err := os.ReadDir("."); err == nil {
+							for _, file := range files {
+								filename := file.Name()
+								if strings.HasPrefix(filename, toComplete) {
+									if file.IsDir() {
+										filename = fmt.Sprintf("%s/", filename)
+									}
+									// We are completing a file prefix
+									comps = append(comps, filename)
+								}
+							}
+						}
+					} else if len(toComplete) == 0 {
+						// If the user didn't provide any input to complete,
+						// we provide a hint that a path can also be used
+						comps = append(comps, "./", "/")
+					}
+				}
+			}
+			return comps, cobra.ShellCompDirectiveNoSpace
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			cmdutil.CheckErr(o.Complete(f, cmd, args))
+			cmdutil.CheckErr(o.Validate())
+			cmdutil.CheckErr(o.Run())
+		},
+	}
+	cmdutil.AddContainerVarFlags(cmd, &o.Container, o.Container)
+	cmd.Flags().BoolVarP(&o.NoPreserve, "no-preserve", "", false, "The copied file/directory's ownership and permissions will not be preserved in the container")
+	cmd.Flags().IntVarP(&o.MaxTries, "retries", "", 0, "Set number of retries to complete a copy operation from a container. Specify 0 to disable or any negative value for infinite retrying. The default is 0 (no retry).")
+
+	pkgssh.AddSshFlags(cmd.Flags(), sshConf)
+	return cmd
+}

cmd/kubevpn/cmds/daemon.go

@@ -0,0 +1,90 @@
+package cmds
+
+import (
+	"crypto/rand"
+	"encoding/base64"
+	"errors"
+	"net/http"
+	"os"
+	"path/filepath"
+	"runtime/pprof"
+
+	"github.com/spf13/cobra"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/config"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon/action"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/dns"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
+)
+
+func CmdDaemon(cmdutil.Factory) *cobra.Command {
+	var opt = &daemon.SvrOption{}
+	cmd := &cobra.Command{
+		Use:   "daemon",
+		Short: i18n.T("Startup kubevpn daemon server"),
+		Long:  templates.LongDesc(i18n.T(`Startup kubevpn daemon server`)),
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			b := make([]byte, 32)
+			if _, err := rand.Read(b); err != nil {
+				return err
+			}
+			opt.ID = base64.URLEncoding.EncodeToString(b)
+
+			if opt.IsSudo {
+				go util.StartupPProf(config.SudoPProfPort)
+				_ = os.RemoveAll("/etc/resolver")
+				_ = dns.CleanupHosts()
+				_ = util.CleanupTempKubeConfigFile()
+			} else {
+				go util.StartupPProf(config.PProfPort)
+			}
+			return initLogfile(action.GetDaemonLogPath(opt.IsSudo))
+		},
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer opt.Stop()
+			defer func() {
+				if errors.Is(err, http.ErrServerClosed) {
+					err = nil
+				}
+				if opt.IsSudo {
+					for _, profile := range pprof.Profiles() {
+						func() {
+							file, e := os.Create(filepath.Join(config.PprofPath, profile.Name()))
+							if e != nil {
+								return
+							}
+							defer file.Close()
+							e = profile.WriteTo(file, 1)
+							if e != nil {
+								return
+							}
+						}()
+					}
+				}
+			}()
+			return opt.Start(cmd.Context())
+		},
+		Hidden:                true,
+		DisableFlagsInUseLine: true,
+	}
+	cmd.Flags().BoolVar(&opt.IsSudo, "sudo", false, "is sudo or not")
+	return cmd
+}
+
+func initLogfile(path string) error {
+	_, err := os.Lstat(path)
+	if os.IsNotExist(err) {
+		var f *os.File
+		f, err = os.Create(path)
+		if err != nil {
+			return err
+		}
+		_ = f.Close()
+		return os.Chmod(path, 0644)
+	}
+	return nil
+}

cmd/kubevpn/cmds/dev.go

@@ -0,0 +1,156 @@
+package cmds
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/docker/cli/cli/command"
+	"github.com/spf13/cobra"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/completion"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/config"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/dev"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/handler"
+	plog "github.com/wencaiwulue/kubevpn/v2/pkg/log"
+	pkgssh "github.com/wencaiwulue/kubevpn/v2/pkg/ssh"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util/regctl"
+)
+
+func CmdDev(f cmdutil.Factory) *cobra.Command {
+	var options = &dev.Options{
+		NoProxy:        false,
+		ExtraRouteInfo: handler.ExtraRouteInfo{},
+	}
+	var sshConf = &pkgssh.SshConfig{}
+	var transferImage bool
+	var imagePullSecretName string
+	var connectNamespace string
+	cmd := &cobra.Command{
+		Use:   "dev TYPE/NAME [-c CONTAINER] [flags] -- [args...]",
+		Short: i18n.T("Startup your kubernetes workloads in local Docker container"),
+		Long: templates.LongDesc(i18n.T(`
+		Startup your kubernetes workloads in local Docker container with same volume、env、and network
+
+		## What did it do:
+		- Download volume which MountPath point to, mount to docker container
+		- Connect to cluster network, set network to docker container
+		- Get all environment with command (env), set env to docker container
+		`)),
+		Example: templates.Examples(i18n.T(`
+        # Develop workloads
+		- develop deployment
+		  kubevpn dev deployment/productpage
+		- develop service
+		  kubevpn dev service/productpage
+
+		# Develop workloads with mesh, traffic with header foo=bar, will hit local PC, otherwise no effect
+		kubevpn dev service/productpage --headers foo=bar
+
+        # Develop workloads without proxy traffic
+		kubevpn dev service/productpage --no-proxy
+
+		# Develop workloads which api-server behind of bastion host or ssh jump host
+		kubevpn dev deployment/productpage --ssh-addr 192.168.1.100:22 --ssh-username root --ssh-keyfile ~/.ssh/ssh.pem
+
+		# It also supports ProxyJump, like
+		┌──────┐     ┌──────┐     ┌──────┐     ┌──────┐                 ┌────────────┐
+		│  pc  ├────►│ ssh1 ├────►│ ssh2 ├────►│ ssh3 ├─────►... ─────► │ api-server │
+		└──────┘     └──────┘     └──────┘     └──────┘                 └────────────┘
+		kubevpn dev deployment/productpage --ssh-alias <alias>
+
+		# Switch to terminal mode; send stdin to 'bash' and sends stdout/stderror from 'bash' back to the client
+        kubevpn dev deployment/authors -n default --kubeconfig ~/.kube/config --ssh-alias dev --entrypoint /bin/bash
+		  or
+        kubevpn dev deployment/authors -n default --kubeconfig ~/.kube/config --ssh-alias dev --entrypoint /bin/bash
+
+		# Support ssh auth GSSAPI
+        kubevpn dev deployment/authors -n default --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-keytab /path/to/keytab --entrypoint /bin/bash
+        kubevpn dev deployment/authors -n default --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-cache /path/to/cache --entrypoint /bin/bash
+        kubevpn dev deployment/authors -n default --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-password <PASSWORD> --entrypoint /bin/bash
+		`)),
+		ValidArgsFunction:     completion.ResourceTypeAndNameCompletionFunc(f),
+		Args:                  cobra.MatchAll(cobra.OnlyValidArgs),
+		DisableFlagsInUseLine: true,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				_, _ = fmt.Fprintf(os.Stdout, "You must specify the type of resource to proxy. %s\n\n", cmdutil.SuggestAPIResources("kubevpn"))
+				fullCmdName := cmd.Parent().CommandPath()
+				usageString := "Required resource not specified."
+				if len(fullCmdName) > 0 && cmdutil.IsSiblingCommandExists(cmd, "explain") {
+					usageString = fmt.Sprintf("%s\nUse \"%s explain <resource>\" for a detailed description of that resource (e.g. %[2]s explain pods).", usageString, fullCmdName)
+				}
+				return cmdutil.UsageErrorf(cmd, usageString)
+			}
+			err := cmd.Flags().Parse(args[1:])
+			if err != nil {
+				return err
+			}
+			plog.InitLoggerForClient()
+			err = daemon.StartupDaemon(cmd.Context())
+			if err != nil {
+				return err
+			}
+			if transferImage {
+				err = regctl.TransferImageWithRegctl(cmd.Context(), config.OriginImage, config.Image)
+			}
+			if err != nil {
+				return err
+			}
+			return pkgssh.SshJumpAndSetEnv(cmd.Context(), sshConf, cmd.Flags(), false)
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			options.Workload = args[0]
+			for i, arg := range args {
+				if arg == "--" && i != len(args)-1 {
+					options.ContainerOptions.Args = args[i+1:]
+					break
+				}
+			}
+
+			defer func() {
+				for _, function := range options.GetRollbackFuncList() {
+					if function != nil {
+						if err := function(); err != nil {
+							plog.G(context.Background()).Errorf("Rollback failed, error: %s", err.Error())
+						}
+					}
+				}
+			}()
+
+			if err := options.InitClient(f); err != nil {
+				return err
+			}
+
+			conf, hostConfig, err := dev.Parse(cmd.Flags(), options.ContainerOptions)
+			if err != nil {
+				return err
+			}
+
+			return options.Main(cmd.Context(), sshConf, conf, hostConfig, imagePullSecretName, connectNamespace)
+		},
+	}
+	cmd.Flags().SortFlags = false
+	cmd.Flags().StringToStringVarP(&options.Headers, "headers", "H", map[string]string{}, "Traffic with special headers with reverse it to local PC, you should startup your service after reverse workloads successfully, If not special, redirect all traffic to local PC, format is k=v, like: k1=v1,k2=v2")
+	cmd.Flags().BoolVar(&options.NoProxy, "no-proxy", false, "Whether proxy remote workloads traffic into local or not, true: just startup container on local without inject containers to intercept traffic, false: intercept traffic and forward to local")
+	cmdutil.AddContainerVarFlags(cmd, &options.ContainerName, options.ContainerName)
+	cmdutil.CheckErr(cmd.RegisterFlagCompletionFunc("container", completion.ContainerCompletionFunc(f)))
+	cmd.Flags().StringVar((*string)(&options.ConnectMode), "connect-mode", string(dev.ConnectModeHost), "Connect to kubernetes network in container or in host, eg: ["+string(dev.ConnectModeContainer)+"|"+string(dev.ConnectModeHost)+"]")
+	handler.AddCommonFlags(cmd.Flags(), &transferImage, &imagePullSecretName, &options.Engine)
+	cmd.Flags().StringVarP(&connectNamespace, "connect-namespace", "C", config.DefaultNamespaceKubevpn, "Connect to special namespace which kubevpn server installed by helm in cluster mode")
+
+	// diy docker options
+	cmd.Flags().StringVar(&options.DevImage, "dev-image", "", "Use to startup docker container, Default is pod image")
+	// -- origin docker options -- start
+	options.ContainerOptions = dev.AddFlags(cmd.Flags())
+	cmd.Flags().StringVar(&options.RunOptions.Pull, "pull", dev.PullImageMissing, `Pull image before running ("`+dev.PullImageAlways+`"|"`+dev.PullImageMissing+`"|"`+dev.PullImageNever+`")`)
+	command.AddPlatformFlag(cmd.Flags(), &options.RunOptions.Platform)
+	// -- origin docker options -- end
+	handler.AddExtraRoute(cmd.Flags(), &options.ExtraRouteInfo)
+	pkgssh.AddSshFlags(cmd.Flags(), sshConf)
+	return cmd
+}

cmd/kubevpn/cmds/disconnect.go

@@ -0,0 +1,93 @@
+package cmds
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+
+	"github.com/spf13/cobra"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+	"k8s.io/utils/pointer"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon/rpc"
+	plog "github.com/wencaiwulue/kubevpn/v2/pkg/log"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
+)
+
+func CmdDisconnect(f cmdutil.Factory) *cobra.Command {
+	var all = false
+	var clusterIDs []string
+	cmd := &cobra.Command{
+		Use:   "disconnect",
+		Short: i18n.T("Disconnect from kubernetes cluster network"),
+		Long: templates.LongDesc(i18n.T(`
+		Disconnect from kubernetes cluster network
+		
+		This command is to disconnect from cluster. after use command 'kubevpn connect',
+		you can use this command to disconnect from a specific cluster. 
+		before disconnect, it will leave proxy resource and clone resource if resource depends on this cluster 
+		after disconnect it will also cleanup DNS and host
+		`)),
+		Example: templates.Examples(i18n.T(`
+		# disconnect from cluster network and restore proxy resource
+        kubevpn disconnect
+		`)),
+		PreRunE: func(cmd *cobra.Command, args []string) (err error) {
+			plog.InitLoggerForClient()
+			err = daemon.StartupDaemon(cmd.Context())
+			return err
+		},
+		Args: cobra.MatchAll(cobra.OnlyValidArgs),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) > 0 && all {
+				return fmt.Errorf("either specify --all or ID, not both")
+			}
+			if len(clusterIDs) > 0 && all {
+				return fmt.Errorf("either specify --all or cluster-id, not both")
+			}
+			if len(args) == 0 && !all && len(clusterIDs) == 0 {
+				return fmt.Errorf("either specify --all or ID or cluster-id")
+			}
+			var ids *int32
+			if len(args) > 0 {
+				integer, err := strconv.Atoi(args[0])
+				if err != nil {
+					return fmt.Errorf("invalid ID: %s: %v", args[0], err)
+				}
+				ids = pointer.Int32(int32(integer))
+			}
+			cli, err := daemon.GetClient(false)
+			if err != nil {
+				return err
+			}
+			client, err := cli.Disconnect(
+				cmd.Context(),
+				&rpc.DisconnectRequest{
+					ID:         ids,
+					ClusterIDs: clusterIDs,
+					All:        pointer.Bool(all),
+				},
+			)
+			if err != nil {
+				return err
+			}
+			err = util.PrintGRPCStream[rpc.DisconnectResponse](client)
+			if err != nil {
+				if status.Code(err) == codes.Canceled {
+					return nil
+				}
+				return err
+			}
+			_, _ = fmt.Fprint(os.Stdout, "Disconnect completed")
+			return nil
+		},
+	}
+	cmd.Flags().BoolVar(&all, "all", all, "Disconnect all cluster, disconnect from all cluster network")
+	cmd.Flags().StringArrayVar(&clusterIDs, "cluster-id", []string{}, "Cluster id, command status -o yaml/json will show cluster-id")
+	return cmd
+}

cmd/kubevpn/cmds/get.go

@@ -0,0 +1,113 @@
+package cmds
+
+import (
+	"cmp"
+	"encoding/json"
+	"os"
+	"slices"
+	"strings"
+
+	"github.com/spf13/cobra"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/cli-runtime/pkg/printers"
+	cmdget "k8s.io/kubectl/pkg/cmd/get"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/scheme"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon/rpc"
+)
+
+func CmdGet(f cmdutil.Factory) *cobra.Command {
+	var printFlags = cmdget.NewGetPrintFlags()
+	cmd := &cobra.Command{
+		Use:    "get",
+		Hidden: true,
+		Short:  i18n.T("Get cluster resources which connected"),
+		Long:   templates.LongDesc(i18n.T(`Get cluster resources which connected`)),
+		Example: templates.Examples(i18n.T(`
+		# Get resource to k8s cluster network
+		kubevpn get pods
+
+		# Get api-server behind of bastion host or ssh jump host
+		kubevpn get deployment --ssh-addr 192.168.1.100:22 --ssh-username root --ssh-keyfile ~/.ssh/ssh.pem
+
+		# It also supports ProxyJump, like
+		┌──────┐     ┌──────┐     ┌──────┐     ┌──────┐                 ┌────────────┐
+		│  pc  ├────►│ ssh1 ├────►│ ssh2 ├────►│ ssh3 ├─────►... ─────► │ api-server │
+		└──────┘     └──────┘     └──────┘     └──────┘                 └────────────┘
+		kubevpn get service --ssh-alias <alias>
+		`)),
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			// startup daemon process and sudo process
+			return daemon.StartupDaemon(cmd.Context())
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ns, _, err := f.ToRawKubeConfigLoader().Namespace()
+			if err != nil {
+				return err
+			}
+			cli, err := daemon.GetClient(true)
+			if err != nil {
+				return err
+			}
+			client, err := cli.Get(
+				cmd.Context(),
+				&rpc.GetRequest{
+					Namespace: ns,
+					Resource:  args[0],
+				},
+			)
+			if err != nil {
+				return err
+			}
+			w := printers.GetNewTabWriter(os.Stdout)
+			var toPrinter = func() (printers.ResourcePrinterFunc, error) {
+				var flags = printFlags.Copy()
+				_ = flags.EnsureWithNamespace()
+				printer, err := flags.ToPrinter()
+				if err != nil {
+					return nil, err
+				}
+				printer, err = printers.NewTypeSetter(scheme.Scheme).WrapToPrinter(printer, nil)
+				if err != nil {
+					return nil, err
+				}
+				outputOption := cmd.Flags().Lookup("output").Value.String()
+				if strings.Contains(outputOption, "custom-columns") || outputOption == "yaml" || strings.Contains(outputOption, "json") {
+				} else {
+					printer = &cmdget.TablePrinter{Delegate: printer}
+				}
+				return printer.PrintObj, nil
+			}
+			var list []*v1.PartialObjectMetadata
+			for _, m := range client.Metadata {
+				var data v1.PartialObjectMetadata
+				err = json.Unmarshal([]byte(m), &data)
+				if err != nil {
+					continue
+				}
+				list = append(list, &data)
+			}
+			slices.SortStableFunc(list, func(a, b *v1.PartialObjectMetadata) int {
+				compare := cmp.Compare(a.GetNamespace(), b.GetNamespace())
+				if compare == 0 {
+					return cmp.Compare(a.GetName(), b.GetName())
+				}
+				return compare
+			})
+			for _, m := range list {
+				printer, err := toPrinter()
+				if err != nil {
+					return err
+				}
+				_ = printer.PrintObj(m, w)
+			}
+			return w.Flush()
+		},
+	}
+	printFlags.AddFlags(cmd)
+	return cmd
+}

cmd/kubevpn/cmds/imagecopy.go

@@ -0,0 +1,43 @@
+package cmds
+
+import (
+	"github.com/spf13/cobra"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+
+	plog "github.com/wencaiwulue/kubevpn/v2/pkg/log"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util/regctl"
+)
+
+func CmdImageCopy(cmdutil.Factory) *cobra.Command {
+	var imageCmd = &cobra.Command{
+		Use:   "image <cmd>",
+		Short: "copy images",
+	}
+
+	copyCmd := &cobra.Command{
+		Use:     "copy <src_image_ref> <dst_image_ref>",
+		Aliases: []string{"cp"},
+		Short:   "copy or re-tag image",
+		Long: `Copy or re-tag an image. This works between registries and only pulls layers
+that do not exist at the target. In the same registry it attempts to mount
+the layers between repositories. And within the same repository it only
+sends the manifest with the new tag.`,
+		Example: `
+# copy an image
+kubevpn image copy ghcr.io/kubenetworks/kubevpn:latest registry.example.org/kubevpn/kubevpn:latest
+
+# re-tag an image
+kubevpn image copy ghcr.io/kubenetworks/kubevpn:latest ghcr.io/kubenetworks/kubevpn:v2.3.4`,
+		Args: cobra.MatchAll(cobra.ExactArgs(2), cobra.OnlyValidArgs),
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			plog.InitLoggerForClient()
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			err := regctl.TransferImageWithRegctl(cmd.Context(), args[0], args[1])
+			return err
+		},
+	}
+	imageCmd.AddCommand(copyCmd)
+	return imageCmd
+}

cmd/kubevpn/cmds/leave.go

@@ -0,0 +1,63 @@
+package cmds
+
+import (
+	"github.com/spf13/cobra"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon/rpc"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
+)
+
+func CmdLeave(f cmdutil.Factory) *cobra.Command {
+	var leaveCmd = &cobra.Command{
+		Use:   "leave",
+		Short: i18n.T("Leave proxy resource"),
+		Long: templates.LongDesc(i18n.T(`
+		Leave proxy resource and restore it to origin
+	
+		This command is used to leave proxy resources. after use command 'kubevpn proxy xxx',
+		you can use this command to leave proxy resources.
+		you can just leave proxy resources which do proxy by yourself.
+		and the last one leave proxy resource, it will also restore workloads container.
+		otherwise it will keep containers [vpn, envoy-proxy] until last one to leave.
+		`)),
+		Example: templates.Examples(i18n.T(`
+		# leave proxy resource and restore it to origin
+		kubevpn leave deployment/authors
+		`)),
+		PreRunE: func(cmd *cobra.Command, args []string) (err error) {
+			return daemon.StartupDaemon(cmd.Context())
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			_, ns, err := util.ConvertToKubeConfigBytes(f)
+			if err != nil {
+				return err
+			}
+			cli, err := daemon.GetClient(false)
+			if err != nil {
+				return err
+			}
+			resp, err := cli.Leave(cmd.Context(), &rpc.LeaveRequest{
+				Namespace: ns,
+				Workloads: args,
+			})
+			if err != nil {
+				return err
+			}
+			err = util.PrintGRPCStream[rpc.LeaveResponse](resp)
+			if err != nil {
+				if status.Code(err) == codes.Canceled {
+					return nil
+				}
+				return err
+			}
+			return nil
+		},
+	}
+	return leaveCmd
+}

cmd/kubevpn/cmds/list.go

@@ -0,0 +1,45 @@
+package cmds
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon/rpc"
+)
+
+func CmdList(f cmdutil.Factory) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "list",
+		Short: i18n.T("List proxy resources"),
+		Long:  templates.LongDesc(i18n.T(`List proxy resources`)),
+		Example: templates.Examples(i18n.T(`
+	    # list proxy resources
+        kubevpn list
+		`)),
+		PreRunE: func(cmd *cobra.Command, args []string) (err error) {
+			return daemon.StartupDaemon(cmd.Context())
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cli, err := daemon.GetClient(true)
+			if err != nil {
+				return err
+			}
+			client, err := cli.List(
+				cmd.Context(),
+				&rpc.ListRequest{},
+			)
+			if err != nil {
+				return err
+			}
+			fmt.Println(client.GetMessage())
+			return nil
+		},
+		Hidden: true,
+	}
+	return cmd
+}

cmd/kubevpn/cmds/logs.go

@@ -0,0 +1,58 @@
+package cmds
+
+import (
+	"github.com/spf13/cobra"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon/rpc"
+	plog "github.com/wencaiwulue/kubevpn/v2/pkg/log"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
+)
+
+func CmdLogs(f cmdutil.Factory) *cobra.Command {
+	req := &rpc.LogRequest{}
+	cmd := &cobra.Command{
+		Use:   "logs",
+		Short: i18n.T("Log kubevpn daemon grpc server"),
+		Long: templates.LongDesc(i18n.T(`
+		Print the logs for kubevpn daemon grpc server. it will show sudo daemon and daemon grpc server log in both
+		`)),
+		Example: templates.Examples(i18n.T(`
+        # show log for kubevpn daemon server
+        kubevpn logs
+        # follow more log
+        kubevpn logs -f
+		`)),
+		PreRunE: func(cmd *cobra.Command, args []string) (err error) {
+			plog.InitLoggerForClient()
+			// startup daemon process and sudo process
+			return daemon.StartupDaemon(cmd.Context())
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cli, err := daemon.GetClient(true)
+			if err != nil {
+				return err
+			}
+			client, err := cli.Logs(cmd.Context(), req)
+			if err != nil {
+				return err
+			}
+			err = util.PrintGRPCStream[rpc.LogResponse](client)
+			if err != nil {
+				if status.Code(err) == codes.Canceled {
+					return nil
+				}
+				return err
+			}
+			return nil
+		},
+	}
+	cmd.Flags().BoolVarP(&req.Follow, "follow", "f", false, "Specify if the logs should be streamed.")
+	cmd.Flags().Int32VarP(&req.Lines, "number", "N", 10, "Lines of recent log file to display.")
+	return cmd
+}

cmd/kubevpn/cmds/options.go

@@ -0,0 +1,36 @@
+package cmds
+
+import (
+	"os"
+
+	"github.com/spf13/cobra"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+)
+
+var (
+	optionsExample = templates.Examples(i18n.T(`
+		# Print flags inherited by all commands
+		kubevpn options
+		`))
+)
+
+func CmdOptions(cmdutil.Factory) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "options",
+		Short: i18n.T("Print the list of flags inherited by all commands"),
+		Long: templates.LongDesc(i18n.T(`
+		Print the list of flags inherited by all commands
+		`)),
+		Example: optionsExample,
+		Run: func(cmd *cobra.Command, args []string) {
+			cmd.Usage()
+		},
+	}
+
+	cmd.SetOut(os.Stdout)
+
+	templates.UseOptionsTemplates(cmd)
+	return cmd
+}

cmd/kubevpn/cmds/proxy.go

@@ -0,0 +1,201 @@
+package cmds
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	utilcomp "k8s.io/kubectl/pkg/util/completion"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/config"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon/rpc"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/handler"
+	plog "github.com/wencaiwulue/kubevpn/v2/pkg/log"
+	pkgssh "github.com/wencaiwulue/kubevpn/v2/pkg/ssh"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util/regctl"
+)
+
+func CmdProxy(f cmdutil.Factory) *cobra.Command {
+	var headers = make(map[string]string)
+	var portmap []string
+	var connect = handler.ConnectOptions{}
+	var extraRoute = &handler.ExtraRouteInfo{}
+	var sshConf = &pkgssh.SshConfig{}
+	var transferImage, foreground bool
+	var imagePullSecretName string
+	var connectNamespace string
+	cmd := &cobra.Command{
+		Use:   "proxy",
+		Short: i18n.T("Proxy kubernetes workloads inbound traffic into local PC"),
+		Long: templates.LongDesc(i18n.T(`
+		Proxy kubernetes workloads inbound traffic into local PC
+
+		Proxy k8s workloads inbound traffic into local PC with/without service mesh. 
+		Without service mesh, it will proxy all inbound traffic into local PC, even traffic protocol is layer 4(Transport layer).
+		With service mesh, it will proxy traffic which has special header to local PC, support protocol HTTP,GRPC,THRIFT, WebSocket...
+		After proxy resource, it also connected to cluster network automatically. so just startup your app in local PC
+		and waiting for inbound traffic, make debug more easier.
+		
+		`)),
+		Example: templates.Examples(i18n.T(`
+		# Reverse proxy
+		- proxy deployment
+		  kubevpn proxy deployment/productpage
+
+		- proxy service
+		  kubevpn proxy service/productpage
+
+        - proxy multiple workloads
+          kubevpn proxy deployment/authors deployment/productpage
+          or 
+          kubevpn proxy deployment authors productpage
+
+		# Reverse proxy with mesh, traffic with header foo=bar, will hit local PC, otherwise no effect
+		kubevpn proxy service/productpage --headers foo=bar
+		
+		# Reverse proxy with mesh, traffic with header foo=bar and env=dev, will hit local PC, otherwise no effect
+		kubevpn proxy service/productpage --headers foo=bar --headers env=dev
+
+		# Connect to api-server behind of bastion host or ssh jump host and proxy kubernetes resource traffic into local PC
+		kubevpn proxy deployment/productpage --ssh-addr 192.168.1.100:22 --ssh-username root --ssh-keyfile ~/.ssh/ssh.pem --headers foo=bar
+
+		# It also supports ProxyJump, like
+		┌──────┐     ┌──────┐     ┌──────┐     ┌──────┐                 ┌────────────┐
+		│  pc  ├────►│ ssh1 ├────►│ ssh2 ├────►│ ssh3 ├─────►... ─────► │ api-server │
+		└──────┘     └──────┘     └──────┘     └──────┘                 └────────────┘
+		kubevpn proxy service/productpage --ssh-alias <alias> --headers foo=bar
+
+		# Support ssh auth GSSAPI
+        kubevpn proxy service/productpage --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-keytab /path/to/keytab
+        kubevpn proxy service/productpage --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-cache /path/to/cache
+        kubevpn proxy service/productpage --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-password <PASSWORD>
+
+		# Support port map, you can proxy container port to local port by command:
+		kubevpn proxy deployment/productpage --portmap 80:8080
+		
+		# Proxy container port 9080 to local port 8080 of TCP protocol
+		kubevpn proxy deployment/productpage --portmap 9080:8080
+
+		# Proxy container port 9080 to local port 5000 of UDP protocol
+		kubevpn proxy deployment/productpage --portmap udp/9080:5000
+
+		# Auto proxy container port to same local port, and auto detect protocol
+		kubevpn proxy deployment/productpage
+		`)),
+		PreRunE: func(cmd *cobra.Command, args []string) (err error) {
+			plog.InitLoggerForClient()
+			if err = daemon.StartupDaemon(cmd.Context()); err != nil {
+				return err
+			}
+			if transferImage {
+				err = regctl.TransferImageWithRegctl(cmd.Context(), config.OriginImage, config.Image)
+			}
+			return err
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				fmt.Fprintf(os.Stdout, "You must specify the type of resource to proxy. %s\n\n", cmdutil.SuggestAPIResources("kubevpn"))
+				fullCmdName := cmd.Parent().CommandPath()
+				usageString := "Required resource not specified."
+				if len(fullCmdName) > 0 && cmdutil.IsSiblingCommandExists(cmd, "explain") {
+					usageString = fmt.Sprintf("%s\nUse \"%s explain <resource>\" for a detailed description of that resource (e.g. %[2]s explain pods).", usageString, fullCmdName)
+				}
+				return cmdutil.UsageErrorf(cmd, usageString)
+			}
+
+			bytes, ns, err := util.ConvertToKubeConfigBytes(f)
+			if err != nil {
+				return err
+			}
+			if !sshConf.IsEmpty() {
+				if ip := util.GetAPIServerFromKubeConfigBytes(bytes); ip != nil {
+					extraRoute.ExtraCIDR = append(extraRoute.ExtraCIDR, ip.String())
+				}
+			}
+			// todo 将 doConnect 方法封装？内部使用 client 发送到daemon？
+			cli, err := daemon.GetClient(false)
+			if err != nil {
+				return err
+			}
+			resp, err := cli.Proxy(
+				cmd.Context(),
+				&rpc.ProxyRequest{
+					KubeconfigBytes:      string(bytes),
+					Namespace:            ns,
+					Headers:              headers,
+					PortMap:              portmap,
+					Workloads:            args,
+					ExtraRoute:           extraRoute.ToRPC(),
+					Engine:               string(connect.Engine),
+					SshJump:              sshConf.ToRPC(),
+					TransferImage:        transferImage,
+					Image:                config.Image,
+					ImagePullSecretName:  imagePullSecretName,
+					Level:                int32(util.If(config.Debug, log.DebugLevel, log.InfoLevel)),
+					OriginKubeconfigPath: util.GetKubeConfigPath(f),
+					ConnectNamespace:     connectNamespace,
+				},
+			)
+			if err != nil {
+				return err
+			}
+			err = util.PrintGRPCStream[rpc.ConnectResponse](resp)
+			if err != nil {
+				if status.Code(err) == codes.Canceled {
+					err = leave(cli, ns, args)
+					return err
+				}
+				return err
+			}
+			util.Print(os.Stdout, config.Slogan)
+			// hangup
+			if foreground {
+				// leave from cluster resources
+				<-cmd.Context().Done()
+
+				err = leave(cli, ns, args)
+				if err != nil {
+					return err
+				}
+			}
+			return nil
+		},
+	}
+	cmd.Flags().StringToStringVarP(&headers, "headers", "H", map[string]string{}, "Traffic with special headers (use `and` to match all headers) with reverse it to local PC, If not special, redirect all traffic to local PC. format: <KEY>=<VALUE> eg: --headers foo=bar --headers env=dev")
+	cmd.Flags().StringArrayVar(&portmap, "portmap", []string{}, "Port map, map container port to local port, format: [tcp/udp]/containerPort:localPort, If not special, localPort will use containerPort. eg: tcp/80:8080 or udp/5000:5001 or 80 or 80:8080")
+	handler.AddCommonFlags(cmd.Flags(), &transferImage, &imagePullSecretName, &connect.Engine)
+	cmd.Flags().BoolVar(&foreground, "foreground", false, "foreground hang up")
+	cmd.Flags().StringVarP(&connectNamespace, "connect-namespace", "C", config.DefaultNamespaceKubevpn, "Connect to special namespace which kubevpn server installed by helm in cluster mode")
+
+	handler.AddExtraRoute(cmd.Flags(), extraRoute)
+	pkgssh.AddSshFlags(cmd.Flags(), sshConf)
+	cmd.ValidArgsFunction = utilcomp.ResourceTypeAndNameCompletionFunc(f)
+	return cmd
+}
+
+func leave(cli rpc.DaemonClient, ns string, args []string) error {
+	stream, err := cli.Leave(context.Background(), &rpc.LeaveRequest{
+		Namespace: ns,
+		Workloads: args,
+	})
+	if err != nil {
+		return err
+	}
+	err = util.PrintGRPCStream[rpc.LeaveResponse](stream)
+	if err != nil {
+		if status.Code(err) == codes.Canceled {
+			return nil
+		}
+		return err
+	}
+	return nil
+}

cmd/kubevpn/cmds/quit.go

@@ -0,0 +1,59 @@
+package cmds
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon/rpc"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
+)
+
+func CmdQuit(f cmdutil.Factory) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "quit",
+		Short: i18n.T("Quit kubevpn daemon grpc server"),
+		Long: templates.LongDesc(i18n.T(`
+		Disconnect from cluster, leave proxy resources, quit daemon grpc server and cleanup dns/hosts
+		`)),
+		Example: templates.Examples(i18n.T(`
+        # before quit kubevpn, it will leave proxy resources to origin and disconnect from cluster
+        kubevpn quit
+		`)),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			_ = quit(cmd.Context(), true)
+			_ = quit(cmd.Context(), false)
+			util.CleanExtensionLib()
+			_, _ = fmt.Fprint(os.Stdout, "Exited")
+			return nil
+		},
+	}
+	return cmd
+}
+
+func quit(ctx context.Context, isSudo bool) error {
+	cli, err := daemon.GetClient(isSudo)
+	if err != nil {
+		return err
+	}
+	client, err := cli.Quit(ctx, &rpc.QuitRequest{})
+	if err != nil {
+		return err
+	}
+	err = util.PrintGRPCStream[rpc.QuitResponse](client)
+	if err != nil {
+		if status.Code(err) == codes.Canceled {
+			return nil
+		}
+		return err
+	}
+	return nil
+}

cmd/kubevpn/cmds/remove.go

@@ -0,0 +1,56 @@
+package cmds
+
+import (
+	"github.com/spf13/cobra"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon/rpc"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
+)
+
+func CmdRemove(f cmdutil.Factory) *cobra.Command {
+	var cmd = &cobra.Command{
+		Use:   "remove",
+		Short: "Remove clone resource",
+		Long: templates.LongDesc(i18n.T(`
+		Remove clone resource
+
+		This command is design to remove clone resources, after use command 'kubevpn clone xxx', 
+		it will generate and create a new resource in target k8s cluster with format [resource_name]_clone_xxxxx,
+		use this command to remove this created resources.
+		`)),
+		Example: templates.Examples(i18n.T(`
+        # leave proxy resources to origin
+        kubevpn remove deployment/authors
+		`)),
+		PreRunE: func(cmd *cobra.Command, args []string) (err error) {
+			return daemon.StartupDaemon(cmd.Context())
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cli, err := daemon.GetClient(false)
+			if err != nil {
+				return err
+			}
+			resp, err := cli.Remove(cmd.Context(), &rpc.RemoveRequest{
+				Workloads: args,
+			})
+			if err != nil {
+				return err
+			}
+			err = util.PrintGRPCStream[rpc.RemoveResponse](resp)
+			if err != nil {
+				if status.Code(err) == codes.Canceled {
+					return nil
+				}
+				return err
+			}
+			return nil
+		},
+	}
+	return cmd
+}

cmd/kubevpn/cmds/reset.go

@@ -0,0 +1,86 @@
+package cmds
+
+import (
+	"github.com/spf13/cobra"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon/rpc"
+	plog "github.com/wencaiwulue/kubevpn/v2/pkg/log"
+	pkgssh "github.com/wencaiwulue/kubevpn/v2/pkg/ssh"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
+)
+
+func CmdReset(f cmdutil.Factory) *cobra.Command {
+	var sshConf = &pkgssh.SshConfig{}
+	cmd := &cobra.Command{
+		Use:   "reset",
+		Short: "Reset workloads to origin status",
+		Long: templates.LongDesc(i18n.T(`
+		Reset workloads to origin status
+		
+		Reset will remove injected container envoy-proxy and vpn, and restore service mesh rules.
+		`)),
+		Example: templates.Examples(i18n.T(`
+        # Reset default namespace workloads depooyment/productpage
+		  kubevpn reset deployment/productpage
+
+		# Reset another namespace test workloads depooyment/productpage
+		  kubevpn reset deployment/productpage -n test
+
+		# Reset workloads depooyment/productpage which api-server behind of bastion host or ssh jump host
+		kubevpn reset deployment/productpage --ssh-addr 192.168.1.100:22 --ssh-username root --ssh-keyfile ~/.ssh/ssh.pem
+
+		# It also supports ProxyJump, like
+		┌──────┐     ┌──────┐     ┌──────┐     ┌──────┐                 ┌────────────┐
+		│  pc  ├────►│ ssh1 ├────►│ ssh2 ├────►│ ssh3 ├─────►... ─────► │ api-server │
+		└──────┘     └──────┘     └──────┘     └──────┘                 └────────────┘
+		kubevpn reset deployment/productpage --ssh-alias <alias>
+
+		# Support ssh auth GSSAPI
+        kubevpn reset deployment/productpage --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-keytab /path/to/keytab
+        kubevpn reset deployment/productpage --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-cache /path/to/cache
+        kubevpn reset deployment/productpage --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-password <PASSWORD>
+		`)),
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			plog.InitLoggerForClient()
+			return daemon.StartupDaemon(cmd.Context())
+		},
+		Args: cobra.MatchAll(cobra.ExactArgs(1)),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			bytes, ns, err := util.ConvertToKubeConfigBytes(f)
+			if err != nil {
+				return err
+			}
+			cli, err := daemon.GetClient(false)
+			if err != nil {
+				return err
+			}
+			req := &rpc.ResetRequest{
+				KubeconfigBytes: string(bytes),
+				Namespace:       ns,
+				Workloads:       args,
+				SshJump:         sshConf.ToRPC(),
+			}
+			resp, err := cli.Reset(cmd.Context(), req)
+			if err != nil {
+				return err
+			}
+			err = util.PrintGRPCStream[rpc.ResetResponse](resp)
+			if err != nil {
+				if status.Code(err) == codes.Canceled {
+					return nil
+				}
+				return err
+			}
+			return nil
+		},
+	}
+
+	pkgssh.AddSshFlags(cmd.Flags(), sshConf)
+	return cmd
+}

cmd/kubevpn/cmds/root.go

@@ -1,9 +1,112 @@
 package cmds
 
-import "github.com/spf13/cobra"
+import (
+	"os"
+	"strings"
 
-var RootCmd = &cobra.Command{
-	Use:   "kubevpn",
-	Short: "kubevpn",
-	Long:  `kubevpn`,
+	"github.com/spf13/cobra"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"k8s.io/client-go/rest"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/homedir"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+	"k8s.io/utils/ptr"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/config"
+)
+
+func NewKubeVPNCommand() *cobra.Command {
+	var cmd = &cobra.Command{
+		Use:   "kubevpn",
+		Short: i18n.T("KubeVPN offers a Cloud-Native Dev Environment that seamlessly connects to your Kubernetes cluster network."),
+		Long: templates.LongDesc(`
+        KubeVPN offers a Cloud-Native Dev Environment that seamlessly connects to your Kubernetes cluster network.
+        `),
+		Run: func(cmd *cobra.Command, args []string) {
+			cmd.Help()
+		},
+	}
+
+	flags := cmd.PersistentFlags()
+	configFlags := genericclioptions.NewConfigFlags(true)
+	configFlags.WrapConfigFn = func(c *rest.Config) *rest.Config {
+		if path, ok := os.LookupEnv(config.EnvSSHJump); ok {
+			kubeconfigBytes, err := os.ReadFile(path)
+			cmdutil.CheckErr(err)
+			var conf *restclient.Config
+			conf, err = clientcmd.RESTConfigFromKubeConfig(kubeconfigBytes)
+			cmdutil.CheckErr(err)
+			return conf
+		}
+		return c
+	}
+	configFlags.AddFlags(flags)
+	matchVersionFlags := cmdutil.NewMatchVersionFlags(&warp{ConfigFlags: configFlags})
+	matchVersionFlags.AddFlags(flags)
+	factory := cmdutil.NewFactory(matchVersionFlags)
+
+	groups := templates.CommandGroups{
+		{
+			Message: "Develop commands:",
+			Commands: []*cobra.Command{
+				CmdConnect(factory),
+				CmdDisconnect(factory),
+				CmdProxy(factory),
+				CmdLeave(factory),
+				CmdClone(factory),
+				CmdRemove(factory),
+				CmdDev(factory),
+				// Hidden, Server Commands (DO NOT USE IT !!!)
+				CmdControlPlane(factory),
+				CmdServer(factory),
+				CmdDaemon(factory),
+				CmdWebhook(factory),
+				CmdSyncthing(factory),
+			},
+		},
+		{
+			Message: "Management commands:",
+			Commands: []*cobra.Command{
+				CmdStatus(factory),
+				CmdList(factory),
+				CmdAlias(factory),
+				CmdGet(factory),
+				CmdConfig(factory),
+				CmdSSH(factory),
+				CmdSSHDaemon(factory),
+				CmdImageCopy(factory),
+				CmdLogs(factory),
+				CmdCp(factory),
+				CmdReset(factory),
+				CmdUninstall(factory),
+				CmdQuit(factory),
+			},
+		},
+		{
+			Message: "Other commands:",
+			Commands: []*cobra.Command{
+				CmdUpgrade(factory),
+				CmdVersion(factory),
+			},
+		},
+	}
+	groups.Add(cmd)
+	templates.ActsAsRootCommand(cmd, []string{"options"}, groups...)
+	cmd.AddCommand(CmdOptions(factory))
+	return cmd
+}
+
+type warp struct {
+	*genericclioptions.ConfigFlags
+}
+
+func (f *warp) ToRawKubeConfigLoader() clientcmd.ClientConfig {
+	if strings.HasPrefix(ptr.Deref[string](f.KubeConfig, ""), "~") {
+		home := homedir.HomeDir()
+		f.KubeConfig = ptr.To(strings.Replace(*f.KubeConfig, "~", home, 1))
+	}
+	return f.ConfigFlags.ToRawKubeConfigLoader()
 }

cmd/kubevpn/cmds/serve.go

@@ -1,33 +0,0 @@
-package cmds
-
-import (
-	"context"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-	"github.com/wencaiwulue/kubevpn/pkg"
-	"github.com/wencaiwulue/kubevpn/util"
-)
-
-var config pkg.Route
-
-func init() {
-	ServerCmd.Flags().StringArrayVarP(&config.ServeNodes, "nodeCommand", "L", []string{}, "command needs to be executed")
-	ServerCmd.Flags().StringVarP(&config.ChainNode, "chainCommand", "F", "", "command needs to be executed")
-	ServerCmd.Flags().BoolVar(&util.Debug, "debug", false, "true/false")
-	RootCmd.AddCommand(ServerCmd)
-}
-
-var ServerCmd = &cobra.Command{
-	Use:   "serve",
-	Short: "serve",
-	Long:  `serve`,
-	PreRun: func(*cobra.Command, []string) {
-		util.InitLogger(util.Debug)
-	},
-	Run: func(cmd *cobra.Command, args []string) {
-		if err := pkg.Start(context.TODO(), config); err != nil {
-			log.Fatal(err)
-		}
-		select {}
-	},
-}

cmd/kubevpn/cmds/server.go

@@ -0,0 +1,60 @@
+package cmds
+
+import (
+	"math/rand"
+	"os"
+	"runtime"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"go.uber.org/automaxprocs/maxprocs"
+	glog "gvisor.dev/gvisor/pkg/log"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/config"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/core"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/handler"
+	plog "github.com/wencaiwulue/kubevpn/v2/pkg/log"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
+)
+
+func CmdServer(cmdutil.Factory) *cobra.Command {
+	var route = &core.Route{}
+	cmd := &cobra.Command{
+		Use:    "server",
+		Hidden: true,
+		Short:  "Server side, startup traffic manager, forward inbound and outbound traffic",
+		Long: templates.LongDesc(i18n.T(`
+		Server side, startup traffic manager, forward inbound and outbound traffic.
+		`)),
+		Example: templates.Examples(i18n.T(`
+        # server listener
+        kubevpn server -l "tcp://:10800" -l "tun://127.0.0.1:8422?net=198.19.0.123/32"
+		`)),
+		PreRun: func(*cobra.Command, []string) {
+			runtime.GOMAXPROCS(0)
+			go util.StartupPProfForServer(config.PProfPort)
+			glog.SetTarget(plog.ServerEmitter{Writer: &glog.Writer{Next: os.Stderr}})
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			rand.Seed(time.Now().UnixNano())
+			_, _ = maxprocs.Set(maxprocs.Logger(nil))
+			ctx := cmd.Context()
+			logger := plog.InitLoggerForServer()
+			logger.SetLevel(util.If(config.Debug, log.DebugLevel, log.InfoLevel))
+			servers, err := handler.Parse(*route)
+			if err != nil {
+				plog.G(ctx).Errorf("Parse server failed: %v", err)
+				return err
+			}
+			return handler.Run(plog.WithLogger(ctx, logger), servers)
+		},
+	}
+	cmd.Flags().StringArrayVarP(&route.Listeners, "listener", "l", []string{}, "Startup listener server. eg: tcp://localhost:1080")
+	cmd.Flags().StringVarP(&route.Forwarder, "forwarder", "f", "", "Special forwarder. eg: tcp://192.168.1.100:2345")
+	cmd.Flags().BoolVar(&config.Debug, "debug", false, "Enable debug log or not")
+	return cmd
+}

cmd/kubevpn/cmds/ssh.go

@@ -0,0 +1,195 @@
+package cmds
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/containerd/containerd/platforms"
+	"github.com/google/uuid"
+	"github.com/spf13/cobra"
+	"golang.org/x/crypto/ssh/terminal"
+	"golang.org/x/net/websocket"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+	"k8s.io/kubectl/pkg/util/term"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon/handler"
+	plog "github.com/wencaiwulue/kubevpn/v2/pkg/log"
+	pkgssh "github.com/wencaiwulue/kubevpn/v2/pkg/ssh"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
+)
+
+// CmdSSH
+// Remember to use network mask 32, because ssh using unique network CIDR 198.18.0.0/16
+func CmdSSH(cmdutil.Factory) *cobra.Command {
+	var sshConf = &pkgssh.SshConfig{}
+	var extraCIDR []string
+	var platform string
+	var lite bool
+	cmd := &cobra.Command{
+		Use:   "ssh",
+		Short: "Ssh to jump server",
+		Long: templates.LongDesc(i18n.T(`
+		Ssh to jump server
+		`)),
+		Example: templates.Examples(i18n.T(`
+        # Jump to server behind of bastion host or ssh jump host
+		kubevpn ssh --ssh-addr 192.168.1.100:22 --ssh-username root --ssh-keyfile ~/.ssh/ssh.pem
+
+		# It also supports ProxyJump, like
+		┌──────┐     ┌──────┐     ┌──────┐     ┌──────┐                 ┌────────┐
+		│  pc  ├────►│ ssh1 ├────►│ ssh2 ├────►│ ssh3 ├─────►... ─────► │ server │
+		└──────┘     └──────┘     └──────┘     └──────┘                 └────────┘
+		kubevpn ssh --ssh-alias <alias>
+
+		# Support ssh auth GSSAPI
+        kubevpn ssh --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-keytab /path/to/keytab
+        kubevpn ssh --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-cache /path/to/cache
+        kubevpn ssh --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-password <PASSWORD>
+		`)),
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			plog.InitLoggerForClient()
+			return daemon.StartupDaemon(cmd.Context())
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			plat, err := platforms.Parse(platform)
+			if err != nil {
+				return err
+			}
+			config, err := websocket.NewConfig("ws://test/ws", "http://test")
+			if err != nil {
+				return err
+			}
+			fd := int(os.Stdin.Fd())
+			if !terminal.IsTerminal(fd) {
+				return fmt.Errorf("stdin is not a terminal")
+			}
+			width, height, err := terminal.GetSize(fd)
+			if err != nil {
+				return fmt.Errorf("terminal get size: %s", err)
+			}
+			sessionID := uuid.NewString()
+			ssh := handler.Ssh{
+				Config:    *sshConf,
+				ExtraCIDR: extraCIDR,
+				Width:     width,
+				Height:    height,
+				Platform:  platforms.Format(platforms.Normalize(plat)),
+				SessionID: sessionID,
+				Lite:      lite,
+			}
+			marshal, err := json.Marshal(ssh)
+			if err != nil {
+				return err
+			}
+			config.Header.Set("ssh", string(marshal))
+			client := daemon.GetTCPClient(true)
+			if client == nil {
+				return fmt.Errorf("client is nil")
+			}
+			conn, err := websocket.NewClient(config, client)
+			if err != nil {
+				return err
+			}
+			defer conn.Close()
+
+			errChan := make(chan error, 3)
+			go func() {
+				errChan <- monitorSize(cmd.Context(), sessionID)
+			}()
+
+			readyCtx, cancelFunc := context.WithCancel(cmd.Context())
+			checker := func(log string) bool {
+				isReady := strings.Contains(log, fmt.Sprintf(handler.SshTerminalReadyFormat, sessionID))
+				if isReady {
+					cancelFunc()
+				}
+				return isReady
+			}
+			var state *terminal.State
+			go func() {
+				select {
+				case <-cmd.Context().Done():
+					return
+				case <-readyCtx.Done():
+				}
+				if state, err = terminal.MakeRaw(fd); err != nil {
+					plog.G(context.Background()).Errorf("terminal make raw: %s", err)
+				}
+			}()
+
+			go func() {
+				_, err := io.Copy(conn, os.Stdin)
+				errChan <- err
+			}()
+			go func() {
+				_, err := io.Copy(io.MultiWriter(os.Stdout, util.NewWriter(checker)), conn)
+				errChan <- err
+			}()
+
+			defer func() {
+				if state != nil {
+					terminal.Restore(fd, state)
+				}
+			}()
+
+			select {
+			case err := <-errChan:
+				return err
+			case <-cmd.Context().Done():
+				return cmd.Context().Err()
+			}
+		},
+	}
+	pkgssh.AddSshFlags(cmd.Flags(), sshConf)
+	cmd.Flags().StringArrayVar(&extraCIDR, "extra-cidr", []string{}, "Extra network CIDR string, eg: --extra-cidr 192.168.0.159/24 --extra-cidr 192.168.1.160/32")
+	cmd.Flags().StringVar(&platform, "platform", util.If(os.Getenv("KUBEVPN_DEFAULT_PLATFORM") != "", os.Getenv("KUBEVPN_DEFAULT_PLATFORM"), "linux/amd64"), "Set ssh server platform if needs to install command kubevpn")
+	cmd.Flags().BoolVar(&lite, "lite", false, "connect to ssh server in lite mode. mode \"lite\": design for only connect to ssh server. mode \"full\": not only connect to ssh server, it also create a two-way tunnel communicate with inner ip")
+	return cmd
+}
+
+func monitorSize(ctx context.Context, sessionID string) error {
+	conn := daemon.GetTCPClient(true)
+	if conn == nil {
+		return fmt.Errorf("conn is nil")
+	}
+	var tt = term.TTY{
+		In:     os.Stdin,
+		Out:    os.Stdout,
+		Raw:    false,
+		TryDev: false,
+		Parent: nil,
+	}
+	sizeQueue := tt.MonitorSize(tt.GetSize())
+	if sizeQueue == nil {
+		return fmt.Errorf("sizeQueue is nil")
+	}
+	//defer runtime.HandleCrash()
+	config, err := websocket.NewConfig("ws://test/resize", "http://test")
+	if err != nil {
+		return err
+	}
+	config.Header.Set("session-id", sessionID)
+	client, err := websocket.NewClient(config, conn)
+	if err != nil {
+		return err
+	}
+	encoder := json.NewEncoder(client)
+	for ctx.Err() == nil {
+		size := sizeQueue.Next()
+		if size == nil {
+			return nil
+		}
+		if err = encoder.Encode(&size); err != nil {
+			plog.G(ctx).Errorf("Encode resize: %s", err)
+			return err
+		}
+	}
+	return nil
+}

cmd/kubevpn/cmds/sshdaemon.go

@@ -0,0 +1,53 @@
+package cmds
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon/rpc"
+)
+
+// CmdSSHDaemon
+// set local tun ip 198.19.0.1/32, remember to use mask 32
+func CmdSSHDaemon(cmdutil.Factory) *cobra.Command {
+	var clientIP string
+	cmd := &cobra.Command{
+		Use:    "ssh-daemon",
+		Hidden: true,
+		Short:  "Ssh daemon server",
+		Long:   templates.LongDesc(i18n.T(`Ssh daemon server`)),
+		Example: templates.Examples(i18n.T(`
+        # SSH daemon server
+        kubevpn ssh-daemon --client-ip 198.19.0.123/32
+		`)),
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			err := daemon.StartupDaemon(cmd.Context())
+			return err
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cli, err := daemon.GetClient(true)
+			if err != nil {
+				return err
+			}
+			resp, err := cli.SshStart(
+				cmd.Context(),
+				&rpc.SshStartRequest{
+					ClientIP: clientIP,
+				},
+			)
+			if err != nil {
+				return err
+			}
+			_, err = fmt.Fprint(os.Stdout, resp.ServerIP)
+			return err
+		},
+	}
+	cmd.Flags().StringVar(&clientIP, "client-ip", "", "Client cidr")
+	return cmd
+}

cmd/kubevpn/cmds/status.go

@@ -0,0 +1,289 @@
+package cmds
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/liggitt/tabwriter"
+	"github.com/spf13/cobra"
+	flag "github.com/spf13/pflag"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"k8s.io/cli-runtime/pkg/printers"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+	"sigs.k8s.io/yaml"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/config"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon/rpc"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/handler"
+	plog "github.com/wencaiwulue/kubevpn/v2/pkg/log"
+	pkgssh "github.com/wencaiwulue/kubevpn/v2/pkg/ssh"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
+)
+
+const (
+	FormatJson  = "json"
+	FormatYaml  = "yaml"
+	FormatTable = "table"
+)
+
+func CmdStatus(f cmdutil.Factory) *cobra.Command {
+	var aliasName string
+	var localFile string
+	var remoteAddr string
+	var format string
+	cmd := &cobra.Command{
+		Use:   "status",
+		Short: i18n.T("Show connect status and list proxy/clone resource"),
+		Long: templates.LongDesc(i18n.T(`
+		Show connect status and list proxy/clone resource
+
+		Show connect status and list proxy or clone resource, you can check connect status by filed status and netif.
+		if netif is empty, means tun device closed, so it's unhealthy, it will also show route info, if proxy workloads, 
+		not only show myself proxy resource, another route info will also display.
+		`)),
+		Example: templates.Examples(i18n.T(`
+        # show status for connect status and list proxy/clone resource 
+        kubevpn status
+
+		# query status by alias config name dev_new 
+        kubevpn status --alias dev_new
+
+		# query status with output json format
+		kubevpn status -o json
+
+		# query status with output yaml format
+		kubevpn status -o yaml
+		`)),
+		PreRunE: func(cmd *cobra.Command, args []string) (err error) {
+			plog.InitLoggerForClient()
+			return daemon.StartupDaemon(cmd.Context())
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			var clusterIDs []string
+			if aliasName != "" {
+				configs, err := ParseAndGet(localFile, remoteAddr, aliasName)
+				if err != nil {
+					return err
+				}
+				for _, conf := range configs {
+					clusterID, err := GetClusterIDByConfig(cmd, conf)
+					if err != nil {
+						return err
+					}
+					clusterIDs = append(clusterIDs, clusterID)
+				}
+			}
+
+			cli, err := daemon.GetClient(false)
+			if err != nil {
+				return err
+			}
+			resp, err := cli.Status(
+				cmd.Context(),
+				&rpc.StatusRequest{
+					ClusterIDs: clusterIDs,
+				},
+			)
+			if err != nil {
+				return err
+			}
+			output, err := genOutput(resp, format)
+			if err != nil {
+				return err
+			}
+			_, _ = fmt.Fprint(os.Stdout, output)
+			return nil
+		},
+	}
+	cmd.Flags().StringVar(&aliasName, "alias", "", "Alias name, query connect status by alias config name")
+	cmd.Flags().StringVarP(&localFile, "kubevpnconfig", "f", util.If(os.Getenv("KUBEVPNCONFIG") != "", os.Getenv("KUBEVPNCONFIG"), config.GetConfigFilePath()), "Path to the kubevpnconfig file to use for CLI requests.")
+	cmd.Flags().StringVarP(&remoteAddr, "remote", "r", "", "Remote config file, eg: https://raw.githubusercontent.com/kubenetworks/kubevpn/master/pkg/config/config.yaml")
+	cmd.Flags().StringVarP(&format, "output", "o", FormatTable, fmt.Sprintf("Output format. One of: (%s, %s, %s)", FormatJson, FormatYaml, FormatTable))
+	return cmd
+}
+
+func genOutput(status *rpc.StatusResponse, format string) (string, error) {
+	switch format {
+	case FormatJson:
+		if len(status.List) == 0 {
+			return "", nil
+		}
+		marshal, err := json.Marshal(status.List)
+		if err != nil {
+			return "", err
+		}
+		return string(marshal), nil
+
+	case FormatYaml:
+		if len(status.List) == 0 {
+			return "", nil
+		}
+		marshal, err := yaml.Marshal(status.List)
+		if err != nil {
+			return "", err
+		}
+		return string(marshal), nil
+	default:
+		var sb = new(bytes.Buffer)
+		w := printers.GetNewTabWriter(sb)
+		genConnectMsg(w, status.List)
+		genProxyMsg(w, status.List)
+		genCloneMsg(w, status.List)
+		_ = w.Flush()
+		return sb.String(), nil
+	}
+}
+
+func genConnectMsg(w *tabwriter.Writer, status []*rpc.Status) {
+	_, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", "ID", "Mode", "Cluster", "Kubeconfig", "Namespace", "Status", "Netif")
+	for _, c := range status {
+		_, _ = fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%s\t%s\t%s\n", c.ID, c.Mode, c.Cluster, c.Kubeconfig, c.Namespace, c.Status, c.Netif)
+	}
+}
+
+func genProxyMsg(w *tabwriter.Writer, list []*rpc.Status) {
+	var needsPrint bool
+	for _, status := range list {
+		if len(status.ProxyList) != 0 {
+			needsPrint = true
+			break
+		}
+	}
+	if !needsPrint {
+		return
+	}
+
+	_, _ = fmt.Fprintf(w, "\n")
+	w.SetRememberedWidths(nil)
+	_, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", "ID", "Namespace", "Name", "Headers", "IP", "PortMap", "CurrentPC")
+	for _, c := range list {
+		for _, proxy := range c.ProxyList {
+			for _, rule := range proxy.RuleList {
+				var headers []string
+				for k, v := range rule.Headers {
+					headers = append(headers, fmt.Sprintf("%s=%s", k, v))
+				}
+				if len(headers) == 0 {
+					headers = []string{"*"}
+				}
+				var portmap []string
+				for k, v := range rule.PortMap {
+					portmap = append(portmap, fmt.Sprintf("%d->%d", k, v))
+				}
+				_, _ = fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%s\t%s\t%v\n",
+					c.ID,
+					proxy.Namespace,
+					proxy.Workload,
+					strings.Join(headers, ","),
+					rule.LocalTunIPv4,
+					strings.Join(portmap, ","),
+					rule.CurrentDevice,
+				)
+			}
+		}
+	}
+}
+
+func genCloneMsg(w *tabwriter.Writer, list []*rpc.Status) {
+	var needsPrint bool
+	for _, status := range list {
+		if len(status.CloneList) != 0 {
+			needsPrint = true
+			break
+		}
+	}
+	if !needsPrint {
+		return
+	}
+
+	_, _ = fmt.Fprintf(w, "\n")
+	w.SetRememberedWidths(nil)
+	_, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", "ID", "Namespace", "Name", "Headers", "ToName", "ToKubeconfig", "ToNamespace", "SyncthingGUI")
+	for _, c := range list {
+		for _, clone := range c.CloneList {
+			//_, _ = fmt.Fprintf(w, "%s\n", clone.Workload)
+			for _, rule := range clone.RuleList {
+				var headers []string
+				for k, v := range rule.Headers {
+					headers = append(headers, fmt.Sprintf("%s=%s", k, v))
+				}
+				if len(headers) == 0 {
+					headers = []string{"*"}
+				}
+				_, _ = fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
+					c.ID,
+					clone.Namespace,
+					clone.Workload,
+					strings.Join(headers, ","),
+					rule.DstWorkload,
+					rule.DstKubeconfig,
+					rule.DstNamespace,
+					clone.SyncthingGUIAddr,
+				)
+			}
+		}
+	}
+}
+
+func GetClusterIDByConfig(cmd *cobra.Command, config Config) (string, error) {
+	flags := flag.NewFlagSet("", flag.ContinueOnError)
+	var sshConf = &pkgssh.SshConfig{}
+	pkgssh.AddSshFlags(flags, sshConf)
+	handler.AddExtraRoute(flags, &handler.ExtraRouteInfo{})
+	configFlags := genericclioptions.NewConfigFlags(true)
+	configFlags.AddFlags(flags)
+	matchVersionFlags := cmdutil.NewMatchVersionFlags(&warp{ConfigFlags: configFlags})
+	matchVersionFlags.AddFlags(flags)
+	factory := cmdutil.NewFactory(matchVersionFlags)
+
+	for _, command := range cmd.Parent().Commands() {
+		command.Flags().VisitAll(func(f *flag.Flag) {
+			if flags.Lookup(f.Name) == nil && flags.ShorthandLookup(f.Shorthand) == nil {
+				flags.AddFlag(f)
+			}
+		})
+	}
+
+	err := flags.ParseAll(config.Flags, func(flag *flag.Flag, value string) error {
+		_ = flags.Set(flag.Name, value)
+		return nil
+	})
+	bytes, ns, err := util.ConvertToKubeConfigBytes(factory)
+	if err != nil {
+		return "", err
+	}
+	file, err := util.ConvertToTempKubeconfigFile(bytes)
+	if err != nil {
+		return "", err
+	}
+	flags = flag.NewFlagSet("", flag.ContinueOnError)
+	flags.AddFlag(&flag.Flag{
+		Name:     "kubeconfig",
+		DefValue: file,
+	})
+	flags.AddFlag(&flag.Flag{
+		Name:     "namespace",
+		DefValue: ns,
+	})
+	var path string
+	path, err = pkgssh.SshJump(cmd.Context(), sshConf, flags, false)
+	if err != nil {
+		return "", err
+	}
+	var c = &handler.ConnectOptions{}
+	err = c.InitClient(util.InitFactoryByPath(path, ns))
+	if err != nil {
+		return "", err
+	}
+	err = c.InitDHCP(cmd.Context())
+	if err != nil {
+		return "", err
+	}
+	return c.GetClusterID(), nil
+}

cmd/kubevpn/cmds/status_test.go

@@ -0,0 +1,218 @@
+package cmds
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon/rpc"
+)
+
+func TestPrintProxyAndClone(t *testing.T) {
+	var status = &rpc.StatusResponse{
+		List: []*rpc.Status{
+			{
+				ID:         0,
+				ClusterID:  "ac6d8dfb-1d23-4f2a-b11e-9c775fd22b84",
+				Cluster:    "ccm6epn7qvcplhs3o8p00",
+				Mode:       "full",
+				Kubeconfig: "/Users/bytedance/.kube/test-feiyan-config-private-new",
+				Namespace:  "vke-system",
+				Status:     "Connected",
+				Netif:      "utun4",
+				ProxyList: []*rpc.Proxy{
+					{
+						ClusterID:  "ac6d8dfb-1d23-4f2a-b11e-9c775fd22b84",
+						Cluster:    "ccm6epn7qvcplhs3o8p00",
+						Kubeconfig: "/Users/bytedance/.kube/test-feiyan-config-private-new",
+						Namespace:  "vke-system",
+						Workload:   "deployment.apps/authors",
+						RuleList: []*rpc.ProxyRule{
+							{
+								Headers:       map[string]string{"user": "naison"},
+								LocalTunIPv4:  "198.19.0.103",
+								LocalTunIPv6:  "2001:2::999d",
+								CurrentDevice: false,
+								PortMap:       map[int32]int32{8910: 8910},
+							},
+						},
+					},
+				},
+				CloneList: []*rpc.Clone{
+					{
+						ClusterID:  "ac6d8dfb-1d23-4f2a-b11e-9c775fd22b84",
+						Cluster:    "ccm6epn7qvcplhs3o8p00",
+						Kubeconfig: "/Users/bytedance/.kube/test-feiyan-config-private-new",
+						Namespace:  "vke-system",
+						Workload:   "deployment.apps/ratings",
+						RuleList: []*rpc.CloneRule{{
+							Headers:       map[string]string{"user": "naison"},
+							DstClusterID:  "ac6d8dfb-1d23-4f2a-b11e-9c775fd22b84",
+							DstCluster:    "ccm6epn7qvcplhs3o8p00",
+							DstKubeconfig: "/Users/bytedance/.kube/test-feiyan-config-private-new",
+							DstNamespace:  "vke-system",
+							DstWorkload:   "deployment.apps/ratings-clone-5ngn6",
+						}},
+					},
+				},
+			},
+			{
+				ID:         1,
+				ClusterID:  "c08cae70-0021-46c9-a1dc-38e6a2f11443",
+				Cluster:    "ccnepblsebp68ivej4a20",
+				Mode:       "full",
+				Kubeconfig: "/Users/bytedance/.kube/dev_fy_config_new",
+				Namespace:  "vke-system",
+				Status:     "Connected",
+				Netif:      "utun5",
+				ProxyList:  []*rpc.Proxy{},
+				CloneList:  []*rpc.Clone{},
+			},
+		},
+	}
+	output, err := genOutput(status, FormatTable)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Println(output)
+}
+
+func TestPrintProxy(t *testing.T) {
+	var status = &rpc.StatusResponse{
+		List: []*rpc.Status{
+			{
+				ID:         0,
+				ClusterID:  "ac6d8dfb-1d23-4f2a-b11e-9c775fd22b84",
+				Cluster:    "ccm6epn7qvcplhs3o8p00",
+				Mode:       "full",
+				Kubeconfig: "/Users/bytedance/.kube/test-feiyan-config-private-new",
+				Namespace:  "vke-system",
+				Status:     "Connected",
+				Netif:      "utun4",
+				ProxyList: []*rpc.Proxy{
+					{
+						ClusterID:  "ac6d8dfb-1d23-4f2a-b11e-9c775fd22b84",
+						Cluster:    "ccm6epn7qvcplhs3o8p00",
+						Kubeconfig: "/Users/bytedance/.kube/test-feiyan-config-private-new",
+						Namespace:  "vke-system",
+						Workload:   "deployment.apps/authors",
+						RuleList: []*rpc.ProxyRule{
+							{
+								Headers:       map[string]string{"user": "naison"},
+								LocalTunIPv4:  "198.19.0.103",
+								LocalTunIPv6:  "2001:2::999d",
+								CurrentDevice: false,
+								PortMap:       map[int32]int32{8910: 8910},
+							},
+						},
+					},
+				},
+				CloneList: []*rpc.Clone{},
+			},
+			{
+				ID:         1,
+				ClusterID:  "c08cae70-0021-46c9-a1dc-38e6a2f11443",
+				Cluster:    "ccnepblsebp68ivej4a20",
+				Mode:       "full",
+				Kubeconfig: "/Users/bytedance/.kube/dev_fy_config_new",
+				Namespace:  "vke-system",
+				Status:     "Connected",
+				Netif:      "utun5",
+				ProxyList:  []*rpc.Proxy{},
+				CloneList:  []*rpc.Clone{},
+			},
+		},
+	}
+	output, err := genOutput(status, FormatTable)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Println(output)
+}
+
+func TestPrintClone(t *testing.T) {
+	var status = &rpc.StatusResponse{
+		List: []*rpc.Status{
+			{
+				ID:         0,
+				ClusterID:  "ac6d8dfb-1d23-4f2a-b11e-9c775fd22b84",
+				Cluster:    "ccm6epn7qvcplhs3o8p00",
+				Mode:       "full",
+				Kubeconfig: "/Users/bytedance/.kube/test-feiyan-config-private-new",
+				Namespace:  "vke-system",
+				Status:     "Connected",
+				Netif:      "utun4",
+				ProxyList:  []*rpc.Proxy{},
+				CloneList: []*rpc.Clone{
+					{
+						ClusterID:  "ac6d8dfb-1d23-4f2a-b11e-9c775fd22b84",
+						Cluster:    "ccm6epn7qvcplhs3o8p00",
+						Kubeconfig: "/Users/bytedance/.kube/test-feiyan-config-private-new",
+						Namespace:  "vke-system",
+						Workload:   "deployment.apps/ratings",
+						RuleList: []*rpc.CloneRule{{
+							Headers:       map[string]string{"user": "naison"},
+							DstClusterID:  "ac6d8dfb-1d23-4f2a-b11e-9c775fd22b84",
+							DstCluster:    "ccm6epn7qvcplhs3o8p00",
+							DstKubeconfig: "/Users/bytedance/.kube/test-feiyan-config-private-new",
+							DstNamespace:  "vke-system",
+							DstWorkload:   "deployment.apps/ratings-clone-5ngn6",
+						}},
+					},
+				},
+			},
+			{
+				ID:         1,
+				ClusterID:  "c08cae70-0021-46c9-a1dc-38e6a2f11443",
+				Cluster:    "ccnepblsebp68ivej4a20",
+				Mode:       "full",
+				Kubeconfig: "/Users/bytedance/.kube/dev_fy_config_new",
+				Namespace:  "vke-system",
+				Status:     "Connected",
+				Netif:      "utun5",
+				ProxyList:  []*rpc.Proxy{},
+				CloneList:  []*rpc.Clone{},
+			},
+		},
+	}
+	output, err := genOutput(status, FormatTable)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Println(output)
+}
+
+func TestPrint(t *testing.T) {
+	var status = &rpc.StatusResponse{
+		List: []*rpc.Status{
+			{
+				ID:         0,
+				ClusterID:  "ac6d8dfb-1d23-4f2a-b11e-9c775fd22b84",
+				Cluster:    "ccm6epn7qvcplhs3o8p00",
+				Mode:       "full",
+				Kubeconfig: "/Users/bytedance/.kube/test-feiyan-config-private-new",
+				Namespace:  "vke-system",
+				Status:     "Connected",
+				Netif:      "utun4",
+				ProxyList:  []*rpc.Proxy{},
+				CloneList:  []*rpc.Clone{},
+			},
+			{
+				ID:         1,
+				ClusterID:  "c08cae70-0021-46c9-a1dc-38e6a2f11443",
+				Cluster:    "ccnepblsebp68ivej4a20",
+				Mode:       "full",
+				Kubeconfig: "/Users/bytedance/.kube/dev_fy_config_new",
+				Namespace:  "vke-system",
+				Status:     "Connected",
+				Netif:      "utun5",
+				ProxyList:  []*rpc.Proxy{},
+				CloneList:  []*rpc.Clone{},
+			},
+		},
+	}
+	output, err := genOutput(status, FormatTable)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Println(output)
+}

cmd/kubevpn/cmds/syncthing.go

@@ -0,0 +1,30 @@
+package cmds
+
+import (
+	"github.com/spf13/cobra"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/syncthing"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
+)
+
+func CmdSyncthing(cmdutil.Factory) *cobra.Command {
+	var detach bool
+	var dir string
+	cmd := &cobra.Command{
+		Use:   "syncthing",
+		Short: i18n.T("Syncthing"),
+		Long:  templates.LongDesc(i18n.T(`Syncthing`)),
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			go util.StartupPProfForServer(0)
+			return syncthing.StartServer(cmd.Context(), detach, dir)
+		},
+		Hidden:                true,
+		DisableFlagsInUseLine: true,
+	}
+	cmd.Flags().StringVar(&dir, "dir", "", "dir")
+	cmd.Flags().BoolVarP(&detach, "detach", "d", false, "Run syncthing in background")
+	return cmd
+}

cmd/kubevpn/cmds/uninstall.go

@@ -0,0 +1,98 @@
+package cmds
+
+import (
+	"github.com/spf13/cobra"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+	"k8s.io/utils/ptr"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon/rpc"
+	plog "github.com/wencaiwulue/kubevpn/v2/pkg/log"
+	pkgssh "github.com/wencaiwulue/kubevpn/v2/pkg/ssh"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
+)
+
+func CmdUninstall(f cmdutil.Factory) *cobra.Command {
+	var sshConf = &pkgssh.SshConfig{}
+	cmd := &cobra.Command{
+		Use:   "uninstall",
+		Short: "Uninstall all resource create by kubevpn in k8s cluster",
+		Long: templates.LongDesc(i18n.T(`
+		Uninstall all resource create by kubevpn in k8s cluster
+		
+		Uninstall will delete all resources create by kubevpn in k8s cluster, like deployment, service, serviceAccount...
+		and it will also delete local develop docker containers, docker networks. delete hosts entry added by kubevpn,
+		cleanup DNS settings.
+		`)),
+		Example: templates.Examples(i18n.T(`
+        # Uninstall default namespace
+		  kubevpn uninstall
+
+		# Uninstall another namespace test
+		  kubevpn uninstall -n test
+
+		# Uninstall cluster api-server behind of bastion host or ssh jump host
+		kubevpn uninstall --ssh-addr 192.168.1.100:22 --ssh-username root --ssh-keyfile ~/.ssh/ssh.pem
+
+		# It also supports ProxyJump, like
+		┌──────┐     ┌──────┐     ┌──────┐     ┌──────┐                 ┌────────────┐
+		│  pc  ├────►│ ssh1 ├────►│ ssh2 ├────►│ ssh3 ├─────►... ─────► │ api-server │
+		└──────┘     └──────┘     └──────┘     └──────┘                 └────────────┘
+		kubevpn uninstall --ssh-alias <alias>
+
+		# Support ssh auth GSSAPI
+        kubevpn uninstall --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-keytab /path/to/keytab
+        kubevpn uninstall --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-cache /path/to/cache
+        kubevpn uninstall --ssh-addr <HOST:PORT> --ssh-username <USERNAME> --gssapi-password <PASSWORD>
+		`)),
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			plog.InitLoggerForClient()
+			return daemon.StartupDaemon(cmd.Context())
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			bytes, ns, err := util.ConvertToKubeConfigBytes(f)
+			if err != nil {
+				return err
+			}
+			cli, err := daemon.GetClient(false)
+			if err != nil {
+				return err
+			}
+			disconnectResp, err := cli.Disconnect(cmd.Context(), &rpc.DisconnectRequest{
+				KubeconfigBytes: ptr.To(string(bytes)),
+				Namespace:       ptr.To(ns),
+				SshJump:         sshConf.ToRPC(),
+			})
+			if err != nil {
+				plog.G(cmd.Context()).Warnf("Failed to disconnect from cluter: %v", err)
+			} else {
+				_ = util.PrintGRPCStream[rpc.DisconnectResponse](disconnectResp)
+			}
+
+			req := &rpc.UninstallRequest{
+				KubeconfigBytes: string(bytes),
+				Namespace:       ns,
+				SshJump:         sshConf.ToRPC(),
+			}
+			resp, err := cli.Uninstall(cmd.Context(), req)
+			if err != nil {
+				return err
+			}
+			err = util.PrintGRPCStream[rpc.UninstallResponse](resp)
+			if err != nil {
+				if status.Code(err) == codes.Canceled {
+					return nil
+				}
+				return err
+			}
+			return nil
+		},
+	}
+
+	pkgssh.AddSshFlags(cmd.Flags(), sshConf)
+	return cmd
+}

cmd/kubevpn/cmds/upgrade.go

@@ -0,0 +1,59 @@
+package cmds
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+
+	"github.com/spf13/cobra"
+	"golang.org/x/oauth2"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/config"
+	plog "github.com/wencaiwulue/kubevpn/v2/pkg/log"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/upgrade"
+)
+
+func CmdUpgrade(cmdutil.Factory) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "upgrade",
+		Short: i18n.T("Upgrade kubevpn client to latest version"),
+		Long: templates.LongDesc(i18n.T(`
+		Upgrade kubevpn client to latest version, automatically download and install latest kubevpn from GitHub.
+		disconnect all from k8s cluster, leave all resources, remove all clone resource, and then,  
+		upgrade local daemon grpc server to latest version.
+		`)),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			const (
+				envLatestUrl = "KUBEVPN_LATEST_VERSION_URL"
+			)
+			plog.InitLoggerForClient()
+			var client = http.DefaultClient
+			if config.GitHubOAuthToken != "" {
+				client = oauth2.NewClient(cmd.Context(), oauth2.StaticTokenSource(&oauth2.Token{AccessToken: config.GitHubOAuthToken, TokenType: "Bearer"}))
+			}
+			var url = os.Getenv(envLatestUrl)
+			if url == "" {
+				var latestVersion string
+				var needsUpgrade bool
+				var err error
+				url, latestVersion, needsUpgrade, err = upgrade.NeedsUpgrade(cmd.Context(), client, config.Version)
+				if err != nil {
+					return err
+				}
+				if !needsUpgrade {
+					_, _ = fmt.Fprintln(os.Stdout, fmt.Sprintf("Already up to date, don't needs to upgrade, version: %s", latestVersion))
+					return nil
+				}
+				_, _ = fmt.Fprintln(os.Stdout, fmt.Sprintf("Current version is: %s less than latest version: %s, needs to upgrade", config.Version, latestVersion))
+				_ = os.Setenv(envLatestUrl, url)
+				_ = quit(cmd.Context(), true)
+				_ = quit(cmd.Context(), false)
+			}
+			return upgrade.Main(cmd.Context(), client, url)
+		},
+	}
+	return cmd
+}

cmd/kubevpn/cmds/version.go

@@ -1,18 +1,25 @@
 package cmds
 
 import (
+	"context"
 	"fmt"
-	"github.com/spf13/cobra"
 	"runtime"
 	"runtime/debug"
 	"time"
+
+	"github.com/spf13/cobra"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/config"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/daemon/rpc"
 )
 
 // --ldflags -X
 var (
-	Version   = ""
 	OsArch    = ""
-	GitCommit = ""
 	BuildTime = ""
 	Branch    = ""
 )
@@ -25,27 +32,45 @@ func reformatDate(buildTime string) string {
 	return buildTime
 }
 
-var versionCmd = &cobra.Command{
-	Use:   "version",
-	Short: "Print the version number of KubeVPN",
-	Long:  `This is the version of KubeVPN`,
-	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Printf("KubeVPN: CLI\n")
-		fmt.Printf("    Version: %s\n", Version)
-		fmt.Printf("    Branch: %s\n", Branch)
-		fmt.Printf("    Git commit: %s\n", GitCommit)
-		fmt.Printf("    Built time: %s\n", reformatDate(BuildTime))
-		fmt.Printf("    Built OS/Arch: %s\n", OsArch)
-		fmt.Printf("    Built Go version: %s\n", runtime.Version())
-	},
+func CmdVersion(cmdutil.Factory) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "version",
+		Short: i18n.T("Print the client version information"),
+		Long: templates.LongDesc(i18n.T(`
+		Print the client version information
+		`)),
+		Run: func(cmd *cobra.Command, args []string) {
+			fmt.Printf("KubeVPN: CLI\n")
+			fmt.Printf("    Version: %s\n", config.Version)
+			fmt.Printf("    Daemon: %s\n", getDaemonVersion())
+			fmt.Printf("    Image: %s\n", config.Image)
+			fmt.Printf("    Branch: %s\n", Branch)
+			fmt.Printf("    Git commit: %s\n", config.GitCommit)
+			fmt.Printf("    Built time: %s\n", reformatDate(BuildTime))
+			fmt.Printf("    Built OS/Arch: %s\n", OsArch)
+			fmt.Printf("    Built Go version: %s\n", runtime.Version())
+		},
+	}
+	return cmd
 }
 
 func init() {
-	RootCmd.AddCommand(versionCmd)
 	// Prefer version number inserted at build using --ldflags
-	if Version == "" {
+	if config.Version == "" {
 		if i, ok := debug.ReadBuildInfo(); ok {
-			Version = i.Main.Version
+			config.Version = i.Main.Version
 		}
 	}
 }
+
+func getDaemonVersion() string {
+	cli, err := daemon.GetClient(false)
+	if err != nil {
+		return "unknown"
+	}
+	version, err := cli.Version(context.Background(), &rpc.VersionRequest{})
+	if err != nil {
+		return "unknown"
+	}
+	return version.Version
+}

cmd/kubevpn/cmds/webhook.go

@@ -0,0 +1,32 @@
+package cmds
+
+import (
+	"github.com/spf13/cobra"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/util/i18n"
+	"k8s.io/kubectl/pkg/util/templates"
+
+	"github.com/wencaiwulue/kubevpn/v2/pkg/util"
+	"github.com/wencaiwulue/kubevpn/v2/pkg/webhook"
+)
+
+func CmdWebhook(f cmdutil.Factory) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:    "webhook",
+		Hidden: true,
+		Short:  i18n.T("Starts a HTTP server, useful for creating MutatingAdmissionWebhook"),
+		Long: templates.LongDesc(i18n.T(`
+		Starts a HTTP server, useful for creating MutatingAdmissionWebhook.
+		After deploying it to Kubernetes cluster, the Administrator needs to create a MutatingWebhookConfiguration
+		in the Kubernetes cluster to register remote webhook admission controllers.
+		`)),
+		Args: cobra.MaximumNArgs(0),
+		PreRun: func(cmd *cobra.Command, args []string) {
+			go util.StartupPProfForServer(0)
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return webhook.Main(f)
+		},
+	}
+	return cmd
+}

cmd/kubevpn/main.go

@@ -1,15 +1,15 @@
 package main
 
 import (
-	log "github.com/sirupsen/logrus"
-	"github.com/wencaiwulue/kubevpn/cmd/kubevpn/cmds"
-	"net/http"
+	ctrl "sigs.k8s.io/controller-runtime"
+
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	_ "net/http/pprof"
+
+	"github.com/wencaiwulue/kubevpn/v2/cmd/kubevpn/cmds"
 )
 
 func main() {
-	go func() {
-		log.Println(http.ListenAndServe("localhost:6060", nil))
-	}()
-	_ = cmds.RootCmd.Execute()
+	ctx := ctrl.SetupSignalHandler()
+	_ = cmds.NewKubeVPNCommand().ExecuteContext(ctx)
 }

core/chain.go

@@ -1,87 +0,0 @@
-package core
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"math"
-	"net"
-)
-
-var (
-	// ErrorEmptyChain is an error that implies the chain is empty.
-	ErrorEmptyChain = errors.New("empty chain")
-)
-
-// Chain is a proxy chain that holds a list of proxy node groups.
-type Chain struct {
-	Retries int
-	node    *Node
-}
-
-// NewChain creates a proxy chain with a list of proxy nodes.
-// It creates the node groups automatically, one group per node.
-func NewChain(retry int, node *Node) *Chain {
-	return &Chain{Retries: retry, node: node}
-}
-
-func (c *Chain) Node() *Node {
-	return c.node
-}
-
-// IsEmpty checks if the chain is empty.
-// An empty chain means that there is no proxy node or node group in the chain.
-func (c *Chain) IsEmpty() bool {
-	return c == nil || c.node == nil
-}
-
-// DialContext connects to the address on the named network using the provided context.
-func (c *Chain) DialContext(ctx context.Context, network, address string) (conn net.Conn, err error) {
-	for i := 0; i < int(math.Max(float64(1), float64(c.Retries))); i++ {
-		conn, err = c.dial(ctx, network, address)
-		if err == nil {
-			break
-		}
-	}
-	return
-}
-
-func (c *Chain) dial(ctx context.Context, network, address string) (net.Conn, error) {
-	ipAddr := address
-	if address != "" {
-		ipAddr = c.resolve(address)
-	}
-
-	if c.IsEmpty() {
-		return nil, ErrorEmptyChain
-	}
-
-	conn, err := c.getConn(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	cc, err := c.Node().Client.ConnectContext(ctx, conn, network, ipAddr)
-	if err != nil {
-		_ = conn.Close()
-		return nil, err
-	}
-	return cc, nil
-}
-
-func (*Chain) resolve(addr string) string {
-	if host, port, err := net.SplitHostPort(addr); err == nil {
-		if ips, err := net.LookupIP(host); err == nil && len(ips) > 0 {
-			return fmt.Sprintf("%s:%s", ips[0].String(), port)
-		}
-	}
-	return addr
-}
-
-// getConn obtains a connection to the last node of the chain.
-func (c *Chain) getConn(_ context.Context) (net.Conn, error) {
-	if c.IsEmpty() {
-		return nil, ErrorEmptyChain
-	}
-	return c.Node().Client.Dial(c.Node().Addr)
-}

core/client.go

@@ -1,25 +0,0 @@
-package core
-
-import (
-	"context"
-	"net"
-)
-
-// Client is a proxy client.
-// A client is divided into two layers: connector and transporter.
-// Connector is responsible for connecting to the destination address through this proxy.
-// Transporter performs a handshake with this proxy.
-type Client struct {
-	Connector
-	Transporter
-}
-
-// Connector is responsible for connecting to the destination address.
-type Connector interface {
-	ConnectContext(ctx context.Context, conn net.Conn, network, address string) (net.Conn, error)
-}
-
-// Transporter is responsible for handshaking with the proxy server.
-type Transporter interface {
-	Dial(addr string) (net.Conn, error)
-}

core/handler.go

@@ -1,44 +0,0 @@
-package core
-
-import (
-	"context"
-	"github.com/wencaiwulue/kubevpn/tun"
-	"net"
-)
-
-// Handler is a proxy server handler
-type Handler interface {
-	Init(options ...HandlerOption)
-	Handle(ctx context.Context, conn net.Conn)
-}
-
-// HandlerOptions describes the options for Handler.
-type HandlerOptions struct {
-	Chain    *Chain
-	Node     *Node
-	IPRoutes []tun.IPRoute
-}
-
-// HandlerOption allows a common way to set handler options.
-type HandlerOption func(opts *HandlerOptions)
-
-// ChainHandlerOption sets the Chain option of HandlerOptions.
-func ChainHandlerOption(chain *Chain) HandlerOption {
-	return func(opts *HandlerOptions) {
-		opts.Chain = chain
-	}
-}
-
-// NodeHandlerOption set the server node for server handler.
-func NodeHandlerOption(node *Node) HandlerOption {
-	return func(opts *HandlerOptions) {
-		opts.Node = node
-	}
-}
-
-// IPRoutesHandlerOption sets the IP routes for tun tunnel.
-func IPRoutesHandlerOption(routes ...tun.IPRoute) HandlerOption {
-	return func(opts *HandlerOptions) {
-		opts.IPRoutes = routes
-	}
-}

core/node.go

@@ -1,67 +0,0 @@
-package core
-
-import (
-	"errors"
-	"net/url"
-	"strconv"
-	"strings"
-)
-
-var (
-	ErrorInvalidNode = errors.New("invalid node")
-)
-
-// Node is a proxy node, mainly used to construct a proxy chain.
-type Node struct {
-	Addr      string
-	Protocol  string
-	Transport string
-	Remote    string // remote address, used by tcp/udp port forwarding
-	Values    url.Values
-	Client    *Client
-}
-
-// ParseNode parses the node info.
-// The proxy node string pattern is [scheme://][user:pass@host]:port.
-func ParseNode(s string) (node *Node, err error) {
-	s = strings.TrimSpace(s)
-	if s == "" {
-		return nil, ErrorInvalidNode
-	}
-	u, err := url.Parse(s)
-	if err != nil {
-		return
-	}
-
-	node = &Node{
-		Addr:   u.Host,
-		Remote: strings.Trim(u.EscapedPath(), "/"),
-		Values: u.Query(),
-	}
-
-	u.RawQuery = ""
-	u.User = nil
-
-	switch u.Scheme {
-	case "tun":
-		node.Protocol = u.Scheme
-		node.Transport = u.Scheme
-	case "tcp":
-		node.Protocol = "tcp"
-		node.Transport = "tcp"
-	default:
-		return nil, ErrorInvalidNode
-	}
-	return
-}
-
-// Get returns node parameter specified by key.
-func (node *Node) Get(key string) string {
-	return node.Values.Get(key)
-}
-
-// GetInt converts node parameter value to int.
-func (node *Node) GetInt(key string) int {
-	n, _ := strconv.Atoi(node.Get(key))
-	return n
-}

core/server.go

@@ -1,72 +0,0 @@
-package core
-
-import (
-	"context"
-	"k8s.io/client-go/util/retry"
-	"net"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-)
-
-// Server is a proxy server.
-type Server struct {
-	Listener net.Listener
-	Handler  Handler
-}
-
-// Addr returns the address of the server
-func (s *Server) Addr() net.Addr {
-	return s.Listener.Addr()
-}
-
-// Close closes the server
-func (s *Server) Close() error {
-	return s.Listener.Close()
-}
-
-// Serve serves as a proxy server.
-func (s *Server) Serve(ctx context.Context, h Handler) error {
-	if h == nil {
-		h = s.Handler
-	}
-
-	l := s.Listener
-	var tempDelay time.Duration
-	go func() {
-		select {
-		case <-ctx.Done():
-			err := retry.OnError(retry.DefaultBackoff, func(err error) bool {
-				return err != nil
-			}, func() error {
-				return l.Close()
-			})
-			if err != nil {
-				log.Warnf("error while close listener, err: %v", err)
-			}
-		}
-	}()
-	for ctx.Err() == nil {
-		conn, e := l.Accept()
-		if e != nil {
-			if ne, ok := e.(net.Error); ok && ne.Temporary() {
-				if tempDelay == 0 {
-					tempDelay = 5 * time.Millisecond
-				} else {
-					tempDelay *= 2
-				}
-				if max := 1 * time.Second; tempDelay > max {
-					tempDelay = max
-				}
-				log.Warnf("server: Accept error: %v; retrying in %v", e, tempDelay)
-				time.Sleep(tempDelay)
-				continue
-			}
-			return e
-		}
-		tempDelay = 0
-
-		go h.Handle(ctx, conn)
-	}
-	return nil
-}

core/tcp.go

@@ -1,52 +0,0 @@
-package core
-
-import (
-	"crypto/tls"
-	"github.com/wencaiwulue/kubevpn/tlsconfig"
-	"github.com/wencaiwulue/kubevpn/util"
-	"net"
-)
-
-// tcpTransporter is a raw TCP transporter.
-type tcpTransporter struct{}
-
-// TCPTransporter creates a raw TCP client.
-func TCPTransporter() Transporter {
-	return &tcpTransporter{}
-}
-
-func (tr *tcpTransporter) Dial(addr string) (net.Conn, error) {
-	dialer := &net.Dialer{Timeout: util.DialTimeout}
-	return tls.DialWithDialer(dialer, "tcp", addr, tlsconfig.TlsconfigClient)
-}
-
-type tcpListener struct {
-	net.Listener
-}
-
-// TCPListener creates a Listener for TCP proxy server.
-func TCPListener(addr string) (net.Listener, error) {
-	laddr, err := net.ResolveTCPAddr("tcp", addr)
-	if err != nil {
-		return nil, err
-	}
-	ln, err := net.ListenTCP("tcp", laddr)
-	if err != nil {
-		return nil, err
-	}
-	return &tcpListener{Listener: tcpKeepAliveListener{TCPListener: ln}}, nil
-}
-
-type tcpKeepAliveListener struct {
-	*net.TCPListener
-}
-
-func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
-	tc, err := ln.AcceptTCP()
-	if err != nil {
-		return
-	}
-	_ = tc.SetKeepAlive(true)
-	_ = tc.SetKeepAlivePeriod(util.KeepAliveTime)
-	return tc, nil
-}

core/tcpserver.go

@@ -1,255 +0,0 @@
-package core
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	log "github.com/sirupsen/logrus"
-	"github.com/wencaiwulue/kubevpn/util"
-	"net"
-	"time"
-)
-
-type fakeUDPTunConnector struct {
-}
-
-// UDPOverTCPTunnelConnector creates a connector for UDP-over-TCP
-func UDPOverTCPTunnelConnector() Connector {
-	return &fakeUDPTunConnector{}
-}
-
-func (c *fakeUDPTunConnector) ConnectContext(_ context.Context, conn net.Conn, network, address string) (net.Conn, error) {
-	switch network {
-	case "tcp", "tcp4", "tcp6":
-		return nil, fmt.Errorf("%s unsupported", network)
-	}
-	_ = conn.SetDeadline(time.Now().Add(util.ConnectTimeout))
-	defer conn.SetDeadline(time.Time{})
-
-	targetAddr, _ := net.ResolveUDPAddr("udp", address)
-	return newFakeUDPTunnelConnOverTCP(conn, targetAddr)
-}
-
-type fakeUdpHandler struct {
-}
-
-// TCPHandler creates a server Handler
-func TCPHandler() Handler {
-	return &fakeUdpHandler{}
-}
-
-func (h *fakeUdpHandler) Init(...HandlerOption) {
-}
-
-func (h *fakeUdpHandler) Handle(ctx context.Context, conn net.Conn) {
-	defer conn.Close()
-	if util.Debug {
-		log.Debugf("[tcpserver] %s -> %s\n", conn.RemoteAddr(), conn.LocalAddr())
-	}
-	h.handleUDPTunnel(conn)
-}
-
-func (h *fakeUdpHandler) transportUDP(relay, peer net.PacketConn) (err error) {
-	errChan := make(chan error, 2)
-	var clientAddr net.Addr
-	go func() {
-		b := util.MPool.Get().([]byte)
-		defer util.MPool.Put(b)
-
-		for {
-			n, laddr, err := relay.ReadFrom(b)
-			if err != nil {
-				errChan <- err
-				return
-			}
-			if clientAddr == nil {
-				clientAddr = laddr
-			}
-			dgram, err := ReadDatagramPacket(bytes.NewReader(b[:n]))
-			if err != nil {
-				log.Errorln(err)
-				errChan <- err
-				return
-			}
-
-			raddr, err := net.ResolveUDPAddr("udp", dgram.Addr())
-			if err != nil {
-				log.Debugf("[tcpserver-udp] addr error, addr: %s, err: %v", dgram.Addr(), err)
-				continue // drop silently
-			}
-			if _, err := peer.WriteTo(dgram.Data, raddr); err != nil {
-				errChan <- err
-				return
-			}
-			if util.Debug {
-				log.Debugf("[tcpserver-udp] %s >>> %s length: %d", relay.LocalAddr(), raddr, len(dgram.Data))
-			}
-		}
-	}()
-
-	go func() {
-		b := util.MPool.Get().([]byte)
-		defer util.MPool.Put(b)
-
-		for {
-			n, raddr, err := peer.ReadFrom(b)
-			if err != nil {
-				errChan <- err
-				return
-			}
-			if clientAddr == nil {
-				continue
-			}
-			buf := bytes.Buffer{}
-			dgram := NewDatagramPacket(raddr, b[:n])
-			_ = dgram.Write(&buf)
-			if _, err := relay.WriteTo(buf.Bytes(), clientAddr); err != nil {
-				errChan <- err
-				return
-			}
-			if util.Debug {
-				log.Debugf("[tcpserver-udp] %s <<< %s length: %d", relay.LocalAddr(), raddr, len(dgram.Data))
-			}
-		}
-	}()
-
-	return <-errChan
-}
-
-func (h *fakeUdpHandler) handleUDPTunnel(conn net.Conn) {
-	// serve tunnel udp, tunnel <-> remote, handle tunnel udp request
-	bindAddr, _ := net.ResolveUDPAddr("udp", ":0")
-	uc, err := net.ListenUDP("udp", bindAddr)
-	if err != nil {
-		log.Debugf("[tcpserver] udp-tun %s -> %s : %s", conn.RemoteAddr(), bindAddr, err)
-		return
-	}
-	defer uc.Close()
-	if util.Debug {
-		log.Debugf("[tcpserver] udp-tun %s <- %s\n", conn.RemoteAddr(), uc.LocalAddr())
-	}
-	log.Debugf("[tcpserver] udp-tun %s <-> %s", conn.RemoteAddr(), uc.LocalAddr())
-	_ = h.tunnelServerUDP(conn, uc)
-	log.Debugf("[tcpserver] udp-tun %s >-< %s", conn.RemoteAddr(), uc.LocalAddr())
-	return
-}
-
-func (h *fakeUdpHandler) tunnelServerUDP(cc net.Conn, pc net.PacketConn) (err error) {
-	errChan := make(chan error, 2)
-
-	go func() {
-		b := util.MPool.Get().([]byte)
-		defer util.MPool.Put(b)
-
-		for {
-			n, addr, err := pc.ReadFrom(b)
-			if err != nil {
-				log.Debugf("[udp-tun] %s : %s", cc.RemoteAddr(), err)
-				errChan <- err
-				return
-			}
-
-			// pipe from peer to tunnel
-			dgram := NewDatagramPacket(addr, b[:n])
-			if err := dgram.Write(cc); err != nil {
-				log.Debugf("[tcpserver] udp-tun %s <- %s : %s", cc.RemoteAddr(), dgram.Addr(), err)
-				errChan <- err
-				return
-			}
-			if util.Debug {
-				log.Debugf("[tcpserver] udp-tun %s <<< %s length: %d", cc.RemoteAddr(), dgram.Addr(), len(dgram.Data))
-			}
-		}
-	}()
-
-	go func() {
-		for {
-			dgram, err := ReadDatagramPacket(cc)
-			if err != nil {
-				log.Debugf("[udp-tun] %s -> 0 : %v", cc.RemoteAddr(), err)
-				errChan <- err
-				return
-			}
-
-			// pipe from tunnel to peer
-			addr, err := net.ResolveUDPAddr("udp", dgram.Addr())
-			if err != nil {
-				log.Debugf("[tcpserver-udp] addr error, addr: %s, err: %v", dgram.Addr(), err)
-				continue // drop silently
-			}
-			if _, err := pc.WriteTo(dgram.Data, addr); err != nil {
-				log.Debugf("[tcpserver] udp-tun %s -> %s : %s", cc.RemoteAddr(), addr, err)
-				errChan <- err
-				return
-			}
-			if util.Debug {
-				log.Debugf("[tcpserver] udp-tun %s >>> %s length: %d", cc.RemoteAddr(), addr, len(dgram.Data))
-			}
-		}
-	}()
-
-	return <-errChan
-}
-
-// fake udp connect over tcp
-type fakeUDPTunnelConn struct {
-	// tcp connection
-	net.Conn
-	targetAddr net.Addr
-}
-
-func newFakeUDPTunnelConnOverTCP(conn net.Conn, targetAddr net.Addr) (net.Conn, error) {
-	return &fakeUDPTunnelConn{
-		Conn:       conn,
-		targetAddr: targetAddr,
-	}, nil
-}
-
-func (c *fakeUDPTunnelConn) Read(b []byte) (n int, err error) {
-	n, _, err = c.ReadFrom(b)
-	return
-}
-
-func (c *fakeUDPTunnelConn) ReadFrom(b []byte) (n int, addr net.Addr, err error) {
-	dgram, err := ReadDatagramPacket(c.Conn)
-	if err != nil {
-		log.Debug(err)
-		return
-	}
-	n = copy(b, dgram.Data)
-	addr, err = net.ResolveUDPAddr("udp", dgram.Addr())
-	if err != nil {
-		log.Debugf("[tcpserver-udp] addr error, addr: %s, err: %v", dgram.Addr(), err)
-	}
-	return
-}
-
-func (c *fakeUDPTunnelConn) Write(b []byte) (n int, err error) {
-	return c.WriteTo(b, c.targetAddr)
-}
-
-func (c *fakeUDPTunnelConn) WriteTo(b []byte, addr net.Addr) (n int, err error) {
-	dgram := NewDatagramPacket(addr, b)
-	if err = dgram.Write(c.Conn); err != nil {
-		return
-	}
-	return len(b), nil
-}
-
-func (c *fakeUDPTunnelConn) Close() error {
-	return c.Conn.Close()
-}
-
-func (c *fakeUDPTunnelConn) CloseWrite() error {
-	if cc, ok := c.Conn.(interface{ CloseWrite() error }); ok {
-		return cc.CloseWrite()
-	}
-	return nil
-}
-
-func (c *fakeUDPTunnelConn) CloseRead() error {
-	if cc, ok := c.Conn.(interface{ CloseRead() error }); ok {
-		return cc.CloseRead()
-	}
-	return nil
-}

core/tunhandler.go

@@ -1,296 +0,0 @@
-package core
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"github.com/wencaiwulue/kubevpn/util"
-	"net"
-	"sync"
-	"time"
-
-	"github.com/shadowsocks/go-shadowsocks2/shadowaead"
-	log "github.com/sirupsen/logrus"
-	"github.com/songgao/water/waterutil"
-	"golang.org/x/net/ipv4"
-	"golang.org/x/net/ipv6"
-)
-
-func ipToTunRouteKey(ip net.IP) string {
-	return ip.To16().String()
-}
-
-type tunHandler struct {
-	options *HandlerOptions
-	routes  sync.Map
-	chExit  chan struct{}
-}
-
-// TunHandler creates a handler for tun tunnel.
-func TunHandler(opts ...HandlerOption) Handler {
-	h := &tunHandler{
-		options: &HandlerOptions{},
-		chExit:  make(chan struct{}, 1),
-	}
-	for _, opt := range opts {
-		opt(h.options)
-	}
-	return h
-}
-
-func (h *tunHandler) Init(options ...HandlerOption) {
-	if h.options == nil {
-		h.options = &HandlerOptions{}
-	}
-	for _, opt := range options {
-		opt(h.options)
-	}
-}
-
-func (h *tunHandler) Handle(ctx context.Context, conn net.Conn) {
-	defer conn.Close()
-	var err error
-	var raddr net.Addr
-	if addr := h.options.Node.Remote; addr != "" {
-		raddr, err = net.ResolveUDPAddr("udp", addr)
-		if err != nil {
-			log.Debugf("[tun] %s: remote addr: %v", conn.LocalAddr(), err)
-			return
-		}
-	}
-
-	var tempDelay time.Duration
-	for ctx.Err() == nil {
-		err = func() error {
-			var err error
-			var pc net.PacketConn
-			if raddr != nil && !h.options.Chain.IsEmpty() {
-				cc, err := h.options.Chain.DialContext(ctx, "udp", raddr.String())
-				if err != nil {
-					return err
-				}
-				var ok bool
-				pc, ok = cc.(net.PacketConn)
-				if !ok {
-					err = errors.New("not a packet connection")
-					log.Debugf("[tun] %s - %s: %s", conn.LocalAddr(), raddr, err)
-					return err
-				}
-			} else {
-				laddr, _ := net.ResolveUDPAddr("udp", h.options.Node.Addr)
-				pc, err = net.ListenUDP("udp", laddr)
-			}
-			if err != nil {
-				return err
-			}
-
-			return h.transportTun(ctx, conn, pc, raddr)
-		}()
-		if err != nil {
-			log.Debugf("[tun] %s: %v", conn.LocalAddr(), err)
-		}
-
-		select {
-		case <-h.chExit:
-			return
-		case <-ctx.Done():
-			h.chExit <- struct{}{}
-		default:
-			fmt.Println("next loop")
-		}
-
-		if err != nil {
-			if tempDelay == 0 {
-				tempDelay = 1000 * time.Millisecond
-			} else {
-				tempDelay *= 2
-			}
-			if max := 6 * time.Second; tempDelay > max {
-				tempDelay = max
-			}
-			time.Sleep(tempDelay)
-			continue
-		}
-		tempDelay = 0
-	}
-}
-
-func (h *tunHandler) findRouteFor(dst net.IP) net.Addr {
-	if v, ok := h.routes.Load(ipToTunRouteKey(dst)); ok {
-		return v.(net.Addr)
-	}
-	for _, route := range h.options.IPRoutes {
-		if route.Dest.Contains(dst) && route.Gateway != nil {
-			if v, ok := h.routes.Load(ipToTunRouteKey(route.Gateway)); ok {
-				return v.(net.Addr)
-			}
-		}
-	}
-	return nil
-}
-
-func (h *tunHandler) transportTun(ctx context.Context, tun net.Conn, conn net.PacketConn, raddr net.Addr) error {
-	errChan := make(chan error, 2)
-	defer func() {
-		if c, ok := conn.(interface{ CloseRead() error }); ok {
-			_ = c.CloseRead()
-		}
-		if c, ok := conn.(interface{ CloseWrite() error }); ok {
-			_ = c.CloseWrite()
-		}
-		_ = conn.Close()
-	}()
-	go func() {
-		for ctx.Err() == nil {
-			err := func() error {
-				b := util.SPool.Get().([]byte)
-				defer util.SPool.Put(b)
-
-				n, err := tun.Read(b)
-				if err != nil {
-					select {
-					case h.chExit <- struct{}{}:
-					default:
-					}
-					return err
-				}
-
-				var src, dst net.IP
-				if waterutil.IsIPv4(b[:n]) {
-					header, err := ipv4.ParseHeader(b[:n])
-					if err != nil {
-						log.Debugf("[tun] %s: %v", tun.LocalAddr(), err)
-						return nil
-					}
-					if util.Debug {
-						log.Debugf("[tun] %s", header.String())
-					}
-					src, dst = header.Src, header.Dst
-				} else if waterutil.IsIPv6(b[:n]) {
-					header, err := ipv6.ParseHeader(b[:n])
-					if err != nil {
-						log.Debugf("[tun] %s: %v", tun.LocalAddr(), err)
-						return nil
-					}
-					if util.Debug {
-						log.Debugf("[tun] %s", header.String())
-					}
-					src, dst = header.Src, header.Dst
-				} else {
-					log.Debugf("[tun] unknown packet")
-					return nil
-				}
-
-				// client side, deliver packet directly.
-				if raddr != nil {
-					_, err := conn.WriteTo(b[:n], raddr)
-					return err
-				}
-
-				addr := h.findRouteFor(dst)
-				if addr == nil {
-					log.Debugf("[tun] no route for %s -> %s", src, dst)
-					return nil
-				}
-
-				if util.Debug {
-					log.Debugf("[tun] find route: %s -> %s", dst, addr)
-				}
-				if _, err := conn.WriteTo(b[:n], addr); err != nil {
-					return err
-				}
-				return nil
-			}()
-
-			if err != nil {
-				errChan <- err
-				return
-			}
-		}
-	}()
-
-	go func() {
-		for ctx.Err() == nil {
-			err := func() error {
-				b := util.SPool.Get().([]byte)
-				defer util.SPool.Put(b)
-
-				n, addr, err := conn.ReadFrom(b)
-				if err != nil && err != shadowaead.ErrShortPacket {
-					return err
-				}
-
-				var src, dst net.IP
-				if waterutil.IsIPv4(b[:n]) {
-					header, err := ipv4.ParseHeader(b[:n])
-					if err != nil {
-						log.Debugf("[tun] %s: %v", tun.LocalAddr(), err)
-						return nil
-					}
-					if util.Debug {
-						log.Debugf("[tun] %s", header.String())
-					}
-					src, dst = header.Src, header.Dst
-				} else if waterutil.IsIPv6(b[:n]) {
-					header, err := ipv6.ParseHeader(b[:n])
-					if err != nil {
-						log.Debugf("[tun] %s: %v", tun.LocalAddr(), err)
-						return nil
-					}
-					if util.Debug {
-						log.Debugf("[tun] %s", header.String())
-					}
-					src, dst = header.Src, header.Dst
-				} else {
-					log.Debugf("[tun] unknown packet")
-					return nil
-				}
-
-				// client side, deliver packet to tun device.
-				if raddr != nil {
-					_, err := tun.Write(b[:n])
-					return err
-				}
-
-				routeKey := ipToTunRouteKey(src)
-				if actual, loaded := h.routes.LoadOrStore(routeKey, addr); loaded {
-					if actual.(net.Addr).String() != addr.String() {
-						log.Debugf("[tun] update route: %s -> %s (old %s)", src, addr, actual.(net.Addr))
-						h.routes.Store(routeKey, addr)
-					}
-				} else {
-					log.Debugf("[tun] new route: %s -> %s", src, addr)
-				}
-
-				if addr := h.findRouteFor(dst); addr != nil {
-					if util.Debug {
-						log.Debugf("[tun] find route: %s -> %s", dst, addr)
-					}
-					_, err := conn.WriteTo(b[:n], addr)
-					return err
-				}
-
-				if _, err := tun.Write(b[:n]); err != nil {
-					select {
-					case h.chExit <- struct{}{}:
-					default:
-					}
-					return err
-				}
-				return nil
-			}()
-
-			if err != nil {
-				errChan <- err
-				return
-			}
-		}
-	}()
-
-	select {
-	case err := <-errChan:
-		return err
-	case <-ctx.Done():
-		return nil
-	}
-}

core/udpovertcp.go

@@ -1,146 +0,0 @@
-package core
-
-import (
-	"bytes"
-	"encoding/binary"
-	"errors"
-	"fmt"
-	"github.com/wencaiwulue/kubevpn/util"
-	"io"
-	"net"
-	"strconv"
-	"strings"
-)
-
-const (
-	AddrIPv4   uint8 = 1
-	AddrIPv6         = 2
-	AddrDomain       = 3
-)
-
-type DatagramPacket struct {
-	Type       uint8  // [1]byte
-	Host       string // [?]byte, first byte is length if it's a domain
-	Port       uint16 // [2]byte
-	DataLength uint16 // [2]byte
-	Data       []byte // []byte
-}
-
-func (addr *DatagramPacket) String() string {
-	if addr == nil {
-		return ""
-	}
-	return fmt.Sprintf("Type: %d, Host: %s, Port: %d, DataLength: %d, Data: %v\n",
-		addr.Type, addr.Host, addr.Port, addr.DataLength, addr.Data)
-}
-
-func NewDatagramPacket(addr net.Addr, data []byte) (r *DatagramPacket) {
-	s := addr.String()
-	var t uint8
-	if strings.Count(s, ":") >= 2 {
-		t = AddrIPv6
-	} else {
-		if ip := net.ParseIP(strings.Split(s, ":")[0]); ip != nil {
-			t = AddrIPv4
-		} else {
-			t = AddrDomain
-		}
-	}
-	host, port, _ := net.SplitHostPort(s)
-	atoi, _ := strconv.Atoi(port)
-	// todo if host is a domain
-	r = &DatagramPacket{
-		Host:       host,
-		Port:       uint16(atoi),
-		Type:       t,
-		DataLength: uint16(len(data)),
-		Data:       data,
-	}
-	return r
-}
-
-func (addr *DatagramPacket) Addr() string {
-	return net.JoinHostPort(addr.Host, strconv.Itoa(int(addr.Port)))
-}
-
-func ReadDatagramPacket(r io.Reader) (rr *DatagramPacket, errsss error) {
-	b := util.LPool.Get().([]byte)
-	defer util.LPool.Put(b)
-	_, err := io.ReadFull(r, b[:1])
-	if err != nil {
-		return nil, err
-	}
-
-	atype := b[0]
-	d := &DatagramPacket{Type: atype}
-	hostLength := 0
-	switch atype {
-	case AddrIPv4:
-		hostLength = net.IPv4len
-	case AddrIPv6:
-		hostLength = net.IPv6len
-	case AddrDomain:
-		_, err = io.ReadFull(r, b[:1])
-		if err != nil {
-			return nil, err
-		}
-		hostLength = int(b[0])
-	default:
-		return nil, errors.New("")
-	}
-
-	if _, err = io.ReadFull(r, b[:hostLength]); err != nil {
-		return nil, err
-	}
-	var host string
-	switch atype {
-	case AddrIPv4:
-		host = net.IPv4(b[0], b[1], b[2], b[3]).String()
-	case AddrIPv6:
-		p := make(net.IP, net.IPv6len)
-		copy(p, b[:hostLength])
-		host = p.String()
-	case AddrDomain:
-		host = string(b[:hostLength])
-	}
-	d.Host = host
-
-	if _, err = io.ReadFull(r, b[:4]); err != nil {
-		return nil, err
-	}
-	d.Port = binary.BigEndian.Uint16(b[:2])
-	d.DataLength = binary.BigEndian.Uint16(b[2:4])
-
-	if _, err = io.ReadFull(r, b[:d.DataLength]); err != nil && (err != io.ErrUnexpectedEOF || err != io.EOF) {
-		return nil, err
-	}
-	i := make([]byte, d.DataLength)
-	copy(i, b[:d.DataLength])
-	d.Data = i
-	rr = d
-	return d, nil
-}
-
-func (addr *DatagramPacket) Write(w io.Writer) error {
-	buf := bytes.Buffer{}
-	buf.WriteByte(addr.Type)
-	switch addr.Type {
-	case AddrIPv4:
-		buf.Write(net.ParseIP(addr.Host).To4())
-	case AddrIPv6:
-		buf.Write(net.ParseIP(addr.Host).To16())
-	case AddrDomain:
-		buf.WriteByte(byte(len(addr.Host)))
-		buf.WriteString(addr.Host)
-	}
-	i := make([]byte, 2)
-	binary.BigEndian.PutUint16(i, addr.Port)
-	buf.Write(i)
-	binary.BigEndian.PutUint16(i, uint16(len(addr.Data)))
-	buf.Write(i)
-	if _, err := buf.Write(addr.Data); err != nil {
-		return err
-	}
-	_, err := buf.WriteTo(w)
-	return err
-}

dns/dns.go

@@ -1,55 +0,0 @@
-package dns
-
-import (
-	"bytes"
-	"context"
-	miekgdns "github.com/miekg/dns"
-	"github.com/pkg/errors"
-	"github.com/wencaiwulue/kubevpn/util"
-	v12 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/fields"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
-)
-
-func GetDNSServiceIPFromPod(clientset *kubernetes.Clientset, restclient *rest.RESTClient, config *rest.Config, podName, namespace string) (*miekgdns.ClientConfig, error) {
-	var ipp []string
-	if ips, err := getDNSIPFromDnsPod(clientset); err == nil {
-		ipp = ips
-	}
-	resolvConfStr, err := util.Shell(clientset, restclient, config, podName, namespace, "cat /etc/resolv.conf")
-	if err != nil {
-		return nil, err
-	}
-	resolvConf, err := miekgdns.ClientConfigFromReader(bytes.NewBufferString(resolvConfStr))
-	if err != nil {
-		return nil, err
-	}
-	if len(ipp) != 0 {
-		resolvConf.Servers = append(resolvConf.Servers, make([]string, len(ipp))...)
-		copy(resolvConf.Servers[len(ipp):], resolvConf.Servers[:len(resolvConf.Servers)-len(ipp)])
-		for i := range ipp {
-			resolvConf.Servers[i] = ipp[i]
-		}
-	}
-	return resolvConf, nil
-}
-
-func getDNSIPFromDnsPod(clientset *kubernetes.Clientset) (ips []string, err error) {
-	serviceList, err := clientset.CoreV1().Pods(v1.NamespaceSystem).List(context.Background(), v1.ListOptions{
-		LabelSelector: fields.OneTermEqualSelector("k8s-app", "kube-dns").String(),
-	})
-	if err != nil {
-		return
-	}
-	for _, pod := range serviceList.Items {
-		if pod.Status.Phase == v12.PodRunning && pod.DeletionTimestamp == nil {
-			ips = append(ips, pod.Status.PodIP)
-		}
-	}
-	if len(ips) == 0 {
-		return nil, errors.New("")
-	}
-	return ips, nil
-}

dns/dns_linux.go

@@ -1,37 +0,0 @@
-//go:build linux
-// +build linux
-
-package dns
-
-import (
-	miekgdns "github.com/miekg/dns"
-	log "github.com/sirupsen/logrus"
-	"os"
-	"os/exec"
-)
-
-// systemd-resolve --status, systemd-resolve --flush-caches
-func SetupDNS(config *miekgdns.ClientConfig) error {
-	tunName := os.Getenv("tunName")
-	if len(tunName) == 0 {
-		tunName = "tun0"
-	}
-	cmd := exec.Command("systemd-resolve", []string{
-		"--set-dns",
-		config.Servers[0],
-		"--interface",
-		tunName,
-		"--set-domain=" + config.Search[0],
-		"--set-domain=" + config.Search[1],
-		"--set-domain=" + config.Search[2],
-	}...)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		log.Warnf("cmd: %s, output: %s, error: %v\n", cmd.Args, string(output), err)
-	}
-
-	return nil
-}
-
-func CancelDNS() {
-}

dns/dns_unix.go

@@ -1,257 +0,0 @@
-//go:build darwin
-// +build darwin
-
-package dns
-
-import (
-	"context"
-	"fmt"
-	"github.com/fsnotify/fsnotify"
-	miekgdns "github.com/miekg/dns"
-	log "github.com/sirupsen/logrus"
-	"github.com/wencaiwulue/kubevpn/util"
-	"io/fs"
-	"io/ioutil"
-	"k8s.io/apimachinery/pkg/util/sets"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"time"
-)
-
-var cancel context.CancelFunc
-var resolv = "/etc/resolv.conf"
-
-// SetupDNS support like
-// service:port
-// service.namespace:port
-// service.namespace.svc:port
-// service.namespace.svc.cluster:port
-// service.namespace.svc.cluster.local:port
-func SetupDNS(config *miekgdns.ClientConfig) error {
-	usingResolver(config)
-	_ = exec.Command("killall", "mDNSResponderHelper").Run()
-	_ = exec.Command("killall", "-HUP", "mDNSResponder").Run()
-	_ = exec.Command("dscacheutil", "-flushcache").Run()
-	return nil
-}
-
-func usingResolver(clientConfig *miekgdns.ClientConfig) {
-	var err error
-	_ = os.RemoveAll(filepath.Join("/", "etc", "resolver"))
-	if err = os.MkdirAll(filepath.Join("/", "etc", "resolver"), fs.ModePerm); err != nil {
-		log.Error(err)
-	}
-	config := miekgdns.ClientConfig{
-		Servers: clientConfig.Servers,
-		Search:  clientConfig.Search,
-		Ndots:   5,
-		Timeout: 1,
-	}
-	// for support like: service:port, service.namespace.svc.cluster.local:port
-	filename := filepath.Join("/", "etc", "resolver", "local")
-	_ = ioutil.WriteFile(filename, []byte(toString(config)), 0644)
-
-	// for support like: service.namespace:port, service.namespace.svc:port, service.namespace.svc.cluster:port
-	port := util.GetAvailableUDPPortOrDie()
-	go func(port int, clientConfig *miekgdns.ClientConfig) {
-		if err = NewDNSServer("udp", "127.0.0.1:"+strconv.Itoa(port), clientConfig); err != nil {
-			log.Warnln(err)
-		}
-	}(port, clientConfig)
-	config = miekgdns.ClientConfig{
-		Servers: []string{"127.0.0.1"},
-		Search:  clientConfig.Search,
-		Port:    strconv.Itoa(port),
-		Ndots:   clientConfig.Ndots,
-		Timeout: 1,
-	}
-	for _, s := range strings.Split(clientConfig.Search[0], ".") {
-		filename = filepath.Join("/", "etc", "resolver", s)
-		_ = ioutil.WriteFile(filename, []byte(toString(config)), 0644)
-	}
-}
-
-func usingNetworkSetup(ip string, namespace string) {
-	networkSetup(ip, namespace)
-	var ctx context.Context
-	ctx, cancel = context.WithCancel(context.TODO())
-	go func() {
-		ticker := time.NewTicker(time.Second * 10)
-		newWatcher, _ := fsnotify.NewWatcher()
-		defer newWatcher.Close()
-		defer ticker.Stop()
-		_ = newWatcher.Add(resolv)
-		c := make(chan struct{}, 1)
-		c <- struct{}{}
-		for {
-			select {
-			case <-ticker.C:
-				c <- struct{}{}
-			case /*e :=*/ <-newWatcher.Events:
-				//if e.Op == fsnotify.Write {
-				c <- struct{}{}
-				//}
-			case <-c:
-				if rc, err := miekgdns.ClientConfigFromFile(resolv); err == nil && rc.Timeout != 1 {
-					if !sets.NewString(rc.Servers...).Has(ip) {
-						rc.Servers = append(rc.Servers, ip)
-						for _, s := range []string{namespace + ".svc.cluster.local", "svc.cluster.local", "cluster.local"} {
-							rc.Search = append(rc.Search, s)
-						}
-						//rc.Ndots = 5
-					}
-					//rc.Attempts = 1
-					rc.Timeout = 1
-					_ = ioutil.WriteFile(resolv, []byte(toString(*rc)), 0644)
-				}
-			case <-ctx.Done():
-				return
-			}
-		}
-	}()
-}
-
-func toString(config miekgdns.ClientConfig) string {
-	var builder strings.Builder
-	//	builder.WriteString(`#
-	//# macOS Notice
-	//#
-	//# This file is not consulted for DNS hostname resolution, address
-	//# resolution, or the DNS query routing mechanism used by most
-	//# processes on this system.
-	//#
-	//# To view the DNS configuration used by this system, use:
-	//#   scutil --dns
-	//#
-	//# SEE ALSO
-	//#   dns-sd(1), scutil(8)
-	//#
-	//# This file is automatically generated.
-	//#`)
-	//	builder.WriteString("\n")
-	if len(config.Search) > 0 {
-		builder.WriteString(fmt.Sprintf("search %s\n", strings.Join(config.Search, " ")))
-	}
-	for i := range config.Servers {
-		builder.WriteString(fmt.Sprintf("nameserver %s\n", config.Servers[i]))
-	}
-	if len(config.Port) != 0 {
-		builder.WriteString(fmt.Sprintf("port %s\n", config.Port))
-	}
-	builder.WriteString(fmt.Sprintf("options ndots:%d\n", config.Ndots))
-	builder.WriteString(fmt.Sprintf("options timeout:%d\n", config.Timeout))
-	//builder.WriteString(fmt.Sprintf("options attempts:%d\n", config.Attempts))
-	return builder.String()
-}
-
-func CancelDNS() {
-	if cancel != nil {
-		cancel()
-	}
-	_ = os.RemoveAll(filepath.Join("/", "etc", "resolver"))
-	//networkCancel()
-}
-
-/*
-➜  resolver sudo networksetup -setdnsservers Wi-Fi 172.20.135.131 1.1.1.1
-➜  resolver sudo networksetup -setsearchdomains Wi-Fi test.svc.cluster.local svc.cluster.local cluster.local
-➜  resolver sudo networksetup -getsearchdomains Wi-Fi
-test.svc.cluster.local
-svc.cluster.local
-cluster.local
-➜  resolver sudo networksetup -getdnsservers Wi-Fi
-172.20.135.131
-1.1.1.1
-*/
-func networkSetup(ip string, namespace string) {
-	networkCancel()
-	b, err := exec.Command("networksetup", "-listallnetworkservices").Output()
-	if err != nil {
-		return
-	}
-	services := strings.Split(string(b), "\n")
-	for _, s := range services[:len(services)-1] {
-		cmd := exec.Command("networksetup", "-getdnsservers", s)
-		output, err := cmd.Output()
-		if err == nil {
-			var nameservers []string
-			if strings.Contains(string(output), "There aren't any DNS Servers") {
-				nameservers = make([]string, 0, 0)
-				// fix networksetup -getdnsservers is empty, but resolv.conf nameserver is not empty
-				if rc, err := miekgdns.ClientConfigFromFile(resolv); err == nil {
-					nameservers = rc.Servers
-				}
-			} else {
-				nameservers = strings.Split(string(output), "\n")
-				nameservers = nameservers[:len(nameservers)-1]
-			}
-			// add to tail
-			nameservers = append(nameservers, ip)
-			args := []string{"-setdnsservers", s}
-			output, err = exec.Command("networksetup", append(args, nameservers...)...).Output()
-			if err != nil {
-				log.Warnf("error while set dnsserver for %s, err: %v, output: %s\n", s, err, string(output))
-			}
-		}
-		output, err = exec.Command("networksetup", "-getsearchdomains", s).Output()
-		if err == nil {
-			var searchDomains []string
-			if strings.Contains(string(output), "There aren't any Search Domains") {
-				searchDomains = make([]string, 0, 0)
-			} else {
-				searchDomains = strings.Split(string(output), "\n")
-				searchDomains = searchDomains[:len(searchDomains)-1]
-			}
-			newSearchDomains := make([]string, len(searchDomains)+3, len(searchDomains)+3)
-			copy(newSearchDomains[3:], searchDomains)
-			newSearchDomains[0] = fmt.Sprintf("%s.svc.cluster.local", namespace)
-			newSearchDomains[1] = "svc.cluster.local"
-			newSearchDomains[2] = "cluster.local"
-			args := []string{"-setsearchdomains", s}
-			bytes, err := exec.Command("networksetup", append(args, newSearchDomains...)...).Output()
-			if err != nil {
-				log.Warnf("error while set search domain for %s, err: %v, output: %s\n", s, err, string(bytes))
-			}
-		}
-	}
-}
-
-func networkCancel() {
-	b, err := exec.Command("networksetup", "-listallnetworkservices").CombinedOutput()
-	if err != nil {
-		return
-	}
-	services := strings.Split(string(b), "\n")
-	for _, s := range services[:len(services)-1] {
-		output, err := exec.Command("networksetup", "-getsearchdomains", s).Output()
-		if err == nil {
-			i := strings.Split(string(output), "\n")
-			if i[1] == "svc.cluster.local" && i[2] == "cluster.local" {
-				bytes, err := exec.Command("networksetup", "-setsearchdomains", s, strings.Join(i[3:], " ")).Output()
-				if err != nil {
-					log.Warnf("error while remove search domain for %s, err: %v, output: %s\n", s, err, string(bytes))
-				}
-
-				output, err := exec.Command("networksetup", "-getdnsservers", s).Output()
-				if err == nil {
-					dnsServers := strings.Split(string(output), "\n")
-					// dnsServers[len(dnsServers)-1]=""
-					// dnsServers[len(dnsServers)-2]="ip which added by KubeVPN"
-					dnsServers = dnsServers[:len(dnsServers)-2]
-					if len(dnsServers) == 0 {
-						// set default dns server to 1.1.1.1 or just keep on empty
-						dnsServers = append(dnsServers, "empty")
-					}
-					args := []string{"-setdnsservers", s}
-					combinedOutput, err := exec.Command("networksetup", append(args, dnsServers...)...).Output()
-					if err != nil {
-						log.Warnf("error while remove dnsserver for %s, err: %v, output: %s\n", s, err, string(combinedOutput))
-					}
-				}
-			}
-		}
-	}
-}

dns/dns_windows.go

@@ -1,77 +0,0 @@
-//go:build windows
-// +build windows
-
-package dns
-
-import (
-	"context"
-	"fmt"
-	miekgdns "github.com/miekg/dns"
-	log "github.com/sirupsen/logrus"
-	"golang.org/x/sys/windows"
-	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
-	"net"
-	"os"
-	"os/exec"
-	"strconv"
-)
-
-func SetupDNS(config *miekgdns.ClientConfig) error {
-	getenv := os.Getenv("luid")
-	parseUint, err := strconv.ParseUint(getenv, 10, 64)
-	if err != nil {
-		log.Warningln(err)
-		return err
-	}
-	luid := winipcfg.LUID(parseUint)
-	err = luid.SetDNS(windows.AF_INET, []net.IP{net.ParseIP(config.Servers[0])}, config.Search)
-	_ = exec.CommandContext(context.Background(), "ipconfig", "/flushdns").Run()
-	if err != nil {
-		log.Warningln(err)
-		return err
-	}
-	//_ = updateNicMetric(tunName)
-	_ = addNicSuffixSearchList(config.Search)
-	return nil
-}
-
-func CancelDNS() {
-	getenv := os.Getenv("luid")
-	parseUint, err := strconv.ParseUint(getenv, 10, 64)
-	if err != nil {
-		log.Warningln(err)
-		return
-	}
-	luid := winipcfg.LUID(parseUint)
-	_ = luid.FlushDNS(windows.AF_INET)
-}
-
-func updateNicMetric(name string) error {
-	cmd := exec.Command("PowerShell", []string{
-		"Set-NetIPInterface",
-		"-InterfaceAlias",
-		fmt.Sprintf("\"%s\"", name),
-		"-InterfaceMetric",
-		"1",
-	}...)
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		log.Warnf("error while update nic metrics, error: %v, output: %s, command: %v", err, string(out), cmd.Args)
-	}
-	return err
-}
-
-// @see https://docs.microsoft.com/en-us/powershell/module/dnsclient/set-dnsclientglobalsetting?view=windowsserver2019-ps#example-1--set-the-dns-suffix-search-list
-func addNicSuffixSearchList(search []string) error {
-	cmd := exec.Command("PowerShell", []string{
-		"Set-DnsClientGlobalSetting",
-		"-SuffixSearchList",
-		fmt.Sprintf("@(\"%s\", \"%s\", \"%s\")", search[0], search[1], search[2]),
-	}...)
-	output, err := cmd.CombinedOutput()
-	log.Info(cmd.Args)
-	if err != nil {
-		log.Warnf("error while set dns suffix search list, err: %v, output: %s, command: %v", err, string(output), cmd.Args)
-	}
-	return err
-}

dns/dnsserver.go

@@ -1,69 +0,0 @@
-package dns
-
-import (
-	miekgdns "github.com/miekg/dns"
-	log "github.com/sirupsen/logrus"
-	"k8s.io/apimachinery/pkg/util/cache"
-	"strings"
-	"time"
-)
-
-type server struct {
-	// todo using cache to speed up dns resolve process
-	dnsCache   *cache.LRUExpireCache
-	forwardDNS *miekgdns.ClientConfig
-}
-
-func NewDNSServer(network, address string, forwardDNS *miekgdns.ClientConfig) error {
-	return miekgdns.ListenAndServe(address, network, &server{
-		dnsCache: cache.NewLRUExpireCache(1000), forwardDNS: forwardDNS,
-	})
-}
-
-// ServeDNS consider using a cache
-/*
-
-nameserver 172.20.135.131
-search nocalhost.svc.cluster.local svc.cluster.local cluster.local
-options ndots:5
-
-*/
-func (s *server) ServeDNS(w miekgdns.ResponseWriter, r *miekgdns.Msg) {
-	q := r.Question
-	r.Question = make([]miekgdns.Question, 0, len(q))
-	question := q[0]
-	name := question.Name
-	switch strings.Count(question.Name, ".") {
-	case 1:
-		question.Name = question.Name + s.forwardDNS.Search[0] + "."
-	case 2:
-		question.Name = question.Name + s.forwardDNS.Search[1] + "."
-	case 3:
-		question.Name = question.Name + s.forwardDNS.Search[2] + "."
-	case 4:
-		question.Name = question.Name + strings.Split(s.forwardDNS.Search[2], ".")[1] + "."
-	case 5:
-	}
-	r.Question = []miekgdns.Question{question}
-	client := miekgdns.Client{Net: "udp", Timeout: time.Second * 5}
-	answer, _, err := client.Exchange(r, s.forwardDNS.Servers[0]+":53")
-	//answer, err := miekgdns.Exchange(r, s.forwardDNS.Servers[0]+":53")
-	if err != nil {
-		log.Warnln(err)
-		err = w.WriteMsg(r)
-		if err != nil {
-			log.Warnln(err)
-		}
-	} else {
-		if len(answer.Answer) != 0 {
-			answer.Answer[0].Header().Name = name
-		}
-		if len(answer.Question) != 0 {
-			answer.Question[0].Name = name
-		}
-		err = w.WriteMsg(answer)
-		if err != nil {
-			log.Warnln(err)
-		}
-	}
-}

dns/dnsserver_test.go

@@ -1,27 +0,0 @@
-package dns
-
-import (
-	"fmt"
-	miekgdns "github.com/miekg/dns"
-	"github.com/wencaiwulue/kubevpn/util"
-	"strconv"
-	"testing"
-	"time"
-)
-
-func TestSetupDnsServer(t *testing.T) {
-	port := util.GetAvailableUDPPortOrDie()
-	fmt.Println(port)
-	go func() {
-		err := NewDNSServer("udp", "127.0.0.1:"+strconv.Itoa(port), &miekgdns.ClientConfig{
-			Servers: []string{""},
-			Search:  []string{"test.svc.cluster.local", "svc.cluster.local", "cluster.local"},
-			Port:    "53",
-			Ndots:   0,
-		})
-		if err != nil {
-			t.FailNow()
-		}
-	}()
-	time.Sleep(time.Second * 5)
-}

dockerfile/controlplane/Dockerfile

@@ -1,2 +0,0 @@
-FROM scratch
-COPY envoy-xds-server /bin/envoy-xds-server

dockerfile/mesh/Dockerfile

@@ -1,6 +0,0 @@
-FROM envoyproxy/envoy-dev:5f7d6efb5786ee3de31b1fb37c78fa281718b704
-
-WORKDIR /app
-RUN sed -i s@/security.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list \
-    && sed -i s@/archive.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list
-RUN apt-get clean && apt-get update && apt-get install -y wget dnsutils vim curl net-tools iptables iputils-ping lsof iproute2 tcpdump

dockerfile/server/Dockerfile

@@ -1,6 +0,0 @@
-FROM ubuntu:latest
-WORKDIR /app
-RUN sed -i s@/security.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list \
-    && sed -i s@/archive.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list
-RUN apt-get clean && apt-get update && apt-get install -y wget dnsutils vim curl net-tools iptables iputils-ping lsof iproute2 tcpdump
-COPY ./kubevpn /usr/local/bin/kubevpn

docs/en/Architecture.md

@@ -0,0 +1,45 @@
+## Architecture
+
+### Connect mode
+
+create a tunnel with port-forward, add route to virtual interface, like tun0, forward traffic though tunnel to remote
+traffic manager.  
+![connect-mode](/docs/en/images/connect-mode.drawio.svg)
+
+### Reverse mode
+
+base on connect mode, inject a container to controller, use iptables to block all inbound traffic and forward to local
+though tunnel.
+
+```text
+┌──────────┐    ┌─────────┌──────────┐    ┌──────────┐
+│ ServiceA ├───►│ sidecar │ ServiceB │ ┌─►│ ServiceC │
+└──────────┘    └────┌────┘──────────┘ │  └──────────┘
+                     │                 │
+                     │                 │                     cloud
+ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ─ ─ ─ ─┘ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
+                     │                 │                     local
+                 ┌───┘──────┐          │
+                 │ ServiceB'├──────────┘
+                 └──────────┘
+```
+
+### Mesh mode
+
+base on reverse mode, using envoy as proxy, if headers have special key-value pair, it will route to local machine, if
+not, use origin service.
+
+```text
+┌──────────┐    ┌─────────┌────────────┐     ┌──────────┐
+│ ServiceA ├───►│ sidecar ├─► ServiceB │─►┌─►│ ServiceC │
+└──────────┘    └────┌────┘────────────┘  │  └──────────┘
+                     │                    │                     cloud
+─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─┘─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
+                     │                    │                     local
+                header: foo=bar           │
+                 ┌───┘──────┐             │
+                 │ ServiceB'├─────────────┘
+                 └──────────┘
+```
+
+![arch.svg](/docs/en/images/proxy-arch.svg)

driver/driver.go

@@ -1,62 +0,0 @@
-package driver
-
-import (
-	log "github.com/sirupsen/logrus"
-	"github.com/wencaiwulue/kubevpn/driver/openvpn"
-	"github.com/wencaiwulue/kubevpn/driver/wintun"
-	"k8s.io/client-go/util/retry"
-	"os"
-	"os/exec"
-	"path/filepath"
-)
-
-func InstallTunTapDriver() {
-	if err := retry.OnError(retry.DefaultRetry, func(err error) bool {
-		return err != nil
-	}, func() error {
-		return openvpn.Install()
-	}); err != nil {
-		log.Warn(err)
-	}
-}
-
-func InstallWireGuardTunDriver() {
-	if err := retry.OnError(retry.DefaultRetry, func(err error) bool {
-		return err != nil
-	}, func() error {
-		return wintun.InstallWintunDriver()
-	}); err != nil {
-		log.Warn(err)
-	}
-}
-
-func UninstallWireGuardTunDriver() error {
-	wd, err := os.Getwd()
-	if err != nil {
-		return err
-	}
-	filename := filepath.Join(wd, "wintun.dll")
-	return os.Remove(filename)
-}
-
-func UninstallTunTapDriver() {
-	filepath.VolumeName("C")
-	path := filepath.Join(getDiskName()+":\\", "Program Files", "TAP-Windows", "Uninstall.exe")
-	cmd := exec.Command(path, "/S")
-	b, e := cmd.CombinedOutput()
-	if e != nil {
-		log.Warn(e)
-	}
-	log.Info(string(b))
-}
-
-func getDiskName() string {
-	for _, drive := range "ABCDEFGHIJKLMNOPQRSTUVWXYZ" {
-		f, err := os.Open(string(drive) + ":\\")
-		if err == nil {
-			_ = f.Close()
-			return string(drive)
-		}
-	}
-	return ""
-}

driver/openvpn/other.go

@@ -1,12 +0,0 @@
-//go:build !windows
-// +build !windows
-
-package openvpn
-
-import (
-	"errors"
-)
-
-func Install() error {
-	return errors.New("not need to implement")
-}

driver/openvpn/windows.go

@@ -1,35 +0,0 @@
-//go:build windows
-// +build windows
-
-package openvpn
-
-import (
-	"embed"
-	"io/ioutil"
-	"os"
-	"os/exec"
-)
-
-//go:embed exe/tap-windows-9.21.2.exe
-var fs embed.FS
-
-//	driver download from https://build.openvpn.net/downloads/releases/
-func Install() error {
-	bytes, err := fs.ReadFile("exe/tap-windows-9.21.2.exe")
-	if err != nil {
-		return err
-	}
-	tempFile, err := ioutil.TempFile("", "*.exe")
-	defer func() { _ = os.Remove(tempFile.Name()) }()
-	if err != nil {
-		return err
-	}
-	if _, err = tempFile.Write(bytes); err != nil {
-		return err
-	}
-	_ = tempFile.Sync()
-	_ = tempFile.Close()
-	_ = os.Chmod(tempFile.Name(), 0700)
-	cmd := exec.Command(tempFile.Name(), "/S")
-	return cmd.Run()
-}

driver/wintun/amd64.go

@@ -1,29 +0,0 @@
-//go:build windows && amd64
-// +build windows,amd64
-
-package wintun
-
-import (
-	"embed"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-)
-
-//go:embed bin/amd64/wintun.dll
-var wintunFs embed.FS
-
-func InstallWintunDriver() error {
-	bytes, err := wintunFs.ReadFile("bin/amd64/wintun.dll")
-	if err != nil {
-		return err
-	}
-	executable, err := os.Executable()
-	if err != nil {
-		return err
-	}
-	filename := filepath.Join(filepath.Dir(executable), "wintun.dll")
-	_ = os.Remove(filename)
-	err = ioutil.WriteFile(filename, bytes, 644)
-	return err
-}

driver/wintun/arm.go

@@ -1,29 +0,0 @@
-//go:build windows && arm
-// +build windows,arm
-
-package wintun
-
-import (
-	"embed"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-)
-
-//go:embed bin/arm/wintun.dll
-var wintunFs embed.FS
-
-func InstallWintunDriver() error {
-	bytes, err := wintunFs.ReadFile("bin/arm/wintun.dll")
-	if err != nil {
-		return err
-	}
-	executable, err := os.Executable()
-	if err != nil {
-		return err
-	}
-	filename := filepath.Join(filepath.Dir(executable), "wintun.dll")
-	_ = os.Remove(filename)
-	err = ioutil.WriteFile(filename, bytes, 644)
-	return err
-}

driver/wintun/arm64.go

@@ -1,29 +0,0 @@
-//go:build windows && arm64
-// +build windows,arm64
-
-package wintun
-
-import (
-	"embed"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-)
-
-//go:embed bin/arm64/wintun.dll
-var wintunFs embed.FS
-
-func InstallWintunDriver() error {
-	bytes, err := wintunFs.ReadFile("bin/arm64/wintun.dll")
-	if err != nil {
-		return err
-	}
-	executable, err := os.Executable()
-	if err != nil {
-		return err
-	}
-	filename := filepath.Join(filepath.Dir(executable), "wintun.dll")
-	_ = os.Remove(filename)
-	err = ioutil.WriteFile(filename, bytes, 644)
-	return err
-}

driver/wintun/others.go

@@ -1,10 +0,0 @@
-//go:build !windows
-// +build !windows
-
-package wintun
-
-import "github.com/pkg/errors"
-
-func InstallWintunDriver() error {
-	return errors.New("not implement")
-}

driver/wintun/x86.go

@@ -1,30 +0,0 @@
-//go:build windows && (x86 || 386)
-// +build windows
-// +build x86 386
-
-package wintun
-
-import (
-	"embed"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-)
-
-//go:embed bin/x86/wintun.dll
-var wintunFs embed.FS
-
-func InstallWintunDriver() error {
-	bytes, err := wintunFs.ReadFile("bin/x86/wintun.dll")
-	if err != nil {
-		return err
-	}
-	executable, err := os.Executable()
-	if err != nil {
-		return err
-	}
-	filename := filepath.Join(filepath.Dir(executable), "wintun.dll")
-	_ = os.Remove(filename)
-	err = ioutil.WriteFile(filename, bytes, 644)
-	return err
-}