feat: add sub-command reset

2025-12-24 11:51:13 +08:00 · 2023-01-05 14:06:09 +08:00
parent 1b85450ef4
commit 45b4c9c98d
7 changed files with 125 additions and 41 deletions
--- a/cmd/kubevpn/cmds/reset.go
+++ b/cmd/kubevpn/cmds/reset.go
@@ -0,0 +1,28 @@
+package cmds
+
+import (
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	resetCmd.Flags().StringVar(&connect.KubeconfigPath, "kubeconfig", "", "kubeconfig")
+	resetCmd.Flags().StringVarP(&connect.Namespace, "namespace", "n", "", "namespace")
+	RootCmd.AddCommand(resetCmd)
+}
+
+var resetCmd = &cobra.Command{
+	Use:   "reset",
+	Short: "Reset KubeVPN",
+	Long:  `Reset KubeVPN if any error occurs`,
+	Run: func(cmd *cobra.Command, args []string) {
+		if err := connect.InitClient(); err != nil {
+			log.Fatal(err)
+		}
+		err := connect.Reset(cmd.Context())
+		if err != nil {
+			log.Fatal(err)
+		}
+		log.Infoln("done")
+	},
+}
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -13,6 +13,8 @@ const (
 	KeyDHCP             = "DHCP"
 	KeyEnvoy            = "ENVOY_CONFIG"
 	KeyClusterIPv4POOLS = "IPv4_POOLS"
+	// config map annotation
+	AnnoRefCount = "ref-count"

 	// container name
 	ContainerSidecarEnvoyProxy   = "envoy-proxy"
@@ -21,7 +23,7 @@ const (

 	VolumeEnvoyConfig = "envoy-config"

-	innerIPv4Pool = "223.254.254.100/24"
+	innerIPv4Pool = "223.254.0.100/16"

 	DefaultNetDir = "/etc/cni/net.d"

--- a/pkg/handler/cleaner.go
+++ b/pkg/handler/cleaner.go
@@ -43,7 +43,7 @@ func (c *ConnectOptions) addCleanUpResourceHandler(clientset *kubernetes.Clients
 			}
 		}
 		_ = clientset.CoreV1().Pods(namespace).Delete(context.Background(), config.CniNetName, v1.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)})
-		cleanUpTrafficManagerIfRefCountIsZero(clientset, namespace)
+		cleanupIfRefCountIsZero(clientset, namespace, config.ConfigMapPodTrafficManager)
 		log.Info("clean up successful")
 		os.Exit(0)
 	}()
@@ -57,28 +57,28 @@ func Cleanup(s os.Signal) {
 }

 // vendor/k8s.io/kubectl/pkg/polymorphichelpers/rollback.go:99
-func updateServiceRefCount(serviceInterface v12.ServiceInterface, name string, increment int) {
+func updateRefCount(configMapInterface v12.ConfigMapInterface, name string, increment int) {
 	err := retry.OnError(
 		retry.DefaultRetry,
 		func(err error) bool { return !k8serrors.IsNotFound(err) },
 		func() error {
-			service, err := serviceInterface.Get(context.Background(), name, v1.GetOptions{})
+			cm, err := configMapInterface.Get(context.Background(), name, v1.GetOptions{})
 			if err != nil {
 				log.Errorf("update ref-count failed, increment: %d, error: %v", increment, err)
 				return err
 			}
 			curCount := 0
-			if ref := service.GetAnnotations()["ref-count"]; len(ref) > 0 {
+			if ref := cm.GetAnnotations()[config.AnnoRefCount]; len(ref) > 0 {
 				curCount, err = strconv.Atoi(ref)
 			}
 			p, _ := json.Marshal([]interface{}{
 				map[string]interface{}{
 					"op":    "replace",
-					"path":  "/metadata/annotations/ref-count",
+					"path":  fmt.Sprintf("/metadata/annotations/%s", config.AnnoRefCount),
 					"value": strconv.Itoa(curCount + increment),
 				},
 			})
-			_, err = serviceInterface.Patch(context.Background(), config.ConfigMapPodTrafficManager, types.JSONPatchType, p, v1.PatchOptions{})
+			_, err = configMapInterface.Patch(context.Background(), name, types.JSONPatchType, p, v1.PatchOptions{})
 			return err
 		})
 	if err != nil {
@@ -88,30 +88,36 @@ func updateServiceRefCount(serviceInterface v12.ServiceInterface, name string, i
 	}
 }

-func cleanUpTrafficManagerIfRefCountIsZero(clientset *kubernetes.Clientset, namespace string) {
-	updateServiceRefCount(clientset.CoreV1().Services(namespace), config.ConfigMapPodTrafficManager, -1)
-	pod, err := clientset.CoreV1().Services(namespace).Get(context.Background(), config.ConfigMapPodTrafficManager, v1.GetOptions{})
+func cleanupIfRefCountIsZero(clientset *kubernetes.Clientset, namespace, name string) {
+	updateRefCount(clientset.CoreV1().ConfigMaps(namespace), name, -1)
+	cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(context.Background(), name, v1.GetOptions{})
 	if err != nil {
 		log.Error(err)
 		return
 	}
-	refCount, err := strconv.Atoi(pod.GetAnnotations()["ref-count"])
+	refCount, err := strconv.Atoi(cm.GetAnnotations()[config.AnnoRefCount])
 	if err != nil {
 		log.Error(err)
 		return
 	}
-	// if refcount is less than zero or equals to zero, means nobody is using this traffic pod, so clean it
+	// if ref-count is less than zero or equals to zero, means nobody is using this traffic pod, so clean it
 	if refCount <= 0 {
 		log.Info("refCount is zero, prepare to clean up resource")
 		// keep configmap
-		p := []byte(fmt.Sprintf(`[{"op": "remove", "path": "/data/%s"}]`, config.KeyDHCP))
-		_, err = clientset.CoreV1().ConfigMaps(namespace).Patch(context.Background(), config.ConfigMapPodTrafficManager, types.JSONPatchType, p, v1.PatchOptions{})
-		deleteOptions := v1.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)}
-		_ = clientset.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.Background(), config.ConfigMapPodTrafficManager+"."+namespace, deleteOptions)
-		_ = clientset.RbacV1().RoleBindings(namespace).Delete(context.Background(), config.ConfigMapPodTrafficManager, deleteOptions)
-		_ = clientset.CoreV1().ServiceAccounts(namespace).Delete(context.Background(), config.ConfigMapPodTrafficManager, deleteOptions)
-		_ = clientset.RbacV1().Roles(namespace).Delete(context.Background(), config.ConfigMapPodTrafficManager, deleteOptions)
-		_ = clientset.CoreV1().Services(namespace).Delete(context.Background(), config.ConfigMapPodTrafficManager, deleteOptions)
-		_ = clientset.AppsV1().Deployments(namespace).Delete(context.Background(), config.ConfigMapPodTrafficManager, deleteOptions)
+		p := []byte(
+			fmt.Sprintf(`[{"op": "remove", "path": "/data/%s"}]`, config.KeyDHCP),
+		)
+		_, err = clientset.CoreV1().ConfigMaps(namespace).Patch(context.Background(), name, types.JSONPatchType, p, v1.PatchOptions{})
+		p = []byte(
+			fmt.Sprintf(`[{"op": "replace", "path": "/metadata/annotations/%s", "value": "0"}]`, config.AnnoRefCount),
+		)
+		_, err = clientset.CoreV1().ConfigMaps(namespace).Patch(context.Background(), name, types.JSONPatchType, p, v1.PatchOptions{})
+		options := v1.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)}
+		_ = clientset.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.Background(), name+"."+namespace, options)
+		_ = clientset.RbacV1().RoleBindings(namespace).Delete(context.Background(), name, options)
+		_ = clientset.CoreV1().ServiceAccounts(namespace).Delete(context.Background(), name, options)
+		_ = clientset.RbacV1().Roles(namespace).Delete(context.Background(), name, options)
+		_ = clientset.CoreV1().Services(namespace).Delete(context.Background(), name, options)
+		_ = clientset.AppsV1().Deployments(namespace).Delete(context.Background(), name, options)
 	}
 }
--- a/pkg/handler/dhcp.go
+++ b/pkg/handler/dhcp.go
@@ -31,14 +31,14 @@ func NewDHCPManager(client corev1.ConfigMapInterface, namespace string, cidr *ne
 	}
 }

-//	todo optimize dhcp, using mac address, ip and deadline as unit
+// todo optimize dhcp, using mac address, ip and deadline as unit
 func (d *DHCPManager) InitDHCP() error {
-	configMap, err := d.client.Get(context.Background(), config.ConfigMapPodTrafficManager, metav1.GetOptions{})
+	cm, err := d.client.Get(context.Background(), config.ConfigMapPodTrafficManager, metav1.GetOptions{})
 	if err == nil {
-		if _, found := configMap.Data[config.KeyEnvoy]; !found {
+		if _, found := cm.Data[config.KeyEnvoy]; !found {
 			_, err = d.client.Patch(
 				context.Background(),
-				configMap.Name,
+				cm.Name,
 				types.MergePatchType,
 				[]byte(fmt.Sprintf(`{"data":{"%s":"%s"}}`, config.KeyEnvoy, "")),
 				metav1.PatchOptions{},
@@ -47,15 +47,16 @@ func (d *DHCPManager) InitDHCP() error {
 		}
 		return nil
 	}
-	result := &v1.ConfigMap{
+	cm = &v1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      config.ConfigMapPodTrafficManager,
-			Namespace: d.namespace,
-			Labels:    map[string]string{},
+			Name:        config.ConfigMapPodTrafficManager,
+			Namespace:   d.namespace,
+			Labels:      map[string]string{},
+			Annotations: map[string]string{config.AnnoRefCount: "0"},
 		},
 		Data: map[string]string{config.KeyEnvoy: ""},
 	}
-	_, err = d.client.Create(context.Background(), result, metav1.CreateOptions{})
+	_, err = d.client.Create(context.Background(), cm, metav1.CreateOptions{})
 	if err != nil {
 		log.Errorf("create dhcp error, err: %v", err)
 		return err
--- a/pkg/handler/envoy.go
+++ b/pkg/handler/envoy.go
@@ -28,7 +28,7 @@ import (

 // https://istio.io/latest/docs/ops/deployment/requirements/#ports-used-by-istio

-//	patch a sidecar, using iptables to do port-forward let this pod decide should go to 233.254.254.100 or request to 127.0.0.1
+// patch a sidecar, using iptables to do port-forward let this pod decide should go to 233.254.254.100 or request to 127.0.0.1
 func InjectVPNAndEnvoySidecar(factory cmdutil.Factory, clientset v12.ConfigMapInterface, namespace, workloads string, c util.PodRouteConfig, headers map[string]string) error {
 	object, err := util.GetUnstructuredObject(factory, namespace, workloads)
 	if err != nil {
--- a/pkg/handler/remote.go
+++ b/pkg/handler/remote.go
@@ -39,14 +39,12 @@ import (
 )

 func CreateOutboundPod(factory cmdutil.Factory, clientset *kubernetes.Clientset, namespace string, trafficManagerIP string, nodeCIDR []*net.IPNet) (ip net.IP, err error) {
-	podInterface := clientset.CoreV1().Pods(namespace)
-	serviceInterface := clientset.CoreV1().Services(namespace)
-	service, err := serviceInterface.Get(context.Background(), config.ConfigMapPodTrafficManager, metav1.GetOptions{})
+	service, err := clientset.CoreV1().Services(namespace).Get(context.Background(), config.ConfigMapPodTrafficManager, metav1.GetOptions{})
 	if err == nil {
 		_, err = polymorphichelpers.AttachablePodForObjectFn(factory, service, 2*time.Second)
 		if err == nil {
 			log.Infoln("traffic manager already exist, reuse it")
-			updateServiceRefCount(serviceInterface, service.GetName(), 1)
+			updateRefCount(clientset.CoreV1().ConfigMaps(namespace), config.ConfigMapPodTrafficManager, 1)
 			return net.ParseIP(service.Spec.ClusterIP), nil
 		}
 	}
@@ -55,7 +53,7 @@ func CreateOutboundPod(factory cmdutil.Factory, clientset *kubernetes.Clientset,
 		_ = clientset.RbacV1().RoleBindings(namespace).Delete(context.Background(), config.ConfigMapPodTrafficManager, metav1.DeleteOptions{})
 		_ = clientset.RbacV1().Roles(namespace).Delete(context.Background(), config.ConfigMapPodTrafficManager, metav1.DeleteOptions{})
 		_ = clientset.CoreV1().ServiceAccounts(namespace).Delete(context.Background(), config.ConfigMapPodTrafficManager, metav1.DeleteOptions{})
-		_ = serviceInterface.Delete(context.Background(), config.ConfigMapPodTrafficManager, metav1.DeleteOptions{})
+		_ = clientset.CoreV1().Services(namespace).Delete(context.Background(), config.ConfigMapPodTrafficManager, metav1.DeleteOptions{})
 		_ = clientset.AppsV1().Deployments(namespace).Delete(context.Background(), config.ConfigMapPodTrafficManager, metav1.DeleteOptions{})
 	}
 	defer func() {
@@ -135,11 +133,10 @@ func CreateOutboundPod(factory cmdutil.Factory, clientset *kubernetes.Clientset,
 	tcp10800 := "10800-for-tcp"
 	tcp9002 := "9002-for-envoy"
 	tcp80 := "80-for-webhook"
-	svc, err := serviceInterface.Create(context.Background(), &v1.Service{
+	svc, err := clientset.CoreV1().Services(namespace).Create(context.Background(), &v1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:        config.ConfigMapPodTrafficManager,
-			Namespace:   namespace,
-			Annotations: map[string]string{"ref-count": "1"},
+			Name:      config.ConfigMapPodTrafficManager,
+			Namespace: namespace,
 		},
 		Spec: v1.ServiceSpec{
 			Ports: []v1.ServicePort{{
@@ -318,7 +315,7 @@ kubevpn serve -L "tcp://:10800" -L "tun://:8422?net=${TrafficManagerIP}" --debug
 			},
 		},
 	}
-	watchStream, err := podInterface.Watch(context.Background(), metav1.ListOptions{
+	watchStream, err := clientset.CoreV1().Pods(namespace).Watch(context.Background(), metav1.ListOptions{
 		LabelSelector: fields.OneTermEqualSelector("app", config.ConfigMapPodTrafficManager).String(),
 	})
 	if err != nil {
--- a/pkg/handler/reset.go
+++ b/pkg/handler/reset.go
@@ -0,0 +1,50 @@
+package handler
+
+import (
+	"context"
+	"strconv"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/yaml"
+
+	"github.com/wencaiwulue/kubevpn/pkg/config"
+	"github.com/wencaiwulue/kubevpn/pkg/controlplane"
+)
+
+// Reset
+// 1, get all proxy-resources from configmap
+// 2, cleanup all containers
+func (c *ConnectOptions) Reset(ctx2 context.Context) error {
+	cm, err := c.clientset.CoreV1().ConfigMaps(c.Namespace).Get(ctx2, config.ConfigMapPodTrafficManager, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	var v = make([]*controlplane.Virtual, 0)
+	if str, ok := cm.Data[config.KeyEnvoy]; ok {
+		if err = yaml.Unmarshal([]byte(str), &v); err != nil {
+			log.Error(err)
+			return err
+		}
+		for _, virtual := range v {
+			// deployments.apps.ry-server --> deployments.apps/ry-server
+			lastIndex := strings.LastIndex(virtual.Uid, ".")
+			uid := virtual.Uid[:lastIndex] + "/" + virtual.Uid[lastIndex+1:]
+			for _, rule := range virtual.Rules {
+				err = UnPatchContainer(c.factory, c.clientset.CoreV1().ConfigMaps(c.Namespace), c.Namespace, uid, rule.Headers)
+				if err != nil {
+					log.Error(err)
+					continue
+				}
+			}
+		}
+	}
+	curCount := 0
+	if ref := cm.GetAnnotations()[config.AnnoRefCount]; len(ref) > 0 {
+		curCount, err = strconv.Atoi(ref)
+	}
+	updateRefCount(c.clientset.CoreV1().ConfigMaps(c.Namespace), config.ConfigMapPodTrafficManager, 0-curCount)
+	cleanupIfRefCountIsZero(c.clientset, c.Namespace, config.ConfigMapPodTrafficManager)
+	return nil
+}