feat: rent and release ip use api

cmd/kubevpn/cmds/serve.go

@@ -3,7 +3,6 @@ package cmds
 import (
 	"context"
 	"fmt"
-	"k8s.io/klog/v2"
 	"net"
 	"net/http"
 	"os"
@@ -31,25 +30,25 @@ func CmdServe(factory cmdutil.Factory) *cobra.Command {
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if v, ok := os.LookupEnv(config.EnvInboundPodTunIP); ok && v == "" {
-				clientset, err := factory.KubernetesClientSet()
+				namespace := os.Getenv(config.EnvPodNamespace)
-				if err != nil {
+				if namespace == "" {
-					klog.Error(err)
-					return err
-				}
-				namespace, found, _ := factory.ToRawKubeConfigLoader().Namespace()
-				if !found {
 					return fmt.Errorf("can not get namespace")
 				}
-				cmi := clientset.CoreV1().ConfigMaps(namespace)
+				url := fmt.Sprintf("%s.%s:80/rent/ip", config.ConfigMapPodTrafficManager, namespace)
-				dhcp := handler.NewDHCPManager(cmi, namespace, &net.IPNet{IP: config.RouterIP, Mask: config.CIDR.Mask})
+				request, err := http.NewRequest("GET", url, nil)
-				random, err := dhcp.RentIPRandom()
 				if err != nil {
-					klog.Error(err)
+					return fmt.Errorf("can not new request, err: %v", err)
+				}
+				request.Header.Set(config.HeaderPodName, os.Getenv(config.EnvPodName))
+				request.Header.Set(config.HeaderPodNamespace, namespace)
+				ip, err := util.DoReq(request)
+				if err != nil {
+					log.Error(err)
 					return err
 				}
-				err = os.Setenv(config.EnvInboundPodTunIP, random.String())
+				err = os.Setenv(config.EnvInboundPodTunIP, string(ip))
 				if err != nil {
-					klog.Error(err)
+					log.Error(err)
 					return err
 				}
 			}
@@ -72,21 +71,20 @@ func CmdServe(factory cmdutil.Factory) *cobra.Command {
 			if !ok || v == "" {
 				return nil
 			}
-			_, ipNet, err := net.ParseCIDR(v)
+			_, _, err := net.ParseCIDR(v)
 			if err != nil {
 				return err
 			}
-			clientset, err := factory.KubernetesClientSet()
+			namespace := os.Getenv(config.EnvPodNamespace)
+			url := fmt.Sprintf("%s.%s:80/release/ip", config.ConfigMapPodTrafficManager, namespace)
+			request, err := http.NewRequest("DELETE", url, nil)
 			if err != nil {
-				return err
+				return fmt.Errorf("can not new request, err: %v", err)
 			}
-			namespace, found, _ := factory.ToRawKubeConfigLoader().Namespace()
+			request.Header.Set(config.HeaderPodName, os.Getenv(config.EnvPodName))
-			if !found {
+			request.Header.Set(config.HeaderPodNamespace, namespace)
-				return fmt.Errorf("can not get namespace")
+			request.Header.Set(config.HeaderIP, v)
-			}
+			_, err = util.DoReq(request)
-			cmi := clientset.CoreV1().ConfigMaps(namespace)
-			dhcp := handler.NewDHCPManager(cmi, namespace, &net.IPNet{IP: config.RouterIP, Mask: config.CIDR.Mask})
-			err = dhcp.ReleaseIpToDHCP(ipNet)
 			return err
 		},
 	}

pkg/config/config.go

@@ -33,9 +33,16 @@ const (
 	// env name
 	EnvTunNameOrLUID   = "TunNameOrLUID"
 	EnvInboundPodTunIP = "InboundPodTunIP"
+	EnvPodName         = "POD_NAME"
+	EnvPodNamespace    = "POD_NAMESPACE"
 
 	// annotation
 	AnnoServiceAccountName = "service_account_name_backup_by_kubevpn"
+
+	// header name
+	HeaderPodName      = "POD_NAME"
+	HeaderPodNamespace = "POD_NAMESPACE"
+	HeaderIP           = "IP"
 )
 
 var (

pkg/exchange/controller.go

@@ -24,6 +24,13 @@ func AddContainer(spec *corev1.PodSpec, c util.PodRouteConfig) {
 	spec.Containers = append(spec.Containers, corev1.Container{
 		Name:  config.ContainerSidecarVPN,
 		Image: config.Image,
+		EnvFrom: []corev1.EnvFromSource{{
+			SecretRef: &corev1.SecretEnvSource{
+				LocalObjectReference: corev1.LocalObjectReference{
+					Name: config.ConfigMapPodTrafficManager,
+				},
+			},
+		}},
 		Env: []corev1.EnvVar{
 			{
 				Name:  "LocalTunIP",

pkg/handler/cleaner.go

@@ -131,6 +131,7 @@ func cleanup(clientset *kubernetes.Clientset, namespace, name string) {
 	p = []byte(fmt.Sprintf(`{"data":{"%s":"%s"}}`, config.KeyRefCount, strconv.Itoa(0)))
 	_, _ = clientset.CoreV1().ConfigMaps(namespace).Patch(context.Background(), name, types.MergePatchType, p, v1.PatchOptions{})
 	options := v1.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)}
+	_ = clientset.CoreV1().Secrets(namespace).Delete(context.Background(), name, options)
 	_ = clientset.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.Background(), name+"."+namespace, options)
 	_ = clientset.RbacV1().RoleBindings(namespace).Delete(context.Background(), name, options)
 	_ = clientset.CoreV1().ServiceAccounts(namespace).Delete(context.Background(), name, options)

pkg/handler/remote.go

@@ -3,7 +3,6 @@ package handler
 import (
 	"bytes"
 	"context"
-	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -187,6 +186,22 @@ func CreateOutboundPod(ctx context.Context, factory cmdutil.Factory, clientset *
 	h := config.ConfigMapPodTrafficManager + "." + namespace + "." + "svc"
 	certificate, key, _ := cert.GenerateSelfSignedCertKey(h, nil, nil)
 
+	_, err = clientset.CoreV1().Secrets(namespace).Create(ctx, &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      config.ConfigMapPodTrafficManager,
+			Namespace: namespace,
+		},
+		Data: map[string][]byte{
+			v1.TLSCertKey:       certificate,
+			v1.TLSPrivateKeyKey: key,
+		},
+		Type: v1.SecretTypeTLS,
+	}, metav1.CreateOptions{})
+
+	if err != nil && !k8serrors.IsAlreadyExists(err) {
+		return nil, err
+	}
+
 	deployment := &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      config.ConfigMapPodTrafficManager,
@@ -234,6 +249,13 @@ iptables -P FORWARD ACCEPT
 iptables -t nat -A POSTROUTING -s ${CIDR} -o eth0 -j MASQUERADE
 kubevpn serve -L "tcp://:10800" -L "tun://:8422?net=${TrafficManagerIP}" --debug=true`,
 							},
+							EnvFrom: []v1.EnvFromSource{{
+								SecretRef: &v1.SecretEnvSource{
+									LocalObjectReference: v1.LocalObjectReference{
+										Name: config.ConfigMapPodTrafficManager,
+									},
+								},
+							}},
 							Env: []v1.EnvVar{
 								{
 									Name:  "CIDR",
@@ -296,15 +318,13 @@ kubevpn serve -L "tcp://:10800" -L "tun://:8422?net=${TrafficManagerIP}" --debug
 								ContainerPort: 80,
 								Protocol:      v1.ProtocolTCP,
 							}},
-							Env: []v1.EnvVar{
+							EnvFrom: []v1.EnvFromSource{{
-								{
+								SecretRef: &v1.SecretEnvSource{
-									Name:  "CERT",
+									LocalObjectReference: v1.LocalObjectReference{
-									Value: base64.StdEncoding.EncodeToString(certificate),
+										Name: config.ConfigMapPodTrafficManager,
-								}, {
-									Name:  "KEY",
-									Value: base64.StdEncoding.EncodeToString(key),
 									},
 								},
+							}},
 							ImagePullPolicy: v1.PullIfNotPresent,
 							Resources:       Resources,
 						},

pkg/mesh/controller.go

@@ -38,6 +38,13 @@ iptables -t nat -A PREROUTING ! -p icmp ! -s 127.0.0.1 ! -d ${CIDR} -j DNAT --to
 iptables -t nat -A POSTROUTING ! -p icmp ! -s 127.0.0.1 ! -d ${CIDR} -j MASQUERADE
 kubevpn serve -L "tun:/${TrafficManagerRealIP}:8422?net=${InboundPodTunIP}&route=${CIDR}" --debug=true`,
 		},
+		EnvFrom: []v1.EnvFromSource{{
+			SecretRef: &v1.SecretEnvSource{
+				LocalObjectReference: v1.LocalObjectReference{
+					Name: config.ConfigMapPodTrafficManager,
+				},
+			},
+		}},
 		Env: []v1.EnvVar{
 			{
 				Name:  "CIDR",
@@ -52,23 +59,21 @@ kubevpn serve -L "tun:/${TrafficManagerRealIP}:8422?net=${InboundPodTunIP}&route
 				Value: c.InboundPodTunIP,
 			},
 			{
-				Name: "POD_NAMESPACE",
+				Name: config.EnvPodNamespace,
 				ValueFrom: &v1.EnvVarSource{
 					FieldRef: &v1.ObjectFieldSelector{
 						FieldPath: "metadata.namespace",
 					},
 				},
 			},
-		},
+			{
-		SecurityContext: &v1.SecurityContext{
+				Name: config.EnvPodName,
-			Capabilities: &v1.Capabilities{
+				ValueFrom: &v1.EnvVarSource{
-				Add: []v1.Capability{
+					FieldRef: &v1.ObjectFieldSelector{
-					"NET_ADMIN",
+						FieldPath: "metadata.name",
-					//"SYS_MODULE",
+					},
 				},
 			},
-			RunAsUser:  pointer.Int64(0),
-			Privileged: pointer.Bool(true),
 		},
 		Resources: v1.ResourceRequirements{
 			Requests: map[v1.ResourceName]resource.Quantity{
@@ -81,6 +86,16 @@ kubevpn serve -L "tun:/${TrafficManagerRealIP}:8422?net=${InboundPodTunIP}&route
 			},
 		},
 		ImagePullPolicy: v1.PullIfNotPresent,
+		SecurityContext: &v1.SecurityContext{
+			Capabilities: &v1.Capabilities{
+				Add: []v1.Capability{
+					"NET_ADMIN",
+					//"SYS_MODULE",
+				},
+			},
+			RunAsUser:  pointer.Int64(0),
+			Privileged: pointer.Bool(true),
+		},
 	})
 	spec.Spec.Containers = append(spec.Spec.Containers, v1.Container{
 		Name:    config.ContainerSidecarEnvoyProxy,

pkg/util/util.go

@@ -3,6 +3,8 @@ package util
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
@@ -547,3 +549,35 @@ func CanI(clientset *kubernetes.Clientset, sa, ns string, resource *rbacv1.Polic
 
 	return false, nil
 }
+
+func DoReq(request *http.Request) (body []byte, err error) {
+	cert, ok := os.LookupEnv(v1.TLSCertKey)
+	if !ok {
+		return nil, fmt.Errorf("can not get %s from env", v1.TLSCertKey)
+	}
+	caCertPool := x509.NewCertPool()
+	caCertPool.AppendCertsFromPEM([]byte(cert))
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				RootCAs: caCertPool,
+			},
+		},
+	}
+
+	var resp *http.Response
+	resp, err = client.Do(request)
+	if err != nil {
+		return nil, fmt.Errorf("err: %v", err)
+	}
+	defer resp.Body.Close()
+	body, err = io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("can not read body, err: %v", err)
+	}
+	if resp.StatusCode == http.StatusOK {
+		return body, nil
+	}
+	return body, fmt.Errorf("http status is %d", resp.StatusCode)
+}

pkg/webhook/dhcp.go

@@ -0,0 +1,74 @@
+package webhook
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+
+	log "github.com/sirupsen/logrus"
+	"k8s.io/kubectl/pkg/cmd/util"
+
+	"github.com/wencaiwulue/kubevpn/pkg/config"
+	"github.com/wencaiwulue/kubevpn/pkg/handler"
+)
+
+type dhcpServer struct {
+	f util.Factory
+}
+
+func (d *dhcpServer) rentIP(w http.ResponseWriter, r *http.Request) {
+	podName := r.Header.Get("POD_NAME")
+	namespace := r.Header.Get("POD_NAMESPACE")
+
+	log.Infof("handling rent ip request, pod name: %s, ns: %s", podName, namespace)
+	clientset, err := d.f.KubernetesClientSet()
+	if err != nil {
+		log.Error(err)
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+	cmi := clientset.CoreV1().ConfigMaps(namespace)
+	dhcp := handler.NewDHCPManager(cmi, namespace, &net.IPNet{IP: config.RouterIP, Mask: config.CIDR.Mask})
+	random, err := dhcp.RentIPRandom()
+	if err != nil {
+		log.Error(err)
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+	w.WriteHeader(http.StatusOK)
+	_, err = w.Write([]byte(random.String()))
+	if err != nil {
+		log.Error(err)
+	}
+}
+
+func (d *dhcpServer) releaseIP(w http.ResponseWriter, r *http.Request) {
+	podName := r.Header.Get("POD_NAME")
+	namespace := r.Header.Get("POD_NAMESPACE")
+	ip := r.Header.Get("IP")
+
+	_, ipNet, err := net.ParseCIDR(ip)
+	if err != nil {
+		log.Errorf("ip is invailed, ip: %s, err: %v", ip, err)
+		w.WriteHeader(http.StatusBadRequest)
+		w.Write([]byte(fmt.Sprintf("ip is invailed, ip: %s, err: %v", ip, err)))
+		return
+	}
+
+	log.Infof("handling rent ip request, pod name: %s, ns: %s", podName, namespace)
+	clientset, err := d.f.KubernetesClientSet()
+	if err != nil {
+		log.Error(err)
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+	cmi := clientset.CoreV1().ConfigMaps(namespace)
+	dhcp := handler.NewDHCPManager(cmi, namespace, &net.IPNet{IP: config.RouterIP, Mask: config.CIDR.Mask})
+	err = dhcp.ReleaseIpToDHCP(ipNet)
+	if err != nil {
+		log.Error(err)
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+	w.WriteHeader(http.StatusOK)
+}

pkg/webhook/mutateadmissionwebhook.go

@@ -2,7 +2,6 @@ package webhook
 
 import (
 	"crypto/tls"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -12,6 +11,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	v1 "k8s.io/api/admission/v1"
 	"k8s.io/api/admission/v1beta1"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	cmdutil "k8s.io/kubectl/pkg/cmd/util"
 )
@@ -122,21 +122,20 @@ func serve(w http.ResponseWriter, r *http.Request, admit admitHandler) {
 
 func Main(f cmdutil.Factory) error {
 	h := &admissionReviewHandler{f: f}
-	http.HandleFunc("/pods", func(w http.ResponseWriter, r *http.Request) {
+	http.HandleFunc("/pods", func(w http.ResponseWriter, r *http.Request) { serve(w, r, newDelegateToV1AdmitHandler(h.admitPods)) })
-		serve(w, r, newDelegateToV1AdmitHandler(h.admitPods))
+	http.HandleFunc("/readyz", func(w http.ResponseWriter, req *http.Request) { _, _ = w.Write([]byte("ok")) })
-	})
+	s := dhcpServer{f: f}
-	http.HandleFunc("/readyz", func(w http.ResponseWriter, req *http.Request) {
+	http.HandleFunc("/rent/ip", s.rentIP)
-		_, _ = w.Write([]byte("ok"))
+	http.HandleFunc("/release/ip", s.releaseIP)
-	})
+	cert, ok := os.LookupEnv(corev1.TLSCertKey)
-	cert, err := base64.StdEncoding.DecodeString(os.Getenv("CERT"))
+	if !ok {
-	if err != nil {
+		return fmt.Errorf("can not get %s from env", corev1.TLSCertKey)
-		return err
 	}
-	key, err := base64.StdEncoding.DecodeString(os.Getenv("KEY"))
+	key, ok := os.LookupEnv(corev1.TLSPrivateKeyKey)
-	if err != nil {
+	if !ok {
-		return err
+		return fmt.Errorf("can not get %s from env", corev1.TLSPrivateKeyKey)
 	}
-	pair, err := tls.X509KeyPair(cert, key)
+	pair, err := tls.X509KeyPair([]byte(cert), []byte(key))
 	if err != nil {
 		return err
 	}