diff --git a/cmd/kubevpn/cmds/serve.go b/cmd/kubevpn/cmds/serve.go index 1f354824..54247fb7 100644 --- a/cmd/kubevpn/cmds/serve.go +++ b/cmd/kubevpn/cmds/serve.go @@ -3,7 +3,6 @@ package cmds import ( "context" "fmt" - "k8s.io/klog/v2" "net" "net/http" "os" @@ -31,25 +30,25 @@ func CmdServe(factory cmdutil.Factory) *cobra.Command { }, RunE: func(cmd *cobra.Command, args []string) error { if v, ok := os.LookupEnv(config.EnvInboundPodTunIP); ok && v == "" { - clientset, err := factory.KubernetesClientSet() - if err != nil { - klog.Error(err) - return err - } - namespace, found, _ := factory.ToRawKubeConfigLoader().Namespace() - if !found { + namespace := os.Getenv(config.EnvPodNamespace) + if namespace == "" { return fmt.Errorf("can not get namespace") } - cmi := clientset.CoreV1().ConfigMaps(namespace) - dhcp := handler.NewDHCPManager(cmi, namespace, &net.IPNet{IP: config.RouterIP, Mask: config.CIDR.Mask}) - random, err := dhcp.RentIPRandom() + url := fmt.Sprintf("%s.%s:80/rent/ip", config.ConfigMapPodTrafficManager, namespace) + request, err := http.NewRequest("GET", url, nil) if err != nil { - klog.Error(err) + return fmt.Errorf("can not new request, err: %v", err) + } + request.Header.Set(config.HeaderPodName, os.Getenv(config.EnvPodName)) + request.Header.Set(config.HeaderPodNamespace, namespace) + ip, err := util.DoReq(request) + if err != nil { + log.Error(err) return err } - err = os.Setenv(config.EnvInboundPodTunIP, random.String()) + err = os.Setenv(config.EnvInboundPodTunIP, string(ip)) if err != nil { - klog.Error(err) + log.Error(err) return err } } @@ -72,21 +71,20 @@ func CmdServe(factory cmdutil.Factory) *cobra.Command { if !ok || v == "" { return nil } - _, ipNet, err := net.ParseCIDR(v) + _, _, err := net.ParseCIDR(v) if err != nil { return err } - clientset, err := factory.KubernetesClientSet() + namespace := os.Getenv(config.EnvPodNamespace) + url := fmt.Sprintf("%s.%s:80/release/ip", config.ConfigMapPodTrafficManager, namespace) + request, err := http.NewRequest("DELETE", url, nil) if err != nil { - return err + return fmt.Errorf("can not new request, err: %v", err) } - namespace, found, _ := factory.ToRawKubeConfigLoader().Namespace() - if !found { - return fmt.Errorf("can not get namespace") - } - cmi := clientset.CoreV1().ConfigMaps(namespace) - dhcp := handler.NewDHCPManager(cmi, namespace, &net.IPNet{IP: config.RouterIP, Mask: config.CIDR.Mask}) - err = dhcp.ReleaseIpToDHCP(ipNet) + request.Header.Set(config.HeaderPodName, os.Getenv(config.EnvPodName)) + request.Header.Set(config.HeaderPodNamespace, namespace) + request.Header.Set(config.HeaderIP, v) + _, err = util.DoReq(request) return err }, } diff --git a/pkg/config/config.go b/pkg/config/config.go index 0cd7c0dd..6c6137b8 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -33,9 +33,16 @@ const ( // env name EnvTunNameOrLUID = "TunNameOrLUID" EnvInboundPodTunIP = "InboundPodTunIP" + EnvPodName = "POD_NAME" + EnvPodNamespace = "POD_NAMESPACE" // annotation AnnoServiceAccountName = "service_account_name_backup_by_kubevpn" + + // header name + HeaderPodName = "POD_NAME" + HeaderPodNamespace = "POD_NAMESPACE" + HeaderIP = "IP" ) var ( diff --git a/pkg/exchange/controller.go b/pkg/exchange/controller.go index b4f6420b..6a429eaa 100644 --- a/pkg/exchange/controller.go +++ b/pkg/exchange/controller.go @@ -24,6 +24,13 @@ func AddContainer(spec *corev1.PodSpec, c util.PodRouteConfig) { spec.Containers = append(spec.Containers, corev1.Container{ Name: config.ContainerSidecarVPN, Image: config.Image, + EnvFrom: []corev1.EnvFromSource{{ + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: config.ConfigMapPodTrafficManager, + }, + }, + }}, Env: []corev1.EnvVar{ { Name: "LocalTunIP", diff --git a/pkg/handler/cleaner.go b/pkg/handler/cleaner.go index 516b92e6..d6839466 100644 --- a/pkg/handler/cleaner.go +++ b/pkg/handler/cleaner.go @@ -131,6 +131,7 @@ func cleanup(clientset *kubernetes.Clientset, namespace, name string) { p = []byte(fmt.Sprintf(`{"data":{"%s":"%s"}}`, config.KeyRefCount, strconv.Itoa(0))) _, _ = clientset.CoreV1().ConfigMaps(namespace).Patch(context.Background(), name, types.MergePatchType, p, v1.PatchOptions{}) options := v1.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)} + _ = clientset.CoreV1().Secrets(namespace).Delete(context.Background(), name, options) _ = clientset.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.Background(), name+"."+namespace, options) _ = clientset.RbacV1().RoleBindings(namespace).Delete(context.Background(), name, options) _ = clientset.CoreV1().ServiceAccounts(namespace).Delete(context.Background(), name, options) diff --git a/pkg/handler/remote.go b/pkg/handler/remote.go index 8b898f0e..b0b7564c 100644 --- a/pkg/handler/remote.go +++ b/pkg/handler/remote.go @@ -3,7 +3,6 @@ package handler import ( "bytes" "context" - "encoding/base64" "encoding/json" "errors" "fmt" @@ -187,6 +186,22 @@ func CreateOutboundPod(ctx context.Context, factory cmdutil.Factory, clientset * h := config.ConfigMapPodTrafficManager + "." + namespace + "." + "svc" certificate, key, _ := cert.GenerateSelfSignedCertKey(h, nil, nil) + _, err = clientset.CoreV1().Secrets(namespace).Create(ctx, &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: config.ConfigMapPodTrafficManager, + Namespace: namespace, + }, + Data: map[string][]byte{ + v1.TLSCertKey: certificate, + v1.TLSPrivateKeyKey: key, + }, + Type: v1.SecretTypeTLS, + }, metav1.CreateOptions{}) + + if err != nil && !k8serrors.IsAlreadyExists(err) { + return nil, err + } + deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: config.ConfigMapPodTrafficManager, @@ -234,6 +249,13 @@ iptables -P FORWARD ACCEPT iptables -t nat -A POSTROUTING -s ${CIDR} -o eth0 -j MASQUERADE kubevpn serve -L "tcp://:10800" -L "tun://:8422?net=${TrafficManagerIP}" --debug=true`, }, + EnvFrom: []v1.EnvFromSource{{ + SecretRef: &v1.SecretEnvSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: config.ConfigMapPodTrafficManager, + }, + }, + }}, Env: []v1.EnvVar{ { Name: "CIDR", @@ -296,15 +318,13 @@ kubevpn serve -L "tcp://:10800" -L "tun://:8422?net=${TrafficManagerIP}" --debug ContainerPort: 80, Protocol: v1.ProtocolTCP, }}, - Env: []v1.EnvVar{ - { - Name: "CERT", - Value: base64.StdEncoding.EncodeToString(certificate), - }, { - Name: "KEY", - Value: base64.StdEncoding.EncodeToString(key), + EnvFrom: []v1.EnvFromSource{{ + SecretRef: &v1.SecretEnvSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: config.ConfigMapPodTrafficManager, + }, }, - }, + }}, ImagePullPolicy: v1.PullIfNotPresent, Resources: Resources, }, diff --git a/pkg/mesh/controller.go b/pkg/mesh/controller.go index 3f2af162..a1671fa9 100644 --- a/pkg/mesh/controller.go +++ b/pkg/mesh/controller.go @@ -38,6 +38,13 @@ iptables -t nat -A PREROUTING ! -p icmp ! -s 127.0.0.1 ! -d ${CIDR} -j DNAT --to iptables -t nat -A POSTROUTING ! -p icmp ! -s 127.0.0.1 ! -d ${CIDR} -j MASQUERADE kubevpn serve -L "tun:/${TrafficManagerRealIP}:8422?net=${InboundPodTunIP}&route=${CIDR}" --debug=true`, }, + EnvFrom: []v1.EnvFromSource{{ + SecretRef: &v1.SecretEnvSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: config.ConfigMapPodTrafficManager, + }, + }, + }}, Env: []v1.EnvVar{ { Name: "CIDR", @@ -52,23 +59,21 @@ kubevpn serve -L "tun:/${TrafficManagerRealIP}:8422?net=${InboundPodTunIP}&route Value: c.InboundPodTunIP, }, { - Name: "POD_NAMESPACE", + Name: config.EnvPodNamespace, ValueFrom: &v1.EnvVarSource{ FieldRef: &v1.ObjectFieldSelector{ FieldPath: "metadata.namespace", }, }, }, - }, - SecurityContext: &v1.SecurityContext{ - Capabilities: &v1.Capabilities{ - Add: []v1.Capability{ - "NET_ADMIN", - //"SYS_MODULE", + { + Name: config.EnvPodName, + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, }, }, - RunAsUser: pointer.Int64(0), - Privileged: pointer.Bool(true), }, Resources: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ @@ -81,6 +86,16 @@ kubevpn serve -L "tun:/${TrafficManagerRealIP}:8422?net=${InboundPodTunIP}&route }, }, ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{ + Add: []v1.Capability{ + "NET_ADMIN", + //"SYS_MODULE", + }, + }, + RunAsUser: pointer.Int64(0), + Privileged: pointer.Bool(true), + }, }) spec.Spec.Containers = append(spec.Spec.Containers, v1.Container{ Name: config.ContainerSidecarEnvoyProxy, diff --git a/pkg/util/util.go b/pkg/util/util.go index 9ea497ad..eedabc72 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -3,6 +3,8 @@ package util import ( "bytes" "context" + "crypto/tls" + "crypto/x509" "encoding/binary" "encoding/json" "fmt" @@ -547,3 +549,35 @@ func CanI(clientset *kubernetes.Clientset, sa, ns string, resource *rbacv1.Polic return false, nil } + +func DoReq(request *http.Request) (body []byte, err error) { + cert, ok := os.LookupEnv(v1.TLSCertKey) + if !ok { + return nil, fmt.Errorf("can not get %s from env", v1.TLSCertKey) + } + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM([]byte(cert)) + + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: caCertPool, + }, + }, + } + + var resp *http.Response + resp, err = client.Do(request) + if err != nil { + return nil, fmt.Errorf("err: %v", err) + } + defer resp.Body.Close() + body, err = io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("can not read body, err: %v", err) + } + if resp.StatusCode == http.StatusOK { + return body, nil + } + return body, fmt.Errorf("http status is %d", resp.StatusCode) +} diff --git a/pkg/webhook/dhcp.go b/pkg/webhook/dhcp.go new file mode 100644 index 00000000..ec581bf2 --- /dev/null +++ b/pkg/webhook/dhcp.go @@ -0,0 +1,74 @@ +package webhook + +import ( + "fmt" + "net" + "net/http" + + log "github.com/sirupsen/logrus" + "k8s.io/kubectl/pkg/cmd/util" + + "github.com/wencaiwulue/kubevpn/pkg/config" + "github.com/wencaiwulue/kubevpn/pkg/handler" +) + +type dhcpServer struct { + f util.Factory +} + +func (d *dhcpServer) rentIP(w http.ResponseWriter, r *http.Request) { + podName := r.Header.Get("POD_NAME") + namespace := r.Header.Get("POD_NAMESPACE") + + log.Infof("handling rent ip request, pod name: %s, ns: %s", podName, namespace) + clientset, err := d.f.KubernetesClientSet() + if err != nil { + log.Error(err) + w.WriteHeader(http.StatusBadRequest) + return + } + cmi := clientset.CoreV1().ConfigMaps(namespace) + dhcp := handler.NewDHCPManager(cmi, namespace, &net.IPNet{IP: config.RouterIP, Mask: config.CIDR.Mask}) + random, err := dhcp.RentIPRandom() + if err != nil { + log.Error(err) + w.WriteHeader(http.StatusBadRequest) + return + } + w.WriteHeader(http.StatusOK) + _, err = w.Write([]byte(random.String())) + if err != nil { + log.Error(err) + } +} + +func (d *dhcpServer) releaseIP(w http.ResponseWriter, r *http.Request) { + podName := r.Header.Get("POD_NAME") + namespace := r.Header.Get("POD_NAMESPACE") + ip := r.Header.Get("IP") + + _, ipNet, err := net.ParseCIDR(ip) + if err != nil { + log.Errorf("ip is invailed, ip: %s, err: %v", ip, err) + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(fmt.Sprintf("ip is invailed, ip: %s, err: %v", ip, err))) + return + } + + log.Infof("handling rent ip request, pod name: %s, ns: %s", podName, namespace) + clientset, err := d.f.KubernetesClientSet() + if err != nil { + log.Error(err) + w.WriteHeader(http.StatusBadRequest) + return + } + cmi := clientset.CoreV1().ConfigMaps(namespace) + dhcp := handler.NewDHCPManager(cmi, namespace, &net.IPNet{IP: config.RouterIP, Mask: config.CIDR.Mask}) + err = dhcp.ReleaseIpToDHCP(ipNet) + if err != nil { + log.Error(err) + w.WriteHeader(http.StatusBadRequest) + return + } + w.WriteHeader(http.StatusOK) +} diff --git a/pkg/webhook/mutateadmissionwebhook.go b/pkg/webhook/mutateadmissionwebhook.go index 7b95b1f5..11df94d4 100644 --- a/pkg/webhook/mutateadmissionwebhook.go +++ b/pkg/webhook/mutateadmissionwebhook.go @@ -2,7 +2,6 @@ package webhook import ( "crypto/tls" - "encoding/base64" "encoding/json" "fmt" "io" @@ -12,6 +11,7 @@ import ( log "github.com/sirupsen/logrus" v1 "k8s.io/api/admission/v1" "k8s.io/api/admission/v1beta1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" cmdutil "k8s.io/kubectl/pkg/cmd/util" ) @@ -122,21 +122,20 @@ func serve(w http.ResponseWriter, r *http.Request, admit admitHandler) { func Main(f cmdutil.Factory) error { h := &admissionReviewHandler{f: f} - http.HandleFunc("/pods", func(w http.ResponseWriter, r *http.Request) { - serve(w, r, newDelegateToV1AdmitHandler(h.admitPods)) - }) - http.HandleFunc("/readyz", func(w http.ResponseWriter, req *http.Request) { - _, _ = w.Write([]byte("ok")) - }) - cert, err := base64.StdEncoding.DecodeString(os.Getenv("CERT")) - if err != nil { - return err + http.HandleFunc("/pods", func(w http.ResponseWriter, r *http.Request) { serve(w, r, newDelegateToV1AdmitHandler(h.admitPods)) }) + http.HandleFunc("/readyz", func(w http.ResponseWriter, req *http.Request) { _, _ = w.Write([]byte("ok")) }) + s := dhcpServer{f: f} + http.HandleFunc("/rent/ip", s.rentIP) + http.HandleFunc("/release/ip", s.releaseIP) + cert, ok := os.LookupEnv(corev1.TLSCertKey) + if !ok { + return fmt.Errorf("can not get %s from env", corev1.TLSCertKey) } - key, err := base64.StdEncoding.DecodeString(os.Getenv("KEY")) - if err != nil { - return err + key, ok := os.LookupEnv(corev1.TLSPrivateKeyKey) + if !ok { + return fmt.Errorf("can not get %s from env", corev1.TLSPrivateKeyKey) } - pair, err := tls.X509KeyPair(cert, key) + pair, err := tls.X509KeyPair([]byte(cert), []byte(key)) if err != nil { return err }