feat: use webhook to manage ip allocate

pkg/webhook/convert.go

@@ -0,0 +1,89 @@
+package webhook
+
+import (
+	v1 "k8s.io/api/admission/v1"
+	"k8s.io/api/admission/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func convertAdmissionRequestToV1(r *v1beta1.AdmissionRequest) *v1.AdmissionRequest {
+	return &v1.AdmissionRequest{
+		Kind:               r.Kind,
+		Namespace:          r.Namespace,
+		Name:               r.Name,
+		Object:             r.Object,
+		Resource:           r.Resource,
+		Operation:          v1.Operation(r.Operation),
+		UID:                r.UID,
+		DryRun:             r.DryRun,
+		OldObject:          r.OldObject,
+		Options:            r.Options,
+		RequestKind:        r.RequestKind,
+		RequestResource:    r.RequestResource,
+		RequestSubResource: r.RequestSubResource,
+		SubResource:        r.SubResource,
+		UserInfo:           r.UserInfo,
+	}
+}
+
+func convertAdmissionRequestToV1beta1(r *v1.AdmissionRequest) *v1beta1.AdmissionRequest {
+	return &v1beta1.AdmissionRequest{
+		Kind:               r.Kind,
+		Namespace:          r.Namespace,
+		Name:               r.Name,
+		Object:             r.Object,
+		Resource:           r.Resource,
+		Operation:          v1beta1.Operation(r.Operation),
+		UID:                r.UID,
+		DryRun:             r.DryRun,
+		OldObject:          r.OldObject,
+		Options:            r.Options,
+		RequestKind:        r.RequestKind,
+		RequestResource:    r.RequestResource,
+		RequestSubResource: r.RequestSubResource,
+		SubResource:        r.SubResource,
+		UserInfo:           r.UserInfo,
+	}
+}
+
+func convertAdmissionResponseToV1(r *v1beta1.AdmissionResponse) *v1.AdmissionResponse {
+	var pt *v1.PatchType
+	if r.PatchType != nil {
+		t := v1.PatchType(*r.PatchType)
+		pt = &t
+	}
+	return &v1.AdmissionResponse{
+		UID:              r.UID,
+		Allowed:          r.Allowed,
+		AuditAnnotations: r.AuditAnnotations,
+		Patch:            r.Patch,
+		PatchType:        pt,
+		Result:           r.Result,
+		Warnings:         r.Warnings,
+	}
+}
+
+func convertAdmissionResponseToV1beta1(r *v1.AdmissionResponse) *v1beta1.AdmissionResponse {
+	var pt *v1beta1.PatchType
+	if r.PatchType != nil {
+		t := v1beta1.PatchType(*r.PatchType)
+		pt = &t
+	}
+	return &v1beta1.AdmissionResponse{
+		UID:              r.UID,
+		Allowed:          r.Allowed,
+		AuditAnnotations: r.AuditAnnotations,
+		Patch:            r.Patch,
+		PatchType:        pt,
+		Result:           r.Result,
+		Warnings:         r.Warnings,
+	}
+}
+
+func toV1AdmissionResponse(err error) *v1.AdmissionResponse {
+	return &v1.AdmissionResponse{
+		Result: &metav1.Status{
+			Message: err.Error(),
+		},
+	}
+}

pkg/webhook/mutateadmissionwebhook.go

@@ -0,0 +1,138 @@
+package webhook
+
+import (
+	"crypto/tls"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	v1 "k8s.io/api/admission/v1"
+	"k8s.io/api/admission/v1beta1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/klog/v2"
+)
+
+// admitv1beta1Func handles a v1beta1 admission
+type admitv1beta1Func func(v1beta1.AdmissionReview) *v1beta1.AdmissionResponse
+
+// admitv1beta1Func handles a v1 admission
+type admitv1Func func(v1.AdmissionReview) *v1.AdmissionResponse
+
+// admitHandler is a handler, for both validators and mutators, that supports multiple admission review versions
+type admitHandler struct {
+	v1beta1 admitv1beta1Func
+	v1      admitv1Func
+}
+
+func newDelegateToV1AdmitHandler(f admitv1Func) admitHandler {
+	return admitHandler{
+		v1beta1: delegateV1beta1AdmitToV1(f),
+		v1:      f,
+	}
+}
+
+func delegateV1beta1AdmitToV1(f admitv1Func) admitv1beta1Func {
+	return func(review v1beta1.AdmissionReview) *v1beta1.AdmissionResponse {
+		in := v1.AdmissionReview{Request: convertAdmissionRequestToV1(review.Request)}
+		out := f(in)
+		return convertAdmissionResponseToV1beta1(out)
+	}
+}
+
+// serve handles the http portion of a request prior to handing to an admit
+// function
+func serve(w http.ResponseWriter, r *http.Request, admit admitHandler) {
+	var body []byte
+	if r.Body != nil {
+		if data, err := ioutil.ReadAll(r.Body); err == nil {
+			body = data
+		}
+	}
+
+	// verify the content type is accurate
+	contentType := r.Header.Get("Content-Type")
+	if contentType != "application/json" {
+		klog.Errorf("contentType=%s, expect application/json", contentType)
+		return
+	}
+
+	klog.V(2).Info(fmt.Sprintf("handling request: %s", body))
+
+	deserializer := codecs.UniversalDeserializer()
+	obj, gvk, err := deserializer.Decode(body, nil, nil)
+	if err != nil {
+		msg := fmt.Sprintf("Request could not be decoded: %v", err)
+		klog.Error(msg)
+		http.Error(w, msg, http.StatusBadRequest)
+		return
+	}
+
+	var responseObj runtime.Object
+	switch *gvk {
+	case v1beta1.SchemeGroupVersion.WithKind("AdmissionReview"):
+		requestedAdmissionReview, ok := obj.(*v1beta1.AdmissionReview)
+		if !ok {
+			klog.Errorf("Expected v1beta1.AdmissionReview but got: %T", obj)
+			return
+		}
+		responseAdmissionReview := &v1beta1.AdmissionReview{}
+		responseAdmissionReview.SetGroupVersionKind(*gvk)
+		responseAdmissionReview.Response = admit.v1beta1(*requestedAdmissionReview)
+		responseAdmissionReview.Response.UID = requestedAdmissionReview.Request.UID
+		responseObj = responseAdmissionReview
+	case v1.SchemeGroupVersion.WithKind("AdmissionReview"):
+		requestedAdmissionReview, ok := obj.(*v1.AdmissionReview)
+		if !ok {
+			klog.Errorf("Expected v1.AdmissionReview but got: %T", obj)
+			return
+		}
+		responseAdmissionReview := &v1.AdmissionReview{}
+		responseAdmissionReview.SetGroupVersionKind(*gvk)
+		responseAdmissionReview.Response = admit.v1(*requestedAdmissionReview)
+		responseAdmissionReview.Response.UID = requestedAdmissionReview.Request.UID
+		responseObj = responseAdmissionReview
+	default:
+		msg := fmt.Sprintf("Unsupported group version kind: %v", gvk)
+		klog.Error(msg)
+		http.Error(w, msg, http.StatusBadRequest)
+		return
+	}
+
+	klog.V(2).Info(fmt.Sprintf("sending response: %v", responseObj))
+	respBytes, err := json.Marshal(responseObj)
+	if err != nil {
+		klog.Error(err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	if _, err := w.Write(respBytes); err != nil {
+		klog.Error(err)
+	}
+}
+
+func servePods(w http.ResponseWriter, r *http.Request) {
+	serve(w, r, newDelegateToV1AdmitHandler(admitPods))
+}
+
+func Main(cmd *cobra.Command, args []string) {
+	http.HandleFunc("/pods", servePods)
+	http.HandleFunc("/readyz", func(w http.ResponseWriter, req *http.Request) { w.Write([]byte("ok")) })
+	cert, _ := base64.StdEncoding.DecodeString(os.Getenv("CERT"))
+	key, _ := base64.StdEncoding.DecodeString(os.Getenv("KEY"))
+	pair, _ := tls.X509KeyPair(cert, key)
+	t := &tls.Config{Certificates: []tls.Certificate{pair}}
+	server := &http.Server{
+		Addr:      fmt.Sprintf(":%d", 80),
+		TLSConfig: t,
+	}
+	err := server.ListenAndServeTLS("", "")
+	if err != nil {
+		panic(err)
+	}
+}

pkg/webhook/pods.go

@@ -0,0 +1,202 @@
+package webhook
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+
+	"github.com/mattbaird/jsonpatch"
+	"k8s.io/api/admission/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/klog/v2"
+	"k8s.io/kubectl/pkg/cmd/util/podcmd"
+
+	"github.com/wencaiwulue/kubevpn/pkg/config"
+	"github.com/wencaiwulue/kubevpn/pkg/handler"
+)
+
+// only allow pods to pull images from specific registry.
+func admitPods(ar v1.AdmissionReview) *v1.AdmissionResponse {
+	klog.V(2).Info("admitting pods")
+	podResource := metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}
+	if ar.Request.Resource != podResource {
+		err := fmt.Errorf("expect resource to be %s", podResource)
+		klog.Error(err)
+		return toV1AdmissionResponse(err)
+	}
+
+	raw := ar.Request.Object.Raw
+	pod := corev1.Pod{}
+
+	deserializer := codecs.UniversalDeserializer()
+	if _, _, err := deserializer.Decode(raw, nil, &pod); err != nil {
+		klog.Error(err)
+		return toV1AdmissionResponse(err)
+	}
+
+	switch ar.Request.Operation {
+	case v1.Create:
+		from, _ := json.Marshal(pod)
+		fmt.Println(ar.String())
+		var found bool
+		for i := 0; i < len(pod.Spec.Containers); i++ {
+			if pod.Spec.Containers[i].Name == config.ContainerSidecarVPN {
+				for j := 0; j < len(pod.Spec.Containers[i].Env); j++ {
+					pair := pod.Spec.Containers[i].Env[j]
+					if pair.Name == "InboundPodTunIP" {
+						found = true
+						conf, err := rest.InClusterConfig()
+						if err != nil {
+							klog.Error(err)
+							return toV1AdmissionResponse(err)
+						}
+						clientset, err := kubernetes.NewForConfig(conf)
+						if err != nil {
+							klog.Error(err)
+							return toV1AdmissionResponse(err)
+						}
+						cmi := clientset.CoreV1().ConfigMaps(ar.Request.Namespace)
+						if cmi == nil {
+							err = fmt.Errorf("why cmi is nil")
+							klog.Error(err)
+							return toV1AdmissionResponse(err)
+						}
+						_, err = cmi.Get(context.Background(), config.ConfigMapPodTrafficManager, metav1.GetOptions{})
+						if err != nil {
+							klog.Error(err)
+							return toV1AdmissionResponse(err)
+						}
+						dhcp := handler.NewDHCPManager(cmi, ar.Request.Namespace, &net.IPNet{IP: config.RouterIP, Mask: config.CIDR.Mask})
+						random, err := dhcp.RentIPRandom()
+						if err != nil {
+							klog.Error(err)
+							return toV1AdmissionResponse(err)
+						}
+						pod.Spec.Containers[i].Env[j].Value = random.String()
+					}
+				}
+			}
+		}
+		fmt.Println(found)
+		if found {
+			to, _ := json.Marshal(pod)
+			patch, _ := jsonpatch.CreatePatch(from, to)
+			marshal, _ := json.Marshal(patch)
+			return applyPodPatch(ar, func(pod *corev1.Pod) bool {
+				name, _ := podcmd.FindContainerByName(pod, config.ContainerSidecarVPN)
+				return name != nil
+			}, string(marshal))
+		}
+		return &v1.AdmissionResponse{
+			Allowed: true,
+		}
+	case v1.Delete:
+		name, _ := podcmd.FindContainerByName(&pod, config.ContainerSidecarVPN)
+		if name != nil {
+			for _, envVar := range name.Env {
+				if envVar.Name == "InboundPodTunIP" {
+					ip, cidr, err := net.ParseCIDR(envVar.Value)
+					if err == nil {
+						conf, err := rest.InClusterConfig()
+						if err != nil {
+							klog.Error(err)
+							return toV1AdmissionResponse(err)
+						}
+						clientset, err := kubernetes.NewForConfig(conf)
+						if err != nil {
+							klog.Error(err)
+							return toV1AdmissionResponse(err)
+						}
+						cmi := clientset.CoreV1().ConfigMaps(ar.Request.Namespace)
+						if cmi == nil {
+							err = fmt.Errorf("why cmi is nil")
+							klog.Error(err)
+							return toV1AdmissionResponse(err)
+						}
+						ipnet := &net.IPNet{
+							IP:   ip,
+							Mask: cidr.Mask,
+						}
+						err = handler.NewDHCPManager(cmi, ar.Request.Namespace, &net.IPNet{IP: config.RouterIP, Mask: config.CIDR.Mask}).ReleaseIpToDHCP(ipnet)
+						if err != nil {
+							klog.V(1).Infof("release ip to dhcp err: %v", err)
+						}
+					}
+				}
+			}
+		}
+
+		return &v1.AdmissionResponse{
+			Allowed: true,
+		}
+	default:
+		return toV1AdmissionResponse(fmt.Errorf("expect operation is %s or %s, not %s", v1.Create, v1.Delete, ar.Request.Operation))
+	}
+}
+
+func applyPodPatch(ar v1.AdmissionReview, shouldPatchPod func(*corev1.Pod) bool, patch string) *v1.AdmissionResponse {
+	klog.V(2).Info("mutating pods")
+	podResource := metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}
+	if ar.Request.Resource != podResource {
+		klog.Errorf("expect resource to be %s", podResource)
+		return nil
+	}
+
+	raw := ar.Request.Object.Raw
+	pod := corev1.Pod{}
+	deserializer := codecs.UniversalDeserializer()
+	if _, _, err := deserializer.Decode(raw, nil, &pod); err != nil {
+		klog.Error(err)
+		return toV1AdmissionResponse(err)
+	}
+	reviewResponse := v1.AdmissionResponse{}
+	reviewResponse.Allowed = true
+	if shouldPatchPod(&pod) {
+		reviewResponse.Patch = []byte(patch)
+		pt := v1.PatchTypeJSONPatch
+		reviewResponse.PatchType = &pt
+	}
+	return &reviewResponse
+}
+
+// denySpecificAttachment denies `kubectl attach to-be-attached-pod -i -c=container1"
+// or equivalent client requests.
+func denySpecificAttachment(ar v1.AdmissionReview) *v1.AdmissionResponse {
+	klog.V(2).Info("handling attaching pods")
+	if ar.Request.Name != "to-be-attached-pod" {
+		return &v1.AdmissionResponse{Allowed: true}
+	}
+	podResource := metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}
+	if e, a := podResource, ar.Request.Resource; e != a {
+		err := fmt.Errorf("expect resource to be %s, got %s", e, a)
+		klog.Error(err)
+		return toV1AdmissionResponse(err)
+	}
+	if e, a := "attach", ar.Request.SubResource; e != a {
+		err := fmt.Errorf("expect subresource to be %s, got %s", e, a)
+		klog.Error(err)
+		return toV1AdmissionResponse(err)
+	}
+
+	raw := ar.Request.Object.Raw
+	podAttachOptions := corev1.PodAttachOptions{}
+	deserializer := codecs.UniversalDeserializer()
+	if _, _, err := deserializer.Decode(raw, nil, &podAttachOptions); err != nil {
+		klog.Error(err)
+		return toV1AdmissionResponse(err)
+	}
+	klog.V(2).Info(fmt.Sprintf("podAttachOptions=%#v\n", podAttachOptions))
+	if !podAttachOptions.Stdin || podAttachOptions.Container != "container1" {
+		return &v1.AdmissionResponse{Allowed: true}
+	}
+	return &v1.AdmissionResponse{
+		Allowed: false,
+		Result: &metav1.Status{
+			Message: "attaching to pod 'to-be-attached-pod' is not allowed",
+		},
+	}
+}

pkg/webhook/scheme.go

@@ -0,0 +1,27 @@
+package webhook
+
+import (
+	admissionv1 "k8s.io/api/admission/v1"
+	admissionv1beta1 "k8s.io/api/admission/v1beta1"
+	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+	admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+)
+
+var scheme = runtime.NewScheme()
+var codecs = serializer.NewCodecFactory(scheme)
+
+func init() {
+	addToScheme(scheme)
+}
+
+func addToScheme(scheme *runtime.Scheme) {
+	utilruntime.Must(corev1.AddToScheme(scheme))
+	utilruntime.Must(admissionv1beta1.AddToScheme(scheme))
+	utilruntime.Must(admissionregistrationv1beta1.AddToScheme(scheme))
+	utilruntime.Must(admissionv1.AddToScheme(scheme))
+	utilruntime.Must(admissionregistrationv1.AddToScheme(scheme))
+}