mirror of
https://github.com/kubenetworks/kubevpn.git
synced 2025-09-27 03:36:09 +08:00
38584da9d3
* refactor: remove options netstack * refactor: remove options netstack * refactor: forward chain use gvisor tcp * refactor: docs * refactor: remove forwarder options * refactor: optimize code * refactor: remove node type "tcp://" * hotfix: packet read from tun needs to handle by gvisor * hotfix: fix charts * refactor: remove parameter engine
488 lines
15 KiB
Go
488 lines
15 KiB
Go
package handler
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
admissionv1 "k8s.io/api/admissionregistration/v1"
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
v1 "k8s.io/api/core/v1"
|
|
rbacv1 "k8s.io/api/rbac/v1"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/util/intstr"
|
|
"k8s.io/apimachinery/pkg/util/wait"
|
|
"k8s.io/client-go/kubernetes"
|
|
corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
|
|
"k8s.io/kubectl/pkg/polymorphichelpers"
|
|
"k8s.io/kubectl/pkg/util/podutils"
|
|
"k8s.io/utils/pointer"
|
|
"k8s.io/utils/ptr"
|
|
|
|
"github.com/wencaiwulue/kubevpn/v2/pkg/config"
|
|
plog "github.com/wencaiwulue/kubevpn/v2/pkg/log"
|
|
"github.com/wencaiwulue/kubevpn/v2/pkg/util"
|
|
)
|
|
|
|
func createOutboundPod(ctx context.Context, clientset *kubernetes.Clientset, namespace, image, imagePullSecretName string) (err error) {
|
|
var exists bool
|
|
exists, err = util.DetectPodExists(ctx, clientset, namespace)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if exists {
|
|
plog.G(ctx).Infof("Use exist traffic manager in namespace %s", namespace)
|
|
return nil
|
|
}
|
|
|
|
var deleteResource = func(ctx context.Context) {
|
|
options := metav1.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)}
|
|
name := config.ConfigMapPodTrafficManager
|
|
_ = clientset.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(ctx, name+"."+namespace, options)
|
|
_ = clientset.RbacV1().RoleBindings(namespace).Delete(ctx, name, options)
|
|
_ = clientset.RbacV1().Roles(namespace).Delete(ctx, name, options)
|
|
_ = clientset.CoreV1().ServiceAccounts(namespace).Delete(ctx, name, options)
|
|
_ = clientset.CoreV1().Services(namespace).Delete(ctx, name, options)
|
|
_ = clientset.CoreV1().Secrets(namespace).Delete(ctx, name, options)
|
|
_ = clientset.CoreV1().Pods(namespace).Delete(ctx, config.CniNetName, options)
|
|
_ = clientset.BatchV1().Jobs(namespace).Delete(ctx, name, options)
|
|
_ = clientset.AppsV1().Deployments(namespace).Delete(ctx, name, options)
|
|
}
|
|
defer func() {
|
|
if err != nil {
|
|
deleteResource(context.Background())
|
|
}
|
|
}()
|
|
deleteResource(ctx)
|
|
|
|
// 1) label namespace
|
|
plog.G(ctx).Infof("Labeling Namespace %s", namespace)
|
|
ns, err := clientset.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{})
|
|
if err != nil {
|
|
plog.G(ctx).Errorf("Get Namespace error: %s", err.Error())
|
|
return err
|
|
}
|
|
if ns.Labels == nil {
|
|
ns.Labels = map[string]string{}
|
|
}
|
|
ns.Labels["ns"] = namespace
|
|
_, err = clientset.CoreV1().Namespaces().Update(ctx, ns, metav1.UpdateOptions{})
|
|
if err != nil {
|
|
plog.G(ctx).Infof("Labeling Namespace error: %s", err.Error())
|
|
return err
|
|
}
|
|
|
|
// 2) create serviceAccount
|
|
plog.G(ctx).Infof("Creating ServiceAccount %s", config.ConfigMapPodTrafficManager)
|
|
_, err = clientset.CoreV1().ServiceAccounts(namespace).Create(ctx, genServiceAccount(namespace), metav1.CreateOptions{})
|
|
if err != nil {
|
|
plog.G(ctx).Infof("Creating ServiceAccount error: %s", err.Error())
|
|
return err
|
|
}
|
|
|
|
// 3) create roles
|
|
plog.G(ctx).Infof("Creating Roles %s", config.ConfigMapPodTrafficManager)
|
|
_, err = clientset.RbacV1().Roles(namespace).Create(ctx, genRole(namespace), metav1.CreateOptions{})
|
|
if err != nil {
|
|
plog.G(ctx).Errorf("Creating Roles error: %s", err.Error())
|
|
return err
|
|
}
|
|
|
|
// 4) create roleBinding
|
|
plog.G(ctx).Infof("Creating RoleBinding %s", config.ConfigMapPodTrafficManager)
|
|
_, err = clientset.RbacV1().RoleBindings(namespace).Create(ctx, genRoleBinding(namespace), metav1.CreateOptions{})
|
|
if err != nil {
|
|
plog.G(ctx).Errorf("Creating RoleBinding error: %s", err.Error())
|
|
return err
|
|
}
|
|
|
|
// 5) create service
|
|
plog.G(ctx).Infof("Creating Service %s", config.ConfigMapPodTrafficManager)
|
|
tcp10801 := "10801-for-tcp"
|
|
tcp9002 := "9002-for-envoy"
|
|
tcp80 := "80-for-webhook"
|
|
udp53 := "53-for-dns"
|
|
svcSpec := genService(namespace, tcp10801, tcp9002, tcp80, udp53)
|
|
_, err = clientset.CoreV1().Services(namespace).Create(ctx, svcSpec, metav1.CreateOptions{})
|
|
if err != nil {
|
|
plog.G(ctx).Errorf("Creating Service error: %s", err.Error())
|
|
return err
|
|
}
|
|
|
|
crt, key, host, err := util.GenTLSCert(ctx, namespace)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// reason why not use v1.SecretTypeTls is because it needs key called tls.crt and tls.key, but tls.key can not as env variable
|
|
// ➜ ~ export tls.key=a
|
|
//export: not valid in this context: tls.key
|
|
secret := genSecret(namespace, crt, key, host)
|
|
_, err = clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
|
|
if err != nil {
|
|
plog.G(ctx).Errorf("Creating secret error: %s", err.Error())
|
|
return err
|
|
}
|
|
|
|
// 6) create mutatingWebhookConfigurations
|
|
plog.G(ctx).Infof("Creating MutatingWebhookConfiguration %s", config.ConfigMapPodTrafficManager)
|
|
mutatingWebhookConfiguration := genMutatingWebhookConfiguration(namespace, crt)
|
|
_, err = clientset.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(ctx, mutatingWebhookConfiguration, metav1.CreateOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create MutatingWebhookConfigurations: %v", err)
|
|
}
|
|
|
|
// 7) create deployment
|
|
plog.G(ctx).Infof("Creating Deployment %s", config.ConfigMapPodTrafficManager)
|
|
deploy := genDeploySpec(namespace, tcp10801, tcp9002, udp53, tcp80, image, imagePullSecretName)
|
|
deploy, err = clientset.AppsV1().Deployments(namespace).Create(ctx, deploy, metav1.CreateOptions{})
|
|
if err != nil {
|
|
plog.G(ctx).Errorf("Failed to create deployment for %s: %v", config.ConfigMapPodTrafficManager, err)
|
|
return err
|
|
}
|
|
|
|
return waitPodReady(ctx, deploy, clientset.CoreV1().Pods(namespace))
|
|
}
|
|
|
|
func genServiceAccount(namespace string) *v1.ServiceAccount {
|
|
return &v1.ServiceAccount{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
Namespace: namespace,
|
|
},
|
|
AutomountServiceAccountToken: pointer.Bool(true),
|
|
}
|
|
}
|
|
|
|
func genRole(namespace string) *rbacv1.Role {
|
|
return &rbacv1.Role{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
Namespace: namespace,
|
|
},
|
|
Rules: []rbacv1.PolicyRule{{
|
|
Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"},
|
|
APIGroups: []string{""},
|
|
Resources: []string{"configmaps", "secrets"},
|
|
ResourceNames: []string{config.ConfigMapPodTrafficManager},
|
|
}},
|
|
}
|
|
}
|
|
|
|
func genRoleBinding(namespace string) *rbacv1.RoleBinding {
|
|
return &rbacv1.RoleBinding{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
Namespace: namespace,
|
|
},
|
|
Subjects: []rbacv1.Subject{{
|
|
Kind: "ServiceAccount",
|
|
//APIGroup: "rbac.authorization.k8s.io",
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
Namespace: namespace,
|
|
}},
|
|
RoleRef: rbacv1.RoleRef{
|
|
APIGroup: "rbac.authorization.k8s.io",
|
|
Kind: "Role",
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
},
|
|
}
|
|
}
|
|
|
|
func genMutatingWebhookConfiguration(namespace string, crt []byte) *admissionv1.MutatingWebhookConfiguration {
|
|
return &admissionv1.MutatingWebhookConfiguration{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: config.ConfigMapPodTrafficManager + "." + namespace,
|
|
Namespace: namespace,
|
|
},
|
|
Webhooks: []admissionv1.MutatingWebhook{{
|
|
Name: config.ConfigMapPodTrafficManager + ".naison.io", // no sense
|
|
ClientConfig: admissionv1.WebhookClientConfig{
|
|
Service: &admissionv1.ServiceReference{
|
|
Namespace: namespace,
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
Path: pointer.String("/pods"),
|
|
Port: pointer.Int32(80),
|
|
},
|
|
CABundle: crt,
|
|
},
|
|
Rules: []admissionv1.RuleWithOperations{{
|
|
Operations: []admissionv1.OperationType{admissionv1.Create, admissionv1.Delete},
|
|
Rule: admissionv1.Rule{
|
|
APIGroups: []string{""},
|
|
APIVersions: []string{"v1"},
|
|
Resources: []string{"pods"},
|
|
Scope: ptr.To(admissionv1.NamespacedScope),
|
|
},
|
|
}},
|
|
FailurePolicy: ptr.To(admissionv1.Ignore),
|
|
// namespace kubevpn is special, if installed to this namespace, means center install mode
|
|
// same as above label ns
|
|
NamespaceSelector: util.If(namespace == config.DefaultNamespaceKubevpn, nil, &metav1.LabelSelector{MatchLabels: map[string]string{"ns": namespace}}),
|
|
SideEffects: ptr.To(admissionv1.SideEffectClassNone),
|
|
TimeoutSeconds: ptr.To[int32](15),
|
|
AdmissionReviewVersions: []string{"v1", "v1beta1"},
|
|
ReinvocationPolicy: ptr.To(admissionv1.NeverReinvocationPolicy),
|
|
/*// needs to enable featureGate=AdmissionWebhookMatchConditions
|
|
MatchConditions: []admissionv1.MatchCondition{
|
|
{
|
|
Name: "",
|
|
Expression: fmt.Sprintf(
|
|
"container_name.exists(c, c == '%s') && environment_variable.find(e, e == '%s').exists()",
|
|
config.ContainerSidecarVPN, config.EnvInboundPodTunIPv4,
|
|
),
|
|
},
|
|
},*/
|
|
}},
|
|
}
|
|
}
|
|
|
|
func genService(namespace string, tcp10801 string, tcp9002 string, tcp80 string, udp53 string) *v1.Service {
|
|
return &v1.Service{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
Namespace: namespace,
|
|
},
|
|
Spec: v1.ServiceSpec{
|
|
Ports: []v1.ServicePort{{
|
|
Name: tcp10801,
|
|
Protocol: v1.ProtocolTCP,
|
|
Port: 10801,
|
|
TargetPort: intstr.FromInt32(10801),
|
|
}, {
|
|
Name: tcp9002,
|
|
Protocol: v1.ProtocolTCP,
|
|
Port: 9002,
|
|
TargetPort: intstr.FromInt32(9002),
|
|
}, {
|
|
Name: tcp80,
|
|
Protocol: v1.ProtocolTCP,
|
|
Port: 80,
|
|
TargetPort: intstr.FromInt32(80),
|
|
}, {
|
|
Name: udp53,
|
|
Protocol: v1.ProtocolUDP,
|
|
Port: 53,
|
|
TargetPort: intstr.FromInt32(53),
|
|
}},
|
|
Selector: map[string]string{"app": config.ConfigMapPodTrafficManager},
|
|
Type: v1.ServiceTypeClusterIP,
|
|
},
|
|
}
|
|
}
|
|
|
|
func genSecret(namespace string, crt []byte, key []byte, host []byte) *v1.Secret {
|
|
secret := &v1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
Namespace: namespace,
|
|
},
|
|
Data: map[string][]byte{
|
|
config.TLSCertKey: crt,
|
|
config.TLSPrivateKeyKey: key,
|
|
config.TLSServerName: host,
|
|
},
|
|
Type: v1.SecretTypeOpaque,
|
|
}
|
|
return secret
|
|
}
|
|
|
|
func genDeploySpec(namespace, tcp10801, tcp9002, udp53, tcp80, image, imagePullSecretName string) *appsv1.Deployment {
|
|
var resourcesSmall = v1.ResourceRequirements{
|
|
Requests: map[v1.ResourceName]resource.Quantity{
|
|
v1.ResourceCPU: resource.MustParse("100m"),
|
|
v1.ResourceMemory: resource.MustParse("128Mi"),
|
|
},
|
|
Limits: map[v1.ResourceName]resource.Quantity{
|
|
v1.ResourceCPU: resource.MustParse("200m"),
|
|
v1.ResourceMemory: resource.MustParse("256Mi"),
|
|
},
|
|
}
|
|
var resourcesLarge = v1.ResourceRequirements{
|
|
Requests: map[v1.ResourceName]resource.Quantity{
|
|
v1.ResourceCPU: resource.MustParse("500m"),
|
|
v1.ResourceMemory: resource.MustParse("512Mi"),
|
|
},
|
|
Limits: map[v1.ResourceName]resource.Quantity{
|
|
v1.ResourceCPU: resource.MustParse("2000m"),
|
|
v1.ResourceMemory: resource.MustParse("2048Mi"),
|
|
},
|
|
}
|
|
|
|
deploy := &appsv1.Deployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
Namespace: namespace,
|
|
},
|
|
Spec: appsv1.DeploymentSpec{
|
|
Replicas: pointer.Int32(1),
|
|
Selector: &metav1.LabelSelector{
|
|
MatchLabels: map[string]string{"app": config.ConfigMapPodTrafficManager},
|
|
},
|
|
Template: v1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{"app": config.ConfigMapPodTrafficManager},
|
|
Annotations: map[string]string{
|
|
"sidecar.istio.io/inject": "false",
|
|
},
|
|
},
|
|
Spec: v1.PodSpec{
|
|
ServiceAccountName: config.ConfigMapPodTrafficManager,
|
|
Volumes: []v1.Volume{},
|
|
Containers: []v1.Container{
|
|
{
|
|
Name: config.ContainerSidecarVPN,
|
|
Image: image,
|
|
Command: []string{"kubevpn"},
|
|
Args: []string{
|
|
"server",
|
|
"-l gtcp://:10801",
|
|
"-l gudp://:10802",
|
|
},
|
|
EnvFrom: []v1.EnvFromSource{{
|
|
SecretRef: &v1.SecretEnvSource{
|
|
LocalObjectReference: v1.LocalObjectReference{
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
},
|
|
},
|
|
}},
|
|
Env: []v1.EnvVar{},
|
|
Ports: []v1.ContainerPort{{
|
|
Name: tcp10801,
|
|
ContainerPort: 10801,
|
|
Protocol: v1.ProtocolTCP,
|
|
}},
|
|
Resources: resourcesLarge,
|
|
ImagePullPolicy: v1.PullIfNotPresent,
|
|
SecurityContext: &v1.SecurityContext{
|
|
Capabilities: &v1.Capabilities{
|
|
Add: []v1.Capability{},
|
|
},
|
|
Privileged: pointer.Bool(false),
|
|
},
|
|
},
|
|
{
|
|
Name: config.ContainerSidecarControlPlane,
|
|
Image: image,
|
|
Command: []string{"kubevpn"},
|
|
Args: []string{"control-plane"},
|
|
Ports: []v1.ContainerPort{
|
|
{
|
|
Name: tcp9002,
|
|
ContainerPort: 9002,
|
|
Protocol: v1.ProtocolTCP,
|
|
},
|
|
{
|
|
Name: udp53,
|
|
ContainerPort: 53,
|
|
Protocol: v1.ProtocolUDP,
|
|
},
|
|
},
|
|
VolumeMounts: []v1.VolumeMount{},
|
|
ImagePullPolicy: v1.PullIfNotPresent,
|
|
Resources: resourcesSmall,
|
|
},
|
|
{
|
|
Name: config.ContainerSidecarWebhook,
|
|
Image: image,
|
|
Command: []string{"kubevpn"},
|
|
Args: []string{"webhook"},
|
|
Ports: []v1.ContainerPort{{
|
|
Name: tcp80,
|
|
ContainerPort: 80,
|
|
Protocol: v1.ProtocolTCP,
|
|
}},
|
|
EnvFrom: []v1.EnvFromSource{{
|
|
SecretRef: &v1.SecretEnvSource{
|
|
LocalObjectReference: v1.LocalObjectReference{
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
},
|
|
},
|
|
}},
|
|
Env: []v1.EnvVar{{
|
|
Name: config.EnvPodNamespace,
|
|
ValueFrom: &v1.EnvVarSource{
|
|
FieldRef: &v1.ObjectFieldSelector{
|
|
FieldPath: "metadata.namespace",
|
|
},
|
|
},
|
|
}},
|
|
ImagePullPolicy: v1.PullIfNotPresent,
|
|
Resources: resourcesSmall,
|
|
},
|
|
},
|
|
RestartPolicy: v1.RestartPolicyAlways,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
if imagePullSecretName != "" {
|
|
deploy.Spec.Template.Spec.ImagePullSecrets = []v1.LocalObjectReference{{
|
|
Name: imagePullSecretName,
|
|
}}
|
|
}
|
|
return deploy
|
|
}
|
|
|
|
func waitPodReady(ctx context.Context, deploy *appsv1.Deployment, clientset corev1.PodInterface) error {
|
|
_, selector, err := polymorphichelpers.SelectorsForObject(deploy)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var isPodReady bool
|
|
var lastMessage string
|
|
ctx2, cancelFunc := context.WithTimeout(ctx, time.Minute*60)
|
|
defer cancelFunc()
|
|
plog.G(ctx).Infoln()
|
|
wait.UntilWithContext(ctx2, func(ctx context.Context) {
|
|
podList, err := clientset.List(ctx2, metav1.ListOptions{
|
|
LabelSelector: selector.String(),
|
|
})
|
|
if err != nil {
|
|
plog.G(ctx).Errorf("Failed to list pods for %s: %v", deploy.Name, err)
|
|
return
|
|
}
|
|
|
|
for _, pod := range podList.Items {
|
|
if pod.DeletionTimestamp != nil {
|
|
continue
|
|
}
|
|
var sb = bytes.NewBuffer(nil)
|
|
sb.WriteString(fmt.Sprintf("Pod %s is %s...\n", pod.Name, pod.Status.Phase))
|
|
if pod.Status.Reason != "" {
|
|
sb.WriteString(fmt.Sprintf(" reason %s", pod.Status.Reason))
|
|
}
|
|
if pod.Status.Message != "" {
|
|
sb.WriteString(fmt.Sprintf(" message %s", pod.Status.Message))
|
|
}
|
|
util.PrintStatus(&pod, sb)
|
|
if lastMessage != sb.String() {
|
|
plog.G(ctx).Info(sb.String())
|
|
}
|
|
lastMessage = sb.String()
|
|
|
|
readyFunc := func(pod *v1.Pod) bool {
|
|
for _, status := range pod.Status.ContainerStatuses {
|
|
if !status.Ready {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
if podutils.IsPodReady(&pod) && readyFunc(&pod) {
|
|
cancelFunc()
|
|
isPodReady = true
|
|
}
|
|
}
|
|
}, time.Second*3)
|
|
|
|
if !isPodReady {
|
|
plog.G(ctx).Errorf("Wait pod %s to be ready timeout", deploy.Name)
|
|
return errors.New(fmt.Sprintf("wait pod %s to be ready timeout", deploy.Name))
|
|
}
|
|
|
|
return nil
|
|
}
|