From 182356794940738938d649911a38f8aa90884d4c Mon Sep 17 00:00:00 2001 From: naison <895703375@qq.com> Date: Wed, 23 Jul 2025 19:52:12 +0800 Subject: [PATCH] hotfix: fix wait envoy rule to works takes too long time (#674) --- charts/kubevpn/templates/deployment.yaml | 6 -- charts/kubevpn/values.yaml | 10 +- cmd/kubevpn/cmds/controlplane.go | 10 +- pkg/config/config.go | 3 +- pkg/controlplane/main.go | 26 ++---- pkg/controlplane/processor.go | 14 +-- pkg/controlplane/watcher.go | 112 +++++++++++++---------- pkg/handler/remote.go | 28 +----- 8 files changed, 84 insertions(+), 125 deletions(-) diff --git a/charts/kubevpn/templates/deployment.yaml b/charts/kubevpn/templates/deployment.yaml index 65cff00f..f4b0260e 100644 --- a/charts/kubevpn/templates/deployment.yaml +++ b/charts/kubevpn/templates/deployment.yaml @@ -82,8 +82,6 @@ spec: {{- end }} - args: - control-plane - - --watchDirectoryFilename - - /etc/envoy/envoy-config.yaml command: - kubevpn image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" @@ -95,10 +93,6 @@ spec: protocol: TCP resources: {{- toYaml .Values.resourcesSmall | nindent 12 }} - volumeMounts: - - mountPath: /etc/envoy - name: envoy-config - readOnly: true - args: - webhook command: diff --git a/charts/kubevpn/values.yaml b/charts/kubevpn/values.yaml index bc66e00c..1729f1e3 100644 --- a/charts/kubevpn/values.yaml +++ b/charts/kubevpn/values.yaml @@ -96,15 +96,7 @@ autoscaling: # targetMemoryUtilizationPercentage: 80 # Additional volumes on the output Deployment definition. -volumes: - - configMap: - defaultMode: 420 - items: - - key: ENVOY_CONFIG - path: envoy-config.yaml - name: kubevpn-traffic-manager - optional: false - name: envoy-config +volumes: {} # Additional volumeMounts on the output Deployment definition. diff --git a/cmd/kubevpn/cmds/controlplane.go b/cmd/kubevpn/cmds/controlplane.go index a72b8cdb..cfd6c0d6 100644 --- a/cmd/kubevpn/cmds/controlplane.go +++ b/cmd/kubevpn/cmds/controlplane.go @@ -17,11 +17,8 @@ import ( "github.com/wencaiwulue/kubevpn/v2/pkg/util" ) -func CmdControlPlane(cmdutil.Factory) *cobra.Command { - var ( - watchDirectoryFilename string - port uint = 9002 - ) +func CmdControlPlane(f cmdutil.Factory) *cobra.Command { + var port uint = 9002 cmd := &cobra.Command{ Use: "control-plane", Hidden: true, @@ -38,11 +35,10 @@ func CmdControlPlane(cmdutil.Factory) *cobra.Command { } plog.G(context.Background()).Fatal(dns.ListenAndServe("udp", ":53", conf)) }() - err := controlplane.Main(cmd.Context(), watchDirectoryFilename, port, plog.G(context.Background())) + err := controlplane.Main(cmd.Context(), f, port, plog.G(context.Background())) return err }, } - cmd.Flags().StringVarP(&watchDirectoryFilename, "watchDirectoryFilename", "w", "/etc/envoy/envoy-config.yaml", "full path to directory to watch for files") cmd.Flags().BoolVar(&config.Debug, "debug", false, "true/false") return cmd } diff --git a/pkg/config/config.go b/pkg/config/config.go index 17f55b8a..96fc8f9f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -39,8 +39,7 @@ const ( ContainerSidecarVPN = "vpn" ContainerSidecarSyncthing = "syncthing" - VolumeEnvoyConfig = "envoy-config" - VolumeSyncthing = "syncthing" + VolumeSyncthing = "syncthing" // innerIPv4Pool is used as tun ip // 198.19.0.0/16 network is part of the 198.18.0.0/15 (reserved for benchmarking). diff --git a/pkg/controlplane/main.go b/pkg/controlplane/main.go index 6a563bad..aed4f9ed 100644 --- a/pkg/controlplane/main.go +++ b/pkg/controlplane/main.go @@ -2,17 +2,16 @@ package controlplane import ( "context" - "fmt" "github.com/envoyproxy/go-control-plane/pkg/cache/v3" serverv3 "github.com/envoyproxy/go-control-plane/pkg/server/v3" - "github.com/fsnotify/fsnotify" log "github.com/sirupsen/logrus" + cmdutil "k8s.io/kubectl/pkg/cmd/util" plog "github.com/wencaiwulue/kubevpn/v2/pkg/log" ) -func Main(ctx context.Context, filename string, port uint, logger *log.Logger) error { +func Main(ctx context.Context, factory cmdutil.Factory, port uint, logger *log.Logger) error { snapshotCache := cache.NewSnapshotCache(false, cache.IDHash{}, logger) proc := NewProcessor(snapshotCache, logger) @@ -25,33 +24,20 @@ func Main(ctx context.Context, filename string, port uint, logger *log.Logger) e notifyCh := make(chan NotifyMessage, 100) - notifyCh <- NotifyMessage{ - Operation: Create, - FilePath: filename, - } - - watcher, err := fsnotify.NewWatcher() - if err != nil { - return fmt.Errorf("failed to create file watcher: %v", err) - } - defer watcher.Close() - err = watcher.Add(filename) - if err != nil { - return fmt.Errorf("failed to add file: %s to wather: %v", filename, err) - } + notifyCh <- NotifyMessage{} go func() { - errChan <- Watch(watcher, filename, notifyCh) + errChan <- Watch(ctx, factory, notifyCh) }() for { select { case msg := <-notifyCh: - err = proc.ProcessFile(msg) + err := proc.ProcessFile(msg) if err != nil { plog.G(ctx).Errorf("Failed to process file: %v", err) return err } - case err = <-errChan: + case err := <-errChan: return err case <-ctx.Done(): return ctx.Err() diff --git a/pkg/controlplane/processor.go b/pkg/controlplane/processor.go index cf581445..1353995c 100644 --- a/pkg/controlplane/processor.go +++ b/pkg/controlplane/processor.go @@ -5,7 +5,6 @@ import ( "encoding/json" "math" "math/rand" - "os" "reflect" "strconv" "time" @@ -46,7 +45,7 @@ func (p *Processor) newVersion() string { } func (p *Processor) ProcessFile(file NotifyMessage) error { - configList, err := ParseYaml(file.FilePath) + configList, err := ParseYaml(file.Content) if err != nil { p.logger.Errorf("failed to parse config file: %v", err) return err @@ -66,7 +65,7 @@ func (p *Processor) ProcessFile(file NotifyMessage) error { uid := util.GenEnvoyUID(config.Namespace, config.Uid) lastConfig, ok := p.expireCache.Get(uid) if ok && reflect.DeepEqual(lastConfig.(*Virtual), config) { - p.logger.Infof("not needs to update, config: %s", string(marshal)) + p.logger.Infof("no need to update, config: %s", string(marshal)) continue } @@ -104,15 +103,10 @@ func (p *Processor) ProcessFile(file NotifyMessage) error { return nil } -func ParseYaml(file string) ([]*Virtual, error) { +func ParseYaml(content string) ([]*Virtual, error) { var virtualList = make([]*Virtual, 0) - yamlFile, err := os.ReadFile(file) - if err != nil { - return nil, err - } - - err = yaml.Unmarshal(yamlFile, &virtualList) + err := yaml.Unmarshal([]byte(content), &virtualList) if err != nil { return nil, err } diff --git a/pkg/controlplane/watcher.go b/pkg/controlplane/watcher.go index 23bd7e43..167af0df 100644 --- a/pkg/controlplane/watcher.go +++ b/pkg/controlplane/watcher.go @@ -1,62 +1,82 @@ package controlplane import ( - "fmt" + "context" "time" - "github.com/fsnotify/fsnotify" -) + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + informerv1 "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + cmdutil "k8s.io/kubectl/pkg/cmd/util" -type OperationType int - -const ( - Create OperationType = iota - Remove - Modify + "github.com/wencaiwulue/kubevpn/v2/pkg/config" + plog "github.com/wencaiwulue/kubevpn/v2/pkg/log" ) type NotifyMessage struct { - Operation OperationType - FilePath string + Content string } -func Watch(watcher *fsnotify.Watcher, filename string, notifyCh chan<- NotifyMessage) error { - ticker := time.NewTicker(time.Second * 2) - defer ticker.Stop() - for { - select { - case event, ok := <-watcher.Events: - if !ok { - return fmt.Errorf("watcher has closed") - } - if event.Op&fsnotify.Write == fsnotify.Write { - notifyCh <- NotifyMessage{ - Operation: Modify, - FilePath: event.Name, - } - } else if event.Op&fsnotify.Create == fsnotify.Create { - notifyCh <- NotifyMessage{ - Operation: Create, - FilePath: event.Name, - } - } else if event.Op&fsnotify.Remove == fsnotify.Remove { - notifyCh <- NotifyMessage{ - Operation: Remove, - FilePath: event.Name, - } - } +func Watch(ctx context.Context, f cmdutil.Factory, notifyCh chan<- NotifyMessage) error { + namespace, _, err := f.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + restConfig, err := f.ToRESTConfig() + if err != nil { + return err + } + conf := rest.CopyConfig(restConfig) + conf.QPS = 1 + conf.Burst = 2 + clientSet, err := kubernetes.NewForConfig(conf) + if err != nil { + plog.G(ctx).Errorf("Failed to create clientset: %v", err) + return err + } + cmIndexers := cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc} + cmInformer := informerv1.NewFilteredConfigMapInformer(clientSet, namespace, 0, cmIndexers, func(options *metav1.ListOptions) { + options.FieldSelector = fields.OneTermEqualSelector("metadata.name", config.ConfigMapPodTrafficManager).String() + }) + cmTicker := time.NewTicker(time.Second * 5) + _, err = cmInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + cmTicker.Reset(time.Nanosecond * 1) + }, + UpdateFunc: func(oldObj, newObj interface{}) { + cmTicker.Reset(time.Nanosecond * 1) + }, + DeleteFunc: func(obj interface{}) { + cmTicker.Reset(time.Nanosecond * 1) + }, + }) + if err != nil { + plog.G(ctx).Errorf("Failed to add service event handler: %v", err) + return err + } - case err, ok := <-watcher.Errors: - if !ok { - return fmt.Errorf("watcher error closed") - } - return err - - case <-ticker.C: - notifyCh <- NotifyMessage{ - Operation: Modify, - FilePath: filename, + go cmInformer.Run(ctx.Done()) + defer cmTicker.Stop() + for ; ctx.Err() == nil; <-cmTicker.C { + cmTicker.Reset(time.Second * 5) + cmList := cmInformer.GetIndexer().List() + if len(cmList) == 0 { + continue + } + for _, cm := range cmList { + configMap, ok := cm.(*v1.ConfigMap) + if ok { + if configMap.Data == nil { + configMap.Data = make(map[string]string) + } + notifyCh <- NotifyMessage{Content: configMap.Data[config.KeyEnvoy]} + continue } } } + return ctx.Err() } diff --git a/pkg/handler/remote.go b/pkg/handler/remote.go index 05004797..ab5aef5c 100644 --- a/pkg/handler/remote.go +++ b/pkg/handler/remote.go @@ -340,23 +340,7 @@ func genDeploySpec(namespace string, udp8422 string, tcp10800 string, tcp9002 st }, Spec: v1.PodSpec{ ServiceAccountName: config.ConfigMapPodTrafficManager, - Volumes: []v1.Volume{{ - Name: config.VolumeEnvoyConfig, - VolumeSource: v1.VolumeSource{ - ConfigMap: &v1.ConfigMapVolumeSource{ - LocalObjectReference: v1.LocalObjectReference{ - Name: config.ConfigMapPodTrafficManager, - }, - Items: []v1.KeyToPath{ - { - Key: config.KeyEnvoy, - Path: "envoy-config.yaml", - }, - }, - Optional: pointer.Bool(false), - }, - }, - }}, + Volumes: []v1.Volume{}, Containers: []v1.Container{ { Name: config.ContainerSidecarVPN, @@ -434,7 +418,7 @@ kubevpn server -l "tcp://:10800" -l "tun://:8422?net=${TunIPv4}&net6=${TunIPv6}" Name: config.ContainerSidecarControlPlane, Image: image, Command: []string{"kubevpn"}, - Args: []string{"control-plane", "--watchDirectoryFilename", "/etc/envoy/envoy-config.yaml"}, + Args: []string{"control-plane"}, Ports: []v1.ContainerPort{ { Name: tcp9002, @@ -447,13 +431,7 @@ kubevpn server -l "tcp://:10800" -l "tun://:8422?net=${TunIPv4}&net6=${TunIPv6}" Protocol: v1.ProtocolUDP, }, }, - VolumeMounts: []v1.VolumeMount{ - { - Name: config.VolumeEnvoyConfig, - ReadOnly: true, - MountPath: "/etc/envoy", - }, - }, + VolumeMounts: []v1.VolumeMount{}, ImagePullPolicy: v1.PullIfNotPresent, Resources: resourcesSmall, },