From 1b7794aa922928adb78fefe432b5fccaa6b6ff11 Mon Sep 17 00:00:00 2001 From: fengcaiwen Date: Wed, 8 Mar 2023 20:50:29 +0800 Subject: [PATCH] feat: optimize get cidr logic --- build/Dockerfile | 2 +- build/local.Dockerfile | 2 +- cmd/kubevpn/cmds/reset.go | 2 +- pkg/dns/dns.go | 6 +- pkg/dns/dns_unix.go | 4 +- pkg/dns/dns_windows.go | 7 +- pkg/handler/cleaner.go | 23 +++-- pkg/handler/connect.go | 24 ++--- pkg/handler/dhcp.go | 4 +- pkg/handler/envoy.go | 2 +- pkg/handler/reset.go | 2 +- pkg/mesh/controller.go | 2 +- pkg/tun/tun_windows.go | 3 +- pkg/util/cidr.go | 19 ++-- pkg/util/const.go | 6 -- pkg/util/getcidr.go | 185 ++++++++++++++++++++++---------------- pkg/util/pod.go | 23 +++++ pkg/util/util.go | 31 ++++++- pkg/util/util_test.go | 23 +++-- 19 files changed, 234 insertions(+), 136 deletions(-) delete mode 100644 pkg/util/const.go diff --git a/build/Dockerfile b/build/Dockerfile index bbddf852..76580678 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -16,7 +16,7 @@ ARG BASE=github.com/wencaiwulue/kubevpn RUN sed -i s@/security.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list \ && sed -i s@/archive.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list RUN apt-get clean && apt-get update && apt-get install -y wget dnsutils vim curl \ - net-tools iptables iputils-ping lsof iproute2 tcpdump + net-tools iptables iputils-ping lsof iproute2 tcpdump binutils WORKDIR /app diff --git a/build/local.Dockerfile b/build/local.Dockerfile index 1c1b31e3..a6f47e21 100644 --- a/build/local.Dockerfile +++ b/build/local.Dockerfile @@ -4,7 +4,7 @@ FROM ubuntu:latest RUN sed -i s@/security.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list \ && sed -i s@/archive.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list RUN apt-get clean && apt-get update && apt-get install -y wget dnsutils vim curl \ - net-tools iptables iputils-ping lsof iproute2 tcpdump + net-tools iptables iputils-ping lsof iproute2 tcpdump binutils WORKDIR /app diff --git a/cmd/kubevpn/cmds/reset.go b/cmd/kubevpn/cmds/reset.go index 0cbc4cdc..b9157d42 100644 --- a/cmd/kubevpn/cmds/reset.go +++ b/cmd/kubevpn/cmds/reset.go @@ -46,7 +46,7 @@ func CmdReset(factory cmdutil.Factory) *cobra.Command { if err != nil { log.Fatal(err) } - log.Infoln("done") + log.Println("done") }, } diff --git a/pkg/dns/dns.go b/pkg/dns/dns.go index 96c2bf0f..10de0d24 100644 --- a/pkg/dns/dns.go +++ b/pkg/dns/dns.go @@ -46,7 +46,7 @@ func GetDNSServiceIPFromPod(clientset *kubernetes.Clientset, restclient *rest.RE } // duplicate server - set := sets.NewString() + set := sets.New[string]() for i := 0; i < len(resolvConf.Servers); i++ { if set.Has(resolvConf.Servers[i]) { resolvConf.Servers = append(resolvConf.Servers[:i], resolvConf.Servers[i+1:]...) @@ -183,8 +183,8 @@ func generateHostsEntry(list []v12.Service) string { if strings.EqualFold(item.Name, ServiceKubernetes) { continue } - ipList := sets.NewString(item.Spec.ClusterIPs...).Insert(item.Spec.ExternalIPs...).List() - domainList := sets.NewString(item.Name).Insert(item.Spec.ExternalName).List() + ipList := sets.New[string](item.Spec.ClusterIPs...).Insert(item.Spec.ExternalIPs...).UnsortedList() + domainList := sets.New[string](item.Name).Insert(item.Spec.ExternalName).UnsortedList() for _, ip := range ipList { for _, domain := range domainList { if net.ParseIP(ip) == nil || domain == "" { diff --git a/pkg/dns/dns_unix.go b/pkg/dns/dns_unix.go index 00fd4082..733ecc42 100644 --- a/pkg/dns/dns_unix.go +++ b/pkg/dns/dns_unix.go @@ -69,7 +69,7 @@ func usingResolver(clientConfig *miekgdns.ClientConfig, ns []string) { Ndots: clientConfig.Ndots, Timeout: 2, } - for _, s := range sets.NewString(strings.Split(clientConfig.Search[0], ".")...).Insert(ns...).List() { + for _, s := range sets.New[string](strings.Split(clientConfig.Search[0], ".")...).Insert(ns...).UnsortedList() { filename = filepath.Join("/", "etc", "resolver", s) _ = os.WriteFile(filename, []byte(toString(config)), 0644) } @@ -97,7 +97,7 @@ func usingNetworkSetup(ip string, namespace string) { //} case <-c: if rc, err := miekgdns.ClientConfigFromFile(resolv); err == nil && rc.Timeout != 1 { - if !sets.NewString(rc.Servers...).Has(ip) { + if !sets.New[string](rc.Servers...).Has(ip) { rc.Servers = append(rc.Servers, ip) for _, s := range []string{namespace + ".svc.cluster.local", "svc.cluster.local", "cluster.local"} { rc.Search = append(rc.Search, s) diff --git a/pkg/dns/dns_windows.go b/pkg/dns/dns_windows.go index 675c66f8..214e5b5e 100644 --- a/pkg/dns/dns_windows.go +++ b/pkg/dns/dns_windows.go @@ -4,7 +4,6 @@ package dns import ( - "context" "fmt" "net/netip" "os" @@ -38,7 +37,11 @@ func SetupDNS(clientConfig *miekgdns.ClientConfig, _ []string) error { servers = append(servers, addr) } err = luid.SetDNS(windows.AF_INET, servers, clientConfig.Search) - _ = exec.CommandContext(context.Background(), "ipconfig", "/flushdns").Run() + if err != nil { + log.Warningln(err) + return err + } + err = luid.FlushDNS(windows.AF_INET) if err != nil { log.Warningln(err) return err diff --git a/pkg/handler/cleaner.go b/pkg/handler/cleaner.go index d6839466..3d2b331b 100644 --- a/pkg/handler/cleaner.go +++ b/pkg/handler/cleaner.go @@ -48,7 +48,8 @@ func (c *ConnectOptions) addCleanUpResourceHandler(clientset *kubernetes.Clients if err == nil { // if ref-count is less than zero or equals to zero, means nobody is using this traffic pod, so clean it if count <= 0 { - cleanup(clientset, namespace, config.ConfigMapPodTrafficManager) + log.Info("ref-count is zero, prepare to clean up resource") + cleanup(clientset, namespace, config.ConfigMapPodTrafficManager, true) } } else { log.Error(err) @@ -123,14 +124,20 @@ func updateRefCount(configMapInterface v12.ConfigMapInterface, name string, incr return } -func cleanup(clientset *kubernetes.Clientset, namespace, name string) { - log.Info("ref-count is zero, prepare to clean up resource") - // keep configmap - p := []byte(fmt.Sprintf(`[{"op": "remove", "path": "/data/%s"}]`, config.KeyDHCP)) - _, _ = clientset.CoreV1().ConfigMaps(namespace).Patch(context.Background(), name, types.JSONPatchType, p, v1.PatchOptions{}) - p = []byte(fmt.Sprintf(`{"data":{"%s":"%s"}}`, config.KeyRefCount, strconv.Itoa(0))) - _, _ = clientset.CoreV1().ConfigMaps(namespace).Patch(context.Background(), name, types.MergePatchType, p, v1.PatchOptions{}) +func cleanup(clientset *kubernetes.Clientset, namespace, name string, keepCidr bool) { options := v1.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)} + + if keepCidr { + // keep configmap + p := []byte(fmt.Sprintf(`[{"op": "remove", "path": "/data/%s"}]`, config.KeyDHCP)) + _, _ = clientset.CoreV1().ConfigMaps(namespace).Patch(context.Background(), name, types.JSONPatchType, p, v1.PatchOptions{}) + p = []byte(fmt.Sprintf(`{"data":{"%s":"%s"}}`, config.KeyRefCount, strconv.Itoa(0))) + _, _ = clientset.CoreV1().ConfigMaps(namespace).Patch(context.Background(), name, types.MergePatchType, p, v1.PatchOptions{}) + } else { + _ = clientset.CoreV1().ConfigMaps(namespace).Delete(context.Background(), name, options) + } + + _ = clientset.CoreV1().Pods(namespace).Delete(context.Background(), config.CniNetName, options) _ = clientset.CoreV1().Secrets(namespace).Delete(context.Background(), name, options) _ = clientset.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.Background(), name+"."+namespace, options) _ = clientset.RbacV1().RoleBindings(namespace).Delete(context.Background(), name, options) diff --git a/pkg/handler/connect.go b/pkg/handler/connect.go index 8bca9eec..f2c6b909 100644 --- a/pkg/handler/connect.go +++ b/pkg/handler/connect.go @@ -247,13 +247,13 @@ func (c *ConnectOptions) startLocalTunServe(ctx context.Context, forwardAddress if util.IsWindows() { c.localTunIP.Mask = net.CIDRMask(0, 32) } - var list = sets.NewString(config.CIDR.String()) + var list = sets.New[string](config.CIDR.String()) for _, ipNet := range c.cidrs { list.Insert(ipNet.String()) } r := core.Route{ ServeNodes: []string{ - fmt.Sprintf("tun:/127.0.0.1:8422?net=%s&route=%s", c.localTunIP.String(), strings.Join(list.List(), ",")), + fmt.Sprintf("tun:/127.0.0.1:8422?net=%s&route=%s", c.localTunIP.String(), strings.Join(list.UnsortedList(), ",")), }, ChainNode: forwardAddress, Retries: 5, @@ -261,7 +261,7 @@ func (c *ConnectOptions) startLocalTunServe(ctx context.Context, forwardAddress log.Debugf("your ip is %s", c.localTunIP.IP.String()) if err = Start(ctx, r); err != nil { - log.Errorf("error while create tunnel, err: %v", err) + log.Errorf("error while create tunnel, err: %v", errors.WithStack(err)) } else { log.Info("tunnel connected") } @@ -417,7 +417,7 @@ func (c *ConnectOptions) setupDNS() error { if relovConf.Port == "" { relovConf.Port = strconv.Itoa(port) } - ns := sets.NewString() + ns := sets.New[string]() list, err := c.clientset.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) if err == nil { for _, item := range list.Items { @@ -430,7 +430,7 @@ func (c *ConnectOptions) setupDNS() error { ns.Insert(item.Name) } } - if err = dns.SetupDNS(relovConf, ns.List()); err != nil { + if err = dns.SetupDNS(relovConf, ns.UnsortedList()); err != nil { return err } // dump service in current namespace for support DNS resolve service:port @@ -604,7 +604,7 @@ func (c *ConnectOptions) PreCheckResource() { controller, err := util.GetTopOwnerReferenceBySelector(c.factory, c.Namespace, selector.String()) if err == nil { if len(controller) > 0 { - c.Workloads[i] = controller.List()[0] + c.Workloads[i] = controller.UnsortedList()[0] } } // only a single service, not support it yet @@ -624,7 +624,7 @@ func (c *ConnectOptions) GetRunningPodList() ([]v1.Pod, error) { return nil, err } for i := 0; i < len(list.Items); i++ { - if list.Items[i].GetDeletionTimestamp() != nil || list.Items[i].Status.Phase != v1.PodRunning { + if list.Items[i].GetDeletionTimestamp() != nil || !util.AllContainerIsRunning(&list.Items[i]) { list.Items = append(list.Items[:i], list.Items[i+1:]...) i-- } @@ -645,12 +645,12 @@ func (c *ConnectOptions) GetRunningPodList() ([]v1.Pod, error) { func (c *ConnectOptions) GetCIDR(ctx context.Context) (err error) { // (1) get cidr from cache var value string - value, err = c.dhcp.Get(config.KeyClusterIPv4POOLS) + value, err = c.dhcp.Get(ctx, config.KeyClusterIPv4POOLS) if err == nil { for _, s := range strings.Split(value, " ") { _, cidr, _ := net.ParseCIDR(s) if cidr != nil { - c.cidrs = append(c.cidrs, cidr) + c.cidrs = util.Deduplicate(append(c.cidrs, cidr)) } } } @@ -662,7 +662,7 @@ func (c *ConnectOptions) GetCIDR(ctx context.Context) (err error) { // (2) get cidr from cni c.cidrs, err = util.GetCIDRElegant(c.clientset, c.restclient, c.config, c.Namespace) if err == nil { - s := sets.NewString() + s := sets.New[string]() for _, cidr := range c.cidrs { s.Insert(cidr.String()) } @@ -670,8 +670,8 @@ func (c *ConnectOptions) GetCIDR(ctx context.Context) (err error) { for _, cidr := range cidrs { s.Insert(cidr.String()) } - c.cidrs = append(c.cidrs, cidrs...) - _ = c.dhcp.Set(config.KeyClusterIPv4POOLS, strings.Join(s.List(), " ")) + c.cidrs = util.Deduplicate(append(c.cidrs, cidrs...)) + _ = c.dhcp.Set(config.KeyClusterIPv4POOLS, strings.Join(s.UnsortedList(), " ")) return } diff --git a/pkg/handler/dhcp.go b/pkg/handler/dhcp.go index 797a485a..10749952 100644 --- a/pkg/handler/dhcp.go +++ b/pkg/handler/dhcp.go @@ -159,8 +159,8 @@ func (d *DHCPManager) Set(key, value string) error { return nil } -func (d *DHCPManager) Get(key string) (string, error) { - cm, err := d.client.Get(context.Background(), config.ConfigMapPodTrafficManager, metav1.GetOptions{}) +func (d *DHCPManager) Get(ctx2 context.Context, key string) (string, error) { + cm, err := d.client.Get(ctx2, config.ConfigMapPodTrafficManager, metav1.GetOptions{}) if err != nil { return "", err } diff --git a/pkg/handler/envoy.go b/pkg/handler/envoy.go index 924a056f..1fd8c144 100644 --- a/pkg/handler/envoy.go +++ b/pkg/handler/envoy.go @@ -60,7 +60,7 @@ func InjectVPNAndEnvoySidecar(ctx1 context.Context, factory cmdutil.Factory, cli } // already inject container vpn and envoy-proxy, do nothing - containerNames := sets.NewString() + containerNames := sets.New[string]() for _, container := range templateSpec.Spec.Containers { containerNames.Insert(container.Name) } diff --git a/pkg/handler/reset.go b/pkg/handler/reset.go index b66c93ab..cafcc01a 100644 --- a/pkg/handler/reset.go +++ b/pkg/handler/reset.go @@ -39,6 +39,6 @@ func (c *ConnectOptions) Reset(ctx2 context.Context) error { } } } - cleanup(c.clientset, c.Namespace, config.ConfigMapPodTrafficManager) + cleanup(c.clientset, c.Namespace, config.ConfigMapPodTrafficManager, false) return nil } diff --git a/pkg/mesh/controller.go b/pkg/mesh/controller.go index 92d393c3..d41b01a0 100644 --- a/pkg/mesh/controller.go +++ b/pkg/mesh/controller.go @@ -19,7 +19,7 @@ var envoyConfig []byte func RemoveContainers(spec *v1.PodTemplateSpec) { for i := 0; i < len(spec.Spec.Containers); i++ { - if sets.NewString(config.ContainerSidecarEnvoyProxy, config.ContainerSidecarVPN).Has(spec.Spec.Containers[i].Name) { + if sets.New[string](config.ContainerSidecarEnvoyProxy, config.ContainerSidecarVPN).Has(spec.Spec.Containers[i].Name) { spec.Spec.Containers = append(spec.Spec.Containers[:i], spec.Spec.Containers[i+1:]...) i-- } diff --git a/pkg/tun/tun_windows.go b/pkg/tun/tun_windows.go index 7fce94d9..a8f6a747 100644 --- a/pkg/tun/tun_windows.go +++ b/pkg/tun/tun_windows.go @@ -84,7 +84,8 @@ func addTunRoutes(luid string, routes ...types.Route) error { if err != nil { return err } - if err = ifName.AddRoute(prefix, addr, 0); err != nil { + err = ifName.AddRoute(prefix, addr, 0) + if err != nil && err != windows.ERROR_OBJECT_ALREADY_EXISTS { return err } } diff --git a/pkg/util/cidr.go b/pkg/util/cidr.go index fb76101f..37c932fb 100644 --- a/pkg/util/cidr.go +++ b/pkg/util/cidr.go @@ -38,25 +38,28 @@ func GetCIDRElegant(clientset *kubernetes.Clientset, restclient *rest.RESTClient result = append(result, cni...) } + pod, err := getPodCIDRFromCNI(clientset, restclient, restconfig, namespace) + if err == nil { + result = append(result, pod...) + } + svc, err := getServiceCIDRByCreateSvc(clientset.CoreV1().Services(namespace)) if err == nil { result = append(result, svc) - fromCNI, err := getPodCIDRFromCNI(clientset, restclient, restconfig, namespace) - if err == nil { - log.Infoln("get cidr from cni ok") - result = append(result, fromCNI...) - } } log.Infoln("get cidr from svc...") - pod, err := getPodCIDRFromPod(clientset, namespace, svc) + pod, err = getPodCIDRFromPod(clientset, namespace, svc) if err == nil { log.Infoln("get cidr from svc ok") result = append(result, pod...) } + result = Deduplicate(result) + if len(result) == 0 { - return nil, fmt.Errorf("can not get any cidr, please make sure you have prilivage") + err = fmt.Errorf("can not get any cidr, please make sure you have prilivage") + return } return } @@ -95,7 +98,7 @@ func GetCIDRFromResourceUgly(clientset *kubernetes.Clientset, namespace string) serviceList, _ := clientset.CoreV1().Services(namespace).List(context.Background(), v1.ListOptions{}) for _, service := range serviceList.Items { if ip := net.ParseIP(service.Spec.ClusterIP); ip != nil { - mask := net.CIDRMask(16, 32) + mask := net.CIDRMask(24, 32) cidrs = append(cidrs, &net.IPNet{IP: ip.Mask(mask), Mask: mask}) } } diff --git a/pkg/util/const.go b/pkg/util/const.go deleted file mode 100644 index 0f41252a..00000000 --- a/pkg/util/const.go +++ /dev/null @@ -1,6 +0,0 @@ -package util - -const ( - v4 = `(([0-9]{1,3}\.){3}[0-9]{1,3}/[0-9]{1,})` - v6 = `(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))/[0-9]{1,}` -) diff --git a/pkg/util/getcidr.go b/pkg/util/getcidr.go index 65ed4281..196856b1 100644 --- a/pkg/util/getcidr.go +++ b/pkg/util/getcidr.go @@ -1,93 +1,68 @@ package util import ( - "bufio" - "bytes" "context" + "encoding/json" "fmt" "net" - "regexp" "strings" "github.com/containernetworking/cni/libcni" log "github.com/sirupsen/logrus" - "github.com/wencaiwulue/kubevpn/pkg/config" v12 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/rest" "k8s.io/utils/pointer" - "sigs.k8s.io/yaml" + + "github.com/wencaiwulue/kubevpn/pkg/config" ) +// root 22008 21846 14 Jan18 ? 6-22:53:35 kube-apiserver --advertise-address=10.56.95.185 --allow-privileged=true --anonymous-auth=True --apiserver-count=3 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --client-ca-file=/etc/kubernetes/ssl/ca.crt --default-not-ready-toleration-seconds=300 --default-unreachable-toleration-seconds=300 --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=False --enable-bootstrap-token-auth=true --endpoint-reconciler-type=lease --etcd-cafile=/etc/ssl/etcd/ssl/ca.pem --etcd-certfile=/etc/ssl/etcd/ssl/node-kube-control-1.pem --etcd-keyfile=/etc/ssl/etcd/ssl/node-kube-control-1-key.pem --etcd-servers=https://10.56.95.185:2379,https://10.56.95.186:2379,https://10.56.95.187:2379 --etcd-servers-overrides=/events#https://10.56.95.185:2381;https://10.56.95.186:2381;https://10.56.95.187:2381 --event-ttl=1h0m0s --insecure-port=0 --kubelet-certificate-authority=/etc/kubernetes/ssl/kubelet/kubelet-ca.crt --kubelet-client-certificate=/etc/kubernetes/ssl/apiserver-kubelet-client.crt --kubelet-client-key=/etc/kubernetes/ssl/apiserver-kubelet-client.key --kubelet-preferred-address-types=InternalDNS,InternalIP,Hostname,ExternalDNS,ExternalIP --profiling=False --proxy-client-cert-file=/etc/kubernetes/ssl/front-proxy-client.crt --proxy-client-key-file=/etc/kubernetes/ssl/front-proxy-client.key --request-timeout=1m0s --requestheader-allowed-names=front-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/front-proxy-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/etc/kubernetes/ssl/sa.pub --service-account-signing-key-file=/etc/kubernetes/ssl/sa.key --service-cluster-ip-range=10.233.0.0/18 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/etc/kubernetes/ssl/apiserver.crt --tls-private-key-file=/etc/kubernetes/ssl/apiserver.key +// ref: https://kubernetes.io/docs/concepts/services-networking/dual-stack/#configure-ipv4-ipv6-dual-stack // get cidr by dump cluster info -func getCIDRByDumpClusterInfo(clientset *kubernetes.Clientset) (result []*net.IPNet, err error) { - p, err := clientset.CoreV1().Pods("kube-system").List(context.Background(), v1.ListOptions{ - FieldSelector: fields.OneTermEqualSelector("status.phase", string(v12.PodRunning)).String(), - }) +func getCIDRByDumpClusterInfo(clientset *kubernetes.Clientset) ([]*net.IPNet, error) { + podList, err := clientset.CoreV1().Pods(v1.NamespaceSystem).List(context.Background(), v1.ListOptions{}) if err != nil { return nil, err } - marshal, err := yaml.Marshal(p) - if err != nil { - return nil, err - } - - svcCIDR := `service-cluster-ip-range` - podCIDR := `cluster-cidr` - reader := bufio.NewReader(bytes.NewBufferString(string(marshal))) - svc := sets.NewString() - v4P := regexp.MustCompile(v4) - v6P := regexp.MustCompile(v6) - for { - line, _, err := reader.ReadLine() - if err != nil { - break - } - if strings.Contains(string(line), svcCIDR) { - ipv4 := v4P.FindAllString(string(line), -1) - ipv6 := v6P.FindAllString(string(line), -1) - svc.Insert(ipv4...).Insert(ipv6...) - } - if strings.Contains(string(line), podCIDR) { - ipv4 := v4P.FindAllString(string(line), -1) - ipv6 := v6P.FindAllString(string(line), -1) - svc.Insert(ipv4...).Insert(ipv6...) + var list []string + for _, item := range podList.Items { + for _, container := range item.Spec.Containers { + list = append(list, container.Args...) + list = append(list, container.Command...) } } - for _, s := range svc.List() { - _, ipnet, err := net.ParseCIDR(s) - if err != nil { - result = append(result, ipnet) - } + var result []*net.IPNet + for _, s := range list { + result = append(result, parseCIDRFromString(s)...) } - - return result, nil + return Deduplicate(result), nil } +// kube-controller-manager--allocate-node-cidrs=true--authentication-kubeconfig=/etc/kubernetes/controller-manager.conf--authorization-kubeconfig=/etc/kubernetes/controller-manager.conf--bind-address=0.0.0.0--client-ca-file=/etc/kubernetes/ssl/ca.crt--cluster-cidr=10.233.64.0/18--cluster-name=cluster.local--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.crt--cluster-signing-key-file=/etc/kubernetes/ssl/ca.key--configure-cloud-routes=false--controllers=*,bootstrapsigner,tokencleaner--kubeconfig=/etc/kubernetes/controller-manager.conf--leader-elect=true--leader-elect-lease-duration=15s--leader-elect-renew-deadline=10s--node-cidr-mask-size=24--node-monitor-grace-period=40s--node-monitor-period=5s--port=0--profiling=False--requestheader-client-ca-file=/etc/kubernetes/ssl/front-proxy-ca.crt--root-ca-file=/etc/kubernetes/ssl/ca.crt--service-account-private-key-file=/etc/kubernetes/ssl/sa.key--service-cluster-ip-range=10.233.0.0/18--terminated-pod-gc-threshold=12500--use-service-account-credentials=true func getCIDRFromCNI(clientset *kubernetes.Clientset, restclient *rest.RESTClient, restconfig *rest.Config, namespace string) ([]*net.IPNet, error) { pod, err := createCIDRPod(clientset, namespace) if err != nil { return nil, err } - var cmd = `grep -a -R "service-cluster-ip-range\|cluster-ip-range\|cluster-cidr" /etc/cni/proc/*/cmdline | grep -a -v grep` + var cmd = `grep -a -R "service-cluster-ip-range\|cluster-cidr" /etc/cni/proc/*/cmdline | grep -a -v grep | tr "\0" "\n"` var result []*net.IPNet content, err := Shell(clientset, restclient, restconfig, pod.Name, "", pod.Namespace, []string{"sh", "-c", cmd}) if err != nil { return nil, err } - result = parseCIDRFromString(content) - if len(result) == 0 { - return nil, fmt.Errorf("can not found any cidr") + for _, s := range strings.Split(content, "\n") { + result = Deduplicate(append(result, parseCIDRFromString(s)...)) } return result, nil @@ -113,30 +88,67 @@ func getServiceCIDRByCreateSvc(serviceInterface corev1.ServiceInterface) (*net.I return nil, err } +/* +* + + { + "name": "cni0", + "cniVersion":"0.3.1", + "plugins":[ + { + "datastore_type": "kubernetes", + "nodename": "10.56.95.185", + "type": "calico", + "log_level": "info", + "log_file_path": "/var/log/calico/cni/cni.log", + "ipam": { + "type": "calico-ipam", + "assign_ipv4": "true", + "ipv4_pools": ["10.233.64.0/18"] + }, + "policy": { + "type": "k8s" + }, + "kubernetes": { + "kubeconfig": "/etc/cni/net.d/calico-kubeconfig" + } + }, + { + "type":"portmap", + "capabilities": { + "portMappings": true + } + } + ] + } +*/ func getPodCIDRFromCNI(clientset *kubernetes.Clientset, restclient *rest.RESTClient, restconfig *rest.Config, namespace string) ([]*net.IPNet, error) { - pod, err := createCIDRPod(clientset, namespace) - if err != nil { - return nil, err - } - //var cmd = "cat /etc/cni/net.d/*.conflist" - content, err := Shell(clientset, restclient, restconfig, pod.Name, "", pod.Namespace, []string{"cat", "/etc/cni/net.d/*.conflist"}) + content, err := Shell(clientset, restclient, restconfig, config.CniNetName, "", namespace, []string{"cat", "/etc/cni/net.d/*.conflist"}) if err != nil { return nil, err } - conf, err := libcni.ConfListFromFile(content) + configList, err := libcni.ConfListFromBytes([]byte(content)) if err == nil { - log.Infoln("get cni config", conf.Name) + log.Infoln("get cni config", configList.Name) + } + var cidr []*net.IPNet + for _, plugin := range configList.Plugins { + switch plugin.Network.Type { + case "calico": + var m map[string]interface{} + _ = json.Unmarshal(plugin.Bytes, &m) + slice, _, _ := unstructured.NestedStringSlice(m, "ipam", "ipv4_pools") + for _, s := range slice { + if _, ipNet, _ := net.ParseCIDR(s); ipNet != nil { + cidr = append(cidr, ipNet) + } + } + } } - result := parseCIDRFromString(content) - - if len(result) == 0 { - return nil, fmt.Errorf("can not found any cidr") - } - - return result, nil + return cidr, nil } func createCIDRPod(clientset *kubernetes.Clientset, namespace string) (*v12.Pod, error) { @@ -240,6 +252,9 @@ func createCIDRPod(clientset *kubernetes.Clientset, namespace string) (*v12.Pod, } get, err := clientset.CoreV1().Pods(pod.Namespace).Get(context.Background(), pod.Name, v1.GetOptions{}) if errors.IsNotFound(err) || get.Status.Phase != v12.PodRunning { + if get.Status.Phase != v12.PodRunning { + _ = clientset.CoreV1().Pods(namespace).Delete(context.Background(), pod.Name, v1.DeleteOptions{GracePeriodSeconds: pointer.Int64(0)}) + } pod, err = clientset.CoreV1().Pods(namespace).Create(context.Background(), pod, v1.CreateOptions{}) if err != nil { return nil, err @@ -247,10 +262,19 @@ func createCIDRPod(clientset *kubernetes.Clientset, namespace string) (*v12.Pod, err = WaitPod(clientset.CoreV1().Pods(namespace), v1.ListOptions{ FieldSelector: fields.OneTermEqualSelector("metadata.name", pod.Name).String(), }, func(pod *v12.Pod) bool { - return pod.Status.Phase == v12.PodRunning + isRunning := pod.Status.Phase == v12.PodRunning + if !isRunning { + if message := PrintStatusInline(pod); message != "" { + fmt.Printf("%s\r", message) + } + } + return isRunning }) if err != nil { + fmt.Printf("wait pod %s to be running timeout, reason %s, ignore\r\n", pod.Name, pod.Status.Reason) return nil, err + } else { + fmt.Printf("\r") } } return pod, nil @@ -278,22 +302,29 @@ func getPodCIDRFromPod(clientset *kubernetes.Clientset, namespace string, svc *n return nil, fmt.Errorf("can not found pod cidr from pod list") } +/* +* +kube-apiserver: +--service-cluster-ip-range=, +kube-controller-manager: +--cluster-cidr=, +--service-cluster-ip-range=, +--node-cidr-mask-size-ipv4|--node-cidr-mask-size-ipv6 defaults to /24 for IPv4 and /64 for IPv6 +kube-proxy: +--cluster-cidr=, +*/ func parseCIDRFromString(content string) (result []*net.IPNet) { - ipv4 := regexp.MustCompile(v4).FindAllString(content, -1) - ipv6 := regexp.MustCompile(v6).FindAllString(content, -1) - - for _, s := range ipv4 { - _, ipNet, err := net.ParseCIDR(s) - if err == nil { - result = append(result, ipNet) + if strings.Contains(content, "cluster-cidr") || strings.Contains(content, "service-cluster-ip-range") { + split := strings.Split(content, "=") + if len(split) == 2 { + cidrList := split[1] + for _, cidr := range strings.Split(cidrList, ",") { + _, c, err := net.ParseCIDR(cidr) + if err == nil { + result = append(result, c) + } + } } } - - for _, s := range ipv6 { - _, ipNet, err := net.ParseCIDR(s) - if err == nil { - result = append(result, ipNet) - } - } - return result + return } diff --git a/pkg/util/pod.go b/pkg/util/pod.go index 4ec6c5b4..adb5819d 100644 --- a/pkg/util/pod.go +++ b/pkg/util/pod.go @@ -1,6 +1,7 @@ package util import ( + "bytes" "fmt" "io" "text/tabwriter" @@ -30,6 +31,28 @@ func PrintStatus(pod *corev1.Pod, writer io.Writer) { } } +func PrintStatusInline(pod *corev1.Pod) string { + var sb = bytes.NewBuffer(nil) + w := tabwriter.NewWriter(sb, 1, 1, 1, ' ', 0) + show := func(v1, v2 any) { + _, _ = fmt.Fprintf(w, "%v\t\t%v", v1, v2) + } + + for _, status := range pod.Status.ContainerStatuses { + if status.State.Waiting != nil { + show(status.State.Waiting.Reason, status.State.Waiting.Message) + } + if status.State.Running != nil { + show("ContainerRunning", "") + } + if status.State.Terminated != nil { + show(status.State.Terminated.Reason, status.State.Terminated.Message) + } + } + _ = w.Flush() + return sb.String() +} + func max[T constraints.Ordered](a T, b T) T { if a > b { return a diff --git a/pkg/util/util.go b/pkg/util/util.go index c1cdab60..8b4617ce 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -45,6 +45,7 @@ import ( "k8s.io/kubectl/pkg/cmd/util" cmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/polymorphichelpers" + "k8s.io/kubectl/pkg/util/podutils" "github.com/wencaiwulue/kubevpn/pkg/config" ) @@ -76,7 +77,7 @@ func GetAvailableTCPPortOrDie() int { } func WaitPod(podInterface v12.PodInterface, list metav1.ListOptions, checker func(*v1.Pod) bool) error { - ctx, cancelFunc := context.WithTimeout(context.Background(), time.Second*10) + ctx, cancelFunc := context.WithTimeout(context.Background(), time.Second*60) defer cancelFunc() w, err := podInterface.Watch(ctx, list) if err != nil { @@ -150,12 +151,12 @@ func GetTopOwnerReference(factory cmdutil.Factory, namespace, workload string) ( } // GetTopOwnerReferenceBySelector assume pods, controller has same labels -func GetTopOwnerReferenceBySelector(factory cmdutil.Factory, namespace, selector string) (sets.String, error) { +func GetTopOwnerReferenceBySelector(factory cmdutil.Factory, namespace, selector string) (sets.Set[string], error) { object, err := GetUnstructuredObjectBySelector(factory, namespace, selector) if err != nil { return nil, err } - set := sets.NewString() + set := sets.New[string]() for _, info := range object { reference, err := GetTopOwnerReference(factory, namespace, fmt.Sprintf("%s/%s", info.Mapping.Resource.GroupResource().String(), info.Name)) if err == nil && reference.Mapping.Resource.Resource != "services" { @@ -594,3 +595,27 @@ func IsIPv4(packet []byte) bool { func IsIPv6(packet []byte) bool { return 6 == (packet[0] >> 4) } + +func Deduplicate(cidr []*net.IPNet) (result []*net.IPNet) { + var set = sets.New[string]() + for _, ipNet := range cidr { + if !set.Has(ipNet.String()) { + result = append(result, ipNet) + } + set.Insert(ipNet.String()) + } + return +} + +func AllContainerIsRunning(pod *v1.Pod) bool { + isReady := podutils.IsPodReady(pod) + if !isReady { + return false + } + for _, status := range pod.Status.ContainerStatuses { + if !status.Ready { + return false + } + } + return true +} diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index b11386e1..602d81cc 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -1,9 +1,12 @@ package util import ( - "fmt" - "regexp" + "encoding/json" "testing" + + "github.com/containernetworking/cni/libcni" + log "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) func TestName(t *testing.T) { @@ -41,8 +44,16 @@ func TestName(t *testing.T) { ` // IPv6 with CIDR - compile := regexp.MustCompile(`(([0-9]{1,3}\.){3}[0-9]{1,3}/[0-9]{1,})`) - v6 := regexp.MustCompile(`(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))/[0-9]{1,}`) - fmt.Println(compile.FindAllString(s, -1)) - fmt.Println(v6.FindAllString(s, -1)) + configList, err := libcni.ConfListFromBytes([]byte(s)) + if err == nil { + log.Infoln("get cni config", configList.Name) + } + for _, plugin := range configList.Plugins { + var m map[string]interface{} + _ = json.Unmarshal(plugin.Bytes, &m) + slice, _, _ := unstructured.NestedStringSlice(m, "ipam", "ipv4_pools") + for _, i := range slice { + println(i) + } + } }