From 2227a82125b94aa809292d4b0f2e0a292eb352e7 Mon Sep 17 00:00:00 2001 From: wencaiwulue <895703375@qq.com> Date: Fri, 31 Mar 2023 22:25:43 +0800 Subject: [PATCH] feat: support ipv6 --- .github/release-note.sh | 5 +- .github/workflows/release.yml | 3 +- cmd/kubevpn/cmds/controlplane.go | 7 +- cmd/kubevpn/cmds/root.go | 6 +- cmd/kubevpn/cmds/serve.go | 11 +-- cmd/kubevpn/cmds/webhook.go | 5 +- pkg/config/config.go | 23 +++-- pkg/controlplane/cache.go | 17 ++-- pkg/core/chain.go | 2 +- pkg/core/route.go | 3 + pkg/core/tcphandler.go | 2 +- pkg/core/tunhandler.go | 94 ++++++++++++++++---- pkg/exchange/controller.go | 42 ++++++--- pkg/handler/cleaner.go | 4 +- pkg/handler/connect.go | 111 +++++++++-------------- pkg/handler/dhcp.go | 116 ++++++++++++++----------- pkg/handler/envoy.go | 14 +-- pkg/handler/remote.go | 62 ++++++++----- pkg/handler/serve.go | 25 +++--- pkg/mesh/controller.go | 33 +++++-- pkg/mesh/envoy.yaml | 10 ++- pkg/test/local.go | 43 +++++++++ pkg/test/pod.yaml | 32 +++++++ pkg/test/run.sh | 10 +++ pkg/test/server/server.go | 39 +++++++++ pkg/tun/tun.go | 6 +- pkg/tun/{tun_freebsd.go => tun_bsd.go} | 50 ++++++++--- pkg/tun/tun_darwin.go | 52 +++++++---- pkg/tun/tun_linux.go | 63 ++++++++------ pkg/tun/tun_openbsd.go | 86 ------------------ pkg/tun/tun_windows.go | 78 ++++++++++++----- pkg/util/cidr.go | 35 ++++++-- pkg/util/getcidr.go | 32 +++++-- pkg/util/pod.go | 22 +++++ pkg/util/route_config.go | 7 -- pkg/webhook/dhcp.go | 35 ++++---- pkg/webhook/pods.go | 53 +++++------ plugins/stable.txt | 1 + 38 files changed, 779 insertions(+), 460 deletions(-) create mode 100644 pkg/test/local.go create mode 100644 pkg/test/pod.yaml create mode 100644 pkg/test/run.sh create mode 100644 pkg/test/server/server.go rename pkg/tun/{tun_freebsd.go => tun_bsd.go} (51%) delete mode 100644 pkg/tun/tun_openbsd.go delete mode 100644 pkg/util/route_config.go create mode 100644 plugins/stable.txt diff --git a/.github/release-note.sh b/.github/release-note.sh index b19f618c..4c2aea2b 100755 --- a/.github/release-note.sh +++ b/.github/release-note.sh @@ -8,9 +8,10 @@ CHANGELOG=$(git log --no-merges --date=short --pretty=format:'- %h %an %ad %s' " cat < plugins/stable.txt - name: Create Pull Request id: cpr uses: peter-evans/create-pull-request@v4 with: add-paths: | *.yaml + plugins/stable.txt token: ${{ secrets.REPOSITORYDISPATCH }} commit-message: "feat: update krew index version to ${{ github.ref }}" committer: GitHub diff --git a/cmd/kubevpn/cmds/controlplane.go b/cmd/kubevpn/cmds/controlplane.go index 80e20f34..5f27e200 100644 --- a/cmd/kubevpn/cmds/controlplane.go +++ b/cmd/kubevpn/cmds/controlplane.go @@ -16,9 +16,10 @@ func CmdControlPlane(_ cmdutil.Factory) *cobra.Command { port uint = 9002 ) cmd := &cobra.Command{ - Use: "control-plane", - Short: "Control-plane is a envoy xds server", - Long: `Control-plane is a envoy xds server, distribute envoy route configuration`, + Use: "control-plane", + Hidden: true, + Short: "Control-plane is a envoy xds server", + Long: `Control-plane is a envoy xds server, distribute envoy route configuration`, Run: func(cmd *cobra.Command, args []string) { util.InitLogger(config.Debug) controlplane.Main(watchDirectoryFilename, port, log.StandardLogger()) diff --git a/cmd/kubevpn/cmds/root.go b/cmd/kubevpn/cmds/root.go index 1746740d..ee6fa8b7 100644 --- a/cmd/kubevpn/cmds/root.go +++ b/cmd/kubevpn/cmds/root.go @@ -39,11 +39,7 @@ func NewKubeVPNCommand() *cobra.Command { CmdUpgrade(factory), CmdReset(factory), CmdVersion(factory), - }, - }, - { - Message: "Server Commands (DO NOT USE IT !!!):", - Commands: []*cobra.Command{ + // Hidden, Server Commands (DO NOT USE IT !!!) CmdControlPlane(factory), CmdServe(factory), CmdWebhook(factory), diff --git a/cmd/kubevpn/cmds/serve.go b/cmd/kubevpn/cmds/serve.go index b129a732..86a23c45 100644 --- a/cmd/kubevpn/cmds/serve.go +++ b/cmd/kubevpn/cmds/serve.go @@ -23,9 +23,10 @@ import ( func CmdServe(factory cmdutil.Factory) *cobra.Command { var route = &core.Route{} cmd := &cobra.Command{ - Use: "serve", - Short: "Server side, startup traffic manager, forward inbound and outbound traffic", - Long: `Server side, startup traffic manager, forward inbound and outbound traffic.`, + Use: "serve", + Hidden: true, + Short: "Server side, startup traffic manager, forward inbound and outbound traffic", + Long: `Server side, startup traffic manager, forward inbound and outbound traffic.`, PreRun: func(*cobra.Command, []string) { util.InitLogger(config.Debug) go func() { log.Info(http.ListenAndServe("localhost:6060", nil)) }() @@ -48,10 +49,10 @@ func CmdServe(factory cmdutil.Factory) *cobra.Command { if err != nil { return err } + //go util.Heartbeats() <-ctx.Done() - return nil + return handler.Final() }, - PostRunE: func(cmd *cobra.Command, args []string) error { return handler.Final() }, } cmd.Flags().StringArrayVarP(&route.ServeNodes, "nodeCommand", "L", []string{}, "command needs to be executed") cmd.Flags().StringVarP(&route.ChainNode, "chainCommand", "F", "", "command needs to be executed") diff --git a/cmd/kubevpn/cmds/webhook.go b/cmd/kubevpn/cmds/webhook.go index e6b9040d..86a629c1 100644 --- a/cmd/kubevpn/cmds/webhook.go +++ b/cmd/kubevpn/cmds/webhook.go @@ -10,8 +10,9 @@ import ( func CmdWebhook(f cmdutil.Factory) *cobra.Command { cmd := &cobra.Command{ - Use: "webhook", - Short: "Starts a HTTP server, useful for creating MutatingAdmissionWebhook", + Use: "webhook", + Hidden: true, + Short: "Starts a HTTP server, useful for creating MutatingAdmissionWebhook", Long: `Starts a HTTP server, useful for creating MutatingAdmissionWebhook. After deploying it to Kubernetes cluster, the Administrator needs to create a MutatingWebhookConfiguration in the Kubernetes cluster to register remote webhook admission controllers.`, diff --git a/pkg/config/config.go b/pkg/config/config.go index 3339ad72..faafe63f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -14,6 +14,7 @@ const ( // config map keys KeyDHCP = "DHCP" + KeyDHCP6 = "DHCP6" KeyEnvoy = "ENVOY_CONFIG" KeyClusterIPv4POOLS = "IPv4_POOLS" KeyRefCount = "REF_COUNT" @@ -49,6 +50,9 @@ const ( // 如果不创建 network，那么是无法请求到 这个 kubernetes 的 service 的 dockerInnerIPv4Pool = "223.255.0.100/16" + //The IPv6 address prefixes FE80::/10 and FF02::/16 are not routable + innerIPv6Pool = "efff:ffff:ffff:ffff:ffff:ffff:ffff:9999/64" + DefaultNetDir = "/etc/cni/net.d" Proc = "/proc" @@ -56,15 +60,17 @@ const ( CniNetName = "cni-net-dir-kubevpn" // env name - EnvTunNameOrLUID = "TunNameOrLUID" - EnvInboundPodTunIP = "InboundPodTunIP" - EnvPodName = "POD_NAME" - EnvPodNamespace = "POD_NAMESPACE" + EnvTunNameOrLUID = "TunNameOrLUID" + EnvInboundPodTunIPv4 = "TunIPv4" + EnvInboundPodTunIPv6 = "TunIPv6" + EnvPodName = "POD_NAME" + EnvPodNamespace = "POD_NAMESPACE" // header name HeaderPodName = "POD_NAME" HeaderPodNamespace = "POD_NAMESPACE" - HeaderIP = "IP" + HeaderIPv4 = "IPv4" + HeaderIPv6 = "IPv6" // api APIRentIP = "/rent/ip" @@ -88,8 +94,10 @@ var ( ) var ( - CIDR *net.IPNet - RouterIP net.IP + CIDR *net.IPNet + CIDR6 *net.IPNet + RouterIP net.IP + RouterIP6 net.IP // for creating docker network DockerCIDR *net.IPNet @@ -98,6 +106,7 @@ var ( func init() { RouterIP, CIDR, _ = net.ParseCIDR(innerIPv4Pool) + RouterIP6, CIDR6, _ = net.ParseCIDR(innerIPv6Pool) DockerRouterIP, DockerCIDR, _ = net.ParseCIDR(dockerInnerIPv4Pool) } diff --git a/pkg/controlplane/cache.go b/pkg/controlplane/cache.go index c4a56c3a..5d500a93 100644 --- a/pkg/controlplane/cache.go +++ b/pkg/controlplane/cache.go @@ -34,8 +34,9 @@ type Virtual struct { } type Rule struct { - Headers map[string]string - LocalTunIP string + Headers map[string]string + LocalTunIPv4 string + LocalTunIPv6 string } func (a *Virtual) To() ( @@ -52,10 +53,12 @@ func (a *Virtual) To() ( var rr []*route.Route for _, rule := range a.Rules { - clusterName := fmt.Sprintf("%s_%v", rule.LocalTunIP, port.ContainerPort) - clusters = append(clusters, ToCluster(clusterName)) - endpoints = append(endpoints, ToEndPoint(clusterName, rule.LocalTunIP, port.ContainerPort)) - rr = append(rr, ToRoute(clusterName, rule.Headers)) + for _, ip := range []string{rule.LocalTunIPv4, rule.LocalTunIPv6} { + clusterName := fmt.Sprintf("%s_%v", ip, port.ContainerPort) + clusters = append(clusters, ToCluster(clusterName)) + endpoints = append(endpoints, ToEndPoint(clusterName, ip, port.ContainerPort)) + rr = append(rr, ToRoute(clusterName, rule.Headers)) + } } rr = append(rr, DefaultRoute()) routes = append(routes, &route.RouteConfiguration{ @@ -122,7 +125,7 @@ func ToCluster(clusterName string) *cluster.Cluster { }, }), }, - DnsLookupFamily: cluster.Cluster_V4_ONLY, + DnsLookupFamily: cluster.Cluster_ALL, } } diff --git a/pkg/core/chain.go b/pkg/core/chain.go index 6328b4b2..1e88cd02 100644 --- a/pkg/core/chain.go +++ b/pkg/core/chain.go @@ -71,7 +71,7 @@ func (c *Chain) getConn(ctx context.Context) (net.Conn, error) { if c.IsEmpty() { return nil, ErrorEmptyChain } - return c.Node().Client.Dial(ctx, c.Node().Addr) + return c.Node().Client.Dial(ctx, c.resolve(c.Node().Addr)) } type Handler interface { diff --git a/pkg/core/route.go b/pkg/core/route.go index 2debd482..b16f7fd5 100644 --- a/pkg/core/route.go +++ b/pkg/core/route.go @@ -2,11 +2,13 @@ package core import ( "net" + "os" "strings" "github.com/containernetworking/cni/pkg/types" "github.com/pkg/errors" + "github.com/wencaiwulue/kubevpn/pkg/config" "github.com/wencaiwulue/kubevpn/pkg/tun" ) @@ -69,6 +71,7 @@ func (r *Route) GenerateServers() ([]Server, error) { ln, err = tun.Listener(tun.Config{ Name: node.Get("name"), Addr: node.Get("net"), + Addr6: os.Getenv(config.EnvInboundPodTunIPv6), MTU: node.GetInt("mtu"), Routes: parseIPRoutes(node.Get("route")), Gateway: node.Get("gw"), diff --git a/pkg/core/tcphandler.go b/pkg/core/tcphandler.go index 57d236dd..e5fc9f54 100644 --- a/pkg/core/tcphandler.go +++ b/pkg/core/tcphandler.go @@ -48,7 +48,7 @@ func TCPHandler() Handler { } } -var Server8422, _ = net.ResolveUDPAddr("udp", "127.0.0.1:8422") +var Server8422, _ = net.ResolveUDPAddr("udp", "localhost:8422") func (h *fakeUdpHandler) Handle(ctx context.Context, tcpConn net.Conn) { defer tcpConn.Close() diff --git a/pkg/core/tunhandler.go b/pkg/core/tunhandler.go index 839f1481..bd7d7d08 100644 --- a/pkg/core/tunhandler.go +++ b/pkg/core/tunhandler.go @@ -15,6 +15,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/wencaiwulue/kubevpn/pkg/config" + pkgtun "github.com/wencaiwulue/kubevpn/pkg/tun" "github.com/wencaiwulue/kubevpn/pkg/util" ) @@ -236,37 +237,78 @@ func (d *Device) Close() { } func (d *Device) heartbeats() { - src := d.tun.LocalAddr().(*net.IPAddr).IP.To4() - dst := config.RouterIP.To4() - if dst.Equal(src) { + tunIface, err := pkgtun.GetInterface() + if err != nil { + return + } + addrs, err := tunIface.Addrs() + if err != nil { + return + } + var srcIPv4, srcIPv6 net.IP + for _, addr := range addrs { + ip, cidr, err := net.ParseCIDR(addr.String()) + if err != nil { + continue + } + if cidr.Contains(config.RouterIP) { + srcIPv4 = ip + } + if cidr.Contains(config.RouterIP6) { + srcIPv6 = ip + } + } + if srcIPv4 == nil || srcIPv6 == nil { + return + } + if config.RouterIP.To4().Equal(srcIPv4) { + return + } + if config.RouterIP6.To4().Equal(srcIPv6) { return } var bytes []byte - var err error + var bytes6 []byte - ticker := time.NewTicker(time.Second * 15) + ticker := time.NewTicker(time.Second * 5) defer ticker.Stop() for ; true; <-ticker.C { for i := 0; i < 4; i++ { if bytes == nil { - bytes, err = genICMPPacket(src, dst) + bytes, err = genICMPPacket(srcIPv4, config.RouterIP) if err != nil { log.Error(err) continue } } - data := config.LPool.Get().([]byte)[:] - length := copy(data, bytes) - if d.closed.Load() { - return + if bytes6 == nil { + bytes6, err = genICMPPacketIPv6(srcIPv6, config.RouterIP6) + if err != nil { + log.Error(err) + continue + } } - d.tunInbound <- &DataElem{ - data: data, - length: length, - src: src, - dst: dst, + for index, i2 := range [][]byte{bytes, bytes6} { + if d.closed.Load() { + return + } + + data := config.LPool.Get().([]byte)[:] + length := copy(data, i2) + var src, dst net.IP + if index == 0 { + src, dst = srcIPv4, config.RouterIP + } else { + src, dst = srcIPv6, config.RouterIP6 + } + d.tunInbound <- &DataElem{ + data: data, + length: length, + src: src, + dst: dst, + } } time.Sleep(time.Second) } @@ -301,6 +343,28 @@ func genICMPPacket(src net.IP, dst net.IP) ([]byte, error) { return buf.Bytes(), nil } +func genICMPPacketIPv6(src net.IP, dst net.IP) ([]byte, error) { + buf := gopacket.NewSerializeBuffer() + icmpLayer := layers.ICMPv6{ + TypeCode: layers.CreateICMPv6TypeCode(layers.ICMPv6TypeEchoRequest, 0), + } + ipLayer := layers.IPv6{ + Version: 6, + SrcIP: src, + DstIP: dst, + NextHeader: layers.IPProtocolICMPv6, + HopLimit: 255, + } + opts := gopacket.SerializeOptions{ + FixLengths: true, + } + err := gopacket.SerializeLayers(buf, opts, &ipLayer, &icmpLayer) + if err != nil { + return nil, fmt.Errorf("failed to serialize icmp6 packet, err: %v", err) + } + return buf.Bytes(), nil +} + func (d *Device) Start() { go d.readFromTun() for i := 0; i < d.thread; i++ { diff --git a/pkg/exchange/controller.go b/pkg/exchange/controller.go index 83beb7d4..d3528c63 100644 --- a/pkg/exchange/controller.go +++ b/pkg/exchange/controller.go @@ -33,35 +33,55 @@ func AddContainer(spec *corev1.PodSpec, c util.PodRouteConfig) { }}, Env: []corev1.EnvVar{ { - Name: "LocalTunIP", - Value: c.LocalTunIP, + Name: "LocalTunIPv4", + Value: c.LocalTunIPv4, }, { - Name: "TrafficManagerRealIP", - Value: c.TrafficManagerRealIP, + Name: "LocalTunIPv6", + Value: c.LocalTunIPv6, }, { - Name: config.EnvInboundPodTunIP, - Value: c.InboundPodTunIP, + Name: config.EnvInboundPodTunIPv4, + Value: "", }, { - Name: "CIDR", + Name: config.EnvInboundPodTunIPv6, + Value: "", + }, + { + Name: "CIDR4", Value: config.CIDR.String(), }, + { + Name: "CIDR6", + Value: config.CIDR6.String(), + }, + { + Name: "TrafficManagerService", + Value: config.ConfigMapPodTrafficManager, + }, }, Command: []string{"/bin/sh", "-c"}, // https://www.netfilter.org/documentation/HOWTO/NAT-HOWTO-6.html#ss6.2 Args: []string{` -sysctl net.ipv4.ip_forward=1 +sysctl -w net.ipv4.ip_forward=1 +sysctl -w net.ipv6.conf.all.forwarding=1 sysctl -w net.ipv4.conf.all.route_localnet=1 update-alternatives --set iptables /usr/sbin/iptables-legacy iptables -F +ip6tables -F iptables -P INPUT ACCEPT +ip6tables -P INPUT ACCEPT iptables -P FORWARD ACCEPT -iptables -t nat -A PREROUTING ! -p icmp -j DNAT --to ${LocalTunIP} +ip6tables -P FORWARD ACCEPT +iptables -t nat -A PREROUTING ! -p icmp -j DNAT --to ${LocalTunIPv4} +ip6tables -t nat -A PREROUTING ! -p icmp -j DNAT --to ${LocalTunIPv6} iptables -t nat -A POSTROUTING ! -p icmp -j MASQUERADE -iptables -t nat -A OUTPUT -o lo ! -p icmp -j DNAT --to-destination ${LocalTunIP} -kubevpn serve -L "tun:/127.0.0.1:8422?net=${InboundPodTunIP}&route=${CIDR}" -F "tcp://${TrafficManagerRealIP}:10800"`, +ip6tables -t nat -A POSTROUTING ! -p icmp -j MASQUERADE +# for curl -g -6 [efff:ffff:ffff:ffff:ffff:ffff:ffff:999a]:9080/health or curl 127.0.0.1:9080/health hit local PC +iptables -t nat -A OUTPUT -o lo ! -p icmp -j DNAT --to-destination ${LocalTunIPv4} +ip6tables -t nat -A OUTPUT -o lo ! -p icmp -j DNAT --to-destination ${LocalTunIPv6} +kubevpn serve -L "tun:/127.0.0.1:8422?net=${TunIPv4}&route=${CIDR4}" -F "tcp://${TrafficManagerService}:10800"`, }, SecurityContext: &corev1.SecurityContext{ Capabilities: &corev1.Capabilities{ diff --git a/pkg/handler/cleaner.go b/pkg/handler/cleaner.go index ca0f72c3..735d20cb 100644 --- a/pkg/handler/cleaner.go +++ b/pkg/handler/cleaner.go @@ -32,7 +32,7 @@ func (c *ConnectOptions) addCleanUpResourceHandler() { go func() { <-stopChan log.Info("prepare to exit, cleaning up") - err := c.dhcp.ReleaseIpToDHCP(append(c.usedIPs, c.localTunIP)...) + err := c.dhcp.ReleaseIpToDHCP(c.localTunIPv4.IP, c.localTunIPv6.IP) if err != nil { log.Errorf("failed to release ip to dhcp, err: %v", err) } @@ -133,7 +133,7 @@ func cleanup(clientset *kubernetes.Clientset, namespace, name string, keepCidr b if keepCidr { // keep configmap - p := []byte(fmt.Sprintf(`[{"op": "remove", "path": "/data/%s"}]`, config.KeyDHCP)) + p := []byte(fmt.Sprintf(`[{"op": "remove", "path": "/data/%s"},{"op": "remove", "path": "/data/%s"}]`, config.KeyDHCP, config.KeyDHCP6)) _, _ = clientset.CoreV1().ConfigMaps(namespace).Patch(context.Background(), name, types.JSONPatchType, p, v1.PatchOptions{}) p = []byte(fmt.Sprintf(`{"data":{"%s":"%s"}}`, config.KeyRefCount, strconv.Itoa(0))) _, _ = clientset.CoreV1().ConfigMaps(namespace).Patch(context.Background(), name, types.MergePatchType, p, v1.PatchOptions{}) diff --git a/pkg/handler/connect.go b/pkg/handler/connect.go index 48341cf9..b794cf97 100644 --- a/pkg/handler/connect.go +++ b/pkg/handler/connect.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "math/rand" "net" "net/netip" "net/url" @@ -74,21 +73,20 @@ type ConnectOptions struct { cidrs []*net.IPNet dhcp *DHCPManager // needs to give it back to dhcp - usedIPs []*net.IPNet - routerIP net.IP - localTunIP *net.IPNet + localTunIPv4 *net.IPNet + localTunIPv6 *net.IPNet } func (c *ConnectOptions) createRemoteInboundPod(ctx1 context.Context) (err error) { - c.localTunIP, err = c.dhcp.RentIPBaseNICAddress() + c.localTunIPv4, c.localTunIPv6, err = c.dhcp.RentIPBaseNICAddress() if err != nil { return } for _, workload := range c.Workloads { configInfo := util.PodRouteConfig{ - LocalTunIP: c.localTunIP.IP.String(), - TrafficManagerRealIP: c.routerIP.String(), + LocalTunIPv4: c.localTunIPv4.IP.String(), + LocalTunIPv6: c.localTunIPv6.IP.String(), } // means mesh mode if len(c.Headers) != 0 { @@ -128,52 +126,44 @@ func Rollback(f cmdutil.Factory, ns, workload string) { } func (c *ConnectOptions) DoConnect() (err error) { - trafficMangerNet := net.IPNet{IP: config.RouterIP, Mask: config.CIDR.Mask} - c.dhcp = NewDHCPManager(c.clientset.CoreV1().ConfigMaps(c.Namespace), c.Namespace, &trafficMangerNet) + c.dhcp = NewDHCPManager(c.clientset.CoreV1().ConfigMaps(c.Namespace), c.Namespace) if err = c.dhcp.InitDHCP(ctx); err != nil { return } c.addCleanUpResourceHandler() - err = c.GetCIDR(ctx) - if err != nil { + if err = c.getCIDR(ctx); err != nil { return } - c.routerIP, err = CreateOutboundPod(ctx, c.factory, c.clientset, c.Namespace, trafficMangerNet.String()) - if err != nil { + if err = createOutboundPod(ctx, c.factory, c.clientset, c.Namespace); err != nil { return } - err = c.SetImage(ctx) - if err != nil { - return err + if err = c.setImage(ctx); err != nil { + return } if err = c.createRemoteInboundPod(ctx); err != nil { return } port := util.GetAvailableTCPPortOrDie() - err = c.portForward(ctx, fmt.Sprintf("%d:10800", port)) - if err != nil { - return err + if err = c.portForward(ctx, fmt.Sprintf("%d:10800", port)); err != nil { + return } if util.IsWindows() { driver.InstallWireGuardTunDriver() } - err = c.startLocalTunServe(ctx, fmt.Sprintf("tcp://127.0.0.1:%d", port)) - if err != nil { - return err + forward := fmt.Sprintf("tcp://127.0.0.1:%d", port) + if err = c.startLocalTunServe(ctx, forward); err != nil { + return } - err = c.addRouteDynamic(ctx) - if err != nil { - return err + if err = c.addRouteDynamic(ctx); err != nil { + return } c.deleteFirewallRule(ctx) - err = c.setupDNS() - if err != nil { - return err + if err = c.setupDNS(); err != nil { + return } - go c.heartbeats() - err = c.addExtraRoute(ctx) - if err != nil { - return err + //go util.Heartbeats() + if err = c.addExtraRoute(ctx); err != nil { + return } log.Info("dns service ok") return @@ -276,7 +266,7 @@ func checkPodStatus(cCtx context.Context, cFunc context.CancelFunc, podName stri func (c *ConnectOptions) startLocalTunServe(ctx context.Context, forwardAddress string) (err error) { // todo figure it out why if util.IsWindows() { - c.localTunIP.Mask = net.CIDRMask(0, 32) + c.localTunIPv4.Mask = net.CIDRMask(0, 32) } var list = sets.New[string](config.CIDR.String()) for _, ipNet := range c.cidrs { @@ -290,15 +280,19 @@ func (c *ConnectOptions) startLocalTunServe(ctx context.Context, forwardAddress } list.Insert(s) } + if err = os.Setenv(config.EnvInboundPodTunIPv6, c.localTunIPv6.String()); err != nil { + return err + } + r := core.Route{ ServeNodes: []string{ - fmt.Sprintf("tun:/127.0.0.1:8422?net=%s&route=%s", c.localTunIP.String(), strings.Join(list.UnsortedList(), ",")), + fmt.Sprintf("tun:/127.0.0.1:8422?net=%s&route=%s", c.localTunIPv4.String(), strings.Join(list.UnsortedList(), ",")), }, ChainNode: forwardAddress, Retries: 5, } - log.Debugf("your ip is %s", c.localTunIP.IP.String()) + log.Debugf("ipv4: %s, ipv6: %s", c.localTunIPv4.IP.String(), c.localTunIPv6.IP.String()) if err = Start(ctx, r); err != nil { log.Errorf("error while create tunnel, err: %v", errors.WithStack(err)) } else { @@ -527,16 +521,22 @@ func Start(ctx context.Context, r core.Route) error { return errors.WithStack(err) } if len(servers) == 0 { - return errors.New("invalid config") + return fmt.Errorf("server is empty, server config: %s", strings.Join(r.ServeNodes, ",")) } for _, server := range servers { go func(ctx context.Context, server core.Server) { l := server.Listener - defer l.Close() for { - conn, err := l.Accept() - if err != nil { - log.Warnf("server: accept error: %v", err) + select { + case <-ctx.Done(): + _ = l.Close() + return + default: + } + + conn, errs := l.Accept() + if errs != nil { + log.Debugf("server accept connect error: %v", errs) continue } go server.Handler.Handle(ctx, conn) @@ -729,14 +729,14 @@ func (c *ConnectOptions) GetRunningPodList() ([]v1.Pod, error) { return list.Items, nil } -// GetCIDR +// getCIDR // 1: get pod cidr // 2: get service cidr // todo optimize code should distinguish service cidr and pod cidr // https://stackoverflow.com/questions/45903123/kubernetes-set-service-cidr-and-pod-cidr-the-same // https://stackoverflow.com/questions/44190607/how-do-you-find-the-cluster-service-cidr-of-a-kubernetes-cluster/54183373#54183373 // https://stackoverflow.com/questions/44190607/how-do-you-find-the-cluster-service-cidr-of-a-kubernetes-cluster -func (c *ConnectOptions) GetCIDR(ctx context.Context) (err error) { +func (c *ConnectOptions) getCIDR(ctx context.Context) (err error) { // (1) get cidr from cache var value string value, err = c.dhcp.Get(ctx, config.KeyClusterIPv4POOLS) @@ -774,31 +774,6 @@ func (c *ConnectOptions) GetCIDR(ctx context.Context) (err error) { return } -func (c *ConnectOptions) heartbeats() { - ticker := time.NewTicker(time.Second * 30) - defer ticker.Stop() - - for ; true; <-ticker.C { - func() { - defer func() { - if err := recover(); err != nil { - log.Debug(err) - } - }() - - err := c.dhcp.ForEach(func(ip net.IP) { - go func() { - time.Sleep(time.Millisecond * time.Duration(rand.Intn(1000))) - _, _ = util.Ping(ip.String()) - }() - }) - if err != nil { - log.Debug(err) - } - }() - } -} - func (c *ConnectOptions) addExtraRoute(ctx context.Context) (err error) { if len(c.ExtraDomain) == 0 { return @@ -971,7 +946,7 @@ func (c *ConnectOptions) UpdateImage(ctx context.Context) error { return err } -func (c *ConnectOptions) SetImage(ctx context.Context) error { +func (c *ConnectOptions) setImage(ctx context.Context) error { deployment, err := c.clientset.AppsV1().Deployments(c.Namespace).Get(ctx, config.ConfigMapPodTrafficManager, metav1.GetOptions{}) if err != nil { return err diff --git a/pkg/handler/dhcp.go b/pkg/handler/dhcp.go index 78c76bdf..0777f3e4 100644 --- a/pkg/handler/dhcp.go +++ b/pkg/handler/dhcp.go @@ -20,14 +20,16 @@ import ( type DHCPManager struct { client corev1.ConfigMapInterface cidr *net.IPNet + cidr6 *net.IPNet namespace string } -func NewDHCPManager(client corev1.ConfigMapInterface, namespace string, cidr *net.IPNet) *DHCPManager { +func NewDHCPManager(client corev1.ConfigMapInterface, namespace string) *DHCPManager { return &DHCPManager{ client: client, namespace: namespace, - cidr: cidr, + cidr: &net.IPNet{IP: config.RouterIP, Mask: config.CIDR.Mask}, + cidr6: &net.IPNet{IP: config.RouterIP6, Mask: config.CIDR6.Mask}, } } @@ -67,35 +69,51 @@ func (d *DHCPManager) InitDHCP(ctx context.Context) error { return nil } -func (d *DHCPManager) RentIPBaseNICAddress() (*net.IPNet, error) { - var ip net.IP - err := d.updateDHCPConfigMap(func(allocator *ipallocator.Range) (err error) { - ip, err = allocator.AllocateNext() +func (d *DHCPManager) RentIPBaseNICAddress() (*net.IPNet, *net.IPNet, error) { + var v4, v6 net.IP + err := d.updateDHCPConfigMap(func(ipv4 *ipallocator.Range, ipv6 *ipallocator.Range) (err error) { + if v4, err = ipv4.AllocateNext(); err != nil { + return err + } + if v6, err = ipv6.AllocateNext(); err != nil { + return err + } return }) if err != nil { - return nil, err + return nil, nil, err } - return &net.IPNet{IP: ip, Mask: d.cidr.Mask}, nil + return &net.IPNet{IP: v4, Mask: d.cidr.Mask}, &net.IPNet{IP: v6, Mask: d.cidr6.Mask}, nil } -func (d *DHCPManager) RentIPRandom() (*net.IPNet, error) { - var ip net.IP - err := d.updateDHCPConfigMap(func(dhcp *ipallocator.Range) (err error) { - ip, err = dhcp.AllocateNext() +func (d *DHCPManager) RentIPRandom() (*net.IPNet, *net.IPNet, error) { + var v4, v6 net.IP + err := d.updateDHCPConfigMap(func(ipv4 *ipallocator.Range, ipv6 *ipallocator.Range) (err error) { + if v4, err = ipv4.AllocateNext(); err != nil { + return err + } + if v6, err = ipv6.AllocateNext(); err != nil { + return err + } return }) if err != nil { log.Errorf("failed to rent ip from DHCP server, err: %v", err) - return nil, err + return nil, nil, err } - return &net.IPNet{IP: ip, Mask: d.cidr.Mask}, nil + return &net.IPNet{IP: v4, Mask: d.cidr.Mask}, &net.IPNet{IP: v6, Mask: d.cidr6.Mask}, nil } -func (d *DHCPManager) ReleaseIpToDHCP(ips ...*net.IPNet) error { - return d.updateDHCPConfigMap(func(r *ipallocator.Range) error { +func (d *DHCPManager) ReleaseIpToDHCP(ips ...net.IP) error { + return d.updateDHCPConfigMap(func(ipv4 *ipallocator.Range, ipv6 *ipallocator.Range) error { for _, ip := range ips { - if err := r.Release(ip.IP); err != nil { + var use *ipallocator.Range + if ip.To4() != nil { + use = ipv4 + } else { + use = ipv6 + } + if err := use.Release(ip); err != nil { return err } } @@ -103,7 +121,7 @@ func (d *DHCPManager) ReleaseIpToDHCP(ips ...*net.IPNet) error { }) } -func (d *DHCPManager) updateDHCPConfigMap(f func(*ipallocator.Range) error) error { +func (d *DHCPManager) updateDHCPConfigMap(f func(ipv4 *ipallocator.Range, ipv6 *ipallocator.Range) error) error { cm, err := d.client.Get(context.Background(), config.ConfigMapPodTrafficManager, metav1.GetOptions{}) if err != nil { return fmt.Errorf("failed to get cm DHCP server, err: %v", err) @@ -124,18 +142,40 @@ func (d *DHCPManager) updateDHCPConfigMap(f func(*ipallocator.Range) error) erro return err } } - if err = f(dhcp); err != nil { - return err - } - _, bytes, err := dhcp.Snapshot() + + dhcp6, err := ipallocator.NewAllocatorCIDRRange(d.cidr6, func(max int, rangeSpec string) (allocator.Interface, error) { + return allocator.NewContiguousAllocationMap(max, rangeSpec), nil + }) if err != nil { return err } - cm.Data[config.KeyDHCP] = base64.StdEncoding.EncodeToString(bytes) + str, err = base64.StdEncoding.DecodeString(cm.Data[config.KeyDHCP6]) + if err == nil { + err = dhcp6.Restore(d.cidr6, str) + if err != nil { + return err + } + } + if err = f(dhcp, dhcp6); err != nil { + return err + } + + for index, i := range []*ipallocator.Range{dhcp, dhcp6} { + var bytes []byte + if _, bytes, err = i.Snapshot(); err != nil { + return err + } + var key string + if index == 0 { + key = config.KeyDHCP + } else { + key = config.KeyDHCP6 + } + cm.Data[key] = base64.StdEncoding.EncodeToString(bytes) + } _, err = d.client.Update(context.Background(), cm, metav1.UpdateOptions{}) if err != nil { - log.Errorf("update dhcp failed, err: %v", err) - return err + return fmt.Errorf("update dhcp failed, err: %v", err) } return nil } @@ -170,29 +210,3 @@ func (d *DHCPManager) Get(ctx2 context.Context, key string) (string, error) { } return "", fmt.Errorf("can not get data") } - -func (d *DHCPManager) ForEach(fn func(net.IP)) error { - cm, err := d.client.Get(context.Background(), config.ConfigMapPodTrafficManager, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("failed to get cm DHCP server, err: %v", err) - } - if cm.Data == nil { - cm.Data = make(map[string]string) - } - dhcp, err := ipallocator.NewAllocatorCIDRRange(d.cidr, func(max int, rangeSpec string) (allocator.Interface, error) { - return allocator.NewContiguousAllocationMap(max, rangeSpec), nil - }) - if err != nil { - return err - } - str, err := base64.StdEncoding.DecodeString(cm.Data[config.KeyDHCP]) - if err != nil { - return err - } - err = dhcp.Restore(d.cidr, str) - if err != nil { - return err - } - dhcp.ForEach(fn) - return nil -} diff --git a/pkg/handler/envoy.go b/pkg/handler/envoy.go index 1fd8c144..ff5e2b67 100644 --- a/pkg/handler/envoy.go +++ b/pkg/handler/envoy.go @@ -53,7 +53,7 @@ func InjectVPNAndEnvoySidecar(ctx1 context.Context, factory cmdutil.Factory, cli } nodeID := fmt.Sprintf("%s.%s", object.Mapping.Resource.GroupResource().String(), object.Name) - err = addEnvoyConfig(clientset, nodeID, c.LocalTunIP, headers, port) + err = addEnvoyConfig(clientset, nodeID, c, headers, port) if err != nil { log.Warnln(err) return err @@ -168,7 +168,7 @@ func UnPatchContainer(factory cmdutil.Factory, mapInterface v12.ConfigMapInterfa return err } -func addEnvoyConfig(mapInterface v12.ConfigMapInterface, nodeID string, localTUNIP string, headers map[string]string, port []v1.ContainerPort) error { +func addEnvoyConfig(mapInterface v12.ConfigMapInterface, nodeID string, tunIP util.PodRouteConfig, headers map[string]string, port []v1.ContainerPort) error { configMap, err := mapInterface.Get(context.Background(), config.ConfigMapPodTrafficManager, metav1.GetOptions{}) if err != nil { return err @@ -191,14 +191,16 @@ func addEnvoyConfig(mapInterface v12.ConfigMapInterface, nodeID string, localTUN Uid: nodeID, Ports: port, Rules: []*controlplane.Rule{{ - Headers: headers, - LocalTunIP: localTUNIP, + Headers: headers, + LocalTunIPv4: tunIP.LocalTunIPv4, + LocalTunIPv6: tunIP.LocalTunIPv6, }}, }) } else { v[index].Rules = append(v[index].Rules, &controlplane.Rule{ - Headers: headers, - LocalTunIP: localTUNIP, + Headers: headers, + LocalTunIPv4: tunIP.LocalTunIPv4, + LocalTunIPv6: tunIP.LocalTunIPv6, }) if v[index].Ports == nil { v[index].Ports = port diff --git a/pkg/handler/remote.go b/pkg/handler/remote.go index c2edc96f..84a79fc8 100644 --- a/pkg/handler/remote.go +++ b/pkg/handler/remote.go @@ -38,7 +38,10 @@ import ( "github.com/wencaiwulue/kubevpn/pkg/util" ) -func CreateOutboundPod(ctx context.Context, factory cmdutil.Factory, clientset *kubernetes.Clientset, namespace string, trafficManagerIP string) (ip net.IP, err error) { +func createOutboundPod(ctx context.Context, factory cmdutil.Factory, clientset *kubernetes.Clientset, namespace string) (err error) { + innerIpv4CIDR := net.IPNet{IP: config.RouterIP, Mask: config.CIDR.Mask} + innerIpv6CIDR := net.IPNet{IP: config.RouterIP6, Mask: config.CIDR6.Mask} + service, err := clientset.CoreV1().Services(namespace).Get(ctx, config.ConfigMapPodTrafficManager, metav1.GetOptions{}) if err == nil { _, err = polymorphichelpers.AttachablePodForObjectFn(factory, service, 2*time.Second) @@ -48,7 +51,7 @@ func CreateOutboundPod(ctx context.Context, factory cmdutil.Factory, clientset * return } log.Infoln("traffic manager already exist, reuse it") - return net.ParseIP(service.Spec.ClusterIP), nil + return nil } } var deleteResource = func(ctx context.Context) { @@ -71,7 +74,7 @@ func CreateOutboundPod(ctx context.Context, factory cmdutil.Factory, clientset * // 1) label namespace ns, err := clientset.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) if err != nil { - return nil, err + return err } if ns.Labels == nil { ns.Labels = map[string]string{} @@ -79,7 +82,7 @@ func CreateOutboundPod(ctx context.Context, factory cmdutil.Factory, clientset * ns.Labels["ns"] = namespace _, err = clientset.CoreV1().Namespaces().Update(ctx, ns, metav1.UpdateOptions{}) if err != nil { - return nil, err + return err } // 2) create serviceAccount @@ -91,7 +94,7 @@ func CreateOutboundPod(ctx context.Context, factory cmdutil.Factory, clientset * AutomountServiceAccountToken: pointer.Bool(true), }, metav1.CreateOptions{}) if err != nil { - return nil, err + return err } // 3) create roles @@ -108,7 +111,7 @@ func CreateOutboundPod(ctx context.Context, factory cmdutil.Factory, clientset * }}, }, metav1.CreateOptions{}) if err != nil { - return nil, err + return err } // 4) create roleBinding @@ -130,14 +133,14 @@ func CreateOutboundPod(ctx context.Context, factory cmdutil.Factory, clientset * }, }, metav1.CreateOptions{}) if err != nil { - return nil, err + return err } udp8422 := "8422-for-udp" tcp10800 := "10800-for-tcp" tcp9002 := "9002-for-envoy" tcp80 := "80-for-webhook" - svc, err := clientset.CoreV1().Services(namespace).Create(ctx, &v1.Service{ + _, err = clientset.CoreV1().Services(namespace).Create(ctx, &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: config.ConfigMapPodTrafficManager, Namespace: namespace, @@ -169,7 +172,7 @@ func CreateOutboundPod(ctx context.Context, factory cmdutil.Factory, clientset * }, }, metav1.CreateOptions{}) if err != nil { - return nil, err + return err } var Resources = v1.ResourceRequirements{ @@ -187,7 +190,7 @@ func CreateOutboundPod(ctx context.Context, factory cmdutil.Factory, clientset * var crt, key []byte crt, key, err = cert.GenerateSelfSignedCertKey(domain, nil, nil) if err != nil { - return nil, err + return err } // reason why not use v1.SecretTypeTls is because it needs key called tls.crt and tls.key, but tls.key can not as env variable @@ -207,7 +210,7 @@ func CreateOutboundPod(ctx context.Context, factory cmdutil.Factory, clientset * _, err = clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) if err != nil && !k8serrors.IsAlreadyExists(err) { - return nil, err + return err } deployment := &appsv1.Deployment{ @@ -249,13 +252,18 @@ func CreateOutboundPod(ctx context.Context, factory cmdutil.Factory, clientset * Image: config.Image, Command: []string{"/bin/sh", "-c"}, Args: []string{` -sysctl net.ipv4.ip_forward=1 +sysctl -w net.ipv4.ip_forward=1 +sysctl -w net.ipv6.conf.all.forwarding=1 update-alternatives --set iptables /usr/sbin/iptables-legacy iptables -F +ip6tables -F iptables -P INPUT ACCEPT +ip6tables -P INPUT ACCEPT iptables -P FORWARD ACCEPT -iptables -t nat -A POSTROUTING -s ${CIDR} -o eth0 -j MASQUERADE -kubevpn serve -L "tcp://:10800" -L "tun://:8422?net=${TrafficManagerIP}" --debug=true`, +ip6tables -P FORWARD ACCEPT +iptables -t nat -A POSTROUTING -s ${CIDR4} -o eth0 -j MASQUERADE +ip6tables -t nat -A POSTROUTING -s ${CIDR6} -o eth0 -j MASQUERADE +kubevpn serve -L "tcp://:10800" -L "tun://:8422?net=${TunIPv4}" --debug=true`, }, EnvFrom: []v1.EnvFromSource{{ SecretRef: &v1.SecretEnvSource{ @@ -266,12 +274,20 @@ kubevpn serve -L "tcp://:10800" -L "tun://:8422?net=${TrafficManagerIP}" --debug }}, Env: []v1.EnvVar{ { - Name: "CIDR", + Name: "CIDR4", Value: config.CIDR.String(), }, { - Name: "TrafficManagerIP", - Value: trafficManagerIP, + Name: "CIDR6", + Value: config.CIDR6.String(), + }, + { + Name: config.EnvInboundPodTunIPv4, + Value: innerIpv4CIDR.String(), + }, + { + Name: config.EnvInboundPodTunIPv6, + Value: innerIpv6CIDR.String(), }, }, Ports: []v1.ContainerPort{{ @@ -348,11 +364,11 @@ kubevpn serve -L "tcp://:10800" -L "tun://:8422?net=${TrafficManagerIP}" --debug LabelSelector: fields.OneTermEqualSelector("app", config.ConfigMapPodTrafficManager).String(), }) if err != nil { - return nil, err + return err } defer watchStream.Stop() if _, err = clientset.AppsV1().Deployments(namespace).Create(ctx, deployment, metav1.CreateOptions{}); err != nil { - return nil, err + return err } var last string out: @@ -360,7 +376,7 @@ out: select { case e, ok := <-watchStream.ResultChan(): if !ok { - return nil, fmt.Errorf("can not wait pod to be ready because of watch chan has closed") + return fmt.Errorf("can not wait pod to be ready because of watch chan has closed") } if podT, ok := e.Object.(*v1.Pod); ok { if podT.DeletionTimestamp != nil { @@ -386,7 +402,7 @@ out: last = sb.String() } case <-time.Tick(time.Minute * 60): - return nil, errors.New(fmt.Sprintf("wait pod %s to be ready timeout", config.ConfigMapPodTrafficManager)) + return errors.New(fmt.Sprintf("wait pod %s to be ready timeout", config.ConfigMapPodTrafficManager)) } } _, err = clientset.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(ctx, &admissionv1.MutatingWebhookConfiguration{ @@ -424,13 +440,13 @@ out: }}, }, metav1.CreateOptions{}) if err != nil && !k8serrors.IsForbidden(err) && !k8serrors.IsAlreadyExists(err) { - return nil, fmt.Errorf("failed to create MutatingWebhookConfigurations, err: %v", err) + return fmt.Errorf("failed to create MutatingWebhookConfigurations, err: %v", err) } _, err = updateRefCount(clientset.CoreV1().ConfigMaps(namespace), config.ConfigMapPodTrafficManager, 1) if err != nil { return } - return net.ParseIP(svc.Spec.ClusterIP), nil + return } func InjectVPNSidecar(ctx1 context.Context, factory cmdutil.Factory, namespace, workloads string, config util.PodRouteConfig) error { diff --git a/pkg/handler/serve.go b/pkg/handler/serve.go index ec10dc4a..c8de9e39 100644 --- a/pkg/handler/serve.go +++ b/pkg/handler/serve.go @@ -2,7 +2,6 @@ package handler import ( "fmt" - "net" "net/http" "os" "strings" @@ -15,7 +14,7 @@ import ( ) func Complete(route *core.Route) error { - if v, ok := os.LookupEnv(config.EnvInboundPodTunIP); ok && v == "" { + if v, ok := os.LookupEnv(config.EnvInboundPodTunIPv4); ok && v == "" { namespace := os.Getenv(config.EnvPodNamespace) if namespace == "" { return fmt.Errorf("can not get namespace") @@ -34,8 +33,15 @@ func Complete(route *core.Route) error { return err } log.Infof("rent an ip %s", strings.TrimSpace(string(ip))) - err = os.Setenv(config.EnvInboundPodTunIP, strings.TrimSpace(string(ip))) - if err != nil { + ips := strings.Split(string(ip), ",") + if len(ips) != 2 { + return fmt.Errorf("can not get ip from %s", string(ip)) + } + if err = os.Setenv(config.EnvInboundPodTunIPv4, ips[0]); err != nil { + log.Error(err) + return err + } + if err = os.Setenv(config.EnvInboundPodTunIPv6, ips[1]); err != nil { log.Error(err) return err } @@ -55,14 +61,6 @@ func Complete(route *core.Route) error { } func Final() error { - v, ok := os.LookupEnv(config.EnvInboundPodTunIP) - if !ok || v == "" { - return nil - } - _, _, err := net.ParseCIDR(v) - if err != nil { - return err - } namespace := os.Getenv(config.EnvPodNamespace) url := fmt.Sprintf("https://%s:80%s", util.GetTlsDomain(namespace), config.APIReleaseIP) req, err := http.NewRequest("DELETE", url, nil) @@ -71,7 +69,8 @@ func Final() error { } req.Header.Set(config.HeaderPodName, os.Getenv(config.EnvPodName)) req.Header.Set(config.HeaderPodNamespace, namespace) - req.Header.Set(config.HeaderIP, v) + req.Header.Set(config.HeaderIPv4, os.Getenv(config.EnvInboundPodTunIPv4)) + req.Header.Set(config.HeaderIPv6, os.Getenv(config.EnvInboundPodTunIPv6)) _, err = util.DoReq(req) return err } diff --git a/pkg/mesh/controller.go b/pkg/mesh/controller.go index d41b01a0..575455c1 100644 --- a/pkg/mesh/controller.go +++ b/pkg/mesh/controller.go @@ -26,6 +26,7 @@ func RemoveContainers(spec *v1.PodTemplateSpec) { } } +// todo envoy support ipv6 func AddMeshContainer(spec *v1.PodTemplateSpec, nodeId string, c util.PodRouteConfig) { // remove envoy proxy containers if already exist RemoveContainers(spec) @@ -34,14 +35,20 @@ func AddMeshContainer(spec *v1.PodTemplateSpec, nodeId string, c util.PodRouteCo Image: config.Image, Command: []string{"/bin/sh", "-c"}, Args: []string{` -sysctl net.ipv4.ip_forward=1 +sysctl -w net.ipv4.ip_forward=1 +sysctl -w net.ipv6.conf.all.forwarding=1 update-alternatives --set iptables /usr/sbin/iptables-legacy iptables -F +ip6tables -F iptables -P INPUT ACCEPT +ip6tables -P INPUT ACCEPT iptables -P FORWARD ACCEPT -iptables -t nat -A PREROUTING ! -p icmp ! -s 127.0.0.1 ! -d ${CIDR} -j DNAT --to 127.0.0.1:15006 -iptables -t nat -A POSTROUTING ! -p icmp ! -s 127.0.0.1 ! -d ${CIDR} -j MASQUERADE -kubevpn serve -L "tun:/127.0.0.1:8422?net=${InboundPodTunIP}&route=${CIDR}" -F "tcp://${TrafficManagerRealIP}:10800"`, +ip6tables -P FORWARD ACCEPT +iptables -t nat -A PREROUTING ! -p icmp ! -s 127.0.0.1 ! -d ${CIDR4} -j DNAT --to 127.0.0.1:15006 +ip6tables -t nat -A PREROUTING ! -p icmp ! -s 0:0:0:0:0:0:0:1 ! -d ${CIDR6} -j DNAT --to 0:0:0:0:0:0:0:1:15006 +iptables -t nat -A POSTROUTING ! -p icmp ! -s 127.0.0.1 ! -d ${CIDR4} -j MASQUERADE +ip6tables -t nat -A POSTROUTING ! -p icmp ! -s 0:0:0:0:0:0:0:1 ! -d ${CIDR6} -j MASQUERADE +kubevpn serve -L "tun:/localhost:8422?net=${TunIPv4}&route=${CIDR4}" -F "tcp://${TrafficManagerService}:10800"`, }, EnvFrom: []v1.EnvFromSource{{ SecretRef: &v1.SecretEnvSource{ @@ -52,16 +59,24 @@ kubevpn serve -L "tun:/127.0.0.1:8422?net=${InboundPodTunIP}&route=${CIDR}" -F " }}, Env: []v1.EnvVar{ { - Name: "CIDR", + Name: "CIDR4", Value: config.CIDR.String(), }, { - Name: "TrafficManagerRealIP", - Value: c.TrafficManagerRealIP, + Name: "CIDR6", + Value: config.CIDR6.String(), }, { - Name: config.EnvInboundPodTunIP, - Value: c.InboundPodTunIP, + Name: config.EnvInboundPodTunIPv4, + Value: "", + }, + { + Name: config.EnvInboundPodTunIPv6, + Value: "", + }, + { + Name: "TrafficManagerService", + Value: config.ConfigMapPodTrafficManager, }, { Name: config.EnvPodNamespace, diff --git a/pkg/mesh/envoy.yaml b/pkg/mesh/envoy.yaml index 49298f3a..f3074cf8 100644 --- a/pkg/mesh/envoy.yaml +++ b/pkg/mesh/envoy.yaml @@ -2,8 +2,9 @@ admin: access_log_path: /dev/null address: socket_address: - address: 0.0.0.0 + address: "::" port_value: 9003 + ipv4_compat: true dynamic_resources: ads_config: api_type: GRPC @@ -23,8 +24,9 @@ static_resources: - name: default_listener address: socket_address: - address: 0.0.0.0 + address: "::" port_value: 15006 + ipv4_compat: true use_original_dst: true filter_chains: - filters: @@ -47,8 +49,10 @@ static_resources: socket_address: address: kubevpn-traffic-manager port_value: 9002 + ipv4_compat: true http2_protocol_options: { } - name: origin_cluster connect_timeout: 5s type: ORIGINAL_DST - lb_policy: CLUSTER_PROVIDED \ No newline at end of file + lb_policy: CLUSTER_PROVIDED + dns_lookup_family: ALL \ No newline at end of file diff --git a/pkg/test/local.go b/pkg/test/local.go new file mode 100644 index 00000000..a9115635 --- /dev/null +++ b/pkg/test/local.go @@ -0,0 +1,43 @@ +package main + +import ( + "io" + "net" + + "github.com/containernetworking/cni/pkg/types" + log "github.com/sirupsen/logrus" + + "github.com/wencaiwulue/kubevpn/pkg/tun" +) + +func main() { + ip := net.ParseIP("fe80::cff4:d42c:7e73:e84a") + listener, err := tun.Listener(tun.Config{ + Addr: ip.String() + "/64", + MTU: 1350, + Routes: []types.Route{ + { + Dst: net.IPNet{ + IP: ip, + Mask: net.CIDRMask(64, 128), + }, + }, { + Dst: net.IPNet{ + IP: net.ParseIP("192.168.0.0"), + Mask: net.CIDRMask(64, 128), + }, + }, + }, + }) + if err != nil { + panic(err) + } + tunConn, err := listener.Accept() + defer tunConn.Close() + tcpConn, err := net.Dial("tcp", ":1080") + if err != nil { + log.Fatal(err) + } + go io.Copy(tunConn, tcpConn) + io.Copy(tcpConn, tunConn) +} diff --git a/pkg/test/pod.yaml b/pkg/test/pod.yaml new file mode 100644 index 00000000..639f77ed --- /dev/null +++ b/pkg/test/pod.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test + labels: + app: test +spec: + terminationGracePeriodSeconds: 0 + containers: + - name: traffic-test + image: naison/kubevpn:v1.1.28 + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + args: + - | + sysctl net.ipv4.ip_forward=1 + sysctl net.ipv6.conf.all.forwarding=1 + update-alternatives --set iptables /usr/sbin/iptables-legacy + iptables -F + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT + ip6tables -t nat -A POSTROUTING -s fe80::cff4:d42c:7e73:e84b/64 -o eth0 -j MASQUERADE + iptables -t nat -A POSTROUTING -s 192.168.0.0/24 -o eth0 -j MASQUERADE + tail -f /dev/null + securityContext: + privileged: true + capabilities: + add: + - NET_ADMIN + restartPolicy: Always diff --git a/pkg/test/run.sh b/pkg/test/run.sh new file mode 100644 index 00000000..36bd661c --- /dev/null +++ b/pkg/test/run.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +export KUBECONFIG=~/.kube/vke +export NS=kube-system +kubectl apply -f pod.yaml -n $NS +kubectl wait --for=condition=Ready pod/test -n $NS +cd ./server && GOARCH=amd64 GOOS=linux go build -o main +kubectl cp main test:/app/main -n $NS +rm -fr main +kubectl port-forward pods/test 1080 -n $NS diff --git a/pkg/test/server/server.go b/pkg/test/server/server.go new file mode 100644 index 00000000..9c006f35 --- /dev/null +++ b/pkg/test/server/server.go @@ -0,0 +1,39 @@ +package main + +import ( + "io" + "net" + + log "github.com/sirupsen/logrus" + + "github.com/wencaiwulue/kubevpn/pkg/tun" +) + +func main() { + ip := net.ParseIP("fe80::cff4:d42c:7e73:e84b") + listener, err := tun.Listener(tun.Config{ + Addr: ip.String() + "/64", + MTU: 1350, + }) + if err != nil { + panic(err) + } + + tunConn, _ := listener.Accept() + + tcpListener, err := net.Listen("tcp", ":1080") + if err != nil { + log.Fatal(err) + } + for { + tcpConn, err := tcpListener.Accept() + if err != nil { + panic(err) + } + go func(tcpConn net.Conn) { + defer tcpConn.Close() + go io.Copy(tunConn, tcpConn) + io.Copy(tcpConn, tunConn) + }(tcpConn) + } +} diff --git a/pkg/tun/tun.go b/pkg/tun/tun.go index 475e8a6a..3f23aec1 100644 --- a/pkg/tun/tun.go +++ b/pkg/tun/tun.go @@ -18,6 +18,7 @@ import ( type Config struct { Name string Addr string + Addr6 string MTU int Routes []types.Route Gateway string @@ -75,8 +76,9 @@ func (l *tunListener) Close() error { } type tunConn struct { - ifce tun.Device - addr net.Addr + ifce tun.Device + addr net.Addr + addr6 net.Addr } func (c *tunConn) Read(b []byte) (n int, err error) { diff --git a/pkg/tun/tun_freebsd.go b/pkg/tun/tun_bsd.go similarity index 51% rename from pkg/tun/tun_freebsd.go rename to pkg/tun/tun_bsd.go index 64d330cf..f4011db3 100644 --- a/pkg/tun/tun_freebsd.go +++ b/pkg/tun/tun_bsd.go @@ -1,4 +1,4 @@ -//go:build freebsd +//go:build freebsd || openbsd package tun @@ -16,11 +16,13 @@ import ( ) func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) { - ip, _, err := net.ParseCIDR(cfg.Addr) - if err != nil { + if cfg.Addr == "" && cfg.Addr6 == "" { + err = fmt.Errorf("ipv4 address and ipv6 address can not be empty at same time") return } + var ipv4, ipv6 net.IP + mtu := cfg.MTU if mtu <= 0 { mtu = config.DefaultMTU @@ -38,12 +40,31 @@ func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) { return } - cmd := fmt.Sprintf("ifconfig %s inet %s mtu %d up", ifce.Name(), cfg.Addr, mtu) - log.Debugf("[tun] %s", cmd) - args := strings.Split(cmd, " ") - if er := exec.Command(args[0], args[1:]...).Run(); er != nil { - err = fmt.Errorf("%s: %v", cmd, er) - return + if cfg.Addr != "" { + ipv4, _, err = net.ParseCIDR(cfg.Addr6) + if err != nil { + return + } + cmd := fmt.Sprintf("ifconfig %s inet %s mtu %d up", ifce.Name(), cfg.Addr, mtu) + log.Debugf("[tun] %s", cmd) + args := strings.Split(cmd, " ") + if err = exec.Command(args[0], args[1:]...).Run(); err != nil { + err = fmt.Errorf("%s: %v", cmd, err) + return + } + } + if cfg.Addr6 != "" { + ipv6, _, err = net.ParseCIDR(cfg.Addr6) + if err != nil { + return + } + cmd := fmt.Sprintf("ifconfig %s add %s", ifce.Name(), cfg.Addr6) + log.Debugf("[tun] %s", cmd) + args := strings.Split(cmd, " ") + if err = exec.Command(args[0], args[1:]...).Run(); err != nil { + err = fmt.Errorf("%s: %v", cmd, err) + return + } } if err = os.Setenv(config.EnvTunNameOrLUID, ifce.Name()); err != nil { @@ -60,8 +81,9 @@ func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) { } conn = &tunConn{ - ifce: ifce, - addr: &net.IPAddr{IP: ip}, + ifce: ifce, + addr: &net.IPAddr{IP: ipv4}, + addr6: &net.IPAddr{IP: ipv6}, } return } @@ -71,7 +93,11 @@ func addTunRoutes(ifName string, routes ...types.Route) error { if route.Dst.String() == "" { continue } - cmd := fmt.Sprintf("route add -net %s -interface %s", route.Dst.String(), ifName) + if route.Dst.IP.To4() != nil { + cmd := fmt.Sprintf("route add -net %s -interface %s", route.Dst.String(), ifName) + } else { + cmd := fmt.Sprintf("route add -inet6 %s -interface %s", route.Dst.String(), ifName) + } log.Debugf("[tun] %s", cmd) args := strings.Split(cmd, " ") if er := exec.Command(args[0], args[1:]...).Run(); er != nil { diff --git a/pkg/tun/tun_darwin.go b/pkg/tun/tun_darwin.go index 9f972ba1..76a06429 100644 --- a/pkg/tun/tun_darwin.go +++ b/pkg/tun/tun_darwin.go @@ -17,10 +17,7 @@ import ( ) func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) { - ip, _, err := net.ParseCIDR(cfg.Addr) - if err != nil { - return - } + var ipv4, ipv6 net.IP mtu := cfg.MTU if mtu <= 0 { @@ -39,29 +36,48 @@ func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) { return } - cmd := fmt.Sprintf("ifconfig %s inet %s %s mtu %d up", name, cfg.Addr, ip.String(), mtu) - log.Debugf("[tun] %s", cmd) - args := strings.Split(cmd, " ") - if er := exec.Command(args[0], args[1:]...).Run(); er != nil { - err = fmt.Errorf("%s: %v", cmd, er) + // set ipv4 address + if ipv4, _, err = net.ParseCIDR(cfg.Addr); err != nil { return } + setIPv4Cmd := fmt.Sprintf("ifconfig %s inet %s %s mtu %d up", name, cfg.Addr, ipv4.String(), mtu) + log.Debugf("[tun] %s", setIPv4Cmd) + args := strings.Split(setIPv4Cmd, " ") + if err = exec.Command(args[0], args[1:]...).Run(); err != nil { + err = fmt.Errorf("%s: %v", setIPv4Cmd, err) + return + } + + // set ipv6 address + var ipv6CIDR *net.IPNet + if ipv6, ipv6CIDR, err = net.ParseCIDR(cfg.Addr6); err != nil { + return + } + ones, _ := ipv6CIDR.Mask.Size() + setIPv6Cmd := fmt.Sprintf("ifconfig %s inet6 %s prefixlen %d alias", name, ipv6.String(), ones) + log.Debugf("[tun] %s", setIPv6Cmd) + args = strings.Split(setIPv6Cmd, " ") + if err = exec.Command(args[0], args[1:]...).Run(); err != nil { + err = fmt.Errorf("%s: %v", setIPv6Cmd, err) + return + } + if err = os.Setenv(config.EnvTunNameOrLUID, name); err != nil { - return nil, nil, err + return } if err = addTunRoutes(name, cfg.Routes...); err != nil { return } - itf, err = net.InterfaceByName(name) - if err != nil { + if itf, err = net.InterfaceByName(name); err != nil { return } conn = &tunConn{ - ifce: ifce, - addr: &net.IPAddr{IP: ip}, + ifce: ifce, + addr: &net.IPAddr{IP: ipv4}, + addr6: &net.IPAddr{IP: ipv6}, } return } @@ -71,7 +87,13 @@ func addTunRoutes(ifName string, routes ...types.Route) error { if route.Dst.String() == "" { continue } - cmd := fmt.Sprintf("route add -net %s -interface %s", route.Dst.String(), ifName) + var cmd string + // ipv4 + if route.Dst.IP.To4() != nil { + cmd = fmt.Sprintf("route add -net %s -interface %s", route.Dst.String(), ifName) + } else { // ipv6 + cmd = fmt.Sprintf("route add -inet6 %s -interface %s", route.Dst.String(), ifName) + } log.Debugf("[tun] %s", cmd) args := strings.Split(cmd, " ") err := exec.Command(args[0], args[1:]...).Run() diff --git a/pkg/tun/tun_linux.go b/pkg/tun/tun_linux.go index 27a42e81..f7c4fe0d 100644 --- a/pkg/tun/tun_linux.go +++ b/pkg/tun/tun_linux.go @@ -18,19 +18,20 @@ import ( ) func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) { - ip, ipNet, err := net.ParseCIDR(cfg.Addr) - if err != nil { + if cfg.Addr == "" && cfg.Addr6 == "" { + err = fmt.Errorf("ipv4 address and ipv6 address can not be empty at same time") return } + var ipv4, ipv6 net.IP + mtu := cfg.MTU if mtu <= 0 { mtu = config.DefaultMTU } var device tun.Device - device, err = tun.CreateTUN("utun", mtu) - if err != nil { + if device, err = tun.CreateTUN("utun", mtu); err != nil { return } @@ -39,49 +40,60 @@ func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) { if err != nil { return } - ifc, err := net.InterfaceByName(name) - if err != nil { + var ifc *net.Interface + if ifc, err = net.InterfaceByName(name); err != nil { err = fmt.Errorf("could not find interface name: %s", err) return } - cmd := fmt.Sprintf("ip link set dev %s mtu %d", name, mtu) - log.Debugf("[tun] %s", cmd) - if er := netlink.NetworkSetMTU(ifc, mtu); er != nil { - err = fmt.Errorf("%s: %v", cmd, er) + if err = netlink.NetworkSetMTU(ifc, mtu); err != nil { + err = fmt.Errorf("can not setup mtu %d to device %s : %v", mtu, name, err) return } - cmd = fmt.Sprintf("ip address add %s dev %s", cfg.Addr, name) - log.Debugf("[tun] %s", cmd) - if er := netlink.NetworkLinkAddIp(ifc, ip, ipNet); er != nil { - err = fmt.Errorf("%s: %v", cmd, er) - return + if cfg.Addr != "" { + var ipv4CIDR *net.IPNet + if ipv4, ipv4CIDR, err = net.ParseCIDR(cfg.Addr); err != nil { + return + } + if err = netlink.NetworkLinkAddIp(ifc, ipv4, ipv4CIDR); err != nil { + err = fmt.Errorf("can not set ipv4 address %s to device %s : %v", ipv4.String(), name, err) + return + } } - cmd = fmt.Sprintf("ip link set dev %s up", name) - log.Debugf("[tun] %s", cmd) - if er := netlink.NetworkLinkUp(ifc); er != nil { - err = fmt.Errorf("%s: %v", cmd, er) + if cfg.Addr6 != "" { + var ipv6CIDR *net.IPNet + if ipv6, ipv6CIDR, err = net.ParseCIDR(cfg.Addr6); err != nil { + return + } + if err = netlink.NetworkLinkAddIp(ifc, ipv6, ipv6CIDR); err != nil { + err = fmt.Errorf("can not setup ipv6 address %s to device %s : %v", ipv6.String(), name, err) + return + } + } + + if err = netlink.NetworkLinkUp(ifc); err != nil { + err = fmt.Errorf("can not up device %s : %v", name, err) return } if err = os.Setenv(config.EnvTunNameOrLUID, name); err != nil { - return nil, nil, err + return } if err = addTunRoutes(name, cfg.Routes...); err != nil { return } - itf, err = net.InterfaceByName(name) - if err != nil { + if itf, err = net.InterfaceByName(name); err != nil { return } conn = &tunConn{ - ifce: device, - addr: &net.IPAddr{IP: ip}, + ifce: device, + addr: &net.IPAddr{IP: ipv4}, + addr6: &net.IPAddr{IP: ipv6}, } return } @@ -93,7 +105,8 @@ func addTunRoutes(ifName string, routes ...types.Route) error { } cmd := fmt.Sprintf("ip route add %s dev %s", route.Dst.String(), ifName) log.Debugf("[tun] %s", cmd) - if err := netlink.AddRoute(route.Dst.String(), "", "", ifName); err != nil && !errors.Is(err, syscall.EEXIST) { + err := netlink.AddRoute(route.Dst.String(), "", "", ifName) + if err != nil && !errors.Is(err, syscall.EEXIST) { return fmt.Errorf("%s: %v", cmd, err) } } diff --git a/pkg/tun/tun_openbsd.go b/pkg/tun/tun_openbsd.go deleted file mode 100644 index 9bbf2ff7..00000000 --- a/pkg/tun/tun_openbsd.go +++ /dev/null @@ -1,86 +0,0 @@ -//go:build openbsd - -package tun - -import ( - "fmt" - "net" - "os/exec" - "strings" - - "github.com/containernetworking/cni/pkg/types" - log "github.com/sirupsen/logrus" - "golang.zx2c4.com/wireguard/tun" - - "github.com/wencaiwulue/kubevpn/pkg/config" -) - -func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) { - ip, _, err := net.ParseCIDR(cfg.Addr) - if err != nil { - return - } - - mtu := cfg.MTU - if mtu <= 0 { - mtu = config.DefaultMTU - } - - var ifce tun.Device - ifce, err = tun.CreateTUN("utun", mtu) - if err != nil { - return - } - - var name string - name, err = ifce.Name() - if err != nil { - return - } - - cmd := fmt.Sprintf("ifconfig %s inet %s mtu %d up", ifce.Name(), cfg.Addr, mtu) - log.Debugf("[tun] %s", cmd) - args := strings.Split(cmd, " ") - if er := exec.Command(args[0], args[1:]...).Run(); er != nil { - err = fmt.Errorf("%s: %v", cmd, er) - return - } - - if err = os.Setenv(config.EnvTunNameOrLUID, ifce.Name()); err != nil { - return nil, nil, err - } - - if err = addTunRoutes(ifce.Name(), cfg.Routes...); err != nil { - return - } - - itf, err = net.InterfaceByName(ifce.Name()) - if err != nil { - return - } - - conn = &tunConn{ - ifce: ifce, - addr: &net.IPAddr{IP: ip}, - } - return -} - -func addTunRoutes(ifName string, routes ...types.Route) error { - for _, route := range routes { - if route.Dst.String() == "" { - continue - } - cmd := fmt.Sprintf("route add -net %s -interface %s", route.Dst.String(), ifName) - log.Debugf("[tun] %s", cmd) - args := strings.Split(cmd, " ") - if er := exec.Command(args[0], args[1:]...).Run(); er != nil { - return fmt.Errorf("%s: %v", cmd, er) - } - } - return nil -} - -func getInterface() (*net.Interface, error) { - return net.InterfaceByName(os.Getenv(config.EnvTunNameOrLUID)) -} diff --git a/pkg/tun/tun_windows.go b/pkg/tun/tun_windows.go index b4d680be..0bd71414 100644 --- a/pkg/tun/tun_windows.go +++ b/pkg/tun/tun_windows.go @@ -22,44 +22,71 @@ import ( "github.com/wencaiwulue/kubevpn/pkg/config" ) -func createTun(cfg Config) (net.Conn, *net.Interface, error) { - ip, _, err := net.ParseCIDR(cfg.Addr) - if err != nil { - return nil, nil, err +func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) { + if cfg.Addr == "" && cfg.Addr6 == "" { + err = fmt.Errorf("ipv4 address and ipv6 address can not be empty at same time") + return } - interfaceName := "wg1" + + interfaceName := "kubevpn" if len(cfg.Name) != 0 { interfaceName = cfg.Name } tunDevice, err := wireguardtun.CreateTUN(interfaceName, cfg.MTU) if err != nil { - return nil, nil, fmt.Errorf("failed to create TUN device: %w", err) + err = fmt.Errorf("failed to create TUN device: %w", err) + return } ifName := winipcfg.LUID(tunDevice.(*wireguardtun.NativeTun).LUID()) - var prefix netip.Prefix - prefix, err = netip.ParsePrefix(cfg.Addr) - if err != nil { - return nil, nil, err + var ipv4, ipv6 net.IP + if cfg.Addr != "" { + if ipv4, _, err = net.ParseCIDR(cfg.Addr); err != nil { + return + } + var prefix netip.Prefix + if prefix, err = netip.ParsePrefix(cfg.Addr); err != nil { + return + } + if err = ifName.AddIPAddress(prefix); err != nil { + return + } } - if err = ifName.AddIPAddress(prefix); err != nil { - return nil, nil, err + if cfg.Addr6 != "" { + if ipv6, _, err = net.ParseCIDR(cfg.Addr6); err != nil { + return + } + var prefix netip.Prefix + if prefix, err = netip.ParsePrefix(cfg.Addr6); err != nil { + return + } + if err = ifName.AddIPAddress(prefix); err != nil { + return + } } luid := fmt.Sprintf("%d", tunDevice.(*wireguardtun.NativeTun).LUID()) if err = os.Setenv(config.EnvTunNameOrLUID, luid); err != nil { - return nil, nil, err - } - _ = ifName.FlushRoutes(windows.AF_INET) - if err = addTunRoutes(luid /*cfg.Gateway,*/, cfg.Routes...); err != nil { - return nil, nil, err + return } - row, _ := ifName.Interface() - iface, _ := net.InterfaceByIndex(int(row.InterfaceIndex)) - return &winTunConn{ifce: tunDevice, addr: &net.IPAddr{IP: ip}}, iface, nil + _ = ifName.FlushRoutes(windows.AF_INET) + _ = ifName.FlushRoutes(windows.AF_INET6) + + if err = addTunRoutes(luid /*cfg.Gateway,*/, cfg.Routes...); err != nil { + return + } + var row *winipcfg.MibIfRow2 + if row, err = ifName.Interface(); err != nil { + return + } + if itf, err = net.InterfaceByIndex(int(row.InterfaceIndex)); err != nil { + return + } + conn = &winTunConn{ifce: tunDevice, addr: &net.IPAddr{IP: ipv4}, addr6: &net.IPAddr{IP: ipv6}} + return } func addTunRoutes(luid string, routes ...types.Route) error { @@ -76,7 +103,11 @@ func addTunRoutes(luid string, routes ...types.Route) error { if gw != "" { route.GW = net.ParseIP(gw) } else { - route.GW = net.IPv4(0, 0, 0, 0) + if route.Dst.IP.To4() != nil { + route.GW = net.IPv4zero + } else { + route.GW = net.IPv6zero + } } prefix, err := netip.ParsePrefix(route.Dst.String()) if err != nil { @@ -96,8 +127,9 @@ func addTunRoutes(luid string, routes ...types.Route) error { } type winTunConn struct { - ifce wireguardtun.Device - addr net.Addr + ifce wireguardtun.Device + addr net.Addr + addr6 net.Addr } func (c *winTunConn) Close() error { diff --git a/pkg/util/cidr.go b/pkg/util/cidr.go index 37c932fb..558758a5 100644 --- a/pkg/util/cidr.go +++ b/pkg/util/cidr.go @@ -7,6 +7,7 @@ import ( log "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/utils/pointer" @@ -88,18 +89,40 @@ func GetCIDRFromResourceUgly(clientset *kubernetes.Clientset, namespace string) if pod.Spec.HostNetwork { continue } - if ip := net.ParseIP(pod.Status.PodIP); ip != nil { - mask := net.CIDRMask(24, 32) - cidrs = append(cidrs, &net.IPNet{IP: ip.Mask(mask), Mask: mask}) + s := sets.Set[string]{}.Insert(pod.Status.PodIP) + for _, p := range pod.Status.PodIPs { + s.Insert(p.IP) + } + for _, t := range s.UnsortedList() { + if ip := net.ParseIP(t); ip != nil { + var mask net.IPMask + if ip.To4() != nil { + mask = net.CIDRMask(24, 32) + } else { + mask = net.CIDRMask(64, 128) + } + cidrs = append(cidrs, &net.IPNet{IP: ip.Mask(mask), Mask: mask}) + } } } // (2) get service CIDR serviceList, _ := clientset.CoreV1().Services(namespace).List(context.Background(), v1.ListOptions{}) for _, service := range serviceList.Items { - if ip := net.ParseIP(service.Spec.ClusterIP); ip != nil { - mask := net.CIDRMask(24, 32) - cidrs = append(cidrs, &net.IPNet{IP: ip.Mask(mask), Mask: mask}) + s := sets.Set[string]{}.Insert(service.Spec.ClusterIP) + for _, p := range service.Spec.ClusterIPs { + s.Insert(p) + } + for _, t := range s.UnsortedList() { + if ip := net.ParseIP(t); ip != nil { + var mask net.IPMask + if ip.To4() != nil { + mask = net.CIDRMask(24, 32) + } else { + mask = net.CIDRMask(64, 128) + } + cidrs = append(cidrs, &net.IPNet{IP: ip.Mask(mask), Mask: mask}) + } } } diff --git a/pkg/util/getcidr.go b/pkg/util/getcidr.go index 196856b1..e51dae93 100644 --- a/pkg/util/getcidr.go +++ b/pkg/util/getcidr.go @@ -15,6 +15,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/rest" @@ -140,7 +141,8 @@ func getPodCIDRFromCNI(clientset *kubernetes.Clientset, restclient *rest.RESTCli var m map[string]interface{} _ = json.Unmarshal(plugin.Bytes, &m) slice, _, _ := unstructured.NestedStringSlice(m, "ipam", "ipv4_pools") - for _, s := range slice { + slice6, _, _ := unstructured.NestedStringSlice(m, "ipam", "ipv6_pools") + for _, s := range sets.New[string]().Insert(slice...).Insert(slice6...).UnsortedList() { if _, ipNet, _ := net.ParseCIDR(s); ipNet != nil { cidr = append(cidr, ipNet) } @@ -286,20 +288,36 @@ func getPodCIDRFromPod(clientset *kubernetes.Clientset, namespace string, svc *n return nil, err } for i := 0; i < len(podList.Items); i++ { - if podList.Items[i].Spec.HostNetwork || net.ParseIP(podList.Items[i].Status.PodIP) == nil { + if podList.Items[i].Spec.HostNetwork { podList.Items = append(podList.Items[:i], podList.Items[i+1:]...) i-- } } + var result []*net.IPNet for _, item := range podList.Items { - if item.Name == config.CniNetName { - return []*net.IPNet{svc, {IP: net.ParseIP(item.Status.PodIP), Mask: /*svc.Mask*/ net.CIDRMask(24, 32)}}, nil + s := sets.New[string]().Insert(item.Status.PodIP) + for _, p := range item.Status.PodIPs { + s.Insert(p.IP) + } + for _, t := range s.UnsortedList() { + if ip := net.ParseIP(t); ip != nil { + var mask net.IPMask + if ip.To4() != nil { + mask = net.CIDRMask(24, 32) + } else { + mask = net.CIDRMask(64, 128) + } + result = append(result, &net.IPNet{IP: ip, Mask: /*svc.Mask*/ mask}) + } } } - for _, item := range podList.Items { - return []*net.IPNet{svc, {IP: net.ParseIP(item.Status.PodIP), Mask: /*svc.Mask*/ net.CIDRMask(24, 32)}}, nil + + if len(result) == 0 { + return nil, fmt.Errorf("can not found pod cidr from pod list") } - return nil, fmt.Errorf("can not found pod cidr from pod list") + + return result, nil + } /* diff --git a/pkg/util/pod.go b/pkg/util/pod.go index bb854076..20679c52 100644 --- a/pkg/util/pod.go +++ b/pkg/util/pod.go @@ -5,15 +5,25 @@ import ( "context" "fmt" "io" + "math/rand" + "net" "strings" "text/tabwriter" + "time" "golang.org/x/exp/constraints" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubectl/pkg/cmd/util" + + "github.com/wencaiwulue/kubevpn/pkg/config" ) +type PodRouteConfig struct { + LocalTunIPv4 string + LocalTunIPv6 string +} + func PrintStatus(pod *corev1.Pod, writer io.Writer) { w := tabwriter.NewWriter(writer, 1, 1, 1, ' ', 0) defer w.Flush() @@ -92,3 +102,15 @@ func GetEnv(ctx context.Context, f util.Factory, ns, pod string) (map[string][]s } return result, nil } + +func Heartbeats() { + ticker := time.NewTicker(time.Second * 5) + defer ticker.Stop() + + for ; true; <-ticker.C { + for _, ip := range []net.IP{config.RouterIP, config.RouterIP6} { + time.Sleep(time.Millisecond * time.Duration(rand.Intn(1000))) + _, _ = Ping(ip.String()) + } + } +} diff --git a/pkg/util/route_config.go b/pkg/util/route_config.go deleted file mode 100644 index b9a28dcd..00000000 --- a/pkg/util/route_config.go +++ /dev/null @@ -1,7 +0,0 @@ -package util - -type PodRouteConfig struct { - LocalTunIP string - InboundPodTunIP string - TrafficManagerRealIP string -} diff --git a/pkg/webhook/dhcp.go b/pkg/webhook/dhcp.go index dfecfdd9..125bf68c 100644 --- a/pkg/webhook/dhcp.go +++ b/pkg/webhook/dhcp.go @@ -19,43 +19,44 @@ type dhcpServer struct { } func (d *dhcpServer) rentIP(w http.ResponseWriter, r *http.Request) { - podName := r.Header.Get("POD_NAME") - namespace := r.Header.Get("POD_NAMESPACE") + podName := r.Header.Get(config.HeaderPodName) + namespace := r.Header.Get(config.HeaderPodNamespace) log.Infof("handling rent ip request, pod name: %s, ns: %s", podName, namespace) cmi := d.clientset.CoreV1().ConfigMaps(namespace) - dhcp := handler.NewDHCPManager(cmi, namespace, &net.IPNet{IP: config.RouterIP, Mask: config.CIDR.Mask}) - random, err := dhcp.RentIPRandom() + dhcp := handler.NewDHCPManager(cmi, namespace) + v4, v6, err := dhcp.RentIPRandom() if err != nil { log.Error(err) w.WriteHeader(http.StatusBadRequest) return } w.WriteHeader(http.StatusOK) - _, err = w.Write([]byte(random.String())) + // todo patch annotation + _, err = w.Write([]byte(fmt.Sprintf("%s,%s", v4.String(), v6.String()))) if err != nil { log.Error(err) } } func (d *dhcpServer) releaseIP(w http.ResponseWriter, r *http.Request) { - podName := r.Header.Get("POD_NAME") - namespace := r.Header.Get("POD_NAMESPACE") - ip := r.Header.Get("IP") + podName := r.Header.Get(config.HeaderPodName) + namespace := r.Header.Get(config.HeaderPodNamespace) - _, ipNet, err := net.ParseCIDR(ip) - if err != nil { - log.Errorf("ip is invailed, ip: %s, err: %v", ip, err) - w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(fmt.Sprintf("ip is invailed, ip: %s, err: %v", ip, err))) - return + var ips []net.IP + for _, s := range []string{r.Header.Get(config.HeaderIPv4), r.Header.Get(config.HeaderIPv6)} { + ip, _, err := net.ParseCIDR(s) + if err != nil { + log.Errorf("ip is invailed, ip: %s, err: %v", ip.String(), err) + continue + } + ips = append(ips, ip) } log.Infof("handling release ip request, pod name: %s, ns: %s", podName, namespace) cmi := d.clientset.CoreV1().ConfigMaps(namespace) - dhcp := handler.NewDHCPManager(cmi, namespace, &net.IPNet{IP: config.RouterIP, Mask: config.CIDR.Mask}) - err = dhcp.ReleaseIpToDHCP(ipNet) - if err != nil { + dhcp := handler.NewDHCPManager(cmi, namespace) + if err := dhcp.ReleaseIpToDHCP(ips...); err != nil { log.Error(err) w.WriteHeader(http.StatusBadRequest) return diff --git a/pkg/webhook/pods.go b/pkg/webhook/pods.go index a30e3998..04a4e235 100644 --- a/pkg/webhook/pods.go +++ b/pkg/webhook/pods.go @@ -46,14 +46,14 @@ func (h *admissionReviewHandler) admitPods(ar v1.AdmissionReview) *v1.AdmissionR var found bool for i := 0; i < len(pod.Spec.Containers); i++ { if pod.Spec.Containers[i].Name == config.ContainerSidecarVPN { + var v4, v6 *net.IPNet for j := 0; j < len(pod.Spec.Containers[i].Env); j++ { pair := pod.Spec.Containers[i].Env[j] - if pair.Name == config.EnvInboundPodTunIP && pair.Value == "" { + if pair.Name == config.EnvInboundPodTunIPv4 && pair.Value == "" { found = true cmi := h.clientset.CoreV1().ConfigMaps(ar.Request.Namespace) - dhcp := handler.NewDHCPManager(cmi, ar.Request.Namespace, &net.IPNet{IP: config.RouterIP, Mask: config.CIDR.Mask}) - var random *net.IPNet - random, err = dhcp.RentIPRandom() + dhcp := handler.NewDHCPManager(cmi, ar.Request.Namespace) + v4, v6, err = dhcp.RentIPRandom() if err != nil { log.Errorf("rent ip random failed, err: %v", err) return toV1AdmissionResponse(err) @@ -62,9 +62,16 @@ func (h *admissionReviewHandler) admitPods(ar v1.AdmissionReview) *v1.AdmissionR if accessor, errT := meta.Accessor(ar.Request.Object); errT == nil { name = accessor.GetName() } - - log.Infof("rent ip %s for pod %s in namespace: %s", random.String(), name, ar.Request.Namespace) - pod.Spec.Containers[i].Env[j].Value = random.String() + log.Infof("rent ipv4: %s ipv6: %s for pod %s in namespace: %s", v4.String(), v6.String(), name, ar.Request.Namespace) + } + } + for j := 0; j < len(pod.Spec.Containers[i].Env); j++ { + pair := pod.Spec.Containers[i].Env[j] + if pair.Name == config.EnvInboundPodTunIPv4 && v4 != nil { + pod.Spec.Containers[i].Env[j].Value = v4.String() + } + if pair.Name == config.EnvInboundPodTunIPv6 && v6 != nil { + pod.Spec.Containers[i].Env[j].Value = v6.String() } } } @@ -110,28 +117,24 @@ func (h *admissionReviewHandler) admitPods(ar v1.AdmissionReview) *v1.AdmissionR return toV1AdmissionResponse(err) } - name, _ := podcmd.FindContainerByName(&pod, config.ContainerSidecarVPN) - if name != nil { - for _, envVar := range name.Env { - if envVar.Name == config.EnvInboundPodTunIP && envVar.Value != "" { - ip, cidr, err := net.ParseCIDR(envVar.Value) - if err == nil { - cmi := h.clientset.CoreV1().ConfigMaps(ar.Request.Namespace) - ipnet := &net.IPNet{ - IP: ip, - Mask: cidr.Mask, - } - err = handler.NewDHCPManager(cmi, ar.Request.Namespace, &net.IPNet{IP: config.RouterIP, Mask: config.CIDR.Mask}).ReleaseIpToDHCP(ipnet) - if err != nil { - log.Errorf("release ip to dhcp err: %v, ip: %s", err, envVar.Value) - } else { - log.Errorf("release ip to dhcp ok, ip: %s", envVar.Value) - } + container, _ := podcmd.FindContainerByName(&pod, config.ContainerSidecarVPN) + if container != nil { + var ips []net.IP + for _, envVar := range container.Env { + if envVar.Name == config.EnvInboundPodTunIPv4 || envVar.Name == config.EnvInboundPodTunIPv6 { + if ip, _, err := net.ParseCIDR(envVar.Value); err == nil { + ips = append(ips, ip) } } } + cmi := h.clientset.CoreV1().ConfigMaps(ar.Request.Namespace) + err := handler.NewDHCPManager(cmi, ar.Request.Namespace).ReleaseIpToDHCP(ips...) + if err != nil { + log.Errorf("release ip to dhcp err: %v, ips: %v", err, ips) + } else { + log.Errorf("release ip to dhcp ok, ip: %v", ips) + } } - return &v1.AdmissionResponse{ Allowed: true, } diff --git a/plugins/stable.txt b/plugins/stable.txt new file mode 100644 index 00000000..9cf67505 --- /dev/null +++ b/plugins/stable.txt @@ -0,0 +1 @@ +v1.1.29 \ No newline at end of file