refactor: use unix library to add route table and setup ip addr instead of use command route or ifconfig (#463)

pkg/tun/route_darwin.go

@@ -0,0 +1,78 @@
+package tun
+
+import (
+	"errors"
+	"net"
+	"net/netip"
+	"os"
+
+	"golang.org/x/net/route"
+	"golang.org/x/sys/unix"
+)
+
+func addRoute(seq int, r netip.Prefix, gw route.Addr) error {
+	return withRouteSocket(func(routeSocket int) error {
+		m := newRouteMessage(unix.RTM_ADD, seq, r, gw)
+		rb, err := m.Marshal()
+		if err != nil {
+			return err
+		}
+		_, err = unix.Write(routeSocket, rb)
+		if errors.Is(err, unix.EEXIST) {
+			err = nil
+		}
+		return err
+	})
+}
+
+func deleteRoute(seq int, r netip.Prefix, gw route.Addr) error {
+	return withRouteSocket(func(routeSocket int) error {
+		m := newRouteMessage(unix.RTM_DELETE, seq, r, gw)
+		rb, err := m.Marshal()
+		if err != nil {
+			return err
+		}
+		_, err = unix.Write(routeSocket, rb)
+		if errors.Is(err, unix.ESRCH) {
+			err = nil
+		}
+		return err
+	})
+}
+
+func withRouteSocket(f func(routeSocket int) error) error {
+	routeSocket, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
+	if err != nil {
+		return err
+	}
+
+	// Avoid the overhead of echoing messages back to sender
+	if err = unix.SetsockoptInt(routeSocket, unix.SOL_SOCKET, unix.SO_USELOOPBACK, 0); err != nil {
+		return err
+	}
+	defer unix.Close(routeSocket)
+	return f(routeSocket)
+}
+
+func newRouteMessage(rtm, seq int, subnet netip.Prefix, gw route.Addr) *route.RouteMessage {
+	var mask, dst route.Addr
+	if subnet.Addr().Is4() {
+		mask = &route.Inet4Addr{IP: [4]byte(net.CIDRMask(subnet.Bits(), 32))}
+		dst = &route.Inet4Addr{IP: subnet.Addr().As4()}
+	} else {
+		mask = &route.Inet6Addr{IP: [16]byte(net.CIDRMask(subnet.Bits(), 128))}
+		dst = &route.Inet6Addr{IP: subnet.Addr().As16()}
+	}
+	return &route.RouteMessage{
+		Version: unix.RTM_VERSION,
+		ID:      uintptr(os.Getpid()),
+		Seq:     seq,
+		Type:    rtm,
+		Flags:   unix.RTF_UP | unix.RTF_STATIC | unix.RTF_CLONING | unix.RTF_GATEWAY,
+		Addrs: []route.Addr{
+			unix.RTAX_DST:     dst,
+			unix.RTAX_GATEWAY: gw,
+			unix.RTAX_NETMASK: mask,
+		},
+	}
+}

pkg/tun/tun_bsd.go

@@ -1,118 +0,0 @@
-//go:build freebsd || openbsd
-
-package tun
-
-import (
-	"fmt"
-	"net"
-	"os/exec"
-	"strings"
-
-	"github.com/containernetworking/cni/pkg/types"
-	log "github.com/sirupsen/logrus"
-	"golang.zx2c4.com/wireguard/tun"
-
-	"github.com/wencaiwulue/kubevpn/v2/pkg/config"
-)
-
-func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) {
-	if cfg.Addr == "" && cfg.Addr6 == "" {
-		err = fmt.Errorf("IPv4 address and IPv6 address can not be empty at same time")
-		return
-	}
-
-	var ipv4, ipv6 net.IP
-
-	mtu := cfg.MTU
-	if mtu <= 0 {
-		mtu = config.DefaultMTU
-	}
-
-	var interfaces []net.Interface
-	interfaces, err = net.Interfaces()
-	if err != nil {
-		return
-	}
-	maxIndex := -1
-	for _, i := range interfaces {
-		ifIndex := -1
-		_, err := fmt.Sscanf(i.Name, "utun%d", &ifIndex)
-		if err == nil && ifIndex >= 0 && maxIndex < ifIndex {
-			maxIndex = ifIndex
-		}
-	}
-
-	var ifce tun.Device
-	ifce, err = tun.CreateTUN(fmt.Sprintf("utun%d", maxIndex+1), mtu)
-	if err != nil {
-		return
-	}
-
-	var name string
-	name, err = ifce.Name()
-	if err != nil {
-		return
-	}
-
-	if cfg.Addr != "" {
-		ipv4, _, err = net.ParseCIDR(cfg.Addr)
-		if err != nil {
-			return
-		}
-		cmd := fmt.Sprintf("ifconfig %s inet %s mtu %d up", ifce.Name(), cfg.Addr, mtu)
-		log.Debugf("[TUN] %s", cmd)
-		args := strings.Split(cmd, " ")
-		if err = exec.Command(args[0], args[1:]...).Run(); err != nil {
-			err = fmt.Errorf("%s: %v", cmd, err)
-			return
-		}
-	}
-	if cfg.Addr6 != "" {
-		ipv6, _, err = net.ParseCIDR(cfg.Addr6)
-		if err != nil {
-			return
-		}
-		cmd := fmt.Sprintf("ifconfig %s add %s", ifce.Name(), cfg.Addr6)
-		log.Debugf("[TUN] %s", cmd)
-		args := strings.Split(cmd, " ")
-		if err = exec.Command(args[0], args[1:]...).Run(); err != nil {
-			err = fmt.Errorf("%s: %v", cmd, err)
-			return
-		}
-	}
-
-	if err = addTunRoutes(ifce.Name(), cfg.Routes...); err != nil {
-		return
-	}
-
-	itf, err = net.InterfaceByName(ifce.Name())
-	if err != nil {
-		return
-	}
-
-	conn = &tunConn{
-		ifce:  ifce,
-		addr:  &net.IPAddr{IP: ipv4},
-		addr6: &net.IPAddr{IP: ipv6},
-	}
-	return
-}
-
-func addTunRoutes(ifName string, routes ...types.Route) error {
-	for _, route := range routes {
-		if route.Dst.String() == "" {
-			continue
-		}
-		if route.Dst.IP.To4() != nil {
-			cmd := fmt.Sprintf("route add -net %s -interface %s", route.Dst.String(), ifName)
-		} else {
-			cmd := fmt.Sprintf("route add -inet6 %s -interface %s", route.Dst.String(), ifName)
-		}
-		log.Debugf("[TUN] %s", cmd)
-		args := strings.Split(cmd, " ")
-		if er := exec.Command(args[0], args[1:]...).Run(); er != nil {
-			return fmt.Errorf("%s: %v", cmd, er)
-		}
-	}
-	return nil
-}

pkg/tun/tun_darwin.go

@@ -1,15 +1,18 @@
-//go:build darwin
+//go:build darwin || freebsd || openbsd
 
 package tun
 
 import (
 	"fmt"
 	"net"
-	"os/exec"
+	"net/netip"
-	"strings"
+	"runtime"
+	"unsafe"
 
 	"github.com/containernetworking/cni/pkg/types"
 	log "github.com/sirupsen/logrus"
+	"golang.org/x/net/route"
+	"golang.org/x/sys/unix"
 	"golang.zx2c4.com/wireguard/tun"
 
 	"github.com/wencaiwulue/kubevpn/v2/pkg/config"
@@ -45,27 +48,29 @@ func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) {
 		if ipv4, _, err = net.ParseCIDR(cfg.Addr); err != nil {
 			return
 		}
-		setIPv4Cmd := fmt.Sprintf("ifconfig %s inet %s %s mtu %d up", name, cfg.Addr, ipv4.String(), mtu)
+		var prefix netip.Prefix
-		log.Debugf("[TUN] %s", setIPv4Cmd)
+		prefix, err = netip.ParsePrefix(cfg.Addr)
-		args := strings.Split(setIPv4Cmd, " ")
+		if err != nil {
-		if err = exec.Command(args[0], args[1:]...).Run(); err != nil {
+			return
-			err = fmt.Errorf("%s: %v", setIPv4Cmd, err)
+		}
+		err = setInterfaceAddress(name, prefix)
+		if err != nil {
 			return
 		}
 	}
 
 	// set ipv6 address
 	if cfg.Addr6 != "" {
-		var ipv6CIDR *net.IPNet
+		if ipv6, _, err = net.ParseCIDR(cfg.Addr6); err != nil {
-		if ipv6, ipv6CIDR, err = net.ParseCIDR(cfg.Addr6); err != nil {
 			return
 		}
-		ones, _ := ipv6CIDR.Mask.Size()
+		var prefix netip.Prefix
-		setIPv6Cmd := fmt.Sprintf("ifconfig %s inet6 %s prefixlen %d alias", name, ipv6.String(), ones)
+		prefix, err = netip.ParsePrefix(cfg.Addr6)
-		log.Debugf("[TUN] %s", setIPv6Cmd)
+		if err != nil {
-		args := strings.Split(setIPv6Cmd, " ")
+			return
-		if err = exec.Command(args[0], args[1:]...).Run(); err != nil {
+		}
-			err = fmt.Errorf("%s: %v", setIPv6Cmd, err)
+		err = setInterfaceAddress(name, prefix)
+		if err != nil {
 			return
 		}
 	}
@@ -88,23 +93,110 @@ func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) {
 }
 
 func addTunRoutes(ifName string, routes ...types.Route) error {
-	for _, route := range routes {
+	tunIfi, err := net.InterfaceByName(ifName)
-		if route.Dst.String() == "" {
+	if err != nil {
+		return err
+	}
+	gw := &route.LinkAddr{Index: tunIfi.Index}
+
+	for _, r := range routes {
+		if r.Dst.String() == "" {
 			continue
 		}
-		var cmd string
+		var prefix netip.Prefix
-		// ipv4
+		prefix, err = netip.ParsePrefix(r.Dst.String())
-		if route.Dst.IP.To4() != nil {
-			cmd = fmt.Sprintf("route add -net %s -interface %s", route.Dst.String(), ifName)
-		} else { // ipv6
-			cmd = fmt.Sprintf("route add -inet6 %s -interface %s", route.Dst.String(), ifName)
-		}
-		log.Debugf("[TUN] %s", cmd)
-		args := strings.Split(cmd, " ")
-		err := exec.Command(args[0], args[1:]...).Run()
 		if err != nil {
-			return fmt.Errorf("run cmd %s: %v", cmd, err)
+			return err
+		}
+		err = addRoute(1, prefix, gw)
+		if err != nil {
+			return fmt.Errorf("failed to add route: %v", err)
 		}
 	}
 	return nil
 }
+
+// struct ifaliasreq
+type inet4AliasReq struct {
+	name    [unix.IFNAMSIZ]byte
+	addr    unix.RawSockaddrInet4
+	dstaddr unix.RawSockaddrInet4
+	mask    unix.RawSockaddrInet4
+}
+
+// ifaliasreq
+type inet6AliasReq struct {
+	name     [unix.IFNAMSIZ]byte
+	addr     unix.RawSockaddrInet6
+	dstaddr  unix.RawSockaddrInet6
+	mask     unix.RawSockaddrInet6
+	flags    int32
+	lifetime struct {
+		expire    float64
+		preferred float64
+		valid     uint32
+		pref      uint32
+	}
+}
+
+// IPv6 SIOCAIFADDR
+const (
+	siocAIFAddrInet6 = (unix.SIOCAIFADDR & 0xe000ffff) | (uint(unsafe.Sizeof(inet6AliasReq{})) << 16)
+	v6InfiniteLife   = 0xffffffff
+	v6IfaceFlags     = 0x0020 | 0x0400 // NODAD + SECURED
+)
+
+func setInterfaceAddress(ifName string, addr netip.Prefix) error {
+	ip := addr.Addr()
+	if ip.Is4() {
+		return setInet4Address(ifName, addr)
+	}
+	return setInet6Address(ifName, addr)
+}
+
+func setInet4Address(ifName string, prefix netip.Prefix) error {
+	fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0)
+	if err != nil {
+		return err
+	}
+	defer unix.Close(fd)
+	ip4 := prefix.Addr().As4()
+	req := &inet4AliasReq{
+		addr:    unix.RawSockaddrInet4{Family: unix.AF_INET, Len: unix.SizeofSockaddrInet4, Addr: ip4},
+		dstaddr: unix.RawSockaddrInet4{Family: unix.AF_INET, Len: unix.SizeofSockaddrInet4, Addr: ip4},
+		mask:    unix.RawSockaddrInet4{Family: unix.AF_INET, Len: unix.SizeofSockaddrInet4, Addr: [4]byte(net.CIDRMask(prefix.Bits(), 32))},
+	}
+	copy(req.name[:], ifName)
+
+	return ioctlRequest(fd, unix.SIOCAIFADDR, unsafe.Pointer(req))
+}
+
+func setInet6Address(ifName string, prefix netip.Prefix) error {
+	fd, err := unix.Socket(unix.AF_INET6, unix.SOCK_DGRAM, 0)
+	if err != nil {
+		return err
+	}
+	defer unix.Close(fd)
+
+	ip6 := prefix.Addr().As16()
+	req := &inet6AliasReq{
+		addr:    unix.RawSockaddrInet6{Family: unix.AF_INET6, Len: unix.SizeofSockaddrInet6, Addr: ip6},
+		dstaddr: unix.RawSockaddrInet6{Family: unix.AF_INET6, Len: unix.SizeofSockaddrInet6, Addr: ip6},
+		mask:    unix.RawSockaddrInet6{Family: unix.AF_INET6, Len: unix.SizeofSockaddrInet6, Addr: [16]byte(net.CIDRMask(prefix.Bits(), 128))},
+		flags:   v6IfaceFlags,
+	}
+	copy(req.name[:], ifName)
+	req.lifetime.valid = v6InfiniteLife
+	req.lifetime.pref = v6InfiniteLife
+
+	return ioctlRequest(fd, siocAIFAddrInet6, unsafe.Pointer(req))
+}
+
+func ioctlRequest(fd int, req uint, ptr unsafe.Pointer) error {
+	err := unix.IoctlSetInt(fd, req, int(uintptr(ptr)))
+	runtime.KeepAlive(ptr)
+	if err != nil {
+		return err
+	}
+	return nil
+}