performance: change tun lib to wg

Makefile

@@ -97,3 +97,8 @@ container-local: kubevpn-linux-amd64
 	docker tag ${IMAGE} ${IMAGE_DEFAULT}
 	docker push ${IMAGE_DEFAULT}
 
+.PHONY: container-test
+container-test: kubevpn-linux-amd64
+	docker build --platform linux/amd64 -t ${IMAGE} -f $(BUILD_DIR)/test.Dockerfile .
+	docker tag ${IMAGE} docker.io/naison/kubevpn:testp
+	docker push docker.io/naison/kubevpn:testp

build/test.Dockerfile

@@ -0,0 +1,5 @@
+FROM naison/kubevpn:test
+
+WORKDIR /app
+
+COPY bin/kubevpn /usr/local/bin/kubevpn

go.mod

@@ -17,7 +17,6 @@ require (
 	github.com/opencontainers/image-spec v1.0.3-0.20220303224323-02efb9a75ee1
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.9.0
-	github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8
 	github.com/spf13/cobra v1.6.1
 	golang.org/x/net v0.5.0
 	golang.org/x/sys v0.4.0

go.sum

@@ -522,8 +522,6 @@ github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic
 github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
-github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8=
-github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=

pkg/config/config.go

@@ -54,9 +54,9 @@ func init() {
 var Debug bool
 
 var (
-	SmallBufferSize  = 2 * 1024  // 2KB small buffer
-	MediumBufferSize = 8 * 1024  // 8KB medium buffer
-	LargeBufferSize  = 32 * 1024 // 32KB large buffer
+	SmallBufferSize  = (1 << 13) - 1 // 8KB small buffer
+	MediumBufferSize = (1 << 15) - 1 // 32KB medium buffer
+	LargeBufferSize  = (1 << 16) - 1 // 64KB large buffer
 )
 
 var (

pkg/core/tcphandler.go

@@ -71,8 +71,8 @@ func (h *fakeUdpHandler) tunnelServerUDP(tcpConn net.Conn, udpConn *net.UDPConn)
 	}()
 
 	go func() {
-		b := MPool.Get().([]byte)
-		defer MPool.Put(b)
+		b := LPool.Get().([]byte)
+		defer LPool.Put(b)
 
 		for {
 			n, err := udpConn.Read(b[:])

pkg/core/tunhandler.go

@@ -9,7 +9,8 @@ import (
 	"sync"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/songgao/water/waterutil"
+
+	"github.com/wencaiwulue/kubevpn/pkg/util"
 )
 
 func ipToTunRouteKey(ip net.IP) string {
@@ -103,8 +104,8 @@ func (h *tunHandler) transportTun(ctx context.Context, tun net.Conn, conn net.Pa
 	errChan := make(chan error, 2)
 	defer conn.Close()
 	go func() {
-		b := SPool.Get().([]byte)
-		defer SPool.Put(b)
+		b := LPool.Get().([]byte)
+		defer LPool.Put(b)
 
 		for {
 			err := func() error {
@@ -124,7 +125,7 @@ func (h *tunHandler) transportTun(ctx context.Context, tun net.Conn, conn net.Pa
 				}
 
 				var src, dst net.IP
-				if waterutil.IsIPv4(b[:n]) {
+				if util.IsIPv4(b[:n]) {
 					header, err := ipv4.ParseHeader(b[:n])
 					if err != nil {
 						log.Errorf("[tun] %s: %v", tun.LocalAddr(), err)
@@ -132,7 +133,7 @@ func (h *tunHandler) transportTun(ctx context.Context, tun net.Conn, conn net.Pa
 					}
 					log.Debugf("[tun] %s", header.String())
 					src, dst = header.Src, header.Dst
-				} else if waterutil.IsIPv6(b[:n]) {
+				} else if util.IsIPv6(b[:n]) {
 					header, err := ipv6.ParseHeader(b[:n])
 					if err != nil {
 						log.Errorf("[tun] %s: %v", tun.LocalAddr(), err)
@@ -184,7 +185,7 @@ func (h *tunHandler) transportTun(ctx context.Context, tun net.Conn, conn net.Pa
 				}
 
 				var src, dst net.IP
-				if waterutil.IsIPv4(b[:n]) {
+				if util.IsIPv4(b[:n]) {
 					header, err := ipv4.ParseHeader(b[:n])
 					if err != nil {
 						log.Errorf("[tun] %s: %v", tun.LocalAddr(), err)
@@ -192,7 +193,7 @@ func (h *tunHandler) transportTun(ctx context.Context, tun net.Conn, conn net.Pa
 					}
 					log.Debugf("[tun] %s", header.String())
 					src, dst = header.Src, header.Dst
-				} else if waterutil.IsIPv6(b[:n]) {
+				} else if util.IsIPv6(b[:n]) {
 					header, err := ipv6.ParseHeader(b[:n])
 					if err != nil {
 						log.Errorf("[tun] %s: %v", tun.LocalAddr(), err)

pkg/tun/tun.go

@@ -8,7 +8,9 @@ import (
 
 	"github.com/containernetworking/cni/pkg/types"
 	log "github.com/sirupsen/logrus"
-	"github.com/songgao/water"
+	"github.com/wencaiwulue/kubevpn/pkg/core"
+	"golang.zx2c4.com/wireguard/device"
+	"golang.zx2c4.com/wireguard/tun"
 
 	"github.com/wencaiwulue/kubevpn/pkg/config"
 )
@@ -74,16 +76,32 @@ func (l *tunListener) Close() error {
 }
 
 type tunConn struct {
-	ifce *water.Interface
+	ifce tun.Device
 	addr net.Addr
 }
 
 func (c *tunConn) Read(b []byte) (n int, err error) {
-	return c.ifce.Read(b)
+	offset := device.MessageTransportHeaderSize
+	bytes := core.LPool.Get().([]byte)
+	core.LPool.Put(bytes)
+
+	size, err := c.ifce.Read(bytes[:], offset)
+	if size == 0 || size > device.MaxSegmentSize-device.MessageTransportHeaderSize {
+		return 0, nil
+	}
+	return copy(b, bytes[offset:offset+size]), nil
 }
 
 func (c *tunConn) Write(b []byte) (n int, err error) {
-	return c.ifce.Write(b)
+	if len(b) < device.MessageTransportHeaderSize {
+		return 0, err
+	}
+	bytes := core.LPool.Get().([]byte)
+	defer core.LPool.Put(bytes)
+
+	copy(bytes[device.MessageTransportOffsetContent:], b)
+
+	return c.ifce.Write(bytes[:device.MessageTransportOffsetContent+len(b)], device.MessageTransportOffsetContent)
 }
 
 func (c *tunConn) Close() (err error) {

pkg/tun/tun_darwin.go

@@ -1,5 +1,4 @@
-//go:build !linux && !windows && darwin
-// +build !linux,!windows,darwin
+//go:build darwin
 
 package tun
 
@@ -12,7 +11,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/types"
 	log "github.com/sirupsen/logrus"
-	"github.com/songgao/water"
+	"golang.zx2c4.com/wireguard/tun"
 
 	"github.com/wencaiwulue/kubevpn/pkg/config"
 )
@@ -23,34 +22,39 @@ func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) {
 		return
 	}
 
-	ifce, err := water.New(water.Config{
-		DeviceType: water.TUN,
-	})
-	if err != nil {
-		return
-	}
-
 	mtu := cfg.MTU
 	if mtu <= 0 {
 		mtu = config.DefaultMTU
 	}
 
-	cmd := fmt.Sprintf("ifconfig %s inet %s %s mtu %d up", ifce.Name(), cfg.Addr, ip.String(), mtu)
+	var ifce tun.Device
+	ifce, err = tun.CreateTUN("utun", mtu)
+	if err != nil {
+		return
+	}
+
+	var name string
+	name, err = ifce.Name()
+	if err != nil {
+		return
+	}
+
+	cmd := fmt.Sprintf("ifconfig %s inet %s %s mtu %d up", name, cfg.Addr, ip.String(), mtu)
 	log.Debugf("[tun] %s", cmd)
 	args := strings.Split(cmd, " ")
 	if er := exec.Command(args[0], args[1:]...).Run(); er != nil {
 		err = fmt.Errorf("%s: %v", cmd, er)
 		return
 	}
-	if err = os.Setenv(config.EnvTunNameOrLUID, ifce.Name()); err != nil {
+	if err = os.Setenv(config.EnvTunNameOrLUID, name); err != nil {
 		return nil, nil, err
 	}
 
-	if err = addTunRoutes(ifce.Name(), cfg.Routes...); err != nil {
+	if err = addTunRoutes(name, cfg.Routes...); err != nil {
 		return
 	}
 
-	itf, err = net.InterfaceByName(ifce.Name())
+	itf, err = net.InterfaceByName(name)
 	if err != nil {
 		return
 	}

pkg/tun/tun_freebsd.go

@@ -1,5 +1,4 @@
-//go:build !linux && !windows && !darwin
-// +build !linux,!windows,!darwin
+//go:build freebsd
 
 package tun
 
@@ -11,7 +10,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/types"
 	log "github.com/sirupsen/logrus"
-	"github.com/songgao/water"
+	"golang.zx2c4.com/wireguard/tun"
 
 	"github.com/wencaiwulue/kubevpn/pkg/config"
 )
@@ -22,16 +21,21 @@ func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) {
 		return
 	}
 
-	ifce, err := water.New(water.Config{
-		DeviceType: water.TUN,
-	})
+	mtu := cfg.MTU
+	if mtu <= 0 {
+		mtu = config.DefaultMTU
+	}
+
+	var ifce tun.Device
+	ifce, err = tun.CreateTUN("utun", mtu)
 	if err != nil {
 		return
 	}
 
-	mtu := cfg.MTU
-	if mtu <= 0 {
-		mtu = config.DefaultMTU
+	var name string
+	name, err = ifce.Name()
+	if err != nil {
+		return
 	}
 
 	cmd := fmt.Sprintf("ifconfig %s inet %s mtu %d up", ifce.Name(), cfg.Addr, mtu)

pkg/tun/tun_linux.go

@@ -1,5 +1,4 @@
-//go:build linux && !windows && !darwin
-// +build linux,!windows,!darwin
+//go:build linux
 
 package tun
 
@@ -14,7 +13,7 @@ import (
 	"github.com/docker/libcontainer/netlink"
 	"github.com/milosgajdos/tenus"
 	log "github.com/sirupsen/logrus"
-	"github.com/songgao/water"
+	"golang.zx2c4.com/wireguard/tun"
 
 	"github.com/wencaiwulue/kubevpn/pkg/config"
 )
@@ -25,56 +24,58 @@ func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) {
 		return
 	}
 
-	ifce, err := water.New(water.Config{
-		DeviceType: water.TUN,
-		PlatformSpecificParams: water.PlatformSpecificParams{
-			Name: cfg.Name,
-		},
-	})
-	if err != nil {
-		return
-	}
-
-	link, err := tenus.NewLinkFrom(ifce.Name())
-	if err != nil {
-		return
-	}
-
 	mtu := cfg.MTU
 	if mtu <= 0 {
 		mtu = config.DefaultMTU
 	}
 
-	cmd := fmt.Sprintf("ip link set dev %s mtu %d", ifce.Name(), mtu)
+	var ifce tun.Device
+	ifce, err = tun.CreateTUN("utun", mtu)
+	if err != nil {
+		return
+	}
+
+	var name string
+	name, err = ifce.Name()
+	if err != nil {
+		return
+	}
+
+	link, err := tenus.NewLinkFrom(name)
+	if err != nil {
+		return
+	}
+
+	cmd := fmt.Sprintf("ip link set dev %s mtu %d", name, mtu)
 	log.Debugf("[tun] %s", cmd)
 	if er := link.SetLinkMTU(mtu); er != nil {
 		err = fmt.Errorf("%s: %v", cmd, er)
 		return
 	}
 
-	cmd = fmt.Sprintf("ip address add %s dev %s", cfg.Addr, ifce.Name())
+	cmd = fmt.Sprintf("ip address add %s dev %s", cfg.Addr, name)
 	log.Debugf("[tun] %s", cmd)
 	if er := link.SetLinkIp(ip, ipNet); er != nil {
 		err = fmt.Errorf("%s: %v", cmd, er)
 		return
 	}
 
-	cmd = fmt.Sprintf("ip link set dev %s up", ifce.Name())
+	cmd = fmt.Sprintf("ip link set dev %s up", name)
 	log.Debugf("[tun] %s", cmd)
 	if er := link.SetLinkUp(); er != nil {
 		err = fmt.Errorf("%s: %v", cmd, er)
 		return
 	}
 
-	if err = os.Setenv(config.EnvTunNameOrLUID, ifce.Name()); err != nil {
+	if err = os.Setenv(config.EnvTunNameOrLUID, name); err != nil {
 		return nil, nil, err
 	}
 
-	if err = addTunRoutes(ifce.Name(), cfg.Routes...); err != nil {
+	if err = addTunRoutes(name, cfg.Routes...); err != nil {
 		return
 	}
 
-	itf, err = net.InterfaceByName(ifce.Name())
+	itf, err = net.InterfaceByName(name)
 	if err != nil {
 		return
 	}

pkg/tun/tun_openbsd.go

@@ -0,0 +1,86 @@
+//go:build openbsd
+
+package tun
+
+import (
+	"fmt"
+	"net"
+	"os/exec"
+	"strings"
+
+	"github.com/containernetworking/cni/pkg/types"
+	log "github.com/sirupsen/logrus"
+	"golang.zx2c4.com/wireguard/tun"
+
+	"github.com/wencaiwulue/kubevpn/pkg/config"
+)
+
+func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) {
+	ip, _, err := net.ParseCIDR(cfg.Addr)
+	if err != nil {
+		return
+	}
+
+	mtu := cfg.MTU
+	if mtu <= 0 {
+		mtu = config.DefaultMTU
+	}
+
+	var ifce tun.Device
+	ifce, err = tun.CreateTUN("utun", mtu)
+	if err != nil {
+		return
+	}
+
+	var name string
+	name, err = ifce.Name()
+	if err != nil {
+		return
+	}
+
+	cmd := fmt.Sprintf("ifconfig %s inet %s mtu %d up", ifce.Name(), cfg.Addr, mtu)
+	log.Debugf("[tun] %s", cmd)
+	args := strings.Split(cmd, " ")
+	if er := exec.Command(args[0], args[1:]...).Run(); er != nil {
+		err = fmt.Errorf("%s: %v", cmd, er)
+		return
+	}
+
+	if err = os.Setenv(config.EnvTunNameOrLUID, ifce.Name()); err != nil {
+		return nil, nil, err
+	}
+
+	if err = addTunRoutes(ifce.Name(), cfg.Routes...); err != nil {
+		return
+	}
+
+	itf, err = net.InterfaceByName(ifce.Name())
+	if err != nil {
+		return
+	}
+
+	conn = &tunConn{
+		ifce: ifce,
+		addr: &net.IPAddr{IP: ip},
+	}
+	return
+}
+
+func addTunRoutes(ifName string, routes ...types.Route) error {
+	for _, route := range routes {
+		if route.Dst.String() == "" {
+			continue
+		}
+		cmd := fmt.Sprintf("route add -net %s -interface %s", route.Dst.String(), ifName)
+		log.Debugf("[tun] %s", cmd)
+		args := strings.Split(cmd, " ")
+		if er := exec.Command(args[0], args[1:]...).Run(); er != nil {
+			return fmt.Errorf("%s: %v", cmd, er)
+		}
+	}
+	return nil
+}
+
+func getInterface() (*net.Interface, error) {
+	return net.InterfaceByName(os.Getenv(config.EnvTunNameOrLUID))
+}

pkg/tun/tun_windows.go

@@ -1,5 +1,4 @@
-//go:build !linux && windows && !darwin
-// +build !linux,windows,!darwin
+//go:build windows
 
 package tun
 

pkg/util/ip.go

@@ -0,0 +1,9 @@
+package util
+
+func IsIPv4(packet []byte) bool {
+	return 4 == (packet[0] >> 4)
+}
+
+func IsIPv6(packet []byte) bool {
+	return 6 == (packet[0] >> 4)
+}