diff --git a/Makefile b/Makefile index 86be1bd3..e49ead72 100644 --- a/Makefile +++ b/Makefile @@ -97,3 +97,8 @@ container-local: kubevpn-linux-amd64 docker tag ${IMAGE} ${IMAGE_DEFAULT} docker push ${IMAGE_DEFAULT} +.PHONY: container-test +container-test: kubevpn-linux-amd64 + docker build --platform linux/amd64 -t ${IMAGE} -f $(BUILD_DIR)/test.Dockerfile . + docker tag ${IMAGE} docker.io/naison/kubevpn:testp + docker push docker.io/naison/kubevpn:testp \ No newline at end of file diff --git a/build/test.Dockerfile b/build/test.Dockerfile new file mode 100644 index 00000000..4b2526ae --- /dev/null +++ b/build/test.Dockerfile @@ -0,0 +1,5 @@ +FROM naison/kubevpn:test + +WORKDIR /app + +COPY bin/kubevpn /usr/local/bin/kubevpn \ No newline at end of file diff --git a/go.mod b/go.mod index 63644572..1b199b69 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,6 @@ require ( github.com/opencontainers/image-spec v1.0.3-0.20220303224323-02efb9a75ee1 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.0 - github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 github.com/spf13/cobra v1.6.1 golang.org/x/net v0.5.0 golang.org/x/sys v0.4.0 diff --git a/go.sum b/go.sum index 80d0e608..a881af5b 100644 --- a/go.sum +++ b/go.sum @@ -522,8 +522,6 @@ github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8= -github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= diff --git a/pkg/config/config.go b/pkg/config/config.go index 255b4856..0cd7c0dd 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -54,9 +54,9 @@ func init() { var Debug bool var ( - SmallBufferSize = 2 * 1024 // 2KB small buffer - MediumBufferSize = 8 * 1024 // 8KB medium buffer - LargeBufferSize = 32 * 1024 // 32KB large buffer + SmallBufferSize = (1 << 13) - 1 // 8KB small buffer + MediumBufferSize = (1 << 15) - 1 // 32KB medium buffer + LargeBufferSize = (1 << 16) - 1 // 64KB large buffer ) var ( diff --git a/pkg/core/tcphandler.go b/pkg/core/tcphandler.go index 2607931d..499a6356 100644 --- a/pkg/core/tcphandler.go +++ b/pkg/core/tcphandler.go @@ -71,8 +71,8 @@ func (h *fakeUdpHandler) tunnelServerUDP(tcpConn net.Conn, udpConn *net.UDPConn) }() go func() { - b := MPool.Get().([]byte) - defer MPool.Put(b) + b := LPool.Get().([]byte) + defer LPool.Put(b) for { n, err := udpConn.Read(b[:]) diff --git a/pkg/core/tunhandler.go b/pkg/core/tunhandler.go index 5c16817b..02fb31db 100644 --- a/pkg/core/tunhandler.go +++ b/pkg/core/tunhandler.go @@ -9,7 +9,8 @@ import ( "sync" log "github.com/sirupsen/logrus" - "github.com/songgao/water/waterutil" + + "github.com/wencaiwulue/kubevpn/pkg/util" ) func ipToTunRouteKey(ip net.IP) string { @@ -103,8 +104,8 @@ func (h *tunHandler) transportTun(ctx context.Context, tun net.Conn, conn net.Pa errChan := make(chan error, 2) defer conn.Close() go func() { - b := SPool.Get().([]byte) - defer SPool.Put(b) + b := LPool.Get().([]byte) + defer LPool.Put(b) for { err := func() error { @@ -124,7 +125,7 @@ func (h *tunHandler) transportTun(ctx context.Context, tun net.Conn, conn net.Pa } var src, dst net.IP - if waterutil.IsIPv4(b[:n]) { + if util.IsIPv4(b[:n]) { header, err := ipv4.ParseHeader(b[:n]) if err != nil { log.Errorf("[tun] %s: %v", tun.LocalAddr(), err) @@ -132,7 +133,7 @@ func (h *tunHandler) transportTun(ctx context.Context, tun net.Conn, conn net.Pa } log.Debugf("[tun] %s", header.String()) src, dst = header.Src, header.Dst - } else if waterutil.IsIPv6(b[:n]) { + } else if util.IsIPv6(b[:n]) { header, err := ipv6.ParseHeader(b[:n]) if err != nil { log.Errorf("[tun] %s: %v", tun.LocalAddr(), err) @@ -184,7 +185,7 @@ func (h *tunHandler) transportTun(ctx context.Context, tun net.Conn, conn net.Pa } var src, dst net.IP - if waterutil.IsIPv4(b[:n]) { + if util.IsIPv4(b[:n]) { header, err := ipv4.ParseHeader(b[:n]) if err != nil { log.Errorf("[tun] %s: %v", tun.LocalAddr(), err) @@ -192,7 +193,7 @@ func (h *tunHandler) transportTun(ctx context.Context, tun net.Conn, conn net.Pa } log.Debugf("[tun] %s", header.String()) src, dst = header.Src, header.Dst - } else if waterutil.IsIPv6(b[:n]) { + } else if util.IsIPv6(b[:n]) { header, err := ipv6.ParseHeader(b[:n]) if err != nil { log.Errorf("[tun] %s: %v", tun.LocalAddr(), err) diff --git a/pkg/tun/tun.go b/pkg/tun/tun.go index bcf69cb1..db9d964a 100644 --- a/pkg/tun/tun.go +++ b/pkg/tun/tun.go @@ -8,7 +8,9 @@ import ( "github.com/containernetworking/cni/pkg/types" log "github.com/sirupsen/logrus" - "github.com/songgao/water" + "github.com/wencaiwulue/kubevpn/pkg/core" + "golang.zx2c4.com/wireguard/device" + "golang.zx2c4.com/wireguard/tun" "github.com/wencaiwulue/kubevpn/pkg/config" ) @@ -74,16 +76,32 @@ func (l *tunListener) Close() error { } type tunConn struct { - ifce *water.Interface + ifce tun.Device addr net.Addr } func (c *tunConn) Read(b []byte) (n int, err error) { - return c.ifce.Read(b) + offset := device.MessageTransportHeaderSize + bytes := core.LPool.Get().([]byte) + core.LPool.Put(bytes) + + size, err := c.ifce.Read(bytes[:], offset) + if size == 0 || size > device.MaxSegmentSize-device.MessageTransportHeaderSize { + return 0, nil + } + return copy(b, bytes[offset:offset+size]), nil } func (c *tunConn) Write(b []byte) (n int, err error) { - return c.ifce.Write(b) + if len(b) < device.MessageTransportHeaderSize { + return 0, err + } + bytes := core.LPool.Get().([]byte) + defer core.LPool.Put(bytes) + + copy(bytes[device.MessageTransportOffsetContent:], b) + + return c.ifce.Write(bytes[:device.MessageTransportOffsetContent+len(b)], device.MessageTransportOffsetContent) } func (c *tunConn) Close() (err error) { diff --git a/pkg/tun/tun_darwin.go b/pkg/tun/tun_darwin.go index e46f7fc2..9f972ba1 100644 --- a/pkg/tun/tun_darwin.go +++ b/pkg/tun/tun_darwin.go @@ -1,5 +1,4 @@ -//go:build !linux && !windows && darwin -// +build !linux,!windows,darwin +//go:build darwin package tun @@ -12,7 +11,7 @@ import ( "github.com/containernetworking/cni/pkg/types" log "github.com/sirupsen/logrus" - "github.com/songgao/water" + "golang.zx2c4.com/wireguard/tun" "github.com/wencaiwulue/kubevpn/pkg/config" ) @@ -23,34 +22,39 @@ func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) { return } - ifce, err := water.New(water.Config{ - DeviceType: water.TUN, - }) - if err != nil { - return - } - mtu := cfg.MTU if mtu <= 0 { mtu = config.DefaultMTU } - cmd := fmt.Sprintf("ifconfig %s inet %s %s mtu %d up", ifce.Name(), cfg.Addr, ip.String(), mtu) + var ifce tun.Device + ifce, err = tun.CreateTUN("utun", mtu) + if err != nil { + return + } + + var name string + name, err = ifce.Name() + if err != nil { + return + } + + cmd := fmt.Sprintf("ifconfig %s inet %s %s mtu %d up", name, cfg.Addr, ip.String(), mtu) log.Debugf("[tun] %s", cmd) args := strings.Split(cmd, " ") if er := exec.Command(args[0], args[1:]...).Run(); er != nil { err = fmt.Errorf("%s: %v", cmd, er) return } - if err = os.Setenv(config.EnvTunNameOrLUID, ifce.Name()); err != nil { + if err = os.Setenv(config.EnvTunNameOrLUID, name); err != nil { return nil, nil, err } - if err = addTunRoutes(ifce.Name(), cfg.Routes...); err != nil { + if err = addTunRoutes(name, cfg.Routes...); err != nil { return } - itf, err = net.InterfaceByName(ifce.Name()) + itf, err = net.InterfaceByName(name) if err != nil { return } diff --git a/pkg/tun/tun_other.go b/pkg/tun/tun_freebsd.go similarity index 89% rename from pkg/tun/tun_other.go rename to pkg/tun/tun_freebsd.go index ab432e15..64d330cf 100644 --- a/pkg/tun/tun_other.go +++ b/pkg/tun/tun_freebsd.go @@ -1,5 +1,4 @@ -//go:build !linux && !windows && !darwin -// +build !linux,!windows,!darwin +//go:build freebsd package tun @@ -11,7 +10,7 @@ import ( "github.com/containernetworking/cni/pkg/types" log "github.com/sirupsen/logrus" - "github.com/songgao/water" + "golang.zx2c4.com/wireguard/tun" "github.com/wencaiwulue/kubevpn/pkg/config" ) @@ -22,16 +21,21 @@ func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) { return } - ifce, err := water.New(water.Config{ - DeviceType: water.TUN, - }) + mtu := cfg.MTU + if mtu <= 0 { + mtu = config.DefaultMTU + } + + var ifce tun.Device + ifce, err = tun.CreateTUN("utun", mtu) if err != nil { return } - mtu := cfg.MTU - if mtu <= 0 { - mtu = config.DefaultMTU + var name string + name, err = ifce.Name() + if err != nil { + return } cmd := fmt.Sprintf("ifconfig %s inet %s mtu %d up", ifce.Name(), cfg.Addr, mtu) diff --git a/pkg/tun/tun_linux.go b/pkg/tun/tun_linux.go index 2d498914..00bfa502 100644 --- a/pkg/tun/tun_linux.go +++ b/pkg/tun/tun_linux.go @@ -1,5 +1,4 @@ -//go:build linux && !windows && !darwin -// +build linux,!windows,!darwin +//go:build linux package tun @@ -14,7 +13,7 @@ import ( "github.com/docker/libcontainer/netlink" "github.com/milosgajdos/tenus" log "github.com/sirupsen/logrus" - "github.com/songgao/water" + "golang.zx2c4.com/wireguard/tun" "github.com/wencaiwulue/kubevpn/pkg/config" ) @@ -25,56 +24,58 @@ func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) { return } - ifce, err := water.New(water.Config{ - DeviceType: water.TUN, - PlatformSpecificParams: water.PlatformSpecificParams{ - Name: cfg.Name, - }, - }) - if err != nil { - return - } - - link, err := tenus.NewLinkFrom(ifce.Name()) - if err != nil { - return - } - mtu := cfg.MTU if mtu <= 0 { mtu = config.DefaultMTU } - cmd := fmt.Sprintf("ip link set dev %s mtu %d", ifce.Name(), mtu) + var ifce tun.Device + ifce, err = tun.CreateTUN("utun", mtu) + if err != nil { + return + } + + var name string + name, err = ifce.Name() + if err != nil { + return + } + + link, err := tenus.NewLinkFrom(name) + if err != nil { + return + } + + cmd := fmt.Sprintf("ip link set dev %s mtu %d", name, mtu) log.Debugf("[tun] %s", cmd) if er := link.SetLinkMTU(mtu); er != nil { err = fmt.Errorf("%s: %v", cmd, er) return } - cmd = fmt.Sprintf("ip address add %s dev %s", cfg.Addr, ifce.Name()) + cmd = fmt.Sprintf("ip address add %s dev %s", cfg.Addr, name) log.Debugf("[tun] %s", cmd) if er := link.SetLinkIp(ip, ipNet); er != nil { err = fmt.Errorf("%s: %v", cmd, er) return } - cmd = fmt.Sprintf("ip link set dev %s up", ifce.Name()) + cmd = fmt.Sprintf("ip link set dev %s up", name) log.Debugf("[tun] %s", cmd) if er := link.SetLinkUp(); er != nil { err = fmt.Errorf("%s: %v", cmd, er) return } - if err = os.Setenv(config.EnvTunNameOrLUID, ifce.Name()); err != nil { + if err = os.Setenv(config.EnvTunNameOrLUID, name); err != nil { return nil, nil, err } - if err = addTunRoutes(ifce.Name(), cfg.Routes...); err != nil { + if err = addTunRoutes(name, cfg.Routes...); err != nil { return } - itf, err = net.InterfaceByName(ifce.Name()) + itf, err = net.InterfaceByName(name) if err != nil { return } diff --git a/pkg/tun/tun_openbsd.go b/pkg/tun/tun_openbsd.go new file mode 100644 index 00000000..9bbf2ff7 --- /dev/null +++ b/pkg/tun/tun_openbsd.go @@ -0,0 +1,86 @@ +//go:build openbsd + +package tun + +import ( + "fmt" + "net" + "os/exec" + "strings" + + "github.com/containernetworking/cni/pkg/types" + log "github.com/sirupsen/logrus" + "golang.zx2c4.com/wireguard/tun" + + "github.com/wencaiwulue/kubevpn/pkg/config" +) + +func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) { + ip, _, err := net.ParseCIDR(cfg.Addr) + if err != nil { + return + } + + mtu := cfg.MTU + if mtu <= 0 { + mtu = config.DefaultMTU + } + + var ifce tun.Device + ifce, err = tun.CreateTUN("utun", mtu) + if err != nil { + return + } + + var name string + name, err = ifce.Name() + if err != nil { + return + } + + cmd := fmt.Sprintf("ifconfig %s inet %s mtu %d up", ifce.Name(), cfg.Addr, mtu) + log.Debugf("[tun] %s", cmd) + args := strings.Split(cmd, " ") + if er := exec.Command(args[0], args[1:]...).Run(); er != nil { + err = fmt.Errorf("%s: %v", cmd, er) + return + } + + if err = os.Setenv(config.EnvTunNameOrLUID, ifce.Name()); err != nil { + return nil, nil, err + } + + if err = addTunRoutes(ifce.Name(), cfg.Routes...); err != nil { + return + } + + itf, err = net.InterfaceByName(ifce.Name()) + if err != nil { + return + } + + conn = &tunConn{ + ifce: ifce, + addr: &net.IPAddr{IP: ip}, + } + return +} + +func addTunRoutes(ifName string, routes ...types.Route) error { + for _, route := range routes { + if route.Dst.String() == "" { + continue + } + cmd := fmt.Sprintf("route add -net %s -interface %s", route.Dst.String(), ifName) + log.Debugf("[tun] %s", cmd) + args := strings.Split(cmd, " ") + if er := exec.Command(args[0], args[1:]...).Run(); er != nil { + return fmt.Errorf("%s: %v", cmd, er) + } + } + return nil +} + +func getInterface() (*net.Interface, error) { + return net.InterfaceByName(os.Getenv(config.EnvTunNameOrLUID)) +} diff --git a/pkg/tun/tun_windows.go b/pkg/tun/tun_windows.go index 5b030bb4..7fce94d9 100644 --- a/pkg/tun/tun_windows.go +++ b/pkg/tun/tun_windows.go @@ -1,5 +1,4 @@ -//go:build !linux && windows && !darwin -// +build !linux,windows,!darwin +//go:build windows package tun diff --git a/pkg/util/ip.go b/pkg/util/ip.go new file mode 100644 index 00000000..8d73a019 --- /dev/null +++ b/pkg/util/ip.go @@ -0,0 +1,9 @@ +package util + +func IsIPv4(packet []byte) bool { + return 4 == (packet[0] >> 4) +} + +func IsIPv6(packet []byte) bool { + return 6 == (packet[0] >> 4) +}