diff --git a/go.mod b/go.mod index bb5b9436..bc2050c9 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( github.com/containernetworking/cni v1.1.2 github.com/hashicorp/go-version v1.6.0 github.com/mattbaird/jsonpatch v0.0.0-20200820163806-098863c1fc24 + github.com/prometheus-community/pro-bing v0.1.0 github.com/schollz/progressbar/v3 v3.13.0 github.com/stretchr/testify v1.8.1 golang.org/x/exp v0.0.0-20230113213754-f9f960f08ad4 @@ -95,6 +96,7 @@ require ( go.starlark.net v0.0.0-20230112144946-fae38c8a6d89 // indirect golang.org/x/crypto v0.5.0 // indirect golang.org/x/mod v0.7.0 // indirect + golang.org/x/sync v0.1.0 // indirect golang.org/x/term v0.4.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.5.0 // indirect diff --git a/go.sum b/go.sum index a576d128..f7a3df52 100644 --- a/go.sum +++ b/go.sum @@ -277,6 +277,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus-community/pro-bing v0.1.0 h1:zjzLGhfNPP0bP1OlzGB+SJcguOViw7df12LPg2vUJh8= +github.com/prometheus-community/pro-bing v0.1.0/go.mod h1:BpWlHurD9flHtzq8wrh8QGWYz9ka9z9ZJAyOel8ej58= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/riobard/go-bloom v0.0.0-20200614022211-cdc8013cb5b3 h1:f/FNXud6gA3MNr8meMVVGxhp+QBTqY91tM8HjEuMjGg= @@ -438,6 +440,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/pkg/config/config.go b/pkg/config/config.go index d9aeb893..d200d5fb 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -29,6 +29,8 @@ const ( Proc = "/proc" CniNetName = "cni-net-dir-kubevpn" + + EnvTunNameOrLUID = "TunNameOrLUID" ) var ( diff --git a/pkg/dns/dns_linux.go b/pkg/dns/dns_linux.go index f7225b3b..cb93472a 100644 --- a/pkg/dns/dns_linux.go +++ b/pkg/dns/dns_linux.go @@ -9,22 +9,24 @@ import ( miekgdns "github.com/miekg/dns" log "github.com/sirupsen/logrus" + + "github.com/wencaiwulue/kubevpn/pkg/config" ) // systemd-resolve --status, systemd-resolve --flush-caches -func SetupDNS(config *miekgdns.ClientConfig, _ []string) error { - tunName := os.Getenv("tunName") +func SetupDNS(clientConfig *miekgdns.ClientConfig, _ []string) error { + tunName := os.Getenv(config.EnvTunNameOrLUID) if len(tunName) == 0 { tunName = "tun0" } cmd := exec.Command("systemd-resolve", []string{ "--set-dns", - config.Servers[0], + clientConfig.Servers[0], "--interface", tunName, - "--set-domain=" + config.Search[0], - "--set-domain=" + config.Search[1], - "--set-domain=" + config.Search[2], + "--set-domain=" + clientConfig.Search[0], + "--set-domain=" + clientConfig.Search[1], + "--set-domain=" + clientConfig.Search[2], }...) output, err := cmd.CombinedOutput() if err != nil { diff --git a/pkg/dns/dns_windows.go b/pkg/dns/dns_windows.go index 7d60b6bc..5c7ac634 100644 --- a/pkg/dns/dns_windows.go +++ b/pkg/dns/dns_windows.go @@ -15,18 +15,20 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/sys/windows" "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg" + + "github.com/wencaiwulue/kubevpn/pkg/config" ) -func SetupDNS(config *miekgdns.ClientConfig, _ []string) error { - getenv := os.Getenv("luid") - parseUint, err := strconv.ParseUint(getenv, 10, 64) +func SetupDNS(clientConfig *miekgdns.ClientConfig, _ []string) error { + env := os.Getenv(config.EnvTunNameOrLUID) + parseUint, err := strconv.ParseUint(env, 10, 64) if err != nil { log.Warningln(err) return err } luid := winipcfg.LUID(parseUint) var servers []netip.Addr - for _, s := range config.Servers { + for _, s := range clientConfig.Servers { var addr netip.Addr addr, err = netip.ParseAddr(s) if err != nil { @@ -35,14 +37,14 @@ func SetupDNS(config *miekgdns.ClientConfig, _ []string) error { } servers = append(servers, addr) } - err = luid.SetDNS(windows.AF_INET, servers, config.Search) + err = luid.SetDNS(windows.AF_INET, servers, clientConfig.Search) _ = exec.CommandContext(context.Background(), "ipconfig", "/flushdns").Run() if err != nil { log.Warningln(err) return err } //_ = updateNicMetric(tunName) - _ = addNicSuffixSearchList(config.Search) + _ = addNicSuffixSearchList(clientConfig.Search) return nil } diff --git a/pkg/handler/connect.go b/pkg/handler/connect.go index 9992c21e..55bec074 100644 --- a/pkg/handler/connect.go +++ b/pkg/handler/connect.go @@ -31,6 +31,7 @@ import ( "github.com/wencaiwulue/kubevpn/pkg/core" "github.com/wencaiwulue/kubevpn/pkg/dns" "github.com/wencaiwulue/kubevpn/pkg/route" + "github.com/wencaiwulue/kubevpn/pkg/tun" "github.com/wencaiwulue/kubevpn/pkg/util" ) @@ -132,7 +133,10 @@ func (c *ConnectOptions) DoConnect() (err error) { if err != nil { return err } - c.deleteFirewallRuleAndSetupDNS(ctx) + go c.addRouteDynamic(ctx) + c.deleteFirewallRule(ctx) + c.setupDNS() + log.Info("dns service ok") //c.detectConflictDevice() return } @@ -259,7 +263,60 @@ func (c *ConnectOptions) startLocalTunServe(ctx context.Context, forwardAddress return } -func (c *ConnectOptions) deleteFirewallRuleAndSetupDNS(ctx context.Context) { +// Listen all pod, add route if needed +func (c *ConnectOptions) addRouteDynamic(ctx context.Context) { + for ctx.Err() == nil { + func() { + w, err := c.clientset.CoreV1().Pods(v1.NamespaceAll).Watch(ctx, metav1.ListOptions{Watch: true, TimeoutSeconds: pointer.Int64(30)}) + if err != nil { + log.Debugf("wait pod failed, err: %v", err) + return + } + defer w.Stop() + for { + select { + case <-ctx.Done(): + return + case e, ok := <-w.ResultChan(): + if !ok || e.Type != watch.Added { + continue + } + var pod *v1.Pod + pod, ok = e.Object.(*v1.Pod) + if !ok { + continue + } + if pod.Spec.HostNetwork { + continue + } + ip := pod.Status.PodIP + if ip == "" { + continue + } + if pod.Status.Phase == v1.PodSucceeded || pod.Status.Phase == v1.PodFailed { + continue + } + go func(phase v1.PodPhase, name, ip string) { + // if pod is running and ping is ok, not need add route + if phase == v1.PodRunning { + if ok, _ := util.Ping(ip); ok { + return + } + } + err := tun.AddRoutes(tun.IPRoute{Dest: &net.IPNet{IP: net.ParseIP(ip), Mask: net.CIDRMask(32, 32)}}) + if err != nil { + log.Debugf("[route] add route failed, pod: %s, ip: %s,err: %v", name, ip, err) + } else { + log.Debugf("[route] add route ok, pod: %s, ip: %s", name, ip) + } + }(pod.Status.Phase, pod.Name, ip) + } + } + }() + } +} + +func (c *ConnectOptions) deleteFirewallRule(ctx context.Context) { if util.IsWindows() { if !util.FindRule() { util.AddFirewallRule() @@ -268,16 +325,14 @@ func (c *ConnectOptions) deleteFirewallRuleAndSetupDNS(ctx context.Context) { go util.DeleteWindowsFirewallRule(ctx) } go util.Heartbeats(ctx) - c.setupDNS() - log.Info("dns service ok") } func (c *ConnectOptions) detectConflictDevice() { - tun := os.Getenv("tunName") - if len(tun) == 0 { + tunName := os.Getenv(config.EnvTunNameOrLUID) + if len(tunName) == 0 { return } - if err := route.DetectAndDisableConflictDevice(tun); err != nil { + if err := route.DetectAndDisableConflictDevice(tunName); err != nil { log.Warnf("error occours while disable conflict devices, err: %v", err) } } diff --git a/pkg/handler/connect_test.go b/pkg/handler/connect_test.go index 8c2f2f21..d4faaaef 100644 --- a/pkg/handler/connect_test.go +++ b/pkg/handler/connect_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + probing "github.com/prometheus-community/pro-bing" log "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -144,3 +145,19 @@ func TestPatchAnnotation(t *testing.T) { } fmt.Println(object.(*unstructured.Unstructured).GetAnnotations()) } + +func TestPing(t *testing.T) { + ip := "10.233.98.197" + ping, _ := Ping(ip) + pinger, err := probing.NewPinger(ip) + if err != nil { + panic(err) + } + pinger.Count = 3 + err = pinger.Run() // Blocks until finished. + if err != nil { + panic(err) + } + stats := pinger.Statistics() // get send/receive/duplicate/rtt stats + fmt.Println(ping) +} diff --git a/pkg/tun/tun.go b/pkg/tun/tun.go index f3837bde..c670c884 100644 --- a/pkg/tun/tun.go +++ b/pkg/tun/tun.go @@ -8,6 +8,8 @@ import ( log "github.com/sirupsen/logrus" "github.com/songgao/water" + + "github.com/wencaiwulue/kubevpn/pkg/config" ) // Config is the config for TUN device. @@ -38,15 +40,11 @@ func Listener(config Config) (net.Listener, error) { if err != nil { return nil, err } - - ln.addr = conn.LocalAddr() - addrs, _ := ifce.Addrs() - _ = os.Setenv("tunName", ifce.Name) log.Debugf("[tun] %s: name: %s, mtu: %d, addrs: %s", conn.LocalAddr(), ifce.Name, ifce.MTU, addrs) + ln.addr = conn.LocalAddr() ln.conns <- conn - return ln, nil } @@ -116,3 +114,9 @@ type IPRoute struct { Dest *net.IPNet Gateway net.IP } + +// AddRoutes for outer called +func AddRoutes(routes ...IPRoute) error { + env := os.Getenv(config.EnvTunNameOrLUID) + return addTunRoutes(env, routes...) +} diff --git a/pkg/tun/tun_darwin.go b/pkg/tun/tun_darwin.go index 5379f110..5fd6f023 100644 --- a/pkg/tun/tun_darwin.go +++ b/pkg/tun/tun_darwin.go @@ -1,8 +1,12 @@ +//go:build !linux && !windows && darwin +// +build !linux,!windows,darwin + package tun import ( "fmt" "net" + "os" "os/exec" "strings" @@ -31,12 +35,15 @@ func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) { } cmd := fmt.Sprintf("ifconfig %s inet %s %s mtu %d up", ifce.Name(), cfg.Addr, ip.String(), mtu) - log.Debug("[tun]", cmd) + log.Debugf("[tun] %s", cmd) args := strings.Split(cmd, " ") if er := exec.Command(args[0], args[1:]...).Run(); er != nil { err = fmt.Errorf("%s: %v", cmd, er) return } + if err = os.Setenv(config.EnvTunNameOrLUID, ifce.Name()); err != nil { + return nil, nil, err + } if err = addTunRoutes(ifce.Name(), cfg.Routes...); err != nil { return @@ -60,7 +67,7 @@ func addTunRoutes(ifName string, routes ...IPRoute) error { continue } cmd := fmt.Sprintf("route add -net %s -interface %s", route.Dest.String(), ifName) - log.Debug("[tun]", cmd) + log.Debugf("[tun] %s", cmd) args := strings.Split(cmd, " ") if er := exec.Command(args[0], args[1:]...).Run(); er != nil { return fmt.Errorf("%s: %v", cmd, er) diff --git a/pkg/tun/tun_linux.go b/pkg/tun/tun_linux.go index fb8e5800..6ff7f9ae 100644 --- a/pkg/tun/tun_linux.go +++ b/pkg/tun/tun_linux.go @@ -1,9 +1,13 @@ +//go:build linux && !windows && !darwin +// +build linux,!windows,!darwin + package tun import ( "errors" "fmt" "net" + "os" "syscall" "github.com/docker/libcontainer/netlink" @@ -41,26 +45,30 @@ func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) { } cmd := fmt.Sprintf("ip link set dev %s mtu %d", ifce.Name(), mtu) - log.Debug("[tun]", cmd) + log.Debugf("[tun] %s", cmd) if er := link.SetLinkMTU(mtu); er != nil { err = fmt.Errorf("%s: %v", cmd, er) return } cmd = fmt.Sprintf("ip address add %s dev %s", cfg.Addr, ifce.Name()) - log.Debug("[tun]", cmd) + log.Debugf("[tun] %s", cmd) if er := link.SetLinkIp(ip, ipNet); er != nil { err = fmt.Errorf("%s: %v", cmd, er) return } cmd = fmt.Sprintf("ip link set dev %s up", ifce.Name()) - log.Debug("[tun]", cmd) + log.Debugf("[tun] %s", cmd) if er := link.SetLinkUp(); er != nil { err = fmt.Errorf("%s: %v", cmd, er) return } + if err = os.Setenv(config.EnvTunNameOrLUID, ifce.Name()); err != nil { + return nil, nil, err + } + if err = addTunRoutes(ifce.Name(), cfg.Routes...); err != nil { return } diff --git a/pkg/tun/tun_unix.go b/pkg/tun/tun_other.go similarity index 91% rename from pkg/tun/tun_unix.go rename to pkg/tun/tun_other.go index 277f0bcf..6bf7e1c6 100644 --- a/pkg/tun/tun_unix.go +++ b/pkg/tun/tun_other.go @@ -34,13 +34,17 @@ func createTun(cfg Config) (conn net.Conn, itf *net.Interface, err error) { } cmd := fmt.Sprintf("ifconfig %s inet %s mtu %d up", ifce.Name(), cfg.Addr, mtu) - log.Debug("[tun]", cmd) + log.Debugf("[tun] %s", cmd) args := strings.Split(cmd, " ") if er := exec.Command(args[0], args[1:]...).Run(); er != nil { err = fmt.Errorf("%s: %v", cmd, er) return } + if err = os.Setenv(config.EnvTunNameOrLUID, ifce.Name()); err != nil { + return nil, nil, err + } + if err = addTunRoutes(ifce.Name(), cfg.Routes...); err != nil { return } diff --git a/pkg/tun/tun_windows.go b/pkg/tun/tun_windows.go index 3b47461d..67f6e448 100644 --- a/pkg/tun/tun_windows.go +++ b/pkg/tun/tun_windows.go @@ -1,3 +1,6 @@ +//go:build !linux && windows && !darwin +// +build !linux,windows,!darwin + package tun import ( @@ -5,12 +8,15 @@ import ( "net" "net/netip" "os" + "strconv" "time" "github.com/pkg/errors" "golang.org/x/sys/windows" wireguardtun "golang.zx2c4.com/wireguard/tun" "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg" + + "github.com/wencaiwulue/kubevpn/pkg/config" ) func createTun(cfg Config) (net.Conn, *net.Interface, error) { @@ -26,9 +32,8 @@ func createTun(cfg Config) (net.Conn, *net.Interface, error) { if err != nil { return nil, nil, fmt.Errorf("failed to create TUN device: %w", err) } - _ = os.Setenv("luid", fmt.Sprintf("%d", tunDevice.(*wireguardtun.NativeTun).LUID())) - luid := winipcfg.LUID(tunDevice.(*wireguardtun.NativeTun).LUID()) + ifName := winipcfg.LUID(tunDevice.(*wireguardtun.NativeTun).LUID()) var prefix netip.Prefix prefix, err = netip.ParsePrefix(cfg.Addr) @@ -36,25 +41,35 @@ func createTun(cfg Config) (net.Conn, *net.Interface, error) { return nil, nil, err } - if err = luid.AddIPAddress(prefix); err != nil { + if err = ifName.AddIPAddress(prefix); err != nil { return nil, nil, err } - if err = addTunRoutes(luid, cfg.Gateway, cfg.Routes...); err != nil { + luid := fmt.Sprintf("%d", tunDevice.(*wireguardtun.NativeTun).LUID()) + if err = os.Setenv(config.EnvTunNameOrLUID, luid); err != nil { + return nil, nil, err + } + if err = addTunRoutes(luid /*cfg.Gateway,*/, cfg.Routes...); err != nil { return nil, nil, err } - row2, _ := luid.Interface() - iface, _ := net.InterfaceByIndex(int(row2.InterfaceIndex)) + row, _ := ifName.Interface() + iface, _ := net.InterfaceByIndex(int(row.InterfaceIndex)) return &winTunConn{ifce: tunDevice, addr: &net.IPAddr{IP: ip}}, iface, nil } -func addTunRoutes(ifName winipcfg.LUID, gw string, routes ...IPRoute) error { +func addTunRoutes(luid string, routes ...IPRoute) error { + parseUint, err := strconv.ParseUint(luid, 10, 64) + if err != nil { + return err + } + ifName := winipcfg.LUID(parseUint) _ = ifName.FlushRoutes(windows.AF_INET) for _, route := range routes { if route.Dest == nil { continue } + var gw string if gw != "" { route.Gateway = net.ParseIP(gw) } else { diff --git a/pkg/util/util.go b/pkg/util/util.go index 6a9a9978..cde961db 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -18,9 +18,8 @@ import ( dockerterm "github.com/moby/term" "github.com/pkg/errors" + probing "github.com/prometheus-community/pro-bing" log "github.com/sirupsen/logrus" - "golang.org/x/net/icmp" - "golang.org/x/net/ipv4" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -305,44 +304,19 @@ func BytesToInt(b []byte) uint32 { } func Ping(targetIP string) (bool, error) { - conn, err := icmp.ListenPacket("ip4:icmp", "0.0.0.0") + pinger, err := probing.NewPinger(targetIP) if err != nil { return false, err } - defer conn.Close() - - message := icmp.Message{ - Type: ipv4.ICMPTypeEcho, Code: 0, - Body: &icmp.Echo{ - ID: os.Getpid() & 0xffff, - Seq: 1, - Data: []byte("HELLO-R-U-THERE"), - }, - } - data, err := message.Marshal(nil) - if err != nil { - return false, nil - } - _, err = conn.WriteTo(data, &net.IPAddr{IP: net.ParseIP(targetIP)}) + pinger.SetPrivileged(true) + pinger.Count = 3 + pinger.Timeout = time.Millisecond * 1500 + err = pinger.Run() // Blocks until finished. if err != nil { return false, err } - - rb := make([]byte, 1500) - n, _, err := conn.ReadFrom(rb) - if err != nil { - return false, err - } - rm, err := icmp.ParseMessage(ipv4.ICMPTypeEchoReply.Protocol(), rb[:n]) - if err != nil { - return false, err - } - switch rm.Type { - case ipv4.ICMPTypeEchoReply: - return true, nil - default: - return false, nil - } + stat := pinger.Statistics() + return stat.PacketsRecv == stat.PacketsSent, err } func RolloutStatus(ctx1 context.Context, factory cmdutil.Factory, namespace, workloads string, timeout time.Duration) error { @@ -463,13 +437,13 @@ func RunWithRollingOutWithChecker(cmd *osexec.Cmd, checker func(log string)) (st } func Heartbeats(ctx context.Context) { - c2 := make(chan struct{}, 1) - c2 <- struct{}{} + c := make(chan struct{}, 1) + c <- struct{}{} for { select { case <-time.Tick(time.Second * 15): - c2 <- struct{}{} - case <-c2: + c <- struct{}{} + case <-c: for i := 0; i < 4; i++ { _, _ = Ping(config.RouterIP.String()) }