diff --git a/cmd/kubevpn/cmds/connect-fork.go b/cmd/kubevpn/cmds/connect-fork.go deleted file mode 100644 index e71efc58..00000000 --- a/cmd/kubevpn/cmds/connect-fork.go +++ /dev/null @@ -1,81 +0,0 @@ -package cmds - -import ( - "fmt" - "os" - - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - cmdutil "k8s.io/kubectl/pkg/cmd/util" - "k8s.io/kubectl/pkg/util/i18n" - "k8s.io/kubectl/pkg/util/templates" - - "github.com/wencaiwulue/kubevpn/pkg/config" - "github.com/wencaiwulue/kubevpn/pkg/handler" - "github.com/wencaiwulue/kubevpn/pkg/util" -) - -func CmdConnectFork(f cmdutil.Factory) *cobra.Command { - var connect = &handler.ConnectOptions{} - var sshConf = &util.SshConfig{} - var transferImage bool - cmd := &cobra.Command{ - Hidden: true, - Use: "connect-fork", - Short: i18n.T("Connect to kubernetes cluster network"), - Long: templates.LongDesc(i18n.T(`Connect to kubernetes cluster network`)), - Example: templates.Examples(i18n.T(` - # Connect to k8s cluster network - kubevpn connect - - # Connect to api-server behind of bastion host or ssh jump host - kubevpn connect --ssh-addr 192.168.1.100:22 --ssh-username root --ssh-keyfile /Users/naison/.ssh/ssh.pem - kubevpn connect --ssh-addr 192.168.1.100:22 --ssh-username root --ssh-keyfile ~/.ssh/ssh.pem - - # it also support ProxyJump, like - ┌──────┐ ┌──────┐ ┌──────┐ ┌──────┐ ┌────────────┐ - │ pc ├────►│ ssh1 ├────►│ ssh2 ├────►│ ssh3 ├─────►... ─────► │ api-server │ - └──────┘ └──────┘ └──────┘ └──────┘ └────────────┘ - kubevpn connect --ssh-alias - -`)), - PreRunE: func(cmd *cobra.Command, args []string) (err error) { - util.InitLogger(false) - if transferImage { - err = util.TransferImage(cmd.Context(), sshConf, config.OriginImage, config.Image, os.Stdout) - if err != nil { - return err - } - } - return handler.SshJumpAndSetEnv(cmd.Context(), sshConf, cmd.Flags(), true) - }, - RunE: func(cmd *cobra.Command, args []string) error { - if err := connect.InitClient(f); err != nil { - return err - } - _, err := connect.RentInnerIP(cmd.Context()) - if err != nil { - return err - } - err = connect.DoConnect(cmd.Context()) - defer connect.Cleanup() - if err != nil { - log.Errorln(err) - } else { - util.Print(os.Stdout, "Now you can access resources in the kubernetes cluster, enjoy it :)") - } - <-cmd.Context().Done() - return nil - }, - } - cmd.Flags().BoolVar(&config.Debug, "debug", false, "enable debug mode or not, true or false") - cmd.Flags().StringVar(&config.Image, "image", config.Image, "use this image to startup container") - cmd.Flags().StringArrayVar(&connect.ExtraCIDR, "extra-cidr", []string{}, "Extra cidr string, eg: --extra-cidr 192.168.0.159/24 --extra-cidr 192.168.1.160/32") - cmd.Flags().StringArrayVar(&connect.ExtraDomain, "extra-domain", []string{}, "Extra domain string, the resolved ip will add to route table, eg: --extra-domain test.abc.com --extra-domain foo.test.com") - cmd.Flags().BoolVar(&transferImage, "transfer-image", false, "transfer image to remote registry, it will transfer image "+config.OriginImage+" to flags `--image` special image, default: "+config.Image) - cmd.Flags().BoolVar(&connect.UseLocalDNS, "use-localdns", false, "if use-lcoaldns is true, kubevpn will start coredns listen at 53 to forward your dns queries. only support on linux now") - cmd.Flags().StringVar((*string)(&connect.Engine), "engine", string(config.EngineRaw), fmt.Sprintf(`transport engine ("%s"|"%s") %s: use gvisor and raw both (both performance and stable), %s: use raw mode (best stable)`, config.EngineMix, config.EngineRaw, config.EngineMix, config.EngineRaw)) - - addSshFlags(cmd, sshConf) - return cmd -} diff --git a/cmd/kubevpn/cmds/connect.go b/cmd/kubevpn/cmds/connect.go index e2dff4ff..ac58a8b8 100644 --- a/cmd/kubevpn/cmds/connect.go +++ b/cmd/kubevpn/cmds/connect.go @@ -23,7 +23,7 @@ import ( func CmdConnect(f cmdutil.Factory) *cobra.Command { var connect = &handler.ConnectOptions{} var sshConf = &util.SshConfig{} - var transferImage, foreground bool + var transferImage, foreground, lite bool cmd := &cobra.Command{ Use: "connect", Short: i18n.T("Connect to kubernetes cluster network"), @@ -66,8 +66,8 @@ func CmdConnect(f cmdutil.Factory) *cobra.Command { Level: int32(log.DebugLevel), } // if is foreground, send to sudo daemon server - if foreground { - cli := daemon.GetClient(true) + cli := daemon.GetClient(false) + if lite { resp, err := cli.ConnectFork(cmd.Context(), req) if err != nil { return err @@ -84,7 +84,6 @@ func CmdConnect(f cmdutil.Factory) *cobra.Command { fmt.Fprint(os.Stdout, recv.GetMessage()) } } else { - cli := daemon.GetClient(false) resp, err := cli.Connect(cmd.Context(), req) if err != nil { return err @@ -100,6 +99,8 @@ func CmdConnect(f cmdutil.Factory) *cobra.Command { } fmt.Fprint(os.Stdout, recv.GetMessage()) } + } + if !req.Foreground { util.Print(os.Stdout, "Now you can access resources in the kubernetes cluster, enjoy it :)") } return nil @@ -112,7 +113,8 @@ func CmdConnect(f cmdutil.Factory) *cobra.Command { cmd.Flags().BoolVar(&transferImage, "transfer-image", false, "transfer image to remote registry, it will transfer image "+config.OriginImage+" to flags `--image` special image, default: "+config.Image) cmd.Flags().BoolVar(&connect.UseLocalDNS, "use-localdns", false, "if use-lcoaldns is true, kubevpn will start coredns listen at 53 to forward your dns queries. only support on linux now") cmd.Flags().StringVar((*string)(&connect.Engine), "engine", string(config.EngineRaw), fmt.Sprintf(`transport engine ("%s"|"%s") %s: use gvisor and raw both (both performance and stable), %s: use raw mode (best stable)`, config.EngineMix, config.EngineRaw, config.EngineMix, config.EngineRaw)) - cmd.Flags().BoolVar(&foreground, "foreground", false, "connect to multiple cluster, you needs to special this options") + cmd.Flags().BoolVar(&foreground, "foreground", false, "Hang up") + cmd.Flags().BoolVar(&lite, "lite", false, "connect to multiple cluster in lite mode, you needs to special this options") addSshFlags(cmd, sshConf) return cmd diff --git a/cmd/kubevpn/cmds/root.go b/cmd/kubevpn/cmds/root.go index 63673ecb..c629ee4c 100644 --- a/cmd/kubevpn/cmds/root.go +++ b/cmd/kubevpn/cmds/root.go @@ -50,7 +50,6 @@ func NewKubeVPNCommand() *cobra.Command { Message: "Develop commands:", Commands: []*cobra.Command{ CmdConnect(factory), - CmdConnectFork(factory), CmdDisconnect(factory), CmdProxy(factory), CmdLeave(factory), diff --git a/cmd/kubevpn/cmds/status.go b/cmd/kubevpn/cmds/status.go index 0646ee7e..0c4d3a66 100644 --- a/cmd/kubevpn/cmds/status.go +++ b/cmd/kubevpn/cmds/status.go @@ -26,7 +26,7 @@ func CmdStatus(f cmdutil.Factory) *cobra.Command { return daemon.StartupDaemon(cmd.Context()) }, RunE: func(cmd *cobra.Command, args []string) error { - client, err := daemon.GetClient(true).Status( + client, err := daemon.GetClient(false).Status( cmd.Context(), &rpc.StatusRequest{}, ) diff --git a/pkg/config/config.go b/pkg/config/config.go index 7a56c87f..529f97de 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -91,6 +91,8 @@ const ( // transport mode ConfigKubeVPNTransportEngine = "transport-engine" + // hosts entry key word + HostsKeyWord = "# Add by KubeVPN" ) var ( diff --git a/pkg/daemon/action/connect-fork.go b/pkg/daemon/action/connect-fork.go index 8265344a..84abda62 100644 --- a/pkg/daemon/action/connect-fork.go +++ b/pkg/daemon/action/connect-fork.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "k8s.io/utils/pointer" defaultlog "log" log "github.com/sirupsen/logrus" @@ -86,6 +87,7 @@ func (svr *Server) ConnectFork(req *rpc.ConnectRequest, resp rpc.Daemon_ConnectF return err } svr.secondaryConnect = append(svr.secondaryConnect, connect) + return nil } @@ -147,7 +149,7 @@ func (svr *Server) redirectConnectForkToSudoDaemon(req *rpc.ConnectRequest, resp return err } - connResp, err := cli.Connect(ctx, req) + connResp, err := cli.ConnectFork(ctx, req) if err != nil { return err } @@ -165,6 +167,36 @@ func (svr *Server) redirectConnectForkToSudoDaemon(req *rpc.ConnectRequest, resp } svr.secondaryConnect = append(svr.secondaryConnect, connect) + + if req.Foreground { + <-resp.Context().Done() + for i := 0; i < len(svr.secondaryConnect); i++ { + if svr.secondaryConnect[i] == connect { + cli := svr.GetClient(false) + if cli == nil { + return fmt.Errorf("sudo daemon not start") + } + disconnect, err := cli.Disconnect(context.Background(), &rpc.DisconnectRequest{ + ID: pointer.Int32(int32(i)), + }) + if err != nil { + log.Errorf("disconnect error: %v", err) + return err + } + for { + recv, err := disconnect.Recv() + if err == io.EOF { + break + } else if err != nil { + return err + } + log.Info(recv.Message) + } + break + } + } + } + return nil } diff --git a/pkg/daemon/action/connect.go b/pkg/daemon/action/connect.go index 7a96b528..2014fcc4 100644 --- a/pkg/daemon/action/connect.go +++ b/pkg/daemon/action/connect.go @@ -181,6 +181,34 @@ func (svr *Server) redirectToSudoDaemon(req *rpc.ConnectRequest, resp rpc.Daemon svr.t = time.Now() svr.connect = connect + + // hangup + if req.Foreground { + <-resp.Context().Done() + + client := svr.GetClient(false) + if client == nil { + return fmt.Errorf("daemon not start") + } + disconnect, err := client.Disconnect(context.Background(), &rpc.DisconnectRequest{ + ID: pointer.Int32(int32(0)), + }) + if err != nil { + log.Errorf("disconnect error: %v", err) + return err + } + for { + recv, err := disconnect.Recv() + if err == io.EOF { + break + } else if err != nil { + log.Error(err) + return err + } + log.Info(recv.Message) + } + } + return nil } diff --git a/pkg/daemon/action/disconnect.go b/pkg/daemon/action/disconnect.go index fe331747..3a34ee41 100644 --- a/pkg/daemon/action/disconnect.go +++ b/pkg/daemon/action/disconnect.go @@ -8,6 +8,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/wencaiwulue/kubevpn/pkg/daemon/rpc" + "github.com/wencaiwulue/kubevpn/pkg/dns" ) func (svr *Server) Disconnect(req *rpc.DisconnectRequest, resp rpc.Daemon_DisconnectServer) error { @@ -77,6 +78,10 @@ func (svr *Server) Disconnect(req *rpc.DisconnectRequest, resp rpc.Daemon_Discon log.Errorf("index %d out of range", req.GetID()) } } + + if svr.connect == nil && len(svr.secondaryConnect) == 0 { + dns.CleanupHosts() + } return nil } diff --git a/pkg/daemon/action/quit.go b/pkg/daemon/action/quit.go index e54f4300..eaecdbb0 100644 --- a/pkg/daemon/action/quit.go +++ b/pkg/daemon/action/quit.go @@ -6,6 +6,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/wencaiwulue/kubevpn/pkg/daemon/rpc" + "github.com/wencaiwulue/kubevpn/pkg/dns" ) func (svr *Server) Quit(req *rpc.QuitRequest, resp rpc.Daemon_QuitServer) error { @@ -33,6 +34,9 @@ func (svr *Server) Quit(req *rpc.QuitRequest, resp rpc.Daemon_QuitServer) error log.Info("quit: cleanup connection") options.Cleanup() } + + dns.CleanupHosts() + return nil } diff --git a/pkg/dev/convert.go b/pkg/dev/convert.go index d40a3025..ead5b333 100644 --- a/pkg/dev/convert.go +++ b/pkg/dev/convert.go @@ -28,7 +28,7 @@ import ( "github.com/wencaiwulue/kubevpn/pkg/config" "github.com/wencaiwulue/kubevpn/pkg/cp" - "github.com/wencaiwulue/kubevpn/pkg/dns" + util2 "github.com/wencaiwulue/kubevpn/pkg/util" ) type RunConfig struct { @@ -186,7 +186,7 @@ func GetDNS(ctx context.Context, f util.Factory, ns, pod string) (*miekgdns.Clie return nil, err } - clientConfig, err := dns.GetDNSServiceIPFromPod(clientSet, client, config, pod, ns) + clientConfig, err := util2.GetDNSServiceIPFromPod(clientSet, client, config, pod, ns) if err != nil { return nil, err } diff --git a/pkg/dns/coredns.go b/pkg/dns/coredns.go index 37985cb6..1d68fd4f 100644 --- a/pkg/dns/coredns.go +++ b/pkg/dns/coredns.go @@ -9,17 +9,17 @@ type CoreFile struct { Content []byte } -// Gets the Caddyfile contents +// Body Gets the Caddyfile contents func (c *CoreFile) Body() []byte { return c.Content } -// Gets the path to the origin file +// Path Gets the path to the origin file func (c *CoreFile) Path() string { return "CoreFile" } -// The type of server this input is intended for +// ServerType The type of server this input is intended for func (c *CoreFile) ServerType() string { return "dns" } diff --git a/pkg/dns/dns.go b/pkg/dns/dns.go index ccf6ab3a..8a2efcbb 100644 --- a/pkg/dns/dns.go +++ b/pkg/dns/dns.go @@ -12,85 +12,39 @@ import ( "time" miekgdns "github.com/miekg/dns" - "github.com/pkg/errors" v12 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/watch" - "k8s.io/client-go/kubernetes" v13 "k8s.io/client-go/kubernetes/typed/core/v1" - "k8s.io/client-go/rest" "k8s.io/client-go/util/flowcontrol" - "github.com/wencaiwulue/kubevpn/pkg/util" + "github.com/wencaiwulue/kubevpn/pkg/config" ) -func GetDNSServiceIPFromPod(clientset *kubernetes.Clientset, restclient *rest.RESTClient, config *rest.Config, podName, namespace string) (*miekgdns.ClientConfig, error) { - resolvConfStr, err := util.Shell(clientset, restclient, config, podName, "", namespace, []string{"cat", "/etc/resolv.conf"}) - if err != nil { - return nil, err - } - resolvConf, err := miekgdns.ClientConfigFromReader(bytes.NewBufferString(resolvConfStr)) - if err != nil { - return nil, err - } - if ips, err := GetDNSIPFromDnsPod(clientset); err == nil && len(ips) != 0 { - resolvConf.Servers = ips - } +type Config struct { + Config *miekgdns.ClientConfig + Ns []string + UseLocalDNS bool + TunName string - // linux nameserver only support amount is 3, so if namespace too much, just use two, left one to system - if len(resolvConf.Servers) > 2 { - resolvConf.Servers = resolvConf.Servers[:2] - } - - return resolvConf, nil + Hosts []Entry } -func GetDNSIPFromDnsPod(clientset *kubernetes.Clientset) (ips []string, err error) { - var serviceList *v12.ServiceList - serviceList, err = clientset.CoreV1().Services(v1.NamespaceSystem).List(context.Background(), v1.ListOptions{ - LabelSelector: fields.OneTermEqualSelector("k8s-app", "kube-dns").String(), - }) - if err != nil { - return - } - for _, item := range serviceList.Items { - if len(item.Spec.ClusterIP) != 0 { - ips = append(ips, item.Spec.ClusterIP) - } - } - var podList *v12.PodList - podList, err = clientset.CoreV1().Pods(v1.NamespaceSystem).List(context.Background(), v1.ListOptions{ - LabelSelector: fields.OneTermEqualSelector("k8s-app", "kube-dns").String(), - }) - if err == nil { - for _, pod := range podList.Items { - if pod.Status.Phase == v12.PodRunning && pod.DeletionTimestamp == nil { - ips = append(ips, pod.Status.PodIP) - } - } - } - if len(ips) == 0 { - err = errors.New("can not found any dns service ip") - return - } - err = nil - return -} - -func AddServiceNameToHosts(ctx context.Context, serviceInterface v13.ServiceInterface, hosts ...Entry) { +func (c *Config) AddServiceNameToHosts(ctx context.Context, serviceInterface v13.ServiceInterface, hosts ...Entry) { rateLimiter := flowcontrol.NewTokenBucketRateLimiter(0.2, 1) defer rateLimiter.Stop() var last string serviceList, err := serviceInterface.List(ctx, v1.ListOptions{}) if err == nil && len(serviceList.Items) != 0 { - entry := generateHostsEntry(serviceList.Items, hosts) - if err = updateHosts(entry); err == nil { - last = entry + entry := c.generateHostsEntry(serviceList.Items, hosts) + if entry != "" { + if err = c.updateHosts(entry); err == nil { + last = entry + } } } for { @@ -102,6 +56,7 @@ func AddServiceNameToHosts(ctx context.Context, serviceInterface v13.ServiceInte w, err := serviceInterface.Watch(ctx, v1.ListOptions{ Watch: true, ResourceVersion: serviceList.ResourceVersion, }) + if err != nil { if utilnet.IsConnectionRefused(err) || apierrors.IsTooManyRequests(err) { time.Sleep(time.Second * 5) @@ -111,11 +66,11 @@ func AddServiceNameToHosts(ctx context.Context, serviceInterface v13.ServiceInte defer w.Stop() for { select { - case c, ok := <-w.ResultChan(): + case event, ok := <-w.ResultChan(): if !ok { return } - if watch.Error == c.Type || watch.Bookmark == c.Type { + if watch.Error == event.Type || watch.Bookmark == event.Type { continue } if !rateLimiter.TryAccept() { @@ -125,11 +80,14 @@ func AddServiceNameToHosts(ctx context.Context, serviceInterface v13.ServiceInte if err != nil { return } - entry := generateHostsEntry(list.Items, hosts) + entry := c.generateHostsEntry(list.Items, hosts) + if entry == "" { + continue + } if entry == last { continue } - if err = updateHosts(entry); err != nil { + if err = c.updateHosts(entry); err != nil { return } last = entry @@ -140,46 +98,44 @@ func AddServiceNameToHosts(ctx context.Context, serviceInterface v13.ServiceInte } } -func updateHosts(str string) error { - if len(str) == 0 { - return nil - } - +func (c *Config) updateHosts(str string) error { path := GetHostFile() file, err := os.ReadFile(path) if err != nil { return err } - split := strings.Split(string(file), "\n") - //for i := 0; i < len(split); i++ { - // if strings.Contains(split[i], "KubeVPN") { - // split = append(split[:i], split[i+1:]...) - // i-- - // } - //} - var sb strings.Builder + lines := strings.Split(string(file), "\n") + for i := 0; i < len(lines); i++ { + line := lines[i] + if strings.Contains(line, config.HostsKeyWord) { + for _, host := range c.Hosts { + if strings.Contains(line, host.Domain) { + lines = append(lines[:i], lines[i+1:]...) + i-- + } + } + } + } + if len(lines) == 0 { + return fmt.Errorf("empty hosts file") + } - sb.WriteString(strings.Join(split, "\n")) + var sb strings.Builder + sb.WriteString(strings.Join(lines, "\n")) if str != "" { sb.WriteString("\n") sb.WriteString(str) } - s := sb.String() // remove last empty line - strList := strings.Split(s, "\n") - for { - if len(strList) > 0 { - if strList[len(strList)-1] == "" { - strList = strList[:len(strList)-1] - continue - } - } - break + s = strings.TrimRight(s, "\n") + + if strings.TrimSpace(s) == "" { + return fmt.Errorf("empty content after update") } - return os.WriteFile(path, []byte(strings.Join(strList, "\n")), 0644) + return os.WriteFile(path, []byte(s), 0644) } type Entry struct { @@ -187,10 +143,11 @@ type Entry struct { Domain string } -func generateHostsEntry(list []v12.Service, hosts []Entry) string { +func (c *Config) generateHostsEntry(list []v12.Service, hosts []Entry) string { const ServiceKubernetes = "kubernetes" var entryList []Entry + // get all service ip for _, item := range list { if strings.EqualFold(item.Name, ServiceKubernetes) { continue @@ -214,7 +171,7 @@ func generateHostsEntry(list []v12.Service, hosts []Entry) string { }) entryList = append(entryList, hosts...) - // 判断是否是通的，或者直接用查询是否有同样条目的记录 + // if dns already works well, not needs to add it to hosts file for i := 0; i < len(entryList); i++ { e := entryList[i] host, err := net.LookupHost(e.Domain) @@ -224,11 +181,53 @@ func generateHostsEntry(list []v12.Service, hosts []Entry) string { } } + // if hosts file already contains item, not needs to add it to hosts file + file, err := os.ReadFile(GetHostFile()) + if err != nil { + return "" + } + lines := strings.Split(string(file), "\n") + for i := 0; i < len(lines); i++ { + line := lines[i] + for j := 0; j < len(entryList); j++ { + entry := entryList[j] + if strings.Contains(line, entry.Domain) && strings.Contains(line, entry.IP) { + entryList = append(entryList[:j], entryList[j+1:]...) + j-- + } + } + } + + c.Hosts = append(c.Hosts, entryList...) var sb = new(bytes.Buffer) w := tabwriter.NewWriter(sb, 1, 1, 1, ' ', 0) for _, e := range entryList { - _, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", e.IP, e.Domain, "", "# Add by KubeVPN") + _, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", e.IP, e.Domain, "", config.HostsKeyWord) } _ = w.Flush() return sb.String() } + +func CleanupHosts() error { + path := GetHostFile() + file, err := os.ReadFile(path) + if err != nil { + return err + } + lines := strings.Split(string(file), "\n") + for i := 0; i < len(lines); i++ { + line := lines[i] + if strings.Contains(line, config.HostsKeyWord) { + lines = append(lines[:i], lines[i+1:]...) + i-- + } + } + if len(lines) == 0 { + return fmt.Errorf("empty hosts file") + } + + var sb strings.Builder + sb.WriteString(strings.Join(lines, "\n")) + + return os.WriteFile(path, []byte(sb.String()), 0644) +} diff --git a/pkg/dns/dns_linux.go b/pkg/dns/dns_linux.go index 2e631dfe..762a6cfd 100644 --- a/pkg/dns/dns_linux.go +++ b/pkg/dns/dns_linux.go @@ -1,5 +1,4 @@ //go:build linux -// +build linux package dns @@ -21,7 +20,11 @@ import ( ) // systemd-resolve --status, systemd-resolve --flush-caches -func SetupDNS(clientConfig *miekgdns.ClientConfig, _ []string, useLocalDNS bool, tunName string) error { +func (c *Config) SetupDNS() error { + clientConfig := c.Config + useLocalDNS := c.UseLocalDNS + tunName := c.TunName + existNameservers := make([]string, 0) existSearches := make([]string, 0) filename := filepath.Join("/", "etc", "resolv.conf") @@ -39,7 +42,7 @@ func SetupDNS(clientConfig *miekgdns.ClientConfig, _ []string, useLocalDNS bool, } if useLocalDNS { - if err := SetupLocalDNS(clientConfig, existNameservers); err != nil { + if err = SetupLocalDNS(clientConfig, existNameservers); err != nil { return err } clientConfig.Servers[0] = "127.0.0.1" @@ -103,8 +106,8 @@ func SetupLocalDNS(clientConfig *miekgdns.ClientConfig, existNameservers []strin return nil } -func CancelDNS(tunName string) { - updateHosts("") +func (c *Config) CancelDNS() { + c.updateHosts("") filename := filepath.Join("/", "etc", "resolv.conf") _ = os.Rename(getBackupFilename(filename), filename) diff --git a/pkg/dns/dns_unix.go b/pkg/dns/dns_unix.go index 2adf511c..41f79517 100644 --- a/pkg/dns/dns_unix.go +++ b/pkg/dns/dns_unix.go @@ -1,5 +1,4 @@ //go:build darwin -// +build darwin package dns @@ -31,15 +30,18 @@ var resolv = "/etc/resolv.conf" // service.namespace.svc:port // service.namespace.svc.cluster:port // service.namespace.svc.cluster.local:port -func SetupDNS(config *miekgdns.ClientConfig, ns []string, _ bool, tunName string) error { - usingResolver(config, ns, tunName) +func (c *Config) SetupDNS() error { + c.usingResolver() _ = exec.Command("killall", "mDNSResponderHelper").Run() _ = exec.Command("killall", "-HUP", "mDNSResponder").Run() _ = exec.Command("dscacheutil", "-flushcache").Run() return nil } -func usingResolver(clientConfig *miekgdns.ClientConfig, ns []string, tunName string) { +func (c *Config) usingResolver() { + var clientConfig = c.Config + var ns = c.Ns + var err error _ = os.RemoveAll(filepath.Join("/", "etc", "resolver")) if err = os.MkdirAll(filepath.Join("/", "etc", "resolver"), fs.ModePerm); err != nil { @@ -80,7 +82,7 @@ func usingResolver(clientConfig *miekgdns.ClientConfig, ns []string, tunName str } } -func usingNetworkSetup(ip string, namespace string) { +func (c *Config) usingNetworkSetup(ip string, namespace string) { networkSetup(ip, namespace) var ctx context.Context ctx, cancel = context.WithCancel(context.Background()) @@ -153,13 +155,13 @@ func toString(config miekgdns.ClientConfig) string { return builder.String() } -func CancelDNS(tunName string) { +func (c *Config) CancelDNS() { if cancel != nil { cancel() } _ = os.RemoveAll(filepath.Join("/", "etc", "resolver")) //networkCancel() - //updateHosts("") + c.updateHosts("") } /* diff --git a/pkg/dns/dns_windows.go b/pkg/dns/dns_windows.go index e093d532..744ae060 100644 --- a/pkg/dns/dns_windows.go +++ b/pkg/dns/dns_windows.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package dns @@ -9,18 +8,23 @@ import ( "net/netip" "os/exec" - miekgdns "github.com/miekg/dns" log "github.com/sirupsen/logrus" "golang.org/x/sys/windows" "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg" ) -func SetupDNS(clientConfig *miekgdns.ClientConfig, _ []string, _ bool, tunName string) error { +func (c *Config) SetupDNS() error { + clientConfig := c.Config + tunName := c.TunName + tun, err := net.InterfaceByName(tunName) if err != nil { return err } - luid := winipcfg.LUIDFromIndex(tun.Index) + luid, err := winipcfg.LUIDFromIndex(uint32(tun.Index)) + if err != nil { + return err + } var servers []netip.Addr for _, s := range clientConfig.Servers { var addr netip.Addr @@ -41,13 +45,16 @@ func SetupDNS(clientConfig *miekgdns.ClientConfig, _ []string, _ bool, tunName s return nil } -func CancelDNS(tunName string) { - updateHosts("") - tun, err := net.InterfaceByName(tunName) +func (c *Config) CancelDNS() { + c.updateHosts("") + tun, err := net.InterfaceByName(c.TunName) + if err != nil { + return + } + luid, err := winipcfg.LUIDFromIndex(uint32(tun.Index)) if err != nil { return } - luid := winipcfg.LUIDFromIndex(tun.Index) _ = luid.FlushDNS(windows.AF_INET) _ = luid.FlushRoutes(windows.AF_INET) } diff --git a/pkg/dns/dns_server.go b/pkg/dns/forwardserver.go similarity index 100% rename from pkg/dns/dns_server.go rename to pkg/dns/forwardserver.go diff --git a/pkg/handler/cleaner.go b/pkg/handler/cleaner.go index 636297ea..e7e1b6bc 100644 --- a/pkg/handler/cleaner.go +++ b/pkg/handler/cleaner.go @@ -21,7 +21,6 @@ import ( "k8s.io/utils/pointer" "github.com/wencaiwulue/kubevpn/pkg/config" - "github.com/wencaiwulue/kubevpn/pkg/dns" "github.com/wencaiwulue/kubevpn/pkg/util" ) @@ -79,11 +78,10 @@ func (c *ConnectOptions) Cleanup() { c.cancel() } c.RollbackFuncList = c.RollbackFuncList[:] - name, err := c.GetTunDeviceName() - if err == nil { - log.Errorf("get tun device error: %v", err) + if c.dnsConfig != nil { + log.Infof("clean up dns") + c.dnsConfig.CancelDNS() } - dns.CancelDNS(name) log.Info("clean up successfully") util.CleanExtensionLib() } diff --git a/pkg/handler/connect.go b/pkg/handler/connect.go index c19644e8..7b11d9cc 100644 --- a/pkg/handler/connect.go +++ b/pkg/handler/connect.go @@ -92,6 +92,7 @@ type ConnectOptions struct { localTunIPv4 *net.IPNet localTunIPv6 *net.IPNet RollbackFuncList []func() + dnsConfig *dns.Config apiServerIPs []net.IP extraHost []dns.Entry @@ -620,7 +621,7 @@ func (c *ConnectOptions) setupDNS(ctx context.Context) error { log.Errorf("get running pod list failed, err: %v", err) return err } - relovConf, err := dns.GetDNSServiceIPFromPod(c.clientset, c.restclient, c.config, pod[0].GetName(), c.Namespace) + relovConf, err := util.GetDNSServiceIPFromPod(c.clientset, c.restclient, c.config, pod[0].GetName(), c.Namespace) if err != nil { log.Errorln(err) return err @@ -645,11 +646,18 @@ func (c *ConnectOptions) setupDNS(ctx context.Context) error { if err != nil { return err } - if err = dns.SetupDNS(relovConf, ns.UnsortedList(), c.UseLocalDNS, tunName); err != nil { + c.dnsConfig = &dns.Config{ + Config: relovConf, + Ns: ns.UnsortedList(), + UseLocalDNS: c.UseLocalDNS, + TunName: tunName, + Hosts: c.extraHost, + } + if err = c.dnsConfig.SetupDNS(); err != nil { return err } // dump service in current namespace for support DNS resolve service:port - go dns.AddServiceNameToHosts(ctx, c.clientset.CoreV1().Services(c.Namespace), c.extraHost...) + go c.dnsConfig.AddServiceNameToHosts(ctx, c.clientset.CoreV1().Services(c.Namespace), c.extraHost...) return nil } @@ -1077,7 +1085,7 @@ func (c *ConnectOptions) addExtraRoute(ctx context.Context) error { if len(c.ExtraDomain) == 0 { return nil } - ips, err := dns.GetDNSIPFromDnsPod(c.clientset) + ips, err := util.GetDNSIPFromDnsPod(c.clientset) if err != nil { return err } diff --git a/pkg/tun/tun_windows.go b/pkg/tun/tun_windows.go index f87abe5f..1752ac4e 100644 --- a/pkg/tun/tun_windows.go +++ b/pkg/tun/tun_windows.go @@ -91,7 +91,10 @@ func addTunRoutes(tunName string, routes ...types.Route) error { if err2 != nil { return err2 } - ifName := winipcfg.LUIDFromIndex(name.Index) + ifName, err := winipcfg.LUIDFromIndex(uint32(name.Index)) + if err != nil { + return err + } for _, route := range routes { if route.Dst.String() == "" { continue diff --git a/pkg/util/dns.go b/pkg/util/dns.go new file mode 100644 index 00000000..352179cd --- /dev/null +++ b/pkg/util/dns.go @@ -0,0 +1,67 @@ +package util + +import ( + "bytes" + "context" + + "github.com/miekg/dns" + "github.com/pkg/errors" + "k8s.io/api/core/v1" + v12 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +func GetDNSServiceIPFromPod(clientset *kubernetes.Clientset, restclient *rest.RESTClient, config *rest.Config, podName, namespace string) (*dns.ClientConfig, error) { + resolvConfStr, err := Shell(clientset, restclient, config, podName, "", namespace, []string{"cat", "/etc/resolv.conf"}) + if err != nil { + return nil, err + } + resolvConf, err := dns.ClientConfigFromReader(bytes.NewBufferString(resolvConfStr)) + if err != nil { + return nil, err + } + if ips, err := GetDNSIPFromDnsPod(clientset); err == nil && len(ips) != 0 { + resolvConf.Servers = ips + } + + // linux nameserver only support amount is 3, so if namespace too much, just use two, left one to system + if len(resolvConf.Servers) > 2 { + resolvConf.Servers = resolvConf.Servers[:2] + } + + return resolvConf, nil +} + +func GetDNSIPFromDnsPod(clientset *kubernetes.Clientset) (ips []string, err error) { + var serviceList *v1.ServiceList + serviceList, err = clientset.CoreV1().Services(v12.NamespaceSystem).List(context.Background(), v12.ListOptions{ + LabelSelector: fields.OneTermEqualSelector("k8s-app", "kube-dns").String(), + }) + if err != nil { + return + } + for _, item := range serviceList.Items { + if len(item.Spec.ClusterIP) != 0 { + ips = append(ips, item.Spec.ClusterIP) + } + } + var podList *v1.PodList + podList, err = clientset.CoreV1().Pods(v12.NamespaceSystem).List(context.Background(), v12.ListOptions{ + LabelSelector: fields.OneTermEqualSelector("k8s-app", "kube-dns").String(), + }) + if err == nil { + for _, pod := range podList.Items { + if pod.Status.Phase == v1.PodRunning && pod.DeletionTimestamp == nil { + ips = append(ips, pod.Status.PodIP) + } + } + } + if len(ips) == 0 { + err = errors.New("can not found any dns service ip") + return + } + err = nil + return +}