diff --git a/cmd/kubevpn/cmds/connect.go b/cmd/kubevpn/cmds/connect.go index 46603276..9810cf82 100644 --- a/cmd/kubevpn/cmds/connect.go +++ b/cmd/kubevpn/cmds/connect.go @@ -41,8 +41,18 @@ func CmdConnect(f cmdutil.Factory) *cobra.Command { # Reverse proxy with mesh, traffic with header a=1, will hit local PC, otherwise no effect kubevpn connect --workloads=service/productpage --headers a=1 + + # Connect to api-server behind of bastion host or ssh jump host + kubevpn connect --ssh-addr 192.168.1.100:22 --ssh-username root --ssh-keyfile /Users/naison/.ssh/ssh.pem + + # it also support ProxyJump, like + ┌─────┐ ┌──────┐ ┌──────┐ ┌──────┐ ┌────────────┐ + │ pc ├─────►│ ssh1 ├────►│ ssh2 ├────►│ ssh3 ├─────►... ─────► │ api-server │ + └─────┘ └──────┘ └──────┘ └──────┘ └────────────┘ + kubevpn connect --ssh-alias + `)), - PreRun: func(*cobra.Command, []string) { + PreRunE: func(cmd *cobra.Command, args []string) (err error) { if !util.IsAdmin() { util.RunWithElevated() os.Exit(0) @@ -53,9 +63,10 @@ func CmdConnect(f cmdutil.Factory) *cobra.Command { if util.IsWindows() { driver.InstallWireGuardTunDriver() } + return handler.SshJump(sshConf, cmd.Flags()) }, RunE: func(cmd *cobra.Command, args []string) error { - if err := connect.InitClient(f, cmd.Flags(), sshConf); err != nil { + if err := connect.InitClient(f); err != nil { return err } connect.PreCheckResource() @@ -105,9 +116,10 @@ func CmdConnect(f cmdutil.Factory) *cobra.Command { cmd.Flags().StringVar(&config.Image, "image", config.Image, "use this image to startup container") // for ssh jumper host - cmd.Flags().StringVar(&sshConf.Addr, "ssh-addr", "", "ssh connection string address to dial as :, eg: 127.0.0.1:22") - cmd.Flags().StringVar(&sshConf.User, "ssh-username", "", "username for ssh") - cmd.Flags().StringVar(&sshConf.Password, "ssh-password", "", "password for ssh") - cmd.Flags().StringVar(&sshConf.Keyfile, "ssh-keyfile", "", "file with private key for SSH authentication") + cmd.Flags().StringVar(&sshConf.Addr, "ssh-addr", "", "Optional ssh jump server address to dial as :, eg: 127.0.0.1:22") + cmd.Flags().StringVar(&sshConf.User, "ssh-username", "", "Optional username for ssh jump server") + cmd.Flags().StringVar(&sshConf.Password, "ssh-password", "", "Optional password for ssh jump server") + cmd.Flags().StringVar(&sshConf.Keyfile, "ssh-keyfile", "", "Optional file with private key for SSH authentication") + cmd.Flags().StringVar(&sshConf.ConfigAlias, "ssh-alias", "", "Optional config alias with ~/.ssh/config for SSH authentication") return cmd } diff --git a/cmd/kubevpn/cmds/dev.go b/cmd/kubevpn/cmds/dev.go index 535f3d16..8db80194 100644 --- a/cmd/kubevpn/cmds/dev.go +++ b/cmd/kubevpn/cmds/dev.go @@ -50,9 +50,19 @@ func CmdDev(f cmdutil.Factory) *cobra.Command { # Reverse proxy with mesh, traffic with header a=1, will hit local PC, otherwise no effect kubevpn dev service/productpage --headers a=1 + + # Dev reverse proxy api-server behind of bastion host or ssh jump host + kubevpn dev deployment/productpage --ssh-addr 192.168.1.100:22 --ssh-username root --ssh-keyfile /Users/naison/.ssh/ssh.pem + + # it also support ProxyJump, like + ┌─────┐ ┌──────┐ ┌──────┐ ┌──────┐ ┌────────────┐ + │ pc ├─────►│ ssh1 ├────►│ ssh2 ├────►│ ssh3 ├─────►... ─────► │ api-server │ + └─────┘ └──────┘ └──────┘ └──────┘ └────────────┘ + kubevpn dev deployment/productpage --ssh-alias + `)), Args: cli.ExactArgs(1), - PreRun: func(cmd *cobra.Command, args []string) { + PreRunE: func(cmd *cobra.Command, args []string) error { if !util.IsAdmin() { util.RunWithElevated() os.Exit(0) @@ -62,6 +72,7 @@ func CmdDev(f cmdutil.Factory) *cobra.Command { if util.IsWindows() { driver.InstallWireGuardTunDriver() } + return handler.SshJump(sshConf, cmd.Flags()) }, RunE: func(cmd *cobra.Command, args []string) error { devOptions.Workload = args[0] @@ -89,7 +100,7 @@ func CmdDev(f cmdutil.Factory) *cobra.Command { } } - if err := connect.InitClient(f, cmd.Flags(), sshConf); err != nil { + if err := connect.InitClient(f); err != nil { return err } connect.PreCheckResource() @@ -159,9 +170,10 @@ func CmdDev(f cmdutil.Factory) *cobra.Command { _ = cmd.Flags().SetAnnotation("platform", "version", []string{"1.32"}) // for ssh jumper host - cmd.Flags().StringVar(&sshConf.Addr, "ssh-addr", "", "ssh connection string address to dial as :, eg: 127.0.0.1:22") - cmd.Flags().StringVar(&sshConf.User, "ssh-username", "", "username for ssh") - cmd.Flags().StringVar(&sshConf.Password, "ssh-password", "", "password for ssh") - cmd.Flags().StringVar(&sshConf.Keyfile, "ssh-keyfile", "", "file with private key for SSH authentication") + cmd.Flags().StringVar(&sshConf.Addr, "ssh-addr", "", "Optional ssh jump server address to dial as :, eg: 127.0.0.1:22") + cmd.Flags().StringVar(&sshConf.User, "ssh-username", "", "Optional username for ssh jump server") + cmd.Flags().StringVar(&sshConf.Password, "ssh-password", "", "Optional password for ssh jump server") + cmd.Flags().StringVar(&sshConf.Keyfile, "ssh-keyfile", "", "Optional file with private key for SSH authentication") + cmd.Flags().StringVar(&sshConf.ConfigAlias, "ssh-alias", "", "Optional config alias with ~/.ssh/config for SSH authentication") return cmd } diff --git a/cmd/kubevpn/cmds/reset.go b/cmd/kubevpn/cmds/reset.go index 968f8aba..0cbc4cdc 100644 --- a/cmd/kubevpn/cmds/reset.go +++ b/cmd/kubevpn/cmds/reset.go @@ -4,6 +4,8 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" "github.com/wencaiwulue/kubevpn/pkg/handler" "github.com/wencaiwulue/kubevpn/pkg/util" @@ -16,8 +18,28 @@ func CmdReset(factory cmdutil.Factory) *cobra.Command { Use: "reset", Short: "Reset KubeVPN", Long: `Reset KubeVPN if any error occurs`, + Example: templates.Examples(i18n.T(` + # Reset default namespace + kubevpn reset + + # Reset another namespace test + kubevpn reset -n test + + # Reset cluster api-server behind of bastion host or ssh jump host + kubevpn reset --ssh-addr 192.168.1.100:22 --ssh-username root --ssh-keyfile /Users/naison/.ssh/ssh.pem + + # it also support ProxyJump, like + ┌─────┐ ┌──────┐ ┌──────┐ ┌──────┐ ┌────────────┐ + │ pc ├─────►│ ssh1 ├────►│ ssh2 ├────►│ ssh3 ├─────►... ─────► │ api-server │ + └─────┘ └──────┘ └──────┘ └──────┘ └────────────┘ + kubevpn reset --ssh-alias + +`)), + PreRunE: func(cmd *cobra.Command, args []string) error { + return handler.SshJump(sshConf, cmd.Flags()) + }, Run: func(cmd *cobra.Command, args []string) { - if err := connect.InitClient(factory, cmd.Flags(), sshConf); err != nil { + if err := connect.InitClient(factory); err != nil { log.Fatal(err) } err := connect.Reset(cmd.Context()) @@ -29,9 +51,10 @@ func CmdReset(factory cmdutil.Factory) *cobra.Command { } // for ssh jumper host - cmd.Flags().StringVar(&sshConf.Addr, "ssh-addr", "", "ssh connection string address to dial as :, eg: 127.0.0.1:22") - cmd.Flags().StringVar(&sshConf.User, "ssh-username", "", "username for ssh") - cmd.Flags().StringVar(&sshConf.Password, "ssh-password", "", "password for ssh") - cmd.Flags().StringVar(&sshConf.Keyfile, "ssh-keyfile", "", "file with private key for SSH authentication") + cmd.Flags().StringVar(&sshConf.Addr, "ssh-addr", "", "Optional ssh jump server address to dial as :, eg: 127.0.0.1:22") + cmd.Flags().StringVar(&sshConf.User, "ssh-username", "", "Optional username for ssh jump server") + cmd.Flags().StringVar(&sshConf.Password, "ssh-password", "", "Optional password for ssh jump server") + cmd.Flags().StringVar(&sshConf.Keyfile, "ssh-keyfile", "", "Optional file with private key for SSH authentication") + cmd.Flags().StringVar(&sshConf.ConfigAlias, "ssh-alias", "", "Optional config alias with ~/.ssh/config for SSH authentication") return cmd } diff --git a/go.mod b/go.mod index 6f8d93ac..fda8e28f 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/containernetworking/cni v1.1.2 github.com/google/uuid v1.3.0 github.com/hashicorp/go-version v1.6.0 + github.com/kevinburke/ssh_config v1.2.0 github.com/libp2p/go-netroute v0.2.1 github.com/mattbaird/jsonpatch v0.0.0-20200820163806-098863c1fc24 github.com/prometheus-community/pro-bing v0.1.0 diff --git a/go.sum b/go.sum index 48f6b253..03d1f705 100644 --- a/go.sum +++ b/go.sum @@ -570,6 +570,8 @@ github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVE github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/pkg/handler/connect.go b/pkg/handler/connect.go index c2eee778..b7ede118 100644 --- a/pkg/handler/connect.go +++ b/pkg/handler/connect.go @@ -463,11 +463,8 @@ func Start(ctx context.Context, r core.Route) error { return nil } -func (c *ConnectOptions) InitClient(f cmdutil.Factory, flags *pflag.FlagSet, conf util.SshConfig) (err error) { +func (c *ConnectOptions) InitClient(f cmdutil.Factory) (err error) { c.factory = f - if err = sshJump(conf, flags); err != nil { - return err - } if c.config, err = c.factory.ToRESTConfig(); err != nil { return } @@ -483,9 +480,9 @@ func (c *ConnectOptions) InitClient(f cmdutil.Factory, flags *pflag.FlagSet, con return } -func sshJump(conf util.SshConfig, flags *pflag.FlagSet) (err error) { - if conf.Addr == "" { - return nil +func SshJump(conf util.SshConfig, flags *pflag.FlagSet) (err error) { + if conf.Addr == "" && conf.ConfigAlias == "" { + return } defer func() { if er := recover(); er != nil { @@ -493,9 +490,11 @@ func sshJump(conf util.SshConfig, flags *pflag.FlagSet) (err error) { } }() configFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag() - lookup := flags.Lookup("kubeconfig") - if lookup != nil && lookup.Value != nil && lookup.Value.String() != "" { - configFlags.KubeConfig = pointer.String(lookup.Value.String()) + if flags != nil { + lookup := flags.Lookup("kubeconfig") + if lookup != nil && lookup.Value != nil && lookup.Value.String() != "" { + configFlags.KubeConfig = pointer.String(lookup.Value.String()) + } } matchVersionFlags := cmdutil.NewMatchVersionFlags(configFlags) rawConfig, err := matchVersionFlags.ToRawKubeConfigLoader().RawConfig() @@ -517,12 +516,13 @@ func sshJump(conf util.SshConfig, flags *pflag.FlagSet) (err error) { errChan := make(chan error, 1) readyChan := make(chan struct{}, 1) go func() { - err := util.Main(ctx, &remote, local, conf, readyChan) + err := util.Main(&remote, local, conf, readyChan) if err != nil { errChan <- err return } }() + log.Infof("wait jump to bastion host...") select { case <-readyChan: case err = <-errChan: @@ -549,6 +549,7 @@ func sshJump(conf util.SshConfig, flags *pflag.FlagSet) (err error) { if err != nil { return err } + log.Infof("using temp kubeconfig %s", temp.Name()) err = os.Setenv(clientcmd.RecommendedConfigPathEnvVar, temp.Name()) return err } diff --git a/pkg/handler/remote_test.go b/pkg/handler/remote_test.go index 9e2abc38..e085a382 100644 --- a/pkg/handler/remote_test.go +++ b/pkg/handler/remote_test.go @@ -141,7 +141,7 @@ func TestPreCheck(t *testing.T) { Namespace: "naison-test", Workloads: []string{"services/authors"}, } - err := options.InitClient(factory, cmd.Flags(), util.SshConfig{}) + err := options.InitClient(factory) assert.Nil(t, err) options.PreCheckResource() fmt.Println(options.Workloads) diff --git a/pkg/util/ssh.go b/pkg/util/ssh.go index c1c30213..4fe2fe02 100644 --- a/pkg/util/ssh.go +++ b/pkg/util/ssh.go @@ -1,75 +1,94 @@ package util import ( - "context" + "errors" "fmt" "io" - "log" "net" "net/netip" "os" + "path/filepath" + "strconv" + "sync" + "time" + "github.com/kevinburke/ssh_config" + log "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" + "k8s.io/client-go/util/homedir" ) type SshConfig struct { - Addr string - User string - Password string - Keyfile string + Addr string + User string + Password string + Keyfile string + ConfigAlias string } -func Main(ctx context.Context, remoteEndpoint, localEndpoint *netip.AddrPort, conf SshConfig, done chan struct{}) error { - var auth []ssh.AuthMethod - if conf.Keyfile != "" { - auth = append(auth, publicKeyFile(conf.Keyfile)) +func Main(remoteEndpoint, localEndpoint *netip.AddrPort, conf SshConfig, done chan struct{}) error { + var remote *ssh.Client + var err error + if conf.ConfigAlias != "" { + remote, err = jumpRecursion(conf.ConfigAlias) + } else { + var auth []ssh.AuthMethod + if conf.Keyfile != "" { + auth = append(auth, publicKeyFile(conf.Keyfile)) + } + if conf.Password != "" { + auth = append(auth, ssh.Password(conf.Password)) + } + // refer to https://godoc.org/golang.org/x/crypto/ssh for other authentication types + sshConfig := &ssh.ClientConfig{ + // SSH connection username + User: conf.User, + Auth: auth, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + // Connect to SSH remote server using serverEndpoint + remote, err = ssh.Dial("tcp", conf.Addr, sshConfig) } - if conf.Password != "" { - auth = append(auth, ssh.Password(conf.Password)) - } - // refer to https://godoc.org/golang.org/x/crypto/ssh for other authentication types - sshConfig := &ssh.ClientConfig{ - // SSH connection username - User: conf.User, - Auth: auth, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - - // Connect to SSH remote server using serverEndpoint - serverConn, err := ssh.Dial("tcp", conf.Addr, sshConfig) if err != nil { - log.Fatalln(fmt.Printf("Dial INTO remote server error: %s", err)) + log.Errorf("Dial INTO remote server error: %s", err) + return err } // Listen on remote server port - var lc net.ListenConfig - listen, err := lc.Listen(ctx, "tcp", "localhost:0") + listen, err := net.Listen("tcp", "localhost:0") if err != nil { return err } defer listen.Close() - local, err := netip.ParseAddrPort(listen.Addr().String()) + + *localEndpoint, err = netip.ParseAddrPort(listen.Addr().String()) if err != nil { return err } - *localEndpoint = local done <- struct{}{} // handle incoming connections on reverse forwarded tunnel for { - select { - case <-ctx.Done(): - return nil - default: - } - accept, err := listen.Accept() + local, err := listen.Accept() if err != nil { - return err + log.Error(err) + continue } - listener, err := serverConn.Dial("tcp", remoteEndpoint.String()) - if err != nil { - return err - } - go handleClient(accept, listener) + go func() { + defer local.Close() + var conn net.Conn + var err error + for i := 0; i < 5; i++ { + conn, err = remote.Dial("tcp", remoteEndpoint.String()) + if err == nil { + break + } + time.Sleep(time.Second) + } + if conn == nil { + return + } + handleClient(local, conn) + }() } } @@ -88,16 +107,14 @@ func publicKeyFile(file string) ssh.AuthMethod { return ssh.PublicKeys(key) } -// From https://sosedoff.com/2015/05/25/ssh-port-forwarding-with-go.html func handleClient(client net.Conn, remote net.Conn) { - defer client.Close() chDone := make(chan bool) // start remote -> local data transfer go func() { _, err := io.Copy(client, remote) if err != nil { - log.Println(fmt.Sprintf("error while copy remote->local: %s", err)) + log.Debugf("error while copy remote->local: %s", err) } chDone <- true }() @@ -106,10 +123,131 @@ func handleClient(client net.Conn, remote net.Conn) { go func() { _, err := io.Copy(remote, client) if err != nil { - log.Println(fmt.Sprintf("error while copy local->remote: %s", err)) + log.Debugf("error while copy local->remote: %s", err) } chDone <- true }() <-chDone } + +func jumpRecursion(name string) (client *ssh.Client, err error) { + var jumper = "ProxyJump" + var bastionList = []*SshConfig{getBastion(name)} + for { + value := confList.Get(name, jumper) + if value != "" { + bastionList = append(bastionList, getBastion(value)) + name = value + continue + } + break + } + for i := len(bastionList) - 1; i >= 0; i-- { + if bastionList[i] == nil { + return nil, errors.New("config is nil") + } + if client == nil { + client, err = dial(bastionList[i]) + if err != nil { + return + } + } else { + client, err = jump(client, bastionList[i]) + if err != nil { + return + } + } + } + return +} + +func getBastion(name string) *SshConfig { + var host, port string + config := SshConfig{ + ConfigAlias: name, + } + var propertyList = []string{"ProxyJump", "Hostname", "User", "Port", "IdentityFile"} + for i, s := range propertyList { + value := confList.Get(name, s) + switch i { + case 0: + + case 1: + host = value + case 2: + config.User = value + case 3: + if port = value; port == "" { + port = strconv.Itoa(22) + } + case 4: + config.Keyfile = value + } + } + config.Addr = fmt.Sprintf("%s:%s", host, port) + return &config +} + +func dial(from *SshConfig) (*ssh.Client, error) { + // connect to the bastion host + return ssh.Dial("tcp", from.Addr, &ssh.ClientConfig{ + User: from.User, + Auth: []ssh.AuthMethod{publicKeyFile(from.Keyfile)}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + }) +} + +func jump(bClient *ssh.Client, to *SshConfig) (*ssh.Client, error) { + // Dial a connection to the service host, from the bastion + conn, err := bClient.Dial("tcp", to.Addr) + if err != nil { + return nil, err + } + + ncc, chans, reqs, err := ssh.NewClientConn(conn, to.Addr, &ssh.ClientConfig{ + User: to.User, + Auth: []ssh.AuthMethod{publicKeyFile(to.Keyfile)}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + }) + if err != nil { + return nil, err + } + + sClient := ssh.NewClient(ncc, chans, reqs) + return sClient, nil +} + +type conf []*ssh_config.Config + +func (c conf) Get(alias string, key string) string { + for _, s := range c { + if v, err := s.Get(alias, key); err == nil { + return v + } + } + return "" +} + +var once sync.Once +var confList conf + +func init() { + once.Do(func() { + strings := []string{ + filepath.Join(homedir.HomeDir(), ".ssh", "config"), + filepath.Join("/", "etc", "ssh", "ssh_config"), + } + for _, s := range strings { + file, err := os.ReadFile(s) + if err != nil { + continue + } + cfg, err := ssh_config.DecodeBytes(file) + if err != nil { + continue + } + confList = append(confList, cfg) + } + }) +} diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 1a5f6eec..064ec17a 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -7,6 +7,7 @@ import ( "regexp" "testing" + "github.com/kevinburke/ssh_config" log "github.com/sirupsen/logrus" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -109,3 +110,62 @@ func TestName(t *testing.T) { fmt.Println(compile.FindAllString(s, -1)) fmt.Println(v6.FindAllString(s, -1)) } + +func TestParse(t *testing.T) { + all, _ := ssh_config.GetAllStrict("ry-agd-of", "ProxyJump") + for _, s := range all { + println(s) + } +} + +func TestGetProxyJump(t *testing.T) { + value := confList.Get("ry-agd-of", "ProxyJump") + println(value) +} + +func TestJ(t *testing.T) { + + //sshConfig := &ssh.ClientConfig{ + // // SSH connection username + // User: "root", + // Auth: []ssh.AuthMethod{publicKeyFile("/Users/bytedance/.ssh/byte.pem")}, + // HostKeyCallback: ssh.InsecureIgnoreHostKey(), + //} + + // sClient is an ssh client connected to the service host, through the bastion host. + + var lc net.ListenConfig + ctx := context.Background() + listen, err := lc.Listen(ctx, "tcp", "localhost:8088") + if err != nil { + log.Fatal(err) + } + defer listen.Close() + fmt.Println(listen.Addr().String()) + + sClient, err := jump(nil, nil) + if err != nil { + log.Fatal(err) + } + + dial, err := sClient.Dial("tcp", "10.1.1.22:5443") + if err != nil { + log.Fatal(err) + } + + // handle incoming connections on reverse forwarded tunnel + for { + select { + case <-ctx.Done(): + return + default: + } + accept, err := listen.Accept() + if err != nil { + log.Fatal(err) + } + go func() { + handleClient(accept, dial) + }() + } +}