feat: support ssh jumper

This commit is contained in:
fengcaiwen
2023-03-03 18:12:59 +08:00
parent 0ba0659ce3
commit ac4c254cec
8 changed files with 232 additions and 19 deletions

View File

@@ -24,6 +24,7 @@ import (
func CmdConnect(f cmdutil.Factory) *cobra.Command {
var connect = handler.ConnectOptions{}
var sshConf = util.SshConfig{}
cmd := &cobra.Command{
Use: "connect",
Short: i18n.T("Connect to kubernetes cluster network, or proxy kubernetes workloads inbound traffic into local PC"),
@@ -54,7 +55,7 @@ func CmdConnect(f cmdutil.Factory) *cobra.Command {
}
},
RunE: func(cmd *cobra.Command, args []string) error {
if err := connect.InitClient(f); err != nil {
if err := connect.InitClient(f, cmd.Flags(), sshConf); err != nil {
return err
}
connect.PreCheckResource()
@@ -102,5 +103,11 @@ func CmdConnect(f cmdutil.Factory) *cobra.Command {
cmd.Flags().StringToStringVarP(&connect.Headers, "headers", "H", map[string]string{}, "Traffic with special headers with reverse it to local PC, you should startup your service after reverse workloads successfully, If not special, redirect all traffic to local PC, format is k=v, like: k1=v1,k2=v2")
cmd.Flags().BoolVar(&config.Debug, "debug", false, "enable debug mode or not, true or false")
cmd.Flags().StringVar(&config.Image, "image", config.Image, "use this image to startup container")
// for ssh jumper host
cmd.Flags().StringVar(&sshConf.Addr, "ssh-addr", "", "ssh connection string address to dial as <hostname>:<port>, eg: 127.0.0.1:22")
cmd.Flags().StringVar(&sshConf.User, "ssh-username", "", "username for ssh")
cmd.Flags().StringVar(&sshConf.Password, "ssh-password", "", "password for ssh")
cmd.Flags().StringVar(&sshConf.Keyfile, "ssh-keyfile", "", "file with private key for SSH authentication")
return cmd
}

View File

@@ -36,6 +36,7 @@ func CmdDev(f cmdutil.Factory) *cobra.Command {
Volumes: opts.NewListOpts(nil),
ExtraHosts: opts.NewListOpts(nil),
}
var sshConf = util.SshConfig{}
cmd := &cobra.Command{
Use: "dev",
Short: i18n.T("Proxy kubernetes workloads inbound traffic into local PC and dev in docker container"),
@@ -88,7 +89,7 @@ func CmdDev(f cmdutil.Factory) *cobra.Command {
}
}
if err := connect.InitClient(f); err != nil {
if err := connect.InitClient(f, cmd.Flags(), sshConf); err != nil {
return err
}
connect.PreCheckResource()
@@ -156,5 +157,11 @@ func CmdDev(f cmdutil.Factory) *cobra.Command {
cmd.Flags().StringVar(&devOptions.Platform, "platform", os.Getenv("DOCKER_DEFAULT_PLATFORM"), "Set platform if server is multi-platform capable")
cmd.Flags().StringVar(&devOptions.VolumeDriver, "volume-driver", "", "Optional volume driver for the container")
_ = cmd.Flags().SetAnnotation("platform", "version", []string{"1.32"})
// for ssh jumper host
cmd.Flags().StringVar(&sshConf.Addr, "ssh-addr", "", "ssh connection string address to dial as <hostname>:<port>, eg: 127.0.0.1:22")
cmd.Flags().StringVar(&sshConf.User, "ssh-username", "", "username for ssh")
cmd.Flags().StringVar(&sshConf.Password, "ssh-password", "", "password for ssh")
cmd.Flags().StringVar(&sshConf.Keyfile, "ssh-keyfile", "", "file with private key for SSH authentication")
return cmd
}

View File

@@ -6,16 +6,18 @@ import (
cmdutil "k8s.io/kubectl/pkg/cmd/util"
"github.com/wencaiwulue/kubevpn/pkg/handler"
"github.com/wencaiwulue/kubevpn/pkg/util"
)
func CmdReset(factory cmdutil.Factory) *cobra.Command {
var connect = handler.ConnectOptions{}
var sshConf = util.SshConfig{}
cmd := &cobra.Command{
Use: "reset",
Short: "Reset KubeVPN",
Long: `Reset KubeVPN if any error occurs`,
Run: func(cmd *cobra.Command, args []string) {
if err := connect.InitClient(factory); err != nil {
if err := connect.InitClient(factory, cmd.Flags(), sshConf); err != nil {
log.Fatal(err)
}
err := connect.Reset(cmd.Context())
@@ -25,5 +27,11 @@ func CmdReset(factory cmdutil.Factory) *cobra.Command {
log.Infoln("done")
},
}
// for ssh jumper host
cmd.Flags().StringVar(&sshConf.Addr, "ssh-addr", "", "ssh connection string address to dial as <hostname>:<port>, eg: 127.0.0.1:22")
cmd.Flags().StringVar(&sshConf.User, "ssh-username", "", "username for ssh")
cmd.Flags().StringVar(&sshConf.Password, "ssh-password", "", "password for ssh")
cmd.Flags().StringVar(&sshConf.Keyfile, "ssh-keyfile", "", "file with private key for SSH authentication")
return cmd
}

12
go.mod
View File

@@ -18,7 +18,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.9.0
github.com/spf13/cobra v1.6.1
golang.org/x/net v0.5.0
golang.org/x/net v0.5.0 // indirect
golang.org/x/sys v0.4.0
golang.zx2c4.com/wireguard v0.0.0-20220920152132-bb719d3a6e2c
golang.zx2c4.com/wireguard/windows v0.5.3
@@ -30,7 +30,7 @@ require (
k8s.io/apimachinery v0.26.1
k8s.io/cli-runtime v0.26.1
k8s.io/client-go v0.26.1
k8s.io/klog/v2 v2.80.1
k8s.io/klog/v2 v2.80.1 // indirect
k8s.io/kubectl v0.26.1
)
@@ -42,7 +42,10 @@ require (
github.com/mattbaird/jsonpatch v0.0.0-20200820163806-098863c1fc24
github.com/prometheus-community/pro-bing v0.1.0
github.com/schollz/progressbar/v3 v3.13.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.8.1
go.uber.org/automaxprocs v1.5.1
golang.org/x/crypto v0.2.0
golang.org/x/exp v0.0.0-20230113213754-f9f960f08ad4
golang.org/x/oauth2 v0.4.0
golang.org/x/text v0.6.0
@@ -54,7 +57,6 @@ require (
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/MakeNowJust/heredoc v1.0.0 // indirect
github.com/Microsoft/go-winio v0.6.0 // indirect
github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
@@ -121,16 +123,12 @@ require (
github.com/prometheus/procfs v0.8.0 // indirect
github.com/rivo/uniseg v0.4.3 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/viper v1.4.0 // indirect
github.com/theupdateframework/notary v0.7.0 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xlab/treeprint v1.1.0 // indirect
go.starlark.net v0.0.0-20230112144946-fae38c8a6d89 // indirect
go.uber.org/automaxprocs v1.5.1 // indirect
golang.org/x/crypto v0.2.0 // indirect
golang.org/x/mod v0.7.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/term v0.4.0 // indirect

7
go.sum
View File

@@ -68,7 +68,6 @@ github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2
github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00=
github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600=
github.com/Microsoft/hcsshim v0.8.25 h1:fRMwXiwk3qDwc0P05eHnh+y2v07JdtsfQ1fuAc69m9g=
github.com/Microsoft/hcsshim v0.9.6 h1:VwnDOgLeoi2du6dAznfmspNqTiwczvjv4K7NxuY9jsY=
github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU=
github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY=
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
@@ -184,8 +183,6 @@ github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoT
github.com/containerd/containerd v1.5.2/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g=
github.com/containerd/containerd v1.5.18 h1:doHr6cNxfOLTotWmZs6aZF6LrfJFcjmYFcWlRmQgYPM=
github.com/containerd/containerd v1.5.18/go.mod h1:7IN9MtIzTZH4WPEmD1gNH8bbTQXVX68yd3ZXxSHYCis=
github.com/containerd/containerd v1.6.17 h1:XDnJIeJW0cLf6v7/+N+6L9kGrChHeXekZp2VHu6OpiY=
github.com/containerd/containerd v1.6.17/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw=
github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
@@ -648,8 +645,6 @@ github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR
github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
github.com/moby/buildkit v0.9.0-rc1 h1:QxjQrpwQmCF3cbcf25kAebzXtIC9NV1dBqWTkscPHY0=
github.com/moby/buildkit v0.9.0-rc1/go.mod h1:en1WhqkDW8foqaeDAXvVxu2bcervCV7n5RJYE+w89bw=
github.com/moby/buildkit v0.11.2 h1:hNNsYuRssvFnp/qJ8FifStEUzROl5riPAEwk7cRzMjg=
github.com/moby/buildkit v0.11.2/go.mod h1:b5hR8j3BZaOj5+gf6yielP9YLT9mU92zy3zZtdoUTrw=
github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M7DBo=
github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
@@ -748,7 +743,6 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
github.com/pelletier/go-toml v1.9.1 h1:a6qW1EVNZWH9WGI6CsYdD8WAylkoXBS5yv0XHlh17Tc=
github.com/pelletier/go-toml v1.9.1/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -761,6 +755,7 @@ github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
github.com/prometheus-community/pro-bing v0.1.0 h1:zjzLGhfNPP0bP1OlzGB+SJcguOViw7df12LPg2vUJh8=
github.com/prometheus-community/pro-bing v0.1.0/go.mod h1:BpWlHurD9flHtzq8wrh8QGWYz9ka9z9ZJAyOel8ej58=
github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=

View File

@@ -2,8 +2,12 @@ package handler
import (
"context"
"encoding/json"
"fmt"
"net"
"net/netip"
"net/url"
"os"
"strconv"
"strings"
"time"
@@ -12,16 +16,23 @@ import (
netroute "github.com/libp2p/go-netroute"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/pflag"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/cli-runtime/pkg/genericclioptions"
"k8s.io/cli-runtime/pkg/resource"
"k8s.io/client-go/kubernetes"
v12 "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/tools/clientcmd/api"
"k8s.io/client-go/tools/clientcmd/api/latest"
clientcmdlatest "k8s.io/client-go/tools/clientcmd/api/latest"
cmdutil "k8s.io/kubectl/pkg/cmd/util"
"k8s.io/kubectl/pkg/polymorphichelpers"
"k8s.io/kubectl/pkg/scheme"
@@ -452,9 +463,11 @@ func Start(ctx context.Context, r core.Route) error {
return nil
}
func (c *ConnectOptions) InitClient(f cmdutil.Factory) (err error) {
func (c *ConnectOptions) InitClient(f cmdutil.Factory, flags *pflag.FlagSet, conf util.SshConfig) (err error) {
c.factory = f
if err = sshJump(conf, flags); err != nil {
return err
}
if c.config, err = c.factory.ToRESTConfig(); err != nil {
return
}
@@ -470,6 +483,76 @@ func (c *ConnectOptions) InitClient(f cmdutil.Factory) (err error) {
return
}
func sshJump(conf util.SshConfig, flags *pflag.FlagSet) (err error) {
if conf.Addr == "" {
return nil
}
defer func() {
if er := recover(); er != nil {
err = er.(error)
}
}()
configFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag()
lookup := flags.Lookup("kubeconfig")
if lookup != nil && lookup.Value != nil && lookup.Value.String() != "" {
configFlags.KubeConfig = pointer.String(lookup.Value.String())
}
matchVersionFlags := cmdutil.NewMatchVersionFlags(configFlags)
rawConfig, err := matchVersionFlags.ToRawKubeConfigLoader().RawConfig()
if err != nil {
return err
}
err = api.FlattenConfig(&rawConfig)
server := rawConfig.Clusters[rawConfig.Contexts[rawConfig.CurrentContext].Cluster].Server
u, err := url.Parse(server)
if err != nil {
return err
}
remote, err := netip.ParseAddrPort(u.Host)
if err != nil {
return err
}
var local = &netip.AddrPort{}
errChan := make(chan error, 1)
readyChan := make(chan struct{}, 1)
go func() {
err := util.Main(ctx, &remote, local, conf, readyChan)
if err != nil {
errChan <- err
return
}
}()
select {
case <-readyChan:
case err = <-errChan:
return err
}
rawConfig.Clusters[rawConfig.Contexts[rawConfig.CurrentContext].Cluster].Server = fmt.Sprintf("%s://%s", u.Scheme, local.String())
rawConfig.SetGroupVersionKind(schema.GroupVersionKind{Version: clientcmdlatest.Version, Kind: "Config"})
convertedObj, err := latest.Scheme.ConvertToVersion(&rawConfig, latest.ExternalVersion)
if err != nil {
return err
}
marshal, err := json.Marshal(convertedObj)
if err != nil {
return err
}
temp, err := os.CreateTemp("", "*.kubeconfig")
if err != nil {
return err
}
_ = temp.Close()
err = os.WriteFile(temp.Name(), marshal, 0644)
if err != nil {
return err
}
err = os.Setenv(clientcmd.RecommendedConfigPathEnvVar, temp.Name())
return err
}
// PreCheckResource transform user parameter to normal, example:
// pod: productpage-7667dfcddb-cbsn5
// replicast: productpage-7667dfcddb

View File

@@ -141,7 +141,7 @@ func TestPreCheck(t *testing.T) {
Namespace: "naison-test",
Workloads: []string{"services/authors"},
}
err := options.InitClient(factory)
err := options.InitClient(factory, cmd.Flags(), util.SshConfig{})
assert.Nil(t, err)
options.PreCheckResource()
fmt.Println(options.Workloads)

115
pkg/util/ssh.go Normal file
View File

@@ -0,0 +1,115 @@
package util
import (
"context"
"fmt"
"io"
"log"
"net"
"net/netip"
"os"
"golang.org/x/crypto/ssh"
)
type SshConfig struct {
Addr string
User string
Password string
Keyfile string
}
func Main(ctx context.Context, remoteEndpoint, localEndpoint *netip.AddrPort, conf SshConfig, done chan struct{}) error {
var auth []ssh.AuthMethod
if conf.Keyfile != "" {
auth = append(auth, publicKeyFile(conf.Keyfile))
}
if conf.Password != "" {
auth = append(auth, ssh.Password(conf.Password))
}
// refer to https://godoc.org/golang.org/x/crypto/ssh for other authentication types
sshConfig := &ssh.ClientConfig{
// SSH connection username
User: conf.User,
Auth: auth,
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
}
// Connect to SSH remote server using serverEndpoint
serverConn, err := ssh.Dial("tcp", conf.Addr, sshConfig)
if err != nil {
log.Fatalln(fmt.Printf("Dial INTO remote server error: %s", err))
}
// Listen on remote server port
var lc net.ListenConfig
listen, err := lc.Listen(ctx, "tcp", "localhost:0")
if err != nil {
return err
}
defer listen.Close()
local, err := netip.ParseAddrPort(listen.Addr().String())
if err != nil {
return err
}
*localEndpoint = local
done <- struct{}{}
// handle incoming connections on reverse forwarded tunnel
for {
select {
case <-ctx.Done():
return nil
default:
}
accept, err := listen.Accept()
if err != nil {
return err
}
listener, err := serverConn.Dial("tcp", remoteEndpoint.String())
if err != nil {
return err
}
go handleClient(accept, listener)
}
}
func publicKeyFile(file string) ssh.AuthMethod {
buffer, err := os.ReadFile(file)
if err != nil {
log.Fatalln(fmt.Sprintf("Cannot read SSH public key file %s", file))
return nil
}
key, err := ssh.ParsePrivateKey(buffer)
if err != nil {
log.Fatalln(fmt.Sprintf("Cannot parse SSH public key file %s", file))
return nil
}
return ssh.PublicKeys(key)
}
// From https://sosedoff.com/2015/05/25/ssh-port-forwarding-with-go.html
func handleClient(client net.Conn, remote net.Conn) {
defer client.Close()
chDone := make(chan bool)
// start remote -> local data transfer
go func() {
_, err := io.Copy(client, remote)
if err != nil {
log.Println(fmt.Sprintf("error while copy remote->local: %s", err))
}
chDone <- true
}()
// start local -> remote data transfer
go func() {
_, err := io.Copy(remote, client)
if err != nil {
log.Println(fmt.Sprintf("error while copy local->remote: %s", err))
}
chDone <- true
}()
<-chDone
}