fea: support route func for network (#53)

cmd/api/v5/cmd.go

@@ -48,4 +48,5 @@ func Commands(app *api.App) {
 	Guest{}.Commands(app)
 	Knock{}.Commands(app)
 	Output{}.Commands(app)
+	Route{}.Commands(app)
 }

cmd/api/v5/network.go

@@ -26,6 +26,7 @@ func (u Network) List(c *cli.Context) error {
 }
 
 func (u Network) Commands(app *api.App) {
+	openvpn := OpenVpn{}
 	app.Command(&cli.Command{
 		Name:    "network",
 		Aliases: []string{"net"},
@@ -37,6 +38,46 @@ func (u Network) Commands(app *api.App) {
 				Aliases: []string{"ls"},
 				Action:  u.List,
 			},
+			openvpn.Commands(),
 		},
 	})
 }
+
+type OpenVpn struct {
+	Cmd
+}
+
+func (o OpenVpn) Url(prefix, name string) string {
+	return prefix + "/api/network/" + name + "/openvpn/restart"
+}
+
+func (o OpenVpn) Restart(c *cli.Context) error {
+	network := c.String("network")
+	url := o.Url(c.String("url"), network)
+
+	clt := o.NewHttp(c.String("token"))
+	if err := clt.PostJSON(url, nil, nil); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (o OpenVpn) Commands() *cli.Command {
+	return &cli.Command{
+		Name:    "openvpn",
+		Usage:   "control openvpn",
+		Aliases: []string{"ov"},
+		Subcommands: []*cli.Command{
+			{
+				Name:    "restart",
+				Usage:   "restart openvpn for the network",
+				Aliases: []string{"ro"},
+				Flags: []cli.Flag{
+					&cli.StringFlag{Name: "network", Required: true},
+				},
+				Action: o.Restart,
+			},
+		},
+	}
+}

cmd/api/v5/route.go

@@ -0,0 +1,136 @@
+package v5
+
+import (
+	"github.com/luscis/openlan/cmd/api"
+	"github.com/luscis/openlan/pkg/libol"
+	"github.com/luscis/openlan/pkg/schema"
+	"github.com/urfave/cli/v2"
+)
+
+type Route struct {
+	Cmd
+}
+
+func (r Route) Url(prefix, name string) string {
+	return prefix + "/api/network/" + name + "/route"
+}
+
+func (r Route) Add(c *cli.Context) error {
+	network := c.String("network")
+	if len(network) == 0 {
+		return libol.NewErr("invalid network")
+	}
+	pr := &schema.PrefixRoute{
+		Prefix:  c.String("prefix"),
+		NextHop: c.String("nexthop"),
+		Metric:  c.Int("metric"),
+		Mode:    c.String("mode"),
+	}
+	url := r.Url(c.String("url"), network)
+	clt := r.NewHttp(c.String("token"))
+	if err := clt.PostJSON(url, pr, nil); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (r Route) Remove(c *cli.Context) error {
+	network := c.String("network")
+	if len(network) == 0 {
+		return libol.NewErr("invalid network")
+	}
+	pr := &schema.PrefixRoute{
+		Prefix:  c.String("prefix"),
+		NextHop: c.String("nexthop"),
+		Metric:  c.Int("metric"),
+		Mode:    c.String("mode"),
+	}
+	url := r.Url(c.String("url"), network)
+	clt := r.NewHttp(c.String("token"))
+	if err := clt.DeleteJSON(url, pr, nil); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (r Route) Save(c *cli.Context) error {
+	network := c.String("network")
+	url := r.Url(c.String("url"), network)
+
+	clt := r.NewHttp(c.String("token"))
+	if err := clt.PutJSON(url, nil, nil); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (r Route) Tmpl() string {
+	return `# total {{ len . }}
+{{ps -25 "prefix"}} {{ps -25 "nexthop"}} {{ps -8 "metric"}} {{ps -8 "mode"}} {{ps -15 "origin"}}
+{{- range . }}
+{{ps -25 .Prefix}} {{ps -25 .NextHop}} {{pi -8 .Metric }} {{ps -8 .Mode}} {{ps -15 .Origin}}
+{{- end }}
+`
+}
+
+func (r Route) List(c *cli.Context) error {
+	url := r.Url(c.String("url"), c.String("network"))
+	clt := r.NewHttp(c.String("token"))
+	var items []schema.PrefixRoute
+	if err := clt.GetJSON(url, &items); err != nil {
+		return err
+	}
+	return r.Out(items, c.String("format"), r.Tmpl())
+}
+
+func (r Route) Commands(app *api.App) {
+	app.Command(&cli.Command{
+		Name:    "route",
+		Aliases: []string{"op"},
+		Usage:   "Route configuration",
+		Subcommands: []*cli.Command{
+			{
+				Name:  "add",
+				Usage: "Add a route for the network",
+				Flags: []cli.Flag{
+					&cli.StringFlag{Name: "network", Required: true},
+					&cli.StringFlag{Name: "prefix"},
+					&cli.StringFlag{Name: "nexthop"},
+					&cli.IntFlag{Name: "metric"},
+					&cli.StringFlag{Name: "mode"},
+				},
+				Action: r.Add,
+			},
+			{
+				Name:    "remove",
+				Usage:   "Remove a route from the network",
+				Aliases: []string{"rm"},
+				Flags: []cli.Flag{
+					&cli.StringFlag{Name: "network", Required: true},
+					&cli.StringFlag{Name: "prefix"},
+					&cli.StringFlag{Name: "nexthop"},
+				},
+				Action: r.Remove,
+			},
+			{
+				Name:    "list",
+				Usage:   "Display all outputs of the network",
+				Aliases: []string{"ls"},
+				Flags: []cli.Flag{
+					&cli.StringFlag{Name: "network", Required: true},
+				},
+				Action: r.List,
+			},
+			{
+				Name:    "save",
+				Usage:   "Save all routes",
+				Aliases: []string{"sa"},
+				Flags: []cli.Flag{
+					&cli.StringFlag{Name: "network", Required: true},
+				},
+				Action: r.Save,
+			},
+		},
+	})
+}

pkg/api/api.go

@@ -47,6 +47,16 @@ type ZTruster interface {
 	ListKnock(name string, call func(obj schema.KnockRule))
 }
 
+type Router interface {
+	AddRoute(route *schema.PrefixRoute, switcher Switcher) error
+	DelRoute(route *schema.PrefixRoute, switcher Switcher) error
+	SaveRoute()
+}
+
+type Vpner interface {
+	RestartVpn()
+}
+
 type Qoser interface {
 	AddQosUser(name string, inSpeed float64) error
 	UpdateQosUser(name string, inSpeed float64) error
@@ -77,6 +87,8 @@ type Networker interface {
 	IfAddr() string
 	ACLer() ACLer
 	Outputer
+	Router
+	Vpner
 }
 
 var workers = make(map[string]Networker)

pkg/api/network.go

@@ -17,6 +17,7 @@ func (h Network) Router(router *mux.Router) {
 	router.HandleFunc("/api/network", h.List).Methods("GET")
 	router.HandleFunc("/api/network/{id}", h.Get).Methods("GET")
 	router.HandleFunc("/get/network/{id}/ovpn", h.Profile).Methods("GET")
+	router.HandleFunc("/api/network/{id}/openvpn/restart", h.RestartVPN).Methods("POST")
 }
 
 func (h Network) List(w http.ResponseWriter, r *http.Request) {
@@ -50,3 +51,18 @@ func (h Network) Profile(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, err.Error(), http.StatusNotFound)
 	}
 }
+
+func (h Network) RestartVPN(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	id := vars["id"]
+
+	worker := GetWorker(id)
+	if worker == nil {
+		http.Error(w, "Network not found", http.StatusInternalServerError)
+		return
+	}
+
+	worker.RestartVpn()
+
+	ResponseJson(w, true)
+}

pkg/api/route.go

@@ -0,0 +1,101 @@
+package api
+
+import (
+	"github.com/gorilla/mux"
+	"github.com/luscis/openlan/pkg/cache"
+	"github.com/luscis/openlan/pkg/models"
+	"github.com/luscis/openlan/pkg/schema"
+	"net/http"
+)
+
+type Route struct {
+	Switcher Switcher
+}
+
+func (rt Route) Router(router *mux.Router) {
+	router.HandleFunc("/api/network/{id}/route", rt.List).Methods("GET")
+	router.HandleFunc("/api/network/{id}/route", rt.Add).Methods("POST")
+	router.HandleFunc("/api/network/{id}/route", rt.Del).Methods("DELETE")
+	router.HandleFunc("/api/network/{id}/route", rt.Save).Methods("PUT")
+}
+
+func (rt Route) List(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	id := vars["id"]
+
+	routes := make([]schema.PrefixRoute, 0, 1024)
+
+	for u := range cache.Network.ListRoute(id) {
+		if u == nil {
+			break
+		}
+		routes = append(routes, models.NewRouteSchema(u))
+	}
+	ResponseJson(w, routes)
+}
+
+func (rt Route) Add(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	id := vars["id"]
+
+	worker := GetWorker(id)
+	if worker == nil {
+		http.Error(w, "Network not found", http.StatusInternalServerError)
+		return
+	}
+
+	pr := &schema.PrefixRoute{}
+	if err := GetData(r, pr); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if err := worker.AddRoute(pr, rt.Switcher); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	ResponseJson(w, true)
+
+}
+
+func (rt Route) Del(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	id := vars["id"]
+
+	worker := GetWorker(id)
+	if worker == nil {
+		http.Error(w, "Network not found", http.StatusInternalServerError)
+		return
+	}
+
+	pr := &schema.PrefixRoute{}
+	if err := GetData(r, pr); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if err := worker.DelRoute(pr, rt.Switcher); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	ResponseJson(w, true)
+
+}
+
+func (rt Route) Save(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	id := vars["id"]
+
+	worker := GetWorker(id)
+	if worker == nil {
+		http.Error(w, "Network not found", http.StatusInternalServerError)
+		return
+	}
+
+	worker.SaveRoute()
+
+	ResponseJson(w, true)
+
+}

pkg/api/url.go

@@ -26,4 +26,5 @@ func Add(router *mux.Router, switcher Switcher) {
 	QosApi{}.Router(router)
 	Output{Switcher: switcher}.Router(router)
 	ACL{}.Router(router)
+	Route{Switcher: switcher}.Router(router)
 }

pkg/cache/network.go

@@ -2,11 +2,11 @@ package cache
 
 import (
 	"encoding/binary"
-	"net"
+	co "github.com/luscis/openlan/pkg/config"
-
 	"github.com/luscis/openlan/pkg/libol"
 	"github.com/luscis/openlan/pkg/models"
 	"github.com/luscis/openlan/pkg/schema"
+	"net"
 )
 
 type network struct {
@@ -32,7 +32,45 @@ func (w *network) Get(name string) *models.Network {
 	return nil
 }
 
-//TODO add/del route
+// add/del route
+func (w *network) ListRoute(name string) <-chan *models.Route {
+	c := make(chan *models.Route, 128)
+
+	n := w.Get(name)
+
+	if n != nil {
+		go func() {
+			for _, route := range n.Routes {
+				if route != nil {
+					c <- route
+				}
+			}
+			c <- nil //Finish channel by nil.
+		}()
+	} else {
+		c <- nil
+	}
+	return c
+}
+
+func (w *network) DelRoute(name string, rt co.PrefixRoute) {
+	n := w.Get(name)
+	if n != nil {
+		for i, route := range n.Routes {
+			if route.Prefix == rt.Prefix && (route.NextHop == rt.NextHop || route.Origin == rt.NextHop) {
+				n.Routes = append(n.Routes[:i], n.Routes[i+1:]...)
+				break
+			}
+		}
+	}
+}
+
+func (w *network) AddRoute(name string, route *models.Route) {
+	n := w.Get(name)
+	if n != nil && route != nil {
+		n.Routes = append(n.Routes, route)
+	}
+}
 
 func (w *network) List() <-chan *models.Network {
 	c := make(chan *models.Network, 128)

pkg/config/subnet.go

@@ -45,17 +45,21 @@ func (r *PrefixRoute) String() string {
 	return fmt.Sprintf("{%s}", strings.Join(elems, " "))
 }
 
+func (r *PrefixRoute) CorrectRoute(nexthop string) {
+	if r.Metric == 0 {
+		r.Metric = 660
+	}
+	if r.NextHop == "" {
+		r.NextHop = nexthop
+	}
+	if r.Mode == "" {
+		r.Mode = "snat"
+	}
+}
+
 func CorrectRoutes(routes []PrefixRoute, nexthop string) {
 	for i := range routes {
-		if routes[i].Metric == 0 {
+		routes[i].CorrectRoute(nexthop)
-			routes[i].Metric = 660
-		}
-		if routes[i].NextHop == "" {
-			routes[i].NextHop = nexthop
-		}
-		if routes[i].Mode == "" {
-			routes[i].Mode = "snat"
-		}
 	}
 }
 

pkg/libol/utils.go

@@ -347,3 +347,12 @@ func Exec(bin string, args ...string) (string, error) {
 	out, err := exec.Command(bin, args...).CombinedOutput()
 	return string(out), err
 }
+
+func IsProcessRunning(pid int) bool {
+	process, err := os.FindProcess(pid)
+	if err != nil {
+		return false
+	}
+	err = process.Signal(syscall.Signal(0))
+	return err == nil
+}

pkg/models/schema.go

@@ -117,6 +117,17 @@ func NewNetworkSchema(n *Network) schema.Network {
 	return sn
 }
 
+func NewRouteSchema(route *Route) schema.PrefixRoute {
+	pr := schema.PrefixRoute{
+		Prefix:  route.Prefix,
+		NextHop: route.NextHop,
+		Metric:  route.Metric,
+		Mode:    route.Mode,
+		Origin:  route.Origin,
+	}
+	return pr
+}
+
 func NewOutputSchema(o *Output) schema.Output {
 	return schema.Output{
 		Network:   o.Network,

pkg/switch/ipsec.go

@@ -215,7 +215,7 @@ func (w *EspWorker) Initialize() {
 	w.updateXfrm()
 }
 
-func (w *EspWorker) AddRoute(device, src, remote string) error {
+func (w *EspWorker) addRoute(device, src, remote string) error {
 	link, err := nl.LinkByName(device)
 	if link == nil {
 		return err
@@ -233,7 +233,7 @@ func (w *EspWorker) AddRoute(device, src, remote string) error {
 		Priority:  650,
 		AdvMSS:    w.spec.TcpMss,
 	}
-	w.out.Debug("EspWorker.AddRoute: %s", rte)
+	w.out.Debug("EspWorker.addRoute: %s", rte)
 	if err := nl.RouteReplace(rte); err != nil {
 		return libol.NewErr("%s %s.", err, remote)
 	}
@@ -370,8 +370,8 @@ func (w *EspWorker) upMember() {
 			w.out.Warn("EspWorker.UpDummy %d %s", mem.Spi, err)
 		}
 		for _, po := range mem.Policies {
-			if err := w.AddRoute(w.spec.Name, mem.Address, po.Dest); err != nil {
+			if err := w.addRoute(w.spec.Name, mem.Address, po.Dest); err != nil {
-				w.out.Warn("EspWorker.AddRoute %d %s", mem.Spi, err)
+				w.out.Warn("EspWorker.addRoute %d %s", mem.Spi, err)
 			}
 		}
 	}

pkg/switch/network.go

@@ -75,6 +75,21 @@ func (w *WorkerImpl) Provider() string {
 	return w.cfg.Provider
 }
 
+func (w *WorkerImpl) newRoute(rt *co.PrefixRoute) *models.Route {
+	if rt.NextHop == "" {
+		w.out.Warn("WorkerImpl.NewRoute: %s noNextHop", rt.Prefix)
+		return nil
+	}
+	rte := models.NewRoute(rt.Prefix, w.IfAddr(), rt.Mode)
+	if rt.Metric > 0 {
+		rte.Metric = rt.Metric
+	}
+	if rt.NextHop != "" {
+		rte.Origin = rt.NextHop
+	}
+	return rte
+}
+
 func (w *WorkerImpl) Initialize() {
 	cfg := w.cfg
 
@@ -95,18 +110,10 @@ func (w *WorkerImpl) Initialize() {
 		Routes:  make([]*models.Route, 0, 2),
 	}
 	for _, rt := range cfg.Routes {
-		if rt.NextHop == "" {
+		nRoute := w.newRoute(&rt)
-			w.out.Warn("WorkerImpl.Initialize: %s noNextHop", rt.Prefix)
+		if nRoute != nil {
-			continue
+			n.Routes = append(n.Routes, nRoute)
 		}
-		rte := models.NewRoute(rt.Prefix, w.IfAddr(), rt.Mode)
-		if rt.Metric > 0 {
-			rte.Metric = rt.Metric
-		}
-		if rt.NextHop != "" {
-			rte.Origin = rt.NextHop
-		}
-		n.Routes = append(n.Routes, rte)
 	}
 
 	cache.Network.Add(&n)
@@ -251,47 +258,54 @@ func (w *WorkerImpl) addOutput(bridge string, port *LinuxPort) {
 	w.AddPhysical(bridge, port.link)
 }
 
+func (w *WorkerImpl) loadRoute(rt co.PrefixRoute) {
+	// install routes
+	ifAddr := w.IfAddr()
+
+	dst, err := libol.ParseNet(rt.Prefix)
+	if err != nil {
+		return
+	}
+	if ifAddr == rt.NextHop && rt.MultiPath == nil {
+		// route's next-hop is local not install again.
+		return
+	}
+	nlr := nl.Route{
+		Dst:   dst,
+		Table: w.table,
+	}
+	for _, hop := range rt.MultiPath {
+		nxhe := &nl.NexthopInfo{
+			Hops: hop.Weight,
+			Gw:   net.ParseIP(hop.NextHop),
+		}
+		nlr.MultiPath = append(nlr.MultiPath, nxhe)
+	}
+	if rt.MultiPath == nil {
+		nlr.Gw = net.ParseIP(rt.NextHop)
+		nlr.Priority = rt.Metric
+	}
+	w.out.Debug("WorkerImpl.loadRoute: %s", nlr.String())
+	rt_c := rt
+	promise := libol.NewPromise()
+	promise.Go(func() error {
+		if err := nl.RouteReplace(&nlr); err != nil {
+			w.out.Warn("WorkerImpl.loadRoute: %v %s", nlr, err)
+			return err
+		}
+		w.out.Info("WorkerImpl.loadRoute: %v success", rt_c.String())
+		return nil
+	})
+
+}
+
 func (w *WorkerImpl) loadRoutes() {
 	// install routes
 	cfg := w.cfg
 	w.out.Debug("WorkerImpl.LoadRoute: %v", cfg.Routes)
-	ifAddr := w.IfAddr()
 
 	for _, rt := range cfg.Routes {
-		_, dst, err := net.ParseCIDR(rt.Prefix)
+		w.loadRoute(rt)
-		if err != nil {
-			continue
-		}
-		if ifAddr == rt.NextHop && rt.MultiPath == nil {
-			// route's next-hop is local not install again.
-			continue
-		}
-		nlr := nl.Route{
-			Dst:   dst,
-			Table: w.table,
-		}
-		for _, hop := range rt.MultiPath {
-			nxhe := &nl.NexthopInfo{
-				Hops: hop.Weight,
-				Gw:   net.ParseIP(hop.NextHop),
-			}
-			nlr.MultiPath = append(nlr.MultiPath, nxhe)
-		}
-		if rt.MultiPath == nil {
-			nlr.Gw = net.ParseIP(rt.NextHop)
-			nlr.Priority = rt.Metric
-		}
-		w.out.Debug("WorkerImpl.LoadRoute: %s", nlr.String())
-		rt_c := rt
-		promise := libol.NewPromise()
-		promise.Go(func() error {
-			if err := nl.RouteReplace(&nlr); err != nil {
-				w.out.Warn("WorkerImpl.LoadRoute: %v %s", nlr, err)
-				return err
-			}
-			w.out.Info("WorkerImpl.LoadRoute: %v success", rt_c.String())
-			return nil
-		})
 	}
 }
 
@@ -375,7 +389,7 @@ func (w *WorkerImpl) Start(v api.Switcher) {
 					return err
 				}
 
-				_, dest, _ := net.ParseCIDR(vpn.Subnet)
+				dest, _ := libol.ParseNet(vpn.Subnet)
 				rt := &nl.Route{
 					Dst:       dest,
 					Table:     w.table,
@@ -467,27 +481,37 @@ func (w *WorkerImpl) delOutput(bridge string, port *LinuxPort) {
 	}
 }
 
+func (w *WorkerImpl) unloadRoute(rt co.PrefixRoute) {
+	dst, err := libol.ParseNet(rt.Prefix)
+	if err != nil {
+		return
+	}
+	nlr := nl.Route{
+		Dst:   dst,
+		Table: w.table,
+	}
+	if rt.MultiPath == nil {
+		nlr.Gw = net.ParseIP(rt.NextHop)
+		nlr.Priority = rt.Metric
+	}
+	w.out.Debug("WorkerImpl.UnLoadRoute: %s", nlr.String())
+	if err := nl.RouteDel(&nlr); err != nil {
+		w.out.Warn("WorkerImpl.UnLoadRoute: %s", err)
+		return
+	}
+	w.out.Info("WorkerImpl.UnLoadRoute: %v", rt.String())
+}
+
 func (w *WorkerImpl) unloadRoutes() {
 	cfg := w.cfg
 	for _, rt := range cfg.Routes {
-		_, dst, err := net.ParseCIDR(rt.Prefix)
+		w.unloadRoute(rt)
-		if err != nil {
+	}
-			continue
+}
-		}
+
-		nlr := nl.Route{
+func (w *WorkerImpl) RestartVpn() {
-			Dst:   dst,
+	if w.vpn != nil {
-			Table: w.table,
+		w.vpn.Restart()
-		}
-		if rt.MultiPath == nil {
-			nlr.Gw = net.ParseIP(rt.NextHop)
-			nlr.Priority = rt.Metric
-		}
-		w.out.Debug("WorkerImpl.UnLoadRoute: %s", nlr.String())
-		if err := nl.RouteDel(&nlr); err != nil {
-			w.out.Warn("WorkerImpl.UnLoadRoute: %s", err)
-			continue
-		}
-		w.out.Info("WorkerImpl.UnLoadRoute: %v", rt.String())
 	}
 }
 
@@ -522,6 +546,7 @@ func (w *WorkerImpl) Stop() {
 
 	w.setR.Destroy()
 	w.setV.Destroy()
+
 }
 
 func (w *WorkerImpl) String() string {
@@ -558,7 +583,7 @@ func (w *WorkerImpl) Subnet() *net.IPNet {
 		ifAddr := strings.SplitN(ipAddr, "/", 2)[0]
 		addr = fmt.Sprintf("%s/%d", ifAddr, prefix)
 	}
-	if _, inet, err := net.ParseCIDR(addr); err == nil {
+	if inet, err := libol.ParseNet(addr); err == nil {
 		return inet
 	}
 
@@ -655,6 +680,25 @@ func (w *WorkerImpl) GetCfgs() (*co.Network, *co.OpenVPN) {
 	return cfg, vpn
 }
 
+func (w *WorkerImpl) updateVPNRoute(routes []string, rt co.PrefixRoute) []string {
+	_, vpn := w.GetCfgs()
+	if vpn == nil {
+		return routes
+	}
+
+	addr := rt.Prefix
+	if addr == "0.0.0.0/0" {
+		vpn.Push = append(vpn.Push, "redirect-gateway def1")
+		routes = append(routes, addr)
+		return routes
+	}
+	if inet, err := libol.ParseNet(addr); err == nil {
+		routes = append(routes, inet.String())
+	}
+
+	return routes
+}
+
 func (w *WorkerImpl) updateVPN() {
 	cfg, vpn := w.GetCfgs()
 	if vpn == nil {
@@ -669,15 +713,7 @@ func (w *WorkerImpl) updateVPN() {
 	}
 
 	for _, rt := range cfg.Routes {
-		addr := rt.Prefix
+		routes = w.updateVPNRoute(routes, rt)
-		if addr == "0.0.0.0/0" {
-			vpn.Push = append(vpn.Push, "redirect-gateway def1")
-			routes = append(routes, addr)
-			continue
-		}
-		if _, inet, err := net.ParseCIDR(addr); err == nil {
-			routes = append(routes, inet.String())
-		}
 	}
 	vpn.Routes = routes
 }
@@ -708,6 +744,24 @@ func (w *WorkerImpl) forwardZone(input string) {
 	})
 }
 
+func (w *WorkerImpl) forwardVPNIpSet(rt string) {
+	if rt == "0.0.0.0/0" {
+		w.setV.Add("0.0.0.0/1")
+		w.setV.Add("128.0.0.0/1")
+		return
+	}
+	w.setV.Add(rt)
+}
+
+func (w *WorkerImpl) delForwardVPNIpSet(rt string) {
+	if rt == "0.0.0.0/0" {
+		w.setV.Del("0.0.0.0/1")
+		w.setV.Del("128.0.0.0/1")
+		return
+	}
+	w.setV.Del(rt)
+}
+
 func (w *WorkerImpl) forwardVPN() {
 	_, vpn := w.GetCfgs()
 	if vpn == nil {
@@ -729,12 +783,7 @@ func (w *WorkerImpl) forwardVPN() {
 	w.toACL(devName)
 
 	for _, rt := range vpn.Routes {
-		if rt == "0.0.0.0/0" {
+		w.forwardVPNIpSet(rt)
-			w.setV.Add("0.0.0.0/1")
-			w.setV.Add("128.0.0.0/1")
-			break
-		}
-		w.setV.Add(rt)
 	}
 	if w.vrf != nil {
 		w.toForward_r(w.vrf.Name(), vpn.Subnet, w.setV.Name, "From VPN")
@@ -744,6 +793,33 @@ func (w *WorkerImpl) forwardVPN() {
 	w.toMasq_r(vpn.Subnet, w.setV.Name, "From VPN")
 }
 
+func (w *WorkerImpl) forwardSubnetIpSet(rt co.PrefixRoute) bool {
+	if rt.MultiPath != nil {
+		return true
+	}
+	if rt.Prefix == "0.0.0.0/0" {
+		w.setR.Add("0.0.0.0/1")
+		w.setR.Add("128.0.0.0/1")
+		return false
+	}
+	w.setR.Add(rt.Prefix)
+
+	return true
+}
+
+func (w *WorkerImpl) delForwardIpSet(rt co.PrefixRoute) {
+	if rt.MultiPath != nil {
+		return
+	}
+	if rt.Prefix == "0.0.0.0/0" {
+		w.setR.Del("0.0.0.0/1")
+		w.setR.Del("128.0.0.0/1")
+		return
+	}
+	w.setR.Del(rt.Prefix)
+	return
+}
+
 func (w *WorkerImpl) forwardSubnet() {
 	cfg, vpn := w.GetCfgs()
 
@@ -762,15 +838,9 @@ func (w *WorkerImpl) forwardSubnet() {
 	// Enable MASQUERADE, and FORWARD it.
 	w.toRelated(input, "Accept related")
 	for _, rt := range cfg.Routes {
-		if rt.MultiPath != nil {
+		if !w.forwardSubnetIpSet(rt) {
-			continue
-		}
-		if rt.Prefix == "0.0.0.0/0" {
-			w.setR.Add("0.0.0.0/1")
-			w.setR.Add("128.0.0.0/1")
 			break
 		}
-		w.setR.Add(rt.Prefix)
 	}
 
 	if w.vrf != nil {
@@ -797,6 +867,171 @@ func (w *WorkerImpl) createVPN() {
 	w.vpn = obj
 }
 
+func (w *WorkerImpl) delCacheRoute(rt co.PrefixRoute) {
+	if rt.NextHop == "" {
+		w.out.Warn("WorkerImpl.DelCacheRoute: %s noNextHop", rt.Prefix)
+		return
+	}
+	rte := models.NewRoute(rt.Prefix, w.IfAddr(), rt.Mode)
+	if rt.Metric > 0 {
+		rte.Metric = rt.Metric
+	}
+	if rt.NextHop != "" {
+		rte.Origin = rt.NextHop
+	}
+
+	cache.Network.DelRoute(w.cfg.Name, rt)
+}
+
+func (w *WorkerImpl) addCacheRoute(rt co.PrefixRoute) {
+	if rt.NextHop == "" {
+		w.out.Warn("WorkerImpl.AddCacheRoute: %s ", rt.Prefix)
+		return
+	}
+	rte := models.NewRoute(rt.Prefix, w.IfAddr(), rt.Mode)
+	if rt.Metric > 0 {
+		rte.Metric = rt.Metric
+	}
+	if rt.NextHop != "" {
+		rte.Origin = rt.NextHop
+	}
+	cache.Network.AddRoute(w.cfg.Name, rte)
+}
+
+func (w *WorkerImpl) addVPNRoute(rt co.PrefixRoute) {
+
+	vpn := w.cfg.OpenVPN
+	if vpn == nil {
+		return
+	}
+	routes := vpn.Routes
+	vpn.Routes = w.updateVPNRoute(routes, rt)
+}
+
+func (w *WorkerImpl) delVPNRoute(rt co.PrefixRoute) {
+
+	vpn := w.cfg.OpenVPN
+	if vpn == nil {
+		return
+	}
+
+	routes := vpn.Routes
+
+	addr := rt.Prefix
+	if addr == "0.0.0.0/0" {
+		for i, s := range vpn.Push {
+			if s == "redirect-gateway def1" {
+				vpn.Push = append(vpn.Push[:i], vpn.Push[i+1:]...)
+				break
+			}
+		}
+		for i2, r := range routes {
+			if r == addr {
+				routes = append(routes[:i2], routes[i2+1:]...)
+				break
+			}
+		}
+
+		return
+	}
+	if inet, err := libol.ParseNet(addr); err == nil {
+		for i, r := range routes {
+			if r == inet.String() {
+				routes = append(routes[:i], routes[i+1:]...)
+				break
+			}
+		}
+	}
+
+	vpn.Routes = routes
+}
+
+func (w *WorkerImpl) correctRoute(route *schema.PrefixRoute) co.PrefixRoute {
+
+	rt := co.PrefixRoute{
+		Prefix:  route.Prefix,
+		NextHop: route.NextHop,
+		Mode:    route.Mode,
+		Metric:  route.Metric,
+	}
+
+	br := w.cfg.Bridge
+	ipAddr := ""
+	if _i, _, err := net.ParseCIDR(br.Address); err == nil {
+		ipAddr = _i.String()
+	}
+
+	rt.CorrectRoute(ipAddr)
+
+	return rt
+}
+
+func (w *WorkerImpl) findRoute(rt co.PrefixRoute) (co.PrefixRoute, int) {
+	for i, ert := range w.cfg.Routes {
+		if ert.Prefix == rt.Prefix && rt.NextHop == ert.NextHop {
+			return ert, i
+		}
+	}
+	return co.PrefixRoute{}, -1
+}
+
+func (w *WorkerImpl) AddRoute(route *schema.PrefixRoute, switcher api.Switcher) error {
+
+	rt := w.correctRoute(route)
+
+	if _, index := w.findRoute(rt); index != -1 {
+		w.out.Warn("WorkerImpl.AddRoute: route exist")
+		return nil
+	}
+
+	w.cfg.Routes = append(w.cfg.Routes, rt)
+
+	libol.Info("WorkerImpl.AddRoute: %v", rt)
+
+	w.forwardSubnetIpSet(rt)
+
+	if inet, err := libol.ParseNet(rt.Prefix); err == nil {
+		w.forwardVPNIpSet(inet.String())
+	}
+
+	w.addCacheRoute(rt)
+	w.addVPNRoute(rt)
+	w.loadRoute(rt)
+
+	return nil
+}
+
+func (w *WorkerImpl) DelRoute(route *schema.PrefixRoute, switcher api.Switcher) error {
+
+	correctRt := w.correctRoute(route)
+
+	delRt, index := w.findRoute(correctRt)
+
+	if index == -1 {
+
+		w.out.Warn("WorkerImpl.DelRoute: route not found")
+		return nil
+	}
+
+	w.cfg.Routes = append(w.cfg.Routes[:index], w.cfg.Routes[index+1:]...)
+
+	w.delForwardIpSet(delRt)
+
+	if inet, err := libol.ParseNet(delRt.Prefix); err == nil {
+		w.delForwardVPNIpSet(inet.String())
+	}
+
+	w.delCacheRoute(delRt)
+	w.delVPNRoute(delRt)
+	w.unloadRoute(delRt)
+
+	return nil
+}
+
+func (w *WorkerImpl) SaveRoute() {
+	w.cfg.SaveRoute()
+}
+
 func (w *WorkerImpl) ZTruster() api.ZTruster {
 	return w.ztrust
 }
@@ -805,6 +1040,10 @@ func (w *WorkerImpl) Qoser() api.Qoser {
 	return w.qos
 }
 
+func (w *WorkerImpl) Router() api.Router {
+	return w
+}
+
 func (w *WorkerImpl) IfAddr() string {
 	return strings.SplitN(w.cfg.Bridge.Address, "/", 2)[0]
 }

pkg/switch/openvpn.go

@@ -8,8 +8,10 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"text/template"
+	"time"
 
 	co "github.com/luscis/openlan/pkg/config"
 	"github.com/luscis/openlan/pkg/libol"
@@ -251,6 +253,15 @@ func (o *OpenVPN) FileIpp(full bool) string {
 	return filepath.Join(o.Cfg.Directory, name)
 }
 
+func (o *OpenVPN) Pid(full bool) string {
+	if data, err := ioutil.ReadFile(o.FilePid(true)); err != nil {
+		o.out.Debug("OpenVPN.Stop %s", err)
+		return ""
+	} else {
+		return strings.TrimSpace(string(data))
+	}
+}
+
 func (o *OpenVPN) DirectoryClientConfig() string {
 	if o.Cfg == nil {
 		return path.Join(DefaultCurDir, "ccd")
@@ -387,8 +398,12 @@ func (o *OpenVPN) Stop() {
 	if data, err := ioutil.ReadFile(o.FilePid(true)); err != nil {
 		o.out.Debug("OpenVPN.Stop %s", err)
 	} else {
+		killPath, err := exec.LookPath("kill")
+		if err != nil {
+			o.out.Warn("kill cmd not found :", err)
+		}
 		pid := strings.TrimSpace(string(data))
-		cmd := exec.Command("/usr/bin/kill", pid)
+		cmd := exec.Command(killPath, pid)
 		if err := cmd.Run(); err != nil {
 			o.out.Warn("OpenVPN.Stop %s: %s", pid, err)
 		}
@@ -396,6 +411,41 @@ func (o *OpenVPN) Stop() {
 	o.Clean()
 }
 
+func (o *OpenVPN) Restart() {
+	o.Stop()
+	o.checkAlreadyClose(o.Pid(true))
+	o.Initialize()
+	o.Start()
+}
+
+func (o *OpenVPN) checkAlreadyClose(pid string) {
+	timeout := 10 * time.Second
+
+	if pid != "" {
+		ticker := time.Tick(200 * time.Millisecond)
+		timer := time.After(timeout)
+		pidInt, err := strconv.Atoi(pid)
+		if err != nil {
+			return
+		}
+		for {
+			select {
+			case <-ticker:
+
+				running := libol.IsProcessRunning(pidInt)
+				if !running {
+					o.out.Debug("OpenVPN.CheckAlreadyClose:vpn is close")
+					return
+				}
+			case <-timer:
+				o.out.Warn("OpenVPN.CheckAlreadyClose:vpn close timeout")
+				return
+			}
+		}
+
+	}
+}
+
 func (o *OpenVPN) ProfileTmpl() string {
 	tmplStr := xAuthClientProfile
 	if o.Cfg.Auth == "cert" {