fea: switch: support router network

dist/rootfs/etc/openlan/switch/network/router.json.example

@@ -0,0 +1,20 @@
+{
+    "name": "router",
+    "provider": "router",
+    "specifies": {
+	"link": "eth0",
+        "subnets": [
+            {
+                "cidr": "192.168.0.0/24"
+            },
+	    {
+		"cidr": "192.168.1.0/24"
+	    }
+	 ]
+    },
+    "routes": [
+       {
+	  "prefix": "172.16.3.0/24"
+       }
+    ]
+}

pkg/config/firewall.go

@@ -14,5 +14,6 @@ type FlowRule struct {
 	Match    string `json:"match,omitempty"`
 	DstPort  string `json:"destPort,omitempty"`
 	SrcPort  string `json:"sourcePort,omitempty"`
+	CtState  string `json:"ctState,omitempty"`
 	Jump     string `json:"jump,omitempty"` // SNAT/RETURN/MASQUERADE
 }

pkg/config/network.go

@@ -1,9 +1,10 @@
 package config
 
 import (
-	"github.com/luscis/openlan/pkg/libol"
 	"net"
 	"path/filepath"
+
+	"github.com/luscis/openlan/pkg/libol"
 )
 
 type Network struct {
@@ -24,6 +25,22 @@ type Network struct {
 	Outputs   []Output      `json:"outputs"`
 }
 
+func (n *Network) NewSpecifies() interface{} {
+	switch n.Provider {
+	case "esp":
+		n.Specifies = &ESPSpecifies{}
+	case "vxlan":
+		n.Specifies = &VxLANSpecifies{}
+	case "fabric":
+		n.Specifies = &FabricSpecifies{}
+	case "router":
+		n.Specifies = &RouterSpecifies{}
+	default:
+		n.Specifies = nil
+	}
+	return n.Specifies
+}
+
 func (n *Network) Correct() {
 	if n.Bridge == nil {
 		n.Bridge = &Bridge{}
@@ -51,7 +68,13 @@ func (n *Network) Correct() {
 			obj.Correct()
 			obj.Name = n.Name
 		}
-	default:
+	case "router":
+		spec := n.Specifies
+		if obj, ok := spec.(*RouterSpecifies); ok {
+			obj.Correct()
+			obj.Name = n.Name
+		}
+	}
 	if n.Subnet == nil {
 		n.Subnet = &Subnet{}
 	}
@@ -64,24 +87,13 @@ func (n *Network) Correct() {
 	if n.Subnet.Netmask == "" {
 		n.Subnet.Netmask = ipMask
 	}
-		for i := range n.Routes {
-			if n.Routes[i].Metric == 0 {
-				n.Routes[i].Metric = 660
-			}
-			if n.Routes[i].NextHop == "" {
-				n.Routes[i].NextHop = ipAddr
-			}
-			if n.Routes[i].Mode == "" {
-				n.Routes[i].Mode = "snat"
-			}
-		}
+	CorrectRoutes(n.Routes, ipAddr)
 	if n.OpenVPN != nil {
 		n.OpenVPN.Network = n.Name
 		obj := DefaultOpenVPN()
 		n.OpenVPN.Merge(obj)
 		n.OpenVPN.Correct()
 	}
-	}
 }
 
 func (n *Network) Dir(elem ...string) string {

pkg/config/router.go

@@ -0,0 +1,11 @@
+package config
+
+type RouterSpecifies struct {
+	Mss     int      `json:"tcpMss,omitempty"`
+	Name    string   `json:"name,omitempty"`
+	Link    string   `json:"link,omitempty"`
+	Subnets []Subnet `json:"subnets"`
+}
+
+func (n *RouterSpecifies) Correct() {
+}

pkg/config/subnet.go

@@ -10,6 +10,7 @@ type Subnet struct {
 	Start   string `json:"startAt,omitempty"`
 	End     string `json:"endAt,omitempty"`
 	Netmask string `json:"netmask,omitempty"`
+	CIDR    string `json:"cidr,omitempty"`
 }
 
 type MultiPath struct {
@@ -44,6 +45,20 @@ func (r *PrefixRoute) String() string {
 	return fmt.Sprintf("{%s}", strings.Join(elems, " "))
 }
 
+func CorrectRoutes(routes []PrefixRoute, nexthop string) {
+	for i := range routes {
+		if routes[i].Metric == 0 {
+			routes[i].Metric = 660
+		}
+		if routes[i].NextHop == "" {
+			routes[i].NextHop = nexthop
+		}
+		if routes[i].Mode == "" {
+			routes[i].Mode = "snat"
+		}
+	}
+}
+
 type HostLease struct {
 	Network  string `json:"network,omitempty"`
 	Hostname string `json:"hostname"`

pkg/config/switch.go

@@ -2,8 +2,9 @@ package config
 
 import (
 	"flag"
-	"github.com/luscis/openlan/pkg/libol"
 	"path/filepath"
+
+	"github.com/luscis/openlan/pkg/libol"
 )
 
 type Perf struct {
@@ -150,22 +151,13 @@ func (s *Switch) Dir(elem ...string) string {
 
 func (s *Switch) Format() {
 	for _, obj := range s.Network {
-		libol.Debug("Switch.Format %s", obj)
 		context := obj.Specifies
-		switch obj.Provider {
-		case "esp":
-			obj.Specifies = &ESPSpecifies{}
-		case "vxlan":
-			obj.Specifies = &VxLANSpecifies{}
-		case "fabric":
-			obj.Specifies = &FabricSpecifies{}
-		default:
-			obj.Specifies = nil
-			continue
-		}
+		obj.NewSpecifies()
 		if data, err := libol.Marshal(context, true); err == nil {
 			if err := libol.Unmarshal(obj.Specifies, data); err != nil {
 				libol.Warn("Switch.Format %s", err)
+			} else {
+				libol.Info("Switch.Format %v", obj.Specifies)
 			}
 		}
 	}

pkg/network/firewall.go

@@ -44,6 +44,7 @@ func NewFireWallGlobal(flows []config.FlowRule) *FireWallGlobal {
 			SrcPort:  rule.SrcPort,
 			Input:    rule.Input,
 			Output:   rule.Output,
+			CtState:  rule.CtState,
 		})
 	}
 	return f

pkg/network/iptables.go

@@ -47,6 +47,7 @@ type IpRule struct {
 	SetMss    int
 	Order     string
 	Match     string
+	CtState   string
 	TcpFlag   []string
 }
 
@@ -81,6 +82,9 @@ func (ru IpRule) Args() []string {
 		args = append(args, "!")
 		args = append(args, "-m", "set", "--match-set", ru.NoDestSet, "dst")
 	}
+	if ru.CtState != "" {
+		args = append(args, "-m", "conntrack", "--ctstate", ru.CtState)
+	}
 	if ru.Proto != "" {
 		args = append(args, "-p", ru.Proto)
 	}

pkg/switch/network.go

@@ -36,6 +36,8 @@ func NewNetworker(c *co.Network) Networker {
 		obj = NewVxLANWorker(c)
 	case "fabric":
 		obj = NewFabricWorker(c)
+	case "router":
+		obj = NewRouterWorker(c)
 	default:
 		obj = NewOpenLANWorker(c)
 	}
@@ -66,12 +68,17 @@ type WorkerImpl struct {
 	dhcp    *Dhcp
 	outputs []*LinuxPort
 	fire    *cn.FireWallTable
+	setR    *cn.IPSet
+	setV    *cn.IPSet
+	vpn     *OpenVPN
 }
 
 func NewWorkerApi(c *co.Network) *WorkerImpl {
 	return &WorkerImpl{
 		cfg:  c,
 		out:  libol.NewSubLogger(c.Name),
+		setR: cn.NewIPSet(c.Name+"_r", "hash:net"),
+		setV: cn.NewIPSet(c.Name+"_v", "hash:net"),
 	}
 }
 
@@ -88,6 +95,12 @@ func (w *WorkerImpl) Initialize() {
 		})
 	}
 	w.fire = cn.NewFireWallTable(w.cfg.Name)
+	if out, err := w.setV.Clear(); err != nil {
+		w.out.Error("WorkImpl.Initialize: create ipset: %s %s", out, err)
+	}
+	if out, err := w.setR.Clear(); err != nil {
+		w.out.Error("WorkImpl.Initialize: create ipset: %s %s", out, err)
+	}
 }
 
 func (w *WorkerImpl) AddPhysical(bridge string, vlan int, output string) {
@@ -176,6 +189,7 @@ func (w *WorkerImpl) Start(v api.Switcher) {
 	cfg := w.cfg
 	fire := w.fire
 
+	w.out.Info("WorkerImpl.Start")
 	if cfg.Acl != "" {
 		fire.Raw.Pre.AddRule(cn.IpRule{
 			Input: cfg.Bridge.Name,
@@ -214,7 +228,7 @@ func (w *WorkerImpl) Start(v api.Switcher) {
 		w.AddOutput(cfg.Bridge.Name, port)
 		w.outputs = append(w.outputs, port)
 	}
-	if w.dhcp != nil {
+	if !(w.dhcp == nil) {
 		w.dhcp.Start()
 		fire.Nat.Post.AddRule(cn.IpRule{
 			Source:  cfg.Bridge.Address,
@@ -223,6 +237,10 @@ func (w *WorkerImpl) Start(v api.Switcher) {
 			Comment: "Default Gateway for DHCP",
 		})
 	}
+	if !(w.vpn == nil) {
+		w.vpn.Start()
+	}
+	w.fire.Start()
 }
 
 func (w *WorkerImpl) DelPhysical(bridge string, vlan int, output string) {
@@ -272,13 +290,20 @@ func (w *WorkerImpl) DelOutput(bridge string, port *LinuxPort) {
 }
 
 func (w *WorkerImpl) Stop() {
-	if w.dhcp != nil {
+	w.out.Info("WorkerImpl.Stop")
+	w.fire.Stop()
+	if !(w.vpn == nil) {
+		w.vpn.Stop()
+	}
+	if !(w.dhcp == nil) {
 		w.dhcp.Stop()
 	}
 	for _, output := range w.outputs {
 		w.DelOutput(w.cfg.Bridge.Name, output)
 	}
 	w.outputs = nil
+	w.setR.Destroy()
+	w.setV.Destroy()
 }
 
 func (w *WorkerImpl) String() string {
@@ -303,3 +328,82 @@ func (w *WorkerImpl) Subnet() string {
 
 func (w *WorkerImpl) Reload(v api.Switcher) {
 }
+
+func (w *WorkerImpl) toACL(acl, input string) {
+	if input == "" {
+		return
+	}
+	if acl != "" {
+		w.fire.Raw.Pre.AddRule(cn.IpRule{
+			Input: input,
+			Jump:  acl,
+		})
+	}
+}
+
+func (w *WorkerImpl) openPort(protocol, port, comment string) {
+	w.out.Info("WorkerImpl.openPort %s %s", protocol, port)
+	// allowed forward between source and prefix.
+	w.fire.Filter.In.AddRule(cn.IpRule{
+		Proto:   protocol,
+		Match:   "multiport",
+		DstPort: port,
+		Comment: comment,
+	})
+}
+
+func (w *WorkerImpl) toForward_r(input, output, source, pfxSet, comment string) {
+	w.out.Debug("WorkerImpl.toForward %s:%s %s:%s", input, output, source, pfxSet)
+	// Allowed forward between source and prefix.
+	w.fire.Filter.For.AddRule(cn.IpRule{
+		Input:   input,
+		Output:  output,
+		Source:  source,
+		DestSet: pfxSet,
+		Comment: comment,
+	})
+}
+
+func (w *WorkerImpl) toForward_s(input, output, srcSet, prefix, comment string) {
+	w.out.Debug("WorkerImpl.toForward %s:%s %s:%s", input, output, srcSet, prefix)
+	// Allowed forward between source and prefix.
+	w.fire.Filter.For.AddRule(cn.IpRule{
+		Input:   input,
+		Output:  output,
+		SrcSet:  srcSet,
+		Dest:    prefix,
+		Comment: comment,
+	})
+}
+
+func (w *WorkerImpl) toMasq_r(source, pfxSet, comment string) {
+	// Enable masquerade from source to prefix.
+	w.fire.Nat.Post.AddRule(cn.IpRule{
+		Source:  source,
+		DestSet: pfxSet,
+		Jump:    cn.CMasq,
+		Comment: comment,
+	})
+
+}
+
+func (w *WorkerImpl) toMasq_s(srcSet, prefix, comment string) {
+	// Enable masquerade from source to prefix.
+	w.fire.Nat.Post.AddRule(cn.IpRule{
+		SrcSet:  srcSet,
+		Dest:    prefix,
+		Jump:    cn.CMasq,
+		Comment: comment,
+	})
+
+}
+
+func (w *WorkerImpl) toRelated(output, comment string) {
+	w.out.Debug("WorkerImpl.toRelated %s", output)
+	// Allowed forward between source and prefix.
+	w.fire.Filter.For.AddRule(cn.IpRule{
+		Output:  output,
+		CtState: "RELATED,ESTABLISHED",
+		Comment: comment,
+	})
+}

pkg/switch/openlan.go

@@ -11,8 +11,8 @@ import (
 	co "github.com/luscis/openlan/pkg/config"
 	"github.com/luscis/openlan/pkg/libol"
 	"github.com/luscis/openlan/pkg/models"
-	"github.com/luscis/openlan/pkg/network"
-	"github.com/vishvananda/netlink"
+	cn "github.com/luscis/openlan/pkg/network"
+	nl "github.com/vishvananda/netlink"
 )
 
 func PeerName(name, prefix string) (string, string) {
@@ -25,10 +25,7 @@ type OpenLANWorker struct {
 	newTime   int64
 	startTime int64
 	links     *Links
-	bridge    network.Bridger
-	vpn       *OpenVPN
-	setR      *network.IPSet
-	setV      *network.IPSet
+	bridge    cn.Bridger
 }
 
 func NewOpenLANWorker(c *co.Network) *OpenLANWorker {
@@ -38,80 +35,9 @@ func NewOpenLANWorker(c *co.Network) *OpenLANWorker {
 		newTime:    time.Now().Unix(),
 		startTime:  0,
 		links:      NewLinks(),
-		setR:       network.NewIPSet(c.Name+"_r", "hash:net"),
-		setV:       network.NewIPSet(c.Name+"_v", "hash:net"),
 	}
 }
 
-func (w *OpenLANWorker) toACL(acl, input string) {
-	if input == "" {
-		return
-	}
-	if acl != "" {
-		w.fire.Raw.Pre.AddRule(network.IpRule{
-			Input: input,
-			Jump:  acl,
-		})
-	}
-}
-
-func (w *OpenLANWorker) openPort(protocol, port, comment string) {
-	w.out.Info("OpenLANWorker.openPort %s %s", protocol, port)
-	// allowed forward between source and prefix.
-	w.fire.Filter.In.AddRule(network.IpRule{
-		Proto:   protocol,
-		Match:   "multiport",
-		DstPort: port,
-		Comment: comment,
-	})
-}
-
-func (w *OpenLANWorker) toForward_r(input, output, source, pfxSet, comment string) {
-	w.out.Debug("OpenLANWorker.toForward %s:%s %s:%s", input, output, source, pfxSet)
-	// Allowed forward between source and prefix.
-	w.fire.Filter.For.AddRule(network.IpRule{
-		Input:   input,
-		Output:  output,
-		Source:  source,
-		DestSet: pfxSet,
-		Comment: comment,
-	})
-}
-
-func (w *OpenLANWorker) toForward_s(input, output, srcSet, prefix, comment string) {
-	w.out.Debug("OpenLANWorker.toForward %s:%s %s:%s", input, output, srcSet, prefix)
-	// Allowed forward between source and prefix.
-	w.fire.Filter.For.AddRule(network.IpRule{
-		Input:   input,
-		Output:  output,
-		SrcSet:  srcSet,
-		Dest:    prefix,
-		Comment: comment,
-	})
-}
-
-func (w *OpenLANWorker) toMasq_r(source, pfxSet, comment string) {
-	// Enable masquerade from source to prefix.
-	w.fire.Nat.Post.AddRule(network.IpRule{
-		Source:  source,
-		DestSet: pfxSet,
-		Jump:    network.CMasq,
-		Comment: comment,
-	})
-
-}
-
-func (w *OpenLANWorker) toMast_s(srcSet, prefix, comment string) {
-	// Enable masquerade from source to prefix.
-	w.fire.Nat.Post.AddRule(network.IpRule{
-		SrcSet:  srcSet,
-		Dest:    prefix,
-		Jump:    network.CMasq,
-		Comment: comment,
-	})
-
-}
-
 func (w *OpenLANWorker) updateVPN() {
 	cfg := w.Config()
 	vCfg := cfg.OpenVPN
@@ -152,6 +78,8 @@ func (w *OpenLANWorker) allowedVPN() {
 	}
 
 	devName := vCfg.Device
+	// Enable MASQUERADE, and allowed forward.
+	w.toRelated(devName, "Accept related")
 	w.toACL(cfg.Acl, devName)
 
 	for _, rt := range vCfg.Routes {
@@ -179,6 +107,7 @@ func (w *OpenLANWorker) allowedSubnet() {
 	subnet := w.Subnet()
 
 	// Enable MASQUERADE, and allowed forward.
+	w.toRelated(br.Name, "Accept related")
 	for _, rt := range cfg.Routes {
 		if rt.MultiPath != nil {
 			continue
@@ -193,7 +122,7 @@ func (w *OpenLANWorker) allowedSubnet() {
 	w.toForward_r(br.Name, "", subnet, w.setR.Name, "To route")
 	w.toForward_s("", br.Name, w.setR.Name, subnet, "From route")
 	if vCfg != nil {
-		w.toMast_s(w.setR.Name, vCfg.Subnet, "To VPN")
+		w.toMasq_s(w.setR.Name, vCfg.Subnet, "To VPN")
 	}
 	w.toMasq_r(subnet, w.setR.Name, "To Masq")
 }
@@ -231,7 +160,7 @@ func (w *OpenLANWorker) Initialize() {
 			lease.Network = w.cfg.Name
 		}
 	}
-	w.bridge = network.NewBridger(brCfg.Provider, brCfg.Name, brCfg.IPMtu)
+	w.bridge = cn.NewBridger(brCfg.Provider, brCfg.Name, brCfg.IPMtu)
 
 	w.updateVPN()
 	vCfg := w.cfg.OpenVPN
@@ -241,12 +170,6 @@ func (w *OpenLANWorker) Initialize() {
 		w.vpn = obj
 	}
 	w.WorkerImpl.Initialize()
-	if out, err := w.setV.Clear(); err != nil {
-		w.out.Error("OpenLANWorker.Initialize: create ipset: %s %s", out, err)
-	}
-	if out, err := w.setR.Clear(); err != nil {
-		w.out.Error("OpenLANWorker.Initialize: create ipset: %s %s", out, err)
-	}
 	w.allowedSubnet()
 	w.allowedVPN()
 }
@@ -282,9 +205,9 @@ func (w *OpenLANWorker) LoadRoutes() {
 			// route's next-hop is local not install again.
 			continue
 		}
-		nlrt := netlink.Route{Dst: dst}
+		nlrt := nl.Route{Dst: dst}
 		for _, hop := range rt.MultiPath {
-			nxhe := &netlink.NexthopInfo{
+			nxhe := &nl.NexthopInfo{
 				Hops: hop.Weight,
 				Gw:   net.ParseIP(hop.NextHop),
 			}
@@ -302,7 +225,7 @@ func (w *OpenLANWorker) LoadRoutes() {
 		}
 		rt_c := rt
 		promise.Go(func() error {
-			if err := netlink.RouteReplace(&nlrt); err != nil {
+			if err := nl.RouteReplace(&nlrt); err != nil {
 				w.out.Warn("OpenLANWorker.LoadRoute: %v %s", nlrt, err)
 				return err
 			}
@@ -319,13 +242,13 @@ func (w *OpenLANWorker) UnLoadRoutes() {
 		if err != nil {
 			continue
 		}
-		nlRt := netlink.Route{Dst: dst}
+		nlRt := nl.Route{Dst: dst}
 		if rt.MultiPath == nil {
 			nlRt.Gw = net.ParseIP(rt.NextHop)
 			nlRt.Priority = rt.Metric
 		}
 		w.out.Debug("OpenLANWorker.UnLoadRoute: %s", nlRt.String())
-		if err := netlink.RouteDel(&nlRt); err != nil {
+		if err := nl.RouteDel(&nlRt); err != nil {
 			w.out.Warn("OpenLANWorker.UnLoadRoute: %s", err)
 			continue
 		}
@@ -364,11 +287,11 @@ func (w *OpenLANWorker) connectPeer(cfg *co.Bridge) {
 		return
 	}
 	in, ex := PeerName(cfg.Network, "-e")
-	link := &netlink.Veth{
-		LinkAttrs: netlink.LinkAttrs{Name: in},
+	link := &nl.Veth{
+		LinkAttrs: nl.LinkAttrs{Name: in},
 		PeerName:  ex,
 	}
-	br := network.NewBrCtl(cfg.Peer, cfg.IPMtu)
+	br := cn.NewBrCtl(cfg.Peer, cfg.IPMtu)
 	promise := &libol.Promise{
 		First:  time.Second * 2,
 		MaxInt: time.Minute,
@@ -379,16 +302,16 @@ func (w *OpenLANWorker) connectPeer(cfg *co.Bridge) {
 			w.out.Warn("%s notFound", br.Name)
 			return libol.NewErr("%s notFound", br.Name)
 		}
-		err := netlink.LinkAdd(link)
+		err := nl.LinkAdd(link)
 		if err != nil {
 			w.out.Error("OpenLANWorker.connectPeer: %s", err)
 			return nil
 		}
-		br0 := network.NewBrCtl(cfg.Name, cfg.IPMtu)
+		br0 := cn.NewBrCtl(cfg.Name, cfg.IPMtu)
 		if err := br0.AddPort(in); err != nil {
 			w.out.Error("OpenLANWorker.connectPeer: %s", err)
 		}
-		br1 := network.NewBrCtl(cfg.Peer, cfg.IPMtu)
+		br1 := cn.NewBrCtl(cfg.Peer, cfg.IPMtu)
 		if err := br1.AddPort(ex); err != nil {
 			w.out.Error("OpenLANWorker.connectPeer: %s", err)
 		}
@@ -403,11 +326,7 @@ func (w *OpenLANWorker) Start(v api.Switcher) {
 	w.UpBridge(w.cfg.Bridge)
 	w.LoadLinks()
 	w.LoadRoutes()
-	if !(w.vpn == nil) {
-		w.vpn.Start()
-	}
 	w.WorkerImpl.Start(v)
-	w.fire.Start()
 }
 
 func (w *OpenLANWorker) downBridge(cfg *co.Bridge) {
@@ -420,11 +339,11 @@ func (w *OpenLANWorker) closePeer(cfg *co.Bridge) {
 		return
 	}
 	in, ex := PeerName(cfg.Network, "-e")
-	link := &netlink.Veth{
-		LinkAttrs: netlink.LinkAttrs{Name: in},
+	link := &nl.Veth{
+		LinkAttrs: nl.LinkAttrs{Name: in},
 		PeerName:  ex,
 	}
-	err := netlink.LinkDel(link)
+	err := nl.LinkDel(link)
 	if err != nil {
 		w.out.Error("OpenLANWorker.closePeer: %s", err)
 		return
@@ -433,17 +352,11 @@ func (w *OpenLANWorker) closePeer(cfg *co.Bridge) {
 
 func (w *OpenLANWorker) Stop() {
 	w.out.Info("OpenLANWorker.Close")
-	w.fire.Stop()
 	w.WorkerImpl.Stop()
-	if !(w.vpn == nil) {
-		w.vpn.Stop()
-	}
 	w.UnLoadRoutes()
 	w.UnLoadLinks()
 	w.startTime = 0
 	w.downBridge(w.cfg.Bridge)
-	w.setR.Destroy()
-	w.setV.Destroy()
 }
 
 func (w *OpenLANWorker) UpTime() int64 {
@@ -459,7 +372,7 @@ func (w *OpenLANWorker) AddLink(c co.Point) {
 
 	c.Alias = w.alias
 	c.Network = w.cfg.Name
-	c.Interface.Name = network.Taps.GenName()
+	c.Interface.Name = cn.Taps.GenName()
 	c.Interface.Bridge = br.Name
 	c.Interface.Address = br.Address
 	c.Interface.Provider = br.Provider
@@ -501,7 +414,7 @@ func (w *OpenLANWorker) Subnet() string {
 	return ""
 }
 
-func (w *OpenLANWorker) Bridge() network.Bridger {
+func (w *OpenLANWorker) Bridge() cn.Bridger {
 	return w.bridge
 }
 

pkg/switch/router.go

@@ -0,0 +1,125 @@
+package _switch
+
+import (
+	"net"
+	"time"
+
+	"github.com/luscis/openlan/pkg/api"
+	co "github.com/luscis/openlan/pkg/config"
+	"github.com/luscis/openlan/pkg/libol"
+	nl "github.com/vishvananda/netlink"
+)
+
+type RouterWorker struct {
+	*WorkerImpl
+	spec *co.RouterSpecifies
+}
+
+func NewRouterWorker(c *co.Network) *RouterWorker {
+	w := &RouterWorker{
+		WorkerImpl: NewWorkerApi(c),
+	}
+	w.spec, _ = c.Specifies.(*co.RouterSpecifies)
+	return w
+}
+
+func (w *RouterWorker) Initialize() {
+	w.WorkerImpl.Initialize()
+}
+
+func (w *RouterWorker) LoadRoutes() {
+	// install routes
+	cfg := w.cfg
+	w.out.Debug("RouterWorker.LoadRoute: %v", cfg.Routes)
+	for _, rt := range cfg.Routes {
+		_, dst, err := net.ParseCIDR(rt.Prefix)
+		if err != nil {
+			continue
+		}
+		if rt.NextHop == "" && rt.MultiPath == nil {
+			// route's next-hop is local not install again.
+			continue
+		}
+		nlrt := nl.Route{Dst: dst}
+		for _, hop := range rt.MultiPath {
+			nxhe := &nl.NexthopInfo{
+				Hops: hop.Weight,
+				Gw:   net.ParseIP(hop.NextHop),
+			}
+			nlrt.MultiPath = append(nlrt.MultiPath, nxhe)
+		}
+		if rt.MultiPath == nil {
+			nlrt.Gw = net.ParseIP(rt.NextHop)
+			nlrt.Priority = rt.Metric
+		}
+		w.out.Debug("RouterWorker.LoadRoute: %s", nlrt.String())
+		promise := &libol.Promise{
+			First:  time.Second * 2,
+			MaxInt: time.Minute,
+			MinInt: time.Second * 10,
+		}
+		rt_c := rt
+		promise.Go(func() error {
+			if err := nl.RouteReplace(&nlrt); err != nil {
+				w.out.Warn("RouterWorker.LoadRoute: %v %s", nlrt, err)
+				return err
+			}
+			w.out.Info("RouterWorker.LoadRoute: %v success", rt_c.String())
+			return nil
+		})
+	}
+}
+
+func (w *RouterWorker) UnLoadRoutes() {
+	cfg := w.cfg
+	for _, rt := range cfg.Routes {
+		_, dst, err := net.ParseCIDR(rt.Prefix)
+		if err != nil {
+			continue
+		}
+		nlRt := nl.Route{Dst: dst}
+		if rt.MultiPath == nil {
+			nlRt.Gw = net.ParseIP(rt.NextHop)
+			nlRt.Priority = rt.Metric
+		}
+		w.out.Debug("RouterWorker.UnLoadRoute: %s", nlRt.String())
+		if err := nl.RouteDel(&nlRt); err != nil {
+			w.out.Warn("RouterWorker.UnLoadRoute: %s", err)
+			continue
+		}
+		w.out.Info("RouterWorker.UnLoadRoute: %v", rt.String())
+	}
+}
+
+func (w *RouterWorker) Forward() {
+	spec := w.spec
+	// Enable MASQUERADE, and allowed forward.
+	w.out.Debug("RouterWorker.Forward %v", w.cfg)
+	for _, sub := range spec.Subnets {
+		if sub.CIDR == "" {
+			continue
+		}
+		w.setR.Add(sub.CIDR)
+	}
+	w.toRelated(spec.Link, "Accept related")
+	w.toForward_s(spec.Link, "", w.setR.Name, "", "From route")
+	w.toMasq_s(w.setR.Name, "", "To Masq")
+}
+
+func (w *RouterWorker) Start(v api.Switcher) {
+	w.uuid = v.UUID()
+	w.LoadRoutes()
+	w.Forward()
+	w.WorkerImpl.Start(v)
+}
+
+func (w *RouterWorker) Stop() {
+	w.WorkerImpl.Stop()
+	w.UnLoadRoutes()
+}
+
+func (w *RouterWorker) Reload(v api.Switcher) {
+	w.Stop()
+	w.Initialize()
+	w.Start(v)
+}