fix CI tests and quite a lot of races

Signed-off-by: Steffen Vogel <post@steffenvogel.de>

.github/workflows/build.yaml

@@ -127,9 +127,11 @@ jobs:
         go install github.com/onsi/ginkgo/v2/ginkgo@latest
         go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
         go install honnef.co/go/tools/cmd/staticcheck@latest
+        go install github.com/amobe/gocov-merger@latest
+        go install github.com/jandelgado/gcov2lcov@latest
 
     - name: Run wice build
-      run: go build -o wice ./cmd
+      run: go build -o wice ./cmd/wice
       
     - name: Run vet
       run: go vet --copylocks=false ./...
@@ -144,7 +146,7 @@ jobs:
       run: make tests
 
     - name: Run tests requiring root privileges
-      if: ${{ matrix.target_arch == matrix.host_arch && matrix.target_os == matrix.host_os && matrix.host_os != 'windows' }}
+      if: ${{ matrix.target_arch == matrix.host_arch && matrix.target_os == matrix.host_os && matrix.host_os == 'linuxcoverprofile_merged.out' }}
       env:
         GINKGO_PKG: ./test
       run: |
@@ -159,6 +161,11 @@ jobs:
         name: test-logs-${{ matrix.target_os }}-${{ matrix.target_arch }}
         path: test/logs/
 
+    - name: Create merged coverage profile
+      if: ${{ matrix.target_os == 'linux' && matrix.target_arch == 'amd64' }}
+      run: |
+        make coverprofile_merged.out
+
     - name: Report test coverage to Codacy
       if: ${{ matrix.target_os == 'linux' && matrix.target_arch == 'amd64' }}
       uses: codacy/codacy-coverage-reporter-action@v1
@@ -166,7 +173,7 @@ jobs:
         CODACY_REPORTER_TMP_FOLDER: .codacy
       with:
         project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
-        coverage-reports: coverprofile.out
+        coverage-reports: coverprofile_merged.out
         force-coverage-parser: go
 
     - name: Report test coverage to Codecov
@@ -174,4 +181,4 @@ jobs:
       uses: codecov/codecov-action@v3
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
-        files: coverprofile.out
+        files: coverprofile_merged.out

Makefile

@@ -6,7 +6,7 @@ GIT_VERSION = $(shell git describe --tags --dirty || echo unknown)
 LDFLAGS = -X main.version=$(GIT_VERSION) -X main.commit=$(GIT_COMMIT) -X main.date=$(shell date -Iseconds)
 
 GINKGO_PKG ?= ./...
-GINKGO_OPTS += --covermode=count --randomize-all $(GINKGO_EXTRA_OPTS) $(GINKGO_PKG) -- $(GINKGO_TEST_OPTS)
+GINKGO_OPTS += --covermode=count --coverpkg=./... --coverprofile=coverprofile.out --randomize-all $(GINKGO_EXTRA_OPTS) $(GINKGO_PKG) -- $(GINKGO_TEST_OPTS)
 
 all: wice
 
@@ -17,10 +17,11 @@ wice:
 tests:
 	ginkgo run $(GINKGO_OPTS)
 
-lcov.info: $(wildcard *.out) 
+coverprofile_merged.out: $(shell find . -name "*.out" -type f)
-	@echo "Merging and converting coverage data..."
+	gocov-merger -o $@ $^
-	gocovmerge $(wildcard *.out) | gcov2lcov > $@
+
-	@echo "Done. $@ updated"
+lcov_merged.info: coverprofile_merged.out
+	gcov2lcov > $@ < $^ 
 
 coverage: lcov.info
 

cmd/wice/cmd_test.go

@@ -7,5 +7,5 @@ import (
 )
 
 func TestRunMain(t *testing.T) {
-	cmd.WGCmd.Execute()
+	wgCmd.Execute()
 }

cmd/wice/signal.go

@@ -46,11 +46,10 @@ func signal(cmd *cobra.Command, args []string) {
 	svr := grpcx.NewServer(opts...)
 
 	go func() {
-		signals := util.SetupSignals()
+		for sig := range util.SetupSignals() {
-		for sig := range signals {
 			logger.Debug("Received signal", zap.Any("signal", sig))
 
-			svr.GracefulStop()
+			svr.Close()
 		}
 	}()
 

docs/Installation.md

@@ -67,7 +67,7 @@ Building ɯice is fairly easy and allows you to install the latest unreleased ve
 You must have a working Go environment.
 
 ```bash
-go install riasc.eu/wice@latest
+go install riasc.eu/wice/cmd/wice@latest
 ```
 
 If required, it will fetch the dependencies and cache them, and validate configuration.

etc/wice.yaml

@@ -24,9 +24,9 @@ wireguard:
     min: 52820
     max: 65535
 
-# Control socket settings
+# RPC control socket settings
-socket:
+rpc:
-  path: /var/run/wice.sock
+  socket: /var/run/wice.sock
 
   # Start of wice daemon will block until its unblocked via the control socket
   # Mostly useful for testing automation

go.mod

@@ -19,7 +19,7 @@ require (
 	github.com/spf13/cobra v1.5.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.12.0
-	github.com/stv0g/gont v1.6.1
+	github.com/stv0g/gont v1.6.2
 	github.com/vishvananda/netlink v1.2.1-beta.2
 	github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74
 	go.uber.org/atomic v1.10.0
@@ -112,4 +112,4 @@ require (
 )
 
 // Temporary, until https://github.com/google/gopacket/pull/1042 is merged
-// replace github.com/google/gopacket => github.com/stv0g/gopacket v0.0.0-20220819110231-82599fdade4a
+replace github.com/google/gopacket => github.com/stv0g/gopacket v0.0.0-20220819110231-82599fdade4a

go.sum

@@ -258,8 +258,6 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
-github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@@ -544,16 +542,21 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stv0g/gont v1.6.1 h1:Xx7aYvOX9zZprgI/WhMG8stRtvlEjYBmgs0KhECvstw=
+github.com/stv0g/gont v1.6.2 h1:Bs2g93ONr67fF/s1ZAqboTnTf5l1jrbIXxPj9jPdfGk=
-github.com/stv0g/gont v1.6.1/go.mod h1:lZW2pE8w4aUf6VMUbdSIyJ5yS41macEeULMbZRDyQ58=
+github.com/stv0g/gont v1.6.2/go.mod h1:lZW2pE8w4aUf6VMUbdSIyJ5yS41macEeULMbZRDyQ58=
+github.com/stv0g/gopacket v0.0.0-20220819110231-82599fdade4a h1:oA2GX2QLOgJcyvdsQ8XCwvdbj3lBjk7RrWychSiIb8U=
+github.com/stv0g/gopacket v0.0.0-20220819110231-82599fdade4a/go.mod h1:riddUzxTSBpJXk3qBHtYr4qOhFhT6k/1c0E3qkQjQpA=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/subosito/gotenv v1.4.0 h1:yAzM1+SmVcz5R4tXGsNMu1jUl2aOJXoiWUCEwwnGrvs=
 github.com/subosito/gotenv v1.4.0/go.mod h1:mZd6rFysKEcUhUHXJk0C/08wAgyDBFuwEYL7vWWGaGo=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
 github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs=
 github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
+github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg=
 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -773,6 +776,7 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

pkg/feat/disc/epice/doc.go

@@ -1,3 +0,0 @@
-// This package implements endpoint (EP) discovery using Interactive Connection Establishment (ICE).
-
-package epice

pkg/feat/disc/epice/epice.go

@@ -1,3 +1,4 @@
+// This package implements endpoint (EP) discovery using Interactive Connection Establishment (ICE).
 package epice
 
 import (
@@ -9,6 +10,7 @@ import (
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 	"riasc.eu/wice/pkg/config"
 	"riasc.eu/wice/pkg/core"
+	icex "riasc.eu/wice/pkg/ice"
 	"riasc.eu/wice/pkg/signaling"
 	"riasc.eu/wice/pkg/watcher"
 	"riasc.eu/wice/pkg/wg"
@@ -28,7 +30,7 @@ type EndpointDiscovery struct {
 	logger *zap.Logger
 }
 
-func New(w *watcher.Watcher, cfg *config.Config, client *wgctrl.Client, backend signaling.Backend) (*EndpointDiscovery, error) {
+func New(w *watcher.Watcher, cfg *config.Config, client *wgctrl.Client, backend signaling.Backend) *EndpointDiscovery {
 	e := &EndpointDiscovery{
 		Peers:      map[*core.Peer]*Peer{},
 		Interfaces: map[*core.Interface]*Interface{},
@@ -40,15 +42,26 @@ func New(w *watcher.Watcher, cfg *config.Config, client *wgctrl.Client, backend
 		client:  client,
 		backend: backend,
 
-		logger: zap.L().Named("ep-disc"),
+		logger: zap.L().Named("epice"),
 	}
 
 	w.OnAll(e)
 
-	return e, nil
+	return e
+}
+
+func (e *EndpointDiscovery) Start() error {
+	e.logger.Info("Started ICE endpoint discovery")
+
+	return nil
 }
 
 func (e *EndpointDiscovery) Close() error {
+	// First switch all sessions to closing so they dont get restarted
+	for _, p := range e.Peers {
+		p.setConnectionState(icex.ConnectionStateClosing)
+	}
+
 	for _, p := range e.Peers {
 		if err := p.Close(); err != nil {
 			return fmt.Errorf("failed to close peer: %w", err)
@@ -64,10 +77,6 @@ func (e *EndpointDiscovery) Close() error {
 	return nil
 }
 
-func (e *EndpointDiscovery) OnConnectionStateChange(h OnConnectionStateHandler) {
-	e.onConnectionStateChange = append(e.onConnectionStateChange, h)
-}
-
 func (e *EndpointDiscovery) OnInterfaceAdded(ci *core.Interface) {
 	i, err := NewInterface(ci, e)
 	if err != nil {

pkg/feat/disc/epice/handlers.go

@@ -5,3 +5,7 @@ import icex "riasc.eu/wice/pkg/ice"
 type OnConnectionStateHandler interface {
 	OnConnectionStateChange(p *Peer, new, prev icex.ConnectionState)
 }
+
+func (e *EndpointDiscovery) OnConnectionStateChange(h OnConnectionStateHandler) {
+	e.onConnectionStateChange = append(e.onConnectionStateChange, h)
+}

pkg/feat/disc/epice/interface.go

@@ -31,7 +31,7 @@ func NewInterface(ci *core.Interface, d *EndpointDiscovery) (*Interface, error)
 		Interface: ci,
 		Discovery: d,
 
-		logger: zap.L().Named("ep.interface"),
+		logger: zap.L().Named("epice.interface"),
 	}
 
 	// Create per-interface UDPMux

pkg/feat/disc/epice/peer.go

@@ -16,6 +16,7 @@ import (
 	"riasc.eu/wice/pkg/pb"
 	"riasc.eu/wice/pkg/proxy"
 	"riasc.eu/wice/pkg/signaling"
+	"riasc.eu/wice/pkg/util"
 )
 
 type Peer struct {
@@ -23,18 +24,16 @@ type Peer struct {
 
 	Interface *Interface
 
-	config *config.Config
+	config          *config.Config
+	agentConfig     *ice.AgentConfig
+	agent           *ice.Agent
+	backend         signaling.Backend
+	proxy           proxy.Proxy
+	connectionState util.AtomicEnum[icex.ConnectionState]
+	credentials     pb.Credentials
 
-	backend signaling.Backend
+	signalingMessages      chan *signaling.Message
-	proxy   proxy.Proxy
+	connectionStateChanges chan icex.ConnectionState
-
-	// TODO: Avoid races around connection state
-	ConnectionState icex.ConnectionState
-
-	agentConfig *ice.AgentConfig
-	agent       *ice.Agent
-	conn        *ice.Conn
-	credentials pb.Credentials
 
 	logger *zap.Logger
 }
@@ -43,19 +42,30 @@ func NewPeer(cp *core.Peer, i *Interface) (*Peer, error) {
 	var err error
 
 	p := &Peer{
-		Peer:            cp,
+		Peer:      cp,
-		Interface:       i,
+		Interface: i,
-		ConnectionState: icex.ConnectionStateUnknown,
 
 		backend: i.Discovery.backend,
 		config:  i.Discovery.config,
 
+		signalingMessages:      make(chan *pb.SignalingMessage, 32),
+		connectionStateChanges: make(chan icex.ConnectionState, 32),
+
 		logger: zap.L().Named("ice.peer").With(
 			zap.String("intf", i.Name()),
 			zap.Any("peer", cp.PublicKey()),
 		),
 	}
 
+	p.connectionState.Store(ice.ConnectionStateClosed)
+
+	// Initialize signaling channel
+	kp := p.PublicPrivateKeyPair()
+	if err := p.backend.Subscribe(context.Background(), kp, p); err != nil {
+		p.logger.Fatal("Failed to subscribe to offers", zap.Error(err))
+	}
+	p.logger.Info("Subscribed to messages from peer", zap.Any("kp", kp))
+
 	// Prepare ICE agent configuration
 	p.agentConfig, err = p.config.AgentConfig()
 	if err != nil {
@@ -85,24 +95,33 @@ func NewPeer(cp *core.Peer, i *Interface) (*Peer, error) {
 		return nil, fmt.Errorf("failed tp setup peer. Neither NAT or Bind is configured")
 	}
 
-	// Initialize signaling channel
+	if err = p.createAgent(); err != nil {
-	kp := p.PublicPrivateKeyPair()
+		return nil, fmt.Errorf("failed to create initial agent: %w", err)
-	if err := p.backend.Subscribe(context.Background(), kp, p); err != nil {
-		p.logger.Fatal("Failed to subscribe to offers", zap.Error(err))
 	}
 
-	p.logger.Info("Subscribed to messages from peer", zap.Any("kp", kp))
+	go p.run()
-
-	// Initialize new agent by simulating a closed event
-	p.onConnectionStateChange(ice.ConnectionStateClosed)
 
 	return p, nil
 }
 
+func (p *Peer) ConnectionState() icex.ConnectionState {
+	return p.connectionState.Load()
+}
+
+func (p *Peer) run() {
+	for {
+		select {
+		case msg := <-p.signalingMessages:
+			p.onSignalingMessage(msg)
+
+		case sc := <-p.connectionStateChanges:
+			p.onConnectionStateChange(sc)
+		}
+	}
+}
+
 // Close destroys the peer as well as the ICE agent and proxies
 func (p *Peer) Close() error {
-	p.setConnectionState(icex.ConnectionStateClosing)
-
 	if err := p.agent.Close(); err != nil {
 		return fmt.Errorf("failed to close ICE agent: %w", err)
 	}
@@ -135,7 +154,7 @@ func (p *Peer) sendCredentials(need bool) error {
 		Credentials: &p.credentials,
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 35*time.Second)
 	defer cancel()
 
 	if err := p.backend.Publish(ctx, p.PublicPrivateKeyPair(), msg); err != nil {
@@ -164,10 +183,13 @@ func (p *Peer) sendCandidate(c ice.Candidate) error {
 	return nil
 }
 
-func (p *Peer) newAgent() (*ice.Agent, error) {
+func (p *Peer) createAgent() error {
-	var agent *ice.Agent
 	var err error
 
+	if !p.setConnectionStateIf(ice.ConnectionStateClosed, icex.ConnectionStateCreating) {
+		return fmt.Errorf("failed to create new agent if previous one is not closed")
+	}
+
 	p.logger.Info("Creating new agent")
 
 	p.credentials = pb.NewCredentials()
@@ -176,28 +198,36 @@ func (p *Peer) newAgent() (*ice.Agent, error) {
 	p.agentConfig.LocalPwd = p.credentials.Pwd
 
 	// Setup new ICE Agent
-	if agent, err = ice.NewAgent(p.agentConfig); err != nil {
+	if p.agent, err = ice.NewAgent(p.agentConfig); err != nil {
-		return nil, fmt.Errorf("failed to create ICE agent: %w", err)
+		return fmt.Errorf("failed to create ICE agent: %w", err)
 	}
 
 	// When we have gathered a new ICE Candidate send it to the remote peer
-	if err := agent.OnCandidate(p.onCandidate); err != nil {
+	if err := p.agent.OnCandidate(p.onCandidate); err != nil {
-		return nil, fmt.Errorf("failed to setup on candidate handler: %w", err)
+		return fmt.Errorf("failed to setup on candidate handler: %w", err)
 	}
 
 	// When selected candidate pair changes
-	if err := agent.OnSelectedCandidatePairChange(p.onSelectedCandidatePairChange); err != nil {
+	if err := p.agent.OnSelectedCandidatePairChange(p.onSelectedCandidatePairChange); err != nil {
-		return nil, fmt.Errorf("failed to setup on selected candidate pair handler: %w", err)
+		return fmt.Errorf("failed to setup on selected candidate pair handler: %w", err)
 	}
 
 	// When ICE Connection state has change print to stdout
-	if err := agent.OnConnectionStateChange(p.onConnectionStateChange); err != nil {
+	if err := p.agent.OnConnectionStateChange(func(cs ice.ConnectionState) {
-		return nil, fmt.Errorf("failed to setup on connection state handler: %w", err)
+		p.onConnectionStateChange(icex.ConnectionState(cs))
+	}); err != nil {
+		return fmt.Errorf("failed to setup on connection state handler: %w", err)
 	}
 
-	p.setConnectionState(icex.ConnectionStateIdle)
+	if !p.setConnectionStateIf(icex.ConnectionStateCreating, icex.ConnectionStateIdle) {
+		return fmt.Errorf("failed to switch to idle state")
+	}
 
-	return agent, nil
+	if err := p.sendCredentials(true); err != nil {
+		return fmt.Errorf("failed to send peer credentials: %w", err)
+	}
+
+	return nil
 }
 
 // isSessionRestart checks if a received offer should restart the
@@ -210,7 +240,7 @@ func (p *Peer) isSessionRestart(c *pb.Credentials) bool {
 
 	credsChanged := (ufrag != "" && pwd != "") && (c.Ufrag != "" && c.Pwd != "") && (ufrag != c.Ufrag || pwd != c.Pwd)
 
-	return p.conn != nil && credsChanged
+	return p.ConnectionState() != ice.ConnectionStateClosed && credsChanged
 }
 
 func (p *Peer) addRemoteCandidate(c *pb.Candidate) error {
@@ -230,14 +260,15 @@ func (p *Peer) addRemoteCandidate(c *pb.Candidate) error {
 
 func (p *Peer) connect(ufrag, pwd string) error {
 	var err error
+	var conn *ice.Conn
 
 	// TODO: use proper context
 	if p.IsControlling() {
 		p.logger.Debug("Dialing...")
-		p.conn, err = p.agent.Dial(context.Background(), ufrag, pwd)
+		conn, err = p.agent.Dial(context.Background(), ufrag, pwd)
 	} else {
 		p.logger.Debug("Accepting...")
-		p.conn, err = p.agent.Accept(context.Background(), ufrag, pwd)
+		conn, err = p.agent.Accept(context.Background(), ufrag, pwd)
 	}
 	if err != nil {
 		return err
@@ -248,7 +279,7 @@ func (p *Peer) connect(ufrag, pwd string) error {
 		return fmt.Errorf("failed to get selected candidate pair: %w", err)
 	}
 
-	ep, err := p.proxy.Update(cp, p.conn)
+	ep, err := p.proxy.Update(cp, conn)
 	if err != nil {
 		return fmt.Errorf("failed to update proxy: %w", err)
 	}
@@ -261,11 +292,22 @@ func (p *Peer) connect(ufrag, pwd string) error {
 }
 
 func (p *Peer) setConnectionState(new icex.ConnectionState) icex.ConnectionState {
-	prev := p.ConnectionState
+	prev := p.connectionState.Swap(new)
-	p.ConnectionState = new
 
-	// Suppress initial invocation of handler from NewPeer()
+	p.logger.Info("Connection state changed",
-	if prev != icex.ConnectionStateUnknown {
+		zap.String("new", strings.ToLower(new.String())),
+		zap.String("previous", strings.ToLower(prev.String())))
+
+	for _, h := range p.Interface.Discovery.onConnectionStateChange {
+		h.OnConnectionStateChange(p, new, prev)
+	}
+
+	return prev
+}
+
+func (p *Peer) setConnectionStateIf(prev, new icex.ConnectionState) bool {
+	swapped := p.connectionState.CompareAndSwap(prev, new)
+	if swapped {
 		p.logger.Info("Connection state changed",
 			zap.String("new", strings.ToLower(new.String())),
 			zap.String("previous", strings.ToLower(prev.String())))
@@ -273,7 +315,8 @@ func (p *Peer) setConnectionState(new icex.ConnectionState) icex.ConnectionState
 		for _, h := range p.Interface.Discovery.onConnectionStateChange {
 			h.OnConnectionStateChange(p, new, prev)
 		}
+
 	}
 
-	return prev
+	return swapped
 }

pkg/feat/disc/epice/peer_handlers.go

@@ -13,28 +13,25 @@ import (
 
 // onConnectionStateChange is a callback which gets called by the ICE agent
 // whenever the state of the ICE connection has changed
-func (p *Peer) onConnectionStateChange(cs ice.ConnectionState) {
+func (p *Peer) onConnectionStateChange(new icex.ConnectionState) {
 	var err error
 
-	csx := icex.ConnectionState(cs)
+	if p.ConnectionState() == icex.ConnectionStateClosing {
-	prevConnectionState := p.setConnectionState(csx)
+		p.logger.Debug("Ignoring state transition as we are closing the session")
+		return
+	}
 
-	if cs == ice.ConnectionStateFailed || cs == ice.ConnectionStateDisconnected {
+	p.setConnectionState(new)
-		// TODO: Add some random delay?
 
+	if new == ice.ConnectionStateFailed || new == ice.ConnectionStateDisconnected {
 		if err := p.Restart(); err != nil {
 			p.logger.Error("Failed to restart ICE session", zap.Error(err))
 		}
-	} else if cs == ice.ConnectionStateClosed && prevConnectionState != icex.ConnectionStateClosing {
+	} else if new == ice.ConnectionStateClosed {
-		if p.agent, err = p.newAgent(); err != nil {
+		if err = p.createAgent(); err != nil {
 			p.logger.Error("Failed to create agent", zap.Error(err))
 			return
 		}
-
-		if err := p.sendCredentials(true); err != nil {
-			p.logger.Error("Failed to send peer credentials", zap.Error(err))
-			return
-		}
 	}
 }
 
@@ -70,9 +67,14 @@ func (p *Peer) onRemoteCredentials(c *pb.Credentials) {
 			p.logger.Error("Failed to restart ICE session", zap.Error(err))
 		}
 	} else {
-		if p.ConnectionState == icex.ConnectionStateIdle {
+		if c.NeedCreds {
-			p.setConnectionState(ice.ConnectionStateNew)
+			if err := p.sendCredentials(false); err != nil {
+				p.logger.Error("Failed to send credentials", zap.Error(err))
+				return
+			}
+		}
 
+		if p.setConnectionStateIf(icex.ConnectionStateIdle, ice.ConnectionStateNew) {
 			if err := p.agent.SetRemoteCredentials(c.Ufrag, c.Pwd); err != nil {
 				p.logger.Error("Failed to set remote credentials", zap.Error(err))
 				return
@@ -84,13 +86,6 @@ func (p *Peer) onRemoteCredentials(c *pb.Credentials) {
 			}
 			p.logger.Info("Started gathering local ICE candidates")
 		}
-
-		if c.NeedCreds {
-			if err := p.sendCredentials(false); err != nil {
-				p.logger.Error("Failed to send credentials", zap.Error(err))
-				return
-			}
-		}
 	}
 }
 
@@ -104,23 +99,23 @@ func (p *Peer) onRemoteCandidate(c *pb.Candidate) {
 		return
 	}
 
-	if p.ConnectionState == ice.ConnectionStateNew {
+	if p.setConnectionStateIf(ice.ConnectionStateNew, icex.ConnectionStateConnecting) {
-		p.setConnectionState(icex.ConnectionStateConnecting)
-
 		ufrag, pwd, err := p.agent.GetRemoteUserCredentials()
 		if err != nil {
 			p.logger.Error("Failed to get remote credentials", zap.Error(err))
 			return
 		}
 
-		if err := p.connect(ufrag, pwd); err != nil && !errors.Is(err, ice.ErrClosed) {
+		go func() {
-			p.logger.Error("Failed to connect", zap.Error(err))
+			if err := p.connect(ufrag, pwd); err != nil && !errors.Is(err, ice.ErrClosed) {
-			return
+				p.logger.Error("Failed to connect", zap.Error(err))
-		}
+				return
+			}
+		}()
 	}
 }
 
-func (p *Peer) onSignalingMessage(kp *crypto.PublicKeyPair, msg *signaling.Message) {
+func (p *Peer) onSignalingMessage(msg *signaling.Message) {
 	if msg.Credentials != nil {
 		p.onRemoteCredentials(msg.Credentials)
 	}
@@ -132,9 +127,5 @@ func (p *Peer) onSignalingMessage(kp *crypto.PublicKeyPair, msg *signaling.Messa
 
 // OnSignalingMessage is invoked for every message received via the signaling backend
 func (p *Peer) OnSignalingMessage(kp *crypto.PublicKeyPair, msg *signaling.Message) {
-	if p.agent == nil {
+	p.signalingMessages <- msg
-		p.logger.Warn("Ignoring message as agent has not been created yet")
-	} else {
-		go p.onSignalingMessage(kp, msg)
-	}
 }

pkg/pb/candidate.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.27.1
+// 	protoc-gen-go v1.28.1
-// 	protoc        v3.19.4
+// 	protoc        v3.6.1
 // source: candidate.proto
 
 package pb

pkg/pb/common.pb.go

@@ -95,77 +95,14 @@ func (ConnectionState) EnumDescriptor() ([]byte, []int) {
 	return file_common_proto_rawDescGZIP(), []int{0}
 }
 
-// The Error code inspired by POSIX's errno
+type Empty struct {
-// See: https://pubs.opengroup.org/onlinepubs/009696899/functions/xsh_chap02_03.html
-type Error_Code int32
-
-const (
-	Error_SUCCESS  Error_Code = 0
-	Error_EPERM    Error_Code = 1
-	Error_ENOENT   Error_Code = 2
-	Error_EEXIST   Error_Code = 17
-	Error_EALREADY Error_Code = 18
-	Error_ENOTSUP  Error_Code = 10
-	Error_EUNKNOWN Error_Code = 255
-)
-
-// Enum value maps for Error_Code.
-var (
-	Error_Code_name = map[int32]string{
-		0:   "SUCCESS",
-		1:   "EPERM",
-		2:   "ENOENT",
-		17:  "EEXIST",
-		18:  "EALREADY",
-		10:  "ENOTSUP",
-		255: "EUNKNOWN",
-	}
-	Error_Code_value = map[string]int32{
-		"SUCCESS":  0,
-		"EPERM":    1,
-		"ENOENT":   2,
-		"EEXIST":   17,
-		"EALREADY": 18,
-		"ENOTSUP":  10,
-		"EUNKNOWN": 255,
-	}
-)
-
-func (x Error_Code) Enum() *Error_Code {
-	p := new(Error_Code)
-	*p = x
-	return p
-}
-
-func (x Error_Code) String() string {
-	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
-}
-
-func (Error_Code) Descriptor() protoreflect.EnumDescriptor {
-	return file_common_proto_enumTypes[1].Descriptor()
-}
-
-func (Error_Code) Type() protoreflect.EnumType {
-	return &file_common_proto_enumTypes[1]
-}
-
-func (x Error_Code) Number() protoreflect.EnumNumber {
-	return protoreflect.EnumNumber(x)
-}
-
-// Deprecated: Use Error_Code.Descriptor instead.
-func (Error_Code) EnumDescriptor() ([]byte, []int) {
-	return file_common_proto_rawDescGZIP(), []int{2, 0}
-}
-
-type Void struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 }
 
-func (x *Void) Reset() {
+func (x *Empty) Reset() {
-	*x = Void{}
+	*x = Empty{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_common_proto_msgTypes[0]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -173,13 +110,13 @@ func (x *Void) Reset() {
 	}
 }
 
-func (x *Void) String() string {
+func (x *Empty) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*Void) ProtoMessage() {}
+func (*Empty) ProtoMessage() {}
 
-func (x *Void) ProtoReflect() protoreflect.Message {
+func (x *Empty) ProtoReflect() protoreflect.Message {
 	mi := &file_common_proto_msgTypes[0]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -191,8 +128,8 @@ func (x *Void) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use Void.ProtoReflect.Descriptor instead.
+// Deprecated: Use Empty.ProtoReflect.Descriptor instead.
-func (*Void) Descriptor() ([]byte, []int) {
+func (*Empty) Descriptor() ([]byte, []int) {
 	return file_common_proto_rawDescGZIP(), []int{0}
 }
 
@@ -252,96 +189,28 @@ func (x *Timestamp) GetNanos() int32 {
 	return 0
 }
 
-// A common error type used as return value for gRPC calls
-type Error struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	// The error code
-	Code Error_Code `protobuf:"varint,1,opt,name=code,proto3,enum=wice.Error_Code" json:"code,omitempty"`
-	// A human readable error message
-	Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
-}
-
-func (x *Error) Reset() {
-	*x = Error{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_common_proto_msgTypes[2]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *Error) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*Error) ProtoMessage() {}
-
-func (x *Error) ProtoReflect() protoreflect.Message {
-	mi := &file_common_proto_msgTypes[2]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use Error.ProtoReflect.Descriptor instead.
-func (*Error) Descriptor() ([]byte, []int) {
-	return file_common_proto_rawDescGZIP(), []int{2}
-}
-
-func (x *Error) GetCode() Error_Code {
-	if x != nil {
-		return x.Code
-	}
-	return Error_SUCCESS
-}
-
-func (x *Error) GetMessage() string {
-	if x != nil {
-		return x.Message
-	}
-	return ""
-}
-
 var File_common_proto protoreflect.FileDescriptor
 
 var file_common_proto_rawDesc = []byte{
 	0x0a, 0x0c, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04,
-	0x77, 0x69, 0x63, 0x65, 0x22, 0x06, 0x0a, 0x04, 0x56, 0x6f, 0x69, 0x64, 0x22, 0x3b, 0x0a, 0x09,
+	0x77, 0x69, 0x63, 0x65, 0x22, 0x07, 0x0a, 0x05, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x3b, 0x0a,
-	0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x63,
+	0x09, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65,
-	0x6f, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x73, 0x65, 0x63, 0x6f,
+	0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x73, 0x65, 0x63,
-	0x6e, 0x64, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x61, 0x6e, 0x6f, 0x73, 0x18, 0x02, 0x20, 0x01,
+	0x6f, 0x6e, 0x64, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x61, 0x6e, 0x6f, 0x73, 0x18, 0x02, 0x20,
-	0x28, 0x05, 0x52, 0x05, 0x6e, 0x61, 0x6e, 0x6f, 0x73, 0x22, 0xa9, 0x01, 0x0a, 0x05, 0x45, 0x72,
+	0x01, 0x28, 0x05, 0x52, 0x05, 0x6e, 0x61, 0x6e, 0x6f, 0x73, 0x2a, 0xa5, 0x01, 0x0a, 0x0f, 0x43,
-	0x72, 0x6f, 0x72, 0x12, 0x24, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x07,
-	0x0e, 0x32, 0x10, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x2e, 0x43,
+	0x0a, 0x03, 0x4e, 0x45, 0x57, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x48, 0x45, 0x43, 0x4b,
-	0x6f, 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73,
+	0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54,
-	0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73,
+	0x45, 0x44, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45,
-	0x61, 0x67, 0x65, 0x22, 0x60, 0x0a, 0x04, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x53,
+	0x44, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x04, 0x12,
-	0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x50, 0x45, 0x52,
+	0x10, 0x0a, 0x0c, 0x44, 0x49, 0x53, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x45, 0x44, 0x10,
-	0x4d, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x45, 0x4e, 0x4f, 0x45, 0x4e, 0x54, 0x10, 0x02, 0x12,
+	0x05, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x44, 0x10, 0x06, 0x12, 0x0c, 0x0a,
-	0x0a, 0x0a, 0x06, 0x45, 0x45, 0x58, 0x49, 0x53, 0x54, 0x10, 0x11, 0x12, 0x0c, 0x0a, 0x08, 0x45,
+	0x08, 0x43, 0x52, 0x45, 0x41, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x64, 0x12, 0x08, 0x0a, 0x04, 0x49,
-	0x41, 0x4c, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x12, 0x12, 0x0b, 0x0a, 0x07, 0x45, 0x4e, 0x4f,
+	0x44, 0x4c, 0x45, 0x10, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54,
-	0x54, 0x53, 0x55, 0x50, 0x10, 0x0a, 0x12, 0x0d, 0x0a, 0x08, 0x45, 0x55, 0x4e, 0x4b, 0x4e, 0x4f,
+	0x49, 0x4e, 0x47, 0x10, 0x66, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4c, 0x4f, 0x53, 0x49, 0x4e, 0x47,
-	0x57, 0x4e, 0x10, 0xff, 0x01, 0x2a, 0xa5, 0x01, 0x0a, 0x0f, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63,
+	0x10, 0x67, 0x42, 0x16, 0x5a, 0x14, 0x72, 0x69, 0x61, 0x73, 0x63, 0x2e, 0x65, 0x75, 0x2f, 0x77,
-	0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x07, 0x0a, 0x03, 0x4e, 0x45, 0x57,
+	0x69, 0x63, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
-	0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x48, 0x45, 0x43, 0x4b, 0x49, 0x4e, 0x47, 0x10, 0x01,
+	0x6f, 0x33,
-	0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x45, 0x44, 0x10, 0x02, 0x12,
-	0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x03, 0x12, 0x0a,
-	0x0a, 0x06, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x04, 0x12, 0x10, 0x0a, 0x0c, 0x44, 0x49,
-	0x53, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x45, 0x44, 0x10, 0x05, 0x12, 0x0a, 0x0a, 0x06,
-	0x43, 0x4c, 0x4f, 0x53, 0x45, 0x44, 0x10, 0x06, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x52, 0x45, 0x41,
-	0x54, 0x49, 0x4e, 0x47, 0x10, 0x64, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x44, 0x4c, 0x45, 0x10, 0x65,
-	0x12, 0x0e, 0x0a, 0x0a, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x66,
-	0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4c, 0x4f, 0x53, 0x49, 0x4e, 0x47, 0x10, 0x67, 0x42, 0x16, 0x5a,
-	0x14, 0x72, 0x69, 0x61, 0x73, 0x63, 0x2e, 0x65, 0x75, 0x2f, 0x77, 0x69, 0x63, 0x65, 0x2f, 0x70,
-	0x6b, 0x67, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -356,22 +225,19 @@ func file_common_proto_rawDescGZIP() []byte {
 	return file_common_proto_rawDescData
 }
 
-var file_common_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
+var file_common_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
-var file_common_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
+var file_common_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
 var file_common_proto_goTypes = []interface{}{
 	(ConnectionState)(0), // 0: wice.ConnectionState
-	(Error_Code)(0),      // 1: wice.Error.Code
+	(*Empty)(nil),        // 1: wice.Empty
-	(*Void)(nil),         // 2: wice.Void
+	(*Timestamp)(nil),    // 2: wice.Timestamp
-	(*Timestamp)(nil),    // 3: wice.Timestamp
-	(*Error)(nil),        // 4: wice.Error
 }
 var file_common_proto_depIdxs = []int32{
-	1, // 0: wice.Error.code:type_name -> wice.Error.Code
+	0, // [0:0] is the sub-list for method output_type
-	1, // [1:1] is the sub-list for method output_type
+	0, // [0:0] is the sub-list for method input_type
-	1, // [1:1] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
-	1, // [1:1] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
-	1, // [1:1] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
-	0, // [0:1] is the sub-list for field type_name
 }
 
 func init() { file_common_proto_init() }
@@ -381,7 +247,7 @@ func file_common_proto_init() {
 	}
 	if !protoimpl.UnsafeEnabled {
 		file_common_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*Void); i {
+			switch v := v.(*Empty); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -404,26 +270,14 @@ func file_common_proto_init() {
 				return nil
 			}
 		}
-		file_common_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*Error); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
 	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
 		File: protoimpl.DescBuilder{
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_common_proto_rawDesc,
-			NumEnums:      2,
+			NumEnums:      1,
-			NumMessages:   3,
+			NumMessages:   2,
 			NumExtensions: 0,
 			NumServices:   0,
 		},

pkg/pb/config.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.27.1
+// 	protoc-gen-go v1.28.1
-// 	protoc        v3.19.4
+// 	protoc        v3.6.1
 // source: config.proto
 
 package pb

pkg/pb/event.go

@@ -21,7 +21,11 @@ func (e *Event) Log(l *zap.Logger, msg string, fields ...zap.Field) {
 	}
 
 	if e.Peer != nil {
-		pk, _ := crypto.ParseKeyBytes(e.Peer)
+		pk, err := crypto.ParseKeyBytes(e.Peer)
+		if err != nil {
+			panic(err)
+		}
+
 		fields = append(fields, zap.Any("peer", pk))
 	}
 

pkg/pb/event.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.27.1
+// 	protoc-gen-go v1.28.1
-// 	protoc        v3.19.4
+// 	protoc        v3.6.1
 // source: event.proto
 
 package pb

pkg/pb/interface.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.27.1
+// 	protoc-gen-go v1.28.1
-// 	protoc        v3.19.4
+// 	protoc        v3.6.1
 // source: interface.proto
 
 package pb

pkg/pb/message.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.27.1
+// 	protoc-gen-go v1.28.1
-// 	protoc        v3.19.4
+// 	protoc        v3.6.1
 // source: message.proto
 
 package pb

pkg/pb/peer.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.27.1
+// 	protoc-gen-go v1.28.1
-// 	protoc        v3.19.4
+// 	protoc        v3.6.1
 // source: peer.proto
 
 package pb

pkg/pb/rpc.pb.go

@@ -577,61 +577,61 @@ var file_rpc_proto_rawDesc = []byte{
 	0x73, 0x1a, 0x0b, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x22, 0x00,
 	0x30, 0x01, 0x12, 0x2b, 0x0a, 0x06, 0x55, 0x6e, 0x57, 0x61, 0x69, 0x74, 0x12, 0x12, 0x2e, 0x77,
 	0x69, 0x63, 0x65, 0x2e, 0x55, 0x6e, 0x57, 0x61, 0x69, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73,
-	0x1a, 0x0b, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x00, 0x12,
+	0x1a, 0x0b, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12,
 	0x27, 0x0a, 0x04, 0x53, 0x74, 0x6f, 0x70, 0x12, 0x10, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x53,
 	0x74, 0x6f, 0x70, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x0b, 0x2e, 0x77, 0x69, 0x63, 0x65,
-	0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x00, 0x32, 0x81, 0x04, 0x0a, 0x07, 0x57, 0x61, 0x74,
+	0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x32, 0x82, 0x04, 0x0a, 0x07, 0x57, 0x61, 0x74,
 	0x63, 0x68, 0x65, 0x72, 0x12, 0x27, 0x0a, 0x04, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x10, 0x2e, 0x77,
 	0x69, 0x63, 0x65, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x0b,
-	0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x00, 0x12, 0x27, 0x0a,
+	0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x28, 0x0a,
-	0x09, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0a, 0x2e, 0x77, 0x69, 0x63,
+	0x09, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x2e, 0x77, 0x69, 0x63,
-	0x65, 0x2e, 0x56, 0x6f, 0x69, 0x64, 0x1a, 0x0c, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x53, 0x74,
+	0x65, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x0c, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x53,
-	0x61, 0x74, 0x75, 0x73, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x0f, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65,
+	0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x0f, 0x52, 0x65, 0x6d, 0x6f, 0x76,
-	0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12, 0x1b, 0x2e, 0x77, 0x69, 0x63, 0x65,
+	0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12, 0x1b, 0x2e, 0x77, 0x69, 0x63,
-	0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65,
+	0x65, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63,
-	0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x0b, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x45, 0x72,
+	0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x0b, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x45,
-	0x72, 0x6f, 0x72, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x13, 0x53, 0x79, 0x6e, 0x63, 0x49, 0x6e, 0x74,
+	0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x13, 0x53, 0x79, 0x6e, 0x63, 0x49, 0x6e,
-	0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1b, 0x2e, 0x77,
+	0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1b, 0x2e,
-	0x69, 0x63, 0x65, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x43, 0x6f, 0x6e,
+	0x77, 0x69, 0x63, 0x65, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x43, 0x6f,
-	0x66, 0x69, 0x67, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x0b, 0x2e, 0x77, 0x69, 0x63, 0x65,
+	0x6e, 0x66, 0x69, 0x67, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x0b, 0x2e, 0x77, 0x69, 0x63,
-	0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x00, 0x12, 0x40, 0x0a, 0x12, 0x41, 0x64, 0x64, 0x49,
+	0x65, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x40, 0x0a, 0x12, 0x41, 0x64, 0x64,
-	0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1b,
+	0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12,
-	0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x43,
+	0x1b, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65,
-	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x0b, 0x2e, 0x77, 0x69,
+	0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x0b, 0x2e, 0x77,
-	0x63, 0x65, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x00, 0x12, 0x40, 0x0a, 0x12, 0x53, 0x65,
+	0x69, 0x63, 0x65, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x40, 0x0a, 0x12, 0x53,
-	0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
+	0x65, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69,
-	0x12, 0x1b, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63,
+	0x67, 0x12, 0x1b, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61,
-	0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x0b, 0x2e,
+	0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x0b,
-	0x77, 0x69, 0x63, 0x65, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x13,
+	0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x57, 0x0a,
-	0x47, 0x65, 0x74, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73, 0x73,
+	0x13, 0x47, 0x65, 0x74, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73,
-	0x61, 0x67, 0x65, 0x12, 0x1f, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x69,
+	0x73, 0x61, 0x67, 0x65, 0x12, 0x1f, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53,
-	0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x50, 0x61,
+	0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x50,
-	0x72, 0x61, 0x6d, 0x73, 0x1a, 0x1d, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53,
+	0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x1d, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x47, 0x65, 0x74,
-	0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x52,
+	0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
-	0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x45, 0x0a, 0x13, 0x50, 0x75, 0x74, 0x53, 0x69, 0x67, 0x6e,
+	0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x45, 0x0a, 0x13, 0x50, 0x75, 0x74, 0x53, 0x69, 0x67,
-	0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x1f, 0x2e, 0x77,
+	0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x1f, 0x2e,
-	0x69, 0x63, 0x65, 0x2e, 0x50, 0x75, 0x74, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67,
+	0x77, 0x69, 0x63, 0x65, 0x2e, 0x50, 0x75, 0x74, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e,
-	0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x0b, 0x2e,
+	0x67, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x0b,
-	0x77, 0x69, 0x63, 0x65, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x00, 0x32, 0x50, 0x0a, 0x17,
+	0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x32, 0x50, 0x0a,
-	0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x44, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72,
+	0x17, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x44, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65,
-	0x79, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x35, 0x0a, 0x0b, 0x52, 0x65, 0x73, 0x74, 0x61,
+	0x72, 0x79, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x35, 0x0a, 0x0b, 0x52, 0x65, 0x73, 0x74,
-	0x72, 0x74, 0x50, 0x65, 0x65, 0x72, 0x12, 0x17, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x52, 0x65,
+	0x61, 0x72, 0x74, 0x50, 0x65, 0x65, 0x72, 0x12, 0x17, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x52,
-	0x73, 0x74, 0x61, 0x72, 0x74, 0x50, 0x65, 0x65, 0x72, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a,
+	0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x50, 0x65, 0x65, 0x72, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73,
-	0x0b, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x00, 0x32, 0xb1,
+	0x1a, 0x0b, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x32,
-	0x01, 0x0a, 0x0f, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x53, 0x6f, 0x63, 0x6b,
+	0xb1, 0x01, 0x0a, 0x0f, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x53, 0x6f, 0x63,
-	0x65, 0x74, 0x12, 0x57, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69,
+	0x6b, 0x65, 0x74, 0x12, 0x57, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c,
-	0x6e, 0x67, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x1f, 0x2e, 0x77, 0x69, 0x63, 0x65,
+	0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x1f, 0x2e, 0x77, 0x69, 0x63,
-	0x2e, 0x47, 0x65, 0x74, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73,
-	0x73, 0x61, 0x67, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x1d, 0x2e, 0x77, 0x69, 0x63,
 	0x65, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4d, 0x65,
-	0x73, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x45, 0x0a, 0x13, 0x50,
+	0x73, 0x73, 0x61, 0x67, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x1d, 0x2e, 0x77, 0x69,
-	0x75, 0x74, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73, 0x73, 0x61,
+	0x63, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4d,
-	0x67, 0x65, 0x12, 0x1f, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x50, 0x75, 0x74, 0x53, 0x69, 0x67,
+	0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x45, 0x0a, 0x13,
-	0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x50, 0x61, 0x72,
+	0x50, 0x75, 0x74, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73, 0x73,
-	0x61, 0x6d, 0x73, 0x1a, 0x0b, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72,
+	0x61, 0x67, 0x65, 0x12, 0x1f, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x50, 0x75, 0x74, 0x53, 0x69,
-	0x22, 0x00, 0x42, 0x16, 0x5a, 0x14, 0x72, 0x69, 0x61, 0x73, 0x63, 0x2e, 0x65, 0x75, 0x2f, 0x77,
+	0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x50, 0x61,
-	0x69, 0x63, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
+	0x72, 0x61, 0x6d, 0x73, 0x1a, 0x0b, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x45, 0x6d, 0x70, 0x74,
-	0x6f, 0x33,
+	0x79, 0x22, 0x00, 0x42, 0x16, 0x5a, 0x14, 0x72, 0x69, 0x61, 0x73, 0x63, 0x2e, 0x65, 0x75, 0x2f,
+	0x77, 0x69, 0x63, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x33,
 }
 
 var (
@@ -662,9 +662,8 @@ var file_rpc_proto_goTypes = []interface{}{
 	(*Interface)(nil),                 // 11: wice.Interface
 	(*InterfaceConfig)(nil),           // 12: wice.InterfaceConfig
 	(*SignalingEnvelope)(nil),         // 13: wice.SignalingEnvelope
-	(*Void)(nil),                      // 14: wice.Void
+	(*Empty)(nil),                     // 14: wice.Empty
 	(*Event)(nil),                     // 15: wice.Event
-	(*Error)(nil),                     // 16: wice.Error
 }
 var file_rpc_proto_depIdxs = []int32{
 	11, // 0: wice.Status.interfaces:type_name -> wice.Interface
@@ -675,7 +674,7 @@ var file_rpc_proto_depIdxs = []int32{
 	1,  // 5: wice.Socket.UnWait:input_type -> wice.UnWaitParams
 	2,  // 6: wice.Socket.Stop:input_type -> wice.StopParams
 	4,  // 7: wice.Watcher.Sync:input_type -> wice.SyncParams
-	14, // 8: wice.Watcher.GetStatus:input_type -> wice.Void
+	14, // 8: wice.Watcher.GetStatus:input_type -> wice.Empty
 	5,  // 9: wice.Watcher.RemoveInterface:input_type -> wice.RemoveInterfaceParams
 	7,  // 10: wice.Watcher.SyncInterfaceConfig:input_type -> wice.InterfaceConfigParams
 	7,  // 11: wice.Watcher.AddInterfaceConfig:input_type -> wice.InterfaceConfigParams
@@ -686,19 +685,19 @@ var file_rpc_proto_depIdxs = []int32{
 	8,  // 16: wice.SignalingSocket.GetSignalingMessage:input_type -> wice.GetSignalingMessageParams
 	10, // 17: wice.SignalingSocket.PutSignalingMessage:input_type -> wice.PutSignalingMessageParams
 	15, // 18: wice.Socket.StreamEvents:output_type -> wice.Event
-	16, // 19: wice.Socket.UnWait:output_type -> wice.Error
+	14, // 19: wice.Socket.UnWait:output_type -> wice.Empty
-	16, // 20: wice.Socket.Stop:output_type -> wice.Error
+	14, // 20: wice.Socket.Stop:output_type -> wice.Empty
-	16, // 21: wice.Watcher.Sync:output_type -> wice.Error
+	14, // 21: wice.Watcher.Sync:output_type -> wice.Empty
 	0,  // 22: wice.Watcher.GetStatus:output_type -> wice.Status
-	16, // 23: wice.Watcher.RemoveInterface:output_type -> wice.Error
+	14, // 23: wice.Watcher.RemoveInterface:output_type -> wice.Empty
-	16, // 24: wice.Watcher.SyncInterfaceConfig:output_type -> wice.Error
+	14, // 24: wice.Watcher.SyncInterfaceConfig:output_type -> wice.Empty
-	16, // 25: wice.Watcher.AddInterfaceConfig:output_type -> wice.Error
+	14, // 25: wice.Watcher.AddInterfaceConfig:output_type -> wice.Empty
-	16, // 26: wice.Watcher.SetInterfaceConfig:output_type -> wice.Error
+	14, // 26: wice.Watcher.SetInterfaceConfig:output_type -> wice.Empty
 	9,  // 27: wice.Watcher.GetSignalingMessage:output_type -> wice.GetSignalingMessageResp
-	16, // 28: wice.Watcher.PutSignalingMessage:output_type -> wice.Error
+	14, // 28: wice.Watcher.PutSignalingMessage:output_type -> wice.Empty
-	16, // 29: wice.EndpointDiscoverySocket.RestartPeer:output_type -> wice.Error
+	14, // 29: wice.EndpointDiscoverySocket.RestartPeer:output_type -> wice.Empty
 	9,  // 30: wice.SignalingSocket.GetSignalingMessage:output_type -> wice.GetSignalingMessageResp
-	16, // 31: wice.SignalingSocket.PutSignalingMessage:output_type -> wice.Error
+	14, // 31: wice.SignalingSocket.PutSignalingMessage:output_type -> wice.Empty
 	18, // [18:32] is the sub-list for method output_type
 	4,  // [4:18] is the sub-list for method input_type
 	4,  // [4:4] is the sub-list for extension type_name

pkg/pb/session.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.27.1
+// 	protoc-gen-go v1.28.1
-// 	protoc        v3.19.4
+// 	protoc        v3.6.1
 // source: session.proto
 
 package pb
@@ -86,6 +86,61 @@ func (x *Credentials) GetNeedCreds() bool {
 	return false
 }
 
+type PresharedKeyEstablishment struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	PublicKey  []byte `protobuf:"bytes,1,opt,name=public_key,json=publicKey,proto3" json:"public_key,omitempty"`
+	CipherText []byte `protobuf:"bytes,2,opt,name=cipher_text,json=cipherText,proto3" json:"cipher_text,omitempty"`
+}
+
+func (x *PresharedKeyEstablishment) Reset() {
+	*x = PresharedKeyEstablishment{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_session_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *PresharedKeyEstablishment) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*PresharedKeyEstablishment) ProtoMessage() {}
+
+func (x *PresharedKeyEstablishment) ProtoReflect() protoreflect.Message {
+	mi := &file_session_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use PresharedKeyEstablishment.ProtoReflect.Descriptor instead.
+func (*PresharedKeyEstablishment) Descriptor() ([]byte, []int) {
+	return file_session_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *PresharedKeyEstablishment) GetPublicKey() []byte {
+	if x != nil {
+		return x.PublicKey
+	}
+	return nil
+}
+
+func (x *PresharedKeyEstablishment) GetCipherText() []byte {
+	if x != nil {
+		return x.CipherText
+	}
+	return nil
+}
+
 var File_session_proto protoreflect.FileDescriptor
 
 var file_session_proto_rawDesc = []byte{
@@ -95,9 +150,15 @@ var file_session_proto_rawDesc = []byte{
 	0x01, 0x28, 0x09, 0x52, 0x05, 0x75, 0x66, 0x72, 0x61, 0x67, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x77,
 	0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x70, 0x77, 0x64, 0x12, 0x1d, 0x0a, 0x0a,
 	0x6e, 0x65, 0x65, 0x64, 0x5f, 0x63, 0x72, 0x65, 0x64, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08,
-	0x52, 0x09, 0x6e, 0x65, 0x65, 0x64, 0x43, 0x72, 0x65, 0x64, 0x73, 0x42, 0x16, 0x5a, 0x14, 0x72,
+	0x52, 0x09, 0x6e, 0x65, 0x65, 0x64, 0x43, 0x72, 0x65, 0x64, 0x73, 0x22, 0x5b, 0x0a, 0x19, 0x50,
-	0x69, 0x61, 0x73, 0x63, 0x2e, 0x65, 0x75, 0x2f, 0x77, 0x69, 0x63, 0x65, 0x2f, 0x70, 0x6b, 0x67,
+	0x72, 0x65, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x45, 0x73, 0x74, 0x61, 0x62,
-	0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x6c, 0x69, 0x73, 0x68, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x75, 0x62, 0x6c,
+	0x69, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x70, 0x75,
+	0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x69, 0x70, 0x68, 0x65,
+	0x72, 0x5f, 0x74, 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x63, 0x69,
+	0x70, 0x68, 0x65, 0x72, 0x54, 0x65, 0x78, 0x74, 0x42, 0x16, 0x5a, 0x14, 0x72, 0x69, 0x61, 0x73,
+	0x63, 0x2e, 0x65, 0x75, 0x2f, 0x77, 0x69, 0x63, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62,
+	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -112,9 +173,10 @@ func file_session_proto_rawDescGZIP() []byte {
 	return file_session_proto_rawDescData
 }
 
-var file_session_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
+var file_session_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
 var file_session_proto_goTypes = []interface{}{
-	(*Credentials)(nil), // 0: wice.Credentials
+	(*Credentials)(nil),               // 0: wice.Credentials
+	(*PresharedKeyEstablishment)(nil), // 1: wice.PresharedKeyEstablishment
 }
 var file_session_proto_depIdxs = []int32{
 	0, // [0:0] is the sub-list for method output_type
@@ -142,6 +204,18 @@ func file_session_proto_init() {
 				return nil
 			}
 		}
+		file_session_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*PresharedKeyEstablishment); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
 	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
@@ -149,7 +223,7 @@ func file_session_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_session_proto_rawDesc,
 			NumEnums:      0,
-			NumMessages:   1,
+			NumMessages:   2,
 			NumExtensions: 0,
 			NumServices:   0,
 		},

pkg/pb/session.proto

@@ -12,4 +12,9 @@ message Credentials {
 
     // Flag to indicate that the sending peer requests the credentials of the receiving peer
     bool need_creds = 3;
+}
+
+message PresharedKeyEstablishment {
+    bytes public_key = 1;
+    bytes cipher_text = 2;
 }

pkg/pb/signaling.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.27.1
+// 	protoc-gen-go v1.28.1
-// 	protoc        v3.19.4
+// 	protoc        v3.6.1
 // source: signaling.proto
 
 package pb
@@ -143,9 +143,10 @@ type SignalingMessage struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	Credentials *Credentials     `protobuf:"bytes,1,opt,name=credentials,proto3" json:"credentials,omitempty"`
+	Credentials *Credentials               `protobuf:"bytes,1,opt,name=credentials,proto3" json:"credentials,omitempty"`
-	Candidate   *Candidate       `protobuf:"bytes,2,opt,name=candidate,proto3" json:"candidate,omitempty"`
+	Candidate   *Candidate                 `protobuf:"bytes,2,opt,name=candidate,proto3" json:"candidate,omitempty"`
-	Peer        *PeerDescription `protobuf:"bytes,3,opt,name=peer,proto3" json:"peer,omitempty"`
+	Peer        *PeerDescription           `protobuf:"bytes,3,opt,name=peer,proto3" json:"peer,omitempty"`
+	Pskem       *PresharedKeyEstablishment `protobuf:"bytes,4,opt,name=pskem,proto3" json:"pskem,omitempty"`
 }
 
 func (x *SignalingMessage) Reset() {
@@ -201,6 +202,13 @@ func (x *SignalingMessage) GetPeer() *PeerDescription {
 	return nil
 }
 
+func (x *SignalingMessage) GetPskem() *PresharedKeyEstablishment {
+	if x != nil {
+		return x.Pskem
+	}
+	return nil
+}
+
 type SubscribeParams struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -265,7 +273,7 @@ var file_signaling_proto_rawDesc = []byte{
 	0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e,
 	0x77, 0x69, 0x63, 0x65, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65,
 	0x73, 0x73, 0x61, 0x67, 0x65, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x22,
-	0xa1, 0x01, 0x0a, 0x10, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73,
+	0xd8, 0x01, 0x0a, 0x10, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4d, 0x65, 0x73,
 	0x73, 0x61, 0x67, 0x65, 0x12, 0x33, 0x0a, 0x0b, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69,
 	0x61, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x77, 0x69, 0x63, 0x65,
 	0x2e, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x52, 0x0b, 0x63, 0x72,
@@ -275,23 +283,27 @@ var file_signaling_proto_rawDesc = []byte{
 	0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x12, 0x29, 0x0a, 0x04, 0x70, 0x65, 0x65, 0x72,
 	0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x50, 0x65,
 	0x65, 0x72, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x04, 0x70,
-	0x65, 0x65, 0x72, 0x22, 0x23, 0x0a, 0x0f, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65,
+	0x65, 0x65, 0x72, 0x12, 0x35, 0x0a, 0x05, 0x70, 0x73, 0x6b, 0x65, 0x6d, 0x18, 0x04, 0x20, 0x01,
-	0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20,
+	0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x50, 0x72, 0x65, 0x73, 0x68, 0x61,
-	0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x2a, 0x43, 0x0a, 0x0b, 0x42, 0x61, 0x63, 0x6b,
+	0x72, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x45, 0x73, 0x74, 0x61, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x6d,
-	0x65, 0x6e, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, 0x0a, 0x05, 0x4d, 0x55, 0x4c, 0x54, 0x49,
+	0x65, 0x6e, 0x74, 0x52, 0x05, 0x70, 0x73, 0x6b, 0x65, 0x6d, 0x22, 0x23, 0x0a, 0x0f, 0x53, 0x75,
-	0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x50, 0x32, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x4b,
+	0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x10, 0x0a,
-	0x38, 0x53, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x47, 0x52, 0x50, 0x43, 0x10, 0x03, 0x12, 0x0d,
+	0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x2a,
-	0x0a, 0x09, 0x49, 0x4e, 0x50, 0x52, 0x4f, 0x43, 0x45, 0x53, 0x53, 0x10, 0x04, 0x32, 0x7f, 0x0a,
+	0x43, 0x0a, 0x0b, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09,
-	0x09, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x12, 0x3f, 0x0a, 0x09, 0x53, 0x75,
+	0x0a, 0x05, 0x4d, 0x55, 0x4c, 0x54, 0x49, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x50, 0x32, 0x50,
-	0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x12, 0x15, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x53,
+	0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x4b, 0x38, 0x53, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x47,
-	0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x17,
+	0x52, 0x50, 0x43, 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x49, 0x4e, 0x50, 0x52, 0x4f, 0x43, 0x45,
-	0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x45,
+	0x53, 0x53, 0x10, 0x04, 0x32, 0x7f, 0x0a, 0x09, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e,
-	0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x31, 0x0a, 0x07, 0x50,
+	0x67, 0x12, 0x3f, 0x0a, 0x09, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x12, 0x15,
-	0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x12, 0x17, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x53, 0x69,
+	0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x50,
-	0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x45, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x1a,
+	0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x17, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x53, 0x69, 0x67,
-	0x0b, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x00, 0x42, 0x16,
+	0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x45, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x22, 0x00,
-	0x5a, 0x14, 0x72, 0x69, 0x61, 0x73, 0x63, 0x2e, 0x65, 0x75, 0x2f, 0x77, 0x69, 0x63, 0x65, 0x2f,
+	0x30, 0x01, 0x12, 0x31, 0x0a, 0x07, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x12, 0x17, 0x2e,
-	0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x77, 0x69, 0x63, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x45, 0x6e,
+	0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x1a, 0x0b, 0x2e, 0x77, 0x69, 0x63, 0x65, 0x2e, 0x45, 0x6d,
+	0x70, 0x74, 0x79, 0x22, 0x00, 0x42, 0x16, 0x5a, 0x14, 0x72, 0x69, 0x61, 0x73, 0x63, 0x2e, 0x65,
+	0x75, 0x2f, 0x77, 0x69, 0x63, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -309,30 +321,32 @@ func file_signaling_proto_rawDescGZIP() []byte {
 var file_signaling_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
 var file_signaling_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
 var file_signaling_proto_goTypes = []interface{}{
-	(BackendType)(0),          // 0: wice.BackendType
+	(BackendType)(0),                  // 0: wice.BackendType
-	(*SignalingEnvelope)(nil), // 1: wice.SignalingEnvelope
+	(*SignalingEnvelope)(nil),         // 1: wice.SignalingEnvelope
-	(*SignalingMessage)(nil),  // 2: wice.SignalingMessage
+	(*SignalingMessage)(nil),          // 2: wice.SignalingMessage
-	(*SubscribeParams)(nil),   // 3: wice.SubscribeParams
+	(*SubscribeParams)(nil),           // 3: wice.SubscribeParams
-	(*EncryptedMessage)(nil),  // 4: wice.EncryptedMessage
+	(*EncryptedMessage)(nil),          // 4: wice.EncryptedMessage
-	(*Credentials)(nil),       // 5: wice.Credentials
+	(*Credentials)(nil),               // 5: wice.Credentials
-	(*Candidate)(nil),         // 6: wice.Candidate
+	(*Candidate)(nil),                 // 6: wice.Candidate
-	(*PeerDescription)(nil),   // 7: wice.PeerDescription
+	(*PeerDescription)(nil),           // 7: wice.PeerDescription
-	(*Error)(nil),             // 8: wice.Error
+	(*PresharedKeyEstablishment)(nil), // 8: wice.PresharedKeyEstablishment
+	(*Empty)(nil),                     // 9: wice.Empty
 }
 var file_signaling_proto_depIdxs = []int32{
 	4, // 0: wice.SignalingEnvelope.contents:type_name -> wice.EncryptedMessage
 	5, // 1: wice.SignalingMessage.credentials:type_name -> wice.Credentials
 	6, // 2: wice.SignalingMessage.candidate:type_name -> wice.Candidate
 	7, // 3: wice.SignalingMessage.peer:type_name -> wice.PeerDescription
-	3, // 4: wice.Signaling.Subscribe:input_type -> wice.SubscribeParams
+	8, // 4: wice.SignalingMessage.pskem:type_name -> wice.PresharedKeyEstablishment
-	1, // 5: wice.Signaling.Publish:input_type -> wice.SignalingEnvelope
+	3, // 5: wice.Signaling.Subscribe:input_type -> wice.SubscribeParams
-	1, // 6: wice.Signaling.Subscribe:output_type -> wice.SignalingEnvelope
+	1, // 6: wice.Signaling.Publish:input_type -> wice.SignalingEnvelope
-	8, // 7: wice.Signaling.Publish:output_type -> wice.Error
+	1, // 7: wice.Signaling.Subscribe:output_type -> wice.SignalingEnvelope
-	6, // [6:8] is the sub-list for method output_type
+	9, // 8: wice.Signaling.Publish:output_type -> wice.Empty
-	4, // [4:6] is the sub-list for method input_type
+	7, // [7:9] is the sub-list for method output_type
-	4, // [4:4] is the sub-list for extension type_name
+	5, // [5:7] is the sub-list for method input_type
-	4, // [4:4] is the sub-list for extension extendee
+	5, // [5:5] is the sub-list for extension type_name
-	0, // [0:4] is the sub-list for field type_name
+	5, // [5:5] is the sub-list for extension extendee
+	0, // [0:5] is the sub-list for field type_name
 }
 
 func init() { file_signaling_proto_init() }

pkg/pb/signaling.proto

@@ -28,6 +28,7 @@ message SignalingMessage {
     Credentials credentials = 1;
     Candidate candidate = 2;
     PeerDescription peer = 3;
+    PresharedKeyEstablishment pskem = 4;
 }
 
 message SubscribeParams {

pkg/rpc/client.go

@@ -208,3 +208,12 @@ func (c *Client) WaitForPeerConnectionState(ctx context.Context, peer crypto.Key
 
 	return ctx.Err()
 }
+
+func (c *Client) RestartPeer(ctx context.Context, intf string, pk *crypto.Key) error {
+	_, err := c.EndpointDiscoverySocketClient.RestartPeer(ctx, &pb.RestartPeerParams{
+		Intf: intf,
+		Peer: pk.Bytes(),
+	})
+
+	return err
+}

pkg/signaling/grpc/backend.go

@@ -3,7 +3,6 @@ package grpc
 import (
 	"context"
 	"fmt"
-	"io"
 
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
@@ -103,13 +102,11 @@ func (b *Backend) Close() error {
 }
 
 func (b *Backend) subscribeFromServer(ctx context.Context, pk *crypto.Key) error {
-	b.logger.Debug("Creating new subscription", zap.Any("pk", pk))
-
 	params := &pb.SubscribeParams{
 		Key: pk.Bytes(),
 	}
 
-	stream, err := b.client.Subscribe(ctx, params)
+	stream, err := b.client.Subscribe(ctx, params, grpc.WaitForReady(true))
 	if err != nil {
 		return fmt.Errorf("failed to subscribe to offers: %s", err)
 	}
@@ -118,22 +115,25 @@ func (b *Backend) subscribeFromServer(ctx context.Context, pk *crypto.Key) error
 	// This avoids a race between Subscribe() / Publish() when two subscribers are subscribing
 	// to each other.
 	if _, err := stream.Recv(); err != nil {
-		return fmt.Errorf("failed receive sync envelope: %s", err)
+		return fmt.Errorf("failed receive synchronization envelope: %s", err)
 	}
 
+	b.logger.Debug("Created new subscription", zap.Any("pk", pk))
+
 	go func() {
 		for {
-			if env, err := stream.Recv(); err == nil {
+			if env, err := stream.Recv(); err != nil {
-				if err := b.SubscriptionsRegistry.NewMessage(env); err != nil {
+				b.logger.Error("Subscription stream closed. Re-subscribing..", zap.Error(err))
-					b.logger.Error("Failed to decrypt message", zap.Error(err))
+
-				}
+				if err := b.subscribeFromServer(ctx, pk); err != nil {
-			} else {
+					b.logger.Error("Failed to resubscribe", zap.Error(err))
-				if err != io.EOF {
-					// TODO: Attempt reconnect?
-					b.logger.Error("Received error", zap.Error(err))
 				}
 
 				return
+			} else {
+				if err := b.SubscriptionsRegistry.NewMessage(env); err != nil {
+					b.logger.Error("Failed to decrypt message", zap.Error(err))
+				}
 			}
 		}
 	}()

pkg/signaling/grpc/server.go

@@ -76,7 +76,11 @@ func (s *Server) Subscribe(params *pb.SubscribeParams, stream pb.Signaling_Subsc
 out:
 	for {
 		select {
-		case env := <-ch:
+		case env, ok := <-ch:
+			if !ok {
+				break out
+			}
+
 			if err := stream.Send(env); err == io.EOF {
 				break out
 			} else if err != nil {
@@ -114,9 +118,7 @@ func (s *Server) Publish(ctx context.Context, env *signaling.Envelope) (*pb.Empt
 	return &pb.Empty{}, nil
 }
 
-func (s *Server) GracefulStop() {
+func (s *Server) Close() {
-	// Close all subscription streams
 	s.topicRegistry.Close()
-
 	s.Server.GracefulStop()
 }

pkg/util/util.go

@@ -2,6 +2,7 @@ package util
 
 import (
 	"bytes"
+	"encoding/binary"
 	"fmt"
 	mrand "math/rand"
 	"net"
@@ -9,6 +10,8 @@ import (
 	"os/signal"
 	"syscall"
 	"time"
+
+	"golang.org/x/exp/slices"
 )
 
 func CmpEndpoint(a, b *net.UDPAddr) int {
@@ -55,3 +58,17 @@ func SetupSignals(extraSignals ...os.Signal) chan os.Signal {
 
 	return ch
 }
+
+func OffsetIP(ip net.IP, off int) net.IP {
+	oip := slices.Clone(ip)
+
+	if isV6 := ip.To4() == nil; isV6 {
+		num := binary.BigEndian.Uint64(ip[8:])
+		binary.BigEndian.PutUint64(oip[8:], num+uint64(off))
+	} else {
+		num := binary.BigEndian.Uint32(ip[12:])
+		binary.BigEndian.PutUint32(oip[12:], num+uint32(off))
+	}
+
+	return oip
+}

pkg/wg/device_config.go

@@ -156,6 +156,7 @@ func (cfg *Config) Dump(wr io.Writer) error {
 
 	iniFile := ini.Empty(ini.LoadOptions{
 		AllowNonUniqueSections: true,
+		AllowShadows:           true,
 	})
 
 	if err := iniFile.ReflectFrom(iniCfg); err != nil {

test/nat_double_test.go

@@ -3,6 +3,8 @@
 package test_test
 
 import (
+	"fmt"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	g "github.com/stv0g/gont/pkg"
@@ -14,35 +16,41 @@ import (
 /* Carrier Grade NAT setup with two relays and a single signaling server
  *
  * Hosts:
- *  - 2x Relay nodes (Coturn STUN/TURN server)
+ *  - 1x Signaling node     [s1]    (GRPC server)
- *  - 1x Signaling node (GRPC server)
+ *  - 2x Relay nodes        [nat?]  (Coturn STUN/TURN server)
- *  - 3x NAT routers
+ *  - 3x NAT routers        [nat?]
- *  - 2x WAN switches
+ *  - 2x WAN switches       [wan?]
- *  - 2x LAN switches
+ *  - 2x LAN switches       [lan?]
- *  - 2x wice Agent nodes
+ *  - 2-5x wice Agent nodes [n?]
  *
- *             ┌──────┐
+ *
- *             │  s1  │            Signaling
+ *             ┌──────┐   ┌──────┐   ┌──────┐
- *             └──┬───┘
+ *             │  r1  │   │  s1  │   │  r2  │
- *  ┌──────┐      │       ┌──────┐
+ *             └──┬───┘   └──┬───┘   └───┬──┘
- *  │  r1  │      │       │  r2  │ Relays
+ *                │ ┌────────┘           │
- *  └──┬───┘      │       └──┬───┘
+ *  ┌──────┐   ┌──┴─┴─┐   ┌──────┐   ┌───┴──┐   ┌──────┐
- *  ┌──┴───┐      │       ┌──┴───┐
+ *  │ (n5) ├───┤ wan1 ├───┤ nat3 ├───┤ wan2 ├───┤ (n4) │
- *  │ sw1  ├──────┘       │  sw2 │ WAN Switches
+ *  └──────┘   └──┬───┘   └──────┘   └───┬──┘   └──────┘
- *  └──┬─┬─┘              └─┬─┬──┘
+ *             ┌──┴───┐              ┌───┴──┐
- *     │ └───┐          ┌───┘ │
+ *             │ nat1 │              │ nat2 │
- *  ┌──┴───┐ │ ┌──────┐ │ ┌───┴──┐
+ *             └──┬───┘              └───┬──┘
- *  │ nat1 │ └─┤ nat3 ├─┘ │ nat2 │ NAT Routers
+ *             ┌──┴───┐              ┌───┴──┐   ┌──────┐
- *  └──┬───┘   └──────┘   └───┬──┘
+ *             │ lan1 │              │ lan2 ├───┤ (n3) │
- *  ┌──┴───┐              ┌───┴──┐
+ *             └──┬───┘              └───┬──┘   └──────┘
- *  │ lsw1 │              │ lsw2 │ LAN Switches
+ *             ┌──┴───┐              ┌───┴──┐
- *  └──┬───┘              └───┬──┘
+ *             │  n1  │              │  n2  │
- *  ┌──┴───┐              ┌───┴──┐
+ *             └──────┘              └──────┘
- *  │  n1  │              │  n2  │ wice Agents
- *  └──────┘              └──────┘
  */
-var _ = Context("nat double", Serial, func() {
+var _ = Context("nat double", func() {
-	var n Network
+	var (
+		err error
+
+		n Network
+
+		nw         *g.Network
+		wan1, wan2 *g.Switch
+		lan1, lan2 *g.Switch
+	)
 
 	BeforeEach(func() {
 		n.Init()
@@ -52,28 +60,116 @@ var _ = Context("nat double", Serial, func() {
 		n.Close()
 	})
 
+	AddAgent := func(i int, sw *g.Switch) *nodes.Agent {
+		ifOpts := []g.Option{sw}
+
+		switch {
+		case i <= 3: // lan1, lan2
+			ifOpts = append(ifOpts,
+				gopt.AddressIP("10.1.0.%d/24", i),
+				gopt.AddressIP("fc:1::%d/64", i),
+			)
+		case i == 4: // wan2
+			ifOpts = append(ifOpts,
+				gopt.AddressIP("10.11.0.5/24"),
+				gopt.AddressIP("fc:11::5/64"),
+			)
+		case i == 5: // wan1
+			ifOpts = append(ifOpts,
+				gopt.AddressIP("10.10.0.5/24"),
+				gopt.AddressIP("fc:10::5/64"),
+			)
+		}
+
+		opts := gopt.Customize(n.AgentOptions,
+			gopt.Interface("eth0", ifOpts...),
+			wopt.Interface("wg0",
+				wopt.AddressIP("172.16.0.%d/16", i),
+				wopt.FullMeshPeers,
+			),
+		)
+
+		switch {
+		case i <= 3: // lan1, lan2
+			opts = append(opts,
+				gopt.DefaultGatewayIP("10.1.0.254"),
+				gopt.DefaultGatewayIP("fc:1::ff"),
+			)
+		case i == 4: // wan2
+			opts = append(opts,
+				gopt.DefaultGatewayIP("10.11.0.4"),
+				gopt.DefaultGatewayIP("fc:11::4"),
+			)
+		}
+
+		a, err := nodes.NewAgent(nw, fmt.Sprintf("n%d", i), opts...)
+		Expect(err).To(Succeed(), "Failed to created nodes: %s", err)
+
+		n.AgentNodes = append(n.AgentNodes, a)
+
+		return a
+	}
+
+	AddLAN := func(i int, sw *g.Switch) *g.Switch {
+		// LAN Switch
+		lan, err := nw.AddSwitch(fmt.Sprintf("lan%d", i))
+		Expect(err).To(Succeed(), "Failed to create switch: %s", err)
+
+		nbifOpts := []g.Option{sw, gopt.NorthBound}
+
+		switch {
+		case i == 1: // wan1
+			nbifOpts = append(nbifOpts,
+				gopt.AddressIP("10.10.0.3/16"),
+				gopt.AddressIP("fc:10::3/64"),
+			)
+		case i == 2: // wan2
+			nbifOpts = append(nbifOpts,
+				gopt.AddressIP("10.11.0.3/16"),
+				gopt.AddressIP("fc:11::3/64"),
+			)
+		}
+
+		opts := []g.Option{
+			gopt.Interface("eth-nb", nbifOpts...),
+			gopt.Interface("eth-sb", lan,
+				gopt.SouthBound,
+				gopt.AddressIP("10.1.0.254/24"),
+				gopt.AddressIP("fc:1::ff/64"),
+			),
+		}
+
+		switch {
+		case i == 2: // wan2
+			opts = append(opts,
+				gopt.DefaultGatewayIP("10.11.0.4"),
+				gopt.DefaultGatewayIP("fc:11::4"),
+			)
+		}
+
+		// NAT router
+		_, err = nw.AddNAT(fmt.Sprintf("nat%d", i), opts...)
+		Expect(err).To(Succeed(), "Failed to add NAT node: %s", err)
+
+		return lan
+	}
+
 	JustBeforeEach(func() {
 		By("Initializing core network")
 
-		nw, err := g.NewNetwork(n.Name, n.NetworkOptions...)
+		nw, err = g.NewNetwork(n.Name, n.NetworkOptions...)
 		Expect(err).To(Succeed(), "Failed to create network: %s", err)
 
-		sw1, err := nw.AddSwitch("sw1")
+		wan1, err = nw.AddSwitch("wan1")
 		Expect(err).To(Succeed(), "Failed to create switch: %s", err)
 
-		sw2, err := nw.AddSwitch("sw2")
+		wan2, err = nw.AddSwitch("wan2")
-		Expect(err).To(Succeed(), "Failed to create switch: %s", err)
-
-		lsw1, err := nw.AddSwitch("lws1")
-		Expect(err).To(Succeed(), "Failed to create switch: %s", err)
-
-		lsw2, err := nw.AddSwitch("lws2")
 		Expect(err).To(Succeed(), "Failed to create switch: %s", err)
 
 		By("Initializing relay node")
 
 		r1, err := nodes.NewCoturnNode(nw, "r1",
-			gopt.Interface("eth0", sw1,
+			gopt.Interface("eth0", wan1,
 				gopt.AddressIP("10.10.0.1/16"),
 				gopt.AddressIP("fc:10::1/64"),
 			),
@@ -81,7 +177,7 @@ var _ = Context("nat double", Serial, func() {
 		Expect(err).To(Succeed(), "Failed to start relay: %s", err)
 
 		r2, err := nodes.NewCoturnNode(nw, "r2",
-			gopt.Interface("eth0", sw2,
+			gopt.Interface("eth0", wan2,
 				gopt.AddressIP("10.11.0.1/16"),
 				gopt.AddressIP("fc:11::1/64"),
 			),
@@ -91,7 +187,7 @@ var _ = Context("nat double", Serial, func() {
 		By("Initializing signaling node")
 
 		s1, err := nodes.NewGrpcSignalingNode(nw, "s1",
-			gopt.Interface("eth0", sw1,
+			gopt.Interface("eth0", wan1,
 				gopt.AddressIP("10.10.0.2/16"),
 				gopt.AddressIP("fc:10::2/64"),
 			),
@@ -101,12 +197,12 @@ var _ = Context("nat double", Serial, func() {
 		By("Initializing CGNAT node")
 
 		_, err = nw.AddNAT("nat3",
-			gopt.Interface("eth-nb", sw1,
+			gopt.Interface("eth-nb", wan1,
 				gopt.NorthBound,
 				gopt.AddressIP("10.10.0.4/16"),
 				gopt.AddressIP("fc:10::4/64"),
 			),
-			gopt.Interface("eth-sb", sw2,
+			gopt.Interface("eth-sb", wan2,
 				gopt.SouthBound,
 				gopt.AddressIP("10.11.0.4/24"),
 				gopt.AddressIP("fc:11::4/64"),
@@ -116,79 +212,55 @@ var _ = Context("nat double", Serial, func() {
 
 		By("Initializing agent nodes")
 
-		/// Left: n1
+		lan1 = AddLAN(1, wan1)
+		lan2 = AddLAN(2, wan2)
 
-		opts := gopt.Customize(n.AgentOptions,
+		AddAgent(1, lan1)
-			gopt.Interface("eth0", lsw1,
+		AddAgent(2, lan2)
-				gopt.AddressIP("10.1.0.2/24"),
-				gopt.AddressIP("fc:1::2/64"),
-			),
-			gopt.DefaultGatewayIP("10.1.0.1"),
-			gopt.DefaultGatewayIP("fc:1::1"),
-			wopt.Interface("wg0",
-				wopt.AddressIP("172.16.0.1/16"),
-				wopt.FullMeshPeers,
-			),
-		)
-
-		n1, err := nodes.NewAgent(nw, "n1", opts...)
-		Expect(err).To(Succeed(), "ailed to created nodes: %s", err)
-
-		_, err = nw.AddNAT("nat1",
-			gopt.Interface("eth-nb", sw1,
-				gopt.NorthBound,
-				gopt.AddressIP("10.10.0.3/16"),
-				gopt.AddressIP("fc:10::3/64"),
-			),
-			gopt.Interface("eth-sb", lsw1,
-				gopt.SouthBound,
-				gopt.AddressIP("10.1.0.1/24"),
-				gopt.AddressIP("fc:1::1/64"),
-			),
-		)
-		Expect(err).To(Succeed(), "Failed to add NAT node: %s", err)
-
-		/// Right: n2
-
-		opts = gopt.Customize(n.AgentOptions,
-			gopt.Interface("eth0", lsw2,
-				gopt.AddressIP("10.1.0.2/24"),
-				gopt.AddressIP("fc:1::2/64"),
-			),
-			gopt.DefaultGatewayIP("10.1.0.1"),
-			gopt.DefaultGatewayIP("fc:1::1"),
-			wopt.Interface("wg0",
-				wopt.AddressIP("172.16.0.2/16"),
-				wopt.FullMeshPeers,
-			),
-		)
-
-		n2, err := nodes.NewAgent(nw, "n2", opts...)
-		Expect(err).To(Succeed(), "Failed to create nodes: %s", err)
-
-		_, err = nw.AddNAT("nat2",
-			gopt.DefaultGatewayIP("10.11.0.4"),
-			gopt.DefaultGatewayIP("fc:11::4"),
-			gopt.Interface("eth-nb", sw2,
-				gopt.NorthBound,
-				gopt.AddressIP("10.11.0.3/16"),
-				gopt.AddressIP("fc:11::3/64"),
-			),
-			gopt.Interface("eth-sb", lsw2,
-				gopt.SouthBound,
-				gopt.AddressIP("10.1.0.1/24"),
-				gopt.AddressIP("fc:1::1/64"),
-			),
-		)
-		Expect(err).To(Succeed(), "Failed to add NAT node: %s", err)
 
 		n.Network = nw
-		n.AgentNodes = nodes.AgentList{n1, n2}
 		n.RelayNodes = nodes.RelayList{r1, r2}
 		n.SignalingNodes = nodes.SignalingList{s1}
-
-		n.Start()
 	})
 
-	n.ConnectivityTests()
+	Context("2-nodes", func() {
+		JustBeforeEach(func() {
+			n.Start()
+		})
+
+		n.ConnectivityTests()
+	})
+
+	Context("3-nodes", func() {
+		JustBeforeEach(func() {
+			AddAgent(3, lan2)
+
+			n.Start()
+		})
+
+		n.ConnectivityTests()
+	})
+
+	Context("4-nodes", func() {
+		JustBeforeEach(func() {
+			AddAgent(3, lan2)
+			AddAgent(4, wan2)
+
+			n.Start()
+		})
+
+		n.ConnectivityTests()
+	})
+
+	Context("5-nodes", func() {
+		JustBeforeEach(func() {
+			AddAgent(3, lan2)
+			AddAgent(4, wan2)
+			AddAgent(5, wan1)
+
+			n.Start()
+		})
+
+		n.ConnectivityTests()
+	})
 })

test/nat_test.go

@@ -16,35 +16,40 @@ import (
 /* Typical wide-area NAT setup
  *
  * Hosts:
- *  - 1x Relay node (Coturn STUN/TURN server)
+ *  - 1x Relay node       [r1] (Coturn STUN/TURN server)
- *  - 1x Signaling node (GRPC server)
+ *  - 1x Signaling node   [s1] (GRPC server)
- *  - 2x NAT routers
+ *  - 2x NAT routers      [nat?]
- *  - 1x WAN switch
+ *  - 1x WAN switch       [wan?]
- *  - 2x LAN switches
+ *  - 2x LAN switches     [lan?]
- *  - 2x wice Agent nodes
+ *  - 2x wice Agent nodes [n?]
  *
- *                    ┌────┐   ┌────┐
+ *     ┌──────┐   ┌──────┐
- *              Relay │ r1 │   │ s1 │ Signaling
+ *     │  r1  │   │  s1  │
- *                    └──┬─┘   └─┬──┘
+ *     └───┬──┘   └──┬───┘
- *                       └─┐   ┌─┘
+ *         └───┐  ┌──┘
- *                        ┌┴───┴┐
+ *           ┌─┴──┴─┐
- *                        │ sw1 │ WAN Switch
+ *           │ wan1 │ WAN Switch
- *                        └┬───┬┘
+ *           └┬────┬┘
- *                   ┌─────┘   └─────┐
+ *      ┌─────┘    └────┐
- *               ┌───┴──┐        ┌───┴──┐
+ *  ┌───┴──┐        ┌───┴──┐
- *               │ nat1 │        │ nat2 │ NAT Routers
+ *  │ nat1 │        │ nat2 │
- *               └───┬──┘        └───┬──┘
+ *  └───┬──┘        └───┬──┘
- *               ┌───┴──┐        ┌───┴──┐
+ *  ┌───┴──┐        ┌───┴──┐
- *  LAN Switches │ lsw1 │        │ lsw2 │
+ *  │ lan1 │        │ lan2 │
- *               └───┬──┘        └─┬──┬─┘
+ *  └───┬──┘        └─┬──┬─┘
- *                   │           ┌─┘  └─┐
+ *      │           ┌─┘  └─┐
- *               ┌───┴──┐   ┌────┴─┐  ┌─┴────┐
+ *  ┌───┴──┐   ┌────┴─┐  ┌─┴────┐
- *               │  n1  │   │  n2  │  │ (n3) │  wice Agents
+ *  │  n1  │   │  n2  │  │ (n3) │
- *               └──────┘   └──────┘  └──────┘
+ *  └──────┘   └──────┘  └──────┘
  */
-var _ = Context("nat simple", Serial, func() {
+var _ = Context("nat simple", func() {
-	var n Network
+	var (
-	var lsw2 *g.Switch
+		err error
+
+		n    Network
+		nw   *g.Network
+		lan2 *g.Switch
+	)
 
 	BeforeEach(func() {
 		n.Init()
@@ -54,19 +59,39 @@ var _ = Context("nat simple", Serial, func() {
 		n.Close()
 	})
 
+	AddAgent := func(i int, lan *g.Switch) *nodes.Agent {
+		a, err := nodes.NewAgent(nw, fmt.Sprintf("n%d", i),
+			gopt.DefaultGatewayIP("10.1.0.254"),
+			gopt.DefaultGatewayIP("fc:1::254"),
+			gopt.Interface("eth0", lan,
+				gopt.AddressIP("10.1.0.%d/24", i),
+				gopt.AddressIP("fc:1::%d/64", i),
+			),
+			wopt.Interface("wg0",
+				wopt.FullMeshPeers,
+				wopt.AddressIP("172.16.0.%d/16", i),
+			),
+		)
+		Expect(err).To(Succeed(), "Failed to created nodes: %s", err)
+
+		n.AgentNodes = append(n.AgentNodes, a)
+
+		return a
+	}
+
 	JustBeforeEach(func() {
 		By("Initializing core network")
+		nw, err = g.NewNetwork(n.Name, n.NetworkOptions...)
 
-		nw, err := g.NewNetwork(n.Name, n.NetworkOptions...)
 		Expect(err).To(Succeed(), "Failed to create network: %s", err)
 
-		sw1, err := nw.AddSwitch("sw1")
+		wan1, err := nw.AddSwitch("wan1")
 		Expect(err).To(Succeed(), "Failed to create switch: %s", err)
 
 		By("Initializing relay node")
 
 		r1, err := nodes.NewCoturnNode(nw, "r1",
-			gopt.Interface("eth0", sw1,
+			gopt.Interface("eth0", wan1,
 				gopt.AddressIP("10.0.0.1/16"),
 				gopt.AddressIP("fc::1/64"),
 			),
@@ -76,7 +101,7 @@ var _ = Context("nat simple", Serial, func() {
 		By("Initializing signaling node")
 
 		s1, err := nodes.NewGrpcSignalingNode(nw, "s1",
-			gopt.Interface("eth0", sw1,
+			gopt.Interface("eth0", wan1,
 				gopt.AddressIP("10.0.0.2/16"),
 				gopt.AddressIP("fc::2/64"),
 			),
@@ -85,60 +110,40 @@ var _ = Context("nat simple", Serial, func() {
 
 		By("Initializing agent nodes")
 
-		CreateAgent := func(i int) *nodes.Agent {
+		AddLAN := func(i int) *g.Switch {
 			// LAN switch
-			lsw, err := nw.AddSwitch(fmt.Sprintf("lsw%d", i))
+			lan, err := nw.AddSwitch(fmt.Sprintf("lan%d", i))
 			Expect(err).To(Succeed(), "Failed to add LAN switch: %s", err)
 
-			if i == 2 {
-				lsw2 = lsw
-			}
-
 			// NAT router
 			_, err = nw.AddNAT(fmt.Sprintf("nat%d", i),
-				gopt.Interface("eth-nb", sw1,
+				gopt.Interface("eth-nb", wan1,
 					gopt.NorthBound,
 					gopt.AddressIP("10.0.1.%d/16", i),
 					gopt.AddressIP("fc::1:%d/64", i),
 				),
-				gopt.Interface("eth-sb", lsw,
+				gopt.Interface("eth-sb", lan,
 					gopt.SouthBound,
-					gopt.AddressIP("10.1.0.1/24"),
+					gopt.AddressIP("10.1.0.254/24"),
-					gopt.AddressIP("fc:1::1/64"),
+					gopt.AddressIP("fc:1::254/64"),
 				),
 			)
 			Expect(err).To(Succeed(), "Failed to created nodes: %s", err)
 
-			// Agent node
+			AddAgent(i, lan)
-			n, err := nodes.NewAgent(nw, fmt.Sprintf("n%d", i),
-				gopt.Customize(n.AgentOptions,
-					gopt.Interface("eth0", lsw,
-						gopt.AddressIP("10.1.0.2/24"),
-						gopt.AddressIP("fc:1::2/64"),
-					),
-					gopt.DefaultGatewayIP("10.1.0.1"),
-					gopt.DefaultGatewayIP("fc:1::1"),
-					wopt.Interface("wg0",
-						wopt.AddressIP("172.16.0.%d/16", i),
-						wopt.FullMeshPeers,
-					),
-				)...,
-			)
-			Expect(err).To(Succeed(), "Failed to created nodes: %s", err)
 
-			return n
+			return lan
 		}
 
+		AddLAN(1)
+		lan2 = AddLAN(2)
+
 		n.Network = nw
-		n.AgentNodes = nodes.AgentList{
-			CreateAgent(1),
-			CreateAgent(2),
-		}
 		n.RelayNodes = nodes.RelayList{r1}
 		n.SignalingNodes = nodes.SignalingList{s1}
 	})
 
-	Context("without-n3", func() {
+	Context("2-nodes", func() {
 		JustBeforeEach(func() {
 			n.Start()
 		})
@@ -146,25 +151,9 @@ var _ = Context("nat simple", Serial, func() {
 		n.ConnectivityTests()
 	})
 
-	FContext("with-n3", func() {
+	Context("3-nodes", func() {
 		JustBeforeEach(func() {
-			n3, err := nodes.NewAgent(n.Network, "n3",
+			AddAgent(3, lan2)
-				gopt.Customize(n.AgentOptions,
-					gopt.Interface("eth0", lsw2,
-						gopt.AddressIP("10.1.0.3/24"),
-						gopt.AddressIP("fc:1::3/64"),
-					),
-					gopt.DefaultGatewayIP("10.1.0.1"),
-					gopt.DefaultGatewayIP("fc:1::1"),
-					wopt.Interface("wg0",
-						wopt.AddressIP("172.16.0.3/16"),
-						wopt.FullMeshPeers,
-					),
-				)...,
-			)
-			Expect(err).To(Succeed(), "Failed to created node: %s", err)
-
-			n.AgentNodes = append(n.AgentNodes, n3)
 
 			n.Start()
 		})

test/network_others_test.go

@@ -1,4 +1,4 @@
-//go:build !tracer
+//go:build linux && !tracer
 
 package test_test
 

test/network_test.go

@@ -23,6 +23,10 @@ import (
 	"riasc.eu/wice/test/nodes"
 )
 
+var (
+	logger *zap.Logger
+)
+
 type Network struct {
 	*g.Network
 
@@ -38,7 +42,7 @@ type Network struct {
 	RelayNodes     nodes.RelayList
 	AgentNodes     nodes.AgentList
 
-	tracer *HandshakeTracer
+	Tracer *HandshakeTracer
 }
 
 func (n *Network) Start() {
@@ -63,7 +67,7 @@ func (n *Network) Start() {
 		Skip("Aborting test as only network setup has been requested")
 	}
 
-	if capture {
+	if len(n.Captures) > 0 && n.Captures[0].LogKeys {
 		n.StartHandshakeTracer()
 	}
 
@@ -103,6 +107,13 @@ func (n *Network) Start() {
 	err = n.SignalingNodes.Start(n.BasePath)
 	Expect(err).To(Succeed(), "Failed to start signaling node: %s", err)
 
+	By("Starting agent nodes")
+
+	err = n.AgentNodes.Start(n.BasePath, n.AgentArgs()...)
+	Expect(err).To(Succeed(), "Failed to start ɯice: %s", err)
+}
+
+func (n *Network) AgentArgs() []any {
 	extraArgs := []any{}
 
 	if len(n.RelayNodes) > 0 {
@@ -123,10 +134,7 @@ func (n *Network) Start() {
 		extraArgs = append(extraArgs, "--backend", s.URL())
 	}
 
-	By("Starting agent nodes")
+	return extraArgs
-
-	err = n.AgentNodes.Start(n.BasePath, extraArgs...)
-	Expect(err).To(Succeed(), "Failed to start ɯice: %s", err)
 }
 
 func (n *Network) Close() {
@@ -151,13 +159,15 @@ func (n *Network) Close() {
 	Expect(err).To(Succeed(), "Failed to close network; %s", err)
 
 	n.StopHandshakeTracer()
+
+	GinkgoWriter.ClearTeeWriters()
 }
 
 func (n *Network) ConnectivityTests() {
-	It("connectivity", func() {
+	It("", func() {
 		By("Waiting until all peers are connected")
 
-		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
 		defer cancel()
 
 		err := n.AgentNodes.WaitConnectionsReady(ctx)
@@ -207,7 +217,6 @@ func (n *Network) Init() {
 	// Ginkgo log
 	logger = test.SetupLoggingWithFile(logFilename, true)
 
-	// Agent logs
 	n.AgentOptions = append(n.AgentOptions,
 		gopt.LogToDebug(false),
 	)
@@ -220,13 +229,8 @@ func (n *Network) Init() {
 		n.NetworkOptions = append(n.NetworkOptions,
 			gopt.CaptureAll(
 				copt.Filename(pcapFilename),
+				copt.LogKeys(true),
 			),
 		)
 	}
-
-	logger.Info("Current test",
-		zap.String("name", name),
-		zap.String("path", n.BasePath),
-		zap.String("executed", time.Now().String()),
-	)
 }

test/network_tracer_test.go

@@ -1,4 +1,4 @@
-//go:build tracer
+//go:build linux && tracer
 
 package test_test
 
@@ -20,12 +20,12 @@ func (n *Network) StartHandshakeTracer() {
 	tracer, err := tracer.NewHandshakeTracer()
 	Expect(err).To(Succeed(), "Failed to setup WireGuard handshake tracer: %s", err)
 
-	n.tracer = (*HandshakeTracer)(tracer)
+	n.Tracer = (*HandshakeTracer)(tracer)
 
 	go func() {
 		for {
 			select {
-			case hs := <-n.tracer.Handshakes:
+			case hs := <-n.Tracer.Handshakes:
 				b := &bytes.Buffer{}
 				err = hs.DumpKeyLog(b)
 				Expect(err).To(Succeed(), "Failed to dump WireGuard handshake: %s", err)
@@ -35,7 +35,7 @@ func (n *Network) StartHandshakeTracer() {
 					Expect(err).To(Succeed(), "Failed to write decryption secrets to PCAPng file: %s", err)
 				}
 
-			case err := <-n.tracer.Errors:
+			case err := <-n.Tracer.Errors:
 				logger.Error("Failed to trace WireGuard handshake", zap.Error(err))
 			}
 		}
@@ -43,10 +43,10 @@ func (n *Network) StartHandshakeTracer() {
 }
 
 func (n *Network) StopHandshakeTracer() {
-	if n.tracer != nil {
+	if n.Tracer != nil {
 		By("Stopping WireGuard handshake tracer")
 
-		err := (*tracer.HandshakeTracer)(n.tracer).Close()
+		err := (*tracer.HandshakeTracer)(n.Tracer).Close()
 		Expect(err).To(Succeed(), "Failed to close WireGuard handshake tracer; %s", err)
 	}
 }

test/nodes/agent.go

@@ -88,6 +88,9 @@ func (a *Agent) Start(_, dir string, extraArgs ...any) error {
 	var rpcSockPath = fmt.Sprintf("/var/run/wice.%s.sock", a.Name())
 	var logPath = fmt.Sprintf("%s/%s.log", dir, a.Name())
 
+	// Old RPC sockets are also removed by wice.
+	// However we also need to do it here to avoid racing
+	// against rpc.Connect() further down here
 	if err := os.RemoveAll(rpcSockPath); err != nil {
 		return fmt.Errorf("failed to remove old socket: %w", err)
 	}

test/restart_test.go

@@ -0,0 +1,264 @@
+//go:build linux
+
+package test_test
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"riasc.eu/wice/pkg/crypto"
+	"riasc.eu/wice/pkg/util"
+	"riasc.eu/wice/pkg/wg"
+	"riasc.eu/wice/test/nodes"
+	wopt "riasc.eu/wice/test/nodes/options/wg"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	g "github.com/stv0g/gont/pkg"
+	gopt "github.com/stv0g/gont/pkg/options"
+)
+
+/* Simple local-area switched topology with 2 agents
+ *
+ *  - 1x Signaling node    [s1] (GRPC server)
+ *  - 1x Switch            [sw1]
+ *  - 2x  wice Agent nodes [n?]
+ *
+ *         Signaling
+ *          ┌─────┐
+ *          │  s1 │
+ *          └──┬──┘
+ *             │
+ *          ┌──┴──┐
+ *          │ sw1 │ Switch
+ *          └┬───┬┘
+ *       ┌───┘   └───┐
+ *    ┌──┴─┐       ┌─┴──┐
+ *    │ n1 │       │ n2 │
+ *    └────┘       └────┘
+ *         wice Agents
+ */
+var _ = Context("restart", func() {
+	var (
+		err error
+		n   Network
+		nw  *g.Network
+
+		s1     *nodes.GrpcSignalingNode
+		n1, n2 *nodes.Agent
+	)
+
+	BeforeEach(OncePerOrdered, func() {
+		n.Init()
+
+		n.AgentOptions = append(n.AgentOptions,
+			gopt.EmptyDir(wg.ConfigPath),
+			gopt.EmptyDir(wg.SocketPath),
+		)
+
+		n.WireGuardInterfaceOptions = append(n.WireGuardInterfaceOptions,
+			wopt.FullMeshPeers,
+		)
+	})
+
+	AfterEach(OncePerOrdered, func() {
+		n.Close()
+	})
+
+	JustBeforeEach(OncePerOrdered, func() {
+		By("Initializing core network")
+
+		nw, err = g.NewNetwork(n.Name, n.NetworkOptions...)
+		Expect(err).To(Succeed(), "Failed to create network: %s", err)
+
+		sw1, err := nw.AddSwitch("sw1")
+		Expect(err).To(Succeed(), "Failed to create switch: %s", err)
+
+		By("Initializing signaling node")
+
+		s1, err = nodes.NewGrpcSignalingNode(nw, "s1",
+			gopt.Interface("eth0", sw1,
+				gopt.AddressIP("10.0.0.2/16"),
+				gopt.AddressIP("fc::2/64"),
+			),
+		)
+		Expect(err).To(Succeed(), "Failed to create signaling node: %s", err)
+
+		By("Initializing agent nodes")
+
+		AddAgent := func(i int) *nodes.Agent {
+			a, err := nodes.NewAgent(nw, fmt.Sprintf("n%d", i),
+				gopt.Customize(n.AgentOptions,
+					gopt.Interface("eth0", sw1,
+						gopt.AddressIP("10.0.1.%d/16", i),
+						gopt.AddressIP("fc::1:%d/64", i),
+					),
+					wopt.Interface("wg0",
+						gopt.Customize(n.WireGuardInterfaceOptions,
+							wopt.AddressIP("172.16.0.%d/16", i),
+						)...,
+					),
+				)...,
+			)
+			Expect(err).To(Succeed(), "Failed to create agent node: %s", err)
+
+			n.AgentNodes = append(n.AgentNodes, a)
+
+			return a
+		}
+
+		n1 = AddAgent(1)
+		n2 = AddAgent(2)
+
+		By("Starting network")
+
+		n.Network = nw
+		n.SignalingNodes = nodes.SignalingList{s1}
+
+		n.Start()
+	})
+
+	RestartTest := func(restart func(gap time.Duration)) {
+		var gap time.Duration
+
+		ConnectivityTestCycle := func() {
+			n.ConnectivityTests()
+
+			It("", func() {
+				By("Triggering restart")
+
+				restart(gap)
+
+				time.Sleep(gap)
+			})
+
+			n.ConnectivityTests()
+		}
+
+		Context("quick", Ordered, func() {
+			BeforeEach(func() {
+				gap = 3 * time.Second
+			})
+
+			ConnectivityTestCycle()
+		})
+
+		Context("slow", Ordered, func() {
+			BeforeEach(func() {
+				gap = 10 * time.Second // > ICE failed/disconnected timeout (5s)
+			})
+
+			ConnectivityTestCycle()
+		})
+	}
+
+	Context("agent", func() {
+		RestartTest(func(gap time.Duration) {
+			By("Stopping first agent")
+
+			err = n1.Stop()
+			Expect(err).To(Succeed(), "Failed to stop first agent: %s", err)
+
+			By("Waiting some time")
+
+			time.Sleep(gap)
+
+			By("Re-starting first agent again")
+
+			err = n1.Start("", n.BasePath, n.AgentArgs()...)
+			Expect(err).To(Succeed(), "Failed to restart first agent: %s", err)
+		})
+	})
+
+	Context("addresses", Pending, func() {
+		RestartTest(func(gap time.Duration) {
+			i := n1.Interface("eth0")
+			Expect(i).NotTo(BeNil(), "Failed to find agent interface")
+
+			By("Deleting old addresses from agent interface")
+
+			for _, a := range i.Addresses {
+				err = i.DeleteAddress(&a)
+				Expect(err).To(Succeed(), "Failed to remove IP address '%s': %s", a.String(), err)
+			}
+
+			By("Waiting some time")
+
+			time.Sleep(gap)
+
+			By("Assigning new addresses to agent interface")
+
+			for _, a := range i.Addresses {
+				ao := a
+				ao.IP = util.OffsetIP(ao.IP, 128)
+
+				err = i.AddAddress(&ao)
+				Expect(err).To(Succeed(), "Failed to add IP address '%s': %s", a.String(), err)
+			}
+
+			out, _, _ := n1.Run("ip", "a")
+			GinkgoWriter.Write(out)
+
+			out, _, _ = n1.Run("wg")
+			GinkgoWriter.Write(out)
+
+			out, _, _ = n2.Run("wg")
+			GinkgoWriter.Write(out)
+		})
+	})
+
+	Context("link", Pending, func() {
+		RestartTest(func(gap time.Duration) {
+			i := n1.Interface("eth0")
+			Expect(i).NotTo(BeNil(), "Failed to find agent interface")
+
+			By("Bringing interface of first agent down")
+
+			err = i.SetDown()
+			Expect(err).To(Succeed(), "Failed to bring interface down: %s", err)
+
+			By("Waiting some time")
+
+			time.Sleep(gap)
+
+			By("Bringing interface of first agent back up")
+
+			err = i.SetUp()
+			Expect(err).To(Succeed(), "Failed to bring interface back up: %s", err)
+		})
+	})
+
+	Context("rpc", func() {
+		RestartTest(func(gap time.Duration) {
+			ctx := context.Background()
+
+			i := n1.WireGuardInterfaces[0]
+			p := i.Peers[0]
+			pk := (*crypto.Key)(&p.PublicKey)
+
+			By("Initiating restart via RPC")
+
+			err = n1.Client.RestartPeer(ctx, i.Name, pk)
+			Expect(err).To(Succeed(), "Failed to restart peer: %s", err)
+		})
+	})
+
+	Context("signaling", Pending, func() {
+		RestartTest(func(gap time.Duration) {
+			By("Stopping signaling server")
+
+			err = s1.Stop()
+			Expect(err).To(Succeed(), "Failed to stop signaling server")
+
+			By("Waiting some time")
+
+			time.Sleep(gap)
+
+			By("Re-starting signaling server again")
+
+			err = s1.Start("", n.BasePath)
+			Expect(err).To(Succeed(), "Failed to restart signaling server: %s", err)
+		})
+	})
+})

test/scaling_test.go

@@ -1,121 +0,0 @@
-//go:build linux
-
-package test_test
-
-import (
-	"fmt"
-
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-	g "github.com/stv0g/gont/pkg"
-	gopt "github.com/stv0g/gont/pkg/options"
-	"riasc.eu/wice/test/nodes"
-	opt "riasc.eu/wice/test/nodes/options"
-	wopt "riasc.eu/wice/test/nodes/options/wg"
-)
-
-/* Simple local-area switched topology for scalability tests
- *
- * Hosts:
- *  - 1x Relay node (Coturn STUN/TURN server)
- *  - 1x Signaling node (GRPC server)
- *  - 1x Switch
- *  - Yx  wice Agent nodes
- *
- *        ┌────┐            ┌────┐
- *  Relay │ r1 │            │ s1 │ Signaling
- *        └──┬─┘            └─┬──┘
- *           └─────┐   ┌──────┘
- *                ┌┴───┴┐
- *                │ sw1 │ Switch
- *                └┬─┬─┬┘
- *           ┌─────┘ │ └───────┐
- *        ┌──┴─┐  ┌──┴─┐     ┌─┴──┐
- *        │ n1 │  │ n2 │ ... │ nY │ wice Agents
- *        └────┘  └────┘     └────┘
- */
-var _ = Context("scaling", Serial, func() {
-	var (
-		n Network
-
-		NumAgents int
-	)
-
-	BeforeEach(func() {
-		n.Init()
-
-		n.AgentOptions = append(n.AgentOptions,
-			opt.ExtraArgs{
-				"--host-sync=false",
-				"--ice-max-binding-requests=500",
-				// "--ice-check-interval=1s",
-				"--ice-failed-timeout=10s",
-				"--ice-candidate-type=host",
-				"--ice-candidate-type=srflx",
-			},
-		)
-		NumAgents = 4
-	})
-
-	AfterEach(func() {
-		n.Close()
-	})
-
-	JustBeforeEach(func() {
-		By("Initializing core network")
-
-		nw, err := g.NewNetwork(n.Name, n.NetworkOptions...)
-		Expect(err).To(Succeed(), "Failed to create network: %s", err)
-
-		sw1, err := nw.AddSwitch("sw1")
-		Expect(err).To(Succeed(), "Failed to create switch: %s", err)
-
-		By("Initializing relay node")
-
-		r1, err := nodes.NewCoturnNode(nw, "r1",
-			gopt.Interface("eth0", sw1,
-				gopt.AddressIP("10.0.0.1/16"),
-				gopt.AddressIP("fc::1/64"),
-			),
-		)
-		Expect(err).To(Succeed(), "Failed to start relay: %s", err)
-
-		By("Initializing signaling node")
-
-		s1, err := nodes.NewGrpcSignalingNode(nw, "s1",
-			gopt.Interface("eth0", sw1,
-				gopt.AddressIP("10.0.0.2/16"),
-				gopt.AddressIP("fc::2/64"),
-			),
-		)
-		Expect(err).To(Succeed(), "Failed to create signaling node: %s", err)
-
-		By("Initializing agent nodes")
-
-		for i := 1; i <= NumAgents; i++ {
-			node, err := nodes.NewAgent(nw, fmt.Sprintf("n%d", i),
-				gopt.Customize(n.AgentOptions,
-					gopt.Interface("eth0", sw1,
-						gopt.AddressIPv4(10, 0, 1, byte(i), 16),
-						gopt.AddressIP(fmt.Sprintf("fc::1:%d/64", i)),
-					),
-					wopt.Interface("wg0",
-						wopt.AddressIPv4(172, 16, 0, byte(i), 16),
-						wopt.FullMeshPeers,
-					),
-				)...,
-			)
-			Expect(err).To(Succeed(), "Failed to create node: %s", err)
-
-			n.AgentNodes = append(n.AgentNodes, node)
-		}
-
-		n.Network = nw
-		n.RelayNodes = nodes.RelayList{r1}
-		n.SignalingNodes = nodes.SignalingList{s1}
-
-		n.Start()
-	})
-
-	n.ConnectivityTests()
-})

test/simple_test.go

@@ -5,7 +5,6 @@ package test_test
 import (
 	"fmt"
 
-	"go.uber.org/zap"
 	"golang.org/x/sys/unix"
 	"riasc.eu/wice/pkg/wg"
 	"riasc.eu/wice/test/nodes"
@@ -19,36 +18,41 @@ import (
 	gfopt "github.com/stv0g/gont/pkg/options/filters"
 )
 
-var (
+/* Simple local-area switched topology with variable number of agents
-	logger *zap.Logger
-)
-
-/* Simple local-area switched topology
  *
- * Hosts:
+ *  - 1x Relay node        [r1] (Coturn STUN/TURN server)
- *  - 1x Relay node (Coturn STUN/TURN server)
+ *  - 1x Signaling node    [s1] (GRPC server)
- *  - 1x Signaling node (GRPC server)
+ *  - 1x Switch            [sw1]
- *  - 1x Switch
+ *  - Yx  wice Agent nodes [n?]
- *  - 2x wice Agent nodes
  *
- *        ┌────┐   ┌────┐
+ *        Relay            Signaling
- *  Relay │ r1 │   │ s1 │ Signaling
+ *        ┌────┐            ┌────┐
- *        └──┬─┘   └─┬──┘
+ *        │ r1 │            │ s1 │
- *           └─┐   ┌─┘
+ *        └──┬─┘            └─┬──┘
- *            ┌┴───┴┐
+ *           └─────┐   ┌──────┘
- *            │ sw1 │ Switch
+ *                ┌┴───┴┐
- *            └┬───┬┘
+ *                │ sw1 │ Switch
- *           ┌─┘   └─┐
+ *                └┬─┬─┬┘
- *        ┌──┴─┐   ┌─┴──┐
+ *           ┌─────┘ │ └───────┐
- *        │ n1 │   │ n2 │ wice Agents
+ *        ┌──┴─┐  ┌──┴─┐     ┌─┴──┐
- *        └────┘   └────┘
+ *        │ n1 │  │ n2 │ ... │ nY │
+ *        └────┘  └────┘     └────┘
+ *               wice Agents
  */
 var _ = Context("simple", Serial, func() {
-	var n Network
+	var (
+		err error
+		n   Network
+		nw  *g.Network
+
+		NumAgents int
+	)
 
 	BeforeEach(func() {
 		n.Init()
 
+		NumAgents = 3
+
 		n.AgentOptions = append(n.AgentOptions,
 			gopt.EmptyDir(wg.ConfigPath),
 			gopt.EmptyDir(wg.SocketPath),
@@ -66,7 +70,7 @@ var _ = Context("simple", Serial, func() {
 	JustBeforeEach(func() {
 		By("Initializing core network")
 
-		nw, err := g.NewNetwork(n.Name, n.NetworkOptions...)
+		nw, err = g.NewNetwork(n.Name, n.NetworkOptions...)
 		Expect(err).To(Succeed(), "Failed to create network: %s", err)
 
 		sw1, err := nw.AddSwitch("sw1")
@@ -94,9 +98,8 @@ var _ = Context("simple", Serial, func() {
 
 		By("Initializing agent nodes")
 
-		CreateAgent := func(i int) *nodes.Agent {
+		AddAgent := func(i int) *nodes.Agent {
-			name := fmt.Sprintf("n%d", i)
+			a, err := nodes.NewAgent(nw, fmt.Sprintf("n%d", i),
-			n, err := nodes.NewAgent(n.Network, name,
 				gopt.Customize(n.AgentOptions,
 					gopt.Interface("eth0", sw1,
 						gopt.AddressIP("10.0.1.%d/16", i),
@@ -105,11 +108,19 @@ var _ = Context("simple", Serial, func() {
 					wopt.Interface("wg0",
 						gopt.Customize(n.WireGuardInterfaceOptions,
 							wopt.AddressIP("172.16.0.%d/16", i),
-						)...),
+						)...,
-				)...)
+					),
+				)...,
+			)
 			Expect(err).To(Succeed(), "Failed to create agent node: %s", err)
 
-			return n
+			n.AgentNodes = append(n.AgentNodes, a)
+
+			return a
+		}
+
+		for i := 1; i <= NumAgents; i++ {
+			AddAgent(i)
 		}
 
 		By("Starting network")
@@ -117,10 +128,6 @@ var _ = Context("simple", Serial, func() {
 		n.Network = nw
 		n.RelayNodes = nodes.RelayList{r1}
 		n.SignalingNodes = nodes.SignalingList{s1}
-		n.AgentNodes = nodes.AgentList{
-			CreateAgent(1),
-			CreateAgent(2),
-		}
 
 		n.Start()
 	})
@@ -185,20 +192,18 @@ var _ = Context("simple", Serial, func() {
 	})
 
 	Context("userspace", func() {
-		Context("ipv4", func() {
+		BeforeEach(func() {
-			BeforeEach(func() {
+			n.WireGuardInterfaceOptions = append(n.WireGuardInterfaceOptions,
-				n.WireGuardInterfaceOptions = append(n.WireGuardInterfaceOptions,
+				wopt.WriteConfigFile(true),
-					wopt.WriteConfigFile(true),
+				wopt.SetupKernelInterface(false),
-					wopt.SetupKernelInterface(false),
+			)
-				)
 
-				n.AgentOptions = append(n.AgentOptions,
+			n.AgentOptions = append(n.AgentOptions,
-					opt.ExtraArgs{"--wg-userspace", "wg0"},
+				opt.ExtraArgs{"--wg-userspace", "wg0"},
-				)
+			)
-			})
-
-			ConnectivityTestsForAllCandidateTypes()
 		})
+
+		ConnectivityTestsForAllCandidateTypes()
 	})
 
 	Context("filtered", func() {

test/suite_test.go

@@ -7,16 +7,15 @@ import (
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
-	"github.com/onsi/ginkgo/v2/types"
 	. "github.com/onsi/gomega"
+
 	"github.com/onsi/gomega/gexec"
-	"go.uber.org/zap"
 	"riasc.eu/wice/pkg/util"
+
+	"github.com/onsi/ginkgo/v2/types"
 )
 
 var (
-	logger *zap.Logger
-
 	setup   bool
 	persist bool
 	capture bool