Updated test suites to retry connection to tcp server. Use concurrency to setup cluster in Test_Cluster suite.

echovault/echovault_test.go

@@ -40,6 +40,7 @@ type ClientServerPair struct {
 	raftPort         int
 	mlPort           int
 	bootstrapCluster bool
+	joinAddr         string
 	raw              net.Conn
 	client           *resp.Conn
 	server           *EchoVault
@@ -92,9 +93,57 @@ func setupServer(
 	)
 }
 
+func setupNode(node *ClientServerPair, isLeader bool, errChan *chan error) {
+	server, err := setupServer(
+		node.serverId,
+		node.bootstrapCluster,
+		node.bindAddr,
+		node.joinAddr,
+		node.port,
+		node.raftPort,
+		node.mlPort,
+	)
+	if err != nil {
+		*errChan <- fmt.Errorf("could not start server; %v", err)
+	}
+
+	// Start the server.
+	go func() {
+		server.Start()
+	}()
+
+	if isLeader {
+		// If node is a leader, wait until it's established itself as a leader of the raft cluster.
+		for {
+			if server.raft.IsRaftLeader() {
+				break
+			}
+		}
+	} else {
+		// If the node is a follower, wait until it's joined the raft cluster before moving forward.
+		for {
+			if server.raft.HasJoinedCluster() {
+				break
+			}
+		}
+	}
+
+	// Setup client connection.
+	conn, err := internal.GetConnection(node.bindAddr, node.port)
+	if err != nil {
+		*errChan <- fmt.Errorf("could not open tcp connection: %v", err)
+	}
+	client := resp.NewConn(conn)
+
+	node.raw = conn
+	node.client = client
+	node.server = server
+}
+
 func makeCluster(size int) ([]ClientServerPair, error) {
 	pairs := make([]ClientServerPair, size)
 
+	// Set up node metadata.
 	for i := 0; i < len(pairs); i++ {
 		serverId := fmt.Sprintf("SERVER-%d", i)
 		bindAddr := getBindAddr().String()
@@ -115,42 +164,6 @@ func makeCluster(size int) ([]ClientServerPair, error) {
 		if err != nil {
 			return nil, fmt.Errorf("could not get free memberlist port: %v", err)
 		}
-		server, err := setupServer(serverId, bootstrapCluster, bindAddr, joinAddr, port, raftPort, memberlistPort)
-		if err != nil {
-			return nil, fmt.Errorf("could not start server; %v", err)
-		}
-
-		// Start the server
-		wg := sync.WaitGroup{}
-		wg.Add(1)
-		go func() {
-			wg.Done()
-			server.Start()
-		}()
-		wg.Wait()
-
-		if i == 0 {
-			// If node is a leader, wait until it's established itself as a leader of the raft cluster.
-			for {
-				if server.raft.IsRaftLeader() {
-					break
-				}
-			}
-		} else {
-			// If the node is a follower, wait until it's joined the raft cluster before moving forward.
-			for {
-				if server.raft.HasJoinedCluster() {
-					break
-				}
-			}
-		}
-
-		// Setup client connection.
-		conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", bindAddr, port))
-		if err != nil {
-			return nil, fmt.Errorf("could not open tcp connection: %v", err)
-		}
-		client := resp.NewConn(conn)
 
 		pairs[i] = ClientServerPair{
 			serverId:         serverId,
@@ -159,12 +172,37 @@ func makeCluster(size int) ([]ClientServerPair, error) {
 			raftPort:         raftPort,
 			mlPort:           memberlistPort,
 			bootstrapCluster: bootstrapCluster,
-			raw:              conn,
-			client:           client,
-			server:           server,
+			joinAddr:         joinAddr,
 		}
 	}
 
+	errChan := make(chan error)
+	doneChan := make(chan struct{})
+
+	// Set up nodes.
+	wg := sync.WaitGroup{}
+	for i := 0; i < len(pairs); i++ {
+		if i == 0 {
+			setupNode(&pairs[i], pairs[i].bootstrapCluster, &errChan)
+			continue
+		}
+		wg.Add(1)
+		go func(idx int) {
+			setupNode(&pairs[idx], pairs[idx].bootstrapCluster, &errChan)
+			wg.Done()
+		}(i)
+	}
+	go func() {
+		wg.Wait()
+		doneChan <- struct{}{}
+	}()
+
+	select {
+	case err := <-errChan:
+		return nil, err
+	case <-doneChan:
+	}
+
 	return pairs, nil
 }
 
@@ -428,275 +466,283 @@ func Test_Cluster(t *testing.T) {
 		}
 	})
 
-	// t.Run("Test_ForwardCommand", func(t *testing.T) {
-	// 	tests := tests["forward"]
-	// 	// Write all the data a random cluster follower.
-	// 	for i, test := range tests {
-	// 		// Send write command to follower node.
-	// 		node := nodes[1]
-	// 		if err := node.client.WriteArray([]resp.Value{
-	// 			resp.StringValue("SET"),
-	// 			resp.StringValue(test.key),
-	// 			resp.StringValue(test.value),
-	// 		}); err != nil {
-	// 			t.Errorf("could not write data to follower node (test %d): %v", i, err)
-	// 		}
-	// 		// Read response and make sure we received "ok" response.
-	// 		rd, _, err := node.client.ReadValue()
-	// 		if err != nil {
-	// 			t.Errorf("could not read response from follower node (test %d): %v", i, err)
-	// 		}
-	// 		if !strings.EqualFold(rd.String(), "ok") {
-	// 			t.Errorf("expected response for test %d to be \"OK\", got %s", i, rd.String())
-	// 		}
-	// 	}
-	//
-	// 	<-time.After(200 * time.Millisecond) // Short yield to allow change to take effect.
-	//
-	// 	// Check if the data has been replicated on a quorum (majority of the cluster).
-	// 	quorum := int(math.Ceil(float64(len(nodes)/2)) + 1)
-	// 	for i, test := range tests {
-	// 		count := 0
-	// 		for j := 0; j < len(nodes); j++ {
-	// 			node := nodes[j]
-	// 			if err := node.client.WriteArray([]resp.Value{
-	// 				resp.StringValue("GET"),
-	// 				resp.StringValue(test.key),
-	// 			}); err != nil {
-	// 				t.Errorf("could not write data to follower node %d (test %d): %v", j, i, err)
-	// 			}
-	// 			rd, _, err := node.client.ReadValue()
-	// 			if err != nil {
-	// 				t.Errorf("could not read data from follower node %d (test %d): %v", j, i, err)
-	// 			}
-	// 			if rd.String() == test.value {
-	// 				count += 1 // If the expected value is found, increment the count.
-	// 			}
-	// 		}
-	// 		// Fail if count is less than quorum.
-	// 		if count < quorum {
-	// 			t.Errorf("could not find value %s at key %s in cluster quorum", test.value, test.key)
-	// 		}
-	// 	}
-	// })
+	t.Run("Test_ForwardCommand", func(t *testing.T) {
+		tests := tests["forward"]
+		// Write all the data a random cluster follower.
+		for i, test := range tests {
+			// Send write command to follower node.
+			node := nodes[1]
+			if err := node.client.WriteArray([]resp.Value{
+				resp.StringValue("SET"),
+				resp.StringValue(test.key),
+				resp.StringValue(test.value),
+			}); err != nil {
+				t.Errorf("could not write data to follower node (test %d): %v", i, err)
+			}
+			// Read response and make sure we received "ok" response.
+			rd, _, err := node.client.ReadValue()
+			if err != nil {
+				t.Errorf("could not read response from follower node (test %d): %v", i, err)
+			}
+			if !strings.EqualFold(rd.String(), "ok") {
+				t.Errorf("expected response for test %d to be \"OK\", got %s", i, rd.String())
+			}
+		}
+
+		<-time.After(1 * time.Second) // Short yield to allow change to take effect.
+
+		// Check if the data has been replicated on a quorum (majority of the cluster).
+		quorum := int(math.Ceil(float64(len(nodes)/2)) + 1)
+		for i, test := range tests {
+			count := 0
+			for j := 0; j < len(nodes); j++ {
+				node := nodes[j]
+				if err := node.client.WriteArray([]resp.Value{
+					resp.StringValue("GET"),
+					resp.StringValue(test.key),
+				}); err != nil {
+					t.Errorf("could not write data to follower node %d (test %d): %v", j, i, err)
+				}
+				rd, _, err := node.client.ReadValue()
+				if err != nil {
+					t.Errorf("could not read data from follower node %d (test %d): %v", j, i, err)
+				}
+				if rd.String() == test.value {
+					count += 1 // If the expected value is found, increment the count.
+				}
+			}
+			// Fail if count is less than quorum.
+			if count < quorum {
+				t.Errorf("could not find value %s at key %s in cluster quorum", test.value, test.key)
+			}
+		}
+	})
 }
 
-func Test_TLS(t *testing.T) {
-	port, err := internal.GetFreePort()
-	if err != nil {
-		t.Error(err)
-	}
+func Test_Standalone(t *testing.T) {
+	t.Run("Test_TLS", func(t *testing.T) {
+		t.Parallel()
 
-	conf := DefaultConfig()
-	conf.DataDir = ""
-	conf.BindAddr = "localhost"
-	conf.Port = uint16(port)
-	conf.TLS = true
-	conf.CertKeyPairs = [][]string{
-		{
-			path.Join("..", "openssl", "server", "server1.crt"),
-			path.Join("..", "openssl", "server", "server1.key"),
-		},
-		{
-			path.Join("..", "openssl", "server", "server2.crt"),
-			path.Join("..", "openssl", "server", "server2.key"),
-		},
-	}
+		port, err := internal.GetFreePort()
+		if err != nil {
+			t.Error(err)
+			return
+		}
 
-	server, err := NewEchoVault(WithConfig(conf))
-	if err != nil {
-		t.Error(err)
-		return
-	}
+		conf := DefaultConfig()
+		conf.DataDir = ""
+		conf.BindAddr = "localhost"
+		conf.Port = uint16(port)
+		conf.TLS = true
+		conf.CertKeyPairs = [][]string{
+			{
+				path.Join("..", "openssl", "server", "server1.crt"),
+				path.Join("..", "openssl", "server", "server1.key"),
+			},
+			{
+				path.Join("..", "openssl", "server", "server2.crt"),
+				path.Join("..", "openssl", "server", "server2.key"),
+			},
+		}
 
-	wg := sync.WaitGroup{}
-	wg.Add(1)
-	go func() {
-		wg.Done()
-		server.Start()
-	}()
-	wg.Wait()
+		server, err := NewEchoVault(WithConfig(conf))
+		if err != nil {
+			t.Error(err)
+			return
+		}
 
-	// Dial with ServerCAs
-	serverCAs := x509.NewCertPool()
-	f, err := os.Open(path.Join("..", "openssl", "server", "rootCA.crt"))
-	if err != nil {
-		t.Error(err)
-	}
-	cert, err := io.ReadAll(bufio.NewReader(f))
-	if err != nil {
-		t.Error(err)
-	}
-	ok := serverCAs.AppendCertsFromPEM(cert)
-	if !ok {
-		t.Error("could not load server CA")
-	}
+		wg := sync.WaitGroup{}
+		wg.Add(1)
+		go func() {
+			wg.Done()
+			server.Start()
+		}()
+		wg.Wait()
 
-	conn, err := tls.Dial("tcp", fmt.Sprintf("localhost:%d", port), &tls.Config{
-		RootCAs: serverCAs,
-	})
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	defer func() {
-		_ = conn.Close()
-		server.ShutDown()
-	}()
-	client := resp.NewConn(conn)
-
-	// Test that we can set and get a value from the server.
-	key := "key1"
-	value := "value1"
-	err = client.WriteArray([]resp.Value{
-		resp.StringValue("SET"), resp.StringValue(key), resp.StringValue(value),
-	})
-	if err != nil {
-		t.Error(err)
-	}
-
-	res, _, err := client.ReadValue()
-	if err != nil {
-		t.Error(err)
-	}
-
-	if !strings.EqualFold(res.String(), "ok") {
-		t.Errorf("expected response OK, got \"%s\"", res.String())
-	}
-
-	err = client.WriteArray([]resp.Value{resp.StringValue("GET"), resp.StringValue(key)})
-	if err != nil {
-		t.Error(err)
-	}
-
-	res, _, err = client.ReadValue()
-	if err != nil {
-		t.Error(err)
-	}
-
-	if res.String() != value {
-		t.Errorf("expected response at key \"%s\" to be \"%s\", got \"%s\"", key, value, res.String())
-	}
-}
-
-func Test_MTLS(t *testing.T) {
-	port, err := internal.GetFreePort()
-	if err != nil {
-		t.Error(err)
-	}
-
-	conf := DefaultConfig()
-	conf.DataDir = ""
-	conf.BindAddr = "localhost"
-	conf.Port = uint16(port)
-	conf.TLS = true
-	conf.MTLS = true
-	conf.ClientCAs = []string{
-		path.Join("..", "openssl", "client", "rootCA.crt"),
-	}
-	conf.CertKeyPairs = [][]string{
-		{
-			path.Join("..", "openssl", "server", "server1.crt"),
-			path.Join("..", "openssl", "server", "server1.key"),
-		},
-		{
-			path.Join("..", "openssl", "server", "server2.crt"),
-			path.Join("..", "openssl", "server", "server2.key"),
-		},
-	}
-
-	server, err := NewEchoVault(WithConfig(conf))
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	wg := sync.WaitGroup{}
-	wg.Add(1)
-	go func() {
-		wg.Done()
-		server.Start()
-	}()
-	wg.Wait()
-
-	// Dial with ServerCAs and client certificates
-	clientCertKeyPairs := [][]string{
-		{
-			path.Join("..", "openssl", "client", "client1.crt"),
-			path.Join("..", "openssl", "client", "client1.key"),
-		},
-		{
-			path.Join("..", "openssl", "client", "client2.crt"),
-			path.Join("..", "openssl", "client", "client2.key"),
-		},
-	}
-	var certificates []tls.Certificate
-	for _, pair := range clientCertKeyPairs {
-		c, err := tls.LoadX509KeyPair(pair[0], pair[1])
+		// Dial with ServerCAs
+		serverCAs := x509.NewCertPool()
+		f, err := os.Open(path.Join("..", "openssl", "server", "rootCA.crt"))
 		if err != nil {
 			t.Error(err)
 		}
-		certificates = append(certificates, c)
-	}
+		cert, err := io.ReadAll(bufio.NewReader(f))
+		if err != nil {
+			t.Error(err)
+		}
+		ok := serverCAs.AppendCertsFromPEM(cert)
+		if !ok {
+			t.Error("could not load server CA")
+		}
 
-	serverCAs := x509.NewCertPool()
-	f, err := os.Open(path.Join("..", "openssl", "server", "rootCA.crt"))
-	if err != nil {
-		t.Error(err)
-	}
-	cert, err := io.ReadAll(bufio.NewReader(f))
-	if err != nil {
-		t.Error(err)
-	}
-	ok := serverCAs.AppendCertsFromPEM(cert)
-	if !ok {
-		t.Error("could not load server CA")
-	}
+		conn, err := internal.GetTLSConnection("localhost", port, &tls.Config{
+			RootCAs: serverCAs,
+		})
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		defer func() {
+			_ = conn.Close()
+			server.ShutDown()
+		}()
+		client := resp.NewConn(conn)
 
-	conn, err := tls.Dial("tcp", fmt.Sprintf("localhost:%d", port), &tls.Config{
-		RootCAs:      serverCAs,
-		Certificates: certificates,
+		// Test that we can set and get a value from the server.
+		key := "key1"
+		value := "value1"
+		err = client.WriteArray([]resp.Value{
+			resp.StringValue("SET"), resp.StringValue(key), resp.StringValue(value),
+		})
+		if err != nil {
+			t.Error(err)
+		}
+
+		res, _, err := client.ReadValue()
+		if err != nil {
+			t.Error(err)
+		}
+
+		if !strings.EqualFold(res.String(), "ok") {
+			t.Errorf("expected response OK, got \"%s\"", res.String())
+		}
+
+		err = client.WriteArray([]resp.Value{resp.StringValue("GET"), resp.StringValue(key)})
+		if err != nil {
+			t.Error(err)
+		}
+
+		res, _, err = client.ReadValue()
+		if err != nil {
+			t.Error(err)
+		}
+
+		if res.String() != value {
+			t.Errorf("expected response at key \"%s\" to be \"%s\", got \"%s\"", key, value, res.String())
+		}
 	})
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	defer func() {
-		_ = conn.Close()
-		server.ShutDown()
-	}()
-	client := resp.NewConn(conn)
 
-	// Test that we can set and get a value from the server.
-	key := "key1"
-	value := "value1"
-	err = client.WriteArray([]resp.Value{
-		resp.StringValue("SET"), resp.StringValue(key), resp.StringValue(value),
+	t.Run("Test_MTLS", func(t *testing.T) {
+		t.Parallel()
+
+		port, err := internal.GetFreePort()
+		if err != nil {
+			t.Error(err)
+			return
+		}
+
+		conf := DefaultConfig()
+		conf.DataDir = ""
+		conf.BindAddr = "localhost"
+		conf.Port = uint16(port)
+		conf.TLS = true
+		conf.MTLS = true
+		conf.ClientCAs = []string{
+			path.Join("..", "openssl", "client", "rootCA.crt"),
+		}
+		conf.CertKeyPairs = [][]string{
+			{
+				path.Join("..", "openssl", "server", "server1.crt"),
+				path.Join("..", "openssl", "server", "server1.key"),
+			},
+			{
+				path.Join("..", "openssl", "server", "server2.crt"),
+				path.Join("..", "openssl", "server", "server2.key"),
+			},
+		}
+
+		server, err := NewEchoVault(WithConfig(conf))
+		if err != nil {
+			t.Error(err)
+			return
+		}
+
+		wg := sync.WaitGroup{}
+		wg.Add(1)
+		go func() {
+			wg.Done()
+			server.Start()
+		}()
+		wg.Wait()
+
+		// Dial with ServerCAs and client certificates
+		clientCertKeyPairs := [][]string{
+			{
+				path.Join("..", "openssl", "client", "client1.crt"),
+				path.Join("..", "openssl", "client", "client1.key"),
+			},
+			{
+				path.Join("..", "openssl", "client", "client2.crt"),
+				path.Join("..", "openssl", "client", "client2.key"),
+			},
+		}
+		var certificates []tls.Certificate
+		for _, pair := range clientCertKeyPairs {
+			c, err := tls.LoadX509KeyPair(pair[0], pair[1])
+			if err != nil {
+				t.Error(err)
+			}
+			certificates = append(certificates, c)
+		}
+
+		serverCAs := x509.NewCertPool()
+		f, err := os.Open(path.Join("..", "openssl", "server", "rootCA.crt"))
+		if err != nil {
+			t.Error(err)
+		}
+		cert, err := io.ReadAll(bufio.NewReader(f))
+		if err != nil {
+			t.Error(err)
+		}
+		ok := serverCAs.AppendCertsFromPEM(cert)
+		if !ok {
+			t.Error("could not load server CA")
+		}
+
+		conn, err := internal.GetTLSConnection("localhost", port, &tls.Config{
+			RootCAs:      serverCAs,
+			Certificates: certificates,
+		})
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		defer func() {
+			_ = conn.Close()
+			server.ShutDown()
+		}()
+		client := resp.NewConn(conn)
+
+		// Test that we can set and get a value from the server.
+		key := "key1"
+		value := "value1"
+		err = client.WriteArray([]resp.Value{
+			resp.StringValue("SET"), resp.StringValue(key), resp.StringValue(value),
+		})
+		if err != nil {
+			t.Error(err)
+		}
+
+		res, _, err := client.ReadValue()
+		if err != nil {
+			t.Error(err)
+		}
+
+		if !strings.EqualFold(res.String(), "ok") {
+			t.Errorf("expected response OK, got \"%s\"", res.String())
+		}
+
+		err = client.WriteArray([]resp.Value{resp.StringValue("GET"), resp.StringValue(key)})
+		if err != nil {
+			t.Error(err)
+		}
+
+		res, _, err = client.ReadValue()
+		if err != nil {
+			t.Error(err)
+		}
+
+		if res.String() != value {
+			t.Errorf("expected response at key \"%s\" to be \"%s\", got \"%s\"", key, value, res.String())
+		}
 	})
-	if err != nil {
-		t.Error(err)
-	}
-
-	res, _, err := client.ReadValue()
-	if err != nil {
-		t.Error(err)
-	}
-
-	if !strings.EqualFold(res.String(), "ok") {
-		t.Errorf("expected response OK, got \"%s\"", res.String())
-	}
-
-	err = client.WriteArray([]resp.Value{resp.StringValue("GET"), resp.StringValue(key)})
-	if err != nil {
-		t.Error(err)
-	}
-
-	res, _, err = client.ReadValue()
-	if err != nil {
-		t.Error(err)
-	}
-
-	if res.String() != value {
-		t.Errorf("expected response at key \"%s\" to be \"%s\", got \"%s\"", key, value, res.String())
-	}
 }