improve the performance of starting functional test

The proxy must be waiting for the etcd to be running, but the current
implementation hard codes the wating time as 5 seconds. The improvement
is to dynamically check whether the etcd is running, and start the
proxy when etcd port is reachable.

Signed-off-by: Benjamin Wang <wachao@vmware.com>

tests/functional/agent/handler.go

@@ -125,7 +125,7 @@ func (srv *Server) createEtcd(fromSnapshot bool, failpoints string) error {
 func (srv *Server) runEtcd() error {
 	errc := make(chan error)
 	go func() {
-		time.Sleep(5 * time.Second)
+		time.Sleep(1 * time.Second)
 		// server advertise client/peer listener had to start first
 		// before setting up proxy listener
 		errc <- srv.startProxy()
@@ -137,17 +137,19 @@ func (srv *Server) runEtcd() error {
 			zap.String("command-path", srv.etcdCmd.Path),
 		)
 		err := srv.etcdCmd.Start()
-		perr := <-errc
+
 		srv.lg.Info(
 			"started etcd command",
 			zap.String("command-path", srv.etcdCmd.Path),
 			zap.Strings("command-args", srv.etcdCmd.Args),
-			zap.Errors("errors", []error{err, perr}),
+			zap.Strings("envs", srv.etcdCmd.Env),
+			zap.Error(err),
 		)
 		if err != nil {
 			return err
 		}
-		return perr
+
+		return <-errc
 	}
 
 	select {
@@ -218,6 +220,11 @@ func (srv *Server) startProxy() error {
 			return err
 		}
 
+		srv.lg.Info("Checking client target's connectivity", zap.String("target", listenClientURL.Host))
+		if err := checkTCPConnect(srv.lg, listenClientURL.Host); err != nil {
+			return fmt.Errorf("check client target failed, %w", err)
+		}
+
 		srv.lg.Info("starting proxy on client traffic", zap.String("url", advertiseClientURL.String()))
 		srv.advertiseClientPortToProxy[advertiseClientURLPort] = proxy.NewServer(proxy.ServerConfig{
 			Logger: srv.lg,
@@ -226,6 +233,7 @@ func (srv *Server) startProxy() error {
 		})
 		select {
 		case err = <-srv.advertiseClientPortToProxy[advertiseClientURLPort].Error():
+			srv.lg.Info("starting client proxy failed", zap.Error(err))
 			return err
 		case <-time.After(2 * time.Second):
 			srv.lg.Info("started proxy on client traffic", zap.String("url", advertiseClientURL.String()))
@@ -242,6 +250,11 @@ func (srv *Server) startProxy() error {
 			return err
 		}
 
+		srv.lg.Info("Checking peer target's connectivity", zap.String("target", listenPeerURL.Host))
+		if err := checkTCPConnect(srv.lg, listenPeerURL.Host); err != nil {
+			return fmt.Errorf("check peer target failed, %w", err)
+		}
+
 		srv.lg.Info("starting proxy on peer traffic", zap.String("url", advertisePeerURL.String()))
 		srv.advertisePeerPortToProxy[advertisePeerURLPort] = proxy.NewServer(proxy.ServerConfig{
 			Logger: srv.lg,
@@ -250,6 +263,7 @@ func (srv *Server) startProxy() error {
 		})
 		select {
 		case err = <-srv.advertisePeerPortToProxy[advertisePeerURLPort].Error():
+			srv.lg.Info("starting peer proxy failed", zap.Error(err))
 			return err
 		case <-time.After(2 * time.Second):
 			srv.lg.Info("started proxy on peer traffic", zap.String("url", advertisePeerURL.String()))

tests/functional/agent/utils.go

@@ -126,6 +126,23 @@ func loadFileData(filePath string) ([]byte, error) {
 	return data, nil
 }
 
+func checkTCPConnect(lg *zap.Logger, target string) error {
+	for i := 0; i < 10; i++ {
+		if conn, err := net.Dial("tcp", target); err != nil {
+			lg.Error("The target isn't reachable", zap.Int("retries", i), zap.String("target", target), zap.Error(err))
+		} else {
+			if conn != nil {
+				conn.Close()
+				lg.Info("The target is reachable", zap.Int("retries", i), zap.String("target", target))
+				return nil
+			}
+			lg.Error("The target isn't reachable due to the returned conn is nil", zap.Int("retries", i), zap.String("target", target))
+		}
+		time.Sleep(time.Second)
+	}
+	return fmt.Errorf("timed out waiting for the target (%s) to be reachable", target)
+}
+
 func cleanPageCache() error {
 	// https://www.kernel.org/doc/Documentation/sysctl/vm.txt
 	// https://github.com/torvalds/linux/blob/master/fs/drop_caches.c