functional/tester: handle "process already finished"

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>

functional/tester/cluster.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"math/rand"
 	"net/http"
@@ -318,15 +319,27 @@ func (clus *Cluster) broadcastOperation(op rpcpb.Operation) error {
 			continue
 		}
 
-		if err != nil &&
-			op == rpcpb.Operation_DestroyEtcdAgent &&
-			strings.Contains(err.Error(), "rpc error: code = Unavailable desc = transport is closing") {
-			// agent server has already closed;
-			// so this error is expected
-			clus.lg.Info("successfully destroyed all")
-			continue
+		if err != nil {
+			destroyed := false
+			if op == rpcpb.Operation_DestroyEtcdAgent {
+				if err == io.EOF {
+					destroyed = true
+				}
+				if strings.Contains(err.Error(),
+					"rpc error: code = Unavailable desc = transport is closing") {
+					// agent server has already closed;
+					// so this error is expected
+					destroyed = true
+				}
+				if strings.Contains(err.Error(),
+					"desc = os: process already finished") {
+					destroyed = true
+				}
+			}
+			if !destroyed {
+				errs = append(errs, err.Error())
+			}
 		}
-		errs = append(errs, err.Error())
 	}
 
 	if len(errs) == 0 {

functional/tester/cluster_run.go

@@ -292,10 +292,6 @@ func (clus *Cluster) compact(rev int64, timeout time.Duration) (err error) {
 }
 
 func (clus *Cluster) failed() {
-	if !clus.Tester.ExitOnFailure {
-		return
-	}
-
 	clus.lg.Info(
 		"functional-tester FAIL",
 		zap.Int("round", clus.rd),
@@ -303,11 +299,14 @@ func (clus *Cluster) failed() {
 		zap.Int("case-total", len(clus.failures)),
 	)
 	clus.DestroyEtcdAgents()
+
 	os.Exit(2)
 }
 
 func (clus *Cluster) cleanup() error {
-	defer clus.failed()
+	if clus.Tester.ExitOnFailure {
+		defer clus.failed()
+	}
 
 	roundFailedTotalCounter.Inc()
 	desc := "compact/defrag"