Integration tests: Use zaptest.Logger based testing.TB

Thanks to this the logs:
  - are automatically printed if the test fails.
  - are in pretty consistent format.
  - are annotated by 'member' information of the cluster emitting them.

Side changes:
  - Set propert default got DefaultWarningApplyDuration (used to be '0')
  - Name the members based on their 'place' on the list (as opposed to
'random')

raft/logger.go

@@ -48,6 +48,16 @@ func SetLogger(l Logger) {
 	raftLoggerMu.Unlock()
 }
 
+func ResetDefaultLogger() {
+	SetLogger(defaultLogger)
+}
+
+func getLogger() Logger {
+	raftLoggerMu.Lock()
+	defer raftLoggerMu.Unlock()
+	return raftLogger
+}
+
 var (
 	defaultLogger = &DefaultLogger{Logger: log.New(os.Stderr, "raft", log.LstdFlags)}
 	discardLogger = &DefaultLogger{Logger: log.New(ioutil.Discard, "", 0)}

raft/raft.go

@@ -230,7 +230,7 @@ func (c *Config) validate() error {
 	}
 
 	if c.Logger == nil {
-		c.Logger = raftLogger
+		c.Logger = getLogger()
 	}
 
 	if c.ReadOnlyOption == ReadOnlyLeaseBased && !c.CheckQuorum {

raft/status.go

@@ -100,7 +100,7 @@ func (s Status) MarshalJSON() ([]byte, error) {
 func (s Status) String() string {
 	b, err := s.MarshalJSON()
 	if err != nil {
-		raftLogger.Panicf("unexpected error: %v", err)
+		getLogger().Panicf("unexpected error: %v", err)
 	}
 	return string(b)
 }

raft/storage.go

@@ -115,7 +115,7 @@ func (ms *MemoryStorage) Entries(lo, hi, maxSize uint64) ([]pb.Entry, error) {
 		return nil, ErrCompacted
 	}
 	if hi > ms.lastIndex()+1 {
-		raftLogger.Panicf("entries' hi(%d) is out of bound lastindex(%d)", hi, ms.lastIndex())
+		getLogger().Panicf("entries' hi(%d) is out of bound lastindex(%d)", hi, ms.lastIndex())
 	}
 	// only contains dummy entries.
 	if len(ms.ents) == 1 {
@@ -200,7 +200,7 @@ func (ms *MemoryStorage) CreateSnapshot(i uint64, cs *pb.ConfState, data []byte)
 
 	offset := ms.ents[0].Index
 	if i > ms.lastIndex() {
-		raftLogger.Panicf("snapshot %d is out of bound lastindex(%d)", i, ms.lastIndex())
+		getLogger().Panicf("snapshot %d is out of bound lastindex(%d)", i, ms.lastIndex())
 	}
 
 	ms.snapshot.Metadata.Index = i
@@ -223,7 +223,7 @@ func (ms *MemoryStorage) Compact(compactIndex uint64) error {
 		return ErrCompacted
 	}
 	if compactIndex > ms.lastIndex() {
-		raftLogger.Panicf("compact %d is out of bound lastindex(%d)", compactIndex, ms.lastIndex())
+		getLogger().Panicf("compact %d is out of bound lastindex(%d)", compactIndex, ms.lastIndex())
 	}
 
 	i := compactIndex - offset
@@ -266,7 +266,7 @@ func (ms *MemoryStorage) Append(entries []pb.Entry) error {
 	case uint64(len(ms.ents)) == offset:
 		ms.ents = append(ms.ents, entries...)
 	default:
-		raftLogger.Panicf("missing log entry [last: %d, append at: %d]",
+		getLogger().Panicf("missing log entry [last: %d, append at: %d]",
 			ms.lastIndex(), entries[0].Index)
 	}
 	return nil

test.sh

@@ -110,7 +110,7 @@ function integration_extra {
 
 function integration_pass {
   local pkgs=${USERPKG:-"./integration/..."}
-  run_for_module "tests" go_test "${pkgs}" "parallel" : -timeout="${TIMEOUT:-15m}" "-v" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@" || return $?
+  run_for_module "tests" go_test "${pkgs}" "parallel" : -timeout="${TIMEOUT:-15m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@" || return $?
   integration_extra "$@"
 }
 

tests/integration/clientv3/cluster_test.go

@@ -265,7 +265,7 @@ func TestMemberPromote(t *testing.T) {
 	// create and launch learner member based on the response of V3 Member Add API.
 	// (the response has information on peer urls of the existing members in cluster)
 	learnerMember := clus.MustNewMember(t, memberAddResp)
-	clus.Members = append(clus.Members, learnerMember)
+
 	if err := learnerMember.Launch(); err != nil {
 		t.Fatal(err)
 	}

tests/integration/cluster.go

@@ -36,11 +36,11 @@ import (
 	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
 	"go.etcd.io/etcd/client/v2"
 	"go.etcd.io/etcd/client/v3"
-	"go.etcd.io/etcd/pkg/v3/logutil"
 	"go.etcd.io/etcd/pkg/v3/testutil"
 	"go.etcd.io/etcd/pkg/v3/tlsutil"
 	"go.etcd.io/etcd/pkg/v3/transport"
 	"go.etcd.io/etcd/pkg/v3/types"
+	"go.etcd.io/etcd/raft/v3"
 	"go.etcd.io/etcd/server/v3/embed"
 	"go.etcd.io/etcd/server/v3/etcdserver"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp"
@@ -52,6 +52,8 @@ import (
 	"go.etcd.io/etcd/server/v3/etcdserver/api/v3lock"
 	lockpb "go.etcd.io/etcd/server/v3/etcdserver/api/v3lock/v3lockpb"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc"
+	"go.uber.org/zap/zapcore"
+	"go.uber.org/zap/zaptest"
 
 	"github.com/soheilhy/cmux"
 	"go.uber.org/zap"
@@ -169,6 +171,12 @@ type ClusterConfig struct {
 type cluster struct {
 	cfg           *ClusterConfig
 	Members       []*member
+	lastMemberNum int
+}
+
+func (c *cluster) generateMemberName() string {
+	c.lastMemberNum++
+	return fmt.Sprintf("m%v", c.lastMemberNum)
 }
 
 func schemeFromTLSInfo(tls *transport.TLSInfo) string {
@@ -294,7 +302,7 @@ func (c *cluster) HTTPMembers() []client.Member {
 func (c *cluster) mustNewMember(t testing.TB) *member {
 	m := mustNewMember(t,
 		memberConfig{
-			name:                        c.name(rand.Int()),
+			name:                        c.generateMemberName(),
 			authToken:                   c.cfg.AuthToken,
 			peerTLS:                     c.cfg.PeerTLS,
 			clientTLS:                   c.cfg.ClientTLS,
@@ -707,18 +715,28 @@ func mustNewMember(t testing.TB, mcfg memberConfig) *member {
 	m.WatchProgressNotifyInterval = mcfg.WatchProgressNotifyInterval
 
 	m.InitialCorruptCheck = true
+	m.WarningApplyDuration = embed.DefaultWarningApplyDuration
 
-	lcfg := logutil.DefaultZapLoggerConfig
+	level := zapcore.InfoLevel
-	m.LoggerConfig = &lcfg
-	m.LoggerConfig.OutputPaths = []string{"/dev/null"}
-	m.LoggerConfig.ErrorOutputPaths = []string{"/dev/null"}
 	if os.Getenv("CLUSTER_DEBUG") != "" {
-		m.LoggerConfig.OutputPaths = []string{"stderr"}
+		level = zapcore.DebugLevel
-		m.LoggerConfig.ErrorOutputPaths = []string{"stderr"}
 	}
-	m.Logger, err = m.LoggerConfig.Build()
+
+	if t != nil {
+		options := zaptest.WrapOptions(zap.Fields(zap.String("member", mcfg.name)))
+		m.Logger = zaptest.NewLogger(t, zaptest.Level(level), options)
+		if t != nil {
+			t.Cleanup(func() {
+				// if we didn't cleanup the logger, the consecutive test
+				// might reuse this (t).
+				raft.ResetDefaultLogger()
+			})
+		}
+	} else {
+		m.Logger, err = zap.NewDevelopment(zap.IncreaseLevel(level))
 		if err != nil {
-		t.Fatal(err)
+			log.Panic(err)
+		}
 	}
 	return m
 }
@@ -1518,6 +1536,6 @@ func (c *ClusterV3) MustNewMember(t testing.TB, resp *clientv3.MemberAddResponse
 		m.InitialPeerURLsMap[mm.Name] = mm.PeerURLs
 	}
 	m.InitialPeerURLsMap[m.Name] = types.MustNewURLs(resp.Member.PeerURLs)
-
+	c.Members = append(c.Members, m)
 	return m
 }