Merge pull request #5738 from heyitsanthony/fp

build with failpoints

build

@@ -1,22 +1,44 @@
 #!/bin/sh -e
 
+# set some environment variables
 ORG_PATH="github.com/coreos"
 REPO_PATH="${ORG_PATH}/etcd"
-
 export GO15VENDOREXPERIMENT="1"
-
 eval $(go env)
-
 GIT_SHA=`git rev-parse --short HEAD || echo "GitNotFound"`
-
-LINK_OPERATOR="="
-
-if [ -z "${GOARCH}" ] || [ "${GOARCH}" = "$(go env GOHOSTARCH)" ]; then
-	out="bin"
-else
-	out="bin/${GOARCH}"
+if [ ! -z "$FAILPOINTS" ]; then
+	GIT_SHA="$GIT_SHA"-FAILPOINTS
 fi
 
-# Static compilation is useful when etcd is run in a container
-CGO_ENABLED=0 go build $GO_BUILD_FLAGS -installsuffix cgo -ldflags "-s -X ${REPO_PATH}/cmd/vendor/${REPO_PATH}/version.GitSHA${LINK_OPERATOR}${GIT_SHA}" -o ${out}/etcd ${REPO_PATH}/cmd
-CGO_ENABLED=0 go build $GO_BUILD_FLAGS -installsuffix cgo -ldflags "-s" -o ${out}/etcdctl ${REPO_PATH}/cmd/etcdctl
+# enable/disable failpoints
+toggle_failpoints() {
+	FAILPKGS="etcdserver/"
+
+	mode="disable"
+	if [ ! -z "$FAILPOINTS" ]; then mode="enable"; fi
+	if [ ! -z "$1" ]; then mode="$1"; fi
+
+	if which gofail >/dev/null 2>&1; then
+		gofail "$mode" $FAILPKGS
+	elif [ "$mode" != "disable" ]; then
+		echo "FAILPOINTS set but gofail not found"
+		exit 1
+	fi
+}
+
+etcd_build() {
+	if [ -z "${GOARCH}" ] || [ "${GOARCH}" = "$(go env GOHOSTARCH)" ]; then
+		out="bin"
+	else
+		out="bin/${GOARCH}"
+	fi
+	toggle_failpoints
+	# Static compilation is useful when etcd is run in a container
+	CGO_ENABLED=0 go build $GO_BUILD_FLAGS -installsuffix cgo -ldflags "-s -X ${REPO_PATH}/cmd/vendor/${REPO_PATH}/version.GitSHA=${GIT_SHA}" -o ${out}/etcd ${REPO_PATH}/cmd
+	CGO_ENABLED=0 go build $GO_BUILD_FLAGS -installsuffix cgo -ldflags "-s" -o ${out}/etcdctl ${REPO_PATH}/cmd/etcdctl
+}
+
+toggle_failpoints
+
+# don't build when sourced
+(echo "$0" | grep "/build$") && etcd_build || true

etcdserver/raft.go

@@ -213,27 +213,34 @@ func (r *raftNode) start(s *EtcdServer) {
 				// writing to their disks.
 				// For more details, check raft thesis 10.2.1
 				if islead {
+					// gofail: var raftBeforeLeaderSend struct{}
 					r.s.send(rd.Messages)
 				}
 
+				// gofail: var raftBeforeSave struct{}
 				if err := r.storage.Save(rd.HardState, rd.Entries); err != nil {
 					plog.Fatalf("raft save state and entries error: %v", err)
 				}
 				if !raft.IsEmptyHardState(rd.HardState) {
 					proposalsCommitted.Set(float64(rd.HardState.Commit))
 				}
+				// gofail: var raftAfterSave struct{}
 
 				if !raft.IsEmptySnap(rd.Snapshot) {
+					// gofail: var raftBeforeSaveSnap struct{}
 					if err := r.storage.SaveSnap(rd.Snapshot); err != nil {
 						plog.Fatalf("raft save snapshot error: %v", err)
 					}
+					// gofail: var raftAfterSaveSnap struct{}
 					r.raftStorage.ApplySnapshot(rd.Snapshot)
 					plog.Infof("raft applied incoming snapshot at index %d", rd.Snapshot.Metadata.Index)
+					// gofail: var raftAfterApplySnap struct{}
 				}
 
 				r.raftStorage.Append(rd.Entries)
 
 				if !islead {
+					// gofail: var raftBeforeFollowerSend struct{}
 					r.s.send(rd.Messages)
 				}
 				raftDone <- struct{}{}

test

@@ -10,6 +10,8 @@
 # PKG=snap ./test
 set -e
 
+source ./build
+
 # TODO: 'client' pkg fails with gosimple from generated files
 # TODO: 'rafttest' is failing with unused
 GOSIMPLE_UNUSED_PATHS=$(go list ./... | sed -e 's/github.com\/coreos\/etcd\///g' | grep -vE 'cmd|vendor|rafttest|github.com/coreos/etcd$|client$')
@@ -17,15 +19,6 @@ GOSIMPLE_UNUSED_PATHS=$(go list ./... | sed -e 's/github.com\/coreos\/etcd\///g'
 # Invoke ./cover for HTML output
 COVER=${COVER:-"-cover"}
 
-GO_BUILD_FLAGS="-a -v"
-source ./build
-
-# Set up gopath so tests use vendored dependencies
-export GOPATH=${PWD}/gopath
-rm -rf $GOPATH/src
-mkdir -p $GOPATH
-ln -s ${PWD}/cmd/vendor $GOPATH/src
-
 # Hack: gofmt ./ will recursively check the .git directory. So use *.go for gofmt.
 IGNORE_PKGS="(cmd|vendor|etcdserverpb|rafttest)"
 INTEGRATION_PKGS="(integration|e2e|contrib|functional-tester)"
@@ -168,10 +161,20 @@ function dep_tests {
 	fi
 }
 
+# Set up gopath so tests use vendored dependencies
+export GOPATH=${PWD}/gopath
+rm -rf $GOPATH/src
+mkdir -p $GOPATH
+ln -s ${PWD}/cmd/vendor $GOPATH/src
+
 # fail fast on static tests
+toggle_failpoints disable
 fmt_tests
 dep_tests
 
+# fail fast on static tests
+GO_BUILD_FLAGS="-a -v" etcd_build
+
 unit_tests
 if [ -n "$INTEGRATION" ]; then
 	integration_tests