From fc5f94144b4416e9d5dea222efd22f21d5d74e58 Mon Sep 17 00:00:00 2001
From: Wenjia Zhang <wenjiazhang@google.com>
Date: Wed, 16 Oct 2019 13:27:07 -0700
Subject: [PATCH] Add cluster version fix #11233, #11254, #11265

---
 e2e/etcd_release_upgrade_test.go | 6 +-----
 etcdserver/membership/cluster.go | 8 ++++++--
 etcdserver/server.go             | 2 +-
 tests/e2e/metrics_test.go        | 4 ++--
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/e2e/etcd_release_upgrade_test.go b/e2e/etcd_release_upgrade_test.go
index 066feac6b..3ece2f8c7 100644
--- a/e2e/etcd_release_upgrade_test.go
+++ b/e2e/etcd_release_upgrade_test.go
@@ -103,11 +103,7 @@ func TestReleaseUpgrade(t *testing.T) {
 	}
 
 	// expect upgraded cluster version
-	ver := version.Version
-	if strings.HasSuffix(ver, "+git") {
-		ver = strings.Replace(ver, "+git", "", 1)
-	}
-	if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: fmt.Sprintf(`etcd_cluster_version{cluster_version="%s"} 1`, ver), metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil {
+	if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: fmt.Sprintf(`etcd_cluster_version{cluster_version="%s"} 1`, version.Cluster(version.Version)), metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil {
 		cx.t.Fatalf("failed get with curl (%v)", err)
 	}
 }
diff --git a/etcdserver/membership/cluster.go b/etcdserver/membership/cluster.go
index 8485b0081..8e1751530 100644
--- a/etcdserver/membership/cluster.go
+++ b/etcdserver/membership/cluster.go
@@ -369,6 +369,7 @@ func (c *RaftCluster) SetVersion(ver *semver.Version, onSet func(*semver.Version
 	} else {
 		plog.Noticef("set the initial cluster version to %v", version.Cluster(ver.String()))
 	}
+	oldVer := c.version
 	c.version = ver
 	mustDetectDowngrade(c.version)
 	if c.store != nil {
@@ -377,8 +378,11 @@ func (c *RaftCluster) SetVersion(ver *semver.Version, onSet func(*semver.Version
 	if c.be != nil {
 		mustSaveClusterVersionToBackend(c.be, ver)
 	}
-	ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": ver.String()}).Set(1)
-    onSet(ver)
+	if oldVer != nil {
+		ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(oldVer.String())}).Set(0)
+	}
+	ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(ver.String())}).Set(1)
+	onSet(ver)
 }
 
 func (c *RaftCluster) IsReadyToAddNewMember() bool {
diff --git a/etcdserver/server.go b/etcdserver/server.go
index 48e7baf7c..9d28bd61d 100644
--- a/etcdserver/server.go
+++ b/etcdserver/server.go
@@ -591,7 +591,7 @@ func (s *EtcdServer) start() {
 	s.readNotifier = newNotifier()
 	if s.ClusterVersion() != nil {
 		plog.Infof("starting server... [version: %v, cluster version: %v]", version.Version, version.Cluster(s.ClusterVersion().String()))
-		membership.ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": s.ClusterVersion().String()}).Set(1)
+		membership.ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(s.ClusterVersion().String())}).Set(1)
 	} else {
 		plog.Infof("starting server... [version: %v, cluster version: to_be_decided]", version.Version)
 	}
diff --git a/tests/e2e/metrics_test.go b/tests/e2e/metrics_test.go
index 7a32f0031..aade2718f 100644
--- a/tests/e2e/metrics_test.go
+++ b/tests/e2e/metrics_test.go
@@ -19,7 +19,7 @@ import (
 	"strings"
 	"testing"
 
-	"go.etcd.io/etcd/version"
+	"github.com/coreos/etcd/version"
 )
 
 func TestV3MetricsSecure(t *testing.T) {
@@ -50,7 +50,7 @@ func metricsTest(cx ctlCtx) {
 	if strings.HasSuffix(ver, "+git") {
 		ver = strings.Replace(ver, "+git", "", 1)
 	}
-	if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: fmt.Sprintf(`etcd_cluster_version{cluster_version="%s"} 1`, ver), metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil {
+	if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: fmt.Sprintf(`etcd_cluster_version{cluster_version="%s"} 1`, version.Cluster(version.Version)), metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil {
 		cx.t.Fatalf("failed get with curl (%v)", err)
 	}
 	if err := cURLGet(cx.epc, cURLReq{endpoint: "/health", expected: `{"health":"true"}`, metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil {