Merge pull request #12943 from ptabor/20210430-v2-deprecation-flags

--v2-deprecation flag:  opt-in mode to validate that store-v2 has no user-content

server/config/config.go

@@ -192,6 +192,9 @@ type ServerConfig struct {
 	// ExperimentalBootstrapDefragThresholdMegabytes is the minimum number of megabytes needed to be freed for etcd server to
 	// consider running defrag during bootstrap. Needs to be set to non-zero value to take effect.
 	ExperimentalBootstrapDefragThresholdMegabytes uint `json:"experimental-bootstrap-defrag-threshold-megabytes"`
+
+	// V2Deprecation defines a phase of v2store deprecation process.
+	V2Deprecation V2DeprecationEnum `json:"v2-deprecation"`
 }
 
 // VerifyBootstrap sanity-checks the initial config for bootstrap case

server/config/v2_deprecation.go

@@ -0,0 +1,50 @@
+// Copyright 2021 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+type V2DeprecationEnum string
+
+const (
+	// Default in v3.5.  Issues a warning if v2store have meaningful content.
+	V2_DEPR_0_NOT_YET = V2DeprecationEnum("not-yet")
+	// Default in v3.6.  Meaningful v2 state is not allowed.
+	// The V2 files are maintained for v3.5 rollback.
+	V2_DEPR_1_WRITE_ONLY = V2DeprecationEnum("write-only")
+	// V2store is WIPED if found !!!
+	V2_DEPR_1_WRITE_ONLY_DROP = V2DeprecationEnum("write-only-drop-data")
+	// V2store is neither written nor read. Usage of this configuration is blocking
+	// ability to rollback to etcd v3.5.
+	V2_DEPR_2_GONE = V2DeprecationEnum("gone")
+
+	V2_DEPR_DEFAULT = V2_DEPR_0_NOT_YET
+)
+
+func (e V2DeprecationEnum) IsAtLeast(v2d V2DeprecationEnum) bool {
+	return e.level() >= v2d.level()
+}
+
+func (e V2DeprecationEnum) level() int {
+	switch e {
+	case V2_DEPR_0_NOT_YET:
+		return 0
+	case V2_DEPR_1_WRITE_ONLY:
+		return 1
+	case V2_DEPR_1_WRITE_ONLY_DROP:
+		return 2
+	case V2_DEPR_2_GONE:
+		return 3
+	}
+	panic("Unknown V2DeprecationEnum: " + e)
+}

server/config/v2_deprecation_test.go

@@ -0,0 +1,41 @@
+// Copyright 2021 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import "testing"
+
+func TestV2DeprecationEnum_IsAtLeast(t *testing.T) {
+	tests := []struct {
+		e    V2DeprecationEnum
+		v2d  V2DeprecationEnum
+		want bool
+	}{
+		{V2_DEPR_0_NOT_YET, V2_DEPR_0_NOT_YET, true},
+		{V2_DEPR_0_NOT_YET, V2_DEPR_1_WRITE_ONLY_DROP, false},
+		{V2_DEPR_0_NOT_YET, V2_DEPR_2_GONE, false},
+		{V2_DEPR_2_GONE, V2_DEPR_1_WRITE_ONLY_DROP, true},
+		{V2_DEPR_2_GONE, V2_DEPR_0_NOT_YET, true},
+		{V2_DEPR_2_GONE, V2_DEPR_2_GONE, true},
+		{V2_DEPR_1_WRITE_ONLY, V2_DEPR_1_WRITE_ONLY_DROP, false},
+		{V2_DEPR_1_WRITE_ONLY_DROP, V2_DEPR_1_WRITE_ONLY, true},
+	}
+	for _, tt := range tests {
+		t.Run(string(tt.e)+" >= "+string(tt.v2d), func(t *testing.T) {
+			if got := tt.e.IsAtLeast(tt.v2d); got != tt.want {
+				t.Errorf("IsAtLeast() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

server/embed/config.go

@@ -33,6 +33,7 @@ import (
 	"go.etcd.io/etcd/client/pkg/v3/types"
 	"go.etcd.io/etcd/pkg/v3/flags"
 	"go.etcd.io/etcd/pkg/v3/netutil"
+	"go.etcd.io/etcd/server/v3/config"
 	"go.etcd.io/etcd/server/v3/etcdserver"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/v3compactor"
 
@@ -400,6 +401,9 @@ type Config struct {
 
 	// ExperimentalTxnModeWriteWithSharedBuffer enables write transaction to use a shared buffer in its readonly check operations.
 	ExperimentalTxnModeWriteWithSharedBuffer bool `json:"experimental-txn-mode-write-with-shared-buffer"`
+
+	// V2Deprecation describes phase of API & Storage V2 support
+	V2Deprecation config.V2DeprecationEnum `json:"v2-deprecation"`
 }
 
 // configYAML holds the config suitable for yaml parsing
@@ -494,6 +498,8 @@ func NewConfig() *Config {
 		ExperimentalDowngradeCheckTime:           DefaultDowngradeCheckTime,
 		ExperimentalMemoryMlock:                  false,
 		ExperimentalTxnModeWriteWithSharedBuffer: true,
+
+		V2Deprecation: config.V2_DEPR_DEFAULT,
 	}
 	cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name)
 	return cfg
@@ -795,6 +801,13 @@ func (cfg Config) InitialClusterFromName(name string) (ret string) {
 func (cfg Config) IsNewCluster() bool { return cfg.ClusterState == ClusterStateFlagNew }
 func (cfg Config) ElectionTicks() int { return int(cfg.ElectionMs / cfg.TickMs) }
 
+func (cfg Config) V2DeprecationEffective() config.V2DeprecationEnum {
+	if cfg.V2Deprecation == "" {
+		return config.V2_DEPR_DEFAULT
+	}
+	return cfg.V2Deprecation
+}
+
 func (cfg Config) defaultPeerHost() bool {
 	return len(cfg.APUrls) == 1 && cfg.APUrls[0].String() == DefaultInitialAdvertisePeerURLs
 }

server/embed/etcd.go

@@ -226,6 +226,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
 		ExperimentalMemoryMlock:                  cfg.ExperimentalMemoryMlock,
 		ExperimentalTxnModeWriteWithSharedBuffer: cfg.ExperimentalTxnModeWriteWithSharedBuffer,
 		ExperimentalBootstrapDefragThresholdMegabytes: cfg.ExperimentalBootstrapDefragThresholdMegabytes,
+		V2Deprecation: cfg.V2DeprecationEffective(),
 	}
 
 	if srvcfg.ExperimentalEnableDistributedTracing {
@@ -696,6 +697,9 @@ func (e *Etcd) serveClients() (err error) {
 	// Start a client server goroutine for each listen address
 	var h http.Handler
 	if e.Config().EnableV2 {
+		if e.Config().V2DeprecationEffective().IsAtLeast(config.V2_DEPR_1_WRITE_ONLY) {
+			return fmt.Errorf("--enable-v2 and --v2-deprecation=%s are mutually exclusive", e.Config().V2DeprecationEffective())
+		}
 		e.cfg.logger.Warn("Flag `enable-v2` is deprecated and will get removed in etcd 3.6.")
 		if len(e.Config().ExperimentalEnableV2V3) > 0 {
 			e.cfg.logger.Warn("Flag `experimental-enable-v2v3` is deprecated and will get removed in etcd 3.6.")

server/etcdmain/config.go

@@ -27,6 +27,7 @@ import (
 	"go.etcd.io/etcd/api/v3/version"
 	"go.etcd.io/etcd/client/pkg/v3/logutil"
 	"go.etcd.io/etcd/pkg/v3/flags"
+	cconfig "go.etcd.io/etcd/server/v3/config"
 	"go.etcd.io/etcd/server/v3/embed"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp"
 
@@ -86,10 +87,11 @@ type config struct {
 
 // configFlags has the set of flags used for command line parsing a Config
 type configFlags struct {
-	flagSet      *flag.FlagSet
-	clusterState *flags.SelectiveStringValue
-	fallback     *flags.SelectiveStringValue
-	proxy        *flags.SelectiveStringValue
+	flagSet       *flag.FlagSet
+	clusterState  *flags.SelectiveStringValue
+	fallback      *flags.SelectiveStringValue
+	proxy         *flags.SelectiveStringValue
+	v2deprecation *flags.SelectiveStringsValue
 }
 
 func newConfig() *config {
@@ -119,6 +121,11 @@ func newConfig() *config {
 			proxyFlagReadonly,
 			proxyFlagOn,
 		),
+		v2deprecation: flags.NewSelectiveStringsValue(
+			string(cconfig.V2_DEPR_0_NOT_YET),
+			string(cconfig.V2_DEPR_1_WRITE_ONLY),
+			string(cconfig.V2_DEPR_1_WRITE_ONLY_DROP),
+			string(cconfig.V2_DEPR_2_GONE)),
 	}
 
 	fs := cfg.cf.flagSet
@@ -190,9 +197,13 @@ func newConfig() *config {
 	fs.Var(cfg.cf.clusterState, "initial-cluster-state", "Initial cluster state ('new' or 'existing').")
 
 	fs.BoolVar(&cfg.ec.StrictReconfigCheck, "strict-reconfig-check", cfg.ec.StrictReconfigCheck, "Reject reconfiguration requests that would cause quorum loss.")
-	fs.BoolVar(&cfg.ec.EnableV2, "enable-v2", cfg.ec.EnableV2, "Accept etcd V2 client requests. Deprecated in v3.5. Will be decommission in v3.6.")
+
 	fs.BoolVar(&cfg.ec.PreVote, "pre-vote", cfg.ec.PreVote, "Enable to run an additional Raft election phase.")
 
+	fs.BoolVar(&cfg.ec.EnableV2, "enable-v2", cfg.ec.EnableV2, "Accept etcd V2 client requests. Deprecated in v3.5. Will be decommission in v3.6.")
+	fs.StringVar(&cfg.ec.ExperimentalEnableV2V3, "experimental-enable-v2v3", cfg.ec.ExperimentalEnableV2V3, "v3 prefix for serving emulated v2 state. Deprecated in 3.5. Will be decomissioned in 3.6.")
+	fs.Var(cfg.cf.v2deprecation, "v2-deprecation", fmt.Sprintf("v2store deprecation stage: %q. ", cfg.cf.proxy.Valids()))
+
 	// proxy
 	fs.Var(cfg.cf.proxy, "proxy", fmt.Sprintf("Valid values include %q", cfg.cf.proxy.Valids()))
 	fs.UintVar(&cfg.cp.ProxyFailureWaitMs, "proxy-failure-wait", cfg.cp.ProxyFailureWaitMs, "Time (in milliseconds) an endpoint will be held in a failed state.")
@@ -268,7 +279,7 @@ func newConfig() *config {
 	// experimental
 	fs.BoolVar(&cfg.ec.ExperimentalInitialCorruptCheck, "experimental-initial-corrupt-check", cfg.ec.ExperimentalInitialCorruptCheck, "Enable to check data corruption before serving any client/peer traffic.")
 	fs.DurationVar(&cfg.ec.ExperimentalCorruptCheckTime, "experimental-corrupt-check-time", cfg.ec.ExperimentalCorruptCheckTime, "Duration of time between cluster corruption check passes.")
-	fs.StringVar(&cfg.ec.ExperimentalEnableV2V3, "experimental-enable-v2v3", cfg.ec.ExperimentalEnableV2V3, "v3 prefix for serving emulated v2 state. Deprecated in 3.5. Will be decomissioned in 3.6.")
+
 	fs.BoolVar(&cfg.ec.ExperimentalEnableLeaseCheckpoint, "experimental-enable-lease-checkpoint", false, "Enable to persist lease remaining TTL to prevent indefinite auto-renewal of long lived leases.")
 	fs.IntVar(&cfg.ec.ExperimentalCompactionBatchLimit, "experimental-compaction-batch-limit", cfg.ec.ExperimentalCompactionBatchLimit, "Sets the maximum revisions deleted in each compaction batch.")
 	fs.DurationVar(&cfg.ec.ExperimentalWatchProgressNotifyInterval, "experimental-watch-progress-notify-interval", cfg.ec.ExperimentalWatchProgressNotifyInterval, "Duration of periodic watch progress notifications.")
@@ -331,6 +342,11 @@ func (cfg *config) parse(arguments []string) error {
 	} else {
 		err = cfg.configFromCmdLine()
 	}
+
+	if cfg.ec.V2Deprecation == "" {
+		cfg.ec.V2Deprecation = cconfig.V2_DEPR_DEFAULT
+	}
+
 	// now logger is set up
 	return err
 }
@@ -385,6 +401,8 @@ func (cfg *config) configFromCmdLine() error {
 	cfg.cp.Fallback = cfg.cf.fallback.String()
 	cfg.cp.Proxy = cfg.cf.proxy.String()
 
+	cfg.ec.V2Deprecation = cconfig.V2DeprecationEnum(cfg.cf.v2deprecation.String())
+
 	// disable default advertise-client-urls if lcurls is set
 	missingAC := flags.IsSet(cfg.cf.flagSet, "listen-client-urls") && !flags.IsSet(cfg.cf.flagSet, "advertise-client-urls")
 	if !cfg.mayBeProxy() && missingAC {

server/etcdmain/help.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"strconv"
 
+	cconfig "go.etcd.io/etcd/server/v3/config"
 	"go.etcd.io/etcd/server/v3/embed"
 	"golang.org/x/crypto/bcrypt"
 )
@@ -124,6 +125,13 @@ Clustering:
     Interpret 'auto-compaction-retention' one of: periodic|revision. 'periodic' for duration based retention, defaulting to hours if no time unit is provided (e.g. '5m'). 'revision' for revision number based retention.
   --enable-v2 '` + strconv.FormatBool(embed.DefaultEnableV2) + `'
     Accept etcd V2 client requests. Deprecated and to be decommissioned in v3.6.
+  --v2-deprecation '` + string(cconfig.V2_DEPR_DEFAULT) + `'
+    Phase of v2store deprecation. Allows to opt-in for higher compatibility mode.
+    Supported values:
+      'not-yet'                // Issues a warning if v2store have meaningful content (default in v3.5)
+      'write-only'             // Custom v2 state is not allowed (planned default in v3.6)
+      'write-only-drop-data'   // Custom v2 state will get DELETED !
+      'gone'                   // v2store is not maintained any longer. (planned default in v3.7)
 
 Security:
   --cert-file ''

server/etcdserver/api/membership/store.go

@@ -31,7 +31,7 @@ const (
 	attributesSuffix     = "attributes"
 	raftAttributesSuffix = "raftAttributes"
 
-	// the prefix for stroing membership related information in store provided by store pkg.
+	// the prefix for storing membership related information in store provided by store pkg.
 	storePrefix = "/0"
 )
 

server/etcdserver/api/membership/storev2.go

@@ -0,0 +1,36 @@
+// Copyright 2021 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package membership
+
+import (
+	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
+)
+
+// IsMetaStoreOnly verifies if the given `store` contains only
+// a meta-information (members, version) that can be recovered from the
+// backend (storev3) as well as opposed to user-data.
+func IsMetaStoreOnly(store v2store.Store) (bool, error) {
+	event, err := store.Get("/", true, false)
+	if err != nil {
+		return false, err
+	}
+	for _, n := range event.Node.Nodes {
+		if n.Key != storePrefix && n.Nodes.Len() > 0 {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}

server/etcdserver/api/membership/storev2_test.go

@@ -0,0 +1,54 @@
+// Copyright 2021 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package membership
+
+import (
+	"testing"
+
+	"github.com/coreos/go-semver/semver"
+	"github.com/stretchr/testify/assert"
+	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
+	"go.uber.org/zap/zaptest"
+)
+
+func TestIsMetaStoreOnly(t *testing.T) {
+	lg := zaptest.NewLogger(t)
+	s := v2store.New("/0", "/1")
+
+	metaOnly, err := IsMetaStoreOnly(s)
+	assert.NoError(t, err)
+	assert.True(t, metaOnly, "Just created v2store should be meta-only")
+
+	mustSaveClusterVersionToStore(lg, s, semver.New("3.5.17"))
+	metaOnly, err = IsMetaStoreOnly(s)
+	assert.NoError(t, err)
+	assert.True(t, metaOnly, "Just created v2store should be meta-only")
+
+	mustSaveMemberToStore(lg, s, &Member{ID: 0x00abcd})
+	metaOnly, err = IsMetaStoreOnly(s)
+	assert.NoError(t, err)
+	assert.True(t, metaOnly, "Just created v2store should be meta-only")
+
+	_, err = s.Create("/1/foo", false, "v1", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent})
+	assert.NoError(t, err)
+	metaOnly, err = IsMetaStoreOnly(s)
+	assert.NoError(t, err)
+	assert.False(t, metaOnly, "Just created v2store should be meta-only")
+
+	_, err = s.Delete("/1/foo", false, false)
+	assert.NoError(t, err)
+	assert.NoError(t, err)
+	assert.False(t, metaOnly, "Just created v2store should be meta-only")
+}

server/etcdserver/server.go

@@ -479,6 +479,11 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) {
 				cfg.Logger.Panic("failed to recover from snapshot", zap.Error(err))
 			}
 
+			if err = assertNoV2StoreContent(cfg.Logger, st, cfg.V2Deprecation); err != nil {
+				cfg.Logger.Error("illegal v2store content", zap.Error(err))
+				return nil, err
+			}
+
 			cfg.Logger.Info(
 				"recovered v2 store from snapshot",
 				zap.Uint64("snapshot-index", snapshot.Metadata.Index),
@@ -496,6 +501,8 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) {
 				zap.Int64("backend-size-in-use-bytes", s2),
 				zap.String("backend-size-in-use", humanize.Bytes(uint64(s2))),
 			)
+		} else {
+			cfg.Logger.Info("No snapshot found. Recovering WAL from scratch!")
 		}
 
 		if !cfg.ForceNewCluster {
@@ -662,6 +669,23 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) {
 	return srv, nil
 }
 
+// assertNoV2StoreContent -> depending on the deprecation stage, warns or report an error
+// if the v2store contains custom content.
+func assertNoV2StoreContent(lg *zap.Logger, st v2store.Store, deprecationStage config.V2DeprecationEnum) error {
+	metaOnly, err := membership.IsMetaStoreOnly(st)
+	if err != nil {
+		return err
+	}
+	if metaOnly {
+		return nil
+	}
+	if deprecationStage.IsAtLeast(config.V2_DEPR_1_WRITE_ONLY) {
+		return fmt.Errorf("detected disallowed custom content in v2store for stage --v2-deprecation=%s", deprecationStage)
+	}
+	lg.Warn("detected custom v2store content. Etcd v3.5 is the last version allowing to access it using API v2. Please remove the content.")
+	return nil
+}
+
 func (s *EtcdServer) Logger() *zap.Logger {
 	s.lgMu.RLock()
 	l := s.lg
@@ -1252,6 +1276,10 @@ func (s *EtcdServer) applySnapshot(ep *etcdProgress, apply *apply) {
 		lg.Panic("failed to restore v2 store", zap.Error(err))
 	}
 
+	if err := assertNoV2StoreContent(lg, s.v2store, s.Cfg.V2Deprecation); err != nil {
+		lg.Panic("illegal v2store content", zap.Error(err))
+	}
+
 	lg.Info("restored v2 store")
 
 	s.cluster.SetBackend(newbe)

tests/e2e/cluster_proxy_test.go

@@ -22,6 +22,7 @@ import (
 	"net"
 	"net/url"
 	"os"
+	"path"
 	"strconv"
 	"strings"
 
@@ -268,10 +269,10 @@ func newProxyV3Proc(cfg *etcdServerProcessConfig) *proxyV3Proc {
 		// Configure certificates for connection proxy ---> server.
 		// This certificate must NOT have CN set.
 		tlsArgs = append(tlsArgs,
-			"--cert", "../fixtures/client-nocn.crt",
-			"--key", "../fixtures/client-nocn.key.insecure",
-			"--cacert", "../fixtures/ca.crt",
-			"--client-crl-file", "../fixtures/revoke.crl")
+			"--cert", path.Join(fixturesDir, "client-nocn.crt"),
+			"--key", path.Join(fixturesDir, "client-nocn.key.insecure"),
+			"--cacert", path.Join(fixturesDir, "ca.crt"),
+			"--client-crl-file", path.Join(fixturesDir, "revoke.crl"))
 	}
 	return &proxyV3Proc{
 		proxyProc{

tests/e2e/cluster_test.go

@@ -18,17 +18,23 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"path"
 	"strings"
 	"testing"
 	"time"
 
 	"go.etcd.io/etcd/server/v3/etcdserver"
+	"go.etcd.io/etcd/tests/v3/integration"
 )
 
 const etcdProcessBasePort = 20000
 
 type clientConnType int
 
+var (
+	fixturesDir = integration.MustAbsPath("../fixtures")
+)
+
 const (
 	clientNonTLS clientConnType = iota
 	clientTLS
@@ -113,9 +119,10 @@ func newConfigClientTLSCertAuthWithNoCN() *etcdProcessClusterConfig {
 
 func newConfigJWT() *etcdProcessClusterConfig {
 	return &etcdProcessClusterConfig{
-		clusterSize:   1,
-		initialToken:  "new",
-		authTokenOpts: "jwt,pub-key=../fixtures/server.crt,priv-key=../fixtures/server.key.insecure,sign-method=RS256,ttl=1s",
+		clusterSize:  1,
+		initialToken: "new",
+		authTokenOpts: "jwt,pub-key=" + path.Join(fixturesDir, "server.crt") +
+			",priv-key=" + path.Join(fixturesDir, "server.key.insecure") + ",sign-method=RS256,ttl=1s",
 	}
 }
 
@@ -161,6 +168,7 @@ type etcdProcessClusterConfig struct {
 	enableV2            bool
 	initialCorruptCheck bool
 	authTokenOpts       string
+	v2deprecation       string
 
 	rollingStart bool
 }
@@ -246,7 +254,7 @@ func (cfg *etcdProcessClusterConfig) etcdServerProcessConfigs(tb testing.TB) []*
 		}
 
 		purl := url.URL{Scheme: cfg.peerScheme(), Host: fmt.Sprintf("localhost:%d", port+1)}
-		name := fmt.Sprintf("test-%s-%d", tb.Name(), i)
+		name := fmt.Sprintf("test-%d", i)
 		dataDirPath := cfg.dataDirPath
 		if cfg.dataDirPath == "" {
 			dataDirPath = tb.TempDir()
@@ -296,6 +304,10 @@ func (cfg *etcdProcessClusterConfig) etcdServerProcessConfigs(tb testing.TB) []*
 			args = append(args, "--auth-token", cfg.authTokenOpts)
 		}
 
+		if cfg.v2deprecation != "" {
+			args = append(args, "--v2-deprecation", cfg.v2deprecation)
+		}
+
 		etcdCfgs[i] = &etcdServerProcessConfig{
 			execPath:     cfg.execPath,
 			args:         args,

tests/e2e/ctl_v3_migrate_test.go

@@ -21,17 +21,9 @@ import (
 	"testing"
 	"time"
 
-	"go.etcd.io/etcd/client/pkg/v3/testutil"
 	"go.etcd.io/etcd/client/v3"
-	"go.etcd.io/etcd/server/v3/verify"
 )
 
-func BeforeTest(t testing.TB) {
-	skipInShortMode(t)
-	testutil.BeforeTest(t)
-	os.Setenv(verify.ENV_VERIFY, verify.ENV_VERIFY_ALL_VALUE)
-}
-
 func TestCtlV3Migrate(t *testing.T) {
 	BeforeTest(t)
 

tests/e2e/ctl_v3_test.go

@@ -30,7 +30,7 @@ import (
 func TestCtlV3Version(t *testing.T) { testCtl(t, versionTest) }
 
 func TestClusterVersion(t *testing.T) {
-	skipInShortMode(t)
+	BeforeTest(t)
 
 	tests := []struct {
 		name         string

tests/e2e/main_test.go

@@ -6,13 +6,12 @@ package e2e
 
 import (
 	"flag"
-	"log"
 	"os"
-	"path/filepath"
 	"runtime"
 	"testing"
 
 	"go.etcd.io/etcd/client/pkg/v3/testutil"
+	"go.etcd.io/etcd/tests/v3/integration"
 )
 
 var (
@@ -38,15 +37,8 @@ func TestMain(m *testing.M) {
 	os.Setenv("ETCD_UNSUPPORTED_ARCH", runtime.GOARCH)
 	os.Unsetenv("ETCDCTL_API")
 
-	binDirDef, err := filepath.Abs("../../bin")
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	certDirDef, err := filepath.Abs("../fixtures")
-	if err != nil {
-		log.Fatal(err)
-	}
+	binDirDef := integration.MustAbsPath("../../bin")
+	certDirDef := fixturesDir
 
 	flag.StringVar(&binDir, "bin-dir", binDirDef, "The directory for store etcd and etcdctl binaries.")
 	flag.StringVar(&certDir, "cert-dir", certDirDef, "The directory for store certificate files.")

tests/e2e/testing.go

@@ -0,0 +1,38 @@
+// Copyright 2021 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"go.etcd.io/etcd/client/pkg/v3/testutil"
+	"go.etcd.io/etcd/server/v3/verify"
+)
+
+func BeforeTest(t testing.TB) {
+	skipInShortMode(t)
+	testutil.BeforeTest(t)
+	os.Setenv(verify.ENV_VERIFY, verify.ENV_VERIFY_ALL_VALUE)
+
+	path, err := os.Getwd()
+	assert.NoError(t, err)
+	tempDir := t.TempDir()
+	assert.NoError(t, os.Chdir(tempDir))
+	t.Logf("Changing working directory to: %s", tempDir)
+
+	t.Cleanup(func() { assert.NoError(t, os.Chdir(path)) })
+}

tests/e2e/v2store_deprecation_test.go

@@ -0,0 +1,98 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func createV2store(t testing.TB, dataDirPath string) {
+	t.Log("Creating not-yet v2-deprecated etcd")
+
+	cfg := configStandalone(etcdProcessClusterConfig{enableV2: true, dataDirPath: dataDirPath, snapshotCount: 5})
+	epc, err := newEtcdProcessCluster(t, cfg)
+	assert.NoError(t, err)
+
+	defer func() {
+		assert.NoError(t, epc.Stop())
+	}()
+
+	// We need to exceed 'snapshotCount' such that v2 snapshot is dumped.
+	for i := 0; i < 10; i++ {
+		if err := cURLPut(epc, cURLReq{
+			endpoint: "/v2/keys/foo", value: "bar" + fmt.Sprint(i),
+			expected: `{"action":"set","node":{"key":"/foo","value":"bar` + fmt.Sprint(i)}); err != nil {
+			t.Fatalf("failed put with curl (%v)", err)
+		}
+	}
+}
+
+func assertVerifyCanStartV2deprecationNotYet(t testing.TB, dataDirPath string) {
+	t.Log("verify: possible to start etcd with --v2-deprecation=not-yet mode")
+
+	cfg := configStandalone(etcdProcessClusterConfig{enableV2: true, dataDirPath: dataDirPath, v2deprecation: "not-yet", keepDataDir: true})
+	epc, err := newEtcdProcessCluster(t, cfg)
+	assert.NoError(t, err)
+
+	defer func() {
+		assert.NoError(t, epc.Stop())
+	}()
+
+	if err := cURLGet(epc, cURLReq{
+		endpoint: "/v2/keys/foo",
+		expected: `{"action":"get","node":{"key":"/foo","value":"bar9","modifiedIndex":13,"createdIndex":13}}`}); err != nil {
+		t.Fatalf("failed get with curl (%v)", err)
+	}
+
+}
+
+func assertVerifyCannotStartV2deprecationWriteOnly(t testing.TB, dataDirPath string) {
+	t.Log("Verify its infeasible to start etcd with --v2-deprecation=write-only mode")
+	proc, err := spawnCmd([]string{binDir + "/etcd", "--v2-deprecation=write-only", "--data-dir=" + dataDirPath})
+	assert.NoError(t, err)
+
+	_, err = proc.Expect("detected disallowed custom content in v2store for stage --v2-deprecation=write-only")
+	assert.NoError(t, err)
+}
+
+func TestV2Deprecation(t *testing.T) {
+	BeforeTest(t)
+	dataDirPath := t.TempDir()
+
+	t.Run("create-storev2-data", func(t *testing.T) {
+		createV2store(t, dataDirPath)
+	})
+
+	t.Run("--v2-deprecation=write-only fails", func(t *testing.T) {
+		assertVerifyCannotStartV2deprecationWriteOnly(t, dataDirPath)
+	})
+
+	t.Run("--v2-deprecation=not-yet succeeds", func(t *testing.T) {
+		assertVerifyCanStartV2deprecationNotYet(t, dataDirPath)
+	})
+
+}
+
+func TestV2DeprecationWriteOnlyNoV2Api(t *testing.T) {
+	BeforeTest(t)
+	proc, err := spawnCmd([]string{binDir + "/etcd", "--v2-deprecation=write-only", "--enable-v2"})
+	assert.NoError(t, err)
+
+	_, err = proc.Expect("--enable-v2 and --v2-deprecation=write-only are mutually exclusive")
+	assert.NoError(t, err)
+}

tests/integration/cluster.go

@@ -706,6 +706,8 @@ func mustNewMember(t testutil.TB, mcfg memberConfig) *member {
 	m.InitialCorruptCheck = true
 	m.WarningApplyDuration = embed.DefaultWarningApplyDuration
 
+	m.V2Deprecation = config.V2_DEPR_DEFAULT
+
 	m.Logger = memberLogger(t, mcfg.name)
 	t.Cleanup(func() {
 		// if we didn't cleanup the logger, the consecutive test