Merge pull request #18062 from stackbaek/main

featuregate: adds EtcdServer.FeatureEnabled interface.

pkg/featuregate/feature_gate.go

@@ -0,0 +1,411 @@
+// Copyright 2024 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package featuregate is copied from k8s.io/component-base@v0.30.1 to avoid any potential circular dependency between k8s and etcd.
+package featuregate
+
+import (
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+
+	"github.com/spf13/pflag"
+	"go.uber.org/zap"
+)
+
+type Feature string
+
+const (
+	flagName = "feature-gates"
+
+	// allAlphaGate is a global toggle for alpha features. Per-feature key
+	// values override the default set by allAlphaGate. Examples:
+	//   AllAlpha=false,NewFeature=true  will result in newFeature=true
+	//   AllAlpha=true,NewFeature=false  will result in newFeature=false
+	allAlphaGate Feature = "AllAlpha"
+
+	// allBetaGate is a global toggle for beta features. Per-feature key
+	// values override the default set by allBetaGate. Examples:
+	//   AllBeta=false,NewFeature=true  will result in NewFeature=true
+	//   AllBeta=true,NewFeature=false  will result in NewFeature=false
+	allBetaGate Feature = "AllBeta"
+)
+
+var (
+	// The generic features.
+	defaultFeatures = map[Feature]FeatureSpec{
+		allAlphaGate: {Default: false, PreRelease: Alpha},
+		allBetaGate:  {Default: false, PreRelease: Beta},
+	}
+
+	// Special handling for a few gates.
+	specialFeatures = map[Feature]func(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool){
+		allAlphaGate: setUnsetAlphaGates,
+		allBetaGate:  setUnsetBetaGates,
+	}
+)
+
+type FeatureSpec struct {
+	// Default is the default enablement state for the feature
+	Default bool
+	// LockToDefault indicates that the feature is locked to its default and cannot be changed
+	LockToDefault bool
+	// PreRelease indicates the maturity level of the feature
+	PreRelease prerelease
+}
+
+type prerelease string
+
+const (
+	// Values for PreRelease.
+	Alpha = prerelease("ALPHA")
+	Beta  = prerelease("BETA")
+	GA    = prerelease("")
+
+	// Deprecated
+	Deprecated = prerelease("DEPRECATED")
+)
+
+// FeatureGate indicates whether a given feature is enabled or not
+type FeatureGate interface {
+	// Enabled returns true if the key is enabled.
+	Enabled(key Feature) bool
+	// KnownFeatures returns a slice of strings describing the FeatureGate's known features.
+	KnownFeatures() []string
+	// DeepCopy returns a deep copy of the FeatureGate object, such that gates can be
+	// set on the copy without mutating the original. This is useful for validating
+	// config against potential feature gate changes before committing those changes.
+	DeepCopy() MutableFeatureGate
+}
+
+// MutableFeatureGate parses and stores flag gates for known features from
+// a string like feature1=true,feature2=false,...
+type MutableFeatureGate interface {
+	FeatureGate
+
+	// AddFlag adds a flag for setting global feature gates to the specified FlagSet.
+	AddFlag(fs *pflag.FlagSet)
+	// Set parses and stores flag gates for known features
+	// from a string like feature1=true,feature2=false,...
+	Set(value string) error
+	// SetFromMap stores flag gates for known features from a map[string]bool or returns an error
+	SetFromMap(m map[string]bool) error
+	// Add adds features to the featureGate.
+	Add(features map[Feature]FeatureSpec) error
+	// GetAll returns a copy of the map of known feature names to feature specs.
+	GetAll() map[Feature]FeatureSpec
+	// AddMetrics adds feature enablement metrics
+	AddMetrics()
+	// OverrideDefault sets a local override for the registered default value of a named
+	// feature. If the feature has not been previously registered (e.g. by a call to Add), has a
+	// locked default, or if the gate has already registered itself with a FlagSet, a non-nil
+	// error is returned.
+	//
+	// When two or more components consume a common feature, one component can override its
+	// default at runtime in order to adopt new defaults before or after the other
+	// components. For example, a new feature can be evaluated with a limited blast radius by
+	// overriding its default to true for a limited number of components without simultaneously
+	// changing its default for all consuming components.
+	OverrideDefault(name Feature, override bool) error
+}
+
+// featureGate implements FeatureGate as well as pflag.Value for flag parsing.
+type featureGate struct {
+	lg *zap.Logger
+
+	featureGateName string
+
+	special map[Feature]func(map[Feature]FeatureSpec, map[Feature]bool, bool)
+
+	// lock guards writes to known, enabled, and reads/writes of closed
+	lock sync.Mutex
+	// known holds a map[Feature]FeatureSpec
+	known atomic.Value
+	// enabled holds a map[Feature]bool
+	enabled atomic.Value
+	// closed is set to true when AddFlag is called, and prevents subsequent calls to Add
+	closed bool
+}
+
+func setUnsetAlphaGates(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool) {
+	for k, v := range known {
+		if v.PreRelease == Alpha {
+			if _, found := enabled[k]; !found {
+				enabled[k] = val
+			}
+		}
+	}
+}
+
+func setUnsetBetaGates(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool) {
+	for k, v := range known {
+		if v.PreRelease == Beta {
+			if _, found := enabled[k]; !found {
+				enabled[k] = val
+			}
+		}
+	}
+}
+
+// Set, String, and Type implement pflag.Value
+var _ pflag.Value = &featureGate{}
+
+func New(name string, lg *zap.Logger) *featureGate {
+	known := map[Feature]FeatureSpec{}
+	for k, v := range defaultFeatures {
+		known[k] = v
+	}
+
+	f := &featureGate{
+		lg:              lg,
+		featureGateName: name,
+		special:         specialFeatures,
+	}
+	f.known.Store(known)
+	f.enabled.Store(map[Feature]bool{})
+
+	return f
+}
+
+// Set parses a string of the form "key1=value1,key2=value2,..." into a
+// map[string]bool of known keys or returns an error.
+func (f *featureGate) Set(value string) error {
+	m := make(map[string]bool)
+	for _, s := range strings.Split(value, ",") {
+		if len(s) == 0 {
+			continue
+		}
+		arr := strings.SplitN(s, "=", 2)
+		k := strings.TrimSpace(arr[0])
+		if len(arr) != 2 {
+			return fmt.Errorf("missing bool value for %s", k)
+		}
+		v := strings.TrimSpace(arr[1])
+		boolValue, err := strconv.ParseBool(v)
+		if err != nil {
+			return fmt.Errorf("invalid value of %s=%s, err: %v", k, v, err)
+		}
+		m[k] = boolValue
+	}
+	return f.SetFromMap(m)
+}
+
+// SetFromMap stores flag gates for known features from a map[string]bool or returns an error
+func (f *featureGate) SetFromMap(m map[string]bool) error {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+
+	// Copy existing state
+	known := map[Feature]FeatureSpec{}
+	for k, v := range f.known.Load().(map[Feature]FeatureSpec) {
+		known[k] = v
+	}
+	enabled := map[Feature]bool{}
+	for k, v := range f.enabled.Load().(map[Feature]bool) {
+		enabled[k] = v
+	}
+
+	for k, v := range m {
+		k := Feature(k)
+		featureSpec, ok := known[k]
+		if !ok {
+			return fmt.Errorf("unrecognized feature gate: %s", k)
+		}
+		if featureSpec.LockToDefault && featureSpec.Default != v {
+			return fmt.Errorf("cannot set feature gate %v to %v, feature is locked to %v", k, v, featureSpec.Default)
+		}
+		enabled[k] = v
+		// Handle "special" features like "all alpha gates"
+		if fn, found := f.special[k]; found {
+			fn(known, enabled, v)
+		}
+
+		if featureSpec.PreRelease == Deprecated {
+			f.lg.Warn(fmt.Sprintf("Setting deprecated feature gate %s=%t. It will be removed in a future release.", k, v))
+		} else if featureSpec.PreRelease == GA {
+			f.lg.Warn(fmt.Sprintf("Setting GA feature gate %s=%t. It will be removed in a future release.", k, v))
+		}
+	}
+
+	// Persist changes
+	f.known.Store(known)
+	f.enabled.Store(enabled)
+
+	f.lg.Info(fmt.Sprintf("feature gates: %v", f.enabled))
+	return nil
+}
+
+// String returns a string containing all enabled feature gates, formatted as "key1=value1,key2=value2,...".
+func (f *featureGate) String() string {
+	pairs := []string{}
+	for k, v := range f.enabled.Load().(map[Feature]bool) {
+		pairs = append(pairs, fmt.Sprintf("%s=%t", k, v))
+	}
+	sort.Strings(pairs)
+	return strings.Join(pairs, ",")
+}
+
+func (f *featureGate) Type() string {
+	return "mapStringBool"
+}
+
+// Add adds features to the featureGate.
+func (f *featureGate) Add(features map[Feature]FeatureSpec) error {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+
+	if f.closed {
+		return fmt.Errorf("cannot add a feature gate after adding it to the flag set")
+	}
+
+	// Copy existing state
+	known := map[Feature]FeatureSpec{}
+	for k, v := range f.known.Load().(map[Feature]FeatureSpec) {
+		known[k] = v
+	}
+
+	for name, spec := range features {
+		if existingSpec, found := known[name]; found {
+			if existingSpec == spec {
+				continue
+			}
+			return fmt.Errorf("feature gate %q with different spec already exists: %v", name, existingSpec)
+		}
+
+		known[name] = spec
+	}
+
+	// Persist updated state
+	f.known.Store(known)
+
+	return nil
+}
+
+func (f *featureGate) OverrideDefault(name Feature, override bool) error {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+
+	if f.closed {
+		return fmt.Errorf("cannot override default for feature %q: gates already added to a flag set", name)
+	}
+
+	known := map[Feature]FeatureSpec{}
+	for name, spec := range f.known.Load().(map[Feature]FeatureSpec) {
+		known[name] = spec
+	}
+
+	spec, ok := known[name]
+	switch {
+	case !ok:
+		return fmt.Errorf("cannot override default: feature %q is not registered", name)
+	case spec.LockToDefault:
+		return fmt.Errorf("cannot override default: feature %q default is locked to %t", name, spec.Default)
+	case spec.PreRelease == Deprecated:
+		f.lg.Warn(fmt.Sprintf("Overriding default of deprecated feature gate %s=%t. It will be removed in a future release.", name, override))
+	case spec.PreRelease == GA:
+		f.lg.Warn(fmt.Sprintf("Overriding default of GA feature gate %s=%t. It will be removed in a future release.", name, override))
+	}
+
+	spec.Default = override
+	known[name] = spec
+	f.known.Store(known)
+
+	return nil
+}
+
+// GetAll returns a copy of the map of known feature names to feature specs.
+func (f *featureGate) GetAll() map[Feature]FeatureSpec {
+	retval := map[Feature]FeatureSpec{}
+	for k, v := range f.known.Load().(map[Feature]FeatureSpec) {
+		retval[k] = v
+	}
+	return retval
+}
+
+// Enabled returns true if the key is enabled.  If the key is not known, this call will panic.
+func (f *featureGate) Enabled(key Feature) bool {
+	if v, ok := f.enabled.Load().(map[Feature]bool)[key]; ok {
+		return v
+	}
+	if v, ok := f.known.Load().(map[Feature]FeatureSpec)[key]; ok {
+		return v.Default
+	}
+
+	panic(fmt.Errorf("feature %q is not registered in FeatureGate %q", key, f.featureGateName))
+}
+
+// AddFlag adds a flag for setting global feature gates to the specified FlagSet.
+func (f *featureGate) AddFlag(fs *pflag.FlagSet) {
+	f.lock.Lock()
+	// TODO(mtaufen): Shouldn't we just close it on the first Set/SetFromMap instead?
+	// Not all components expose a feature gates flag using this AddFlag method, and
+	// in the future, all components will completely stop exposing a feature gates flag,
+	// in favor of componentconfig.
+	f.closed = true
+	f.lock.Unlock()
+
+	known := f.KnownFeatures()
+	fs.Var(f, flagName, ""+
+		"A set of key=value pairs that describe feature gates for alpha/experimental features. "+
+		"Options are:\n"+strings.Join(known, "\n"))
+}
+
+func (f *featureGate) AddMetrics() {
+	// TODO(henrybear327): implement this.
+}
+
+// KnownFeatures returns a slice of strings describing the FeatureGate's known features.
+// Deprecated and GA features are hidden from the list.
+func (f *featureGate) KnownFeatures() []string {
+	var known []string
+	for k, v := range f.known.Load().(map[Feature]FeatureSpec) {
+		if v.PreRelease == GA || v.PreRelease == Deprecated {
+			continue
+		}
+		known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, v.PreRelease, v.Default))
+	}
+	sort.Strings(known)
+	return known
+}
+
+// DeepCopy returns a deep copy of the FeatureGate object, such that gates can be
+// set on the copy without mutating the original. This is useful for validating
+// config against potential feature gate changes before committing those changes.
+func (f *featureGate) DeepCopy() MutableFeatureGate {
+	// Copy existing state.
+	known := map[Feature]FeatureSpec{}
+	for k, v := range f.known.Load().(map[Feature]FeatureSpec) {
+		known[k] = v
+	}
+	enabled := map[Feature]bool{}
+	for k, v := range f.enabled.Load().(map[Feature]bool) {
+		enabled[k] = v
+	}
+
+	// Construct a new featureGate around the copied state.
+	// Note that specialFeatures is treated as immutable by convention,
+	// and we maintain the value of f.closed across the copy.
+	fg := &featureGate{
+		special: specialFeatures,
+		closed:  f.closed,
+	}
+
+	fg.known.Store(known)
+	fg.enabled.Store(enabled)
+
+	return fg
+}

pkg/featuregate/feature_gate_test.go

@@ -0,0 +1,613 @@
+// Copyright 2024 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package featuregate
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/spf13/pflag"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap/zaptest"
+)
+
+func TestFeatureGateFlag(t *testing.T) {
+	// gates for testing
+	const testAlphaGate Feature = "TestAlpha"
+	const testBetaGate Feature = "TestBeta"
+
+	tests := []struct {
+		arg        string
+		expect     map[Feature]bool
+		parseError string
+	}{
+		{
+			arg: "",
+			expect: map[Feature]bool{
+				allAlphaGate:  false,
+				allBetaGate:   false,
+				testAlphaGate: false,
+				testBetaGate:  false,
+			},
+		},
+		{
+			arg: "fooBarBaz=true",
+			expect: map[Feature]bool{
+				allAlphaGate:  false,
+				allBetaGate:   false,
+				testAlphaGate: false,
+				testBetaGate:  false,
+			},
+			parseError: "unrecognized feature gate: fooBarBaz",
+		},
+		{
+			arg: "AllAlpha=false",
+			expect: map[Feature]bool{
+				allAlphaGate:  false,
+				allBetaGate:   false,
+				testAlphaGate: false,
+				testBetaGate:  false,
+			},
+		},
+		{
+			arg: "AllAlpha=true",
+			expect: map[Feature]bool{
+				allAlphaGate:  true,
+				allBetaGate:   false,
+				testAlphaGate: true,
+				testBetaGate:  false,
+			},
+		},
+		{
+			arg: "AllAlpha=banana",
+			expect: map[Feature]bool{
+				allAlphaGate:  false,
+				allBetaGate:   false,
+				testAlphaGate: false,
+				testBetaGate:  false,
+			},
+			parseError: "invalid value of AllAlpha",
+		},
+		{
+			arg: "AllAlpha=false,TestAlpha=true",
+			expect: map[Feature]bool{
+				allAlphaGate:  false,
+				allBetaGate:   false,
+				testAlphaGate: true,
+				testBetaGate:  false,
+			},
+		},
+		{
+			arg: "TestAlpha=true,AllAlpha=false",
+			expect: map[Feature]bool{
+				allAlphaGate:  false,
+				allBetaGate:   false,
+				testAlphaGate: true,
+				testBetaGate:  false,
+			},
+		},
+		{
+			arg: "AllAlpha=true,TestAlpha=false",
+			expect: map[Feature]bool{
+				allAlphaGate:  true,
+				allBetaGate:   false,
+				testAlphaGate: false,
+				testBetaGate:  false,
+			},
+		},
+		{
+			arg: "TestAlpha=false,AllAlpha=true",
+			expect: map[Feature]bool{
+				allAlphaGate:  true,
+				allBetaGate:   false,
+				testAlphaGate: false,
+				testBetaGate:  false,
+			},
+		},
+		{
+			arg: "TestBeta=true,AllAlpha=false",
+			expect: map[Feature]bool{
+				allAlphaGate:  false,
+				allBetaGate:   false,
+				testAlphaGate: false,
+				testBetaGate:  true,
+			},
+		},
+
+		{
+			arg: "AllBeta=false",
+			expect: map[Feature]bool{
+				allAlphaGate:  false,
+				allBetaGate:   false,
+				testAlphaGate: false,
+				testBetaGate:  false,
+			},
+		},
+		{
+			arg: "AllBeta=true",
+			expect: map[Feature]bool{
+				allAlphaGate:  false,
+				allBetaGate:   true,
+				testAlphaGate: false,
+				testBetaGate:  true,
+			},
+		},
+		{
+			arg: "AllBeta=banana",
+			expect: map[Feature]bool{
+				allAlphaGate:  false,
+				allBetaGate:   false,
+				testAlphaGate: false,
+				testBetaGate:  false,
+			},
+			parseError: "invalid value of AllBeta",
+		},
+		{
+			arg: "AllBeta=false,TestBeta=true",
+			expect: map[Feature]bool{
+				allAlphaGate:  false,
+				allBetaGate:   false,
+				testAlphaGate: false,
+				testBetaGate:  true,
+			},
+		},
+		{
+			arg: "TestBeta=true,AllBeta=false",
+			expect: map[Feature]bool{
+				allAlphaGate:  false,
+				allBetaGate:   false,
+				testAlphaGate: false,
+				testBetaGate:  true,
+			},
+		},
+		{
+			arg: "AllBeta=true,TestBeta=false",
+			expect: map[Feature]bool{
+				allAlphaGate:  false,
+				allBetaGate:   true,
+				testAlphaGate: false,
+				testBetaGate:  false,
+			},
+		},
+		{
+			arg: "TestBeta=false,AllBeta=true",
+			expect: map[Feature]bool{
+				allAlphaGate:  false,
+				allBetaGate:   true,
+				testAlphaGate: false,
+				testBetaGate:  false,
+			},
+		},
+		{
+			arg: "TestAlpha=true,AllBeta=false",
+			expect: map[Feature]bool{
+				allAlphaGate:  false,
+				allBetaGate:   false,
+				testAlphaGate: true,
+				testBetaGate:  false,
+			},
+		},
+	}
+	for i, test := range tests {
+		t.Run(test.arg, func(t *testing.T) {
+			fs := pflag.NewFlagSet("testfeaturegateflag", pflag.ContinueOnError)
+			f := New("test", zaptest.NewLogger(t))
+			f.Add(map[Feature]FeatureSpec{
+				testAlphaGate: {Default: false, PreRelease: Alpha},
+				testBetaGate:  {Default: false, PreRelease: Beta},
+			})
+			f.AddFlag(fs)
+
+			err := fs.Parse([]string{fmt.Sprintf("--%s=%s", flagName, test.arg)})
+			if test.parseError != "" {
+				if !strings.Contains(err.Error(), test.parseError) {
+					t.Errorf("%d: Parse() Expected %v, Got %v", i, test.parseError, err)
+				}
+			} else if err != nil {
+				t.Errorf("%d: Parse() Expected nil, Got %v", i, err)
+			}
+			for k, v := range test.expect {
+				if actual := f.enabled.Load().(map[Feature]bool)[k]; actual != v {
+					t.Errorf("%d: expected %s=%v, Got %v", i, k, v, actual)
+				}
+			}
+		})
+	}
+}
+
+func TestFeatureGateOverride(t *testing.T) {
+	const testAlphaGate Feature = "TestAlpha"
+	const testBetaGate Feature = "TestBeta"
+
+	// Don't parse the flag, assert defaults are used.
+	var f = New("test", zaptest.NewLogger(t))
+	f.Add(map[Feature]FeatureSpec{
+		testAlphaGate: {Default: false, PreRelease: Alpha},
+		testBetaGate:  {Default: false, PreRelease: Beta},
+	})
+
+	f.Set("TestAlpha=true,TestBeta=true")
+	if f.Enabled(testAlphaGate) != true {
+		t.Errorf("Expected true")
+	}
+	if f.Enabled(testBetaGate) != true {
+		t.Errorf("Expected true")
+	}
+
+	f.Set("TestAlpha=false")
+	if f.Enabled(testAlphaGate) != false {
+		t.Errorf("Expected false")
+	}
+	if f.Enabled(testBetaGate) != true {
+		t.Errorf("Expected true")
+	}
+}
+
+func TestFeatureGateFlagDefaults(t *testing.T) {
+	// gates for testing
+	const testAlphaGate Feature = "TestAlpha"
+	const testBetaGate Feature = "TestBeta"
+
+	// Don't parse the flag, assert defaults are used.
+	var f = New("test", zaptest.NewLogger(t))
+	f.Add(map[Feature]FeatureSpec{
+		testAlphaGate: {Default: false, PreRelease: Alpha},
+		testBetaGate:  {Default: true, PreRelease: Beta},
+	})
+
+	if f.Enabled(testAlphaGate) != false {
+		t.Errorf("Expected false")
+	}
+	if f.Enabled(testBetaGate) != true {
+		t.Errorf("Expected true")
+	}
+}
+
+func TestFeatureGateKnownFeatures(t *testing.T) {
+	// gates for testing
+	const (
+		testAlphaGate      Feature = "TestAlpha"
+		testBetaGate       Feature = "TestBeta"
+		testGAGate         Feature = "TestGA"
+		testDeprecatedGate Feature = "TestDeprecated"
+	)
+
+	// Don't parse the flag, assert defaults are used.
+	var f = New("test", zaptest.NewLogger(t))
+	f.Add(map[Feature]FeatureSpec{
+		testAlphaGate:      {Default: false, PreRelease: Alpha},
+		testBetaGate:       {Default: true, PreRelease: Beta},
+		testGAGate:         {Default: true, PreRelease: GA},
+		testDeprecatedGate: {Default: false, PreRelease: Deprecated},
+	})
+
+	known := strings.Join(f.KnownFeatures(), " ")
+
+	assert.Contains(t, known, testAlphaGate)
+	assert.Contains(t, known, testBetaGate)
+	assert.NotContains(t, known, testGAGate)
+	assert.NotContains(t, known, testDeprecatedGate)
+}
+
+func TestFeatureGateSetFromMap(t *testing.T) {
+	// gates for testing
+	const testAlphaGate Feature = "TestAlpha"
+	const testBetaGate Feature = "TestBeta"
+	const testLockedTrueGate Feature = "TestLockedTrue"
+	const testLockedFalseGate Feature = "TestLockedFalse"
+
+	tests := []struct {
+		name        string
+		setmap      map[string]bool
+		expect      map[Feature]bool
+		setmapError string
+	}{
+		{
+			name: "set TestAlpha and TestBeta true",
+			setmap: map[string]bool{
+				"TestAlpha": true,
+				"TestBeta":  true,
+			},
+			expect: map[Feature]bool{
+				testAlphaGate: true,
+				testBetaGate:  true,
+			},
+		},
+		{
+			name: "set TestBeta true",
+			setmap: map[string]bool{
+				"TestBeta": true,
+			},
+			expect: map[Feature]bool{
+				testAlphaGate: false,
+				testBetaGate:  true,
+			},
+		},
+		{
+			name: "set TestAlpha false",
+			setmap: map[string]bool{
+				"TestAlpha": false,
+			},
+			expect: map[Feature]bool{
+				testAlphaGate: false,
+				testBetaGate:  false,
+			},
+		},
+		{
+			name: "set TestInvaild true",
+			setmap: map[string]bool{
+				"TestInvaild": true,
+			},
+			expect: map[Feature]bool{
+				testAlphaGate: false,
+				testBetaGate:  false,
+			},
+			setmapError: "unrecognized feature gate:",
+		},
+		{
+			name: "set locked gates",
+			setmap: map[string]bool{
+				"TestLockedTrue":  true,
+				"TestLockedFalse": false,
+			},
+			expect: map[Feature]bool{
+				testAlphaGate: false,
+				testBetaGate:  false,
+			},
+		},
+		{
+			name: "set locked gates",
+			setmap: map[string]bool{
+				"TestLockedTrue": false,
+			},
+			expect: map[Feature]bool{
+				testAlphaGate: false,
+				testBetaGate:  false,
+			},
+			setmapError: "cannot set feature gate TestLockedTrue to false, feature is locked to true",
+		},
+		{
+			name: "set locked gates",
+			setmap: map[string]bool{
+				"TestLockedFalse": true,
+			},
+			expect: map[Feature]bool{
+				testAlphaGate: false,
+				testBetaGate:  false,
+			},
+			setmapError: "cannot set feature gate TestLockedFalse to true, feature is locked to false",
+		},
+	}
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("SetFromMap %s", test.name), func(t *testing.T) {
+			f := New("test", zaptest.NewLogger(t))
+			f.Add(map[Feature]FeatureSpec{
+				testAlphaGate:       {Default: false, PreRelease: Alpha},
+				testBetaGate:        {Default: false, PreRelease: Beta},
+				testLockedTrueGate:  {Default: true, PreRelease: GA, LockToDefault: true},
+				testLockedFalseGate: {Default: false, PreRelease: GA, LockToDefault: true},
+			})
+			err := f.SetFromMap(test.setmap)
+			if test.setmapError != "" {
+				if err == nil {
+					t.Errorf("expected error, got none")
+				} else if !strings.Contains(err.Error(), test.setmapError) {
+					t.Errorf("%d: SetFromMap(%#v) Expected err:%v, Got err:%v", i, test.setmap, test.setmapError, err)
+				}
+			} else if err != nil {
+				t.Errorf("%d: SetFromMap(%#v) Expected success, Got err:%v", i, test.setmap, err)
+			}
+			for k, v := range test.expect {
+				if actual := f.Enabled(k); actual != v {
+					t.Errorf("%d: SetFromMap(%#v) Expected %s=%v, Got %s=%v", i, test.setmap, k, v, k, actual)
+				}
+			}
+		})
+	}
+}
+
+func TestFeatureGateMetrics(t *testing.T) {
+	// TODO(henrybear327): Add tests once feature gate metrics are added.
+}
+
+func TestFeatureGateString(t *testing.T) {
+	// gates for testing
+	const testAlphaGate Feature = "TestAlpha"
+	const testBetaGate Feature = "TestBeta"
+	const testGAGate Feature = "TestGA"
+
+	featuremap := map[Feature]FeatureSpec{
+		testGAGate:    {Default: true, PreRelease: GA},
+		testAlphaGate: {Default: false, PreRelease: Alpha},
+		testBetaGate:  {Default: true, PreRelease: Beta},
+	}
+
+	tests := []struct {
+		setmap map[string]bool
+		expect string
+	}{
+		{
+			setmap: map[string]bool{
+				"TestAlpha": false,
+			},
+			expect: "TestAlpha=false",
+		},
+		{
+			setmap: map[string]bool{
+				"TestAlpha": false,
+				"TestBeta":  true,
+			},
+			expect: "TestAlpha=false,TestBeta=true",
+		},
+		{
+			setmap: map[string]bool{
+				"TestGA":    true,
+				"TestAlpha": false,
+				"TestBeta":  true,
+			},
+			expect: "TestAlpha=false,TestBeta=true,TestGA=true",
+		},
+	}
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("SetFromMap %s", test.expect), func(t *testing.T) {
+			f := New("test", zaptest.NewLogger(t))
+			f.Add(featuremap)
+			f.SetFromMap(test.setmap)
+			result := f.String()
+			if result != test.expect {
+				t.Errorf("%d: SetFromMap(%#v) Expected %s, Got %s", i, test.setmap, test.expect, result)
+			}
+		})
+	}
+}
+
+func TestFeatureGateOverrideDefault(t *testing.T) {
+	t.Run("overrides take effect", func(t *testing.T) {
+		f := New("test", zaptest.NewLogger(t))
+		if err := f.Add(map[Feature]FeatureSpec{
+			"TestFeature1": {Default: true},
+			"TestFeature2": {Default: false},
+		}); err != nil {
+			t.Fatal(err)
+		}
+		if err := f.OverrideDefault("TestFeature1", false); err != nil {
+			t.Fatal(err)
+		}
+		if err := f.OverrideDefault("TestFeature2", true); err != nil {
+			t.Fatal(err)
+		}
+		if f.Enabled("TestFeature1") {
+			t.Error("expected TestFeature1 to have effective default of false")
+		}
+		if !f.Enabled("TestFeature2") {
+			t.Error("expected TestFeature2 to have effective default of true")
+		}
+	})
+
+	t.Run("overrides are preserved across deep copies", func(t *testing.T) {
+		f := New("test", zaptest.NewLogger(t))
+		if err := f.Add(map[Feature]FeatureSpec{"TestFeature": {Default: false}}); err != nil {
+			t.Fatal(err)
+		}
+		if err := f.OverrideDefault("TestFeature", true); err != nil {
+			t.Fatal(err)
+		}
+		fcopy := f.DeepCopy()
+		if !fcopy.Enabled("TestFeature") {
+			t.Error("default override was not preserved by deep copy")
+		}
+	})
+
+	t.Run("reflected in known features", func(t *testing.T) {
+		f := New("test", zaptest.NewLogger(t))
+		if err := f.Add(map[Feature]FeatureSpec{"TestFeature": {
+			Default:    false,
+			PreRelease: Alpha,
+		}}); err != nil {
+			t.Fatal(err)
+		}
+		if err := f.OverrideDefault("TestFeature", true); err != nil {
+			t.Fatal(err)
+		}
+		var found bool
+		for _, s := range f.KnownFeatures() {
+			if !strings.Contains(s, "TestFeature") {
+				continue
+			}
+			found = true
+			if !strings.Contains(s, "default=true") {
+				t.Errorf("expected override of default to be reflected in known feature description %q", s)
+			}
+		}
+		if !found {
+			t.Error("found no entry for TestFeature in known features")
+		}
+	})
+
+	t.Run("may not change default for specs with locked defaults", func(t *testing.T) {
+		f := New("test", zaptest.NewLogger(t))
+		if err := f.Add(map[Feature]FeatureSpec{
+			"LockedFeature": {
+				Default:       true,
+				LockToDefault: true,
+			},
+		}); err != nil {
+			t.Fatal(err)
+		}
+		if f.OverrideDefault("LockedFeature", false) == nil {
+			t.Error("expected error when attempting to override the default for a feature with a locked default")
+		}
+		if f.OverrideDefault("LockedFeature", true) == nil {
+			t.Error("expected error when attempting to override the default for a feature with a locked default")
+		}
+	})
+
+	t.Run("does not supersede explicitly-set value", func(t *testing.T) {
+		f := New("test", zaptest.NewLogger(t))
+		if err := f.Add(map[Feature]FeatureSpec{"TestFeature": {Default: true}}); err != nil {
+			t.Fatal(err)
+		}
+		if err := f.OverrideDefault("TestFeature", false); err != nil {
+			t.Fatal(err)
+		}
+		if err := f.SetFromMap(map[string]bool{"TestFeature": true}); err != nil {
+			t.Fatal(err)
+		}
+		if !f.Enabled("TestFeature") {
+			t.Error("expected feature to be effectively enabled despite default override")
+		}
+	})
+
+	t.Run("prevents re-registration of feature spec after overriding default", func(t *testing.T) {
+		f := New("test", zaptest.NewLogger(t))
+		if err := f.Add(map[Feature]FeatureSpec{
+			"TestFeature": {
+				Default:    true,
+				PreRelease: Alpha,
+			},
+		}); err != nil {
+			t.Fatal(err)
+		}
+		if err := f.OverrideDefault("TestFeature", false); err != nil {
+			t.Fatal(err)
+		}
+		if err := f.Add(map[Feature]FeatureSpec{
+			"TestFeature": {
+				Default:    true,
+				PreRelease: Alpha,
+			},
+		}); err == nil {
+			t.Error("expected re-registration to return a non-nil error after overriding its default")
+		}
+	})
+
+	t.Run("does not allow override for an unknown feature", func(t *testing.T) {
+		f := New("test", zaptest.NewLogger(t))
+		if err := f.OverrideDefault("TestFeature", true); err == nil {
+			t.Error("expected an error to be returned in attempt to override default for unregistered feature")
+		}
+	})
+
+	t.Run("returns error if already added to flag set", func(t *testing.T) {
+		f := New("test", zaptest.NewLogger(t))
+		fs := pflag.NewFlagSet("test", pflag.ContinueOnError)
+		f.AddFlag(fs)
+
+		if err := f.OverrideDefault("TestFeature", true); err == nil {
+			t.Error("expected a non-nil error to be returned")
+		}
+	})
+}

server/config/config.go

@@ -28,6 +28,7 @@ import (
 	bolt "go.etcd.io/bbolt"
 	"go.etcd.io/etcd/client/pkg/v3/transport"
 	"go.etcd.io/etcd/client/pkg/v3/types"
+	"go.etcd.io/etcd/pkg/v3/featuregate"
 	"go.etcd.io/etcd/pkg/v3/netutil"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/v3discovery"
 	"go.etcd.io/etcd/server/v3/storage/datadir"
@@ -207,6 +208,9 @@ type ServerConfig struct {
 
 	// ExperimentalLocalAddress is the local IP address to use when communicating with a peer.
 	ExperimentalLocalAddress string `json:"experimental-local-address"`
+
+	// ServerFeatureGate is a server level feature gate
+	ServerFeatureGate featuregate.FeatureGate
 }
 
 // VerifyBootstrap sanity-checks the initial config for bootstrap case

server/etcdserver/server.go

@@ -39,6 +39,7 @@ import (
 	"go.etcd.io/etcd/client/pkg/v3/fileutil"
 	"go.etcd.io/etcd/client/pkg/v3/types"
 	"go.etcd.io/etcd/client/pkg/v3/verify"
+	"go.etcd.io/etcd/pkg/v3/featuregate"
 	"go.etcd.io/etcd/pkg/v3/idutil"
 	"go.etcd.io/etcd/pkg/v3/notify"
 	"go.etcd.io/etcd/pkg/v3/pbutil"
@@ -456,6 +457,11 @@ func (s *EtcdServer) Config() config.ServerConfig {
 	return s.Cfg
 }
 
+// FeatureEnabled returns true if the feature is enabled by the etcd server, false otherwise.
+func (s *EtcdServer) FeatureEnabled(f featuregate.Feature) bool {
+	return s.Cfg.ServerFeatureGate.Enabled(f)
+}
+
 func tickToDur(ticks int, tickMs uint) string {
 	return fmt.Sprintf("%v", time.Duration(ticks)*time.Duration(tickMs)*time.Millisecond)
 }