Merge pull request #11427 from YoyinZyc/migration-cluster-attr

Migrate cluster attributes to use v3 backend

clientv3/integration/kv_test.go

@@ -893,7 +893,7 @@ func TestKVLargeRequests(t *testing.T) {
 		expectError error
 	}{
 		{
-			maxRequestBytesServer:  1,
+			maxRequestBytesServer:  256,
 			maxCallSendBytesClient: 0,
 			maxCallRecvBytesClient: 0,
 			valueSize:              1024,

etcdserver/api/membership/cluster.go

@@ -247,7 +247,12 @@ func (c *RaftCluster) Recover(onSet func(*zap.Logger, *semver.Version)) {
 	defer c.Unlock()
 
 	c.members, c.removed = membersFromStore(c.lg, c.v2store)
-	c.version = clusterVersionFromStore(c.lg, c.v2store)
+	if c.be != nil {
+		c.version = clusterVersionFromBackend(c.lg, c.be)
+	} else {
+		c.version = clusterVersionFromStore(c.lg, c.v2store)
+	}
+
 	mustDetectDowngrade(c.lg, c.version)
 	onSet(c.lg, c.version)
 
@@ -753,6 +758,26 @@ func clusterVersionFromStore(lg *zap.Logger, st v2store.Store) *semver.Version {
 	return semver.Must(semver.NewVersion(*e.Node.Value))
 }
 
+func clusterVersionFromBackend(lg *zap.Logger, be backend.Backend) *semver.Version {
+	ckey := backendClusterVersionKey()
+	tx := be.ReadTx()
+	tx.RLock()
+	defer tx.RUnlock()
+	keys, vals := tx.UnsafeRange(clusterBucketName, ckey, nil, 0)
+	if len(keys) == 0 {
+		return nil
+	}
+	if len(keys) != 1 {
+		if lg != nil {
+			lg.Panic(
+				"unexpected number of keys when getting cluster version from backend",
+				zap.Int("number-of-key", len(keys)),
+			)
+		}
+	}
+	return semver.Must(semver.NewVersion(string(vals[0])))
+}
+
 // ValidateClusterAndAssignIDs validates the local cluster by matching the PeerURLs
 // with the existing cluster. If the validation succeeds, it assigns the IDs
 // from the existing cluster to the local cluster.

etcdserver/api/membership/membershippb/membership.proto

@@ -0,0 +1,38 @@
+syntax = "proto3";
+package membershippb;
+
+import "gogoproto/gogo.proto";
+
+option (gogoproto.marshaler_all) = true;
+option (gogoproto.sizer_all) = true;
+option (gogoproto.unmarshaler_all) = true;
+option (gogoproto.goproto_getters_all) = false;
+
+// RaftAttributes represents the raft related attributes of an etcd member.
+message RaftAttributes {
+  // peerURLs is the list of peers in the raft cluster.
+  repeated string peer_urls = 1;
+  // isLearner indicates if the member is raft learner.
+  bool is_learner = 2;
+}
+
+// Attributes represents all the non-raft related attributes of an etcd member.
+message Attributes {
+  string name = 1;
+  repeated string client_urls = 2;
+}
+
+message Member {
+  uint64 ID = 1;
+  RaftAttributes raft_attributes = 2;
+  Attributes member_attributes = 3;
+}
+
+message ClusterVersionSetRequest {
+  string ver = 1;
+}
+
+message ClusterMemberAttrSetRequest {
+  uint64 member_ID = 1;
+  Attributes member_attributes = 2;
+}

etcdserver/apply.go

@@ -21,7 +21,11 @@ import (
 	"sort"
 	"time"
 
+	"github.com/coreos/go-semver/semver"
 	"go.etcd.io/etcd/auth"
+	"go.etcd.io/etcd/etcdserver/api"
+	"go.etcd.io/etcd/etcdserver/api/membership"
+	"go.etcd.io/etcd/etcdserver/api/membership/membershippb"
 	pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
 	"go.etcd.io/etcd/lease"
 	"go.etcd.io/etcd/mvcc"
@@ -47,6 +51,12 @@ type applyResult struct {
 	trace *traceutil.Trace
 }
 
+// applierV3Internal is the interface for processing internal V3 raft request
+type applierV3Internal interface {
+	ClusterVersionSet(r *membershippb.ClusterVersionSetRequest)
+	ClusterMemberAttrSet(r *membershippb.ClusterMemberAttrSetRequest)
+}
+
 // applierV3 is the interface for processing V3 raft messages
 type applierV3 interface {
 	Apply(r *pb.InternalRaftRequest) *applyResult
@@ -104,6 +114,11 @@ func (s *EtcdServer) newApplierV3Backend() applierV3 {
 	return base
 }
 
+func (s *EtcdServer) newApplierV3Internal() applierV3Internal {
+	base := &applierV3backend{s: s}
+	return base
+}
+
 func (s *EtcdServer) newApplierV3() applierV3 {
 	return newAuthApplierV3(
 		s.AuthStore(),
@@ -170,6 +185,10 @@ func (a *applierV3backend) Apply(r *pb.InternalRaftRequest) *applyResult {
 		ar.resp, ar.err = a.s.applyV3.UserList(r.AuthUserList)
 	case r.AuthRoleList != nil:
 		ar.resp, ar.err = a.s.applyV3.RoleList(r.AuthRoleList)
+	case r.ClusterVersionSet != nil:
+		a.s.applyV3Internal.ClusterVersionSet(r.ClusterVersionSet)
+	case r.ClusterMemberAttrSet != nil:
+		a.s.applyV3Internal.ClusterMemberAttrSet(r.ClusterMemberAttrSet)
 	default:
 		panic("not implemented")
 	}
@@ -833,6 +852,20 @@ func (a *applierV3backend) RoleList(r *pb.AuthRoleListRequest) (*pb.AuthRoleList
 	return resp, err
 }
 
+func (a *applierV3backend) ClusterVersionSet(r *membershippb.ClusterVersionSetRequest) {
+	a.s.cluster.SetVersion(semver.Must(semver.NewVersion(r.Ver)), api.UpdateCapability)
+}
+
+func (a *applierV3backend) ClusterMemberAttrSet(r *membershippb.ClusterMemberAttrSetRequest) {
+	a.s.cluster.UpdateAttributes(
+		types.ID(r.Member_ID),
+		membership.Attributes{
+			Name:       r.MemberAttributes.Name,
+			ClientURLs: r.MemberAttributes.ClientUrls,
+		},
+	)
+}
+
 type quotaApplierV3 struct {
 	applierV3
 	q Quota

etcdserver/apply_v2.go

@@ -19,12 +19,10 @@ import (
 	"path"
 	"time"
 
-	"go.etcd.io/etcd/etcdserver/api"
 	"go.etcd.io/etcd/etcdserver/api/membership"
 	"go.etcd.io/etcd/etcdserver/api/v2store"
 	"go.etcd.io/etcd/pkg/pbutil"
 
-	"github.com/coreos/go-semver/semver"
 	"go.uber.org/zap"
 )
 
@@ -91,10 +89,8 @@ func (a *applierV2store) Put(r *RequestV2) Response {
 			// return an empty response since there is no consumer.
 			return Response{}
 		}
+		// remove v2 version set to avoid the conflict between v2 and v3.
 		if r.Path == membership.StoreClusterVersionKey() {
-			if a.cluster != nil {
-				a.cluster.SetVersion(semver.Must(semver.NewVersion(r.Val)), api.UpdateCapability)
-			}
 			// return an empty response since there is no consumer.
 			return Response{}
 		}

etcdserver/etcdserverpb/raft_internal.pb.go

@@ -12,6 +12,8 @@ import (
 
 	_ "github.com/gogo/protobuf/gogoproto"
 
+	membershippb "go.etcd.io/etcd/etcdserver/api/membership/membershippb"
+
 	io "io"
 )
 
@@ -36,34 +38,36 @@ func (*RequestHeader) Descriptor() ([]byte, []int) { return fileDescriptorRaftIn
 // An InternalRaftRequest is the union of all requests which can be
 // sent via raft.
 type InternalRaftRequest struct {
-	Header                   *RequestHeader                   `protobuf:"bytes,100,opt,name=header" json:"header,omitempty"`
+	Header                   *RequestHeader                            `protobuf:"bytes,100,opt,name=header" json:"header,omitempty"`
-	ID                       uint64                           `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"`
+	ID                       uint64                                    `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"`
-	V2                       *Request                         `protobuf:"bytes,2,opt,name=v2" json:"v2,omitempty"`
+	V2                       *Request                                  `protobuf:"bytes,2,opt,name=v2" json:"v2,omitempty"`
-	Range                    *RangeRequest                    `protobuf:"bytes,3,opt,name=range" json:"range,omitempty"`
+	Range                    *RangeRequest                             `protobuf:"bytes,3,opt,name=range" json:"range,omitempty"`
-	Put                      *PutRequest                      `protobuf:"bytes,4,opt,name=put" json:"put,omitempty"`
+	Put                      *PutRequest                               `protobuf:"bytes,4,opt,name=put" json:"put,omitempty"`
-	DeleteRange              *DeleteRangeRequest              `protobuf:"bytes,5,opt,name=delete_range,json=deleteRange" json:"delete_range,omitempty"`
+	DeleteRange              *DeleteRangeRequest                       `protobuf:"bytes,5,opt,name=delete_range,json=deleteRange" json:"delete_range,omitempty"`
-	Txn                      *TxnRequest                      `protobuf:"bytes,6,opt,name=txn" json:"txn,omitempty"`
+	Txn                      *TxnRequest                               `protobuf:"bytes,6,opt,name=txn" json:"txn,omitempty"`
-	Compaction               *CompactionRequest               `protobuf:"bytes,7,opt,name=compaction" json:"compaction,omitempty"`
+	Compaction               *CompactionRequest                        `protobuf:"bytes,7,opt,name=compaction" json:"compaction,omitempty"`
-	LeaseGrant               *LeaseGrantRequest               `protobuf:"bytes,8,opt,name=lease_grant,json=leaseGrant" json:"lease_grant,omitempty"`
+	LeaseGrant               *LeaseGrantRequest                        `protobuf:"bytes,8,opt,name=lease_grant,json=leaseGrant" json:"lease_grant,omitempty"`
-	LeaseRevoke              *LeaseRevokeRequest              `protobuf:"bytes,9,opt,name=lease_revoke,json=leaseRevoke" json:"lease_revoke,omitempty"`
+	LeaseRevoke              *LeaseRevokeRequest                       `protobuf:"bytes,9,opt,name=lease_revoke,json=leaseRevoke" json:"lease_revoke,omitempty"`
-	Alarm                    *AlarmRequest                    `protobuf:"bytes,10,opt,name=alarm" json:"alarm,omitempty"`
+	Alarm                    *AlarmRequest                             `protobuf:"bytes,10,opt,name=alarm" json:"alarm,omitempty"`
-	LeaseCheckpoint          *LeaseCheckpointRequest          `protobuf:"bytes,11,opt,name=lease_checkpoint,json=leaseCheckpoint" json:"lease_checkpoint,omitempty"`
+	LeaseCheckpoint          *LeaseCheckpointRequest                   `protobuf:"bytes,11,opt,name=lease_checkpoint,json=leaseCheckpoint" json:"lease_checkpoint,omitempty"`
-	AuthEnable               *AuthEnableRequest               `protobuf:"bytes,1000,opt,name=auth_enable,json=authEnable" json:"auth_enable,omitempty"`
+	AuthEnable               *AuthEnableRequest                        `protobuf:"bytes,1000,opt,name=auth_enable,json=authEnable" json:"auth_enable,omitempty"`
-	AuthDisable              *AuthDisableRequest              `protobuf:"bytes,1011,opt,name=auth_disable,json=authDisable" json:"auth_disable,omitempty"`
+	AuthDisable              *AuthDisableRequest                       `protobuf:"bytes,1011,opt,name=auth_disable,json=authDisable" json:"auth_disable,omitempty"`
-	Authenticate             *InternalAuthenticateRequest     `protobuf:"bytes,1012,opt,name=authenticate" json:"authenticate,omitempty"`
+	Authenticate             *InternalAuthenticateRequest              `protobuf:"bytes,1012,opt,name=authenticate" json:"authenticate,omitempty"`
-	AuthUserAdd              *AuthUserAddRequest              `protobuf:"bytes,1100,opt,name=auth_user_add,json=authUserAdd" json:"auth_user_add,omitempty"`
+	AuthUserAdd              *AuthUserAddRequest                       `protobuf:"bytes,1100,opt,name=auth_user_add,json=authUserAdd" json:"auth_user_add,omitempty"`
-	AuthUserDelete           *AuthUserDeleteRequest           `protobuf:"bytes,1101,opt,name=auth_user_delete,json=authUserDelete" json:"auth_user_delete,omitempty"`
+	AuthUserDelete           *AuthUserDeleteRequest                    `protobuf:"bytes,1101,opt,name=auth_user_delete,json=authUserDelete" json:"auth_user_delete,omitempty"`
-	AuthUserGet              *AuthUserGetRequest              `protobuf:"bytes,1102,opt,name=auth_user_get,json=authUserGet" json:"auth_user_get,omitempty"`
+	AuthUserGet              *AuthUserGetRequest                       `protobuf:"bytes,1102,opt,name=auth_user_get,json=authUserGet" json:"auth_user_get,omitempty"`
-	AuthUserChangePassword   *AuthUserChangePasswordRequest   `protobuf:"bytes,1103,opt,name=auth_user_change_password,json=authUserChangePassword" json:"auth_user_change_password,omitempty"`
+	AuthUserChangePassword   *AuthUserChangePasswordRequest            `protobuf:"bytes,1103,opt,name=auth_user_change_password,json=authUserChangePassword" json:"auth_user_change_password,omitempty"`
-	AuthUserGrantRole        *AuthUserGrantRoleRequest        `protobuf:"bytes,1104,opt,name=auth_user_grant_role,json=authUserGrantRole" json:"auth_user_grant_role,omitempty"`
+	AuthUserGrantRole        *AuthUserGrantRoleRequest                 `protobuf:"bytes,1104,opt,name=auth_user_grant_role,json=authUserGrantRole" json:"auth_user_grant_role,omitempty"`
-	AuthUserRevokeRole       *AuthUserRevokeRoleRequest       `protobuf:"bytes,1105,opt,name=auth_user_revoke_role,json=authUserRevokeRole" json:"auth_user_revoke_role,omitempty"`
+	AuthUserRevokeRole       *AuthUserRevokeRoleRequest                `protobuf:"bytes,1105,opt,name=auth_user_revoke_role,json=authUserRevokeRole" json:"auth_user_revoke_role,omitempty"`
-	AuthUserList             *AuthUserListRequest             `protobuf:"bytes,1106,opt,name=auth_user_list,json=authUserList" json:"auth_user_list,omitempty"`
+	AuthUserList             *AuthUserListRequest                      `protobuf:"bytes,1106,opt,name=auth_user_list,json=authUserList" json:"auth_user_list,omitempty"`
-	AuthRoleList             *AuthRoleListRequest             `protobuf:"bytes,1107,opt,name=auth_role_list,json=authRoleList" json:"auth_role_list,omitempty"`
+	AuthRoleList             *AuthRoleListRequest                      `protobuf:"bytes,1107,opt,name=auth_role_list,json=authRoleList" json:"auth_role_list,omitempty"`
-	AuthRoleAdd              *AuthRoleAddRequest              `protobuf:"bytes,1200,opt,name=auth_role_add,json=authRoleAdd" json:"auth_role_add,omitempty"`
+	AuthRoleAdd              *AuthRoleAddRequest                       `protobuf:"bytes,1200,opt,name=auth_role_add,json=authRoleAdd" json:"auth_role_add,omitempty"`
-	AuthRoleDelete           *AuthRoleDeleteRequest           `protobuf:"bytes,1201,opt,name=auth_role_delete,json=authRoleDelete" json:"auth_role_delete,omitempty"`
+	AuthRoleDelete           *AuthRoleDeleteRequest                    `protobuf:"bytes,1201,opt,name=auth_role_delete,json=authRoleDelete" json:"auth_role_delete,omitempty"`
-	AuthRoleGet              *AuthRoleGetRequest              `protobuf:"bytes,1202,opt,name=auth_role_get,json=authRoleGet" json:"auth_role_get,omitempty"`
+	AuthRoleGet              *AuthRoleGetRequest                       `protobuf:"bytes,1202,opt,name=auth_role_get,json=authRoleGet" json:"auth_role_get,omitempty"`
-	AuthRoleGrantPermission  *AuthRoleGrantPermissionRequest  `protobuf:"bytes,1203,opt,name=auth_role_grant_permission,json=authRoleGrantPermission" json:"auth_role_grant_permission,omitempty"`
+	AuthRoleGrantPermission  *AuthRoleGrantPermissionRequest           `protobuf:"bytes,1203,opt,name=auth_role_grant_permission,json=authRoleGrantPermission" json:"auth_role_grant_permission,omitempty"`
-	AuthRoleRevokePermission *AuthRoleRevokePermissionRequest `protobuf:"bytes,1204,opt,name=auth_role_revoke_permission,json=authRoleRevokePermission" json:"auth_role_revoke_permission,omitempty"`
+	AuthRoleRevokePermission *AuthRoleRevokePermissionRequest          `protobuf:"bytes,1204,opt,name=auth_role_revoke_permission,json=authRoleRevokePermission" json:"auth_role_revoke_permission,omitempty"`
+	ClusterVersionSet        *membershippb.ClusterVersionSetRequest    `protobuf:"bytes,1300,opt,name=cluster_version_set,json=clusterVersionSet" json:"cluster_version_set,omitempty"`
+	ClusterMemberAttrSet     *membershippb.ClusterMemberAttrSetRequest `protobuf:"bytes,1301,opt,name=cluster_member_attr_set,json=clusterMemberAttrSet" json:"cluster_member_attr_set,omitempty"`
 }
 
 func (m *InternalRaftRequest) Reset()                    { *m = InternalRaftRequest{} }
@@ -460,6 +464,30 @@ func (m *InternalRaftRequest) MarshalTo(dAtA []byte) (int, error) {
 		}
 		i += n27
 	}
+	if m.ClusterVersionSet != nil {
+		dAtA[i] = 0xa2
+		i++
+		dAtA[i] = 0x51
+		i++
+		i = encodeVarintRaftInternal(dAtA, i, uint64(m.ClusterVersionSet.Size()))
+		n28, err := m.ClusterVersionSet.MarshalTo(dAtA[i:])
+		if err != nil {
+			return 0, err
+		}
+		i += n28
+	}
+	if m.ClusterMemberAttrSet != nil {
+		dAtA[i] = 0xaa
+		i++
+		dAtA[i] = 0x51
+		i++
+		i = encodeVarintRaftInternal(dAtA, i, uint64(m.ClusterMemberAttrSet.Size()))
+		n29, err := m.ClusterMemberAttrSet.MarshalTo(dAtA[i:])
+		if err != nil {
+			return 0, err
+		}
+		i += n29
+	}
 	return i, nil
 }
 
@@ -656,6 +684,14 @@ func (m *InternalRaftRequest) Size() (n int) {
 		l = m.AuthRoleRevokePermission.Size()
 		n += 2 + l + sovRaftInternal(uint64(l))
 	}
+	if m.ClusterVersionSet != nil {
+		l = m.ClusterVersionSet.Size()
+		n += 2 + l + sovRaftInternal(uint64(l))
+	}
+	if m.ClusterMemberAttrSet != nil {
+		l = m.ClusterMemberAttrSet.Size()
+		n += 2 + l + sovRaftInternal(uint64(l))
+	}
 	return n
 }
 
@@ -1752,6 +1788,72 @@ func (m *InternalRaftRequest) Unmarshal(dAtA []byte) error {
 				return err
 			}
 			iNdEx = postIndex
+		case 1300:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field ClusterVersionSet", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowRaftInternal
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthRaftInternal
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if m.ClusterVersionSet == nil {
+				m.ClusterVersionSet = &membershippb.ClusterVersionSetRequest{}
+			}
+			if err := m.ClusterVersionSet.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 1301:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field ClusterMemberAttrSet", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowRaftInternal
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthRaftInternal
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if m.ClusterMemberAttrSet == nil {
+				m.ClusterMemberAttrSet = &membershippb.ClusterMemberAttrSetRequest{}
+			}
+			if err := m.ClusterMemberAttrSet.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipRaftInternal(dAtA[iNdEx:])
@@ -2068,60 +2170,66 @@ var (
 func init() { proto.RegisterFile("raft_internal.proto", fileDescriptorRaftInternal) }
 
 var fileDescriptorRaftInternal = []byte{
-	// 867 bytes of a gzipped FileDescriptorProto
+	// 961 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x96, 0xcb, 0x52, 0x33, 0x45,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x96, 0x49, 0x73, 0x1b, 0x45,
-	0x14, 0xc7, 0x49, 0xb8, 0xa6, 0x13, 0x2e, 0x36, 0xa0, 0x6d, 0xa8, 0x8a, 0x10, 0xbc, 0xe0, 0x0d,
+	0x14, 0xc7, 0x23, 0xc5, 0x71, 0xac, 0x96, 0xed, 0x38, 0x6d, 0x87, 0x34, 0x72, 0x95, 0x70, 0x1c,
-	0xad, 0xf0, 0x00, 0x1a, 0x13, 0x0a, 0xa8, 0xa2, 0x90, 0x9a, 0xc2, 0x2a, 0xab, 0x5c, 0x8c, 0xcd,
+	0x96, 0xb0, 0xd9, 0x94, 0x73, 0xa5, 0x0a, 0x84, 0xe4, 0x72, 0x5c, 0x15, 0x82, 0x6b, 0x08, 0x4b,
-	0xcc, 0x21, 0x19, 0x99, 0xcc, 0x8c, 0x3d, 0x9d, 0x88, 0x6f, 0xe2, 0x63, 0x78, 0xdb, 0xbb, 0x65,
+	0x15, 0x87, 0xa1, 0x35, 0xf3, 0x22, 0x0d, 0x9e, 0x8d, 0xee, 0x96, 0x30, 0xdf, 0x03, 0xaa, 0xf8,
-	0xe1, 0x05, 0xf5, 0x05, 0x14, 0x37, 0xee, 0xbf, 0xef, 0x01, 0xbe, 0xea, 0xcb, 0xf4, 0x64, 0x92,
+	0x18, 0x6c, 0x1f, 0x22, 0x07, 0x96, 0x00, 0x5f, 0x00, 0xcc, 0x85, 0x3b, 0xdc, 0xb8, 0xa4, 0x7a,
-	0x0e, 0xbb, 0xc9, 0x39, 0xff, 0xf3, 0xfb, 0x9f, 0x99, 0x3e, 0x07, 0x1a, 0x6d, 0x32, 0x7a, 0xc3,
+	0x99, 0x9e, 0x19, 0xa9, 0xe5, 0xdb, 0xe8, 0xff, 0xfe, 0xef, 0xf7, 0xde, 0x74, 0xbf, 0x1e, 0x35,
-	0xdd, 0x20, 0xe2, 0xc0, 0x22, 0x1a, 0x1e, 0x26, 0x2c, 0xe6, 0x31, 0xae, 0x01, 0xf7, 0xfc, 0x14,
+	0xda, 0x64, 0xf4, 0x91, 0xf0, 0xa3, 0x54, 0x00, 0x4b, 0x69, 0xbc, 0x97, 0xb3, 0x4c, 0x64, 0x78,
-	0xd8, 0x08, 0x58, 0x72, 0x5d, 0xdf, 0xea, 0xc5, 0xbd, 0x58, 0x26, 0x3e, 0x10, 0x4f, 0x4a, 0x53,
+	0x15, 0x44, 0x10, 0x72, 0x60, 0x53, 0x60, 0xf9, 0xb0, 0xb3, 0x35, 0xca, 0x46, 0x99, 0x0a, 0xec,
-	0xdf, 0xc8, 0x35, 0x3a, 0x52, 0x61, 0x89, 0xa7, 0x1e, 0x9b, 0x5f, 0xa2, 0x55, 0x07, 0xbe, 0x1e,
+	0xcb, 0x27, 0xed, 0xe9, 0x6c, 0x94, 0x1e, 0xa3, 0xb4, 0x58, 0x1e, 0x98, 0xc7, 0x37, 0x65, 0x70,
-	0x42, 0xca, 0x4f, 0x81, 0xfa, 0xc0, 0xf0, 0x1a, 0x2a, 0x9f, 0x75, 0x49, 0x69, 0xb7, 0x74, 0xb0,
+	0xbf, 0x74, 0xec, 0xd3, 0x3c, 0xda, 0x4f, 0x20, 0x19, 0x02, 0xe3, 0xe3, 0x28, 0xaf, 0x3c, 0xe6,
-	0xe0, 0x94, 0xcf, 0xba, 0xb8, 0x8e, 0x56, 0x86, 0xa9, 0xb0, 0x1c, 0x00, 0x29, 0xef, 0x96, 0x0e,
+	0xc3, 0xca, 0x0f, 0x9d, 0xbd, 0xfb, 0x29, 0x5a, 0xf3, 0xe0, 0xf3, 0x09, 0x70, 0x71, 0x0f, 0x68,
-	0x2a, 0x8e, 0xf9, 0x8d, 0xf7, 0xd1, 0x2a, 0x1d, 0xf2, 0xbe, 0xcb, 0x60, 0x14, 0xa4, 0x41, 0x1c,
+	0x08, 0x0c, 0xaf, 0xa3, 0xe6, 0xf1, 0x80, 0x34, 0x76, 0x1a, 0x77, 0x96, 0xbc, 0xe6, 0xf1, 0x00,
-	0x91, 0x79, 0x59, 0x56, 0x13, 0x41, 0x47, 0xc7, 0x9a, 0xbf, 0xac, 0xa3, 0xcd, 0x33, 0xdd, 0xb5,
+	0x77, 0xd0, 0xca, 0x84, 0xcb, 0x86, 0x13, 0x20, 0xcd, 0x9d, 0xc6, 0x9d, 0x96, 0x67, 0x7f, 0xe3,
-	0x43, 0x6f, 0xb8, 0xb6, 0x9b, 0x32, 0x7a, 0x03, 0x95, 0x47, 0x2d, 0x69, 0x51, 0x6d, 0x6d, 0x1f,
+	0xdb, 0x68, 0x8d, 0x4e, 0xc4, 0xd8, 0x67, 0x30, 0x8d, 0x78, 0x94, 0xa5, 0xe4, 0xb2, 0x4a, 0x5b,
-	0x8e, 0xbf, 0xd7, 0xa1, 0x2e, 0x71, 0xca, 0xa3, 0x16, 0xfe, 0x10, 0x2d, 0x32, 0x1a, 0xf5, 0x40,
+	0x95, 0xa2, 0x67, 0xb4, 0xdd, 0xff, 0x37, 0xd0, 0xe6, 0xb1, 0x79, 0x67, 0x8f, 0x3e, 0x12, 0xa6,
-	0x7a, 0x55, 0x5b, 0xf5, 0x09, 0xa5, 0x48, 0x65, 0x72, 0x25, 0xc4, 0xef, 0xa0, 0xf9, 0x64, 0xc8,
+	0xdc, 0x5c, 0xa1, 0x17, 0x50, 0x73, 0x7a, 0xa0, 0x4a, 0xb4, 0x0f, 0x6e, 0xec, 0x55, 0x57, 0x65,
-	0xc9, 0x82, 0xd4, 0x93, 0xa2, 0xfe, 0x72, 0x98, 0xf5, 0xe3, 0x08, 0x11, 0xee, 0xa0, 0x9a, 0x0f,
+	0xcf, 0xa4, 0x78, 0xcd, 0xe9, 0x01, 0x7e, 0x03, 0x5d, 0x61, 0x34, 0x1d, 0x81, 0xaa, 0xd5, 0x3e,
-	0x21, 0x70, 0x70, 0x95, 0xc9, 0xa2, 0x2c, 0xda, 0x2d, 0x16, 0x75, 0xa5, 0xa2, 0x60, 0x55, 0xf5,
+	0xe8, 0xcc, 0x38, 0x65, 0xa8, 0xb0, 0x6b, 0x23, 0x7e, 0x05, 0x5d, 0xce, 0x27, 0x82, 0x2c, 0x29,
-	0xf3, 0x98, 0x30, 0xe4, 0x77, 0x11, 0x59, 0xb2, 0x19, 0x5e, 0xdd, 0x45, 0xc6, 0x90, 0xdf, 0x45,
+	0x3f, 0xa9, 0xfb, 0x4f, 0x26, 0x45, 0x3f, 0x9e, 0x34, 0xe1, 0x3e, 0x5a, 0x0d, 0x21, 0x06, 0x01,
-	0xf8, 0x23, 0x84, 0xbc, 0x78, 0x90, 0x50, 0x8f, 0x8b, 0xef, 0xb7, 0x2c, 0x4b, 0x5e, 0x2b, 0x96,
+	0xbe, 0x2e, 0x72, 0x45, 0x25, 0xed, 0xd4, 0x93, 0x06, 0xca, 0x51, 0x2b, 0xd5, 0x0e, 0x4b, 0x4d,
-	0x74, 0x4c, 0x3e, 0xab, 0x1c, 0x2b, 0xc1, 0x1f, 0xa3, 0x6a, 0x08, 0x34, 0x05, 0xb7, 0xc7, 0x68,
+	0x16, 0x14, 0x67, 0x29, 0x59, 0x76, 0x15, 0x7c, 0x78, 0x96, 0xda, 0x82, 0xe2, 0x2c, 0xc5, 0x6f,
-	0xc4, 0xc9, 0x8a, 0x8d, 0x70, 0x2e, 0x04, 0x27, 0x22, 0x6f, 0x08, 0xa1, 0x09, 0x89, 0x77, 0x56,
+	0x21, 0x14, 0x64, 0x49, 0x4e, 0x03, 0x21, 0xd7, 0xef, 0xaa, 0x4a, 0x79, 0xae, 0x9e, 0xd2, 0xb7,
-	0x04, 0x06, 0xa3, 0xf8, 0x16, 0x48, 0xc5, 0xf6, 0xce, 0x12, 0xe1, 0x48, 0x81, 0x79, 0xe7, 0x30,
+	0xf1, 0x22, 0xb3, 0x92, 0x82, 0xdf, 0x46, 0xed, 0x18, 0x28, 0x07, 0x7f, 0xc4, 0x68, 0x2a, 0xc8,
-	0x8f, 0x89, 0x63, 0xa1, 0x21, 0x65, 0x03, 0x82, 0x6c, 0xc7, 0xd2, 0x16, 0x29, 0x73, 0x2c, 0x52,
+	0x8a, 0x8b, 0x70, 0x5f, 0x1a, 0x8e, 0x64, 0xdc, 0x12, 0x62, 0x2b, 0xc9, 0x77, 0xd6, 0x04, 0x06,
-	0x88, 0x3f, 0x45, 0x1b, 0xca, 0xd6, 0xeb, 0x83, 0x77, 0x9b, 0xc4, 0x41, 0xc4, 0x49, 0x55, 0x16,
+	0xd3, 0xec, 0x14, 0x48, 0xcb, 0xf5, 0xce, 0x0a, 0xe1, 0x29, 0x83, 0x7d, 0xe7, 0xb8, 0xd4, 0xe4,
-	0xbf, 0x6e, 0xb1, 0xee, 0x18, 0x51, 0x86, 0x59, 0x0f, 0x8b, 0x71, 0x7c, 0x84, 0x96, 0xfa, 0x72,
+	0xb6, 0xd0, 0x98, 0xb2, 0x84, 0x20, 0xd7, 0xb6, 0xf4, 0x64, 0xc8, 0x6e, 0x8b, 0x32, 0xe2, 0xf7,
-	0x86, 0x89, 0x2f, 0x31, 0x3b, 0xd6, 0x21, 0x52, 0x63, 0xee, 0x68, 0x29, 0x6e, 0xa3, 0xaa, 0x1c,
+	0xd0, 0x86, 0x2e, 0x1b, 0x8c, 0x21, 0x38, 0xcd, 0xb3, 0x28, 0x15, 0xa4, 0xad, 0x92, 0x9f, 0x77,
-	0x61, 0x88, 0xe8, 0x75, 0x08, 0xe4, 0x7f, 0xeb, 0x09, 0xb4, 0x87, 0xbc, 0x7f, 0x2c, 0x05, 0xe6,
+	0x94, 0xee, 0x5b, 0x53, 0x81, 0xb9, 0x16, 0xd7, 0x75, 0x7c, 0x17, 0x2d, 0x8f, 0xd5, 0x0c, 0x93,
-	0xfb, 0x51, 0x13, 0xc2, 0x5d, 0x24, 0x07, 0xde, 0xf5, 0x83, 0x54, 0x32, 0x9e, 0x2d, 0xdb, 0x3e,
+	0x50, 0x61, 0xb6, 0x9d, 0x43, 0xa4, 0xc7, 0xdc, 0x33, 0x56, 0xdc, 0x43, 0x6d, 0x35, 0xc2, 0x90,
-	0xa0, 0x60, 0x74, 0x95, 0xc2, 0x7c, 0x40, 0x9a, 0xc7, 0xf0, 0x85, 0xa2, 0x40, 0xc4, 0x03, 0x8f,
+	0xd2, 0x61, 0x0c, 0xe4, 0x1f, 0xe7, 0x0e, 0xf4, 0x26, 0x62, 0x7c, 0xa8, 0x0c, 0x76, 0xfd, 0xa8,
-	0x72, 0x20, 0xcf, 0x15, 0xe5, 0xed, 0x22, 0x25, 0x5b, 0xa4, 0xf6, 0x98, 0x34, 0xc3, 0x15, 0xea,
+	0x95, 0xf0, 0x00, 0xa9, 0x81, 0xf7, 0xc3, 0x88, 0x2b, 0xc6, 0xbf, 0x57, 0x5d, 0x0b, 0x28, 0x19,
-	0xf1, 0xb1, 0xde, 0x4d, 0xb1, 0xac, 0x2e, 0xf5, 0x7d, 0xf2, 0xeb, 0xca, 0xac, 0xb6, 0x3e, 0x4b,
+	0x03, 0xed, 0xb0, 0x0b, 0x48, 0x4b, 0x0d, 0x3f, 0xd0, 0x14, 0x48, 0x45, 0x14, 0x50, 0x01, 0xe4,
-	0x81, 0xb5, 0x7d, 0xbf, 0xd0, 0x96, 0x8e, 0xe1, 0x0b, 0xb4, 0x91, 0x63, 0xd4, 0x90, 0x93, 0xdf,
+	0x3f, 0x4d, 0x79, 0xb9, 0x4e, 0x29, 0x0e, 0x52, 0xaf, 0x62, 0x2d, 0x70, 0xb5, 0x7c, 0x7c, 0x68,
-	0x14, 0x69, 0xdf, 0x4e, 0xd2, 0xdb, 0xa1, 0x61, 0x6b, 0xb4, 0x10, 0x2e, 0xb6, 0xd5, 0x03, 0x4e,
+	0xce, 0xa6, 0x3c, 0xac, 0x3e, 0x0d, 0x43, 0xf2, 0xd3, 0xca, 0xa2, 0xb6, 0x3e, 0xe0, 0xc0, 0x7a,
-	0x7e, 0x7f, 0xb2, 0xad, 0x13, 0xe0, 0x53, 0x6d, 0x9d, 0x00, 0xc7, 0x3d, 0xf4, 0x6a, 0x8e, 0xf1,
+	0x61, 0x58, 0x6b, 0xcb, 0x68, 0xf8, 0x01, 0xda, 0x28, 0x31, 0x7a, 0xc8, 0xc9, 0xcf, 0x9a, 0x74,
-	0xfa, 0x62, 0xed, 0xdc, 0x84, 0xa6, 0xe9, 0x37, 0x31, 0xf3, 0xc9, 0x1f, 0x0a, 0xf9, 0xae, 0x1d,
+	0xdb, 0x4d, 0x32, 0xa7, 0xc3, 0xc0, 0xd6, 0x69, 0x4d, 0xae, 0xb7, 0x35, 0x02, 0x41, 0x7e, 0xb9,
-	0xd9, 0x91, 0xea, 0x4b, 0x2d, 0xce, 0xe8, 0x2f, 0x53, 0x6b, 0x1a, 0x7f, 0x8e, 0xb6, 0xc6, 0xfa,
+	0xb0, 0xad, 0x23, 0x10, 0x73, 0x6d, 0x1d, 0x81, 0xc0, 0x23, 0xf4, 0x6c, 0x89, 0x09, 0xc6, 0xf2,
-	0x15, 0xfb, 0xe2, 0xb2, 0x38, 0x04, 0xf2, 0xa0, 0x3c, 0xde, 0x9c, 0xd1, 0xb6, 0xdc, 0xb5, 0x38,
+	0xd8, 0xf9, 0x39, 0xe5, 0xfc, 0x8b, 0x8c, 0x85, 0xe4, 0x57, 0x8d, 0x7c, 0xd5, 0x8d, 0xec, 0x2b,
-	0x3f, 0xea, 0x97, 0xe8, 0x64, 0x06, 0x7f, 0x81, 0xb6, 0x73, 0xb2, 0x5a, 0x3d, 0x85, 0xfe, 0x53,
+	0xf7, 0x89, 0x31, 0x17, 0xf4, 0x67, 0xa8, 0x33, 0x8c, 0x3f, 0x46, 0x5b, 0x95, 0x7e, 0xe5, 0x79,
-	0xa1, 0xdf, 0xb2, 0xa3, 0xf5, 0x0e, 0x8e, 0xb1, 0x31, 0x9d, 0x4a, 0xe1, 0x53, 0xb4, 0x96, 0xc3,
+	0xf1, 0x59, 0x16, 0x03, 0x79, 0xa2, 0x6b, 0xbc, 0xb8, 0xa0, 0x6d, 0x75, 0xd6, 0xb2, 0x72, 0xab,
-	0xc3, 0x20, 0xe5, 0xe4, 0x2f, 0x45, 0xdd, 0xb3, 0x53, 0xcf, 0x83, 0x94, 0x17, 0xe6, 0x28, 0x0b,
+	0xaf, 0xd3, 0xd9, 0x08, 0xfe, 0x04, 0xdd, 0x28, 0xc9, 0xfa, 0xe8, 0x69, 0xf4, 0x6f, 0x1a, 0xfd,
-	0x1a, 0x92, 0x68, 0x4d, 0x91, 0xfe, 0x9e, 0x49, 0x12, 0xd6, 0x53, 0xa4, 0x2c, 0x68, 0x8e, 0x5e,
+	0x92, 0x1b, 0x6d, 0xce, 0x60, 0x85, 0x8d, 0xe9, 0x5c, 0x08, 0xdf, 0x43, 0xeb, 0x25, 0x3c, 0x8e,
-	0x92, 0xc4, 0x44, 0x7e, 0x5f, 0x99, 0x75, 0xf4, 0xa2, 0x66, 0x72, 0x22, 0x75, 0xcc, 0x4c, 0xa4,
+	0xb8, 0x20, 0xbf, 0x6b, 0xea, 0x2d, 0x37, 0xf5, 0x7e, 0xc4, 0x45, 0x6d, 0x8e, 0x0a, 0xd1, 0x92,
-	0xc4, 0xe8, 0x89, 0xfc, 0xa1, 0x32, 0x6b, 0x22, 0x45, 0x95, 0x65, 0x22, 0xf3, 0x70, 0xb1, 0x2d,
+	0x64, 0x6b, 0x9a, 0xf4, 0xc7, 0x42, 0x92, 0x2c, 0x3d, 0x47, 0x2a, 0x44, 0xbb, 0xf5, 0x8a, 0x24,
-	0x31, 0x91, 0x3f, 0x3e, 0xd9, 0xd6, 0xe4, 0x44, 0xea, 0x18, 0xfe, 0x0a, 0xd5, 0xc7, 0x30, 0x72,
+	0x27, 0xf2, 0xdb, 0xd6, 0xa2, 0xad, 0x97, 0x39, 0xb3, 0x13, 0x69, 0x34, 0x3b, 0x91, 0x0a, 0x63,
-	0x50, 0x12, 0x60, 0x83, 0x20, 0x95, 0xff, 0x18, 0x7f, 0x52, 0xcc, 0xf7, 0x66, 0x30, 0x85, 0xfc,
+	0x26, 0xf2, 0xbb, 0xd6, 0xa2, 0x89, 0x94, 0x59, 0x8e, 0x89, 0x2c, 0xe5, 0x7a, 0x5b, 0x72, 0x22,
-	0xd2, 0xa8, 0x33, 0xfe, 0x2b, 0xd4, 0x9e, 0xc7, 0x03, 0xb4, 0x93, 0x7b, 0xe9, 0xd1, 0x19, 0x33,
+	0xbf, 0xbf, 0xb0, 0xad, 0xd9, 0x89, 0x34, 0x1a, 0xfe, 0x0c, 0x75, 0x2a, 0x18, 0x35, 0x28, 0x39,
-	0xfb, 0x59, 0x99, 0xbd, 0x6f, 0x37, 0x53, 0x53, 0x32, 0xed, 0x46, 0xe8, 0x0c, 0x41, 0x73, 0x1d,
+	0xb0, 0x24, 0xe2, 0xea, 0x8f, 0xf1, 0x07, 0xcd, 0x7c, 0x6d, 0x01, 0x53, 0xda, 0x4f, 0xac, 0xbb,
-	0xad, 0x1e, 0x0f, 0x12, 0xfe, 0xad, 0x03, 0x69, 0x12, 0x47, 0x29, 0x34, 0x13, 0xb4, 0xf3, 0xc4,
+	0xe0, 0xdf, 0xa4, 0xee, 0x38, 0x4e, 0xd0, 0x76, 0x59, 0xcb, 0x8c, 0x4e, 0xa5, 0xd8, 0x8f, 0xba,
-	0x1f, 0x22, 0x8c, 0xd1, 0x82, 0xbc, 0x2e, 0x94, 0xe4, 0x75, 0x41, 0x3e, 0x8b, 0x6b, 0x84, 0xd9,
+	0xd8, 0xeb, 0xee, 0x62, 0x7a, 0x4a, 0xe6, 0xab, 0x11, 0xba, 0xc0, 0x80, 0x3f, 0x42, 0x9b, 0x41,
-	0x4f, 0x7d, 0x8d, 0xc8, 0x7e, 0xe3, 0x3d, 0x54, 0x4b, 0x83, 0x41, 0x12, 0x82, 0xcb, 0xe3, 0x5b,
+	0x3c, 0xe1, 0x02, 0x98, 0x3f, 0x05, 0x26, 0x25, 0x9f, 0x83, 0x20, 0x5f, 0x21, 0x73, 0x04, 0xaa,
-	0x50, 0xb7, 0x88, 0x8a, 0x53, 0x55, 0xb1, 0x2b, 0x11, 0xfa, 0x64, 0xeb, 0xfe, 0xdf, 0xc6, 0xdc,
+	0x37, 0x8c, 0xbd, 0xbe, 0x76, 0x7e, 0xa8, 0x8d, 0xef, 0x97, 0xab, 0x75, 0x3d, 0x98, 0x8d, 0x60,
-	0xfd, 0x63, 0xa3, 0xf4, 0xf0, 0xd8, 0x28, 0xfd, 0xf3, 0xd8, 0x28, 0x7d, 0xf7, 0x5f, 0x63, 0xee,
+	0x8a, 0x6e, 0x16, 0x60, 0xcd, 0xf0, 0xa9, 0x10, 0x4c, 0xc1, 0xbf, 0x46, 0xe6, 0xf3, 0xe7, 0x82,
-	0x7a, 0x49, 0xde, 0x61, 0x8e, 0x5e, 0x04, 0x00, 0x00, 0xff, 0xff, 0xed, 0x36, 0xf0, 0x6f, 0x1b,
+	0xbf, 0xab, 0xb4, 0x9e, 0x10, 0xac, 0xc2, 0xdf, 0x0a, 0x1c, 0xc1, 0xdd, 0x6b, 0x68, 0xed, 0x30,
-	0x09, 0x00, 0x00,
+	0xc9, 0xc5, 0x97, 0x1e, 0xf0, 0x3c, 0x4b, 0x39, 0xec, 0xe6, 0x68, 0xfb, 0x82, 0x8f, 0x28, 0xc6,
+	0x68, 0x49, 0x5d, 0x75, 0x1a, 0xea, 0xaa, 0xa3, 0x9e, 0xe5, 0x15, 0xc8, 0x7e, 0x5b, 0xcc, 0x15,
+	0xa8, 0xf8, 0x8d, 0x6f, 0xa1, 0x55, 0x1e, 0x25, 0x79, 0x0c, 0xbe, 0xc8, 0x4e, 0x41, 0xdf, 0x80,
+	0x5a, 0x5e, 0x5b, 0x6b, 0x0f, 0xa5, 0xf4, 0xce, 0xd6, 0xe3, 0xbf, 0xba, 0x97, 0x1e, 0x9f, 0x77,
+	0x1b, 0x4f, 0xce, 0xbb, 0x8d, 0x3f, 0xcf, 0xbb, 0x8d, 0x6f, 0xfe, 0xee, 0x5e, 0x1a, 0x2e, 0xab,
+	0xfb, 0xd7, 0xdd, 0xa7, 0x01, 0x00, 0x00, 0xff, 0xff, 0x39, 0xbf, 0x15, 0x6f, 0x15, 0x0a, 0x00,
+	0x00,
 }

etcdserver/etcdserverpb/raft_internal.proto

@@ -4,6 +4,7 @@ package etcdserverpb;
 import "gogoproto/gogo.proto";
 import "etcdserver.proto";
 import "rpc.proto";
+import "etcd/etcdserver/api/membership/membershippb/membership.proto";
 
 option (gogoproto.marshaler_all) = true;
 option (gogoproto.sizer_all) = true;
@@ -58,6 +59,9 @@ message InternalRaftRequest {
   AuthRoleGetRequest auth_role_get = 1202;
   AuthRoleGrantPermissionRequest auth_role_grant_permission = 1203;
   AuthRoleRevokePermissionRequest auth_role_revoke_permission = 1204;
+
+  membershippb.ClusterVersionSetRequest cluster_version_set = 1300;
+  membershippb.ClusterMemberAttrSetRequest cluster_member_attr_set = 1301;
 }
 
 message EmptyResponse {

etcdserver/server.go

@@ -32,6 +32,7 @@ import (
 	"go.etcd.io/etcd/auth"
 	"go.etcd.io/etcd/etcdserver/api"
 	"go.etcd.io/etcd/etcdserver/api/membership"
+	"go.etcd.io/etcd/etcdserver/api/membership/membershippb"
 	"go.etcd.io/etcd/etcdserver/api/rafthttp"
 	"go.etcd.io/etcd/etcdserver/api/snap"
 	"go.etcd.io/etcd/etcdserver/api/v2discovery"
@@ -239,7 +240,9 @@ type EtcdServer struct {
 	applyV3 applierV3
 	// applyV3Base is the core applier without auth or quotas
 	applyV3Base applierV3
-	applyWait   wait.WaitTime
+	// applyV3Internal is the applier for internal request
+	applyV3Internal applierV3Internal
+	applyWait       wait.WaitTime
 
 	kv         mvcc.ConsistentWatchableKV
 	lessor     lease.Lessor
@@ -592,6 +595,7 @@ func NewServer(cfg ServerConfig) (srv *EtcdServer, err error) {
 	}
 
 	srv.applyV3Base = srv.newApplierV3Backend()
+	srv.applyV3Internal = srv.newApplierV3Internal()
 	if err = srv.restoreAlarms(); err != nil {
 		return nil, err
 	}
@@ -1988,6 +1992,62 @@ func (s *EtcdServer) sync(timeout time.Duration) {
 	})
 }
 
+// publishV3 registers server information into the cluster using v3 request. The
+// information is the JSON representation of this server's member struct, updated
+// with the static clientURLs of the server.
+// The function keeps attempting to register until it succeeds,
+// or its server is stopped.
+// TODO: replace publish() in 3.6
+func (s *EtcdServer) publishV3(timeout time.Duration) {
+	req := &membershippb.ClusterMemberAttrSetRequest{
+		Member_ID: uint64(s.id),
+		MemberAttributes: &membershippb.Attributes{
+			Name:       s.attributes.Name,
+			ClientUrls: s.attributes.ClientURLs,
+		},
+	}
+	lg := s.getLogger()
+	for {
+		select {
+		case <-s.stopping:
+			lg.Warn(
+				"stopped publish because server is stopping",
+				zap.String("local-member-id", s.ID().String()),
+				zap.String("local-member-attributes", fmt.Sprintf("%+v", s.attributes)),
+				zap.Duration("publish-timeout", timeout),
+			)
+			return
+
+		default:
+		}
+
+		ctx, cancel := context.WithTimeout(s.ctx, timeout)
+		_, err := s.raftRequest(ctx, pb.InternalRaftRequest{ClusterMemberAttrSet: req})
+		cancel()
+		switch err {
+		case nil:
+			close(s.readych)
+			lg.Info(
+				"published local member to cluster through raft",
+				zap.String("local-member-id", s.ID().String()),
+				zap.String("local-member-attributes", fmt.Sprintf("%+v", s.attributes)),
+				zap.String("cluster-id", s.cluster.ID().String()),
+				zap.Duration("publish-timeout", timeout),
+			)
+			return
+
+		default:
+			lg.Warn(
+				"failed to publish local member to cluster through raft",
+				zap.String("local-member-id", s.ID().String()),
+				zap.String("local-member-attributes", fmt.Sprintf("%+v", s.attributes)),
+				zap.Duration("publish-timeout", timeout),
+				zap.Error(err),
+			)
+		}
+	}
+}
+
 // publish registers server information into the cluster. The information
 // is the JSON representation of this server's member struct, updated with the
 // static clientURLs of the server.
@@ -1998,7 +2058,7 @@ func (s *EtcdServer) sync(timeout time.Duration) {
 // but does not go through v2 API endpoint, which means even with v2
 // client handler disabled (e.g. --enable-v2=false), cluster can still
 // process publish requests through rafthttp
-// TODO: Deprecate v2 store
+// TODO: Deprecate v2 store in 3.6
 func (s *EtcdServer) publish(timeout time.Duration) {
 	b, err := json.Marshal(s.attributes)
 	if err != nil {
@@ -2527,14 +2587,10 @@ func (s *EtcdServer) updateClusterVersion(ver string) {
 		}
 	}
 
-	req := pb.Request{
+	req := membershippb.ClusterVersionSetRequest{Ver: ver}
-		Method: "PUT",
-		Path:   membership.StoreClusterVersionKey(),
-		Val:    ver,
-	}
 
 	ctx, cancel := context.WithTimeout(s.ctx, s.Cfg.ReqTimeout())
-	_, err := s.Do(ctx, req)
+	_, err := s.raftRequest(ctx, pb.InternalRaftRequest{ClusterVersionSet: &req})
 	cancel()
 
 	switch err {

etcdserver/server_test.go

@@ -21,7 +21,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
-	"path"
 	"path/filepath"
 	"reflect"
 	"sync"
@@ -1505,53 +1504,6 @@ func TestPublishRetry(t *testing.T) {
 	<-ch
 }
 
-func TestUpdateVersion(t *testing.T) {
-	n := newNodeRecorder()
-	ch := make(chan interface{}, 1)
-	// simulate that request has gone through consensus
-	ch <- Response{}
-	w := wait.NewWithResponse(ch)
-	ctx, cancel := context.WithCancel(context.TODO())
-	srv := &EtcdServer{
-		lgMu:       new(sync.RWMutex),
-		lg:         zap.NewExample(),
-		id:         1,
-		Cfg:        ServerConfig{Logger: zap.NewExample(), TickMs: 1, SnapshotCatchUpEntries: DefaultSnapshotCatchUpEntries},
-		r:          *newRaftNode(raftNodeConfig{lg: zap.NewExample(), Node: n}),
-		attributes: membership.Attributes{Name: "node1", ClientURLs: []string{"http://node1.com"}},
-		cluster:    &membership.RaftCluster{},
-		w:          w,
-		reqIDGen:   idutil.NewGenerator(0, time.Time{}),
-		SyncTicker: &time.Ticker{},
-
-		ctx:    ctx,
-		cancel: cancel,
-	}
-	srv.updateClusterVersion("2.0.0")
-
-	action := n.Action()
-	if len(action) != 1 {
-		t.Fatalf("len(action) = %d, want 1", len(action))
-	}
-	if action[0].Name != "Propose" {
-		t.Fatalf("action = %s, want Propose", action[0].Name)
-	}
-	data := action[0].Params[0].([]byte)
-	var r pb.Request
-	if err := r.Unmarshal(data); err != nil {
-		t.Fatalf("unmarshal request error: %v", err)
-	}
-	if r.Method != "PUT" {
-		t.Errorf("method = %s, want PUT", r.Method)
-	}
-	if wpath := path.Join(StoreClusterPrefix, "version"); r.Path != wpath {
-		t.Errorf("path = %s, want %s", r.Path, wpath)
-	}
-	if r.Val != "2.0.0" {
-		t.Errorf("val = %s, want %s", r.Val, "2.0.0")
-	}
-}
-
 func TestStopNotify(t *testing.T) {
 	s := &EtcdServer{
 		lgMu: new(sync.RWMutex),

integration/v3_grpc_test.go

@@ -1898,7 +1898,7 @@ func TestV3LargeRequests(t *testing.T) {
 		expectError     error
 	}{
 		// don't set to 0. use 0 as the default.
-		{1, 1024, rpctypes.ErrGRPCRequestTooLarge},
+		{256, 1024, rpctypes.ErrGRPCRequestTooLarge},
 		{10 * 1024 * 1024, 9 * 1024 * 1024, nil},
 		{10 * 1024 * 1024, 10 * 1024 * 1024, rpctypes.ErrGRPCRequestTooLarge},
 		{10 * 1024 * 1024, 10*1024*1024 + 5, rpctypes.ErrGRPCRequestTooLarge},

scripts/genproto.sh

@@ -16,7 +16,7 @@ if [[ $(protoc --version | cut -f2 -d' ') != "3.7.1" ]]; then
 fi
 
 # directories containing protos to be built
-DIRS="./wal/walpb ./etcdserver/etcdserverpb ./etcdserver/api/snap/snappb ./raft/raftpb ./mvcc/mvccpb ./lease/leasepb ./auth/authpb ./etcdserver/api/v3lock/v3lockpb ./etcdserver/api/v3election/v3electionpb"
+DIRS="./wal/walpb ./etcdserver/etcdserverpb ./etcdserver/api/snap/snappb ./raft/raftpb ./mvcc/mvccpb ./lease/leasepb ./auth/authpb ./etcdserver/api/v3lock/v3lockpb ./etcdserver/api/v3election/v3electionpb ./etcdserver/api/membership/membershippb"
 
 # disable go mod
 export GO111MODULE=off

tests/e2e/ctl_v3_migrate_test.go

@@ -28,7 +28,9 @@ import (
 func TestCtlV3Migrate(t *testing.T) {
 	defer testutil.AfterTest(t)
 
-	epc := setupEtcdctlTest(t, &configNoTLS, false)
+	cfg := configNoTLS
+	cfg.enableV2 = true
+	epc := setupEtcdctlTest(t, &cfg, false)
 	defer func() {
 		if errC := epc.Close(); errC != nil {
 			t.Fatalf("error closing etcd processes (%v)", errC)
@@ -69,10 +71,6 @@ func TestCtlV3Migrate(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// to ensure revision increment is continuous from migrated v2 data
-	if err := ctlV3Put(cx, "test", "value", ""); err != nil {
-		t.Fatal(err)
-	}
 	cli, err := clientv3.New(clientv3.Config{
 		Endpoints:   epc.EndpointsV3(),
 		DialTimeout: 3 * time.Second,
@@ -85,11 +83,22 @@ func TestCtlV3Migrate(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	revAfterMigrate := resp.Header.Revision
+	// to ensure revision increment is continuous from migrated v2 data
+	if err := ctlV3Put(cx, "test", "value", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	resp, err = cli.Get(context.TODO(), "test")
+	if err != nil {
+		t.Fatal(err)
+	}
 	if len(resp.Kvs) != 1 {
 		t.Fatalf("len(resp.Kvs) expected 1, got %+v", resp.Kvs)
 	}
-	if resp.Kvs[0].CreateRevision != 7 {
+
-		t.Fatalf("resp.Kvs[0].CreateRevision expected 7, got %d", resp.Kvs[0].CreateRevision)
+	if resp.Kvs[0].CreateRevision != revAfterMigrate+1 {
+		t.Fatalf("expected revision increment is continuous from migrated v2, got %d", resp.Kvs[0].CreateRevision)
 	}
 }