// Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import ( "context" "github.com/coreos/go-semver/semver" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/membershippb" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/auth" "go.etcd.io/etcd/server/v3/etcdserver/api" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/version" "go.etcd.io/etcd/server/v3/lease" serverstorage "go.etcd.io/etcd/server/v3/storage" "go.etcd.io/etcd/server/v3/storage/mvcc" "github.com/gogo/protobuf/proto" "go.uber.org/zap" ) const ( v3Version = "v3" ) type applyResult struct { resp proto.Message err error // physc signals the physical effect of the request has completed in addition // to being logically reflected by the node. Currently, only used for // Compaction requests. physc <-chan struct{} trace *traceutil.Trace } type ApplyFunc func(ctx context.Context, r *pb.InternalRaftRequest, shouldApplyV3 membership.ShouldApplyV3) *applyResult // applierV3 is the interface for processing V3 raft messages type applierV3 interface { WrapApply(ctx context.Context, r *pb.InternalRaftRequest, shouldApplyV3 membership.ShouldApplyV3, applyFunc ApplyFunc) *applyResult //Apply(r *pb.InternalRaftRequest, shouldApplyV3 membership.ShouldApplyV3) *applyResult Put(ctx context.Context, txn mvcc.TxnWrite, p *pb.PutRequest) (*pb.PutResponse, *traceutil.Trace, error) Range(ctx context.Context, txn mvcc.TxnRead, r *pb.RangeRequest) (*pb.RangeResponse, error) DeleteRange(txn mvcc.TxnWrite, dr *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, error) Txn(ctx context.Context, rt *pb.TxnRequest) (*pb.TxnResponse, *traceutil.Trace, error) Compaction(compaction *pb.CompactionRequest) (*pb.CompactionResponse, <-chan struct{}, *traceutil.Trace, error) LeaseGrant(lc *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) LeaseRevoke(lc *pb.LeaseRevokeRequest) (*pb.LeaseRevokeResponse, error) LeaseCheckpoint(lc *pb.LeaseCheckpointRequest) (*pb.LeaseCheckpointResponse, error) Alarm(*pb.AlarmRequest) (*pb.AlarmResponse, error) Authenticate(r *pb.InternalAuthenticateRequest) (*pb.AuthenticateResponse, error) AuthEnable() (*pb.AuthEnableResponse, error) AuthDisable() (*pb.AuthDisableResponse, error) AuthStatus() (*pb.AuthStatusResponse, error) UserAdd(ua *pb.AuthUserAddRequest) (*pb.AuthUserAddResponse, error) UserDelete(ua *pb.AuthUserDeleteRequest) (*pb.AuthUserDeleteResponse, error) UserChangePassword(ua *pb.AuthUserChangePasswordRequest) (*pb.AuthUserChangePasswordResponse, error) UserGrantRole(ua *pb.AuthUserGrantRoleRequest) (*pb.AuthUserGrantRoleResponse, error) UserGet(ua *pb.AuthUserGetRequest) (*pb.AuthUserGetResponse, error) UserRevokeRole(ua *pb.AuthUserRevokeRoleRequest) (*pb.AuthUserRevokeRoleResponse, error) RoleAdd(ua *pb.AuthRoleAddRequest) (*pb.AuthRoleAddResponse, error) RoleGrantPermission(ua *pb.AuthRoleGrantPermissionRequest) (*pb.AuthRoleGrantPermissionResponse, error) RoleGet(ua *pb.AuthRoleGetRequest) (*pb.AuthRoleGetResponse, error) RoleRevokePermission(ua *pb.AuthRoleRevokePermissionRequest) (*pb.AuthRoleRevokePermissionResponse, error) RoleDelete(ua *pb.AuthRoleDeleteRequest) (*pb.AuthRoleDeleteResponse, error) UserList(ua *pb.AuthUserListRequest) (*pb.AuthUserListResponse, error) RoleList(ua *pb.AuthRoleListRequest) (*pb.AuthRoleListResponse, error) // processing internal V3 raft request ClusterVersionSet(r *membershippb.ClusterVersionSetRequest, shouldApplyV3 membership.ShouldApplyV3) ClusterMemberAttrSet(r *membershippb.ClusterMemberAttrSetRequest, shouldApplyV3 membership.ShouldApplyV3) DowngradeInfoSet(r *membershippb.DowngradeInfoSetRequest, shouldApplyV3 membership.ShouldApplyV3) } type applierV3backend struct { s *EtcdServer } func (a *applierV3backend) WrapApply(ctx context.Context, r *pb.InternalRaftRequest, shouldApplyV3 membership.ShouldApplyV3, applyFunc ApplyFunc) *applyResult { return applyFunc(ctx, r, shouldApplyV3) } func (a *applierV3backend) Put(ctx context.Context, txn mvcc.TxnWrite, p *pb.PutRequest) (resp *pb.PutResponse, trace *traceutil.Trace, err error) { return Put(ctx, a.s.Logger(), a.s.lessor, a.s.KV(), txn, p) } func (a *applierV3backend) DeleteRange(txn mvcc.TxnWrite, dr *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, error) { return DeleteRange(a.s.KV(), txn, dr) } func (a *applierV3backend) Range(ctx context.Context, txn mvcc.TxnRead, r *pb.RangeRequest) (*pb.RangeResponse, error) { return Range(ctx, a.s.Logger(), a.s.KV(), txn, r) } func (a *applierV3backend) Txn(ctx context.Context, rt *pb.TxnRequest) (*pb.TxnResponse, *traceutil.Trace, error) { return Txn(ctx, a.s.Logger(), rt, a.s.Cfg.ExperimentalTxnModeWriteWithSharedBuffer, a.s.KV(), a.s.lessor) } func (a *applierV3backend) Compaction(compaction *pb.CompactionRequest) (*pb.CompactionResponse, <-chan struct{}, *traceutil.Trace, error) { resp := &pb.CompactionResponse{} resp.Header = &pb.ResponseHeader{} trace := traceutil.New("compact", a.s.Logger(), traceutil.Field{Key: "revision", Value: compaction.Revision}, ) ch, err := a.s.KV().Compact(trace, compaction.Revision) if err != nil { return nil, ch, nil, err } // get the current revision. which key to get is not important. rr, _ := a.s.KV().Range(context.TODO(), []byte("compaction"), nil, mvcc.RangeOptions{}) resp.Header.Revision = rr.Rev return resp, ch, trace, err } func (a *applierV3backend) LeaseGrant(lc *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) { l, err := a.s.lessor.Grant(lease.LeaseID(lc.ID), lc.TTL) resp := &pb.LeaseGrantResponse{} if err == nil { resp.ID = int64(l.ID) resp.TTL = l.TTL() resp.Header = newHeader(a.s) } return resp, err } func (a *applierV3backend) LeaseRevoke(lc *pb.LeaseRevokeRequest) (*pb.LeaseRevokeResponse, error) { err := a.s.lessor.Revoke(lease.LeaseID(lc.ID)) return &pb.LeaseRevokeResponse{Header: newHeader(a.s)}, err } func (a *applierV3backend) LeaseCheckpoint(lc *pb.LeaseCheckpointRequest) (*pb.LeaseCheckpointResponse, error) { for _, c := range lc.Checkpoints { err := a.s.lessor.Checkpoint(lease.LeaseID(c.ID), c.Remaining_TTL) if err != nil { return &pb.LeaseCheckpointResponse{Header: newHeader(a.s)}, err } } return &pb.LeaseCheckpointResponse{Header: newHeader(a.s)}, nil } func (a *applierV3backend) Alarm(ar *pb.AlarmRequest) (*pb.AlarmResponse, error) { resp := &pb.AlarmResponse{} switch ar.Action { case pb.AlarmRequest_GET: resp.Alarms = a.s.alarmStore.Get(ar.Alarm) case pb.AlarmRequest_ACTIVATE: if ar.Alarm == pb.AlarmType_NONE { break } m := a.s.alarmStore.Activate(types.ID(ar.MemberID), ar.Alarm) if m == nil { break } resp.Alarms = append(resp.Alarms, m) case pb.AlarmRequest_DEACTIVATE: m := a.s.alarmStore.Deactivate(types.ID(ar.MemberID), ar.Alarm) if m == nil { break } resp.Alarms = append(resp.Alarms, m) default: return nil, nil } return resp, nil } type applierV3Capped struct { applierV3 q serverstorage.BackendQuota } // newApplierV3Capped creates an applyV3 that will reject Puts and transactions // with Puts so that the number of keys in the store is capped. func newApplierV3Capped(base applierV3) applierV3 { return &applierV3Capped{applierV3: base} } func (a *applierV3Capped) Put(_ context.Context, _ mvcc.TxnWrite, _ *pb.PutRequest) (*pb.PutResponse, *traceutil.Trace, error) { return nil, nil, ErrNoSpace } func (a *applierV3Capped) Txn(ctx context.Context, r *pb.TxnRequest) (*pb.TxnResponse, *traceutil.Trace, error) { if a.q.Cost(r) > 0 { return nil, nil, ErrNoSpace } return a.applierV3.Txn(ctx, r) } func (a *applierV3Capped) LeaseGrant(_ *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) { return nil, ErrNoSpace } func (a *applierV3backend) AuthEnable() (*pb.AuthEnableResponse, error) { err := a.s.AuthStore().AuthEnable() if err != nil { return nil, err } return &pb.AuthEnableResponse{Header: newHeader(a.s)}, nil } func (a *applierV3backend) AuthDisable() (*pb.AuthDisableResponse, error) { a.s.AuthStore().AuthDisable() return &pb.AuthDisableResponse{Header: newHeader(a.s)}, nil } func (a *applierV3backend) AuthStatus() (*pb.AuthStatusResponse, error) { enabled := a.s.AuthStore().IsAuthEnabled() authRevision := a.s.AuthStore().Revision() return &pb.AuthStatusResponse{Header: newHeader(a.s), Enabled: enabled, AuthRevision: authRevision}, nil } func (a *applierV3backend) Authenticate(r *pb.InternalAuthenticateRequest) (*pb.AuthenticateResponse, error) { ctx := context.WithValue(context.WithValue(a.s.ctx, auth.AuthenticateParamIndex{}, a.s.consistIndex.ConsistentIndex()), auth.AuthenticateParamSimpleTokenPrefix{}, r.SimpleToken) resp, err := a.s.AuthStore().Authenticate(ctx, r.Name, r.Password) if resp != nil { resp.Header = newHeader(a.s) } return resp, err } func (a *applierV3backend) UserAdd(r *pb.AuthUserAddRequest) (*pb.AuthUserAddResponse, error) { resp, err := a.s.AuthStore().UserAdd(r) if resp != nil { resp.Header = newHeader(a.s) } return resp, err } func (a *applierV3backend) UserDelete(r *pb.AuthUserDeleteRequest) (*pb.AuthUserDeleteResponse, error) { resp, err := a.s.AuthStore().UserDelete(r) if resp != nil { resp.Header = newHeader(a.s) } return resp, err } func (a *applierV3backend) UserChangePassword(r *pb.AuthUserChangePasswordRequest) (*pb.AuthUserChangePasswordResponse, error) { resp, err := a.s.AuthStore().UserChangePassword(r) if resp != nil { resp.Header = newHeader(a.s) } return resp, err } func (a *applierV3backend) UserGrantRole(r *pb.AuthUserGrantRoleRequest) (*pb.AuthUserGrantRoleResponse, error) { resp, err := a.s.AuthStore().UserGrantRole(r) if resp != nil { resp.Header = newHeader(a.s) } return resp, err } func (a *applierV3backend) UserGet(r *pb.AuthUserGetRequest) (*pb.AuthUserGetResponse, error) { resp, err := a.s.AuthStore().UserGet(r) if resp != nil { resp.Header = newHeader(a.s) } return resp, err } func (a *applierV3backend) UserRevokeRole(r *pb.AuthUserRevokeRoleRequest) (*pb.AuthUserRevokeRoleResponse, error) { resp, err := a.s.AuthStore().UserRevokeRole(r) if resp != nil { resp.Header = newHeader(a.s) } return resp, err } func (a *applierV3backend) RoleAdd(r *pb.AuthRoleAddRequest) (*pb.AuthRoleAddResponse, error) { resp, err := a.s.AuthStore().RoleAdd(r) if resp != nil { resp.Header = newHeader(a.s) } return resp, err } func (a *applierV3backend) RoleGrantPermission(r *pb.AuthRoleGrantPermissionRequest) (*pb.AuthRoleGrantPermissionResponse, error) { resp, err := a.s.AuthStore().RoleGrantPermission(r) if resp != nil { resp.Header = newHeader(a.s) } return resp, err } func (a *applierV3backend) RoleGet(r *pb.AuthRoleGetRequest) (*pb.AuthRoleGetResponse, error) { resp, err := a.s.AuthStore().RoleGet(r) if resp != nil { resp.Header = newHeader(a.s) } return resp, err } func (a *applierV3backend) RoleRevokePermission(r *pb.AuthRoleRevokePermissionRequest) (*pb.AuthRoleRevokePermissionResponse, error) { resp, err := a.s.AuthStore().RoleRevokePermission(r) if resp != nil { resp.Header = newHeader(a.s) } return resp, err } func (a *applierV3backend) RoleDelete(r *pb.AuthRoleDeleteRequest) (*pb.AuthRoleDeleteResponse, error) { resp, err := a.s.AuthStore().RoleDelete(r) if resp != nil { resp.Header = newHeader(a.s) } return resp, err } func (a *applierV3backend) UserList(r *pb.AuthUserListRequest) (*pb.AuthUserListResponse, error) { resp, err := a.s.AuthStore().UserList(r) if resp != nil { resp.Header = newHeader(a.s) } return resp, err } func (a *applierV3backend) RoleList(r *pb.AuthRoleListRequest) (*pb.AuthRoleListResponse, error) { resp, err := a.s.AuthStore().RoleList(r) if resp != nil { resp.Header = newHeader(a.s) } return resp, err } func (a *applierV3backend) ClusterVersionSet(r *membershippb.ClusterVersionSetRequest, shouldApplyV3 membership.ShouldApplyV3) { prevVersion := a.s.Cluster().Version() newVersion := semver.Must(semver.NewVersion(r.Ver)) a.s.cluster.SetVersion(newVersion, api.UpdateCapability, shouldApplyV3) // Force snapshot after cluster version downgrade. if prevVersion != nil && newVersion.LessThan(*prevVersion) { lg := a.s.Logger() if lg != nil { lg.Info("Cluster version downgrade detected, forcing snapshot", zap.String("prev-cluster-version", prevVersion.String()), zap.String("new-cluster-version", newVersion.String()), ) } a.s.forceSnapshot = true } } func (a *applierV3backend) ClusterMemberAttrSet(r *membershippb.ClusterMemberAttrSetRequest, shouldApplyV3 membership.ShouldApplyV3) { a.s.cluster.UpdateAttributes( types.ID(r.Member_ID), membership.Attributes{ Name: r.MemberAttributes.Name, ClientURLs: r.MemberAttributes.ClientUrls, }, shouldApplyV3, ) } func (a *applierV3backend) DowngradeInfoSet(r *membershippb.DowngradeInfoSetRequest, shouldApplyV3 membership.ShouldApplyV3) { d := version.DowngradeInfo{Enabled: false} if r.Enabled { d = version.DowngradeInfo{Enabled: true, TargetVersion: r.Ver} } a.s.cluster.SetDowngradeInfo(&d, shouldApplyV3) } type quotaApplierV3 struct { applierV3 q serverstorage.Quota } func newQuotaApplierV3(s *EtcdServer, app applierV3) applierV3 { return "aApplierV3{app, serverstorage.NewBackendQuota(s.Cfg, s.Backend(), "v3-applier")} } func (a *quotaApplierV3) Put(ctx context.Context, txn mvcc.TxnWrite, p *pb.PutRequest) (*pb.PutResponse, *traceutil.Trace, error) { ok := a.q.Available(p) resp, trace, err := a.applierV3.Put(ctx, txn, p) if err == nil && !ok { err = ErrNoSpace } return resp, trace, err } func (a *quotaApplierV3) Txn(ctx context.Context, rt *pb.TxnRequest) (*pb.TxnResponse, *traceutil.Trace, error) { ok := a.q.Available(rt) resp, trace, err := a.applierV3.Txn(ctx, rt) if err == nil && !ok { err = ErrNoSpace } return resp, trace, err } func (a *quotaApplierV3) LeaseGrant(lc *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) { ok := a.q.Available(lc) resp, err := a.applierV3.LeaseGrant(lc) if err == nil && !ok { err = ErrNoSpace } return resp, err } func noSideEffect(r *pb.InternalRaftRequest) bool { return r.Range != nil || r.AuthUserGet != nil || r.AuthRoleGet != nil || r.AuthStatus != nil } func removeNeedlessRangeReqs(txn *pb.TxnRequest) { f := func(ops []*pb.RequestOp) []*pb.RequestOp { j := 0 for i := 0; i < len(ops); i++ { if _, ok := ops[i].Request.(*pb.RequestOp_RequestRange); ok { continue } ops[j] = ops[i] j++ } return ops[:j] } txn.Success = f(txn.Success) txn.Failure = f(txn.Failure) } func newHeader(s *EtcdServer) *pb.ResponseHeader { return &pb.ResponseHeader{ ClusterId: uint64(s.Cluster().ID()), MemberId: uint64(s.MemberId()), Revision: s.KV().Rev(), RaftTerm: s.Term(), } }