diff --git a/clientv3/retry.go b/clientv3/retry.go index 5ad815d74..7f89ba641 100644 --- a/clientv3/retry.go +++ b/clientv3/retry.go @@ -25,10 +25,26 @@ import ( "google.golang.org/grpc/status" ) +type retryPolicy uint8 + +const ( + repeatable retryPolicy = iota + nonRepeatable +) + type rpcFunc func(ctx context.Context) error -type retryRPCFunc func(context.Context, rpcFunc) error +type retryRPCFunc func(context.Context, rpcFunc, retryPolicy) error type retryStopErrFunc func(error) bool +// immutable requests (e.g. Get) should be retried unless it's +// an obvious server-side error (e.g. rpctypes.ErrRequestTooLarge). +// +// "isRepeatableStopError" returns "true" when an immutable request +// is interrupted by server-side or gRPC-side error and its status +// code is not transient (!= codes.Unavailable). +// +// Returning "true" means retry should stop, since client cannot +// handle itself even with retries. func isRepeatableStopError(err error) bool { eErr := rpctypes.Error(err) // always stop retry on etcd errors @@ -40,6 +56,17 @@ func isRepeatableStopError(err error) bool { return ev.Code() != codes.Unavailable } +// mutable requests (e.g. Put, Delete, Txn) should only be retried +// when the status code is codes.Unavailable when initial connection +// has not been established (no pinned endpoint). +// +// "isNonRepeatableStopError" returns "true" when a mutable request +// is interrupted by non-transient error that client cannot handle itself, +// or transient error while the connection has already been established +// (pinned endpoint exists). +// +// Returning "true" means retry should stop, otherwise it violates +// write-at-most-once semantics. func isNonRepeatableStopError(err error) bool { ev, _ := status.FromError(err) if ev.Code() != codes.Unavailable { @@ -49,8 +76,15 @@ func isNonRepeatableStopError(err error) bool { return desc != "there is no address available" && desc != "there is no connection available" } -func (c *Client) newRetryWrapper(isStop retryStopErrFunc) retryRPCFunc { - return func(rpcCtx context.Context, f rpcFunc) error { +func (c *Client) newRetryWrapper() retryRPCFunc { + return func(rpcCtx context.Context, f rpcFunc, rp retryPolicy) error { + var isStop retryStopErrFunc + switch rp { + case repeatable: + isStop = isRepeatableStopError + case nonRepeatable: + isStop = isNonRepeatableStopError + } for { if err := readyWait(rpcCtx, c.ctx, c.balancer.ConnectNotify()); err != nil { return err @@ -76,11 +110,11 @@ func (c *Client) newRetryWrapper(isStop retryStopErrFunc) retryRPCFunc { } } -func (c *Client) newAuthRetryWrapper() retryRPCFunc { - return func(rpcCtx context.Context, f rpcFunc) error { +func (c *Client) newAuthRetryWrapper(retryf retryRPCFunc) retryRPCFunc { + return func(rpcCtx context.Context, f rpcFunc, rp retryPolicy) error { for { pinned := c.balancer.pinned() - err := f(rpcCtx) + err := retryf(rpcCtx, f, rp) if err == nil { return nil } @@ -99,390 +133,364 @@ func (c *Client) newAuthRetryWrapper() retryRPCFunc { } } +type retryKVClient struct { + kc pb.KVClient + retryf retryRPCFunc +} + // RetryKVClient implements a KVClient. func RetryKVClient(c *Client) pb.KVClient { - repeatableRetry := c.newRetryWrapper(isRepeatableStopError) - nonRepeatableRetry := c.newRetryWrapper(isNonRepeatableStopError) - conn := pb.NewKVClient(c.conn) - retryBasic := &retryKVClient{&nonRepeatableKVClient{conn, nonRepeatableRetry}, repeatableRetry} - retryAuthWrapper := c.newAuthRetryWrapper() return &retryKVClient{ - &nonRepeatableKVClient{retryBasic, retryAuthWrapper}, - retryAuthWrapper} + kc: pb.NewKVClient(c.conn), + retryf: c.newAuthRetryWrapper(c.newRetryWrapper()), + } } - -type retryKVClient struct { - *nonRepeatableKVClient - repeatableRetry retryRPCFunc -} - func (rkv *retryKVClient) Range(ctx context.Context, in *pb.RangeRequest, opts ...grpc.CallOption) (resp *pb.RangeResponse, err error) { - err = rkv.repeatableRetry(ctx, func(rctx context.Context) error { + err = rkv.retryf(ctx, func(rctx context.Context) error { resp, err = rkv.kc.Range(rctx, in, opts...) return err - }) + }, repeatable) return resp, err } -type nonRepeatableKVClient struct { - kc pb.KVClient - nonRepeatableRetry retryRPCFunc -} - -func (rkv *nonRepeatableKVClient) Put(ctx context.Context, in *pb.PutRequest, opts ...grpc.CallOption) (resp *pb.PutResponse, err error) { - err = rkv.nonRepeatableRetry(ctx, func(rctx context.Context) error { +func (rkv *retryKVClient) Put(ctx context.Context, in *pb.PutRequest, opts ...grpc.CallOption) (resp *pb.PutResponse, err error) { + err = rkv.retryf(ctx, func(rctx context.Context) error { resp, err = rkv.kc.Put(rctx, in, opts...) return err - }) + }, nonRepeatable) return resp, err } -func (rkv *nonRepeatableKVClient) DeleteRange(ctx context.Context, in *pb.DeleteRangeRequest, opts ...grpc.CallOption) (resp *pb.DeleteRangeResponse, err error) { - err = rkv.nonRepeatableRetry(ctx, func(rctx context.Context) error { +func (rkv *retryKVClient) DeleteRange(ctx context.Context, in *pb.DeleteRangeRequest, opts ...grpc.CallOption) (resp *pb.DeleteRangeResponse, err error) { + err = rkv.retryf(ctx, func(rctx context.Context) error { resp, err = rkv.kc.DeleteRange(rctx, in, opts...) return err - }) + }, nonRepeatable) return resp, err } -func (rkv *nonRepeatableKVClient) Txn(ctx context.Context, in *pb.TxnRequest, opts ...grpc.CallOption) (resp *pb.TxnResponse, err error) { - // TODO: repeatableRetry if read-only txn - err = rkv.nonRepeatableRetry(ctx, func(rctx context.Context) error { +func (rkv *retryKVClient) Txn(ctx context.Context, in *pb.TxnRequest, opts ...grpc.CallOption) (resp *pb.TxnResponse, err error) { + // TODO: "repeatable" for read-only txn + err = rkv.retryf(ctx, func(rctx context.Context) error { resp, err = rkv.kc.Txn(rctx, in, opts...) return err - }) + }, nonRepeatable) return resp, err } -func (rkv *nonRepeatableKVClient) Compact(ctx context.Context, in *pb.CompactionRequest, opts ...grpc.CallOption) (resp *pb.CompactionResponse, err error) { - err = rkv.nonRepeatableRetry(ctx, func(rctx context.Context) error { +func (rkv *retryKVClient) Compact(ctx context.Context, in *pb.CompactionRequest, opts ...grpc.CallOption) (resp *pb.CompactionResponse, err error) { + err = rkv.retryf(ctx, func(rctx context.Context) error { resp, err = rkv.kc.Compact(rctx, in, opts...) return err - }) + }, nonRepeatable) return resp, err } type retryLeaseClient struct { - lc pb.LeaseClient - repeatableRetry retryRPCFunc + lc pb.LeaseClient + retryf retryRPCFunc } // RetryLeaseClient implements a LeaseClient. func RetryLeaseClient(c *Client) pb.LeaseClient { - retry := &retryLeaseClient{ - pb.NewLeaseClient(c.conn), - c.newRetryWrapper(isRepeatableStopError), + return &retryLeaseClient{ + lc: pb.NewLeaseClient(c.conn), + retryf: c.newAuthRetryWrapper(c.newRetryWrapper()), } - return &retryLeaseClient{retry, c.newAuthRetryWrapper()} } func (rlc *retryLeaseClient) LeaseTimeToLive(ctx context.Context, in *pb.LeaseTimeToLiveRequest, opts ...grpc.CallOption) (resp *pb.LeaseTimeToLiveResponse, err error) { - err = rlc.repeatableRetry(ctx, func(rctx context.Context) error { + err = rlc.retryf(ctx, func(rctx context.Context) error { resp, err = rlc.lc.LeaseTimeToLive(rctx, in, opts...) return err - }) + }, repeatable) return resp, err } func (rlc *retryLeaseClient) LeaseLeases(ctx context.Context, in *pb.LeaseLeasesRequest, opts ...grpc.CallOption) (resp *pb.LeaseLeasesResponse, err error) { - err = rlc.repeatableRetry(ctx, func(rctx context.Context) error { + err = rlc.retryf(ctx, func(rctx context.Context) error { resp, err = rlc.lc.LeaseLeases(rctx, in, opts...) return err - }) + }, repeatable) return resp, err } func (rlc *retryLeaseClient) LeaseGrant(ctx context.Context, in *pb.LeaseGrantRequest, opts ...grpc.CallOption) (resp *pb.LeaseGrantResponse, err error) { - err = rlc.repeatableRetry(ctx, func(rctx context.Context) error { + err = rlc.retryf(ctx, func(rctx context.Context) error { resp, err = rlc.lc.LeaseGrant(rctx, in, opts...) return err - }) + }, repeatable) return resp, err } func (rlc *retryLeaseClient) LeaseRevoke(ctx context.Context, in *pb.LeaseRevokeRequest, opts ...grpc.CallOption) (resp *pb.LeaseRevokeResponse, err error) { - err = rlc.repeatableRetry(ctx, func(rctx context.Context) error { + err = rlc.retryf(ctx, func(rctx context.Context) error { resp, err = rlc.lc.LeaseRevoke(rctx, in, opts...) return err - }) + }, repeatable) return resp, err } func (rlc *retryLeaseClient) LeaseKeepAlive(ctx context.Context, opts ...grpc.CallOption) (stream pb.Lease_LeaseKeepAliveClient, err error) { - err = rlc.repeatableRetry(ctx, func(rctx context.Context) error { + err = rlc.retryf(ctx, func(rctx context.Context) error { stream, err = rlc.lc.LeaseKeepAlive(rctx, opts...) return err - }) + }, repeatable) return stream, err } type retryClusterClient struct { - *nonRepeatableClusterClient - repeatableRetry retryRPCFunc + cc pb.ClusterClient + retryf retryRPCFunc } // RetryClusterClient implements a ClusterClient. func RetryClusterClient(c *Client) pb.ClusterClient { - repeatableRetry := c.newRetryWrapper(isRepeatableStopError) - nonRepeatableRetry := c.newRetryWrapper(isNonRepeatableStopError) - cc := pb.NewClusterClient(c.conn) - return &retryClusterClient{&nonRepeatableClusterClient{cc, nonRepeatableRetry}, repeatableRetry} + return &retryClusterClient{ + cc: pb.NewClusterClient(c.conn), + retryf: c.newRetryWrapper(), + } } func (rcc *retryClusterClient) MemberList(ctx context.Context, in *pb.MemberListRequest, opts ...grpc.CallOption) (resp *pb.MemberListResponse, err error) { - err = rcc.repeatableRetry(ctx, func(rctx context.Context) error { + err = rcc.retryf(ctx, func(rctx context.Context) error { resp, err = rcc.cc.MemberList(rctx, in, opts...) return err - }) + }, repeatable) return resp, err } -type nonRepeatableClusterClient struct { - cc pb.ClusterClient - nonRepeatableRetry retryRPCFunc -} - -func (rcc *nonRepeatableClusterClient) MemberAdd(ctx context.Context, in *pb.MemberAddRequest, opts ...grpc.CallOption) (resp *pb.MemberAddResponse, err error) { - err = rcc.nonRepeatableRetry(ctx, func(rctx context.Context) error { +func (rcc *retryClusterClient) MemberAdd(ctx context.Context, in *pb.MemberAddRequest, opts ...grpc.CallOption) (resp *pb.MemberAddResponse, err error) { + err = rcc.retryf(ctx, func(rctx context.Context) error { resp, err = rcc.cc.MemberAdd(rctx, in, opts...) return err - }) + }, nonRepeatable) return resp, err } -func (rcc *nonRepeatableClusterClient) MemberRemove(ctx context.Context, in *pb.MemberRemoveRequest, opts ...grpc.CallOption) (resp *pb.MemberRemoveResponse, err error) { - err = rcc.nonRepeatableRetry(ctx, func(rctx context.Context) error { +func (rcc *retryClusterClient) MemberRemove(ctx context.Context, in *pb.MemberRemoveRequest, opts ...grpc.CallOption) (resp *pb.MemberRemoveResponse, err error) { + err = rcc.retryf(ctx, func(rctx context.Context) error { resp, err = rcc.cc.MemberRemove(rctx, in, opts...) return err - }) + }, nonRepeatable) return resp, err } -func (rcc *nonRepeatableClusterClient) MemberUpdate(ctx context.Context, in *pb.MemberUpdateRequest, opts ...grpc.CallOption) (resp *pb.MemberUpdateResponse, err error) { - err = rcc.nonRepeatableRetry(ctx, func(rctx context.Context) error { +func (rcc *retryClusterClient) MemberUpdate(ctx context.Context, in *pb.MemberUpdateRequest, opts ...grpc.CallOption) (resp *pb.MemberUpdateResponse, err error) { + err = rcc.retryf(ctx, func(rctx context.Context) error { resp, err = rcc.cc.MemberUpdate(rctx, in, opts...) return err - }) + }, nonRepeatable) return resp, err } +type retryMaintenanceClient struct { + mc pb.MaintenanceClient + retryf retryRPCFunc +} + // RetryMaintenanceClient implements a Maintenance. func RetryMaintenanceClient(c *Client, conn *grpc.ClientConn) pb.MaintenanceClient { - repeatableRetry := c.newRetryWrapper(isRepeatableStopError) - nonRepeatableRetry := c.newRetryWrapper(isNonRepeatableStopError) - mc := pb.NewMaintenanceClient(conn) - return &retryMaintenanceClient{&nonRepeatableMaintenanceClient{mc, nonRepeatableRetry}, repeatableRetry} -} - -type retryMaintenanceClient struct { - *nonRepeatableMaintenanceClient - repeatableRetry retryRPCFunc + return &retryMaintenanceClient{ + mc: pb.NewMaintenanceClient(conn), + retryf: c.newRetryWrapper(), + } } func (rmc *retryMaintenanceClient) Alarm(ctx context.Context, in *pb.AlarmRequest, opts ...grpc.CallOption) (resp *pb.AlarmResponse, err error) { - err = rmc.repeatableRetry(ctx, func(rctx context.Context) error { + err = rmc.retryf(ctx, func(rctx context.Context) error { resp, err = rmc.mc.Alarm(rctx, in, opts...) return err - }) + }, repeatable) return resp, err } func (rmc *retryMaintenanceClient) Status(ctx context.Context, in *pb.StatusRequest, opts ...grpc.CallOption) (resp *pb.StatusResponse, err error) { - err = rmc.repeatableRetry(ctx, func(rctx context.Context) error { + err = rmc.retryf(ctx, func(rctx context.Context) error { resp, err = rmc.mc.Status(rctx, in, opts...) return err - }) + }, repeatable) return resp, err } func (rmc *retryMaintenanceClient) Hash(ctx context.Context, in *pb.HashRequest, opts ...grpc.CallOption) (resp *pb.HashResponse, err error) { - err = rmc.repeatableRetry(ctx, func(rctx context.Context) error { + err = rmc.retryf(ctx, func(rctx context.Context) error { resp, err = rmc.mc.Hash(rctx, in, opts...) return err - }) + }, repeatable) return resp, err } func (rmc *retryMaintenanceClient) HashKV(ctx context.Context, in *pb.HashKVRequest, opts ...grpc.CallOption) (resp *pb.HashKVResponse, err error) { - err = rmc.repeatableRetry(ctx, func(rctx context.Context) error { + err = rmc.retryf(ctx, func(rctx context.Context) error { resp, err = rmc.mc.HashKV(rctx, in, opts...) return err - }) + }, repeatable) return resp, err } func (rmc *retryMaintenanceClient) Snapshot(ctx context.Context, in *pb.SnapshotRequest, opts ...grpc.CallOption) (stream pb.Maintenance_SnapshotClient, err error) { - err = rmc.repeatableRetry(ctx, func(rctx context.Context) error { + err = rmc.retryf(ctx, func(rctx context.Context) error { stream, err = rmc.mc.Snapshot(rctx, in, opts...) return err - }) + }, repeatable) return stream, err } func (rmc *retryMaintenanceClient) MoveLeader(ctx context.Context, in *pb.MoveLeaderRequest, opts ...grpc.CallOption) (resp *pb.MoveLeaderResponse, err error) { - err = rmc.repeatableRetry(ctx, func(rctx context.Context) error { + err = rmc.retryf(ctx, func(rctx context.Context) error { resp, err = rmc.mc.MoveLeader(rctx, in, opts...) return err - }) + }, repeatable) return resp, err } -type nonRepeatableMaintenanceClient struct { - mc pb.MaintenanceClient - nonRepeatableRetry retryRPCFunc -} - -func (rmc *nonRepeatableMaintenanceClient) Defragment(ctx context.Context, in *pb.DefragmentRequest, opts ...grpc.CallOption) (resp *pb.DefragmentResponse, err error) { - err = rmc.nonRepeatableRetry(ctx, func(rctx context.Context) error { +func (rmc *retryMaintenanceClient) Defragment(ctx context.Context, in *pb.DefragmentRequest, opts ...grpc.CallOption) (resp *pb.DefragmentResponse, err error) { + err = rmc.retryf(ctx, func(rctx context.Context) error { resp, err = rmc.mc.Defragment(rctx, in, opts...) return err - }) + }, nonRepeatable) return resp, err } type retryAuthClient struct { - *nonRepeatableAuthClient - repeatableRetry retryRPCFunc + ac pb.AuthClient + retryf retryRPCFunc } // RetryAuthClient implements a AuthClient. func RetryAuthClient(c *Client) pb.AuthClient { - repeatableRetry := c.newRetryWrapper(isRepeatableStopError) - nonRepeatableRetry := c.newRetryWrapper(isNonRepeatableStopError) - ac := pb.NewAuthClient(c.conn) - return &retryAuthClient{&nonRepeatableAuthClient{ac, nonRepeatableRetry}, repeatableRetry} + return &retryAuthClient{ + ac: pb.NewAuthClient(c.conn), + retryf: c.newRetryWrapper(), + } } func (rac *retryAuthClient) UserList(ctx context.Context, in *pb.AuthUserListRequest, opts ...grpc.CallOption) (resp *pb.AuthUserListResponse, err error) { - err = rac.repeatableRetry(ctx, func(rctx context.Context) error { + err = rac.retryf(ctx, func(rctx context.Context) error { resp, err = rac.ac.UserList(rctx, in, opts...) return err - }) + }, repeatable) return resp, err } func (rac *retryAuthClient) UserGet(ctx context.Context, in *pb.AuthUserGetRequest, opts ...grpc.CallOption) (resp *pb.AuthUserGetResponse, err error) { - err = rac.repeatableRetry(ctx, func(rctx context.Context) error { + err = rac.retryf(ctx, func(rctx context.Context) error { resp, err = rac.ac.UserGet(rctx, in, opts...) return err - }) + }, repeatable) return resp, err } func (rac *retryAuthClient) RoleGet(ctx context.Context, in *pb.AuthRoleGetRequest, opts ...grpc.CallOption) (resp *pb.AuthRoleGetResponse, err error) { - err = rac.repeatableRetry(ctx, func(rctx context.Context) error { + err = rac.retryf(ctx, func(rctx context.Context) error { resp, err = rac.ac.RoleGet(rctx, in, opts...) return err - }) + }, repeatable) return resp, err } func (rac *retryAuthClient) RoleList(ctx context.Context, in *pb.AuthRoleListRequest, opts ...grpc.CallOption) (resp *pb.AuthRoleListResponse, err error) { - err = rac.repeatableRetry(ctx, func(rctx context.Context) error { + err = rac.retryf(ctx, func(rctx context.Context) error { resp, err = rac.ac.RoleList(rctx, in, opts...) return err - }) + }, repeatable) return resp, err } -type nonRepeatableAuthClient struct { - ac pb.AuthClient - nonRepeatableRetry retryRPCFunc -} - -func (rac *nonRepeatableAuthClient) AuthEnable(ctx context.Context, in *pb.AuthEnableRequest, opts ...grpc.CallOption) (resp *pb.AuthEnableResponse, err error) { - err = rac.nonRepeatableRetry(ctx, func(rctx context.Context) error { +func (rac *retryAuthClient) AuthEnable(ctx context.Context, in *pb.AuthEnableRequest, opts ...grpc.CallOption) (resp *pb.AuthEnableResponse, err error) { + err = rac.retryf(ctx, func(rctx context.Context) error { resp, err = rac.ac.AuthEnable(rctx, in, opts...) return err - }) + }, nonRepeatable) return resp, err } -func (rac *nonRepeatableAuthClient) AuthDisable(ctx context.Context, in *pb.AuthDisableRequest, opts ...grpc.CallOption) (resp *pb.AuthDisableResponse, err error) { - err = rac.nonRepeatableRetry(ctx, func(rctx context.Context) error { +func (rac *retryAuthClient) AuthDisable(ctx context.Context, in *pb.AuthDisableRequest, opts ...grpc.CallOption) (resp *pb.AuthDisableResponse, err error) { + err = rac.retryf(ctx, func(rctx context.Context) error { resp, err = rac.ac.AuthDisable(rctx, in, opts...) return err - }) + }, nonRepeatable) return resp, err } -func (rac *nonRepeatableAuthClient) UserAdd(ctx context.Context, in *pb.AuthUserAddRequest, opts ...grpc.CallOption) (resp *pb.AuthUserAddResponse, err error) { - err = rac.nonRepeatableRetry(ctx, func(rctx context.Context) error { +func (rac *retryAuthClient) UserAdd(ctx context.Context, in *pb.AuthUserAddRequest, opts ...grpc.CallOption) (resp *pb.AuthUserAddResponse, err error) { + err = rac.retryf(ctx, func(rctx context.Context) error { resp, err = rac.ac.UserAdd(rctx, in, opts...) return err - }) + }, nonRepeatable) return resp, err } -func (rac *nonRepeatableAuthClient) UserDelete(ctx context.Context, in *pb.AuthUserDeleteRequest, opts ...grpc.CallOption) (resp *pb.AuthUserDeleteResponse, err error) { - err = rac.nonRepeatableRetry(ctx, func(rctx context.Context) error { +func (rac *retryAuthClient) UserDelete(ctx context.Context, in *pb.AuthUserDeleteRequest, opts ...grpc.CallOption) (resp *pb.AuthUserDeleteResponse, err error) { + err = rac.retryf(ctx, func(rctx context.Context) error { resp, err = rac.ac.UserDelete(rctx, in, opts...) return err - }) + }, nonRepeatable) return resp, err } -func (rac *nonRepeatableAuthClient) UserChangePassword(ctx context.Context, in *pb.AuthUserChangePasswordRequest, opts ...grpc.CallOption) (resp *pb.AuthUserChangePasswordResponse, err error) { - err = rac.nonRepeatableRetry(ctx, func(rctx context.Context) error { +func (rac *retryAuthClient) UserChangePassword(ctx context.Context, in *pb.AuthUserChangePasswordRequest, opts ...grpc.CallOption) (resp *pb.AuthUserChangePasswordResponse, err error) { + err = rac.retryf(ctx, func(rctx context.Context) error { resp, err = rac.ac.UserChangePassword(rctx, in, opts...) return err - }) + }, nonRepeatable) return resp, err } -func (rac *nonRepeatableAuthClient) UserGrantRole(ctx context.Context, in *pb.AuthUserGrantRoleRequest, opts ...grpc.CallOption) (resp *pb.AuthUserGrantRoleResponse, err error) { - err = rac.nonRepeatableRetry(ctx, func(rctx context.Context) error { +func (rac *retryAuthClient) UserGrantRole(ctx context.Context, in *pb.AuthUserGrantRoleRequest, opts ...grpc.CallOption) (resp *pb.AuthUserGrantRoleResponse, err error) { + err = rac.retryf(ctx, func(rctx context.Context) error { resp, err = rac.ac.UserGrantRole(rctx, in, opts...) return err - }) + }, nonRepeatable) return resp, err } -func (rac *nonRepeatableAuthClient) UserRevokeRole(ctx context.Context, in *pb.AuthUserRevokeRoleRequest, opts ...grpc.CallOption) (resp *pb.AuthUserRevokeRoleResponse, err error) { - err = rac.nonRepeatableRetry(ctx, func(rctx context.Context) error { +func (rac *retryAuthClient) UserRevokeRole(ctx context.Context, in *pb.AuthUserRevokeRoleRequest, opts ...grpc.CallOption) (resp *pb.AuthUserRevokeRoleResponse, err error) { + err = rac.retryf(ctx, func(rctx context.Context) error { resp, err = rac.ac.UserRevokeRole(rctx, in, opts...) return err - }) + }, nonRepeatable) return resp, err } -func (rac *nonRepeatableAuthClient) RoleAdd(ctx context.Context, in *pb.AuthRoleAddRequest, opts ...grpc.CallOption) (resp *pb.AuthRoleAddResponse, err error) { - err = rac.nonRepeatableRetry(ctx, func(rctx context.Context) error { +func (rac *retryAuthClient) RoleAdd(ctx context.Context, in *pb.AuthRoleAddRequest, opts ...grpc.CallOption) (resp *pb.AuthRoleAddResponse, err error) { + err = rac.retryf(ctx, func(rctx context.Context) error { resp, err = rac.ac.RoleAdd(rctx, in, opts...) return err - }) + }, nonRepeatable) return resp, err } -func (rac *nonRepeatableAuthClient) RoleDelete(ctx context.Context, in *pb.AuthRoleDeleteRequest, opts ...grpc.CallOption) (resp *pb.AuthRoleDeleteResponse, err error) { - err = rac.nonRepeatableRetry(ctx, func(rctx context.Context) error { +func (rac *retryAuthClient) RoleDelete(ctx context.Context, in *pb.AuthRoleDeleteRequest, opts ...grpc.CallOption) (resp *pb.AuthRoleDeleteResponse, err error) { + err = rac.retryf(ctx, func(rctx context.Context) error { resp, err = rac.ac.RoleDelete(rctx, in, opts...) return err - }) + }, nonRepeatable) return resp, err } -func (rac *nonRepeatableAuthClient) RoleGrantPermission(ctx context.Context, in *pb.AuthRoleGrantPermissionRequest, opts ...grpc.CallOption) (resp *pb.AuthRoleGrantPermissionResponse, err error) { - err = rac.nonRepeatableRetry(ctx, func(rctx context.Context) error { +func (rac *retryAuthClient) RoleGrantPermission(ctx context.Context, in *pb.AuthRoleGrantPermissionRequest, opts ...grpc.CallOption) (resp *pb.AuthRoleGrantPermissionResponse, err error) { + err = rac.retryf(ctx, func(rctx context.Context) error { resp, err = rac.ac.RoleGrantPermission(rctx, in, opts...) return err - }) + }, nonRepeatable) return resp, err } -func (rac *nonRepeatableAuthClient) RoleRevokePermission(ctx context.Context, in *pb.AuthRoleRevokePermissionRequest, opts ...grpc.CallOption) (resp *pb.AuthRoleRevokePermissionResponse, err error) { - err = rac.nonRepeatableRetry(ctx, func(rctx context.Context) error { +func (rac *retryAuthClient) RoleRevokePermission(ctx context.Context, in *pb.AuthRoleRevokePermissionRequest, opts ...grpc.CallOption) (resp *pb.AuthRoleRevokePermissionResponse, err error) { + err = rac.retryf(ctx, func(rctx context.Context) error { resp, err = rac.ac.RoleRevokePermission(rctx, in, opts...) return err - }) + }, nonRepeatable) return resp, err } -func (rac *nonRepeatableAuthClient) Authenticate(ctx context.Context, in *pb.AuthenticateRequest, opts ...grpc.CallOption) (resp *pb.AuthenticateResponse, err error) { - err = rac.nonRepeatableRetry(ctx, func(rctx context.Context) error { +func (rac *retryAuthClient) Authenticate(ctx context.Context, in *pb.AuthenticateRequest, opts ...grpc.CallOption) (resp *pb.AuthenticateResponse, err error) { + err = rac.retryf(ctx, func(rctx context.Context) error { resp, err = rac.ac.Authenticate(rctx, in, opts...) return err - }) + }, nonRepeatable) return resp, err }