rpctypes, clientv3: retry RPC on EtcdStopped

Fixes #5983
This commit is contained in:
Anthony Romano 2016-07-19 12:33:33 -07:00
parent 80c2e4098d
commit 8abae076d1
3 changed files with 18 additions and 3 deletions

View File

@ -275,8 +275,13 @@ func isHaltErr(ctx context.Context, err error) bool {
if err == nil { if err == nil {
return false return false
} }
return strings.HasPrefix(grpc.ErrorDesc(err), "etcdserver: ") || eErr := rpctypes.Error(err)
strings.Contains(err.Error(), grpc.ErrClientConnClosing.Error()) if _, ok := eErr.(rpctypes.EtcdError); ok {
return eErr != rpctypes.ErrStopped && eErr != rpctypes.ErrNoLeader
}
// treat etcdserver errors not recognized by the client as halting
return strings.Contains(err.Error(), grpc.ErrClientConnClosing.Error()) ||
strings.Contains(err.Error(), "etcdserver:")
} }
func toErr(ctx context.Context, err error) error { func toErr(ctx context.Context, err error) error {

View File

@ -19,6 +19,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/coreos/etcd/etcdserver"
"golang.org/x/net/context" "golang.org/x/net/context"
"google.golang.org/grpc" "google.golang.org/grpc"
) )
@ -57,7 +58,13 @@ func TestDialTimeout(t *testing.T) {
func TestIsHaltErr(t *testing.T) { func TestIsHaltErr(t *testing.T) {
if !isHaltErr(nil, fmt.Errorf("etcdserver: some etcdserver error")) { if !isHaltErr(nil, fmt.Errorf("etcdserver: some etcdserver error")) {
t.Errorf(`error prefixed with "etcdserver: " should be Halted`) t.Errorf(`error prefixed with "etcdserver: " should be Halted by default`)
}
if isHaltErr(nil, etcdserver.ErrStopped) {
t.Errorf("error %v should not halt", etcdserver.ErrStopped)
}
if isHaltErr(nil, etcdserver.ErrNoLeader) {
t.Errorf("error %v should not halt", etcdserver.ErrNoLeader)
} }
ctx, cancel := context.WithCancel(context.TODO()) ctx, cancel := context.WithCancel(context.TODO())
if isHaltErr(ctx, nil) { if isHaltErr(ctx, nil) {

View File

@ -53,6 +53,7 @@ var (
ErrGRPCNoLeader = grpc.Errorf(codes.Unavailable, "etcdserver: no leader") ErrGRPCNoLeader = grpc.Errorf(codes.Unavailable, "etcdserver: no leader")
ErrGRPCNotCapable = grpc.Errorf(codes.Unavailable, "etcdserver: not capable") ErrGRPCNotCapable = grpc.Errorf(codes.Unavailable, "etcdserver: not capable")
ErrGRPCStopped = grpc.Errorf(codes.Unavailable, "etcdserver: server stopped")
errStringToError = map[string]error{ errStringToError = map[string]error{
grpc.ErrorDesc(ErrGRPCEmptyKey): ErrGRPCEmptyKey, grpc.ErrorDesc(ErrGRPCEmptyKey): ErrGRPCEmptyKey,
@ -87,6 +88,7 @@ var (
grpc.ErrorDesc(ErrGRPCNoLeader): ErrGRPCNoLeader, grpc.ErrorDesc(ErrGRPCNoLeader): ErrGRPCNoLeader,
grpc.ErrorDesc(ErrGRPCNotCapable): ErrGRPCNotCapable, grpc.ErrorDesc(ErrGRPCNotCapable): ErrGRPCNotCapable,
grpc.ErrorDesc(ErrGRPCStopped): ErrGRPCStopped,
} }
// client-side error // client-side error
@ -122,6 +124,7 @@ var (
ErrNoLeader = Error(ErrGRPCNoLeader) ErrNoLeader = Error(ErrGRPCNoLeader)
ErrNotCapable = Error(ErrGRPCNotCapable) ErrNotCapable = Error(ErrGRPCNotCapable)
ErrStopped = Error(ErrGRPCStopped)
) )
// EtcdError defines gRPC server errors. // EtcdError defines gRPC server errors.