Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

clientv3/integration: handle server-side context timeouts from clock-drift

Browse Source 
Due to clock drifts in server-side, client context times out
first in server-side, while original client-side context is
not timed out yet.

Signed-off-by: Gyu-Ho Lee <gyuhox@gmail.com>
This commit is contained in:
Gyu-Ho Lee 2017-11-27 13:10:05 -08:00
parent bdff651428
commit 0d0e8e78f7
3 changed files with 25 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
clientv3/integration/black_hole_test.go
View File

@ -106,7 +106,7 @@ func TestBalancerUnderBlackholeKeepAliveWatch(t *testing.T) {
func TestBalancerUnderBlackholeNoKeepAlivePut(t *testing.T) { func TestBalancerUnderBlackholeNoKeepAlivePut(t *testing.T) {
testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error { testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error {
_, err := cli.Put(ctx, "foo", "bar") _, err := cli.Put(ctx, "foo", "bar")
if err == context.DeadlineExceeded || err == rpctypes.ErrTimeout { if err == context.DeadlineExceeded || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
return errExpected return errExpected
} }
return err return err
@ -116,7 +116,7 @@ func TestBalancerUnderBlackholeNoKeepAlivePut(t *testing.T) {
func TestBalancerUnderBlackholeNoKeepAliveDelete(t *testing.T) { func TestBalancerUnderBlackholeNoKeepAliveDelete(t *testing.T) {
testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error { testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error {
_, err := cli.Delete(ctx, "foo") _, err := cli.Delete(ctx, "foo")
if err == context.DeadlineExceeded || err == rpctypes.ErrTimeout { if err == context.DeadlineExceeded || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
return errExpected return errExpected
} }
return err return err
@ -129,7 +129,7 @@ func TestBalancerUnderBlackholeNoKeepAliveTxn(t *testing.T) {
If(clientv3.Compare(clientv3.Version("foo"), "=", 0)). If(clientv3.Compare(clientv3.Version("foo"), "=", 0)).
Then(clientv3.OpPut("foo", "bar")). Then(clientv3.OpPut("foo", "bar")).
Else(clientv3.OpPut("foo", "baz")).Commit() Else(clientv3.OpPut("foo", "baz")).Commit()
if err == context.DeadlineExceeded || err == rpctypes.ErrTimeout { if err == context.DeadlineExceeded || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
return errExpected return errExpected
} }
return err return err
@ -139,7 +139,7 @@ func TestBalancerUnderBlackholeNoKeepAliveTxn(t *testing.T) {
func TestBalancerUnderBlackholeNoKeepAliveLinearizableGet(t *testing.T) { func TestBalancerUnderBlackholeNoKeepAliveLinearizableGet(t *testing.T) {
testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error { testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error {
_, err := cli.Get(ctx, "a") _, err := cli.Get(ctx, "a")
if err == context.DeadlineExceeded || err == rpctypes.ErrTimeout { if err == context.DeadlineExceeded || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
return errExpected return errExpected
} }
return err return err
@ -149,7 +149,7 @@ func TestBalancerUnderBlackholeNoKeepAliveLinearizableGet(t *testing.T) {
func TestBalancerUnderBlackholeNoKeepAliveSerializableGet(t *testing.T) { func TestBalancerUnderBlackholeNoKeepAliveSerializableGet(t *testing.T) {
testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error { testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error {
_, err := cli.Get(ctx, "a", clientv3.WithSerializable()) _, err := cli.Get(ctx, "a", clientv3.WithSerializable())
if err == context.DeadlineExceeded { if err == context.DeadlineExceeded || isServerCtxTimeout(err) {
return errExpected return errExpected
} }
return err return err

23
clientv3/integration/network_partition_test.go
View File

@ -19,6 +19,7 @@ package integration
import ( import (
"context" "context"
"errors" "errors"
"strings"
"testing" "testing"
"time" "time"
@ -26,6 +27,8 @@ import (
"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes" "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
"github.com/coreos/etcd/integration" "github.com/coreos/etcd/integration"
"github.com/coreos/etcd/pkg/testutil" "github.com/coreos/etcd/pkg/testutil"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
) )
var errExpected = errors.New("expected error") var errExpected = errors.New("expected error")
@ -36,7 +39,7 @@ var errExpected = errors.New("expected error")
func TestBalancerUnderNetworkPartitionPut(t *testing.T) { func TestBalancerUnderNetworkPartitionPut(t *testing.T) {
testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error { testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error {
_, err := cli.Put(ctx, "a", "b") _, err := cli.Put(ctx, "a", "b")
if err == context.DeadlineExceeded || err == rpctypes.ErrTimeout { if err == context.DeadlineExceeded || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
return errExpected return errExpected
} }
return err return err
@ -46,7 +49,7 @@ func TestBalancerUnderNetworkPartitionPut(t *testing.T) {
func TestBalancerUnderNetworkPartitionDelete(t *testing.T) { func TestBalancerUnderNetworkPartitionDelete(t *testing.T) {
testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error { testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error {
_, err := cli.Delete(ctx, "a") _, err := cli.Delete(ctx, "a")
if err == context.DeadlineExceeded || err == rpctypes.ErrTimeout { if err == context.DeadlineExceeded || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
return errExpected return errExpected
} }
return err return err
@ -59,7 +62,7 @@ func TestBalancerUnderNetworkPartitionTxn(t *testing.T) {
If(clientv3.Compare(clientv3.Version("foo"), "=", 0)). If(clientv3.Compare(clientv3.Version("foo"), "=", 0)).
Then(clientv3.OpPut("foo", "bar")). Then(clientv3.OpPut("foo", "bar")).
Else(clientv3.OpPut("foo", "baz")).Commit() Else(clientv3.OpPut("foo", "baz")).Commit()
if err == context.DeadlineExceeded || err == rpctypes.ErrTimeout { if err == context.DeadlineExceeded || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout {
return errExpected return errExpected
} }
return err return err
@ -82,13 +85,25 @@ func TestBalancerUnderNetworkPartitionLinearizableGetWithLongTimeout(t *testing.
func TestBalancerUnderNetworkPartitionLinearizableGetWithShortTimeout(t *testing.T) { func TestBalancerUnderNetworkPartitionLinearizableGetWithShortTimeout(t *testing.T) {
testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error { testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error {
_, err := cli.Get(ctx, "a") _, err := cli.Get(ctx, "a")
if err == context.DeadlineExceeded { if err == context.DeadlineExceeded || isServerCtxTimeout(err) {
return errExpected return errExpected
} }
return err return err
}, time.Second) }, time.Second)
} }
// e.g. due to clock drifts in server-side,
// client context times out first in server-side
// while original client-side context is not timed out yet
func isServerCtxTimeout(err error) bool {
if err == nil {
return false
}
ev, _ := status.FromError(err)
code := ev.Code()
return code == codes.DeadlineExceeded && strings.Contains(err.Error(), "context deadline exceeded")
}
func TestBalancerUnderNetworkPartitionSerializableGet(t *testing.T) { func TestBalancerUnderNetworkPartitionSerializableGet(t *testing.T) {
testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error { testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error {
_, err := cli.Get(ctx, "a", clientv3.WithSerializable()) _, err := cli.Get(ctx, "a", clientv3.WithSerializable())

2
clientv3/integration/server_shutdown_test.go
View File

@ -100,7 +100,7 @@ func TestBalancerUnderServerShutdownWatch(t *testing.T) {
if err == nil { if err == nil {
break break
} }
if err == context.DeadlineExceeded || err == rpctypes.ErrTimeout || err == rpctypes.ErrTimeoutDueToLeaderFail { if err == context.DeadlineExceeded || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout || err == rpctypes.ErrTimeoutDueToLeaderFail {
continue continue
} }
t.Fatal(err) t.Fatal(err)