mirror of
https://github.com/etcd-io/etcd.git
synced 2024-09-27 06:25:44 +00:00
45b1e6b470
The test depended on very subtle timing semantic and on properties of 'copied' clients. https://travis-ci.com/github/etcd-io/etcd/jobs/486191449 Examplar failure: ``` {"level":"warn","ts":"2021-02-25T12:34:47.894Z","caller":"v3/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0000d6fc0/#initially=[unix://localhost:86269902489114839060]","attempt":1,"error":"rpc error: code = Unavailable desc = etcdserver: rpc not supported for learner"} {"level":"warn","ts":"2021-02-25T12:34:48.163Z","caller":"v3/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00035a000/#initially=[unix://localhost:78285857058450835940]","attempt":0,"error":"rpc error: code = FailedPrecondition desc = etcdserver: not leader"} {"level":"info","ts":"2021-02-25T12:34:48.255Z","caller":"v3/maintenance.go:211","msg":"opened snapshot stream; downloading"} {"level":"warn","ts":"2021-02-25T12:34:48.255Z","caller":"v3/maintenance.go:221","msg":"failed to receive from snapshot stream; closing","error":"rpc error: code = Canceled desc = context canceled"} {"level":"info","ts":"2021-02-25T12:34:48.255Z","caller":"v3/maintenance.go:211","msg":"opened snapshot stream; downloading"} {"level":"info","ts":"2021-02-25T12:34:50.255Z","caller":"v3/maintenance.go:219","msg":"completed snapshot read; closing"} {"level":"info","ts":"2021-02-25T12:34:51.717Z","caller":"v3/maintenance.go:211","msg":"opened snapshot stream; downloading"} {"level":"warn","ts":"2021-02-25T12:34:52.017Z","caller":"v3/maintenance.go:221","msg":"failed to receive from snapshot stream; closing","error":"rpc error: code = Canceled desc = context canceled"} {"level":"info","ts":"2021-02-25T12:34:52.018Z","caller":"v3/maintenance.go:211","msg":"opened snapshot stream; downloading"} {"level":"warn","ts":"2021-02-25T12:34:53.018Z","caller":"v3/maintenance.go:221","msg":"failed to receive from snapshot stream; closing","error":"rpc error: code = DeadlineExceeded desc = context deadline exceeded"} --- FAIL: TestEndpointSwitchResolvesViolation (10.12s) ordering_util_test.go:81: failed to resolve order violation etcdclient: no cluster members have a revision higher than the previously received revision ```
43 lines
1.6 KiB
Go
43 lines
1.6 KiB
Go
// Copyright 2017 The etcd Authors
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package ordering
|
|
|
|
import (
|
|
"errors"
|
|
"sync/atomic"
|
|
|
|
"go.etcd.io/etcd/client/v3"
|
|
)
|
|
|
|
type OrderViolationFunc func(op clientv3.Op, resp clientv3.OpResponse, prevRev int64) error
|
|
|
|
var ErrNoGreaterRev = errors.New("etcdclient: no cluster members have a revision higher than the previously received revision")
|
|
|
|
func NewOrderViolationSwitchEndpointClosure(c *clientv3.Client) OrderViolationFunc {
|
|
violationCount := int32(0)
|
|
return func(_ clientv3.Op, _ clientv3.OpResponse, _ int64) error {
|
|
// Each request is assigned by round-robin load-balancer's picker to a different
|
|
// endpoints. If we cycled them 5 times (even with some level of concurrency),
|
|
// with high probability no endpoint points on a member with fresh data.
|
|
// TODO: Ideally we should track members (resp.opp.Header) that returned
|
|
// stale result and explicitly temporarily disable them in 'picker'.
|
|
if atomic.LoadInt32(&violationCount) > int32(5*len(c.Endpoints())) {
|
|
return ErrNoGreaterRev
|
|
}
|
|
atomic.AddInt32(&violationCount, 1)
|
|
return nil
|
|
}
|
|
}
|