mirror of
https://github.com/etcd-io/etcd.git
synced 2024-09-27 06:25:44 +00:00
Merge pull request #16075 from serathius/robustness-randomize-get
tests/robustness: Move get to list of randomized operations
This commit is contained in:
commit
a081d52bd4
@ -37,14 +37,15 @@ var (
|
|||||||
keyCount: 10,
|
keyCount: 10,
|
||||||
leaseTTL: DefaultLeaseTTL,
|
leaseTTL: DefaultLeaseTTL,
|
||||||
largePutSize: 32769,
|
largePutSize: 32769,
|
||||||
writeChoices: []choiceWeight[etcdRequestType]{
|
operations: []choiceWeight[etcdRequestType]{
|
||||||
{choice: Put, weight: 45},
|
{choice: Get, weight: 50},
|
||||||
{choice: LargePut, weight: 5},
|
{choice: Put, weight: 23},
|
||||||
{choice: Delete, weight: 10},
|
{choice: LargePut, weight: 2},
|
||||||
{choice: MultiOpTxn, weight: 10},
|
{choice: Delete, weight: 5},
|
||||||
{choice: PutWithLease, weight: 10},
|
{choice: MultiOpTxn, weight: 5},
|
||||||
{choice: LeaseRevoke, weight: 10},
|
{choice: PutWithLease, weight: 5},
|
||||||
{choice: CompareAndSet, weight: 10},
|
{choice: LeaseRevoke, weight: 5},
|
||||||
|
{choice: CompareAndSet, weight: 5},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@ -57,9 +58,10 @@ var (
|
|||||||
keyCount: 10,
|
keyCount: 10,
|
||||||
largePutSize: 32769,
|
largePutSize: 32769,
|
||||||
leaseTTL: DefaultLeaseTTL,
|
leaseTTL: DefaultLeaseTTL,
|
||||||
writeChoices: []choiceWeight[etcdRequestType]{
|
operations: []choiceWeight[etcdRequestType]{
|
||||||
{choice: Put, weight: 85},
|
{choice: Get, weight: 50},
|
||||||
{choice: MultiOpTxn, weight: 10},
|
{choice: Put, weight: 40},
|
||||||
|
{choice: MultiOpTxn, weight: 5},
|
||||||
{choice: LargePut, weight: 5},
|
{choice: LargePut, weight: 5},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@ -68,7 +70,7 @@ var (
|
|||||||
|
|
||||||
type etcdTraffic struct {
|
type etcdTraffic struct {
|
||||||
keyCount int
|
keyCount int
|
||||||
writeChoices []choiceWeight[etcdRequestType]
|
operations []choiceWeight[etcdRequestType]
|
||||||
leaseTTL int64
|
leaseTTL int64
|
||||||
largePutSize int
|
largePutSize int
|
||||||
}
|
}
|
||||||
@ -80,6 +82,7 @@ func (t etcdTraffic) ExpectUniqueRevision() bool {
|
|||||||
type etcdRequestType string
|
type etcdRequestType string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
Get etcdRequestType = "get"
|
||||||
Put etcdRequestType = "put"
|
Put etcdRequestType = "put"
|
||||||
LargePut etcdRequestType = "largePut"
|
LargePut etcdRequestType = "largePut"
|
||||||
Delete etcdRequestType = "delete"
|
Delete etcdRequestType = "delete"
|
||||||
@ -91,7 +94,7 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func (t etcdTraffic) Run(ctx context.Context, c *RecordingClient, limiter *rate.Limiter, ids identity.Provider, lm identity.LeaseIdStorage, finish <-chan struct{}) {
|
func (t etcdTraffic) Run(ctx context.Context, c *RecordingClient, limiter *rate.Limiter, ids identity.Provider, lm identity.LeaseIdStorage, finish <-chan struct{}) {
|
||||||
|
lastOperationSucceeded := true
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
@ -101,13 +104,16 @@ func (t etcdTraffic) Run(ctx context.Context, c *RecordingClient, limiter *rate.
|
|||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
key := fmt.Sprintf("%d", rand.Int()%t.keyCount)
|
key := fmt.Sprintf("%d", rand.Int()%t.keyCount)
|
||||||
// Execute one read per one write to avoid operation history include too many failed writes when etcd is down.
|
// Avoid multiple failed writes in a row
|
||||||
resp, err := t.Read(ctx, c, key)
|
if !lastOperationSucceeded {
|
||||||
|
_, err := t.Read(ctx, c, key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
limiter.Wait(ctx)
|
limiter.Wait(ctx)
|
||||||
err = t.Write(ctx, c, limiter, key, ids, lm, resp)
|
}
|
||||||
|
err := t.RandomOperation(ctx, c, limiter, key, ids, lm)
|
||||||
|
lastOperationSucceeded = err == nil
|
||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@ -122,30 +128,38 @@ func (t etcdTraffic) Read(ctx context.Context, c *RecordingClient, key string) (
|
|||||||
return resp, err
|
return resp, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t etcdTraffic) Write(ctx context.Context, c *RecordingClient, limiter *rate.Limiter, key string, id identity.Provider, lm identity.LeaseIdStorage, lastValues *mvccpb.KeyValue) error {
|
func (t etcdTraffic) RandomOperation(ctx context.Context, c *RecordingClient, limiter *rate.Limiter, key string, id identity.Provider, lm identity.LeaseIdStorage) error {
|
||||||
writeCtx, cancel := context.WithTimeout(ctx, RequestTimeout)
|
opCtx, cancel := context.WithTimeout(ctx, RequestTimeout)
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
switch pickRandom(t.writeChoices) {
|
switch pickRandom(t.operations) {
|
||||||
|
case Get:
|
||||||
|
_, err = c.Get(opCtx, key)
|
||||||
case Put:
|
case Put:
|
||||||
err = c.Put(writeCtx, key, fmt.Sprintf("%d", id.NewRequestId()))
|
err = c.Put(opCtx, key, fmt.Sprintf("%d", id.NewRequestId()))
|
||||||
case LargePut:
|
case LargePut:
|
||||||
err = c.Put(writeCtx, key, randString(t.largePutSize))
|
err = c.Put(opCtx, key, randString(t.largePutSize))
|
||||||
case Delete:
|
case Delete:
|
||||||
err = c.Delete(writeCtx, key)
|
err = c.Delete(opCtx, key)
|
||||||
case MultiOpTxn:
|
case MultiOpTxn:
|
||||||
_, err = c.Txn(writeCtx, nil, t.pickMultiTxnOps(id), nil)
|
_, err = c.Txn(opCtx, nil, t.pickMultiTxnOps(id), nil)
|
||||||
case CompareAndSet:
|
case CompareAndSet:
|
||||||
|
var kv *mvccpb.KeyValue
|
||||||
|
kv, err = c.Get(opCtx, key)
|
||||||
|
if err == nil {
|
||||||
|
limiter.Wait(ctx)
|
||||||
var expectedRevision int64
|
var expectedRevision int64
|
||||||
if lastValues != nil {
|
if kv != nil {
|
||||||
expectedRevision = lastValues.ModRevision
|
expectedRevision = kv.ModRevision
|
||||||
|
}
|
||||||
|
txnCtx, txnCancel := context.WithTimeout(ctx, RequestTimeout)
|
||||||
|
_, err = c.Txn(txnCtx, []clientv3.Cmp{clientv3.Compare(clientv3.ModRevision(key), "=", expectedRevision)}, []clientv3.Op{clientv3.OpPut(key, fmt.Sprintf("%d", id.NewRequestId()))}, nil)
|
||||||
|
txnCancel()
|
||||||
}
|
}
|
||||||
value := fmt.Sprintf("%d", id.NewRequestId())
|
|
||||||
_, err = c.Txn(ctx, []clientv3.Cmp{clientv3.Compare(clientv3.ModRevision(key), "=", expectedRevision)}, []clientv3.Op{clientv3.OpPut(key, value)}, nil)
|
|
||||||
case PutWithLease:
|
case PutWithLease:
|
||||||
leaseId := lm.LeaseId(c.id)
|
leaseId := lm.LeaseId(c.id)
|
||||||
if leaseId == 0 {
|
if leaseId == 0 {
|
||||||
leaseId, err = c.LeaseGrant(writeCtx, t.leaseTTL)
|
leaseId, err = c.LeaseGrant(opCtx, t.leaseTTL)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
lm.AddLeaseId(c.id, leaseId)
|
lm.AddLeaseId(c.id, leaseId)
|
||||||
limiter.Wait(ctx)
|
limiter.Wait(ctx)
|
||||||
@ -159,14 +173,14 @@ func (t etcdTraffic) Write(ctx context.Context, c *RecordingClient, limiter *rat
|
|||||||
case LeaseRevoke:
|
case LeaseRevoke:
|
||||||
leaseId := lm.LeaseId(c.id)
|
leaseId := lm.LeaseId(c.id)
|
||||||
if leaseId != 0 {
|
if leaseId != 0 {
|
||||||
err = c.LeaseRevoke(writeCtx, leaseId)
|
err = c.LeaseRevoke(opCtx, leaseId)
|
||||||
//if LeaseRevoke has failed, do not remove the mapping.
|
//if LeaseRevoke has failed, do not remove the mapping.
|
||||||
if err == nil {
|
if err == nil {
|
||||||
lm.RemoveLeaseId(c.id)
|
lm.RemoveLeaseId(c.id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
case Defragment:
|
case Defragment:
|
||||||
err = c.Defragment(writeCtx)
|
err = c.Defragment(opCtx)
|
||||||
default:
|
default:
|
||||||
panic("invalid choice")
|
panic("invalid choice")
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user