Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15819 from serathius/robustness-non-deterministic

Browse Source 
tests/robustness: Split model code into deterministic and non-deterministic
This commit is contained in:
Marek Siarkowicz 2023-05-05 13:16:28 +02:00 committed by GitHub
commit 79eabc1cbf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 931 additions and 452 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/robustness/linearizability_test.go
View File

@ -270,7 +270,7 @@ func runScenario(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.Et
func operationsMaxRevision(operations []porcupine.Operation) int64 {
var maxRevision int64
for _, op := range operations {
revision := op.Output.(model.EtcdResponse).Revision
revision := op.Output.(model.EtcdNonDeterministicResponse).Revision
if revision > maxRevision {
maxRevision = revision
}

140
tests/robustness/model/describe.go Normal file
View File

@ -0,0 +1,140 @@
// Copyright 2023 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package model
import (
"fmt"
"strings"
)
func describeEtcdNonDeterministicResponse(request EtcdRequest, response EtcdNonDeterministicResponse) string {
if response.Err != nil {
return fmt.Sprintf("err: %q", response.Err)
}
if response.ResultUnknown {
return fmt.Sprintf("unknown, rev: %d", response.Revision)
}
return describeEtcdResponse(request, response.EtcdResponse)
}
func describeEtcdResponse(request EtcdRequest, response EtcdResponse) string {
if request.Type == Txn {
return fmt.Sprintf("%s, rev: %d", describeTxnResponse(request.Txn, response.Txn), response.Revision)
}
if response.Revision == 0 {
return "ok"
}
return fmt.Sprintf("ok, rev: %d", response.Revision)
}
func describeEtcdRequest(request EtcdRequest) string {
switch request.Type {
case Txn:
describeOperations := describeEtcdOperations(request.Txn.Ops)
if len(request.Txn.Conds) != 0 {
return fmt.Sprintf("if(%s).then(%s)", describeEtcdConditions(request.Txn.Conds), describeOperations)
}
return describeOperations
case LeaseGrant:
return fmt.Sprintf("leaseGrant(%d)", request.LeaseGrant.LeaseID)
case LeaseRevoke:
return fmt.Sprintf("leaseRevoke(%d)", request.LeaseRevoke.LeaseID)
case Defragment:
return fmt.Sprintf("defragment()")
default:
return fmt.Sprintf("<! unknown request type: %q !>", request.Type)
}
}
func describeEtcdConditions(conds []EtcdCondition) string {
opsDescription := make([]string, len(conds))
for i := range conds {
opsDescription[i] = fmt.Sprintf("mod_rev(%s)==%d", conds[i].Key, conds[i].ExpectedRevision)
}
return strings.Join(opsDescription, " && ")
}
func describeEtcdOperations(ops []EtcdOperation) string {
opsDescription := make([]string, len(ops))
for i := range ops {
opsDescription[i] = describeEtcdOperation(ops[i])
}
return strings.Join(opsDescription, ", ")
}
func describeTxnResponse(request *TxnRequest, response *TxnResponse) string {
if response.TxnResult {
return fmt.Sprintf("txn failed")
}
respDescription := make([]string, len(response.OpsResult))
for i := range response.OpsResult {
respDescription[i] = describeEtcdOperationResponse(request.Ops[i], response.OpsResult[i])
}
return strings.Join(respDescription, ", ")
}
func describeEtcdOperation(op EtcdOperation) string {
switch op.Type {
case Range:
if op.WithPrefix {
return fmt.Sprintf("range(%q)", op.Key)
}
return fmt.Sprintf("get(%q)", op.Key)
case Put:
if op.LeaseID != 0 {
return fmt.Sprintf("put(%q, %s, %d)", op.Key, describeValueOrHash(op.Value), op.LeaseID)
}
return fmt.Sprintf("put(%q, %s)", op.Key, describeValueOrHash(op.Value))
case Delete:
return fmt.Sprintf("delete(%q)", op.Key)
default:
return fmt.Sprintf("<! unknown op: %q !>", op.Type)
}
}
func describeEtcdOperationResponse(req EtcdOperation, resp EtcdOperationResult) string {
switch req.Type {
case Range:
if req.WithPrefix {
kvs := make([]string, len(resp.KVs))
for i, kv := range resp.KVs {
kvs[i] = describeValueOrHash(kv.Value)
}
return fmt.Sprintf("[%s]", strings.Join(kvs, ","))
} else {
if len(resp.KVs) == 0 {
return "nil"
} else {
return describeValueOrHash(resp.KVs[0].Value)
}
}
case Put:
return fmt.Sprintf("ok")
case Delete:
return fmt.Sprintf("deleted: %d", resp.Deleted)
default:
return fmt.Sprintf("<! unknown op: %q !>", req.Type)
}
}
func describeValueOrHash(value ValueOrHash) string {
if value.Hash != 0 {
return fmt.Sprintf("hash: %d", value.Hash)
}
if value.Value == "" {
return "nil"
}
return fmt.Sprintf("%q", value.Value)
}

126
tests/robustness/model/describe_test.go Normal file
View File

@ -0,0 +1,126 @@
// Copyright 2023 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package model
import (
"errors"
"testing"
"github.com/stretchr/testify/assert"
"go.etcd.io/etcd/api/v3/mvccpb"
)
func TestModelDescribe(t *testing.T) {
tcs := []struct {
req EtcdRequest
resp EtcdNonDeterministicResponse
expectDescribe string
}{
{
req: getRequest("key1"),
resp: emptyGetResponse(1),
expectDescribe: `get("key1") -> nil, rev: 1`,
},
{
req: getRequest("key2"),
resp: getResponse("key", "2", 2, 2),
expectDescribe: `get("key2") -> "2", rev: 2`,
},
{
req: getRequest("key2b"),
resp: getResponse("key2b", "01234567890123456789", 2, 2),
expectDescribe: `get("key2b") -> hash: 2945867837, rev: 2`,
},
{
req: putRequest("key3", "3"),
resp: putResponse(3),
expectDescribe: `put("key3", "3") -> ok, rev: 3`,
},
{
req: putWithLeaseRequest("key3b", "3b", 3),
resp: putResponse(3),
expectDescribe: `put("key3b", "3b", 3) -> ok, rev: 3`,
},
{
req: putRequest("key3c", "01234567890123456789"),
resp: putResponse(3),
expectDescribe: `put("key3c", hash: 2945867837) -> ok, rev: 3`,
},
{
req: putRequest("key4", "4"),
resp: failedResponse(errors.New("failed")),
expectDescribe: `put("key4", "4") -> err: "failed"`,
},
{
req: putRequest("key4b", "4b"),
resp: unknownResponse(42),
expectDescribe: `put("key4b", "4b") -> unknown, rev: 42`,
},
{
req: deleteRequest("key5"),
resp: deleteResponse(1, 5),
expectDescribe: `delete("key5") -> deleted: 1, rev: 5`,
},
{
req: deleteRequest("key6"),
resp: failedResponse(errors.New("failed")),
expectDescribe: `delete("key6") -> err: "failed"`,
},
{
req: compareAndSetRequest("key7", 7, "77"),
resp: compareAndSetResponse(false, 7),
expectDescribe: `if(mod_rev(key7)==7).then(put("key7", "77")) -> txn failed, rev: 7`,
},
{
req: compareAndSetRequest("key8", 8, "88"),
resp: compareAndSetResponse(true, 8),
expectDescribe: `if(mod_rev(key8)==8).then(put("key8", "88")) -> ok, rev: 8`,
},
{
req: compareAndSetRequest("key9", 9, "99"),
resp: failedResponse(errors.New("failed")),
expectDescribe: `if(mod_rev(key9)==9).then(put("key9", "99")) -> err: "failed"`,
},
{
req: txnRequest(nil, []EtcdOperation{{Type: Range, Key: "10"}, {Type: Put, Key: "11", Value: ValueOrHash{Value: "111"}}, {Type: Delete, Key: "12"}}),
resp: txnResponse([]EtcdOperationResult{{KVs: []KeyValue{{ValueRevision: ValueRevision{Value: ValueOrHash{Value: "110"}}}}}, {}, {Deleted: 1}}, true, 10),
expectDescribe: `get("10"), put("11", "111"), delete("12") -> "110", ok, deleted: 1, rev: 10`,
},
{
req: defragmentRequest(),
resp: defragmentResponse(10),
expectDescribe: `defragment() -> ok, rev: 10`,
},
{
req: rangeRequest("key11", true),
resp: rangeResponse(nil, 11),
expectDescribe: `range("key11") -> [], rev: 11`,
},
{
req: rangeRequest("key12", true),
resp: rangeResponse([]*mvccpb.KeyValue{{Value: []byte("12")}}, 12),
expectDescribe: `range("key12") -> ["12"], rev: 12`,
},
{
req: rangeRequest("key13", true),
resp: rangeResponse([]*mvccpb.KeyValue{{Value: []byte("01234567890123456789")}}, 13),
expectDescribe: `range("key13") -> [hash: 2945867837], rev: 13`,
},
}
for _, tc := range tcs {
assert.Equal(t, tc.expectDescribe, NonDeterministicModel.DescribeOperation(tc.req, tc.resp))
}
}

420
tests/robustness/model/model.go → tests/robustness/model/deterministic.go
View File

@ -1,4 +1,4 @@
// Copyright 2022 The etcd Authors
// Copyright 2023 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -25,290 +25,52 @@ import (
"github.com/anishathalye/porcupine"
)
type OperationType string
const (
Range OperationType = "range"
Put OperationType = "put"
Delete OperationType = "delete"
)
var Etcd = porcupine.Model{
// DeterministicModel assumes that all requests succeed and have a correct response.
var DeterministicModel = porcupine.Model{
Init: func() interface{} {
return "[]" // empty PossibleStates
},
Step: func(st interface{}, in interface{}, out interface{}) (bool, interface{}) {
var states PossibleStates
err := json.Unmarshal([]byte(st.(string)), &states)
var s etcdState
data, err := json.Marshal(s)
if err != nil {
panic(err)
}
ok, states := step(states, in.(EtcdRequest), out.(EtcdResponse))
data, err := json.Marshal(states)
return string(data)
},
Step: func(st interface{}, in interface{}, out interface{}) (bool, interface{}) {
var s etcdState
err := json.Unmarshal([]byte(st.(string)), &s)
if err != nil {
panic(err)
}
ok, s := s.Step(in.(EtcdRequest), out.(EtcdResponse))
data, err := json.Marshal(s)
if err != nil {
panic(err)
}
return ok, string(data)
},
DescribeOperation: func(in, out interface{}) string {
return describeEtcdRequestResponse(in.(EtcdRequest), out.(EtcdResponse))
return fmt.Sprintf("%s -> %s", describeEtcdRequest(in.(EtcdRequest)), describeEtcdResponse(in.(EtcdRequest), out.(EtcdResponse)))
},
}
type RequestType string
const (
Txn RequestType = "txn"
LeaseGrant RequestType = "leaseGrant"
LeaseRevoke RequestType = "leaseRevoke"
Defragment RequestType = "defragment"
)
type EtcdRequest struct {
Type RequestType
LeaseGrant *LeaseGrantRequest
LeaseRevoke *LeaseRevokeRequest
Txn *TxnRequest
Defragment *DefragmentRequest
}
type TxnRequest struct {
Conds []EtcdCondition
Ops []EtcdOperation
}
type EtcdCondition struct {
Key string
ExpectedRevision int64
}
type EtcdOperation struct {
Type OperationType
Key string
WithPrefix bool
Value ValueOrHash
LeaseID int64
}
type LeaseGrantRequest struct {
LeaseID int64
}
type LeaseRevokeRequest struct {
LeaseID int64
}
type DefragmentRequest struct{}
type EtcdResponse struct {
Err error
Revision int64
ResultUnknown bool
Txn *TxnResponse
LeaseGrant *LeaseGrantReponse
LeaseRevoke *LeaseRevokeResponse
Defragment *DefragmentResponse
}
type TxnResponse struct {
TxnResult bool
OpsResult []EtcdOperationResult
}
type LeaseGrantReponse struct {
LeaseID int64
}
type LeaseRevokeResponse struct{}
type DefragmentResponse struct{}
func Match(r1, r2 EtcdResponse) bool {
return ((r1.ResultUnknown || r2.ResultUnknown) && (r1.Revision == r2.Revision)) || reflect.DeepEqual(r1, r2)
}
type EtcdOperationResult struct {
KVs []KeyValue
Deleted int64
}
type KeyValue struct {
Key string
ValueRevision
}
var leased = struct{}{}
type EtcdLease struct {
LeaseID int64
Keys map[string]struct{}
}
type PossibleStates []EtcdState
type EtcdState struct {
type etcdState struct {
Revision int64
KeyValues map[string]ValueRevision
KeyLeases map[string]int64
Leases map[int64]EtcdLease
}
type ValueRevision struct {
Value ValueOrHash
ModRevision int64
}
type ValueOrHash struct {
Value string
Hash uint32
}
func ToValueOrHash(value string) ValueOrHash {
v := ValueOrHash{}
if len(value) < 20 {
v.Value = value
} else {
h := fnv.New32a()
h.Write([]byte(value))
v.Hash = h.Sum32()
func (s etcdState) Step(request EtcdRequest, response EtcdResponse) (bool, etcdState) {
if s.Revision == 0 {
return true, initState(request, response)
}
return v
}
func describeEtcdRequestResponse(request EtcdRequest, response EtcdResponse) string {
return fmt.Sprintf("%s -> %s", describeEtcdRequest(request), describeEtcdResponse(request, response))
}
func describeEtcdResponse(request EtcdRequest, response EtcdResponse) string {
if response.Err != nil {
return fmt.Sprintf("err: %q", response.Err)
}
if response.ResultUnknown {
return fmt.Sprintf("unknown, rev: %d", response.Revision)
}
if request.Type == Txn {
return fmt.Sprintf("%s, rev: %d", describeTxnResponse(request.Txn, response.Txn), response.Revision)
}
if response.Revision == 0 {
return "ok"
}
return fmt.Sprintf("ok, rev: %d", response.Revision)
}
func describeEtcdRequest(request EtcdRequest) string {
switch request.Type {
case Txn:
describeOperations := describeEtcdOperations(request.Txn.Ops)
if len(request.Txn.Conds) != 0 {
return fmt.Sprintf("if(%s).then(%s)", describeEtcdConditions(request.Txn.Conds), describeOperations)
}
return describeOperations
case LeaseGrant:
return fmt.Sprintf("leaseGrant(%d)", request.LeaseGrant.LeaseID)
case LeaseRevoke:
return fmt.Sprintf("leaseRevoke(%d)", request.LeaseRevoke.LeaseID)
case Defragment:
return fmt.Sprintf("defragment()")
default:
return fmt.Sprintf("<! unknown request type: %q !>", request.Type)
}
}
func describeEtcdConditions(conds []EtcdCondition) string {
opsDescription := make([]string, len(conds))
for i := range conds {
opsDescription[i] = fmt.Sprintf("mod_rev(%s)==%d", conds[i].Key, conds[i].ExpectedRevision)
}
return strings.Join(opsDescription, " && ")
}
func describeEtcdOperations(ops []EtcdOperation) string {
opsDescription := make([]string, len(ops))
for i := range ops {
opsDescription[i] = describeEtcdOperation(ops[i])
}
return strings.Join(opsDescription, ", ")
}
func describeTxnResponse(request *TxnRequest, response *TxnResponse) string {
if response.TxnResult {
return fmt.Sprintf("txn failed")
}
respDescription := make([]string, len(response.OpsResult))
for i := range response.OpsResult {
respDescription[i] = describeEtcdOperationResponse(request.Ops[i], response.OpsResult[i])
}
return strings.Join(respDescription, ", ")
}
func describeEtcdOperation(op EtcdOperation) string {
switch op.Type {
case Range:
if op.WithPrefix {
return fmt.Sprintf("range(%q)", op.Key)
}
return fmt.Sprintf("get(%q)", op.Key)
case Put:
if op.LeaseID != 0 {
return fmt.Sprintf("put(%q, %s, %d)", op.Key, describeValueOrHash(op.Value), op.LeaseID)
}
return fmt.Sprintf("put(%q, %s)", op.Key, describeValueOrHash(op.Value))
case Delete:
return fmt.Sprintf("delete(%q)", op.Key)
default:
return fmt.Sprintf("<! unknown op: %q !>", op.Type)
}
}
func describeEtcdOperationResponse(req EtcdOperation, resp EtcdOperationResult) string {
switch req.Type {
case Range:
if req.WithPrefix {
kvs := make([]string, len(resp.KVs))
for i, kv := range resp.KVs {
kvs[i] = describeValueOrHash(kv.Value)
}
return fmt.Sprintf("[%s]", strings.Join(kvs, ","))
} else {
if len(resp.KVs) == 0 {
return "nil"
} else {
return describeValueOrHash(resp.KVs[0].Value)
}
}
case Put:
return fmt.Sprintf("ok")
case Delete:
return fmt.Sprintf("deleted: %d", resp.Deleted)
default:
return fmt.Sprintf("<! unknown op: %q !>", req.Type)
}
}
func describeValueOrHash(value ValueOrHash) string {
if value.Hash != 0 {
return fmt.Sprintf("hash: %d", value.Hash)
}
if value.Value == "" {
return "nil"
}
return fmt.Sprintf("%q", value.Value)
}
func step(states PossibleStates, request EtcdRequest, response EtcdResponse) (bool, PossibleStates) {
if len(states) == 0 {
// states were not initialized
if response.Err != nil || response.ResultUnknown || response.Revision == 0 {
return true, nil
}
return true, PossibleStates{initState(request, response)}
}
if response.Err != nil {
states = applyFailedRequest(states, request)
} else {
states = applyRequest(states, request, response)
}
return len(states) > 0, states
newState, gotResponse := s.step(request)
return reflect.DeepEqual(response, gotResponse), newState
}
// initState tries to create etcd state based on the first request.
func initState(request EtcdRequest, response EtcdResponse) EtcdState {
state := EtcdState{
func initState(request EtcdRequest, response EtcdResponse) etcdState {
state := etcdState{
Revision: response.Revision,
KeyValues: map[string]ValueRevision{},
KeyLeases: map[string]int64{},
@ -356,31 +118,8 @@ func initState(request EtcdRequest, response EtcdResponse) EtcdState {
return state
}
// applyFailedRequest handles a failed requests, one that it's not known if it was persisted or not.
func applyFailedRequest(states PossibleStates, request EtcdRequest) PossibleStates {
for _, s := range states {
newState, _ := applyRequestToSingleState(s, request)
if !reflect.DeepEqual(newState, s) {
states = append(states, newState)
}
}
return states
}
// applyRequest handles a successful request by applying it to possible states and checking if they match the response.
func applyRequest(states PossibleStates, request EtcdRequest, response EtcdResponse) PossibleStates {
newStates := make(PossibleStates, 0, len(states))
for _, s := range states {
newState, expectResponse := applyRequestToSingleState(s, request)
if Match(expectResponse, response) {
newStates = append(newStates, newState)
}
}
return newStates
}
// applyRequestToSingleState handles a successful request, returning updated state and response it would generate.
func applyRequestToSingleState(s EtcdState, request EtcdRequest) (EtcdState, EtcdResponse) {
// step handles a successful request, returning updated state and response it would generate.
func (s etcdState) step(request EtcdRequest) (etcdState, EtcdResponse) {
newKVs := map[string]ValueRevision{}
for k, v := range s.KeyValues {
newKVs[k] = v
@ -480,13 +219,13 @@ func applyRequestToSingleState(s EtcdState, request EtcdRequest) (EtcdState, Etc
}
return s, EtcdResponse{Revision: s.Revision, LeaseRevoke: &LeaseRevokeResponse{}}
case Defragment:
return s, defragmentResponse()
return s, EtcdResponse{Defragment: &DefragmentResponse{}, Revision: s.Revision}
default:
panic(fmt.Sprintf("Unknown request type: %v", request.Type))
}
}
func detachFromOldLease(s EtcdState, key string) EtcdState {
func detachFromOldLease(s etcdState, key string) etcdState {
if oldLeaseId, ok := s.KeyLeases[key]; ok {
delete(s.Leases[oldLeaseId].Keys, key)
delete(s.KeyLeases, key)
@ -494,8 +233,109 @@ func detachFromOldLease(s EtcdState, key string) EtcdState {
return s
}
func attachToNewLease(s EtcdState, leaseID int64, key string) EtcdState {
func attachToNewLease(s etcdState, leaseID int64, key string) etcdState {
s.KeyLeases[key] = leaseID
s.Leases[leaseID].Keys[key] = leased
return s
}
type RequestType string
const (
Txn RequestType = "txn"
LeaseGrant RequestType = "leaseGrant"
LeaseRevoke RequestType = "leaseRevoke"
Defragment RequestType = "defragment"
)
type EtcdRequest struct {
Type RequestType
LeaseGrant *LeaseGrantRequest
LeaseRevoke *LeaseRevokeRequest
Txn *TxnRequest
Defragment *DefragmentRequest
}
type TxnRequest struct {
Conds []EtcdCondition
Ops []EtcdOperation
}
type EtcdCondition struct {
Key string
ExpectedRevision int64
}
type EtcdOperation struct {
Type OperationType
Key string
WithPrefix bool
Value ValueOrHash
LeaseID int64
}
type LeaseGrantRequest struct {
LeaseID int64
}
type LeaseRevokeRequest struct {
LeaseID int64
}
type DefragmentRequest struct{}
type EtcdResponse struct {
Revision int64
Txn *TxnResponse
LeaseGrant *LeaseGrantReponse
LeaseRevoke *LeaseRevokeResponse
Defragment *DefragmentResponse
}
type TxnResponse struct {
TxnResult bool
OpsResult []EtcdOperationResult
}
type LeaseGrantReponse struct {
LeaseID int64
}
type LeaseRevokeResponse struct{}
type DefragmentResponse struct{}
type EtcdOperationResult struct {
KVs []KeyValue
Deleted int64
}
type KeyValue struct {
Key string
ValueRevision
}
var leased = struct{}{}
type EtcdLease struct {
LeaseID int64
Keys map[string]struct{}
}
type ValueRevision struct {
Value ValueOrHash
ModRevision int64
}
type ValueOrHash struct {
Value string
Hash uint32
}
func ToValueOrHash(value string) ValueOrHash {
v := ValueOrHash{}
if len(value) < 20 {
v.Value = value
} else {
h := fnv.New32a()
h.Write([]byte(value))
v.Hash = h.Sum32()
}
return v
}

362
tests/robustness/model/deterministic_test.go Normal file
View File

@ -0,0 +1,362 @@
// Copyright 2023 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package model
import (
"testing"
"go.etcd.io/etcd/api/v3/mvccpb"
)
func TestModelBase(t *testing.T) {
type testOperation struct {
req EtcdRequest
resp EtcdResponse
failure bool
}
tcs := []struct {
name string
operations []testOperation
}{
{
name: "First Get can start from non-empty value and non-zero revision",
operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 42, 42).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "1", 42, 42).EtcdResponse},
},
},
{
name: "First Range can start from non-empty value and non-zero revision",
operations: []testOperation{
{req: rangeRequest("key", true), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key"), Value: []byte("1")}}, 42).EtcdResponse},
{req: rangeRequest("key", true), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key"), Value: []byte("1")}}, 42).EtcdResponse},
},
},
{
name: "First Range can start from non-zero revision",
operations: []testOperation{
{req: rangeRequest("key", true), resp: rangeResponse(nil, 1).EtcdResponse},
{req: rangeRequest("key", true), resp: rangeResponse(nil, 1).EtcdResponse},
},
},
{
name: "First Put can start from non-zero revision",
operations: []testOperation{
{req: putRequest("key", "1"), resp: putResponse(42).EtcdResponse},
},
},
{
name: "First delete can start from non-zero revision",
operations: []testOperation{
{req: deleteRequest("key"), resp: deleteResponse(0, 42).EtcdResponse},
},
},
{
name: "First Txn can start from non-zero revision",
operations: []testOperation{
{req: compareAndSetRequest("key", 0, "42"), resp: compareAndSetResponse(false, 42).EtcdResponse},
},
},
{
name: "Get response data should match put",
operations: []testOperation{
{req: putRequest("key1", "11"), resp: putResponse(1).EtcdResponse},
{req: putRequest("key2", "12"), resp: putResponse(2).EtcdResponse},
{req: getRequest("key1"), resp: getResponse("key1", "11", 1, 1).EtcdResponse, failure: true},
{req: getRequest("key1"), resp: getResponse("key1", "12", 1, 1).EtcdResponse, failure: true},
{req: getRequest("key1"), resp: getResponse("key1", "12", 2, 2).EtcdResponse, failure: true},
{req: getRequest("key1"), resp: getResponse("key1", "11", 1, 2).EtcdResponse},
{req: getRequest("key2"), resp: getResponse("key2", "11", 2, 2).EtcdResponse, failure: true},
{req: getRequest("key2"), resp: getResponse("key2", "12", 1, 1).EtcdResponse, failure: true},
{req: getRequest("key2"), resp: getResponse("key2", "11", 1, 1).EtcdResponse, failure: true},
{req: getRequest("key2"), resp: getResponse("key2", "12", 2, 2).EtcdResponse},
},
},
{
name: "Range response data should match put",
operations: []testOperation{
{req: putRequest("key1", "1"), resp: putResponse(1).EtcdResponse},
{req: putRequest("key2", "2"), resp: putResponse(2).EtcdResponse},
{req: rangeRequest("key", true), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 2}}, 2).EtcdResponse},
{req: rangeRequest("key", true), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 2}}, 2).EtcdResponse},
},
},
{
name: "Range response should be ordered by key",
operations: []testOperation{
{req: rangeRequest("key", true), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key1"), Value: []byte("2"), ModRevision: 3},
{Key: []byte("key2"), Value: []byte("1"), ModRevision: 2},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 1},
}, 3).EtcdResponse},
},
},
{
name: "Range response data should match large put",
operations: []testOperation{
{req: putRequest("key", "012345678901234567890"), resp: putResponse(1).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "123456789012345678901", 1, 1).EtcdResponse, failure: true},
{req: getRequest("key"), resp: getResponse("key", "012345678901234567890", 1, 1).EtcdResponse},
{req: putRequest("key", "123456789012345678901"), resp: putResponse(2).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "123456789012345678901", 2, 2).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "012345678901234567890", 2, 2).EtcdResponse, failure: true},
},
},
{
name: "Put must increase revision by 1",
operations: []testOperation{
{req: getRequest("key"), resp: emptyGetResponse(1).EtcdResponse},
{req: putRequest("key", "1"), resp: putResponse(1).EtcdResponse, failure: true},
{req: putRequest("key", "1"), resp: putResponse(3).EtcdResponse, failure: true},
{req: putRequest("key", "1"), resp: putResponse(2).EtcdResponse},
},
},
{
name: "Delete only increases revision on success",
operations: []testOperation{
{req: putRequest("key1", "11"), resp: putResponse(1).EtcdResponse},
{req: putRequest("key2", "12"), resp: putResponse(2).EtcdResponse},
{req: deleteRequest("key1"), resp: deleteResponse(1, 2).EtcdResponse, failure: true},
{req: deleteRequest("key1"), resp: deleteResponse(1, 3).EtcdResponse},
{req: deleteRequest("key1"), resp: deleteResponse(0, 4).EtcdResponse, failure: true},
{req: deleteRequest("key1"), resp: deleteResponse(0, 3).EtcdResponse},
},
},
{
name: "Delete not existing key",
operations: []testOperation{
{req: getRequest("key"), resp: emptyGetResponse(1).EtcdResponse},
{req: deleteRequest("key"), resp: deleteResponse(1, 2).EtcdResponse, failure: true},
{req: deleteRequest("key"), resp: deleteResponse(0, 1).EtcdResponse},
},
},
{
name: "Delete clears value",
operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1).EtcdResponse},
{req: deleteRequest("key"), resp: deleteResponse(1, 2).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1).EtcdResponse, failure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 2, 2).EtcdResponse, failure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 2).EtcdResponse, failure: true},
{req: getRequest("key"), resp: emptyGetResponse(2).EtcdResponse},
},
},
{
name: "Txn sets new value if value matches expected",
operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1).EtcdResponse},
{req: compareAndSetRequest("key", 1, "2"), resp: compareAndSetResponse(true, 1).EtcdResponse, failure: true},
{req: compareAndSetRequest("key", 1, "2"), resp: compareAndSetResponse(false, 2).EtcdResponse, failure: true},
{req: compareAndSetRequest("key", 1, "2"), resp: compareAndSetResponse(false, 1).EtcdResponse, failure: true},
{req: compareAndSetRequest("key", 1, "2"), resp: compareAndSetResponse(true, 2).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1).EtcdResponse, failure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 2).EtcdResponse, failure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 2, 2).EtcdResponse, failure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 1, 1).EtcdResponse, failure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2).EtcdResponse},
},
},
{
name: "Txn can expect on empty key",
operations: []testOperation{
{req: getRequest("key1"), resp: emptyGetResponse(1).EtcdResponse},
{req: compareAndSetRequest("key1", 0, "2"), resp: compareAndSetResponse(true, 2).EtcdResponse},
{req: compareAndSetRequest("key2", 0, "3"), resp: compareAndSetResponse(true, 3).EtcdResponse},
{req: compareAndSetRequest("key3", 4, "4"), resp: compareAndSetResponse(false, 4).EtcdResponse, failure: true},
},
},
{
name: "Txn doesn't do anything if value doesn't match expected",
operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1).EtcdResponse},
{req: compareAndSetRequest("key", 2, "3"), resp: compareAndSetResponse(true, 2).EtcdResponse, failure: true},
{req: compareAndSetRequest("key", 2, "3"), resp: compareAndSetResponse(true, 1).EtcdResponse, failure: true},
{req: compareAndSetRequest("key", 2, "3"), resp: compareAndSetResponse(false, 2).EtcdResponse, failure: true},
{req: compareAndSetRequest("key", 2, "3"), resp: compareAndSetResponse(false, 1).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "2", 1, 1).EtcdResponse, failure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2).EtcdResponse, failure: true},
{req: getRequest("key"), resp: getResponse("key", "3", 1, 1).EtcdResponse, failure: true},
{req: getRequest("key"), resp: getResponse("key", "3", 1, 2).EtcdResponse, failure: true},
{req: getRequest("key"), resp: getResponse("key", "3", 2, 2).EtcdResponse, failure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1).EtcdResponse},
},
},
{
name: "Put with valid lease id should succeed. Put with invalid lease id should fail",
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse},
{req: putWithLeaseRequest("key", "3", 2), resp: putResponse(3).EtcdResponse, failure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2).EtcdResponse},
},
},
{
name: "Put with valid lease id should succeed. Put with expired lease id should fail",
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2).EtcdResponse},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse},
{req: putWithLeaseRequest("key", "4", 1), resp: putResponse(4).EtcdResponse, failure: true},
{req: getRequest("key"), resp: emptyGetResponse(3).EtcdResponse},
},
},
{
name: "Revoke should increment the revision",
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse},
{req: getRequest("key"), resp: emptyGetResponse(3).EtcdResponse},
},
},
{
name: "Put following a PutWithLease will detach the key from the lease",
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse},
{req: putRequest("key", "3"), resp: putResponse(3).EtcdResponse},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "3", 3, 3).EtcdResponse},
},
},
{
name: "Change lease. Revoking older lease should not increment revision",
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: leaseGrantRequest(2), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse},
{req: putWithLeaseRequest("key", "3", 2), resp: putResponse(3).EtcdResponse},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "3", 3, 3).EtcdResponse},
{req: leaseRevokeRequest(2), resp: leaseRevokeResponse(4).EtcdResponse},
{req: getRequest("key"), resp: emptyGetResponse(4).EtcdResponse},
},
},
{
name: "Update key with same lease",
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse},
{req: putWithLeaseRequest("key", "3", 1), resp: putResponse(3).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "3", 3, 3).EtcdResponse},
},
},
{
name: "Deleting a leased key - revoke should not increment revision",
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse},
{req: deleteRequest("key"), resp: deleteResponse(1, 3).EtcdResponse},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(4).EtcdResponse, failure: true},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse},
},
},
{
name: "Lease a few keys - revoke should increment revision only once",
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key1", "1", 1), resp: putResponse(2).EtcdResponse},
{req: putWithLeaseRequest("key2", "2", 1), resp: putResponse(3).EtcdResponse},
{req: putWithLeaseRequest("key3", "3", 1), resp: putResponse(4).EtcdResponse},
{req: putWithLeaseRequest("key4", "4", 1), resp: putResponse(5).EtcdResponse},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(6).EtcdResponse},
},
},
{
name: "Lease some keys then delete some of them. Revoke should increment revision since some keys were still leased",
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key1", "1", 1), resp: putResponse(2).EtcdResponse},
{req: putWithLeaseRequest("key2", "2", 1), resp: putResponse(3).EtcdResponse},
{req: putWithLeaseRequest("key3", "3", 1), resp: putResponse(4).EtcdResponse},
{req: putWithLeaseRequest("key4", "4", 1), resp: putResponse(5).EtcdResponse},
{req: deleteRequest("key1"), resp: deleteResponse(1, 6).EtcdResponse},
{req: deleteRequest("key3"), resp: deleteResponse(1, 7).EtcdResponse},
{req: deleteRequest("key4"), resp: deleteResponse(1, 8).EtcdResponse},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(9).EtcdResponse},
{req: deleteRequest("key2"), resp: deleteResponse(0, 9).EtcdResponse},
{req: getRequest("key1"), resp: emptyGetResponse(9).EtcdResponse},
{req: getRequest("key2"), resp: emptyGetResponse(9).EtcdResponse},
{req: getRequest("key3"), resp: emptyGetResponse(9).EtcdResponse},
{req: getRequest("key4"), resp: emptyGetResponse(9).EtcdResponse},
},
},
{
name: "Lease some keys then delete all of them. Revoke should not increment",
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key1", "1", 1), resp: putResponse(2).EtcdResponse},
{req: putWithLeaseRequest("key2", "2", 1), resp: putResponse(3).EtcdResponse},
{req: putWithLeaseRequest("key3", "3", 1), resp: putResponse(4).EtcdResponse},
{req: putWithLeaseRequest("key4", "4", 1), resp: putResponse(5).EtcdResponse},
{req: deleteRequest("key1"), resp: deleteResponse(1, 6).EtcdResponse},
{req: deleteRequest("key2"), resp: deleteResponse(1, 7).EtcdResponse},
{req: deleteRequest("key3"), resp: deleteResponse(1, 8).EtcdResponse},
{req: deleteRequest("key4"), resp: deleteResponse(1, 9).EtcdResponse},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(9).EtcdResponse},
},
},
{
name: "All request types",
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key", "1", 1), resp: putResponse(2).EtcdResponse},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse},
{req: putRequest("key", "4"), resp: putResponse(4).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "4", 4, 4).EtcdResponse},
{req: compareAndSetRequest("key", 4, "5"), resp: compareAndSetResponse(true, 5).EtcdResponse},
{req: deleteRequest("key"), resp: deleteResponse(1, 6).EtcdResponse},
{req: defragmentRequest(), resp: defragmentResponse(6).EtcdResponse},
},
},
{
name: "Defragment success between all other request types",
operations: []testOperation{
{req: defragmentRequest(), resp: defragmentResponse(1).EtcdResponse},
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: defragmentRequest(), resp: defragmentResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key", "1", 1), resp: putResponse(2).EtcdResponse},
{req: defragmentRequest(), resp: defragmentResponse(2).EtcdResponse},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse},
{req: defragmentRequest(), resp: defragmentResponse(3).EtcdResponse},
{req: putRequest("key", "4"), resp: putResponse(4).EtcdResponse},
{req: defragmentRequest(), resp: defragmentResponse(4).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "4", 4, 4).EtcdResponse},
{req: defragmentRequest(), resp: defragmentResponse(4).EtcdResponse},
{req: compareAndSetRequest("key", 4, "5"), resp: compareAndSetResponse(true, 5).EtcdResponse},
{req: defragmentRequest(), resp: defragmentResponse(5).EtcdResponse},
{req: deleteRequest("key"), resp: deleteResponse(1, 6).EtcdResponse},
{req: defragmentRequest(), resp: defragmentResponse(6).EtcdResponse},
},
},
}
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
state := DeterministicModel.Init()
for _, op := range tc.operations {
t.Logf("state: %v", state)
ok, newState := DeterministicModel.Step(state, op.req, op.resp)
if op.failure == ok {
t.Errorf("Unexpected operation result, expect: %v, got: %v, operation: %s", !op.failure, ok, DeterministicModel.DescribeOperation(op.req, op.resp))
break
}
if ok {
state = newState
}
}
})
}
}

52
tests/robustness/model/history.go
View File

@ -30,7 +30,7 @@ import (
// ValidateOperationHistoryAndReturnVisualize return visualize as porcupine.linearizationInfo used to generate visualization is private.
func ValidateOperationHistoryAndReturnVisualize(t *testing.T, lg *zap.Logger, operations []porcupine.Operation) (visualize func(basepath string)) {
linearizable, info := porcupine.CheckOperationsVerbose(Etcd, operations, 5*time.Minute)
linearizable, info := porcupine.CheckOperationsVerbose(NonDeterministicModel, operations, 5*time.Minute)
if linearizable == porcupine.Illegal {
t.Error("Model is not linearizable")
}
@ -39,7 +39,7 @@ func ValidateOperationHistoryAndReturnVisualize(t *testing.T, lg *zap.Logger, op
}
return func(path string) {
lg.Info("Saving visualization", zap.String("path", path))
err := porcupine.VisualizePath(Etcd, info, path)
err := porcupine.VisualizePath(NonDeterministicModel, info, path)
if err != nil {
t.Errorf("Failed to visualize, err: %v", err)
}
@ -295,11 +295,15 @@ func (h *AppendableHistory) AppendDefragment(start, end time.Duration, resp *cli
h.appendFailed(request, start, err)
return
}
var revision int64
if resp != nil && resp.Header != nil {
revision = resp.Header.Revision
}
h.successful = append(h.successful, porcupine.Operation{
ClientId: h.id,
Input: request,
Call: start.Nanoseconds(),
Output: defragmentResponse(),
Output: defragmentResponse(revision),
Return: end.Nanoseconds(),
})
}
@ -325,15 +329,15 @@ func rangeRequest(key string, withPrefix bool) EtcdRequest {
return EtcdRequest{Type: Txn, Txn: &TxnRequest{Ops: []EtcdOperation{{Type: Range, Key: key, WithPrefix: withPrefix}}}}
}
func emptyGetResponse(revision int64) EtcdResponse {
func emptyGetResponse(revision int64) EtcdNonDeterministicResponse {
return rangeResponse([]*mvccpb.KeyValue{}, revision)
}
func getResponse(key, value string, modRevision, revision int64) EtcdResponse {
func getResponse(key, value string, modRevision, revision int64) EtcdNonDeterministicResponse {
return rangeResponse([]*mvccpb.KeyValue{{Key: []byte(key), Value: []byte(value), ModRevision: modRevision}}, revision)
}
func rangeResponse(kvs []*mvccpb.KeyValue, revision int64) EtcdResponse {
func rangeResponse(kvs []*mvccpb.KeyValue, revision int64) EtcdNonDeterministicResponse {
result := EtcdOperationResult{KVs: make([]KeyValue, len(kvs))}
for i, kv := range kvs {
@ -345,38 +349,38 @@ func rangeResponse(kvs []*mvccpb.KeyValue, revision int64) EtcdResponse {
},
}
}
return EtcdResponse{Txn: &TxnResponse{OpsResult: []EtcdOperationResult{result}}, Revision: revision}
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{OpsResult: []EtcdOperationResult{result}}, Revision: revision}}
}
func failedResponse(err error) EtcdResponse {
return EtcdResponse{Err: err}
func failedResponse(err error) EtcdNonDeterministicResponse {
return EtcdNonDeterministicResponse{Err: err}
}
func unknownResponse(revision int64) EtcdResponse {
return EtcdResponse{ResultUnknown: true, Revision: revision}
func unknownResponse(revision int64) EtcdNonDeterministicResponse {
return EtcdNonDeterministicResponse{ResultUnknown: true, EtcdResponse: EtcdResponse{Revision: revision}}
}
func putRequest(key, value string) EtcdRequest {
return EtcdRequest{Type: Txn, Txn: &TxnRequest{Ops: []EtcdOperation{{Type: Put, Key: key, Value: ToValueOrHash(value)}}}}
}
func putResponse(revision int64) EtcdResponse {
return EtcdResponse{Txn: &TxnResponse{OpsResult: []EtcdOperationResult{{}}}, Revision: revision}
func putResponse(revision int64) EtcdNonDeterministicResponse {
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{OpsResult: []EtcdOperationResult{{}}}, Revision: revision}}
}
func deleteRequest(key string) EtcdRequest {
return EtcdRequest{Type: Txn, Txn: &TxnRequest{Ops: []EtcdOperation{{Type: Delete, Key: key}}}}
}
func deleteResponse(deleted int64, revision int64) EtcdResponse {
return EtcdResponse{Txn: &TxnResponse{OpsResult: []EtcdOperationResult{{Deleted: deleted}}}, Revision: revision}
func deleteResponse(deleted int64, revision int64) EtcdNonDeterministicResponse {
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{OpsResult: []EtcdOperationResult{{Deleted: deleted}}}, Revision: revision}}
}
func compareAndSetRequest(key string, expectedRevision int64, value string) EtcdRequest {
return txnRequest([]EtcdCondition{{Key: key, ExpectedRevision: expectedRevision}}, []EtcdOperation{{Type: Put, Key: key, Value: ToValueOrHash(value)}})
}
func compareAndSetResponse(succeeded bool, revision int64) EtcdResponse {
func compareAndSetResponse(succeeded bool, revision int64) EtcdNonDeterministicResponse {
var result []EtcdOperationResult
if succeeded {
result = []EtcdOperationResult{{}}
@ -388,8 +392,8 @@ func txnRequest(conds []EtcdCondition, onSuccess []EtcdOperation) EtcdRequest {
return EtcdRequest{Type: Txn, Txn: &TxnRequest{Conds: conds, Ops: onSuccess}}
}
func txnResponse(result []EtcdOperationResult, succeeded bool, revision int64) EtcdResponse {
return EtcdResponse{Txn: &TxnResponse{OpsResult: result, TxnResult: !succeeded}, Revision: revision}
func txnResponse(result []EtcdOperationResult, succeeded bool, revision int64) EtcdNonDeterministicResponse {
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{OpsResult: result, TxnResult: !succeeded}, Revision: revision}}
}
func putWithLeaseRequest(key, value string, leaseID int64) EtcdRequest {
@ -400,24 +404,24 @@ func leaseGrantRequest(leaseID int64) EtcdRequest {
return EtcdRequest{Type: LeaseGrant, LeaseGrant: &LeaseGrantRequest{LeaseID: leaseID}}
}
func leaseGrantResponse(revision int64) EtcdResponse {
return EtcdResponse{LeaseGrant: &LeaseGrantReponse{}, Revision: revision}
func leaseGrantResponse(revision int64) EtcdNonDeterministicResponse {
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{LeaseGrant: &LeaseGrantReponse{}, Revision: revision}}
}
func leaseRevokeRequest(leaseID int64) EtcdRequest {
return EtcdRequest{Type: LeaseRevoke, LeaseRevoke: &LeaseRevokeRequest{LeaseID: leaseID}}
}
func leaseRevokeResponse(revision int64) EtcdResponse {
return EtcdResponse{LeaseRevoke: &LeaseRevokeResponse{}, Revision: revision}
func leaseRevokeResponse(revision int64) EtcdNonDeterministicResponse {
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{LeaseRevoke: &LeaseRevokeResponse{}, Revision: revision}}
}
func defragmentRequest() EtcdRequest {
return EtcdRequest{Type: Defragment, Defragment: &DefragmentRequest{}}
}
func defragmentResponse() EtcdResponse {
return EtcdResponse{Defragment: &DefragmentResponse{}}
func defragmentResponse(revision int64) EtcdNonDeterministicResponse {
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{Defragment: &DefragmentResponse{}, Revision: revision}}
}
type History struct {

120
tests/robustness/model/non_deterministic.go Normal file
View File

@ -0,0 +1,120 @@
// Copyright 2022 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package model
import (
"encoding/json"
"fmt"
"reflect"
"github.com/anishathalye/porcupine"
)
type OperationType string
const (
Range OperationType = "range"
Put OperationType = "put"
Delete OperationType = "delete"
)
// NonDeterministicModel extends DeterministicModel to handle requests that have unknown or error response.
// Unknown/error response doesn't inform whether request was persisted or not, so model
// considers both cases. This is represented as multiple equally possible deterministic states.
// Failed requests fork the possible states, while successful requests merge and filter them.
var NonDeterministicModel = porcupine.Model{
Init: func() interface{} {
var states nonDeterministicState
data, err := json.Marshal(states)
if err != nil {
panic(err)
}
return string(data)
},
Step: func(st interface{}, in interface{}, out interface{}) (bool, interface{}) {
var states nonDeterministicState
err := json.Unmarshal([]byte(st.(string)), &states)
if err != nil {
panic(err)
}
ok, states := states.Step(in.(EtcdRequest), out.(EtcdNonDeterministicResponse))
data, err := json.Marshal(states)
if err != nil {
panic(err)
}
return ok, string(data)
},
DescribeOperation: func(in, out interface{}) string {
return fmt.Sprintf("%s -> %s", describeEtcdRequest(in.(EtcdRequest)), describeEtcdNonDeterministicResponse(in.(EtcdRequest), out.(EtcdNonDeterministicResponse)))
},
}
type nonDeterministicState []etcdState
type EtcdNonDeterministicResponse struct {
EtcdResponse
Err error
ResultUnknown bool
}
func (states nonDeterministicState) Step(request EtcdRequest, response EtcdNonDeterministicResponse) (bool, nonDeterministicState) {
if len(states) == 0 {
// states were not initialized
if response.Err != nil || response.ResultUnknown || response.Revision == 0 {
return true, nil
}
return true, initNonDeterministicState(request, response)
}
var newStates nonDeterministicState
if response.Err != nil {
newStates = states.stepFailedRequest(request)
} else {
newStates = states.stepSuccessfulRequest(request, response)
}
return len(newStates) > 0, newStates
}
func initNonDeterministicState(request EtcdRequest, response EtcdNonDeterministicResponse) nonDeterministicState {
return nonDeterministicState{initState(request, response.EtcdResponse)}
}
// stepFailedRequest duplicates number of states by considering request persisted and lost.
func (states nonDeterministicState) stepFailedRequest(request EtcdRequest) nonDeterministicState {
newStates := make(nonDeterministicState, 0, len(states)*2)
for _, s := range states {
newStates = append(newStates, s)
newState, _ := s.step(request)
if !reflect.DeepEqual(newState, s) {
newStates = append(newStates, newState)
}
}
return newStates
}
// stepSuccessfulRequest filters possible states by leaving ony states that would respond correctly.
func (states nonDeterministicState) stepSuccessfulRequest(request EtcdRequest, response EtcdNonDeterministicResponse) nonDeterministicState {
newStates := make(nonDeterministicState, 0, len(states))
for _, s := range states {
newState, gotResponse := s.step(request)
if Match(EtcdNonDeterministicResponse{EtcdResponse: gotResponse}, response) {
newStates = append(newStates, newState)
}
}
return newStates
}
func Match(r1, r2 EtcdNonDeterministicResponse) bool {
return ((r1.ResultUnknown || r2.ResultUnknown) && (r1.Revision == r2.Revision)) || reflect.DeepEqual(r1, r2)
}

155
tests/robustness/model/model_test.go → tests/robustness/model/non_deterministic_test.go
View File

@ -15,17 +15,20 @@
package model
import (
"encoding/json"
"errors"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
"go.etcd.io/etcd/api/v3/mvccpb"
)
func TestModelStep(t *testing.T) {
func TestModelNonDeterministic(t *testing.T) {
type testOperation struct {
req EtcdRequest
resp EtcdNonDeterministicResponse
failure bool
}
tcs := []struct {
name string
operations []testOperation
@ -583,27 +586,27 @@ func TestModelStep(t *testing.T) {
{req: getRequest("key"), resp: getResponse("key", "4", 4, 4)},
{req: compareAndSetRequest("key", 4, "5"), resp: compareAndSetResponse(true, 5)},
{req: deleteRequest("key"), resp: deleteResponse(1, 6)},
{req: defragmentRequest(), resp: defragmentResponse()},
{req: defragmentRequest(), resp: defragmentResponse(6)},
},
},
{
name: "Defragment success between all other request types",
operations: []testOperation{
{req: defragmentRequest(), resp: defragmentResponse()},
{req: defragmentRequest(), resp: defragmentResponse(1)},
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: defragmentRequest(), resp: defragmentResponse()},
{req: defragmentRequest(), resp: defragmentResponse(1)},
{req: putWithLeaseRequest("key", "1", 1), resp: putResponse(2)},
{req: defragmentRequest(), resp: defragmentResponse()},
{req: defragmentRequest(), resp: defragmentResponse(2)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)},
{req: defragmentRequest(), resp: defragmentResponse()},
{req: defragmentRequest(), resp: defragmentResponse(3)},
{req: putRequest("key", "4"), resp: putResponse(4)},
{req: defragmentRequest(), resp: defragmentResponse()},
{req: defragmentRequest(), resp: defragmentResponse(4)},
{req: getRequest("key"), resp: getResponse("key", "4", 4, 4)},
{req: defragmentRequest(), resp: defragmentResponse()},
{req: defragmentRequest(), resp: defragmentResponse(4)},
{req: compareAndSetRequest("key", 4, "5"), resp: compareAndSetResponse(true, 5)},
{req: defragmentRequest(), resp: defragmentResponse()},
{req: defragmentRequest(), resp: defragmentResponse(5)},
{req: deleteRequest("key"), resp: deleteResponse(1, 6)},
{req: defragmentRequest(), resp: defragmentResponse()},
{req: defragmentRequest(), resp: defragmentResponse(6)},
},
},
{
@ -629,20 +632,12 @@ func TestModelStep(t *testing.T) {
}
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
state := Etcd.Init()
state := NonDeterministicModel.Init()
for _, op := range tc.operations {
ok, newState := Etcd.Step(state, op.req, op.resp)
ok, newState := NonDeterministicModel.Step(state, op.req, op.resp)
if ok != !op.failure {
t.Errorf("Unexpected operation result, expect: %v, got: %v, operation: %s", !op.failure, ok, Etcd.DescribeOperation(op.req, op.resp))
var states PossibleStates
err := json.Unmarshal([]byte(state.(string)), &states)
if err != nil {
panic(err)
}
for _, s := range states {
_, gotResp := applyRequestToSingleState(s, op.req)
t.Logf("For state: %v, diff: %s", state, cmp.Diff(op.resp, gotResp))
}
t.Logf("state: %v", state)
t.Errorf("Unexpected operation result, expect: %v, got: %v, operation: %s", !op.failure, ok, NonDeterministicModel.DescribeOperation(op.req, op.resp))
}
if ok {
state = newState
@ -653,118 +648,10 @@ func TestModelStep(t *testing.T) {
}
}
type testOperation struct {
req EtcdRequest
resp EtcdResponse
failure bool
}
func TestModelDescribe(t *testing.T) {
tcs := []struct {
req EtcdRequest
resp EtcdResponse
expectDescribe string
}{
{
req: getRequest("key1"),
resp: emptyGetResponse(1),
expectDescribe: `get("key1") -> nil, rev: 1`,
},
{
req: getRequest("key2"),
resp: getResponse("key", "2", 2, 2),
expectDescribe: `get("key2") -> "2", rev: 2`,
},
{
req: getRequest("key2b"),
resp: getResponse("key2b", "01234567890123456789", 2, 2),
expectDescribe: `get("key2b") -> hash: 2945867837, rev: 2`,
},
{
req: putRequest("key3", "3"),
resp: putResponse(3),
expectDescribe: `put("key3", "3") -> ok, rev: 3`,
},
{
req: putWithLeaseRequest("key3b", "3b", 3),
resp: putResponse(3),
expectDescribe: `put("key3b", "3b", 3) -> ok, rev: 3`,
},
{
req: putRequest("key3c", "01234567890123456789"),
resp: putResponse(3),
expectDescribe: `put("key3c", hash: 2945867837) -> ok, rev: 3`,
},
{
req: putRequest("key4", "4"),
resp: failedResponse(errors.New("failed")),
expectDescribe: `put("key4", "4") -> err: "failed"`,
},
{
req: putRequest("key4b", "4b"),
resp: unknownResponse(42),
expectDescribe: `put("key4b", "4b") -> unknown, rev: 42`,
},
{
req: deleteRequest("key5"),
resp: deleteResponse(1, 5),
expectDescribe: `delete("key5") -> deleted: 1, rev: 5`,
},
{
req: deleteRequest("key6"),
resp: failedResponse(errors.New("failed")),
expectDescribe: `delete("key6") -> err: "failed"`,
},
{
req: compareAndSetRequest("key7", 7, "77"),
resp: compareAndSetResponse(false, 7),
expectDescribe: `if(mod_rev(key7)==7).then(put("key7", "77")) -> txn failed, rev: 7`,
},
{
req: compareAndSetRequest("key8", 8, "88"),
resp: compareAndSetResponse(true, 8),
expectDescribe: `if(mod_rev(key8)==8).then(put("key8", "88")) -> ok, rev: 8`,
},
{
req: compareAndSetRequest("key9", 9, "99"),
resp: failedResponse(errors.New("failed")),
expectDescribe: `if(mod_rev(key9)==9).then(put("key9", "99")) -> err: "failed"`,
},
{
req: txnRequest(nil, []EtcdOperation{{Type: Range, Key: "10"}, {Type: Put, Key: "11", Value: ValueOrHash{Value: "111"}}, {Type: Delete, Key: "12"}}),
resp: txnResponse([]EtcdOperationResult{{KVs: []KeyValue{{ValueRevision: ValueRevision{Value: ValueOrHash{Value: "110"}}}}}, {}, {Deleted: 1}}, true, 10),
expectDescribe: `get("10"), put("11", "111"), delete("12") -> "110", ok, deleted: 1, rev: 10`,
},
{
req: defragmentRequest(),
resp: defragmentResponse(),
expectDescribe: `defragment() -> ok`,
},
{
req: rangeRequest("key11", true),
resp: rangeResponse(nil, 11),
expectDescribe: `range("key11") -> [], rev: 11`,
},
{
req: rangeRequest("key12", true),
resp: rangeResponse([]*mvccpb.KeyValue{{Value: []byte("12")}}, 12),
expectDescribe: `range("key12") -> ["12"], rev: 12`,
},
{
req: rangeRequest("key13", true),
resp: rangeResponse([]*mvccpb.KeyValue{{Value: []byte("01234567890123456789")}}, 13),
expectDescribe: `range("key13") -> [hash: 2945867837], rev: 13`,
},
}
for _, tc := range tcs {
assert.Equal(t, tc.expectDescribe, Etcd.DescribeOperation(tc.req, tc.resp))
}
}
func TestModelResponseMatch(t *testing.T) {
tcs := []struct {
resp1 EtcdResponse
resp2 EtcdResponse
resp1 EtcdNonDeterministicResponse
resp2 EtcdNonDeterministicResponse
expectMatch bool
}{
{

6
tests/robustness/watch.go
View File

@ -269,7 +269,7 @@ func patchOperationBasedOnWatchEvents(operations []porcupine.Operation, watchEve
for _, op := range operations {
request := op.Input.(model.EtcdRequest)
resp := op.Output.(model.EtcdResponse)
resp := op.Output.(model.EtcdNonDeterministicResponse)
if resp.Err == nil || op.Call > lastObservedOperation.Call || request.Type != model.Txn {
// Cannot patch those requests.
newOperations = append(newOperations, op)
@ -279,8 +279,8 @@ func patchOperationBasedOnWatchEvents(operations []porcupine.Operation, watchEve
if event != nil {
// Set revision and time based on watchEvent.
op.Return = event.Time.UnixNano()
op.Output = model.EtcdResponse{
Revision: event.Revision,
op.Output = model.EtcdNonDeterministicResponse{
EtcdResponse: model.EtcdResponse{Revision: event.Revision},
ResultUnknown: true,
}
newOperations = append(newOperations, op)