Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #16083 from serathius/robustness-perfect-knowledge

Browse Source 
Robustness Allow errors and partial responses from deterministic model
This commit is contained in:
Marek Siarkowicz 2023-06-16 10:07:05 +02:00 committed by GitHub
commit 32ea42b51c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 339 additions and 346 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
tests/robustness/model/describe.go
View File

@ -19,17 +19,13 @@ import (
"strings" "strings"
) )
func describeEtcdNonDeterministicResponse(request EtcdRequest, response EtcdNonDeterministicResponse) string { func describeEtcdResponse(request EtcdRequest, response MaybeEtcdResponse) string {
if response.Err != nil { if response.Err != nil {
return fmt.Sprintf("err: %q", response.Err) return fmt.Sprintf("err: %q", response.Err)
} }
if response.ResultUnknown { if response.PartialResponse {
return fmt.Sprintf("unknown, rev: %d", response.Revision) return fmt.Sprintf("unknown, rev: %d", response.Revision)
} }
return describeEtcdResponse(request, response.EtcdResponse)
}
func describeEtcdResponse(request EtcdRequest, response EtcdResponse) string {
switch request.Type { switch request.Type {
case Range: case Range:
return fmt.Sprintf("%s, rev: %d", describeRangeResponse(request.Range.RangeOptions, *response.Range), response.Revision) return fmt.Sprintf("%s, rev: %d", describeRangeResponse(request.Range.RangeOptions, *response.Range), response.Revision)

4
tests/robustness/model/describe_test.go
View File

@ -26,7 +26,7 @@ import (
func TestModelDescribe(t *testing.T) { func TestModelDescribe(t *testing.T) {
tcs := []struct { tcs := []struct {
req EtcdRequest req EtcdRequest
resp EtcdNonDeterministicResponse resp MaybeEtcdResponse
expectDescribe string expectDescribe string
}{ }{
{ {
@ -66,7 +66,7 @@ func TestModelDescribe(t *testing.T) {
}, },
{ {
req: putRequest("key4b", "4b"), req: putRequest("key4b", "4b"),
resp: unknownResponse(42), resp: partialResponse(42),
expectDescribe: `put("key4b", "4b") -> unknown, rev: 42`, expectDescribe: `put("key4b", "4b") -> unknown, rev: 42`,
}, },
{ {

50
tests/robustness/model/deterministic.go
View File

@ -25,7 +25,18 @@ import (
"github.com/anishathalye/porcupine" "github.com/anishathalye/porcupine"
) )
// DeterministicModel assumes that all requests succeed and have a correct response. // DeterministicModel assumes a deterministic execution of etcd requests. All
// requests that client called were executed and persisted by etcd. This
// assumption is good for simulating etcd behavior (aka writing a fake), but not
// for validating correctness as requests might be lost or interrupted. It
// requires perfect knowledge of what happened to request which is not possible
// in real systems.
//
// Model can still respond with error or partial response.
// - Error for etcd known errors, like future revision or compacted revision.
// - Incomplete response when requests is correct, but model doesn't have all
// to provide a full response. For example stale reads as model doesn't store
// whole change history as real etcd does.
var DeterministicModel = porcupine.Model{ var DeterministicModel = porcupine.Model{
Init: func() interface{} { Init: func() interface{} {
var s etcdState var s etcdState
@ -49,7 +60,7 @@ var DeterministicModel = porcupine.Model{
return ok, string(data) return ok, string(data)
}, },
DescribeOperation: func(in, out interface{}) string { DescribeOperation: func(in, out interface{}) string {
return fmt.Sprintf("%s -> %s", describeEtcdRequest(in.(EtcdRequest)), describeEtcdResponse(in.(EtcdRequest), out.(EtcdResponse))) return fmt.Sprintf("%s -> %s", describeEtcdRequest(in.(EtcdRequest)), describeEtcdResponse(in.(EtcdRequest), MaybeEtcdResponse{EtcdResponse: out.(EtcdResponse)}))
}, },
} }
@ -64,8 +75,8 @@ func (s etcdState) Step(request EtcdRequest, response EtcdResponse) (bool, etcdS
if s.Revision == 0 { if s.Revision == 0 {
return true, initState(request, response) return true, initState(request, response)
} }
newState, gotResponse := s.step(request) newState, modelResponse := s.step(request)
return reflect.DeepEqual(response, gotResponse), newState return Match(MaybeEtcdResponse{EtcdResponse: response}, modelResponse), newState
} }
// initState tries to create etcd state based on the first request. // initState tries to create etcd state based on the first request.
@ -85,7 +96,7 @@ func initState(request EtcdRequest, response EtcdResponse) etcdState {
return state return state
} }
if len(request.Txn.OperationsOnSuccess) != len(response.Txn.Results) { if len(request.Txn.OperationsOnSuccess) != len(response.Txn.Results) {
panic(fmt.Sprintf("Incorrect request %s, response %+v", describeEtcdRequest(request), describeEtcdResponse(request, response))) panic(fmt.Sprintf("Incorrect request %s, response %+v", describeEtcdRequest(request), describeEtcdResponse(request, MaybeEtcdResponse{EtcdResponse: response})))
} }
for i, op := range request.Txn.OperationsOnSuccess { for i, op := range request.Txn.OperationsOnSuccess {
opResp := response.Txn.Results[i] opResp := response.Txn.Results[i]
@ -131,7 +142,7 @@ func emptyState() etcdState {
} }
// step handles a successful request, returning updated state and response it would generate. // step handles a successful request, returning updated state and response it would generate.
func (s etcdState) step(request EtcdRequest) (etcdState, EtcdResponse) { func (s etcdState) step(request EtcdRequest) (etcdState, MaybeEtcdResponse) {
newKVs := map[string]ValueRevision{} newKVs := map[string]ValueRevision{}
for k, v := range s.KeyValues { for k, v := range s.KeyValues {
newKVs[k] = v newKVs[k] = v
@ -140,7 +151,7 @@ func (s etcdState) step(request EtcdRequest) (etcdState, EtcdResponse) {
switch request.Type { switch request.Type {
case Range: case Range:
resp := s.getRange(request.Range.Key, request.Range.RangeOptions) resp := s.getRange(request.Range.Key, request.Range.RangeOptions)
return s, EtcdResponse{Range: &resp, Revision: s.Revision} return s, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Range: &resp, Revision: s.Revision}}
case Txn: case Txn:
failure := false failure := false
for _, cond := range request.Txn.Conditions { for _, cond := range request.Txn.Conditions {
@ -189,14 +200,14 @@ func (s etcdState) step(request EtcdRequest) (etcdState, EtcdResponse) {
if increaseRevision { if increaseRevision {
s.Revision += 1 s.Revision += 1
} }
return s, EtcdResponse{Txn: &TxnResponse{Failure: failure, Results: opResp}, Revision: s.Revision} return s, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Failure: failure, Results: opResp}, Revision: s.Revision}}
case LeaseGrant: case LeaseGrant:
lease := EtcdLease{ lease := EtcdLease{
LeaseID: request.LeaseGrant.LeaseID, LeaseID: request.LeaseGrant.LeaseID,
Keys: map[string]struct{}{}, Keys: map[string]struct{}{},
} }
s.Leases[request.LeaseGrant.LeaseID] = lease s.Leases[request.LeaseGrant.LeaseID] = lease
return s, EtcdResponse{Revision: s.Revision, LeaseGrant: &LeaseGrantReponse{}} return s, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Revision: s.Revision, LeaseGrant: &LeaseGrantReponse{}}}
case LeaseRevoke: case LeaseRevoke:
//Delete the keys attached to the lease //Delete the keys attached to the lease
keyDeleted := false keyDeleted := false
@ -215,9 +226,9 @@ func (s etcdState) step(request EtcdRequest) (etcdState, EtcdResponse) {
if keyDeleted { if keyDeleted {
s.Revision += 1 s.Revision += 1
} }
return s, EtcdResponse{Revision: s.Revision, LeaseRevoke: &LeaseRevokeResponse{}} return s, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Revision: s.Revision, LeaseRevoke: &LeaseRevokeResponse{}}}
case Defragment: case Defragment:
return s, EtcdResponse{Defragment: &DefragmentResponse{}, Revision: s.Revision} return s, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Defragment: &DefragmentResponse{}, Revision: s.Revision}}
default: default:
panic(fmt.Sprintf("Unknown request type: %v", request.Type)) panic(fmt.Sprintf("Unknown request type: %v", request.Type))
} }
@ -339,13 +350,28 @@ type LeaseRevokeRequest struct {
} }
type DefragmentRequest struct{} type DefragmentRequest struct{}
// MaybeEtcdResponse extends EtcdResponse to represent partial or failed responses.
// Possible states:
// * Normal response. Only EtcdResponse is set.
// * Partial response. The EtcdResponse.Revision and PartialResponse are set.
// * Failed response. Only Err is set.
type MaybeEtcdResponse struct {
EtcdResponse
PartialResponse bool
Err error
}
type EtcdResponse struct { type EtcdResponse struct {
Revision int64
Txn *TxnResponse Txn *TxnResponse
Range *RangeResponse Range *RangeResponse
LeaseGrant *LeaseGrantReponse LeaseGrant *LeaseGrantReponse
LeaseRevoke *LeaseRevokeResponse LeaseRevoke *LeaseRevokeResponse
Defragment *DefragmentResponse Defragment *DefragmentResponse
Revision int64
}
func Match(r1, r2 MaybeEtcdResponse) bool {
return ((r1.PartialResponse || r2.PartialResponse) && (r1.Revision == r2.Revision)) || reflect.DeepEqual(r1, r2)
} }
type TxnResponse struct { type TxnResponse struct {

410
tests/robustness/model/deterministic_test.go
View File

@ -24,12 +24,12 @@ import (
) )
func TestModelDeterministic(t *testing.T) { func TestModelDeterministic(t *testing.T) {
for _, tc := range deterministicModelTestScenarios { for _, tc := range commonTestScenarios {
tc := tc tc := tc
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
state := DeterministicModel.Init() state := DeterministicModel.Init()
for _, op := range tc.operations { for _, op := range tc.operations {
ok, newState := DeterministicModel.Step(state, op.req, op.resp) ok, newState := DeterministicModel.Step(state, op.req, op.resp.EtcdResponse)
if op.expectFailure == ok { if op.expectFailure == ok {
t.Logf("state: %v", state) t.Logf("state: %v", state)
t.Errorf("Unexpected operation result, expect: %v, got: %v, operation: %s", !op.expectFailure, ok, DeterministicModel.DescribeOperation(op.req, op.resp)) t.Errorf("Unexpected operation result, expect: %v, got: %v, operation: %s", !op.expectFailure, ok, DeterministicModel.DescribeOperation(op.req, op.resp))
@ -51,363 +51,363 @@ func TestModelDeterministic(t *testing.T) {
} }
} }
type deterministicModelTest struct { type modelTestCase struct {
name string name string
operations []deterministicOperation operations []testOperation
} }
type deterministicOperation struct { type testOperation struct {
req EtcdRequest req EtcdRequest
resp EtcdResponse resp MaybeEtcdResponse
expectFailure bool expectFailure bool
} }
var deterministicModelTestScenarios = []deterministicModelTest{ var commonTestScenarios = []modelTestCase{
{ {
name: "First Get can start from non-empty value and non-zero revision", name: "First Get can start from non-empty value and non-zero revision",
operations: []deterministicOperation{ operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 42, 42).EtcdResponse}, {req: getRequest("key"), resp: getResponse("key", "1", 42, 42)},
{req: getRequest("key"), resp: getResponse("key", "1", 42, 42).EtcdResponse}, {req: getRequest("key"), resp: getResponse("key", "1", 42, 42)},
}, },
}, },
{ {
name: "First Range can start from non-empty value and non-zero revision", name: "First Range can start from non-empty value and non-zero revision",
operations: []deterministicOperation{ operations: []testOperation{
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key"), Value: []byte("1")}}, 1, 42).EtcdResponse}, {req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key"), Value: []byte("1")}}, 1, 42)},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key"), Value: []byte("1")}}, 1, 42).EtcdResponse}, {req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key"), Value: []byte("1")}}, 1, 42)},
}, },
}, },
{ {
name: "First Range can start from non-zero revision", name: "First Range can start from non-zero revision",
operations: []deterministicOperation{ operations: []testOperation{
{req: rangeRequest("key", true, 0), resp: rangeResponse(nil, 0, 1).EtcdResponse}, {req: rangeRequest("key", true, 0), resp: rangeResponse(nil, 0, 1)},
{req: rangeRequest("key", true, 0), resp: rangeResponse(nil, 0, 1).EtcdResponse}, {req: rangeRequest("key", true, 0), resp: rangeResponse(nil, 0, 1)},
}, },
}, },
{ {
name: "First Put can start from non-zero revision", name: "First Put can start from non-zero revision",
operations: []deterministicOperation{ operations: []testOperation{
{req: putRequest("key", "1"), resp: putResponse(42).EtcdResponse}, {req: putRequest("key", "1"), resp: putResponse(42)},
}, },
}, },
{ {
name: "First delete can start from non-zero revision", name: "First delete can start from non-zero revision",
operations: []deterministicOperation{ operations: []testOperation{
{req: deleteRequest("key"), resp: deleteResponse(0, 42).EtcdResponse}, {req: deleteRequest("key"), resp: deleteResponse(0, 42)},
}, },
}, },
{ {
name: "First Txn can start from non-zero revision", name: "First Txn can start from non-zero revision",
operations: []deterministicOperation{ operations: []testOperation{
{req: compareRevisionAndPutRequest("key", 0, "42"), resp: compareRevisionAndPutResponse(false, 42).EtcdResponse}, {req: compareRevisionAndPutRequest("key", 0, "42"), resp: compareRevisionAndPutResponse(false, 42)},
}, },
}, },
{ {
name: "Get response data should match put", name: "Get response data should match put",
operations: []deterministicOperation{ operations: []testOperation{
{req: putRequest("key1", "11"), resp: putResponse(1).EtcdResponse}, {req: putRequest("key1", "11"), resp: putResponse(1)},
{req: putRequest("key2", "12"), resp: putResponse(2).EtcdResponse}, {req: putRequest("key2", "12"), resp: putResponse(2)},
{req: getRequest("key1"), resp: getResponse("key1", "11", 1, 1).EtcdResponse, expectFailure: true}, {req: getRequest("key1"), resp: getResponse("key1", "11", 1, 1), expectFailure: true},
{req: getRequest("key1"), resp: getResponse("key1", "12", 1, 1).EtcdResponse, expectFailure: true}, {req: getRequest("key1"), resp: getResponse("key1", "12", 1, 1), expectFailure: true},
{req: getRequest("key1"), resp: getResponse("key1", "12", 2, 2).EtcdResponse, expectFailure: true}, {req: getRequest("key1"), resp: getResponse("key1", "12", 2, 2), expectFailure: true},
{req: getRequest("key1"), resp: getResponse("key1", "11", 1, 2).EtcdResponse}, {req: getRequest("key1"), resp: getResponse("key1", "11", 1, 2)},
{req: getRequest("key2"), resp: getResponse("key2", "11", 2, 2).EtcdResponse, expectFailure: true}, {req: getRequest("key2"), resp: getResponse("key2", "11", 2, 2), expectFailure: true},
{req: getRequest("key2"), resp: getResponse("key2", "12", 1, 1).EtcdResponse, expectFailure: true}, {req: getRequest("key2"), resp: getResponse("key2", "12", 1, 1), expectFailure: true},
{req: getRequest("key2"), resp: getResponse("key2", "11", 1, 1).EtcdResponse, expectFailure: true}, {req: getRequest("key2"), resp: getResponse("key2", "11", 1, 1), expectFailure: true},
{req: getRequest("key2"), resp: getResponse("key2", "12", 2, 2).EtcdResponse}, {req: getRequest("key2"), resp: getResponse("key2", "12", 2, 2)},
}, },
}, },
{ {
name: "Range response data should match put", name: "Range response data should match put",
operations: []deterministicOperation{ operations: []testOperation{
{req: putRequest("key1", "1"), resp: putResponse(1).EtcdResponse}, {req: putRequest("key1", "1"), resp: putResponse(1)},
{req: putRequest("key2", "2"), resp: putResponse(2).EtcdResponse}, {req: putRequest("key2", "2"), resp: putResponse(2)},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 2}}, 2, 2).EtcdResponse}, {req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 2}}, 2, 2)},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 2}}, 2, 2).EtcdResponse}, {req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 2}}, 2, 2)},
}, },
}, },
{ {
name: "Range limit should reduce number of kvs, but maintain count", name: "Range limit should reduce number of kvs, but maintain count",
operations: []deterministicOperation{ operations: []testOperation{
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{ {req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1}, {Key: []byte("key1"), Value: []byte("1"), ModRevision: 1},
{Key: []byte("key2"), Value: []byte("2"), ModRevision: 2}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 2},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 3}, {Key: []byte("key3"), Value: []byte("3"), ModRevision: 3},
}, 3, 3).EtcdResponse}, }, 3, 3)},
{req: rangeRequest("key", true, 4), resp: rangeResponse([]*mvccpb.KeyValue{ {req: rangeRequest("key", true, 4), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1}, {Key: []byte("key1"), Value: []byte("1"), ModRevision: 1},
{Key: []byte("key2"), Value: []byte("2"), ModRevision: 2}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 2},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 3}, {Key: []byte("key3"), Value: []byte("3"), ModRevision: 3},
}, 3, 3).EtcdResponse}, }, 3, 3)},
{req: rangeRequest("key", true, 3), resp: rangeResponse([]*mvccpb.KeyValue{ {req: rangeRequest("key", true, 3), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1}, {Key: []byte("key1"), Value: []byte("1"), ModRevision: 1},
{Key: []byte("key2"), Value: []byte("2"), ModRevision: 2}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 2},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 3}, {Key: []byte("key3"), Value: []byte("3"), ModRevision: 3},
}, 3, 3).EtcdResponse}, }, 3, 3)},
{req: rangeRequest("key", true, 2), resp: rangeResponse([]*mvccpb.KeyValue{ {req: rangeRequest("key", true, 2), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1}, {Key: []byte("key1"), Value: []byte("1"), ModRevision: 1},
{Key: []byte("key2"), Value: []byte("2"), ModRevision: 2}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 2},
}, 3, 3).EtcdResponse}, }, 3, 3)},
{req: rangeRequest("key", true, 1), resp: rangeResponse([]*mvccpb.KeyValue{ {req: rangeRequest("key", true, 1), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1}, {Key: []byte("key1"), Value: []byte("1"), ModRevision: 1},
}, 3, 3).EtcdResponse}, }, 3, 3)},
}, },
}, },
{ {
name: "Range response should be ordered by key", name: "Range response should be ordered by key",
operations: []deterministicOperation{ operations: []testOperation{
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{ {req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key1"), Value: []byte("2"), ModRevision: 3}, {Key: []byte("key1"), Value: []byte("2"), ModRevision: 3},
{Key: []byte("key2"), Value: []byte("1"), ModRevision: 2}, {Key: []byte("key2"), Value: []byte("1"), ModRevision: 2},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 1}, {Key: []byte("key3"), Value: []byte("3"), ModRevision: 1},
}, 3, 3).EtcdResponse}, }, 3, 3)},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{ {req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key2"), Value: []byte("1"), ModRevision: 2}, {Key: []byte("key2"), Value: []byte("1"), ModRevision: 2},
{Key: []byte("key1"), Value: []byte("2"), ModRevision: 3}, {Key: []byte("key1"), Value: []byte("2"), ModRevision: 3},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 1}, {Key: []byte("key3"), Value: []byte("3"), ModRevision: 1},
}, 3, 3).EtcdResponse, expectFailure: true}, }, 3, 3), expectFailure: true},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{ {req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 1}, {Key: []byte("key3"), Value: []byte("3"), ModRevision: 1},
{Key: []byte("key2"), Value: []byte("1"), ModRevision: 2}, {Key: []byte("key2"), Value: []byte("1"), ModRevision: 2},
{Key: []byte("key1"), Value: []byte("2"), ModRevision: 3}, {Key: []byte("key1"), Value: []byte("2"), ModRevision: 3},
}, 3, 3).EtcdResponse, expectFailure: true}, }, 3, 3), expectFailure: true},
}, },
}, },
{ {
name: "Range response data should match large put", name: "Range response data should match large put",
operations: []deterministicOperation{ operations: []testOperation{
{req: putRequest("key", "012345678901234567890"), resp: putResponse(1).EtcdResponse}, {req: putRequest("key", "012345678901234567890"), resp: putResponse(1)},
{req: getRequest("key"), resp: getResponse("key", "123456789012345678901", 1, 1).EtcdResponse, expectFailure: true}, {req: getRequest("key"), resp: getResponse("key", "123456789012345678901", 1, 1), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "012345678901234567890", 1, 1).EtcdResponse}, {req: getRequest("key"), resp: getResponse("key", "012345678901234567890", 1, 1)},
{req: putRequest("key", "123456789012345678901"), resp: putResponse(2).EtcdResponse}, {req: putRequest("key", "123456789012345678901"), resp: putResponse(2)},
{req: getRequest("key"), resp: getResponse("key", "123456789012345678901", 2, 2).EtcdResponse}, {req: getRequest("key"), resp: getResponse("key", "123456789012345678901", 2, 2)},
{req: getRequest("key"), resp: getResponse("key", "012345678901234567890", 2, 2).EtcdResponse, expectFailure: true}, {req: getRequest("key"), resp: getResponse("key", "012345678901234567890", 2, 2), expectFailure: true},
}, },
}, },
{ {
name: "Put must increase revision by 1", name: "Put must increase revision by 1",
operations: []deterministicOperation{ operations: []testOperation{
{req: getRequest("key"), resp: emptyGetResponse(1).EtcdResponse}, {req: getRequest("key"), resp: emptyGetResponse(1)},
{req: putRequest("key", "1"), resp: putResponse(1).EtcdResponse, expectFailure: true}, {req: putRequest("key", "1"), resp: putResponse(1), expectFailure: true},
{req: putRequest("key", "1"), resp: putResponse(3).EtcdResponse, expectFailure: true}, {req: putRequest("key", "1"), resp: putResponse(3), expectFailure: true},
{req: putRequest("key", "1"), resp: putResponse(2).EtcdResponse}, {req: putRequest("key", "1"), resp: putResponse(2)},
}, },
}, },
{ {
name: "Delete only increases revision on success", name: "Delete only increases revision on success",
operations: []deterministicOperation{ operations: []testOperation{
{req: putRequest("key1", "11"), resp: putResponse(1).EtcdResponse}, {req: putRequest("key1", "11"), resp: putResponse(1)},
{req: putRequest("key2", "12"), resp: putResponse(2).EtcdResponse}, {req: putRequest("key2", "12"), resp: putResponse(2)},
{req: deleteRequest("key1"), resp: deleteResponse(1, 2).EtcdResponse, expectFailure: true}, {req: deleteRequest("key1"), resp: deleteResponse(1, 2), expectFailure: true},
{req: deleteRequest("key1"), resp: deleteResponse(1, 3).EtcdResponse}, {req: deleteRequest("key1"), resp: deleteResponse(1, 3)},
{req: deleteRequest("key1"), resp: deleteResponse(0, 4).EtcdResponse, expectFailure: true}, {req: deleteRequest("key1"), resp: deleteResponse(0, 4), expectFailure: true},
{req: deleteRequest("key1"), resp: deleteResponse(0, 3).EtcdResponse}, {req: deleteRequest("key1"), resp: deleteResponse(0, 3)},
}, },
}, },
{ {
name: "Delete not existing key", name: "Delete not existing key",
operations: []deterministicOperation{ operations: []testOperation{
{req: getRequest("key"), resp: emptyGetResponse(1).EtcdResponse}, {req: getRequest("key"), resp: emptyGetResponse(1)},
{req: deleteRequest("key"), resp: deleteResponse(1, 2).EtcdResponse, expectFailure: true}, {req: deleteRequest("key"), resp: deleteResponse(1, 2), expectFailure: true},
{req: deleteRequest("key"), resp: deleteResponse(0, 1).EtcdResponse}, {req: deleteRequest("key"), resp: deleteResponse(0, 1)},
}, },
}, },
{ {
name: "Delete clears value", name: "Delete clears value",
operations: []deterministicOperation{ operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1).EtcdResponse}, {req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: deleteRequest("key"), resp: deleteResponse(1, 2).EtcdResponse}, {req: deleteRequest("key"), resp: deleteResponse(1, 2)},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1).EtcdResponse, expectFailure: true}, {req: getRequest("key"), resp: getResponse("key", "1", 1, 1), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 2, 2).EtcdResponse, expectFailure: true}, {req: getRequest("key"), resp: getResponse("key", "1", 2, 2), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 2).EtcdResponse, expectFailure: true}, {req: getRequest("key"), resp: getResponse("key", "1", 1, 2), expectFailure: true},
{req: getRequest("key"), resp: emptyGetResponse(2).EtcdResponse}, {req: getRequest("key"), resp: emptyGetResponse(2)},
}, },
}, },
{ {
name: "Txn executes onSuccess if revision matches expected", name: "Txn executes onSuccess if revision matches expected",
operations: []deterministicOperation{ operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1).EtcdResponse}, {req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(true, 1).EtcdResponse, expectFailure: true}, {req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(true, 1), expectFailure: true},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(false, 2).EtcdResponse, expectFailure: true}, {req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(false, 2), expectFailure: true},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(false, 1).EtcdResponse, expectFailure: true}, {req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(false, 1), expectFailure: true},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(true, 2).EtcdResponse}, {req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(true, 2)},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1).EtcdResponse, expectFailure: true}, {req: getRequest("key"), resp: getResponse("key", "1", 1, 1), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 2).EtcdResponse, expectFailure: true}, {req: getRequest("key"), resp: getResponse("key", "1", 1, 2), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 2, 2).EtcdResponse, expectFailure: true}, {req: getRequest("key"), resp: getResponse("key", "1", 2, 2), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 1, 1).EtcdResponse, expectFailure: true}, {req: getRequest("key"), resp: getResponse("key", "2", 1, 1), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2).EtcdResponse}, {req: getRequest("key"), resp: getResponse("key", "2", 2, 2)},
}, },
}, },
{ {
name: "Txn can expect on key not existing", name: "Txn can expect on key not existing",
operations: []deterministicOperation{ operations: []testOperation{
{req: getRequest("key1"), resp: emptyGetResponse(1).EtcdResponse}, {req: getRequest("key1"), resp: emptyGetResponse(1)},
{req: compareRevisionAndPutRequest("key1", 0, "2"), resp: compareRevisionAndPutResponse(true, 2).EtcdResponse}, {req: compareRevisionAndPutRequest("key1", 0, "2"), resp: compareRevisionAndPutResponse(true, 2)},
{req: compareRevisionAndPutRequest("key1", 0, "3"), resp: compareRevisionAndPutResponse(true, 3).EtcdResponse, expectFailure: true}, {req: compareRevisionAndPutRequest("key1", 0, "3"), resp: compareRevisionAndPutResponse(true, 3), expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key1", 0), putOperation("key1", "4"), putOperation("key1", "5")), resp: txnPutResponse(false, 3).EtcdResponse}, {req: txnRequestSingleOperation(compareRevision("key1", 0), putOperation("key1", "4"), putOperation("key1", "5")), resp: txnPutResponse(false, 3)},
{req: getRequest("key1"), resp: getResponse("key1", "5", 3, 3).EtcdResponse}, {req: getRequest("key1"), resp: getResponse("key1", "5", 3, 3)},
{req: compareRevisionAndPutRequest("key2", 0, "6"), resp: compareRevisionAndPutResponse(true, 4).EtcdResponse}, {req: compareRevisionAndPutRequest("key2", 0, "6"), resp: compareRevisionAndPutResponse(true, 4)},
}, },
}, },
{ {
name: "Txn executes onFailure if revision doesn't match expected", name: "Txn executes onFailure if revision doesn't match expected",
operations: []deterministicOperation{ operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1).EtcdResponse}, {req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnPutResponse(false, 2).EtcdResponse, expectFailure: true}, {req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnPutResponse(false, 2), expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnEmptyResponse(false, 2).EtcdResponse, expectFailure: true}, {req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnEmptyResponse(false, 2), expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnEmptyResponse(true, 2).EtcdResponse, expectFailure: true}, {req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnEmptyResponse(true, 2), expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnPutResponse(true, 1).EtcdResponse, expectFailure: true}, {req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnPutResponse(true, 1), expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnEmptyResponse(true, 1).EtcdResponse}, {req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnEmptyResponse(true, 1)},
{req: txnRequestSingleOperation(compareRevision("key", 2), nil, putOperation("key", "2")), resp: txnPutResponse(false, 2).EtcdResponse}, {req: txnRequestSingleOperation(compareRevision("key", 2), nil, putOperation("key", "2")), resp: txnPutResponse(false, 2)},
}, },
}, },
{ {
name: "Put with valid lease id should succeed. Put with invalid lease id should fail", name: "Put with valid lease id should succeed. Put with invalid lease id should fail",
operations: []deterministicOperation{ operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse}, {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse}, {req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)},
{req: putWithLeaseRequest("key", "3", 2), resp: putResponse(3).EtcdResponse, expectFailure: true}, {req: putWithLeaseRequest("key", "3", 2), resp: putResponse(3), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2).EtcdResponse}, {req: getRequest("key"), resp: getResponse("key", "2", 2, 2)},
}, },
}, },
{ {
name: "Put with valid lease id should succeed. Put with expired lease id should fail", name: "Put with valid lease id should succeed. Put with expired lease id should fail",
operations: []deterministicOperation{ operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse}, {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse}, {req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2).EtcdResponse}, {req: getRequest("key"), resp: getResponse("key", "2", 2, 2)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)},
{req: putWithLeaseRequest("key", "4", 1), resp: putResponse(4).EtcdResponse, expectFailure: true}, {req: putWithLeaseRequest("key", "4", 1), resp: putResponse(4), expectFailure: true},
{req: getRequest("key"), resp: emptyGetResponse(3).EtcdResponse}, {req: getRequest("key"), resp: emptyGetResponse(3)},
}, },
}, },
{ {
name: "Revoke should increment the revision", name: "Revoke should increment the revision",
operations: []deterministicOperation{ operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse}, {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse}, {req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)},
{req: getRequest("key"), resp: emptyGetResponse(3).EtcdResponse}, {req: getRequest("key"), resp: emptyGetResponse(3)},
}, },
}, },
{ {
name: "Put following a PutWithLease will detach the key from the lease", name: "Put following a PutWithLease will detach the key from the lease",
operations: []deterministicOperation{ operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse}, {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse}, {req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)},
{req: putRequest("key", "3"), resp: putResponse(3).EtcdResponse}, {req: putRequest("key", "3"), resp: putResponse(3)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)},
{req: getRequest("key"), resp: getResponse("key", "3", 3, 3).EtcdResponse}, {req: getRequest("key"), resp: getResponse("key", "3", 3, 3)},
}, },
}, },
{ {
name: "Change lease. Revoking older lease should not increment revision", name: "Change lease. Revoking older lease should not increment revision",
operations: []deterministicOperation{ operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse}, {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: leaseGrantRequest(2), resp: leaseGrantResponse(1).EtcdResponse}, {req: leaseGrantRequest(2), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse}, {req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)},
{req: putWithLeaseRequest("key", "3", 2), resp: putResponse(3).EtcdResponse}, {req: putWithLeaseRequest("key", "3", 2), resp: putResponse(3)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)},
{req: getRequest("key"), resp: getResponse("key", "3", 3, 3).EtcdResponse}, {req: getRequest("key"), resp: getResponse("key", "3", 3, 3)},
{req: leaseRevokeRequest(2), resp: leaseRevokeResponse(4).EtcdResponse}, {req: leaseRevokeRequest(2), resp: leaseRevokeResponse(4)},
{req: getRequest("key"), resp: emptyGetResponse(4).EtcdResponse}, {req: getRequest("key"), resp: emptyGetResponse(4)},
}, },
}, },
{ {
name: "Update key with same lease", name: "Update key with same lease",
operations: []deterministicOperation{ operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse}, {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse}, {req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)},
{req: putWithLeaseRequest("key", "3", 1), resp: putResponse(3).EtcdResponse}, {req: putWithLeaseRequest("key", "3", 1), resp: putResponse(3)},
{req: getRequest("key"), resp: getResponse("key", "3", 3, 3).EtcdResponse}, {req: getRequest("key"), resp: getResponse("key", "3", 3, 3)},
}, },
}, },
{ {
name: "Deleting a leased key - revoke should not increment revision", name: "Deleting a leased key - revoke should not increment revision",
operations: []deterministicOperation{ operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse}, {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse}, {req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)},
{req: deleteRequest("key"), resp: deleteResponse(1, 3).EtcdResponse}, {req: deleteRequest("key"), resp: deleteResponse(1, 3)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(4).EtcdResponse, expectFailure: true}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(4), expectFailure: true},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)},
}, },
}, },
{ {
name: "Lease a few keys - revoke should increment revision only once", name: "Lease a few keys - revoke should increment revision only once",
operations: []deterministicOperation{ operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse}, {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key1", "1", 1), resp: putResponse(2).EtcdResponse}, {req: putWithLeaseRequest("key1", "1", 1), resp: putResponse(2)},
{req: putWithLeaseRequest("key2", "2", 1), resp: putResponse(3).EtcdResponse}, {req: putWithLeaseRequest("key2", "2", 1), resp: putResponse(3)},
{req: putWithLeaseRequest("key3", "3", 1), resp: putResponse(4).EtcdResponse}, {req: putWithLeaseRequest("key3", "3", 1), resp: putResponse(4)},
{req: putWithLeaseRequest("key4", "4", 1), resp: putResponse(5).EtcdResponse}, {req: putWithLeaseRequest("key4", "4", 1), resp: putResponse(5)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(6).EtcdResponse}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(6)},
}, },
}, },
{ {
name: "Lease some keys then delete some of them. Revoke should increment revision since some keys were still leased", name: "Lease some keys then delete some of them. Revoke should increment revision since some keys were still leased",
operations: []deterministicOperation{ operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse}, {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key1", "1", 1), resp: putResponse(2).EtcdResponse}, {req: putWithLeaseRequest("key1", "1", 1), resp: putResponse(2)},
{req: putWithLeaseRequest("key2", "2", 1), resp: putResponse(3).EtcdResponse}, {req: putWithLeaseRequest("key2", "2", 1), resp: putResponse(3)},
{req: putWithLeaseRequest("key3", "3", 1), resp: putResponse(4).EtcdResponse}, {req: putWithLeaseRequest("key3", "3", 1), resp: putResponse(4)},
{req: putWithLeaseRequest("key4", "4", 1), resp: putResponse(5).EtcdResponse}, {req: putWithLeaseRequest("key4", "4", 1), resp: putResponse(5)},
{req: deleteRequest("key1"), resp: deleteResponse(1, 6).EtcdResponse}, {req: deleteRequest("key1"), resp: deleteResponse(1, 6)},
{req: deleteRequest("key3"), resp: deleteResponse(1, 7).EtcdResponse}, {req: deleteRequest("key3"), resp: deleteResponse(1, 7)},
{req: deleteRequest("key4"), resp: deleteResponse(1, 8).EtcdResponse}, {req: deleteRequest("key4"), resp: deleteResponse(1, 8)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(9).EtcdResponse}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(9)},
{req: deleteRequest("key2"), resp: deleteResponse(0, 9).EtcdResponse}, {req: deleteRequest("key2"), resp: deleteResponse(0, 9)},
{req: getRequest("key1"), resp: emptyGetResponse(9).EtcdResponse}, {req: getRequest("key1"), resp: emptyGetResponse(9)},
{req: getRequest("key2"), resp: emptyGetResponse(9).EtcdResponse}, {req: getRequest("key2"), resp: emptyGetResponse(9)},
{req: getRequest("key3"), resp: emptyGetResponse(9).EtcdResponse}, {req: getRequest("key3"), resp: emptyGetResponse(9)},
{req: getRequest("key4"), resp: emptyGetResponse(9).EtcdResponse}, {req: getRequest("key4"), resp: emptyGetResponse(9)},
}, },
}, },
{ {
name: "Lease some keys then delete all of them. Revoke should not increment", name: "Lease some keys then delete all of them. Revoke should not increment",
operations: []deterministicOperation{ operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse}, {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key1", "1", 1), resp: putResponse(2).EtcdResponse}, {req: putWithLeaseRequest("key1", "1", 1), resp: putResponse(2)},
{req: putWithLeaseRequest("key2", "2", 1), resp: putResponse(3).EtcdResponse}, {req: putWithLeaseRequest("key2", "2", 1), resp: putResponse(3)},
{req: putWithLeaseRequest("key3", "3", 1), resp: putResponse(4).EtcdResponse}, {req: putWithLeaseRequest("key3", "3", 1), resp: putResponse(4)},
{req: putWithLeaseRequest("key4", "4", 1), resp: putResponse(5).EtcdResponse}, {req: putWithLeaseRequest("key4", "4", 1), resp: putResponse(5)},
{req: deleteRequest("key1"), resp: deleteResponse(1, 6).EtcdResponse}, {req: deleteRequest("key1"), resp: deleteResponse(1, 6)},
{req: deleteRequest("key2"), resp: deleteResponse(1, 7).EtcdResponse}, {req: deleteRequest("key2"), resp: deleteResponse(1, 7)},
{req: deleteRequest("key3"), resp: deleteResponse(1, 8).EtcdResponse}, {req: deleteRequest("key3"), resp: deleteResponse(1, 8)},
{req: deleteRequest("key4"), resp: deleteResponse(1, 9).EtcdResponse}, {req: deleteRequest("key4"), resp: deleteResponse(1, 9)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(9).EtcdResponse}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(9)},
}, },
}, },
{ {
name: "All request types", name: "All request types",
operations: []deterministicOperation{ operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse}, {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key", "1", 1), resp: putResponse(2).EtcdResponse}, {req: putWithLeaseRequest("key", "1", 1), resp: putResponse(2)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)},
{req: putRequest("key", "4"), resp: putResponse(4).EtcdResponse}, {req: putRequest("key", "4"), resp: putResponse(4)},
{req: getRequest("key"), resp: getResponse("key", "4", 4, 4).EtcdResponse}, {req: getRequest("key"), resp: getResponse("key", "4", 4, 4)},
{req: compareRevisionAndPutRequest("key", 4, "5"), resp: compareRevisionAndPutResponse(true, 5).EtcdResponse}, {req: compareRevisionAndPutRequest("key", 4, "5"), resp: compareRevisionAndPutResponse(true, 5)},
{req: deleteRequest("key"), resp: deleteResponse(1, 6).EtcdResponse}, {req: deleteRequest("key"), resp: deleteResponse(1, 6)},
{req: defragmentRequest(), resp: defragmentResponse(6).EtcdResponse}, {req: defragmentRequest(), resp: defragmentResponse(6)},
}, },
}, },
{ {
name: "Defragment success between all other request types", name: "Defragment success between all other request types",
operations: []deterministicOperation{ operations: []testOperation{
{req: defragmentRequest(), resp: defragmentResponse(1).EtcdResponse}, {req: defragmentRequest(), resp: defragmentResponse(1)},
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse}, {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: defragmentRequest(), resp: defragmentResponse(1).EtcdResponse}, {req: defragmentRequest(), resp: defragmentResponse(1)},
{req: putWithLeaseRequest("key", "1", 1), resp: putResponse(2).EtcdResponse}, {req: putWithLeaseRequest("key", "1", 1), resp: putResponse(2)},
{req: defragmentRequest(), resp: defragmentResponse(2).EtcdResponse}, {req: defragmentRequest(), resp: defragmentResponse(2)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)},
{req: defragmentRequest(), resp: defragmentResponse(3).EtcdResponse}, {req: defragmentRequest(), resp: defragmentResponse(3)},
{req: putRequest("key", "4"), resp: putResponse(4).EtcdResponse}, {req: putRequest("key", "4"), resp: putResponse(4)},
{req: defragmentRequest(), resp: defragmentResponse(4).EtcdResponse}, {req: defragmentRequest(), resp: defragmentResponse(4)},
{req: getRequest("key"), resp: getResponse("key", "4", 4, 4).EtcdResponse}, {req: getRequest("key"), resp: getResponse("key", "4", 4, 4)},
{req: defragmentRequest(), resp: defragmentResponse(4).EtcdResponse}, {req: defragmentRequest(), resp: defragmentResponse(4)},
{req: compareRevisionAndPutRequest("key", 4, "5"), resp: compareRevisionAndPutResponse(true, 5).EtcdResponse}, {req: compareRevisionAndPutRequest("key", 4, "5"), resp: compareRevisionAndPutResponse(true, 5)},
{req: defragmentRequest(), resp: defragmentResponse(5).EtcdResponse}, {req: defragmentRequest(), resp: defragmentResponse(5)},
{req: deleteRequest("key"), resp: deleteResponse(1, 6).EtcdResponse}, {req: deleteRequest("key"), resp: deleteResponse(1, 6)},
{req: defragmentRequest(), resp: defragmentResponse(6).EtcdResponse}, {req: defragmentRequest(), resp: defragmentResponse(6)},
}, },
}, },
} }

48
tests/robustness/model/history.go
View File

@ -344,15 +344,15 @@ func rangeRequest(key string, withPrefix bool, limit int64) EtcdRequest {
return EtcdRequest{Type: Range, Range: &RangeRequest{Key: key, RangeOptions: RangeOptions{WithPrefix: withPrefix, Limit: limit}}} return EtcdRequest{Type: Range, Range: &RangeRequest{Key: key, RangeOptions: RangeOptions{WithPrefix: withPrefix, Limit: limit}}}
} }
func emptyGetResponse(revision int64) EtcdNonDeterministicResponse { func emptyGetResponse(revision int64) MaybeEtcdResponse {
return rangeResponse([]*mvccpb.KeyValue{}, 0, revision) return rangeResponse([]*mvccpb.KeyValue{}, 0, revision)
} }
func getResponse(key, value string, modRevision, revision int64) EtcdNonDeterministicResponse { func getResponse(key, value string, modRevision, revision int64) MaybeEtcdResponse {
return rangeResponse([]*mvccpb.KeyValue{{Key: []byte(key), Value: []byte(value), ModRevision: modRevision}}, 1, revision) return rangeResponse([]*mvccpb.KeyValue{{Key: []byte(key), Value: []byte(value), ModRevision: modRevision}}, 1, revision)
} }
func rangeResponse(kvs []*mvccpb.KeyValue, count int64, revision int64) EtcdNonDeterministicResponse { func rangeResponse(kvs []*mvccpb.KeyValue, count int64, revision int64) MaybeEtcdResponse {
result := RangeResponse{KVs: make([]KeyValue, len(kvs)), Count: count} result := RangeResponse{KVs: make([]KeyValue, len(kvs)), Count: count}
for i, kv := range kvs { for i, kv := range kvs {
@ -364,38 +364,38 @@ func rangeResponse(kvs []*mvccpb.KeyValue, count int64, revision int64) EtcdNonD
}, },
} }
} }
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{Range: &result, Revision: revision}} return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Range: &result, Revision: revision}}
} }
func failedResponse(err error) EtcdNonDeterministicResponse { func failedResponse(err error) MaybeEtcdResponse {
return EtcdNonDeterministicResponse{Err: err} return MaybeEtcdResponse{Err: err}
} }
func unknownResponse(revision int64) EtcdNonDeterministicResponse { func partialResponse(revision int64) MaybeEtcdResponse {
return EtcdNonDeterministicResponse{ResultUnknown: true, EtcdResponse: EtcdResponse{Revision: revision}} return MaybeEtcdResponse{PartialResponse: true, EtcdResponse: EtcdResponse{Revision: revision}}
} }
func putRequest(key, value string) EtcdRequest { func putRequest(key, value string) EtcdRequest {
return EtcdRequest{Type: Txn, Txn: &TxnRequest{OperationsOnSuccess: []EtcdOperation{{Type: PutOperation, Key: key, PutOptions: PutOptions{Value: ToValueOrHash(value)}}}}} return EtcdRequest{Type: Txn, Txn: &TxnRequest{OperationsOnSuccess: []EtcdOperation{{Type: PutOperation, Key: key, PutOptions: PutOptions{Value: ToValueOrHash(value)}}}}}
} }
func putResponse(revision int64) EtcdNonDeterministicResponse { func putResponse(revision int64) MaybeEtcdResponse {
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: []EtcdOperationResult{{}}}, Revision: revision}} return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: []EtcdOperationResult{{}}}, Revision: revision}}
} }
func deleteRequest(key string) EtcdRequest { func deleteRequest(key string) EtcdRequest {
return EtcdRequest{Type: Txn, Txn: &TxnRequest{OperationsOnSuccess: []EtcdOperation{{Type: DeleteOperation, Key: key}}}} return EtcdRequest{Type: Txn, Txn: &TxnRequest{OperationsOnSuccess: []EtcdOperation{{Type: DeleteOperation, Key: key}}}}
} }
func deleteResponse(deleted int64, revision int64) EtcdNonDeterministicResponse { func deleteResponse(deleted int64, revision int64) MaybeEtcdResponse {
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: []EtcdOperationResult{{Deleted: deleted}}}, Revision: revision}} return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: []EtcdOperationResult{{Deleted: deleted}}}, Revision: revision}}
} }
func compareRevisionAndPutRequest(key string, expectedRevision int64, value string) EtcdRequest { func compareRevisionAndPutRequest(key string, expectedRevision int64, value string) EtcdRequest {
return txnRequestSingleOperation(compareRevision(key, expectedRevision), putOperation(key, value), nil) return txnRequestSingleOperation(compareRevision(key, expectedRevision), putOperation(key, value), nil)
} }
func compareRevisionAndPutResponse(succeeded bool, revision int64) EtcdNonDeterministicResponse { func compareRevisionAndPutResponse(succeeded bool, revision int64) MaybeEtcdResponse {
if succeeded { if succeeded {
return txnPutResponse(succeeded, revision) return txnPutResponse(succeeded, revision)
} }
@ -430,16 +430,16 @@ func txnRequest(conds []EtcdCondition, onSuccess, onFailure []EtcdOperation) Etc
return EtcdRequest{Type: Txn, Txn: &TxnRequest{Conditions: conds, OperationsOnSuccess: onSuccess, OperationsOnFailure: onFailure}} return EtcdRequest{Type: Txn, Txn: &TxnRequest{Conditions: conds, OperationsOnSuccess: onSuccess, OperationsOnFailure: onFailure}}
} }
func txnPutResponse(succeeded bool, revision int64) EtcdNonDeterministicResponse { func txnPutResponse(succeeded bool, revision int64) MaybeEtcdResponse {
return txnResponse([]EtcdOperationResult{{}}, succeeded, revision) return txnResponse([]EtcdOperationResult{{}}, succeeded, revision)
} }
func txnEmptyResponse(succeeded bool, revision int64) EtcdNonDeterministicResponse { func txnEmptyResponse(succeeded bool, revision int64) MaybeEtcdResponse {
return txnResponse([]EtcdOperationResult{}, succeeded, revision) return txnResponse([]EtcdOperationResult{}, succeeded, revision)
} }
func txnResponse(result []EtcdOperationResult, succeeded bool, revision int64) EtcdNonDeterministicResponse { func txnResponse(result []EtcdOperationResult, succeeded bool, revision int64) MaybeEtcdResponse {
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: result, Failure: !succeeded}, Revision: revision}} return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: result, Failure: !succeeded}, Revision: revision}}
} }
func putWithLeaseRequest(key, value string, leaseID int64) EtcdRequest { func putWithLeaseRequest(key, value string, leaseID int64) EtcdRequest {
@ -450,24 +450,24 @@ func leaseGrantRequest(leaseID int64) EtcdRequest {
return EtcdRequest{Type: LeaseGrant, LeaseGrant: &LeaseGrantRequest{LeaseID: leaseID}} return EtcdRequest{Type: LeaseGrant, LeaseGrant: &LeaseGrantRequest{LeaseID: leaseID}}
} }
func leaseGrantResponse(revision int64) EtcdNonDeterministicResponse { func leaseGrantResponse(revision int64) MaybeEtcdResponse {
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{LeaseGrant: &LeaseGrantReponse{}, Revision: revision}} return MaybeEtcdResponse{EtcdResponse: EtcdResponse{LeaseGrant: &LeaseGrantReponse{}, Revision: revision}}
} }
func leaseRevokeRequest(leaseID int64) EtcdRequest { func leaseRevokeRequest(leaseID int64) EtcdRequest {
return EtcdRequest{Type: LeaseRevoke, LeaseRevoke: &LeaseRevokeRequest{LeaseID: leaseID}} return EtcdRequest{Type: LeaseRevoke, LeaseRevoke: &LeaseRevokeRequest{LeaseID: leaseID}}
} }
func leaseRevokeResponse(revision int64) EtcdNonDeterministicResponse { func leaseRevokeResponse(revision int64) MaybeEtcdResponse {
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{LeaseRevoke: &LeaseRevokeResponse{}, Revision: revision}} return MaybeEtcdResponse{EtcdResponse: EtcdResponse{LeaseRevoke: &LeaseRevokeResponse{}, Revision: revision}}
} }
func defragmentRequest() EtcdRequest { func defragmentRequest() EtcdRequest {
return EtcdRequest{Type: Defragment, Defragment: &DefragmentRequest{}} return EtcdRequest{Type: Defragment, Defragment: &DefragmentRequest{}}
} }
func defragmentResponse(revision int64) EtcdNonDeterministicResponse { func defragmentResponse(revision int64) MaybeEtcdResponse {
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{Defragment: &DefragmentResponse{}, Revision: revision}} return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Defragment: &DefragmentResponse{}, Revision: revision}}
} }
type History struct { type History struct {
@ -519,7 +519,7 @@ func (h History) Operations() []porcupine.Operation {
func (h History) MaxRevision() int64 { func (h History) MaxRevision() int64 {
var maxRevision int64 var maxRevision int64
for _, op := range h.successful { for _, op := range h.successful {
revision := op.Output.(EtcdNonDeterministicResponse).Revision revision := op.Output.(MaybeEtcdResponse).Revision
if revision > maxRevision { if revision > maxRevision {
maxRevision = revision maxRevision = revision
} }

45
tests/robustness/model/non_deterministic.go
View File

@ -22,7 +22,7 @@ import (
"github.com/anishathalye/porcupine" "github.com/anishathalye/porcupine"
) )
// NonDeterministicModel extends DeterministicModel to handle requests that have unknown or error response. // NonDeterministicModel extends DeterministicModel to allow for clients with imperfect knowledge of request destiny.
// Unknown/error response doesn't inform whether request was persisted or not, so model // Unknown/error response doesn't inform whether request was persisted or not, so model
// considers both cases. This is represented as multiple equally possible deterministic states. // considers both cases. This is represented as multiple equally possible deterministic states.
// Failed requests fork the possible states, while successful requests merge and filter them. // Failed requests fork the possible states, while successful requests merge and filter them.
@ -41,7 +41,7 @@ var NonDeterministicModel = porcupine.Model{
if err != nil { if err != nil {
panic(err) panic(err)
} }
ok, states := states.Step(in.(EtcdRequest), out.(EtcdNonDeterministicResponse)) ok, states := states.Step(in.(EtcdRequest), out.(MaybeEtcdResponse))
data, err := json.Marshal(states) data, err := json.Marshal(states)
if err != nil { if err != nil {
panic(err) panic(err)
@ -49,30 +49,27 @@ var NonDeterministicModel = porcupine.Model{
return ok, string(data) return ok, string(data)
}, },
DescribeOperation: func(in, out interface{}) string { DescribeOperation: func(in, out interface{}) string {
return fmt.Sprintf("%s -> %s", describeEtcdRequest(in.(EtcdRequest)), describeEtcdNonDeterministicResponse(in.(EtcdRequest), out.(EtcdNonDeterministicResponse))) return fmt.Sprintf("%s -> %s", describeEtcdRequest(in.(EtcdRequest)), describeEtcdResponse(in.(EtcdRequest), out.(MaybeEtcdResponse)))
}, },
} }
type nonDeterministicState []etcdState type nonDeterministicState []etcdState
type EtcdNonDeterministicResponse struct { func (states nonDeterministicState) Step(request EtcdRequest, response MaybeEtcdResponse) (bool, nonDeterministicState) {
EtcdResponse
Err error
ResultUnknown bool
}
func (states nonDeterministicState) Step(request EtcdRequest, response EtcdNonDeterministicResponse) (bool, nonDeterministicState) {
if len(states) == 0 { if len(states) == 0 {
if response.Err == nil && !response.ResultUnknown { if response.Err == nil && !response.PartialResponse {
return true, nonDeterministicState{initState(request, response.EtcdResponse)} return true, nonDeterministicState{initState(request, response.EtcdResponse)}
} }
states = nonDeterministicState{emptyState()} states = nonDeterministicState{emptyState()}
} }
var newStates nonDeterministicState var newStates nonDeterministicState
if response.Err != nil { switch {
case response.Err != nil:
newStates = states.stepFailedRequest(request) newStates = states.stepFailedRequest(request)
} else { case response.PartialResponse:
newStates = states.stepSuccessfulRequest(request, response) newStates = states.stepPartialRequest(request, response.EtcdResponse.Revision)
default:
newStates = states.stepSuccessfulRequest(request, response.EtcdResponse)
} }
return len(newStates) > 0, newStates return len(newStates) > 0, newStates
} }
@ -90,18 +87,26 @@ func (states nonDeterministicState) stepFailedRequest(request EtcdRequest) nonDe
return newStates return newStates
} }
// stepSuccessfulRequest filters possible states by leaving ony states that would respond correctly. // stepPartialRequest filters possible states by leaving ony states that would return proper revision.
func (states nonDeterministicState) stepSuccessfulRequest(request EtcdRequest, response EtcdNonDeterministicResponse) nonDeterministicState { func (states nonDeterministicState) stepPartialRequest(request EtcdRequest, responseRevision int64) nonDeterministicState {
newStates := make(nonDeterministicState, 0, len(states)) newStates := make(nonDeterministicState, 0, len(states))
for _, s := range states { for _, s := range states {
newState, gotResponse := s.step(request) newState, modelResponse := s.step(request)
if Match(EtcdNonDeterministicResponse{EtcdResponse: gotResponse}, response) { if modelResponse.Revision == responseRevision {
newStates = append(newStates, newState) newStates = append(newStates, newState)
} }
} }
return newStates return newStates
} }
func Match(r1, r2 EtcdNonDeterministicResponse) bool { // stepSuccessfulRequest filters possible states by leaving ony states that would respond correctly.
return ((r1.ResultUnknown || r2.ResultUnknown) && (r1.Revision == r2.Revision)) || reflect.DeepEqual(r1, r2) func (states nonDeterministicState) stepSuccessfulRequest(request EtcdRequest, response EtcdResponse) nonDeterministicState {
newStates := make(nonDeterministicState, 0, len(states))
for _, s := range states {
newState, modelResponse := s.step(request)
if Match(modelResponse, MaybeEtcdResponse{EtcdResponse: response}) {
newStates = append(newStates, newState)
}
}
return newStates
} }

113
tests/robustness/model/non_deterministic_test.go
View File

@ -26,15 +26,10 @@ import (
) )
func TestModelNonDeterministic(t *testing.T) { func TestModelNonDeterministic(t *testing.T) {
nonDeterministicTestScenarios := []nonDeterministicModelTest{} nonDeterministicTestScenarios := append(commonTestScenarios, []modelTestCase{
for _, tc := range deterministicModelTestScenarios {
nonDeterministicTestScenarios = append(nonDeterministicTestScenarios, toNonDeterministicTest(tc))
}
nonDeterministicTestScenarios = append(nonDeterministicTestScenarios, []nonDeterministicModelTest{
{ {
name: "First Put request fails, but is persisted", name: "First Put request fails, but is persisted",
operations: []nonDeterministicOperation{ operations: []testOperation{
{req: putRequest("key1", "1"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key1", "1"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key2", "2"), resp: putResponse(3)}, {req: putRequest("key2", "2"), resp: putResponse(3)},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key1"), Value: []byte("1"), ModRevision: 2}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 3}}, 2, 3)}, {req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key1"), Value: []byte("1"), ModRevision: 2}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 3}}, 2, 3)},
@ -42,7 +37,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "First Put request fails, and is lost", name: "First Put request fails, and is lost",
operations: []nonDeterministicOperation{ operations: []testOperation{
{req: putRequest("key1", "1"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key1", "1"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key2", "2"), resp: putResponse(2)}, {req: putRequest("key2", "2"), resp: putResponse(2)},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key2"), Value: []byte("2"), ModRevision: 2}}, 1, 2)}, {req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key2"), Value: []byte("2"), ModRevision: 2}}, 1, 2)},
@ -50,7 +45,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Put can fail and be lost before get", name: "Put can fail and be lost before get",
operations: []nonDeterministicOperation{ operations: []testOperation{
{req: putRequest("key", "1"), resp: putResponse(1)}, {req: putRequest("key", "1"), resp: putResponse(1)},
{req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)}, {req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
@ -61,7 +56,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Put can fail and be lost before put", name: "Put can fail and be lost before put",
operations: []nonDeterministicOperation{ operations: []testOperation{
{req: getRequest("key"), resp: emptyGetResponse(1)}, {req: getRequest("key"), resp: emptyGetResponse(1)},
{req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key", "3"), resp: putResponse(2)}, {req: putRequest("key", "3"), resp: putResponse(2)},
@ -69,7 +64,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Put can fail and be lost before delete", name: "Put can fail and be lost before delete",
operations: []nonDeterministicOperation{ operations: []testOperation{
{req: deleteRequest("key"), resp: deleteResponse(0, 1)}, {req: deleteRequest("key"), resp: deleteResponse(0, 1)},
{req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))},
{req: deleteRequest("key"), resp: deleteResponse(0, 1)}, {req: deleteRequest("key"), resp: deleteResponse(0, 1)},
@ -77,7 +72,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Put can fail and be lost before txn", name: "Put can fail and be lost before txn",
operations: []nonDeterministicOperation{ operations: []testOperation{
// Txn failure // Txn failure
{req: getRequest("key"), resp: emptyGetResponse(1)}, {req: getRequest("key"), resp: emptyGetResponse(1)},
{req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))},
@ -90,11 +85,11 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Put can fail and be lost before txn success", name: "Put can fail and be lost before txn success",
operations: []nonDeterministicOperation{}, operations: []testOperation{},
}, },
{ {
name: "Put can fail but be persisted and increase revision before get", name: "Put can fail but be persisted and increase revision before get",
operations: []nonDeterministicOperation{ operations: []testOperation{
// One failed request, one persisted. // One failed request, one persisted.
{req: putRequest("key", "1"), resp: putResponse(1)}, {req: putRequest("key", "1"), resp: putResponse(1)},
{req: putRequest("key", "2"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key", "2"), resp: failedResponse(errors.New("failed"))},
@ -110,7 +105,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Put can fail but be persisted and increase revision before delete", name: "Put can fail but be persisted and increase revision before delete",
operations: []nonDeterministicOperation{ operations: []testOperation{
// One failed request, one persisted. // One failed request, one persisted.
{req: deleteRequest("key"), resp: deleteResponse(0, 1)}, {req: deleteRequest("key"), resp: deleteResponse(0, 1)},
{req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))},
@ -131,7 +126,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Put can fail but be persisted before txn", name: "Put can fail but be persisted before txn",
operations: []nonDeterministicOperation{ operations: []testOperation{
// Txn success // Txn success
{req: getRequest("key"), resp: emptyGetResponse(1)}, {req: getRequest("key"), resp: emptyGetResponse(1)},
{req: putRequest("key", "2"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key", "2"), resp: failedResponse(errors.New("failed"))},
@ -146,7 +141,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Delete can fail and be lost before get", name: "Delete can fail and be lost before get",
operations: []nonDeterministicOperation{ operations: []testOperation{
{req: putRequest("key", "1"), resp: putResponse(1)}, {req: putRequest("key", "1"), resp: putResponse(1)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)}, {req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
@ -157,7 +152,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Delete can fail and be lost before delete", name: "Delete can fail and be lost before delete",
operations: []nonDeterministicOperation{ operations: []testOperation{
{req: putRequest("key", "1"), resp: putResponse(1)}, {req: putRequest("key", "1"), resp: putResponse(1)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: deleteRequest("key"), resp: deleteResponse(1, 1), expectFailure: true}, {req: deleteRequest("key"), resp: deleteResponse(1, 1), expectFailure: true},
@ -166,7 +161,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Delete can fail and be lost before put", name: "Delete can fail and be lost before put",
operations: []nonDeterministicOperation{ operations: []testOperation{
{req: putRequest("key", "1"), resp: putResponse(1)}, {req: putRequest("key", "1"), resp: putResponse(1)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key", "1"), resp: putResponse(2)}, {req: putRequest("key", "1"), resp: putResponse(2)},
@ -174,7 +169,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Delete can fail but be persisted before get", name: "Delete can fail but be persisted before get",
operations: []nonDeterministicOperation{ operations: []testOperation{
// One failed request, one persisted. // One failed request, one persisted.
{req: putRequest("key", "1"), resp: putResponse(1)}, {req: putRequest("key", "1"), resp: putResponse(1)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
@ -188,7 +183,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Delete can fail but be persisted before put", name: "Delete can fail but be persisted before put",
operations: []nonDeterministicOperation{ operations: []testOperation{
// One failed request, one persisted. // One failed request, one persisted.
{req: putRequest("key", "1"), resp: putResponse(1)}, {req: putRequest("key", "1"), resp: putResponse(1)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
@ -201,7 +196,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Delete can fail but be persisted before delete", name: "Delete can fail but be persisted before delete",
operations: []nonDeterministicOperation{ operations: []testOperation{
// One failed request, one persisted. // One failed request, one persisted.
{req: putRequest("key", "1"), resp: putResponse(1)}, {req: putRequest("key", "1"), resp: putResponse(1)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
@ -215,7 +210,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Delete can fail but be persisted before txn", name: "Delete can fail but be persisted before txn",
operations: []nonDeterministicOperation{ operations: []testOperation{
// Txn success // Txn success
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)}, {req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
@ -228,7 +223,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Txn can fail and be lost before get", name: "Txn can fail and be lost before get",
operations: []nonDeterministicOperation{ operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)}, {req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))}, {req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)}, {req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
@ -237,7 +232,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Txn can fail and be lost before delete", name: "Txn can fail and be lost before delete",
operations: []nonDeterministicOperation{ operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)}, {req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))}, {req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
{req: deleteRequest("key"), resp: deleteResponse(1, 2)}, {req: deleteRequest("key"), resp: deleteResponse(1, 2)},
@ -245,7 +240,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Txn can fail and be lost before put", name: "Txn can fail and be lost before put",
operations: []nonDeterministicOperation{ operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)}, {req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))}, {req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key", "3"), resp: putResponse(2)}, {req: putRequest("key", "3"), resp: putResponse(2)},
@ -253,7 +248,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Txn can fail but be persisted before get", name: "Txn can fail but be persisted before get",
operations: []nonDeterministicOperation{ operations: []testOperation{
// One failed request, one persisted. // One failed request, one persisted.
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)}, {req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))}, {req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
@ -268,7 +263,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Txn can fail but be persisted before put", name: "Txn can fail but be persisted before put",
operations: []nonDeterministicOperation{ operations: []testOperation{
// One failed request, one persisted. // One failed request, one persisted.
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)}, {req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))}, {req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
@ -282,7 +277,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Txn can fail but be persisted before delete", name: "Txn can fail but be persisted before delete",
operations: []nonDeterministicOperation{ operations: []testOperation{
// One failed request, one persisted. // One failed request, one persisted.
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)}, {req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))}, {req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
@ -296,7 +291,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Txn can fail but be persisted before txn", name: "Txn can fail but be persisted before txn",
operations: []nonDeterministicOperation{ operations: []testOperation{
// One failed request, one persisted with success. // One failed request, one persisted with success.
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)}, {req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))}, {req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
@ -314,7 +309,7 @@ func TestModelNonDeterministic(t *testing.T) {
}, },
{ {
name: "Defragment failures between all other request types", name: "Defragment failures between all other request types",
operations: []nonDeterministicOperation{ operations: []testOperation{
{req: defragmentRequest(), resp: failedResponse(errors.New("failed"))}, {req: defragmentRequest(), resp: failedResponse(errors.New("failed"))},
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1)}, {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: defragmentRequest(), resp: failedResponse(errors.New("failed"))}, {req: defragmentRequest(), resp: failedResponse(errors.New("failed"))},
@ -349,7 +344,7 @@ func TestModelNonDeterministic(t *testing.T) {
} }
for i, s := range loadedState { for i, s := range loadedState {
_, resp := s.step(op.req) _, resp := s.step(op.req)
t.Errorf("For state %d, response diff: %s", i, cmp.Diff(op.resp.EtcdResponse, resp)) t.Errorf("For state %d, response diff: %s", i, cmp.Diff(op.resp, resp))
} }
break break
} }
@ -362,36 +357,10 @@ func TestModelNonDeterministic(t *testing.T) {
} }
} }
type nonDeterministicModelTest struct {
name string
operations []nonDeterministicOperation
}
type nonDeterministicOperation struct {
req EtcdRequest
resp EtcdNonDeterministicResponse
expectFailure bool
}
func toNonDeterministicTest(tc deterministicModelTest) nonDeterministicModelTest {
operations := []nonDeterministicOperation{}
for _, op := range tc.operations {
operations = append(operations, nonDeterministicOperation{
req: op.req,
resp: EtcdNonDeterministicResponse{EtcdResponse: op.resp},
expectFailure: op.expectFailure,
})
}
return nonDeterministicModelTest{
name: tc.name,
operations: operations,
}
}
func TestModelResponseMatch(t *testing.T) { func TestModelResponseMatch(t *testing.T) {
tcs := []struct { tcs := []struct {
resp1 EtcdNonDeterministicResponse resp1 MaybeEtcdResponse
resp2 EtcdNonDeterministicResponse resp2 MaybeEtcdResponse
expectMatch bool expectMatch bool
}{ }{
{ {
@ -421,12 +390,12 @@ func TestModelResponseMatch(t *testing.T) {
}, },
{ {
resp1: getResponse("key", "a", 1, 1), resp1: getResponse("key", "a", 1, 1),
resp2: unknownResponse(1), resp2: partialResponse(1),
expectMatch: true, expectMatch: true,
}, },
{ {
resp1: getResponse("key", "a", 1, 1), resp1: getResponse("key", "a", 1, 1),
resp2: unknownResponse(0), resp2: partialResponse(0),
expectMatch: false, expectMatch: false,
}, },
{ {
@ -446,12 +415,12 @@ func TestModelResponseMatch(t *testing.T) {
}, },
{ {
resp1: putResponse(3), resp1: putResponse(3),
resp2: unknownResponse(3), resp2: partialResponse(3),
expectMatch: true, expectMatch: true,
}, },
{ {
resp1: putResponse(3), resp1: putResponse(3),
resp2: unknownResponse(0), resp2: partialResponse(0),
expectMatch: false, expectMatch: false,
}, },
{ {
@ -476,22 +445,22 @@ func TestModelResponseMatch(t *testing.T) {
}, },
{ {
resp1: deleteResponse(1, 5), resp1: deleteResponse(1, 5),
resp2: unknownResponse(5), resp2: partialResponse(5),
expectMatch: true, expectMatch: true,
}, },
{ {
resp1: deleteResponse(0, 5), resp1: deleteResponse(0, 5),
resp2: unknownResponse(0), resp2: partialResponse(0),
expectMatch: false, expectMatch: false,
}, },
{ {
resp1: deleteResponse(1, 5), resp1: deleteResponse(1, 5),
resp2: unknownResponse(0), resp2: partialResponse(0),
expectMatch: false, expectMatch: false,
}, },
{ {
resp1: deleteResponse(0, 5), resp1: deleteResponse(0, 5),
resp2: unknownResponse(2), resp2: partialResponse(2),
expectMatch: false, expectMatch: false,
}, },
{ {
@ -516,22 +485,22 @@ func TestModelResponseMatch(t *testing.T) {
}, },
{ {
resp1: compareRevisionAndPutResponse(true, 7), resp1: compareRevisionAndPutResponse(true, 7),
resp2: unknownResponse(7), resp2: partialResponse(7),
expectMatch: true, expectMatch: true,
}, },
{ {
resp1: compareRevisionAndPutResponse(false, 7), resp1: compareRevisionAndPutResponse(false, 7),
resp2: unknownResponse(7), resp2: partialResponse(7),
expectMatch: true, expectMatch: true,
}, },
{ {
resp1: compareRevisionAndPutResponse(true, 7), resp1: compareRevisionAndPutResponse(true, 7),
resp2: unknownResponse(0), resp2: partialResponse(0),
expectMatch: false, expectMatch: false,
}, },
{ {
resp1: compareRevisionAndPutResponse(false, 7), resp1: compareRevisionAndPutResponse(false, 7),
resp2: unknownResponse(0), resp2: partialResponse(0),
expectMatch: false, expectMatch: false,
}, },
} }

7
tests/robustness/validate/patch_history.go
View File

@ -27,7 +27,7 @@ func patchOperationsWithWatchEvents(operations []porcupine.Operation, watchEvent
for _, op := range operations { for _, op := range operations {
request := op.Input.(model.EtcdRequest) request := op.Input.(model.EtcdRequest)
resp := op.Output.(model.EtcdNonDeterministicResponse) resp := op.Output.(model.MaybeEtcdResponse)
if resp.Err == nil || op.Call > lastObservedOperation.Call || request.Type != model.Txn { if resp.Err == nil || op.Call > lastObservedOperation.Call || request.Type != model.Txn {
// Cannot patch those requests. // Cannot patch those requests.
newOperations = append(newOperations, op) newOperations = append(newOperations, op)
@ -37,10 +37,7 @@ func patchOperationsWithWatchEvents(operations []porcupine.Operation, watchEvent
if event != nil { if event != nil {
// Set revision and time based on watchEvent. // Set revision and time based on watchEvent.
op.Return = event.Time.Nanoseconds() op.Return = event.Time.Nanoseconds()
op.Output = model.EtcdNonDeterministicResponse{ op.Output = model.MaybeEtcdResponse{PartialResponse: true, EtcdResponse: model.EtcdResponse{Revision: event.Revision}}
EtcdResponse: model.EtcdResponse{Revision: event.Revision},
ResultUnknown: true,
}
newOperations = append(newOperations, op) newOperations = append(newOperations, op)
continue continue
} }