Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #16083 from serathius/robustness-perfect-knowledge

Browse Source 
Robustness Allow errors and partial responses from deterministic model
This commit is contained in:
Marek Siarkowicz 2023-06-16 10:07:05 +02:00 committed by GitHub
commit 32ea42b51c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 339 additions and 346 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
tests/robustness/model/describe.go
View File

@ -19,17 +19,13 @@ import (
"strings"
)
func describeEtcdNonDeterministicResponse(request EtcdRequest, response EtcdNonDeterministicResponse) string {
func describeEtcdResponse(request EtcdRequest, response MaybeEtcdResponse) string {
if response.Err != nil {
return fmt.Sprintf("err: %q", response.Err)
}
if response.ResultUnknown {
if response.PartialResponse {
return fmt.Sprintf("unknown, rev: %d", response.Revision)
}
return describeEtcdResponse(request, response.EtcdResponse)
}
func describeEtcdResponse(request EtcdRequest, response EtcdResponse) string {
switch request.Type {
case Range:
return fmt.Sprintf("%s, rev: %d", describeRangeResponse(request.Range.RangeOptions, *response.Range), response.Revision)

4
tests/robustness/model/describe_test.go
View File

@ -26,7 +26,7 @@ import (
func TestModelDescribe(t *testing.T) {
tcs := []struct {
req EtcdRequest
resp EtcdNonDeterministicResponse
resp MaybeEtcdResponse
expectDescribe string
}{
{
@ -66,7 +66,7 @@ func TestModelDescribe(t *testing.T) {
},
{
req: putRequest("key4b", "4b"),
resp: unknownResponse(42),
resp: partialResponse(42),
expectDescribe: `put("key4b", "4b") -> unknown, rev: 42`,
},
{

50
tests/robustness/model/deterministic.go
View File

@ -25,7 +25,18 @@ import (
"github.com/anishathalye/porcupine"
)
// DeterministicModel assumes that all requests succeed and have a correct response.
// DeterministicModel assumes a deterministic execution of etcd requests. All
// requests that client called were executed and persisted by etcd. This
// assumption is good for simulating etcd behavior (aka writing a fake), but not
// for validating correctness as requests might be lost or interrupted. It
// requires perfect knowledge of what happened to request which is not possible
// in real systems.
//
// Model can still respond with error or partial response.
// - Error for etcd known errors, like future revision or compacted revision.
// - Incomplete response when requests is correct, but model doesn't have all
// to provide a full response. For example stale reads as model doesn't store
// whole change history as real etcd does.
var DeterministicModel = porcupine.Model{
Init: func() interface{} {
var s etcdState
@ -49,7 +60,7 @@ var DeterministicModel = porcupine.Model{
return ok, string(data)
},
DescribeOperation: func(in, out interface{}) string {
return fmt.Sprintf("%s -> %s", describeEtcdRequest(in.(EtcdRequest)), describeEtcdResponse(in.(EtcdRequest), out.(EtcdResponse)))
return fmt.Sprintf("%s -> %s", describeEtcdRequest(in.(EtcdRequest)), describeEtcdResponse(in.(EtcdRequest), MaybeEtcdResponse{EtcdResponse: out.(EtcdResponse)}))
},
}
@ -64,8 +75,8 @@ func (s etcdState) Step(request EtcdRequest, response EtcdResponse) (bool, etcdS
if s.Revision == 0 {
return true, initState(request, response)
}
newState, gotResponse := s.step(request)
return reflect.DeepEqual(response, gotResponse), newState
newState, modelResponse := s.step(request)
return Match(MaybeEtcdResponse{EtcdResponse: response}, modelResponse), newState
}
// initState tries to create etcd state based on the first request.
@ -85,7 +96,7 @@ func initState(request EtcdRequest, response EtcdResponse) etcdState {
return state
}
if len(request.Txn.OperationsOnSuccess) != len(response.Txn.Results) {
panic(fmt.Sprintf("Incorrect request %s, response %+v", describeEtcdRequest(request), describeEtcdResponse(request, response)))
panic(fmt.Sprintf("Incorrect request %s, response %+v", describeEtcdRequest(request), describeEtcdResponse(request, MaybeEtcdResponse{EtcdResponse: response})))
}
for i, op := range request.Txn.OperationsOnSuccess {
opResp := response.Txn.Results[i]
@ -131,7 +142,7 @@ func emptyState() etcdState {
}
// step handles a successful request, returning updated state and response it would generate.
func (s etcdState) step(request EtcdRequest) (etcdState, EtcdResponse) {
func (s etcdState) step(request EtcdRequest) (etcdState, MaybeEtcdResponse) {
newKVs := map[string]ValueRevision{}
for k, v := range s.KeyValues {
newKVs[k] = v
@ -140,7 +151,7 @@ func (s etcdState) step(request EtcdRequest) (etcdState, EtcdResponse) {
switch request.Type {
case Range:
resp := s.getRange(request.Range.Key, request.Range.RangeOptions)
return s, EtcdResponse{Range: &resp, Revision: s.Revision}
return s, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Range: &resp, Revision: s.Revision}}
case Txn:
failure := false
for _, cond := range request.Txn.Conditions {
@ -189,14 +200,14 @@ func (s etcdState) step(request EtcdRequest) (etcdState, EtcdResponse) {
if increaseRevision {
s.Revision += 1
}
return s, EtcdResponse{Txn: &TxnResponse{Failure: failure, Results: opResp}, Revision: s.Revision}
return s, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Failure: failure, Results: opResp}, Revision: s.Revision}}
case LeaseGrant:
lease := EtcdLease{
LeaseID: request.LeaseGrant.LeaseID,
Keys: map[string]struct{}{},
}
s.Leases[request.LeaseGrant.LeaseID] = lease
return s, EtcdResponse{Revision: s.Revision, LeaseGrant: &LeaseGrantReponse{}}
return s, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Revision: s.Revision, LeaseGrant: &LeaseGrantReponse{}}}
case LeaseRevoke:
//Delete the keys attached to the lease
keyDeleted := false
@ -215,9 +226,9 @@ func (s etcdState) step(request EtcdRequest) (etcdState, EtcdResponse) {
if keyDeleted {
s.Revision += 1
}
return s, EtcdResponse{Revision: s.Revision, LeaseRevoke: &LeaseRevokeResponse{}}
return s, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Revision: s.Revision, LeaseRevoke: &LeaseRevokeResponse{}}}
case Defragment:
return s, EtcdResponse{Defragment: &DefragmentResponse{}, Revision: s.Revision}
return s, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Defragment: &DefragmentResponse{}, Revision: s.Revision}}
default:
panic(fmt.Sprintf("Unknown request type: %v", request.Type))
}
@ -339,13 +350,28 @@ type LeaseRevokeRequest struct {
}
type DefragmentRequest struct{}
// MaybeEtcdResponse extends EtcdResponse to represent partial or failed responses.
// Possible states:
// * Normal response. Only EtcdResponse is set.
// * Partial response. The EtcdResponse.Revision and PartialResponse are set.
// * Failed response. Only Err is set.
type MaybeEtcdResponse struct {
EtcdResponse
PartialResponse bool
Err error
}
type EtcdResponse struct {
Revision int64
Txn *TxnResponse
Range *RangeResponse
LeaseGrant *LeaseGrantReponse
LeaseRevoke *LeaseRevokeResponse
Defragment *DefragmentResponse
Revision int64
}
func Match(r1, r2 MaybeEtcdResponse) bool {
return ((r1.PartialResponse || r2.PartialResponse) && (r1.Revision == r2.Revision)) || reflect.DeepEqual(r1, r2)
}
type TxnResponse struct {

410
tests/robustness/model/deterministic_test.go
View File

@ -24,12 +24,12 @@ import (
)
func TestModelDeterministic(t *testing.T) {
for _, tc := range deterministicModelTestScenarios {
for _, tc := range commonTestScenarios {
tc := tc
t.Run(tc.name, func(t *testing.T) {
state := DeterministicModel.Init()
for _, op := range tc.operations {
ok, newState := DeterministicModel.Step(state, op.req, op.resp)
ok, newState := DeterministicModel.Step(state, op.req, op.resp.EtcdResponse)
if op.expectFailure == ok {
t.Logf("state: %v", state)
t.Errorf("Unexpected operation result, expect: %v, got: %v, operation: %s", !op.expectFailure, ok, DeterministicModel.DescribeOperation(op.req, op.resp))
@ -51,363 +51,363 @@ func TestModelDeterministic(t *testing.T) {
}
}
type deterministicModelTest struct {
type modelTestCase struct {
name string
operations []deterministicOperation
operations []testOperation
}
type deterministicOperation struct {
type testOperation struct {
req EtcdRequest
resp EtcdResponse
resp MaybeEtcdResponse
expectFailure bool
}
var deterministicModelTestScenarios = []deterministicModelTest{
var commonTestScenarios = []modelTestCase{
{
name: "First Get can start from non-empty value and non-zero revision",
operations: []deterministicOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 42, 42).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "1", 42, 42).EtcdResponse},
operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 42, 42)},
{req: getRequest("key"), resp: getResponse("key", "1", 42, 42)},
},
},
{
name: "First Range can start from non-empty value and non-zero revision",
operations: []deterministicOperation{
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key"), Value: []byte("1")}}, 1, 42).EtcdResponse},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key"), Value: []byte("1")}}, 1, 42).EtcdResponse},
operations: []testOperation{
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key"), Value: []byte("1")}}, 1, 42)},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key"), Value: []byte("1")}}, 1, 42)},
},
},
{
name: "First Range can start from non-zero revision",
operations: []deterministicOperation{
{req: rangeRequest("key", true, 0), resp: rangeResponse(nil, 0, 1).EtcdResponse},
{req: rangeRequest("key", true, 0), resp: rangeResponse(nil, 0, 1).EtcdResponse},
operations: []testOperation{
{req: rangeRequest("key", true, 0), resp: rangeResponse(nil, 0, 1)},
{req: rangeRequest("key", true, 0), resp: rangeResponse(nil, 0, 1)},
},
},
{
name: "First Put can start from non-zero revision",
operations: []deterministicOperation{
{req: putRequest("key", "1"), resp: putResponse(42).EtcdResponse},
operations: []testOperation{
{req: putRequest("key", "1"), resp: putResponse(42)},
},
},
{
name: "First delete can start from non-zero revision",
operations: []deterministicOperation{
{req: deleteRequest("key"), resp: deleteResponse(0, 42).EtcdResponse},
operations: []testOperation{
{req: deleteRequest("key"), resp: deleteResponse(0, 42)},
},
},
{
name: "First Txn can start from non-zero revision",
operations: []deterministicOperation{
{req: compareRevisionAndPutRequest("key", 0, "42"), resp: compareRevisionAndPutResponse(false, 42).EtcdResponse},
operations: []testOperation{
{req: compareRevisionAndPutRequest("key", 0, "42"), resp: compareRevisionAndPutResponse(false, 42)},
},
},
{
name: "Get response data should match put",
operations: []deterministicOperation{
{req: putRequest("key1", "11"), resp: putResponse(1).EtcdResponse},
{req: putRequest("key2", "12"), resp: putResponse(2).EtcdResponse},
{req: getRequest("key1"), resp: getResponse("key1", "11", 1, 1).EtcdResponse, expectFailure: true},
{req: getRequest("key1"), resp: getResponse("key1", "12", 1, 1).EtcdResponse, expectFailure: true},
{req: getRequest("key1"), resp: getResponse("key1", "12", 2, 2).EtcdResponse, expectFailure: true},
{req: getRequest("key1"), resp: getResponse("key1", "11", 1, 2).EtcdResponse},
{req: getRequest("key2"), resp: getResponse("key2", "11", 2, 2).EtcdResponse, expectFailure: true},
{req: getRequest("key2"), resp: getResponse("key2", "12", 1, 1).EtcdResponse, expectFailure: true},
{req: getRequest("key2"), resp: getResponse("key2", "11", 1, 1).EtcdResponse, expectFailure: true},
{req: getRequest("key2"), resp: getResponse("key2", "12", 2, 2).EtcdResponse},
operations: []testOperation{
{req: putRequest("key1", "11"), resp: putResponse(1)},
{req: putRequest("key2", "12"), resp: putResponse(2)},
{req: getRequest("key1"), resp: getResponse("key1", "11", 1, 1), expectFailure: true},
{req: getRequest("key1"), resp: getResponse("key1", "12", 1, 1), expectFailure: true},
{req: getRequest("key1"), resp: getResponse("key1", "12", 2, 2), expectFailure: true},
{req: getRequest("key1"), resp: getResponse("key1", "11", 1, 2)},
{req: getRequest("key2"), resp: getResponse("key2", "11", 2, 2), expectFailure: true},
{req: getRequest("key2"), resp: getResponse("key2", "12", 1, 1), expectFailure: true},
{req: getRequest("key2"), resp: getResponse("key2", "11", 1, 1), expectFailure: true},
{req: getRequest("key2"), resp: getResponse("key2", "12", 2, 2)},
},
},
{
name: "Range response data should match put",
operations: []deterministicOperation{
{req: putRequest("key1", "1"), resp: putResponse(1).EtcdResponse},
{req: putRequest("key2", "2"), resp: putResponse(2).EtcdResponse},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 2}}, 2, 2).EtcdResponse},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 2}}, 2, 2).EtcdResponse},
operations: []testOperation{
{req: putRequest("key1", "1"), resp: putResponse(1)},
{req: putRequest("key2", "2"), resp: putResponse(2)},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 2}}, 2, 2)},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 2}}, 2, 2)},
},
},
{
name: "Range limit should reduce number of kvs, but maintain count",
operations: []deterministicOperation{
operations: []testOperation{
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1},
{Key: []byte("key2"), Value: []byte("2"), ModRevision: 2},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 3},
}, 3, 3).EtcdResponse},
}, 3, 3)},
{req: rangeRequest("key", true, 4), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1},
{Key: []byte("key2"), Value: []byte("2"), ModRevision: 2},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 3},
}, 3, 3).EtcdResponse},
}, 3, 3)},
{req: rangeRequest("key", true, 3), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1},
{Key: []byte("key2"), Value: []byte("2"), ModRevision: 2},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 3},
}, 3, 3).EtcdResponse},
}, 3, 3)},
{req: rangeRequest("key", true, 2), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1},
{Key: []byte("key2"), Value: []byte("2"), ModRevision: 2},
}, 3, 3).EtcdResponse},
}, 3, 3)},
{req: rangeRequest("key", true, 1), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1},
}, 3, 3).EtcdResponse},
}, 3, 3)},
},
},
{
name: "Range response should be ordered by key",
operations: []deterministicOperation{
operations: []testOperation{
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key1"), Value: []byte("2"), ModRevision: 3},
{Key: []byte("key2"), Value: []byte("1"), ModRevision: 2},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 1},
}, 3, 3).EtcdResponse},
}, 3, 3)},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key2"), Value: []byte("1"), ModRevision: 2},
{Key: []byte("key1"), Value: []byte("2"), ModRevision: 3},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 1},
}, 3, 3).EtcdResponse, expectFailure: true},
}, 3, 3), expectFailure: true},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 1},
{Key: []byte("key2"), Value: []byte("1"), ModRevision: 2},
{Key: []byte("key1"), Value: []byte("2"), ModRevision: 3},
}, 3, 3).EtcdResponse, expectFailure: true},
}, 3, 3), expectFailure: true},
},
},
{
name: "Range response data should match large put",
operations: []deterministicOperation{
{req: putRequest("key", "012345678901234567890"), resp: putResponse(1).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "123456789012345678901", 1, 1).EtcdResponse, expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "012345678901234567890", 1, 1).EtcdResponse},
{req: putRequest("key", "123456789012345678901"), resp: putResponse(2).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "123456789012345678901", 2, 2).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "012345678901234567890", 2, 2).EtcdResponse, expectFailure: true},
operations: []testOperation{
{req: putRequest("key", "012345678901234567890"), resp: putResponse(1)},
{req: getRequest("key"), resp: getResponse("key", "123456789012345678901", 1, 1), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "012345678901234567890", 1, 1)},
{req: putRequest("key", "123456789012345678901"), resp: putResponse(2)},
{req: getRequest("key"), resp: getResponse("key", "123456789012345678901", 2, 2)},
{req: getRequest("key"), resp: getResponse("key", "012345678901234567890", 2, 2), expectFailure: true},
},
},
{
name: "Put must increase revision by 1",
operations: []deterministicOperation{
{req: getRequest("key"), resp: emptyGetResponse(1).EtcdResponse},
{req: putRequest("key", "1"), resp: putResponse(1).EtcdResponse, expectFailure: true},
{req: putRequest("key", "1"), resp: putResponse(3).EtcdResponse, expectFailure: true},
{req: putRequest("key", "1"), resp: putResponse(2).EtcdResponse},
operations: []testOperation{
{req: getRequest("key"), resp: emptyGetResponse(1)},
{req: putRequest("key", "1"), resp: putResponse(1), expectFailure: true},
{req: putRequest("key", "1"), resp: putResponse(3), expectFailure: true},
{req: putRequest("key", "1"), resp: putResponse(2)},
},
},
{
name: "Delete only increases revision on success",
operations: []deterministicOperation{
{req: putRequest("key1", "11"), resp: putResponse(1).EtcdResponse},
{req: putRequest("key2", "12"), resp: putResponse(2).EtcdResponse},
{req: deleteRequest("key1"), resp: deleteResponse(1, 2).EtcdResponse, expectFailure: true},
{req: deleteRequest("key1"), resp: deleteResponse(1, 3).EtcdResponse},
{req: deleteRequest("key1"), resp: deleteResponse(0, 4).EtcdResponse, expectFailure: true},
{req: deleteRequest("key1"), resp: deleteResponse(0, 3).EtcdResponse},
operations: []testOperation{
{req: putRequest("key1", "11"), resp: putResponse(1)},
{req: putRequest("key2", "12"), resp: putResponse(2)},
{req: deleteRequest("key1"), resp: deleteResponse(1, 2), expectFailure: true},
{req: deleteRequest("key1"), resp: deleteResponse(1, 3)},
{req: deleteRequest("key1"), resp: deleteResponse(0, 4), expectFailure: true},
{req: deleteRequest("key1"), resp: deleteResponse(0, 3)},
},
},
{
name: "Delete not existing key",
operations: []deterministicOperation{
{req: getRequest("key"), resp: emptyGetResponse(1).EtcdResponse},
{req: deleteRequest("key"), resp: deleteResponse(1, 2).EtcdResponse, expectFailure: true},
{req: deleteRequest("key"), resp: deleteResponse(0, 1).EtcdResponse},
operations: []testOperation{
{req: getRequest("key"), resp: emptyGetResponse(1)},
{req: deleteRequest("key"), resp: deleteResponse(1, 2), expectFailure: true},
{req: deleteRequest("key"), resp: deleteResponse(0, 1)},
},
},
{
name: "Delete clears value",
operations: []deterministicOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1).EtcdResponse},
{req: deleteRequest("key"), resp: deleteResponse(1, 2).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1).EtcdResponse, expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 2, 2).EtcdResponse, expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 2).EtcdResponse, expectFailure: true},
{req: getRequest("key"), resp: emptyGetResponse(2).EtcdResponse},
operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: deleteRequest("key"), resp: deleteResponse(1, 2)},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 2, 2), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 2), expectFailure: true},
{req: getRequest("key"), resp: emptyGetResponse(2)},
},
},
{
name: "Txn executes onSuccess if revision matches expected",
operations: []deterministicOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1).EtcdResponse},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(true, 1).EtcdResponse, expectFailure: true},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(false, 2).EtcdResponse, expectFailure: true},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(false, 1).EtcdResponse, expectFailure: true},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(true, 2).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1).EtcdResponse, expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 2).EtcdResponse, expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 2, 2).EtcdResponse, expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 1, 1).EtcdResponse, expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2).EtcdResponse},
operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(true, 1), expectFailure: true},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(false, 2), expectFailure: true},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(false, 1), expectFailure: true},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(true, 2)},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 2), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 2, 2), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 1, 1), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2)},
},
},
{
name: "Txn can expect on key not existing",
operations: []deterministicOperation{
{req: getRequest("key1"), resp: emptyGetResponse(1).EtcdResponse},
{req: compareRevisionAndPutRequest("key1", 0, "2"), resp: compareRevisionAndPutResponse(true, 2).EtcdResponse},
{req: compareRevisionAndPutRequest("key1", 0, "3"), resp: compareRevisionAndPutResponse(true, 3).EtcdResponse, expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key1", 0), putOperation("key1", "4"), putOperation("key1", "5")), resp: txnPutResponse(false, 3).EtcdResponse},
{req: getRequest("key1"), resp: getResponse("key1", "5", 3, 3).EtcdResponse},
{req: compareRevisionAndPutRequest("key2", 0, "6"), resp: compareRevisionAndPutResponse(true, 4).EtcdResponse},
operations: []testOperation{
{req: getRequest("key1"), resp: emptyGetResponse(1)},
{req: compareRevisionAndPutRequest("key1", 0, "2"), resp: compareRevisionAndPutResponse(true, 2)},
{req: compareRevisionAndPutRequest("key1", 0, "3"), resp: compareRevisionAndPutResponse(true, 3), expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key1", 0), putOperation("key1", "4"), putOperation("key1", "5")), resp: txnPutResponse(false, 3)},
{req: getRequest("key1"), resp: getResponse("key1", "5", 3, 3)},
{req: compareRevisionAndPutRequest("key2", 0, "6"), resp: compareRevisionAndPutResponse(true, 4)},
},
},
{
name: "Txn executes onFailure if revision doesn't match expected",
operations: []deterministicOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1).EtcdResponse},
{req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnPutResponse(false, 2).EtcdResponse, expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnEmptyResponse(false, 2).EtcdResponse, expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnEmptyResponse(true, 2).EtcdResponse, expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnPutResponse(true, 1).EtcdResponse, expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnEmptyResponse(true, 1).EtcdResponse},
{req: txnRequestSingleOperation(compareRevision("key", 2), nil, putOperation("key", "2")), resp: txnPutResponse(false, 2).EtcdResponse},
operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnPutResponse(false, 2), expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnEmptyResponse(false, 2), expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnEmptyResponse(true, 2), expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnPutResponse(true, 1), expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnEmptyResponse(true, 1)},
{req: txnRequestSingleOperation(compareRevision("key", 2), nil, putOperation("key", "2")), resp: txnPutResponse(false, 2)},
},
},
{
name: "Put with valid lease id should succeed. Put with invalid lease id should fail",
operations: []deterministicOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse},
{req: putWithLeaseRequest("key", "3", 2), resp: putResponse(3).EtcdResponse, expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2).EtcdResponse},
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)},
{req: putWithLeaseRequest("key", "3", 2), resp: putResponse(3), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2)},
},
},
{
name: "Put with valid lease id should succeed. Put with expired lease id should fail",
operations: []deterministicOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2).EtcdResponse},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse},
{req: putWithLeaseRequest("key", "4", 1), resp: putResponse(4).EtcdResponse, expectFailure: true},
{req: getRequest("key"), resp: emptyGetResponse(3).EtcdResponse},
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)},
{req: putWithLeaseRequest("key", "4", 1), resp: putResponse(4), expectFailure: true},
{req: getRequest("key"), resp: emptyGetResponse(3)},
},
},
{
name: "Revoke should increment the revision",
operations: []deterministicOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse},
{req: getRequest("key"), resp: emptyGetResponse(3).EtcdResponse},
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)},
{req: getRequest("key"), resp: emptyGetResponse(3)},
},
},
{
name: "Put following a PutWithLease will detach the key from the lease",
operations: []deterministicOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse},
{req: putRequest("key", "3"), resp: putResponse(3).EtcdResponse},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "3", 3, 3).EtcdResponse},
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)},
{req: putRequest("key", "3"), resp: putResponse(3)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)},
{req: getRequest("key"), resp: getResponse("key", "3", 3, 3)},
},
},
{
name: "Change lease. Revoking older lease should not increment revision",
operations: []deterministicOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: leaseGrantRequest(2), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse},
{req: putWithLeaseRequest("key", "3", 2), resp: putResponse(3).EtcdResponse},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "3", 3, 3).EtcdResponse},
{req: leaseRevokeRequest(2), resp: leaseRevokeResponse(4).EtcdResponse},
{req: getRequest("key"), resp: emptyGetResponse(4).EtcdResponse},
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: leaseGrantRequest(2), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)},
{req: putWithLeaseRequest("key", "3", 2), resp: putResponse(3)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)},
{req: getRequest("key"), resp: getResponse("key", "3", 3, 3)},
{req: leaseRevokeRequest(2), resp: leaseRevokeResponse(4)},
{req: getRequest("key"), resp: emptyGetResponse(4)},
},
},
{
name: "Update key with same lease",
operations: []deterministicOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse},
{req: putWithLeaseRequest("key", "3", 1), resp: putResponse(3).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "3", 3, 3).EtcdResponse},
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)},
{req: putWithLeaseRequest("key", "3", 1), resp: putResponse(3)},
{req: getRequest("key"), resp: getResponse("key", "3", 3, 3)},
},
},
{
name: "Deleting a leased key - revoke should not increment revision",
operations: []deterministicOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2).EtcdResponse},
{req: deleteRequest("key"), resp: deleteResponse(1, 3).EtcdResponse},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(4).EtcdResponse, expectFailure: true},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse},
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)},
{req: deleteRequest("key"), resp: deleteResponse(1, 3)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(4), expectFailure: true},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)},
},
},
{
name: "Lease a few keys - revoke should increment revision only once",
operations: []deterministicOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key1", "1", 1), resp: putResponse(2).EtcdResponse},
{req: putWithLeaseRequest("key2", "2", 1), resp: putResponse(3).EtcdResponse},
{req: putWithLeaseRequest("key3", "3", 1), resp: putResponse(4).EtcdResponse},
{req: putWithLeaseRequest("key4", "4", 1), resp: putResponse(5).EtcdResponse},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(6).EtcdResponse},
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key1", "1", 1), resp: putResponse(2)},
{req: putWithLeaseRequest("key2", "2", 1), resp: putResponse(3)},
{req: putWithLeaseRequest("key3", "3", 1), resp: putResponse(4)},
{req: putWithLeaseRequest("key4", "4", 1), resp: putResponse(5)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(6)},
},
},
{
name: "Lease some keys then delete some of them. Revoke should increment revision since some keys were still leased",
operations: []deterministicOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key1", "1", 1), resp: putResponse(2).EtcdResponse},
{req: putWithLeaseRequest("key2", "2", 1), resp: putResponse(3).EtcdResponse},
{req: putWithLeaseRequest("key3", "3", 1), resp: putResponse(4).EtcdResponse},
{req: putWithLeaseRequest("key4", "4", 1), resp: putResponse(5).EtcdResponse},
{req: deleteRequest("key1"), resp: deleteResponse(1, 6).EtcdResponse},
{req: deleteRequest("key3"), resp: deleteResponse(1, 7).EtcdResponse},
{req: deleteRequest("key4"), resp: deleteResponse(1, 8).EtcdResponse},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(9).EtcdResponse},
{req: deleteRequest("key2"), resp: deleteResponse(0, 9).EtcdResponse},
{req: getRequest("key1"), resp: emptyGetResponse(9).EtcdResponse},
{req: getRequest("key2"), resp: emptyGetResponse(9).EtcdResponse},
{req: getRequest("key3"), resp: emptyGetResponse(9).EtcdResponse},
{req: getRequest("key4"), resp: emptyGetResponse(9).EtcdResponse},
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key1", "1", 1), resp: putResponse(2)},
{req: putWithLeaseRequest("key2", "2", 1), resp: putResponse(3)},
{req: putWithLeaseRequest("key3", "3", 1), resp: putResponse(4)},
{req: putWithLeaseRequest("key4", "4", 1), resp: putResponse(5)},
{req: deleteRequest("key1"), resp: deleteResponse(1, 6)},
{req: deleteRequest("key3"), resp: deleteResponse(1, 7)},
{req: deleteRequest("key4"), resp: deleteResponse(1, 8)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(9)},
{req: deleteRequest("key2"), resp: deleteResponse(0, 9)},
{req: getRequest("key1"), resp: emptyGetResponse(9)},
{req: getRequest("key2"), resp: emptyGetResponse(9)},
{req: getRequest("key3"), resp: emptyGetResponse(9)},
{req: getRequest("key4"), resp: emptyGetResponse(9)},
},
},
{
name: "Lease some keys then delete all of them. Revoke should not increment",
operations: []deterministicOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key1", "1", 1), resp: putResponse(2).EtcdResponse},
{req: putWithLeaseRequest("key2", "2", 1), resp: putResponse(3).EtcdResponse},
{req: putWithLeaseRequest("key3", "3", 1), resp: putResponse(4).EtcdResponse},
{req: putWithLeaseRequest("key4", "4", 1), resp: putResponse(5).EtcdResponse},
{req: deleteRequest("key1"), resp: deleteResponse(1, 6).EtcdResponse},
{req: deleteRequest("key2"), resp: deleteResponse(1, 7).EtcdResponse},
{req: deleteRequest("key3"), resp: deleteResponse(1, 8).EtcdResponse},
{req: deleteRequest("key4"), resp: deleteResponse(1, 9).EtcdResponse},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(9).EtcdResponse},
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key1", "1", 1), resp: putResponse(2)},
{req: putWithLeaseRequest("key2", "2", 1), resp: putResponse(3)},
{req: putWithLeaseRequest("key3", "3", 1), resp: putResponse(4)},
{req: putWithLeaseRequest("key4", "4", 1), resp: putResponse(5)},
{req: deleteRequest("key1"), resp: deleteResponse(1, 6)},
{req: deleteRequest("key2"), resp: deleteResponse(1, 7)},
{req: deleteRequest("key3"), resp: deleteResponse(1, 8)},
{req: deleteRequest("key4"), resp: deleteResponse(1, 9)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(9)},
},
},
{
name: "All request types",
operations: []deterministicOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key", "1", 1), resp: putResponse(2).EtcdResponse},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse},
{req: putRequest("key", "4"), resp: putResponse(4).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "4", 4, 4).EtcdResponse},
{req: compareRevisionAndPutRequest("key", 4, "5"), resp: compareRevisionAndPutResponse(true, 5).EtcdResponse},
{req: deleteRequest("key"), resp: deleteResponse(1, 6).EtcdResponse},
{req: defragmentRequest(), resp: defragmentResponse(6).EtcdResponse},
operations: []testOperation{
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: putWithLeaseRequest("key", "1", 1), resp: putResponse(2)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)},
{req: putRequest("key", "4"), resp: putResponse(4)},
{req: getRequest("key"), resp: getResponse("key", "4", 4, 4)},
{req: compareRevisionAndPutRequest("key", 4, "5"), resp: compareRevisionAndPutResponse(true, 5)},
{req: deleteRequest("key"), resp: deleteResponse(1, 6)},
{req: defragmentRequest(), resp: defragmentResponse(6)},
},
},
{
name: "Defragment success between all other request types",
operations: []deterministicOperation{
{req: defragmentRequest(), resp: defragmentResponse(1).EtcdResponse},
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1).EtcdResponse},
{req: defragmentRequest(), resp: defragmentResponse(1).EtcdResponse},
{req: putWithLeaseRequest("key", "1", 1), resp: putResponse(2).EtcdResponse},
{req: defragmentRequest(), resp: defragmentResponse(2).EtcdResponse},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse},
{req: defragmentRequest(), resp: defragmentResponse(3).EtcdResponse},
{req: putRequest("key", "4"), resp: putResponse(4).EtcdResponse},
{req: defragmentRequest(), resp: defragmentResponse(4).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "4", 4, 4).EtcdResponse},
{req: defragmentRequest(), resp: defragmentResponse(4).EtcdResponse},
{req: compareRevisionAndPutRequest("key", 4, "5"), resp: compareRevisionAndPutResponse(true, 5).EtcdResponse},
{req: defragmentRequest(), resp: defragmentResponse(5).EtcdResponse},
{req: deleteRequest("key"), resp: deleteResponse(1, 6).EtcdResponse},
{req: defragmentRequest(), resp: defragmentResponse(6).EtcdResponse},
operations: []testOperation{
{req: defragmentRequest(), resp: defragmentResponse(1)},
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: defragmentRequest(), resp: defragmentResponse(1)},
{req: putWithLeaseRequest("key", "1", 1), resp: putResponse(2)},
{req: defragmentRequest(), resp: defragmentResponse(2)},
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)},
{req: defragmentRequest(), resp: defragmentResponse(3)},
{req: putRequest("key", "4"), resp: putResponse(4)},
{req: defragmentRequest(), resp: defragmentResponse(4)},
{req: getRequest("key"), resp: getResponse("key", "4", 4, 4)},
{req: defragmentRequest(), resp: defragmentResponse(4)},
{req: compareRevisionAndPutRequest("key", 4, "5"), resp: compareRevisionAndPutResponse(true, 5)},
{req: defragmentRequest(), resp: defragmentResponse(5)},
{req: deleteRequest("key"), resp: deleteResponse(1, 6)},
{req: defragmentRequest(), resp: defragmentResponse(6)},
},
},
}

48
tests/robustness/model/history.go
View File

@ -344,15 +344,15 @@ func rangeRequest(key string, withPrefix bool, limit int64) EtcdRequest {
return EtcdRequest{Type: Range, Range: &RangeRequest{Key: key, RangeOptions: RangeOptions{WithPrefix: withPrefix, Limit: limit}}}
}
func emptyGetResponse(revision int64) EtcdNonDeterministicResponse {
func emptyGetResponse(revision int64) MaybeEtcdResponse {
return rangeResponse([]*mvccpb.KeyValue{}, 0, revision)
}
func getResponse(key, value string, modRevision, revision int64) EtcdNonDeterministicResponse {
func getResponse(key, value string, modRevision, revision int64) MaybeEtcdResponse {
return rangeResponse([]*mvccpb.KeyValue{{Key: []byte(key), Value: []byte(value), ModRevision: modRevision}}, 1, revision)
}
func rangeResponse(kvs []*mvccpb.KeyValue, count int64, revision int64) EtcdNonDeterministicResponse {
func rangeResponse(kvs []*mvccpb.KeyValue, count int64, revision int64) MaybeEtcdResponse {
result := RangeResponse{KVs: make([]KeyValue, len(kvs)), Count: count}
for i, kv := range kvs {
@ -364,38 +364,38 @@ func rangeResponse(kvs []*mvccpb.KeyValue, count int64, revision int64) EtcdNonD
},
}
}
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{Range: &result, Revision: revision}}
return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Range: &result, Revision: revision}}
}
func failedResponse(err error) EtcdNonDeterministicResponse {
return EtcdNonDeterministicResponse{Err: err}
func failedResponse(err error) MaybeEtcdResponse {
return MaybeEtcdResponse{Err: err}
}
func unknownResponse(revision int64) EtcdNonDeterministicResponse {
return EtcdNonDeterministicResponse{ResultUnknown: true, EtcdResponse: EtcdResponse{Revision: revision}}
func partialResponse(revision int64) MaybeEtcdResponse {
return MaybeEtcdResponse{PartialResponse: true, EtcdResponse: EtcdResponse{Revision: revision}}
}
func putRequest(key, value string) EtcdRequest {
return EtcdRequest{Type: Txn, Txn: &TxnRequest{OperationsOnSuccess: []EtcdOperation{{Type: PutOperation, Key: key, PutOptions: PutOptions{Value: ToValueOrHash(value)}}}}}
}
func putResponse(revision int64) EtcdNonDeterministicResponse {
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: []EtcdOperationResult{{}}}, Revision: revision}}
func putResponse(revision int64) MaybeEtcdResponse {
return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: []EtcdOperationResult{{}}}, Revision: revision}}
}
func deleteRequest(key string) EtcdRequest {
return EtcdRequest{Type: Txn, Txn: &TxnRequest{OperationsOnSuccess: []EtcdOperation{{Type: DeleteOperation, Key: key}}}}
}
func deleteResponse(deleted int64, revision int64) EtcdNonDeterministicResponse {
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: []EtcdOperationResult{{Deleted: deleted}}}, Revision: revision}}
func deleteResponse(deleted int64, revision int64) MaybeEtcdResponse {
return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: []EtcdOperationResult{{Deleted: deleted}}}, Revision: revision}}
}
func compareRevisionAndPutRequest(key string, expectedRevision int64, value string) EtcdRequest {
return txnRequestSingleOperation(compareRevision(key, expectedRevision), putOperation(key, value), nil)
}
func compareRevisionAndPutResponse(succeeded bool, revision int64) EtcdNonDeterministicResponse {
func compareRevisionAndPutResponse(succeeded bool, revision int64) MaybeEtcdResponse {
if succeeded {
return txnPutResponse(succeeded, revision)
}
@ -430,16 +430,16 @@ func txnRequest(conds []EtcdCondition, onSuccess, onFailure []EtcdOperation) Etc
return EtcdRequest{Type: Txn, Txn: &TxnRequest{Conditions: conds, OperationsOnSuccess: onSuccess, OperationsOnFailure: onFailure}}
}
func txnPutResponse(succeeded bool, revision int64) EtcdNonDeterministicResponse {
func txnPutResponse(succeeded bool, revision int64) MaybeEtcdResponse {
return txnResponse([]EtcdOperationResult{{}}, succeeded, revision)
}
func txnEmptyResponse(succeeded bool, revision int64) EtcdNonDeterministicResponse {
func txnEmptyResponse(succeeded bool, revision int64) MaybeEtcdResponse {
return txnResponse([]EtcdOperationResult{}, succeeded, revision)
}
func txnResponse(result []EtcdOperationResult, succeeded bool, revision int64) EtcdNonDeterministicResponse {
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: result, Failure: !succeeded}, Revision: revision}}
func txnResponse(result []EtcdOperationResult, succeeded bool, revision int64) MaybeEtcdResponse {
return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: result, Failure: !succeeded}, Revision: revision}}
}
func putWithLeaseRequest(key, value string, leaseID int64) EtcdRequest {
@ -450,24 +450,24 @@ func leaseGrantRequest(leaseID int64) EtcdRequest {
return EtcdRequest{Type: LeaseGrant, LeaseGrant: &LeaseGrantRequest{LeaseID: leaseID}}
}
func leaseGrantResponse(revision int64) EtcdNonDeterministicResponse {
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{LeaseGrant: &LeaseGrantReponse{}, Revision: revision}}
func leaseGrantResponse(revision int64) MaybeEtcdResponse {
return MaybeEtcdResponse{EtcdResponse: EtcdResponse{LeaseGrant: &LeaseGrantReponse{}, Revision: revision}}
}
func leaseRevokeRequest(leaseID int64) EtcdRequest {
return EtcdRequest{Type: LeaseRevoke, LeaseRevoke: &LeaseRevokeRequest{LeaseID: leaseID}}
}
func leaseRevokeResponse(revision int64) EtcdNonDeterministicResponse {
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{LeaseRevoke: &LeaseRevokeResponse{}, Revision: revision}}
func leaseRevokeResponse(revision int64) MaybeEtcdResponse {
return MaybeEtcdResponse{EtcdResponse: EtcdResponse{LeaseRevoke: &LeaseRevokeResponse{}, Revision: revision}}
}
func defragmentRequest() EtcdRequest {
return EtcdRequest{Type: Defragment, Defragment: &DefragmentRequest{}}
}
func defragmentResponse(revision int64) EtcdNonDeterministicResponse {
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{Defragment: &DefragmentResponse{}, Revision: revision}}
func defragmentResponse(revision int64) MaybeEtcdResponse {
return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Defragment: &DefragmentResponse{}, Revision: revision}}
}
type History struct {
@ -519,7 +519,7 @@ func (h History) Operations() []porcupine.Operation {
func (h History) MaxRevision() int64 {
var maxRevision int64
for _, op := range h.successful {
revision := op.Output.(EtcdNonDeterministicResponse).Revision
revision := op.Output.(MaybeEtcdResponse).Revision
if revision > maxRevision {
maxRevision = revision
}

45
tests/robustness/model/non_deterministic.go
View File

@ -22,7 +22,7 @@ import (
"github.com/anishathalye/porcupine"
)
// NonDeterministicModel extends DeterministicModel to handle requests that have unknown or error response.
// NonDeterministicModel extends DeterministicModel to allow for clients with imperfect knowledge of request destiny.
// Unknown/error response doesn't inform whether request was persisted or not, so model
// considers both cases. This is represented as multiple equally possible deterministic states.
// Failed requests fork the possible states, while successful requests merge and filter them.
@ -41,7 +41,7 @@ var NonDeterministicModel = porcupine.Model{
if err != nil {
panic(err)
}
ok, states := states.Step(in.(EtcdRequest), out.(EtcdNonDeterministicResponse))
ok, states := states.Step(in.(EtcdRequest), out.(MaybeEtcdResponse))
data, err := json.Marshal(states)
if err != nil {
panic(err)
@ -49,30 +49,27 @@ var NonDeterministicModel = porcupine.Model{
return ok, string(data)
},
DescribeOperation: func(in, out interface{}) string {
return fmt.Sprintf("%s -> %s", describeEtcdRequest(in.(EtcdRequest)), describeEtcdNonDeterministicResponse(in.(EtcdRequest), out.(EtcdNonDeterministicResponse)))
return fmt.Sprintf("%s -> %s", describeEtcdRequest(in.(EtcdRequest)), describeEtcdResponse(in.(EtcdRequest), out.(MaybeEtcdResponse)))
},
}
type nonDeterministicState []etcdState
type EtcdNonDeterministicResponse struct {
EtcdResponse
Err error
ResultUnknown bool
}
func (states nonDeterministicState) Step(request EtcdRequest, response EtcdNonDeterministicResponse) (bool, nonDeterministicState) {
func (states nonDeterministicState) Step(request EtcdRequest, response MaybeEtcdResponse) (bool, nonDeterministicState) {
if len(states) == 0 {
if response.Err == nil && !response.ResultUnknown {
if response.Err == nil && !response.PartialResponse {
return true, nonDeterministicState{initState(request, response.EtcdResponse)}
}
states = nonDeterministicState{emptyState()}
}
var newStates nonDeterministicState
if response.Err != nil {
switch {
case response.Err != nil:
newStates = states.stepFailedRequest(request)
} else {
newStates = states.stepSuccessfulRequest(request, response)
case response.PartialResponse:
newStates = states.stepPartialRequest(request, response.EtcdResponse.Revision)
default:
newStates = states.stepSuccessfulRequest(request, response.EtcdResponse)
}
return len(newStates) > 0, newStates
}
@ -90,18 +87,26 @@ func (states nonDeterministicState) stepFailedRequest(request EtcdRequest) nonDe
return newStates
}
// stepSuccessfulRequest filters possible states by leaving ony states that would respond correctly.
func (states nonDeterministicState) stepSuccessfulRequest(request EtcdRequest, response EtcdNonDeterministicResponse) nonDeterministicState {
// stepPartialRequest filters possible states by leaving ony states that would return proper revision.
func (states nonDeterministicState) stepPartialRequest(request EtcdRequest, responseRevision int64) nonDeterministicState {
newStates := make(nonDeterministicState, 0, len(states))
for _, s := range states {
newState, gotResponse := s.step(request)
if Match(EtcdNonDeterministicResponse{EtcdResponse: gotResponse}, response) {
newState, modelResponse := s.step(request)
if modelResponse.Revision == responseRevision {
newStates = append(newStates, newState)
}
}
return newStates
}
func Match(r1, r2 EtcdNonDeterministicResponse) bool {
return ((r1.ResultUnknown || r2.ResultUnknown) && (r1.Revision == r2.Revision)) || reflect.DeepEqual(r1, r2)
// stepSuccessfulRequest filters possible states by leaving ony states that would respond correctly.
func (states nonDeterministicState) stepSuccessfulRequest(request EtcdRequest, response EtcdResponse) nonDeterministicState {
newStates := make(nonDeterministicState, 0, len(states))
for _, s := range states {
newState, modelResponse := s.step(request)
if Match(modelResponse, MaybeEtcdResponse{EtcdResponse: response}) {
newStates = append(newStates, newState)
}
}
return newStates
}

113
tests/robustness/model/non_deterministic_test.go
View File

@ -26,15 +26,10 @@ import (
)
func TestModelNonDeterministic(t *testing.T) {
nonDeterministicTestScenarios := []nonDeterministicModelTest{}
for _, tc := range deterministicModelTestScenarios {
nonDeterministicTestScenarios = append(nonDeterministicTestScenarios, toNonDeterministicTest(tc))
}
nonDeterministicTestScenarios = append(nonDeterministicTestScenarios, []nonDeterministicModelTest{
nonDeterministicTestScenarios := append(commonTestScenarios, []modelTestCase{
{
name: "First Put request fails, but is persisted",
operations: []nonDeterministicOperation{
operations: []testOperation{
{req: putRequest("key1", "1"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key2", "2"), resp: putResponse(3)},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key1"), Value: []byte("1"), ModRevision: 2}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 3}}, 2, 3)},
@ -42,7 +37,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "First Put request fails, and is lost",
operations: []nonDeterministicOperation{
operations: []testOperation{
{req: putRequest("key1", "1"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key2", "2"), resp: putResponse(2)},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key2"), Value: []byte("2"), ModRevision: 2}}, 1, 2)},
@ -50,7 +45,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Put can fail and be lost before get",
operations: []nonDeterministicOperation{
operations: []testOperation{
{req: putRequest("key", "1"), resp: putResponse(1)},
{req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
@ -61,7 +56,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Put can fail and be lost before put",
operations: []nonDeterministicOperation{
operations: []testOperation{
{req: getRequest("key"), resp: emptyGetResponse(1)},
{req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key", "3"), resp: putResponse(2)},
@ -69,7 +64,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Put can fail and be lost before delete",
operations: []nonDeterministicOperation{
operations: []testOperation{
{req: deleteRequest("key"), resp: deleteResponse(0, 1)},
{req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))},
{req: deleteRequest("key"), resp: deleteResponse(0, 1)},
@ -77,7 +72,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Put can fail and be lost before txn",
operations: []nonDeterministicOperation{
operations: []testOperation{
// Txn failure
{req: getRequest("key"), resp: emptyGetResponse(1)},
{req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))},
@ -90,11 +85,11 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Put can fail and be lost before txn success",
operations: []nonDeterministicOperation{},
operations: []testOperation{},
},
{
name: "Put can fail but be persisted and increase revision before get",
operations: []nonDeterministicOperation{
operations: []testOperation{
// One failed request, one persisted.
{req: putRequest("key", "1"), resp: putResponse(1)},
{req: putRequest("key", "2"), resp: failedResponse(errors.New("failed"))},
@ -110,7 +105,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Put can fail but be persisted and increase revision before delete",
operations: []nonDeterministicOperation{
operations: []testOperation{
// One failed request, one persisted.
{req: deleteRequest("key"), resp: deleteResponse(0, 1)},
{req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))},
@ -131,7 +126,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Put can fail but be persisted before txn",
operations: []nonDeterministicOperation{
operations: []testOperation{
// Txn success
{req: getRequest("key"), resp: emptyGetResponse(1)},
{req: putRequest("key", "2"), resp: failedResponse(errors.New("failed"))},
@ -146,7 +141,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Delete can fail and be lost before get",
operations: []nonDeterministicOperation{
operations: []testOperation{
{req: putRequest("key", "1"), resp: putResponse(1)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
@ -157,7 +152,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Delete can fail and be lost before delete",
operations: []nonDeterministicOperation{
operations: []testOperation{
{req: putRequest("key", "1"), resp: putResponse(1)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: deleteRequest("key"), resp: deleteResponse(1, 1), expectFailure: true},
@ -166,7 +161,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Delete can fail and be lost before put",
operations: []nonDeterministicOperation{
operations: []testOperation{
{req: putRequest("key", "1"), resp: putResponse(1)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key", "1"), resp: putResponse(2)},
@ -174,7 +169,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Delete can fail but be persisted before get",
operations: []nonDeterministicOperation{
operations: []testOperation{
// One failed request, one persisted.
{req: putRequest("key", "1"), resp: putResponse(1)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
@ -188,7 +183,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Delete can fail but be persisted before put",
operations: []nonDeterministicOperation{
operations: []testOperation{
// One failed request, one persisted.
{req: putRequest("key", "1"), resp: putResponse(1)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
@ -201,7 +196,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Delete can fail but be persisted before delete",
operations: []nonDeterministicOperation{
operations: []testOperation{
// One failed request, one persisted.
{req: putRequest("key", "1"), resp: putResponse(1)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
@ -215,7 +210,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Delete can fail but be persisted before txn",
operations: []nonDeterministicOperation{
operations: []testOperation{
// Txn success
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
@ -228,7 +223,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Txn can fail and be lost before get",
operations: []nonDeterministicOperation{
operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
@ -237,7 +232,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Txn can fail and be lost before delete",
operations: []nonDeterministicOperation{
operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
{req: deleteRequest("key"), resp: deleteResponse(1, 2)},
@ -245,7 +240,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Txn can fail and be lost before put",
operations: []nonDeterministicOperation{
operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key", "3"), resp: putResponse(2)},
@ -253,7 +248,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Txn can fail but be persisted before get",
operations: []nonDeterministicOperation{
operations: []testOperation{
// One failed request, one persisted.
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
@ -268,7 +263,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Txn can fail but be persisted before put",
operations: []nonDeterministicOperation{
operations: []testOperation{
// One failed request, one persisted.
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
@ -282,7 +277,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Txn can fail but be persisted before delete",
operations: []nonDeterministicOperation{
operations: []testOperation{
// One failed request, one persisted.
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
@ -296,7 +291,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Txn can fail but be persisted before txn",
operations: []nonDeterministicOperation{
operations: []testOperation{
// One failed request, one persisted with success.
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
@ -314,7 +309,7 @@ func TestModelNonDeterministic(t *testing.T) {
},
{
name: "Defragment failures between all other request types",
operations: []nonDeterministicOperation{
operations: []testOperation{
{req: defragmentRequest(), resp: failedResponse(errors.New("failed"))},
{req: leaseGrantRequest(1), resp: leaseGrantResponse(1)},
{req: defragmentRequest(), resp: failedResponse(errors.New("failed"))},
@ -349,7 +344,7 @@ func TestModelNonDeterministic(t *testing.T) {
}
for i, s := range loadedState {
_, resp := s.step(op.req)
t.Errorf("For state %d, response diff: %s", i, cmp.Diff(op.resp.EtcdResponse, resp))
t.Errorf("For state %d, response diff: %s", i, cmp.Diff(op.resp, resp))
}
break
}
@ -362,36 +357,10 @@ func TestModelNonDeterministic(t *testing.T) {
}
}
type nonDeterministicModelTest struct {
name string
operations []nonDeterministicOperation
}
type nonDeterministicOperation struct {
req EtcdRequest
resp EtcdNonDeterministicResponse
expectFailure bool
}
func toNonDeterministicTest(tc deterministicModelTest) nonDeterministicModelTest {
operations := []nonDeterministicOperation{}
for _, op := range tc.operations {
operations = append(operations, nonDeterministicOperation{
req: op.req,
resp: EtcdNonDeterministicResponse{EtcdResponse: op.resp},
expectFailure: op.expectFailure,
})
}
return nonDeterministicModelTest{
name: tc.name,
operations: operations,
}
}
func TestModelResponseMatch(t *testing.T) {
tcs := []struct {
resp1 EtcdNonDeterministicResponse
resp2 EtcdNonDeterministicResponse
resp1 MaybeEtcdResponse
resp2 MaybeEtcdResponse
expectMatch bool
}{
{
@ -421,12 +390,12 @@ func TestModelResponseMatch(t *testing.T) {
},
{
resp1: getResponse("key", "a", 1, 1),
resp2: unknownResponse(1),
resp2: partialResponse(1),
expectMatch: true,
},
{
resp1: getResponse("key", "a", 1, 1),
resp2: unknownResponse(0),
resp2: partialResponse(0),
expectMatch: false,
},
{
@ -446,12 +415,12 @@ func TestModelResponseMatch(t *testing.T) {
},
{
resp1: putResponse(3),
resp2: unknownResponse(3),
resp2: partialResponse(3),
expectMatch: true,
},
{
resp1: putResponse(3),
resp2: unknownResponse(0),
resp2: partialResponse(0),
expectMatch: false,
},
{
@ -476,22 +445,22 @@ func TestModelResponseMatch(t *testing.T) {
},
{
resp1: deleteResponse(1, 5),
resp2: unknownResponse(5),
resp2: partialResponse(5),
expectMatch: true,
},
{
resp1: deleteResponse(0, 5),
resp2: unknownResponse(0),
resp2: partialResponse(0),
expectMatch: false,
},
{
resp1: deleteResponse(1, 5),
resp2: unknownResponse(0),
resp2: partialResponse(0),
expectMatch: false,
},
{
resp1: deleteResponse(0, 5),
resp2: unknownResponse(2),
resp2: partialResponse(2),
expectMatch: false,
},
{
@ -516,22 +485,22 @@ func TestModelResponseMatch(t *testing.T) {
},
{
resp1: compareRevisionAndPutResponse(true, 7),
resp2: unknownResponse(7),
resp2: partialResponse(7),
expectMatch: true,
},
{
resp1: compareRevisionAndPutResponse(false, 7),
resp2: unknownResponse(7),
resp2: partialResponse(7),
expectMatch: true,
},
{
resp1: compareRevisionAndPutResponse(true, 7),
resp2: unknownResponse(0),
resp2: partialResponse(0),
expectMatch: false,
},
{
resp1: compareRevisionAndPutResponse(false, 7),
resp2: unknownResponse(0),
resp2: partialResponse(0),
expectMatch: false,
},
}

7
tests/robustness/validate/patch_history.go
View File

@ -27,7 +27,7 @@ func patchOperationsWithWatchEvents(operations []porcupine.Operation, watchEvent
for _, op := range operations {
request := op.Input.(model.EtcdRequest)
resp := op.Output.(model.EtcdNonDeterministicResponse)
resp := op.Output.(model.MaybeEtcdResponse)
if resp.Err == nil || op.Call > lastObservedOperation.Call || request.Type != model.Txn {
// Cannot patch those requests.
newOperations = append(newOperations, op)
@ -37,10 +37,7 @@ func patchOperationsWithWatchEvents(operations []porcupine.Operation, watchEvent
if event != nil {
// Set revision and time based on watchEvent.
op.Return = event.Time.Nanoseconds()
op.Output = model.EtcdNonDeterministicResponse{
EtcdResponse: model.EtcdResponse{Revision: event.Revision},
ResultUnknown: true,
}
op.Output = model.MaybeEtcdResponse{PartialResponse: true, EtcdResponse: model.EtcdResponse{Revision: event.Revision}}
newOperations = append(newOperations, op)
continue
}