From 229275d46eb2993f26c1d402ccd861987849f8a8 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Wed, 10 Apr 2024 10:32:04 +0200 Subject: [PATCH 1/4] Refactor appendSuccesful and appendFailed methods to match Signed-off-by: Marek Siarkowicz --- tests/robustness/model/history.go | 110 ++++++++++-------------------- 1 file changed, 35 insertions(+), 75 deletions(-) diff --git a/tests/robustness/model/history.go b/tests/robustness/model/history.go index 936597c0e..9ac5366d3 100644 --- a/tests/robustness/model/history.go +++ b/tests/robustness/model/history.go @@ -59,51 +59,33 @@ func (h *AppendableHistory) AppendRange(startKey, endKey string, revision, limit if resp != nil && resp.Header != nil { respRevision = resp.Header.Revision } - h.appendSuccessful(porcupine.Operation{ - ClientId: h.streamID, - Input: staleRangeRequest(startKey, endKey, limit, revision), - Call: start.Nanoseconds(), - Output: rangeResponse(resp.Kvs, resp.Count, respRevision), - Return: end.Nanoseconds(), - }) + h.appendSuccessful(staleRangeRequest(startKey, endKey, limit, revision), start, end, rangeResponse(resp.Kvs, resp.Count, respRevision)) } func (h *AppendableHistory) AppendPut(key, value string, start, end time.Duration, resp *clientv3.PutResponse, err error) { request := putRequest(key, value) if err != nil { - h.appendFailed(request, start.Nanoseconds(), err) + h.appendFailed(request, start, err) return } var revision int64 if resp != nil && resp.Header != nil { revision = resp.Header.Revision } - h.appendSuccessful(porcupine.Operation{ - ClientId: h.streamID, - Input: request, - Call: start.Nanoseconds(), - Output: putResponse(revision), - Return: end.Nanoseconds(), - }) + h.appendSuccessful(request, start, end, putResponse(revision)) } func (h *AppendableHistory) AppendPutWithLease(key, value string, leaseID int64, start, end time.Duration, resp *clientv3.PutResponse, err error) { request := putWithLeaseRequest(key, value, leaseID) if err != nil { - h.appendFailed(request, start.Nanoseconds(), err) + h.appendFailed(request, start, err) return } var revision int64 if resp != nil && resp.Header != nil { revision = resp.Header.Revision } - h.appendSuccessful(porcupine.Operation{ - ClientId: h.streamID, - Input: request, - Call: start.Nanoseconds(), - Output: putResponse(revision), - Return: end.Nanoseconds(), - }) + h.appendSuccessful(request, start, end, putResponse(revision)) } func (h *AppendableHistory) AppendLeaseGrant(start, end time.Duration, resp *clientv3.LeaseGrantResponse, err error) { @@ -113,45 +95,33 @@ func (h *AppendableHistory) AppendLeaseGrant(start, end time.Duration, resp *cli } request := leaseGrantRequest(leaseID) if err != nil { - h.appendFailed(request, start.Nanoseconds(), err) + h.appendFailed(request, start, err) return } var revision int64 if resp != nil && resp.ResponseHeader != nil { revision = resp.ResponseHeader.Revision } - h.appendSuccessful(porcupine.Operation{ - ClientId: h.streamID, - Input: request, - Call: start.Nanoseconds(), - Output: leaseGrantResponse(revision), - Return: end.Nanoseconds(), - }) + h.appendSuccessful(request, start, end, leaseGrantResponse(revision)) } func (h *AppendableHistory) AppendLeaseRevoke(id int64, start, end time.Duration, resp *clientv3.LeaseRevokeResponse, err error) { request := leaseRevokeRequest(id) if err != nil { - h.appendFailed(request, start.Nanoseconds(), err) + h.appendFailed(request, start, err) return } var revision int64 if resp != nil && resp.Header != nil { revision = resp.Header.Revision } - h.appendSuccessful(porcupine.Operation{ - ClientId: h.streamID, - Input: request, - Call: start.Nanoseconds(), - Output: leaseRevokeResponse(revision), - Return: end.Nanoseconds(), - }) + h.appendSuccessful(request, start, end, leaseRevokeResponse(revision)) } func (h *AppendableHistory) AppendDelete(key string, start, end time.Duration, resp *clientv3.DeleteResponse, err error) { request := deleteRequest(key) if err != nil { - h.appendFailed(request, start.Nanoseconds(), err) + h.appendFailed(request, start, err) return } var revision int64 @@ -160,13 +130,7 @@ func (h *AppendableHistory) AppendDelete(key string, start, end time.Duration, r revision = resp.Header.Revision deleted = resp.Deleted } - h.appendSuccessful(porcupine.Operation{ - ClientId: h.streamID, - Input: request, - Call: start.Nanoseconds(), - Output: deleteResponse(deleted, revision), - Return: end.Nanoseconds(), - }) + h.appendSuccessful(request, start, end, deleteResponse(deleted, revision)) } func (h *AppendableHistory) AppendTxn(cmp []clientv3.Cmp, clientOnSuccessOps, clientOnFailure []clientv3.Op, start, end time.Duration, resp *clientv3.TxnResponse, err error) { @@ -184,7 +148,7 @@ func (h *AppendableHistory) AppendTxn(cmp []clientv3.Cmp, clientOnSuccessOps, cl } request := txnRequest(conds, modelOnSuccess, modelOnFailure) if err != nil { - h.appendFailed(request, start.Nanoseconds(), err) + h.appendFailed(request, start, err) return } var revision int64 @@ -195,16 +159,17 @@ func (h *AppendableHistory) AppendTxn(cmp []clientv3.Cmp, clientOnSuccessOps, cl for _, resp := range resp.Responses { results = append(results, toEtcdOperationResult(resp)) } - h.appendSuccessful(porcupine.Operation{ + h.appendSuccessful(request, start, end, txnResponse(results, resp.Succeeded, revision)) +} + +func (h *AppendableHistory) appendSuccessful(request EtcdRequest, start, end time.Duration, response MaybeEtcdResponse) { + op := porcupine.Operation{ ClientId: h.streamID, Input: request, Call: start.Nanoseconds(), - Output: txnResponse(results, resp.Succeeded, revision), + Output: response, Return: end.Nanoseconds(), - }) -} - -func (h *AppendableHistory) appendSuccessful(op porcupine.Operation) { + } if op.Call >= op.Return { panic(fmt.Sprintf("Invalid operation, call(%d) >= return(%d)", op.Call, op.Return)) } @@ -298,45 +263,40 @@ func toEtcdOperationResult(resp *etcdserverpb.ResponseOp) EtcdOperationResult { func (h *AppendableHistory) AppendDefragment(start, end time.Duration, resp *clientv3.DefragmentResponse, err error) { request := defragmentRequest() if err != nil { - h.appendFailed(request, start.Nanoseconds(), err) + h.appendFailed(request, start, err) return } var revision int64 if resp != nil && resp.Header != nil { revision = resp.Header.Revision } - h.appendSuccessful(porcupine.Operation{ + h.appendSuccessful(request, start, end, defragmentResponse(revision)) +} + +func (h *AppendableHistory) appendFailed(request EtcdRequest, start time.Duration, err error) { + op := porcupine.Operation{ ClientId: h.streamID, Input: request, Call: start.Nanoseconds(), - Output: defragmentResponse(revision), - Return: end.Nanoseconds(), - }) -} - -func (h *AppendableHistory) appendFailed(request EtcdRequest, call int64, err error) { + Output: failedResponse(err), + Return: 0, // For failed writes we don't know when request has really finished. + } if len(h.successful) > 0 { prevSuccessful := h.successful[len(h.successful)-1] - if call <= prevSuccessful.Call { - panic(fmt.Sprintf("Out of order append, new.call(%d) <= prev.call(%d)", call, prevSuccessful.Call)) + if op.Call <= prevSuccessful.Call { + panic(fmt.Sprintf("Out of order append, new.call(%d) <= prev.call(%d)", op.Call, prevSuccessful.Call)) } - if call <= prevSuccessful.Return { - panic(fmt.Sprintf("Overlapping operations, new.call(%d) <= prev.return(%d)", call, prevSuccessful.Return)) + if op.Call <= prevSuccessful.Return { + panic(fmt.Sprintf("Overlapping operations, new.call(%d) <= prev.return(%d)", op.Call, prevSuccessful.Return)) } } if len(h.failed) > 0 { prevFailed := h.failed[len(h.failed)-1] - if call <= prevFailed.Call { - panic(fmt.Sprintf("Out of order append, new.call(%d) <= prev.call(%d)", call, prevFailed.Call)) + if op.Call <= prevFailed.Call { + panic(fmt.Sprintf("Out of order append, new.call(%d) <= prev.call(%d)", op.Call, prevFailed.Call)) } } - h.failed = append(h.failed, porcupine.Operation{ - ClientId: h.streamID, - Input: request, - Call: call, - Output: failedResponse(err), - Return: 0, // For failed writes we don't know when request has really finished. - }) + h.failed = append(h.failed, op) // Operations of single client needs to be sequential. // As we don't know return time of failed operations, all new writes need to be done with new stream id. h.streamID = h.idProvider.NewStreamID() From 65130c6d21d48cce522e23655f91cfb28a7fca2d Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Wed, 10 Apr 2024 10:52:30 +0200 Subject: [PATCH 2/4] Refactor merge succesfull and failed operation in history Signed-off-by: Marek Siarkowicz --- tests/robustness/model/history.go | 107 ++++++++++++------------------ 1 file changed, 44 insertions(+), 63 deletions(-) diff --git a/tests/robustness/model/history.go b/tests/robustness/model/history.go index 9ac5366d3..46b9b8fb0 100644 --- a/tests/robustness/model/history.go +++ b/tests/robustness/model/history.go @@ -48,8 +48,7 @@ func NewAppendableHistory(ids identity.Provider) *AppendableHistory { streamID: ids.NewStreamID(), idProvider: ids, History: History{ - successful: []porcupine.Operation{}, - failed: []porcupine.Operation{}, + operations: []porcupine.Operation{}, }, } } @@ -173,22 +172,16 @@ func (h *AppendableHistory) appendSuccessful(request EtcdRequest, start, end tim if op.Call >= op.Return { panic(fmt.Sprintf("Invalid operation, call(%d) >= return(%d)", op.Call, op.Return)) } - if len(h.successful) > 0 { - prevSuccessful := h.successful[len(h.successful)-1] - if op.Call <= prevSuccessful.Call { - panic(fmt.Sprintf("Out of order append, new.call(%d) <= prev.call(%d)", op.Call, prevSuccessful.Call)) + if len(h.operations) > 0 { + prev := h.operations[len(h.operations)-1] + if op.Call <= prev.Call { + panic(fmt.Sprintf("Out of order append, new.call(%d) <= prev.call(%d)", op.Call, prev.Call)) } - if op.Call <= prevSuccessful.Return { - panic(fmt.Sprintf("Overlapping operations, new.call(%d) <= prev.return(%d)", op.Call, prevSuccessful.Return)) + if op.Call <= prev.Return { + panic(fmt.Sprintf("Overlapping operations, new.call(%d) <= prev.return(%d)", op.Call, prev.Return)) } } - if len(h.failed) > 0 { - prevFailed := h.failed[len(h.failed)-1] - if op.Call <= prevFailed.Call { - panic(fmt.Sprintf("Out of order append, new.call(%d) <= prev.call(%d)", op.Call, prevFailed.Call)) - } - } - h.successful = append(h.successful, op) + h.operations = append(h.operations, op) } func toEtcdCondition(cmp clientv3.Cmp) (cond EtcdCondition) { @@ -279,24 +272,18 @@ func (h *AppendableHistory) appendFailed(request EtcdRequest, start time.Duratio Input: request, Call: start.Nanoseconds(), Output: failedResponse(err), - Return: 0, // For failed writes we don't know when request has really finished. + Return: -1, // For failed writes we don't know when request has really finished. } - if len(h.successful) > 0 { - prevSuccessful := h.successful[len(h.successful)-1] - if op.Call <= prevSuccessful.Call { - panic(fmt.Sprintf("Out of order append, new.call(%d) <= prev.call(%d)", op.Call, prevSuccessful.Call)) + if len(h.operations) > 0 { + prev := h.operations[len(h.operations)-1] + if op.Call <= prev.Call { + panic(fmt.Sprintf("Out of order append, new.call(%d) <= prev.call(%d)", op.Call, prev.Call)) } - if op.Call <= prevSuccessful.Return { - panic(fmt.Sprintf("Overlapping operations, new.call(%d) <= prev.return(%d)", op.Call, prevSuccessful.Return)) + if op.Call <= prev.Return { + panic(fmt.Sprintf("Overlapping operations, new.call(%d) <= prev.return(%d)", op.Call, prev.Return)) } } - if len(h.failed) > 0 { - prevFailed := h.failed[len(h.failed)-1] - if op.Call <= prevFailed.Call { - panic(fmt.Sprintf("Out of order append, new.call(%d) <= prev.call(%d)", op.Call, prevFailed.Call)) - } - } - h.failed = append(h.failed, op) + h.operations = append(h.operations, op) // Operations of single client needs to be sequential. // As we don't know return time of failed operations, all new writes need to be done with new stream id. h.streamID = h.idProvider.NewStreamID() @@ -453,54 +440,48 @@ func defragmentResponse(revision int64) MaybeEtcdResponse { } type History struct { - successful []porcupine.Operation - // failed requests are kept separate as we don't know return time of failed operations. - // Based on https://github.com/anishathalye/porcupine/issues/10 - failed []porcupine.Operation -} - -func (h History) Merge(h2 History) History { - result := History{ - successful: make([]porcupine.Operation, 0, len(h.successful)+len(h2.successful)), - failed: make([]porcupine.Operation, 0, len(h.failed)+len(h2.failed)), - } - result.successful = append(result.successful, h.successful...) - result.successful = append(result.successful, h2.successful...) - result.failed = append(result.failed, h.failed...) - result.failed = append(result.failed, h2.failed...) - return result + operations []porcupine.Operation } func (h History) Len() int { - return len(h.successful) + len(h.failed) + return len(h.operations) } func (h History) Operations() []porcupine.Operation { - operations := make([]porcupine.Operation, 0, len(h.successful)+len(h.failed)) - var maxTime int64 - for _, op := range h.successful { - operations = append(operations, op) - if op.Return > maxTime { - maxTime = op.Return + operations := make([]porcupine.Operation, 0, len(h.operations)) + var maxTime = h.lastObservedTime() + for _, op := range h.operations { + // Failed requests don't have a known return time. + if op.Return == -1 { + // Simulate Infinity by using last observed time. + op.Return = maxTime + time.Second.Nanoseconds() } - } - for _, op := range h.failed { - if op.Call > maxTime { - maxTime = op.Call - } - } - // Failed requests don't have a known return time. - // Simulate Infinity by using last observed time. - for _, op := range h.failed { - op.Return = maxTime + time.Second.Nanoseconds() operations = append(operations, op) } return operations } +func (h History) lastObservedTime() int64 { + var maxTime int64 + for _, op := range h.operations { + if op.Return == -1 { + // Collect call time from failed operations + if op.Call > maxTime { + maxTime = op.Call + } + } else { + // Collect return time from successful operations + if op.Return > maxTime { + maxTime = op.Return + } + } + } + return maxTime +} + func (h History) MaxRevision() int64 { var maxRevision int64 - for _, op := range h.successful { + for _, op := range h.operations { revision := op.Output.(MaybeEtcdResponse).Revision if revision > maxRevision { maxRevision = revision From ae7f79fd631731840fbdd123a3bf1fd4275217f9 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Wed, 10 Apr 2024 10:56:18 +0200 Subject: [PATCH 3/4] Refactor append from appendFailed and appendSuccesfull Signed-off-by: Marek Siarkowicz --- tests/robustness/model/history.go | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/tests/robustness/model/history.go b/tests/robustness/model/history.go index 46b9b8fb0..81fc68608 100644 --- a/tests/robustness/model/history.go +++ b/tests/robustness/model/history.go @@ -169,19 +169,7 @@ func (h *AppendableHistory) appendSuccessful(request EtcdRequest, start, end tim Output: response, Return: end.Nanoseconds(), } - if op.Call >= op.Return { - panic(fmt.Sprintf("Invalid operation, call(%d) >= return(%d)", op.Call, op.Return)) - } - if len(h.operations) > 0 { - prev := h.operations[len(h.operations)-1] - if op.Call <= prev.Call { - panic(fmt.Sprintf("Out of order append, new.call(%d) <= prev.call(%d)", op.Call, prev.Call)) - } - if op.Call <= prev.Return { - panic(fmt.Sprintf("Overlapping operations, new.call(%d) <= prev.return(%d)", op.Call, prev.Return)) - } - } - h.operations = append(h.operations, op) + h.append(op) } func toEtcdCondition(cmp clientv3.Cmp) (cond EtcdCondition) { @@ -274,6 +262,16 @@ func (h *AppendableHistory) appendFailed(request EtcdRequest, start time.Duratio Output: failedResponse(err), Return: -1, // For failed writes we don't know when request has really finished. } + h.append(op) + // Operations of single client needs to be sequential. + // As we don't know return time of failed operations, all new writes need to be done with new stream id. + h.streamID = h.idProvider.NewStreamID() +} + +func (h *AppendableHistory) append(op porcupine.Operation) { + if op.Return != -1 && op.Call >= op.Return { + panic(fmt.Sprintf("Invalid operation, call(%d) >= return(%d)", op.Call, op.Return)) + } if len(h.operations) > 0 { prev := h.operations[len(h.operations)-1] if op.Call <= prev.Call { @@ -284,9 +282,6 @@ func (h *AppendableHistory) appendFailed(request EtcdRequest, start time.Duratio } } h.operations = append(h.operations, op) - // Operations of single client needs to be sequential. - // As we don't know return time of failed operations, all new writes need to be done with new stream id. - h.streamID = h.idProvider.NewStreamID() } func getRequest(key string) EtcdRequest { From 718d5ba2b4f315614d5e403d74a93d62515b514e Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Wed, 10 Apr 2024 11:20:03 +0200 Subject: [PATCH 4/4] Calculate request success rate to provide signal to performance debugging Signed-off-by: Marek Siarkowicz --- tests/robustness/model/deterministic.go | 15 ++++++++ tests/robustness/model/history.go | 38 ++++++++++++------- tests/robustness/report/client.go | 11 ++++++ tests/robustness/report/client_test.go | 2 +- tests/robustness/traffic/client.go | 7 +--- tests/robustness/traffic/traffic.go | 13 ++++--- tests/robustness/validate/patch_history.go | 13 +++++-- .../robustness/validate/patch_history_test.go | 2 +- 8 files changed, 72 insertions(+), 29 deletions(-) diff --git a/tests/robustness/model/deterministic.go b/tests/robustness/model/deterministic.go index 9b743c9a8..f59bb91ab 100644 --- a/tests/robustness/model/deterministic.go +++ b/tests/robustness/model/deterministic.go @@ -248,6 +248,21 @@ type EtcdRequest struct { Defragment *DefragmentRequest } +func (r *EtcdRequest) IsRead() bool { + if r.Type == Range { + return true + } + if r.Type != Txn { + return false + } + for _, op := range append(r.Txn.OperationsOnSuccess, r.Txn.OperationsOnFailure...) { + if op.Type != RangeOperation { + return false + } + } + return true +} + type RangeRequest struct { RangeOptions Revision int64 diff --git a/tests/robustness/model/history.go b/tests/robustness/model/history.go index 81fc68608..edf2c778f 100644 --- a/tests/robustness/model/history.go +++ b/tests/robustness/model/history.go @@ -53,18 +53,23 @@ func NewAppendableHistory(ids identity.Provider) *AppendableHistory { } } -func (h *AppendableHistory) AppendRange(startKey, endKey string, revision, limit int64, start, end time.Duration, resp *clientv3.GetResponse) { +func (h *AppendableHistory) AppendRange(startKey, endKey string, revision, limit int64, start, end time.Duration, resp *clientv3.GetResponse, err error) { + request := staleRangeRequest(startKey, endKey, limit, revision) + if err != nil { + h.appendFailed(request, start, end, err) + return + } var respRevision int64 if resp != nil && resp.Header != nil { respRevision = resp.Header.Revision } - h.appendSuccessful(staleRangeRequest(startKey, endKey, limit, revision), start, end, rangeResponse(resp.Kvs, resp.Count, respRevision)) + h.appendSuccessful(request, start, end, rangeResponse(resp.Kvs, resp.Count, respRevision)) } func (h *AppendableHistory) AppendPut(key, value string, start, end time.Duration, resp *clientv3.PutResponse, err error) { request := putRequest(key, value) if err != nil { - h.appendFailed(request, start, err) + h.appendFailed(request, start, end, err) return } var revision int64 @@ -77,7 +82,7 @@ func (h *AppendableHistory) AppendPut(key, value string, start, end time.Duratio func (h *AppendableHistory) AppendPutWithLease(key, value string, leaseID int64, start, end time.Duration, resp *clientv3.PutResponse, err error) { request := putWithLeaseRequest(key, value, leaseID) if err != nil { - h.appendFailed(request, start, err) + h.appendFailed(request, start, end, err) return } var revision int64 @@ -94,7 +99,7 @@ func (h *AppendableHistory) AppendLeaseGrant(start, end time.Duration, resp *cli } request := leaseGrantRequest(leaseID) if err != nil { - h.appendFailed(request, start, err) + h.appendFailed(request, start, end, err) return } var revision int64 @@ -107,7 +112,7 @@ func (h *AppendableHistory) AppendLeaseGrant(start, end time.Duration, resp *cli func (h *AppendableHistory) AppendLeaseRevoke(id int64, start, end time.Duration, resp *clientv3.LeaseRevokeResponse, err error) { request := leaseRevokeRequest(id) if err != nil { - h.appendFailed(request, start, err) + h.appendFailed(request, start, end, err) return } var revision int64 @@ -120,7 +125,7 @@ func (h *AppendableHistory) AppendLeaseRevoke(id int64, start, end time.Duration func (h *AppendableHistory) AppendDelete(key string, start, end time.Duration, resp *clientv3.DeleteResponse, err error) { request := deleteRequest(key) if err != nil { - h.appendFailed(request, start, err) + h.appendFailed(request, start, end, err) return } var revision int64 @@ -147,7 +152,7 @@ func (h *AppendableHistory) AppendTxn(cmp []clientv3.Cmp, clientOnSuccessOps, cl } request := txnRequest(conds, modelOnSuccess, modelOnFailure) if err != nil { - h.appendFailed(request, start, err) + h.appendFailed(request, start, end, err) return } var revision int64 @@ -244,7 +249,7 @@ func toEtcdOperationResult(resp *etcdserverpb.ResponseOp) EtcdOperationResult { func (h *AppendableHistory) AppendDefragment(start, end time.Duration, resp *clientv3.DefragmentResponse, err error) { request := defragmentRequest() if err != nil { - h.appendFailed(request, start, err) + h.appendFailed(request, start, end, err) return } var revision int64 @@ -254,18 +259,23 @@ func (h *AppendableHistory) AppendDefragment(start, end time.Duration, resp *cli h.appendSuccessful(request, start, end, defragmentResponse(revision)) } -func (h *AppendableHistory) appendFailed(request EtcdRequest, start time.Duration, err error) { +func (h *AppendableHistory) appendFailed(request EtcdRequest, start, end time.Duration, err error) { op := porcupine.Operation{ ClientId: h.streamID, Input: request, Call: start.Nanoseconds(), Output: failedResponse(err), - Return: -1, // For failed writes we don't know when request has really finished. + Return: end.Nanoseconds(), + } + isRead := request.IsRead() + if !isRead { + // Failed writes can still be persisted, setting -1 for now as don't know when request has took effect. + op.Return = -1 + // Operations of single client needs to be sequential. + // As we don't know return time of failed operations, all new writes need to be done with new stream id. + h.streamID = h.idProvider.NewStreamID() } h.append(op) - // Operations of single client needs to be sequential. - // As we don't know return time of failed operations, all new writes need to be done with new stream id. - h.streamID = h.idProvider.NewStreamID() } func (h *AppendableHistory) append(op porcupine.Operation) { diff --git a/tests/robustness/report/client.go b/tests/robustness/report/client.go index cf0b24f52..477e3c4d3 100644 --- a/tests/robustness/report/client.go +++ b/tests/robustness/report/client.go @@ -36,6 +36,17 @@ type ClientReport struct { Watch []model.WatchOperation } +func (r ClientReport) SuccessfulOperations() int { + count := 0 + for _, op := range r.KeyValue { + resp := op.Output.(model.MaybeEtcdResponse) + if resp.Error == "" { + count++ + } + } + return count +} + func (r ClientReport) WatchEventCount() int { count := 0 for _, op := range r.Watch { diff --git a/tests/robustness/report/client_test.go b/tests/robustness/report/client_test.go index 4b3b5ea00..f359955b9 100644 --- a/tests/robustness/report/client_test.go +++ b/tests/robustness/report/client_test.go @@ -42,7 +42,7 @@ func TestPersistLoadClientReports(t *testing.T) { Key: []byte("key"), ModRevision: 2, Value: []byte("value"), - }}}) + }}}, nil) start = time.Since(baseTime) time.Sleep(time.Nanosecond) diff --git a/tests/robustness/traffic/client.go b/tests/robustness/traffic/client.go index 2cfab6a36..94fc3c41e 100644 --- a/tests/robustness/traffic/client.go +++ b/tests/robustness/traffic/client.go @@ -107,12 +107,9 @@ func (c *RecordingClient) Range(ctx context.Context, start, end string, revision defer c.kvMux.Unlock() callTime := time.Since(c.baseTime) resp, err := c.client.Get(ctx, start, ops...) - if err != nil { - return nil, err - } returnTime := time.Since(c.baseTime) - c.kvOperations.AppendRange(start, end, revision, limit, callTime, returnTime, resp) - return resp, nil + c.kvOperations.AppendRange(start, end, revision, limit, callTime, returnTime, resp, err) + return resp, err } func (c *RecordingClient) Put(ctx context.Context, key, value string) (*clientv3.PutResponse, error) { diff --git a/tests/robustness/traffic/traffic.go b/tests/robustness/traffic/traffic.go index 7e4d8d69f..41feed388 100644 --- a/tests/robustness/traffic/traffic.go +++ b/tests/robustness/traffic/traffic.go @@ -93,14 +93,17 @@ func SimulateTraffic(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2 } reports = append(reports, cc.Report()) - var operationCount int + var totalOperations int + var successfulOperations int for _, r := range reports { - operationCount += len(r.KeyValue) + totalOperations += len(r.KeyValue) + successfulOperations += r.SuccessfulOperations() } - lg.Info("Recorded operations", zap.Int("operationCount", operationCount)) + lg.Info("Recorded operations", zap.Int("operations", totalOperations), zap.Float64("successRate", float64(successfulOperations)/float64(totalOperations))) - qps := float64(operationCount) / float64(endTime.Sub(startTime)) * float64(time.Second) - lg.Info("Average traffic", zap.Float64("qps", qps)) + period := endTime.Sub(startTime) + qps := float64(successfulOperations) / period.Seconds() + lg.Info("Traffic from successful requests", zap.Float64("qps", qps), zap.Int("operations", successfulOperations), zap.Duration("period", period)) if qps < profile.MinimalQPS { t.Errorf("Requiring minimal %f qps for test results to be reliable, got %f qps", profile.MinimalQPS, qps) } diff --git a/tests/robustness/validate/patch_history.go b/tests/robustness/validate/patch_history.go index 17e282c8b..e79d17808 100644 --- a/tests/robustness/validate/patch_history.go +++ b/tests/robustness/validate/patch_history.go @@ -23,15 +23,22 @@ import ( ) func patchedOperationHistory(reports []report.ClientReport) []porcupine.Operation { - allOperations := operations(reports) + allOperations := relevantOperations(reports) uniqueEvents := uniqueWatchEvents(reports) return patchOperationsWithWatchEvents(allOperations, uniqueEvents) } -func operations(reports []report.ClientReport) []porcupine.Operation { +func relevantOperations(reports []report.ClientReport) []porcupine.Operation { var ops []porcupine.Operation for _, r := range reports { - ops = append(ops, r.KeyValue...) + for _, op := range r.KeyValue { + request := op.Input.(model.EtcdRequest) + resp := op.Output.(model.MaybeEtcdResponse) + // Remove failed read requests as they are not relevant for linearization. + if resp.Error == "" || !request.IsRead() { + ops = append(ops, op) + } + } } return ops } diff --git a/tests/robustness/validate/patch_history_test.go b/tests/robustness/validate/patch_history_test.go index 9e3877033..3334a6068 100644 --- a/tests/robustness/validate/patch_history_test.go +++ b/tests/robustness/validate/patch_history_test.go @@ -39,7 +39,7 @@ func TestPatchHistory(t *testing.T) { start := time.Since(baseTime) time.Sleep(time.Nanosecond) stop := time.Since(baseTime) - h.AppendRange("key", "", 0, 0, start, stop, &clientv3.GetResponse{}) + h.AppendRange("key", "", 0, 0, start, stop, &clientv3.GetResponse{}, nil) }, expectRemains: true, },