Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Calculate request success rate to provide signal to performance debugging

Browse Source 
Signed-off-by: Marek Siarkowicz <siarkowicz@google.com>
This commit is contained in:
Marek Siarkowicz 2024-04-10 11:20:03 +02:00
parent ae7f79fd63
commit 718d5ba2b4
8 changed files with 72 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
tests/robustness/model/deterministic.go
View File

@ -248,6 +248,21 @@ type EtcdRequest struct {
Defragment *DefragmentRequest
}
func (r *EtcdRequest) IsRead() bool {
if r.Type == Range {
return true
}
if r.Type != Txn {
return false
}
for _, op := range append(r.Txn.OperationsOnSuccess, r.Txn.OperationsOnFailure...) {
if op.Type != RangeOperation {
return false
}
}
return true
}
type RangeRequest struct {
RangeOptions
Revision int64

38
tests/robustness/model/history.go
View File

@ -53,18 +53,23 @@ func NewAppendableHistory(ids identity.Provider) *AppendableHistory {
}
}
func (h *AppendableHistory) AppendRange(startKey, endKey string, revision, limit int64, start, end time.Duration, resp *clientv3.GetResponse) {
func (h *AppendableHistory) AppendRange(startKey, endKey string, revision, limit int64, start, end time.Duration, resp *clientv3.GetResponse, err error) {
request := staleRangeRequest(startKey, endKey, limit, revision)
if err != nil {
h.appendFailed(request, start, end, err)
return
}
var respRevision int64
if resp != nil && resp.Header != nil {
respRevision = resp.Header.Revision
}
h.appendSuccessful(staleRangeRequest(startKey, endKey, limit, revision), start, end, rangeResponse(resp.Kvs, resp.Count, respRevision))
h.appendSuccessful(request, start, end, rangeResponse(resp.Kvs, resp.Count, respRevision))
}
func (h *AppendableHistory) AppendPut(key, value string, start, end time.Duration, resp *clientv3.PutResponse, err error) {
request := putRequest(key, value)
if err != nil {
h.appendFailed(request, start, err)
h.appendFailed(request, start, end, err)
return
}
var revision int64
@ -77,7 +82,7 @@ func (h *AppendableHistory) AppendPut(key, value string, start, end time.Duratio
func (h *AppendableHistory) AppendPutWithLease(key, value string, leaseID int64, start, end time.Duration, resp *clientv3.PutResponse, err error) {
request := putWithLeaseRequest(key, value, leaseID)
if err != nil {
h.appendFailed(request, start, err)
h.appendFailed(request, start, end, err)
return
}
var revision int64
@ -94,7 +99,7 @@ func (h *AppendableHistory) AppendLeaseGrant(start, end time.Duration, resp *cli
}
request := leaseGrantRequest(leaseID)
if err != nil {
h.appendFailed(request, start, err)
h.appendFailed(request, start, end, err)
return
}
var revision int64
@ -107,7 +112,7 @@ func (h *AppendableHistory) AppendLeaseGrant(start, end time.Duration, resp *cli
func (h *AppendableHistory) AppendLeaseRevoke(id int64, start, end time.Duration, resp *clientv3.LeaseRevokeResponse, err error) {
request := leaseRevokeRequest(id)
if err != nil {
h.appendFailed(request, start, err)
h.appendFailed(request, start, end, err)
return
}
var revision int64
@ -120,7 +125,7 @@ func (h *AppendableHistory) AppendLeaseRevoke(id int64, start, end time.Duration
func (h *AppendableHistory) AppendDelete(key string, start, end time.Duration, resp *clientv3.DeleteResponse, err error) {
request := deleteRequest(key)
if err != nil {
h.appendFailed(request, start, err)
h.appendFailed(request, start, end, err)
return
}
var revision int64
@ -147,7 +152,7 @@ func (h *AppendableHistory) AppendTxn(cmp []clientv3.Cmp, clientOnSuccessOps, cl
}
request := txnRequest(conds, modelOnSuccess, modelOnFailure)
if err != nil {
h.appendFailed(request, start, err)
h.appendFailed(request, start, end, err)
return
}
var revision int64
@ -244,7 +249,7 @@ func toEtcdOperationResult(resp *etcdserverpb.ResponseOp) EtcdOperationResult {
func (h *AppendableHistory) AppendDefragment(start, end time.Duration, resp *clientv3.DefragmentResponse, err error) {
request := defragmentRequest()
if err != nil {
h.appendFailed(request, start, err)
h.appendFailed(request, start, end, err)
return
}
var revision int64
@ -254,18 +259,23 @@ func (h *AppendableHistory) AppendDefragment(start, end time.Duration, resp *cli
h.appendSuccessful(request, start, end, defragmentResponse(revision))
}
func (h *AppendableHistory) appendFailed(request EtcdRequest, start time.Duration, err error) {
func (h *AppendableHistory) appendFailed(request EtcdRequest, start, end time.Duration, err error) {
op := porcupine.Operation{
ClientId: h.streamID,
Input: request,
Call: start.Nanoseconds(),
Output: failedResponse(err),
Return: -1, // For failed writes we don't know when request has really finished.
Return: end.Nanoseconds(),
}
isRead := request.IsRead()
if !isRead {
// Failed writes can still be persisted, setting -1 for now as don't know when request has took effect.
op.Return = -1
// Operations of single client needs to be sequential.
// As we don't know return time of failed operations, all new writes need to be done with new stream id.
h.streamID = h.idProvider.NewStreamID()
}
h.append(op)
// Operations of single client needs to be sequential.
// As we don't know return time of failed operations, all new writes need to be done with new stream id.
h.streamID = h.idProvider.NewStreamID()
}
func (h *AppendableHistory) append(op porcupine.Operation) {

11
tests/robustness/report/client.go
View File

@ -36,6 +36,17 @@ type ClientReport struct {
Watch []model.WatchOperation
}
func (r ClientReport) SuccessfulOperations() int {
count := 0
for _, op := range r.KeyValue {
resp := op.Output.(model.MaybeEtcdResponse)
if resp.Error == "" {
count++
}
}
return count
}
func (r ClientReport) WatchEventCount() int {
count := 0
for _, op := range r.Watch {

2
tests/robustness/report/client_test.go
View File

@ -42,7 +42,7 @@ func TestPersistLoadClientReports(t *testing.T) {
Key: []byte("key"),
ModRevision: 2,
Value: []byte("value"),
}}})
}}}, nil)
start = time.Since(baseTime)
time.Sleep(time.Nanosecond)

7
tests/robustness/traffic/client.go
View File

@ -107,12 +107,9 @@ func (c *RecordingClient) Range(ctx context.Context, start, end string, revision
defer c.kvMux.Unlock()
callTime := time.Since(c.baseTime)
resp, err := c.client.Get(ctx, start, ops...)
if err != nil {
return nil, err
}
returnTime := time.Since(c.baseTime)
c.kvOperations.AppendRange(start, end, revision, limit, callTime, returnTime, resp)
return resp, nil
c.kvOperations.AppendRange(start, end, revision, limit, callTime, returnTime, resp, err)
return resp, err
}
func (c *RecordingClient) Put(ctx context.Context, key, value string) (*clientv3.PutResponse, error) {

13
tests/robustness/traffic/traffic.go
View File

@ -93,14 +93,17 @@ func SimulateTraffic(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2
}
reports = append(reports, cc.Report())
var operationCount int
var totalOperations int
var successfulOperations int
for _, r := range reports {
operationCount += len(r.KeyValue)
totalOperations += len(r.KeyValue)
successfulOperations += r.SuccessfulOperations()
}
lg.Info("Recorded operations", zap.Int("operationCount", operationCount))
lg.Info("Recorded operations", zap.Int("operations", totalOperations), zap.Float64("successRate", float64(successfulOperations)/float64(totalOperations)))
qps := float64(operationCount) / float64(endTime.Sub(startTime)) * float64(time.Second)
lg.Info("Average traffic", zap.Float64("qps", qps))
period := endTime.Sub(startTime)
qps := float64(successfulOperations) / period.Seconds()
lg.Info("Traffic from successful requests", zap.Float64("qps", qps), zap.Int("operations", successfulOperations), zap.Duration("period", period))
if qps < profile.MinimalQPS {
t.Errorf("Requiring minimal %f qps for test results to be reliable, got %f qps", profile.MinimalQPS, qps)
}

13
tests/robustness/validate/patch_history.go
View File

@ -23,15 +23,22 @@ import (
)
func patchedOperationHistory(reports []report.ClientReport) []porcupine.Operation {
allOperations := operations(reports)
allOperations := relevantOperations(reports)
uniqueEvents := uniqueWatchEvents(reports)
return patchOperationsWithWatchEvents(allOperations, uniqueEvents)
}
func operations(reports []report.ClientReport) []porcupine.Operation {
func relevantOperations(reports []report.ClientReport) []porcupine.Operation {
var ops []porcupine.Operation
for _, r := range reports {
ops = append(ops, r.KeyValue...)
for _, op := range r.KeyValue {
request := op.Input.(model.EtcdRequest)
resp := op.Output.(model.MaybeEtcdResponse)
// Remove failed read requests as they are not relevant for linearization.
if resp.Error == "" || !request.IsRead() {
ops = append(ops, op)
}
}
}
return ops
}

2
tests/robustness/validate/patch_history_test.go
View File

@ -39,7 +39,7 @@ func TestPatchHistory(t *testing.T) {
start := time.Since(baseTime)
time.Sleep(time.Nanosecond)
stop := time.Since(baseTime)
h.AppendRange("key", "", 0, 0, start, stop, &clientv3.GetResponse{})
h.AppendRange("key", "", 0, 0, start, stop, &clientv3.GetResponse{}, nil)
},
expectRemains: true,
},