etcdserver: add trace for txn request (#11491)

* etcdserver: add trace for txn request

* pkg/traceutil: added StopSubTrace as a sign of the end of subtrace. Added test case for logging out subtrace.

pkg/traceutil/trace.go

@@ -60,12 +60,15 @@ type Trace struct {
 	startTime    time.Time
 	steps        []step
 	stepDisabled bool
+	isEmpty      bool
 }
 
 type step struct {
-	time   time.Time
-	msg    string
-	fields []Field
+	time            time.Time
+	msg             string
+	fields          []Field
+	isSubTraceStart bool
+	isSubTraceEnd   bool
 }
 
 func New(op string, lg *zap.Logger, fields ...Field) *Trace {
@@ -74,7 +77,7 @@ func New(op string, lg *zap.Logger, fields ...Field) *Trace {
 
 // TODO returns a non-nil, empty Trace
 func TODO() *Trace {
-	return &Trace{}
+	return &Trace{isEmpty: true}
 }
 
 func Get(ctx context.Context) *Trace {
@@ -93,7 +96,7 @@ func (t *Trace) SetStartTime(time time.Time) {
 }
 
 func (t *Trace) InsertStep(at int, time time.Time, msg string, fields ...Field) {
-	newStep := step{time, msg, fields}
+	newStep := step{time: time, msg: msg, fields: fields}
 	if at < len(t.steps) {
 		t.steps = append(t.steps[:at+1], t.steps[at:]...)
 		t.steps[at] = newStep
@@ -102,6 +105,18 @@ func (t *Trace) InsertStep(at int, time time.Time, msg string, fields ...Field)
 	}
 }
 
+// StartSubTrace adds step to trace as a start sign of sublevel trace
+// All steps in the subtrace will log out the input fields of this function
+func (t *Trace) StartSubTrace(fields ...Field) {
+	t.steps = append(t.steps, step{fields: fields, isSubTraceStart: true})
+}
+
+// StopSubTrace adds step to trace as a end sign of sublevel trace
+// All steps in the subtrace will log out the input fields of this function
+func (t *Trace) StopSubTrace(fields ...Field) {
+	t.steps = append(t.steps, step{fields: fields, isSubTraceEnd: true})
+}
+
 // Step adds step to trace
 func (t *Trace) Step(msg string, fields ...Field) {
 	if !t.stepDisabled {
@@ -109,22 +124,26 @@ func (t *Trace) Step(msg string, fields ...Field) {
 	}
 }
 
-// DisableStep sets the flag to prevent the trace from adding steps
-func (t *Trace) DisableStep() {
-	t.stepDisabled = true
-}
-
-// EnableStep re-enable the trace to add steps
-func (t *Trace) EnableStep() {
-	t.stepDisabled = false
+// StepWithFunction will measure the input function as a single step
+func (t *Trace) StepWithFunction(f func(), msg string, fields ...Field) {
+	t.disableStep()
+	f()
+	t.enableStep()
+	t.Step(msg, fields...)
 }
 
 func (t *Trace) AddField(fields ...Field) {
 	for _, f := range fields {
-		t.fields = append(t.fields, f)
+		if !t.updateFieldIfExist(f) {
+			t.fields = append(t.fields, f)
+		}
 	}
 }
 
+func (t *Trace) IsEmpty() bool {
+	return t.isEmpty
+}
+
 // Log dumps all steps in the Trace
 func (t *Trace) Log() {
 	t.LogWithStepThreshold(0)
@@ -154,7 +173,25 @@ func (t *Trace) logInfo(threshold time.Duration) (string, []zap.Field) {
 
 	var steps []string
 	lastStepTime := t.startTime
-	for _, step := range t.steps {
+	for i := 0; i < len(t.steps); i++ {
+		step := t.steps[i]
+		// add subtrace common fields which defined at the beginning to each sub-steps
+		if step.isSubTraceStart {
+			for j := i + 1; j < len(t.steps) && !t.steps[j].isSubTraceEnd; j++ {
+				t.steps[j].fields = append(step.fields, t.steps[j].fields...)
+			}
+			continue
+		}
+		// add subtrace common fields which defined at the end to each sub-steps
+		if step.isSubTraceEnd {
+			for j := i - 1; j >= 0 && !t.steps[j].isSubTraceStart; j-- {
+				t.steps[j].fields = append(step.fields, t.steps[j].fields...)
+			}
+			continue
+		}
+	}
+	for i := 0; i < len(t.steps); i++ {
+		step := t.steps[i]
 		stepDuration := step.time.Sub(lastStepTime)
 		if stepDuration > threshold {
 			steps = append(steps, fmt.Sprintf("trace[%d] '%v' %s (duration: %v)",
@@ -167,6 +204,27 @@ func (t *Trace) logInfo(threshold time.Duration) (string, []zap.Field) {
 		zap.Duration("duration", totalDuration),
 		zap.Time("start", t.startTime),
 		zap.Time("end", endTime),
-		zap.Strings("steps", steps)}
+		zap.Strings("steps", steps),
+		zap.Int("step_count", len(steps))}
 	return msg, fs
 }
+
+func (t *Trace) updateFieldIfExist(f Field) bool {
+	for i, v := range t.fields {
+		if v.Key == f.Key {
+			t.fields[i].Value = f.Value
+			return true
+		}
+	}
+	return false
+}
+
+// disableStep sets the flag to prevent the trace from adding steps
+func (t *Trace) disableStep() {
+	t.stepDisabled = true
+}
+
+// enableStep re-enable the trace to add steps
+func (t *Trace) enableStep() {
+	t.stepDisabled = false
+}

pkg/traceutil/trace_test.go

@@ -28,7 +28,7 @@ import (
 )
 
 func TestGet(t *testing.T) {
-	traceForTest := &Trace{operation: "test"}
+	traceForTest := &Trace{operation: "Test"}
 	tests := []struct {
 		name        string
 		inputCtx    context.Context
@@ -151,6 +151,51 @@ func TestLog(t *testing.T) {
 				"msg1", "msg2",
 				"traceKey1:traceValue1", "count:1",
 				"stepKey1:stepValue1", "stepKey2:stepValue2",
+				"\"step_count\":2",
+			},
+		},
+		{
+			name: "When trace has subtrace",
+			trace: &Trace{
+				operation: "Test",
+				startTime: time.Now().Add(-100 * time.Millisecond),
+				steps: []step{
+					{
+						time:   time.Now().Add(-80 * time.Millisecond),
+						msg:    "msg1",
+						fields: []Field{{"stepKey1", "stepValue1"}},
+					},
+					{
+						fields:          []Field{{"beginSubTrace", "true"}},
+						isSubTraceStart: true,
+					},
+					{
+						time:   time.Now().Add(-50 * time.Millisecond),
+						msg:    "submsg",
+						fields: []Field{{"subStepKey", "subStepValue"}},
+					},
+					{
+						fields:        []Field{{"endSubTrace", "true"}},
+						isSubTraceEnd: true,
+					},
+					{
+						time:   time.Now().Add(-30 * time.Millisecond),
+						msg:    "msg2",
+						fields: []Field{{"stepKey2", "stepValue2"}},
+					},
+				},
+			},
+			fields: []Field{
+				{"traceKey1", "traceValue1"},
+				{"count", 1},
+			},
+			expectedMsg: []string{
+				"Test",
+				"msg1", "msg2", "submsg",
+				"traceKey1:traceValue1", "count:1",
+				"stepKey1:stepValue1", "stepKey2:stepValue2", "subStepKey:subStepValue",
+				"beginSubTrace:true", "endSubTrace:true",
+				"\"step_count\":3",
 			},
 		},
 	}