Merge pull request #17983 from thedtripp/feature/addServerRangeDurationMetrics

etcdserver: add server range duration metrics

CHANGELOG/CHANGELOG-3.6.md

@@ -91,6 +91,7 @@ See [List of metrics](https://etcd.io/docs/latest/metrics/) for all metrics per
 
 - Add [`etcd_disk_defrag_inflight`](https://github.com/etcd-io/etcd/pull/13371).
 - Add [`etcd_debugging_server_alarms`](https://github.com/etcd-io/etcd/pull/14276).
+- Add [`etcd_server_range_duration_seconds`](https://github.com/etcd-io/etcd/pull/17983).
 
 ### Go
 - Require [Go 1.22+](https://github.com/etcd-io/etcd/pull/16594).

server/etcdserver/txn/metrics.go

@@ -39,13 +39,29 @@ var (
 		Buckets: prometheus.ExponentialBuckets(0.0001, 2, 20),
 	},
 		[]string{"version", "op", "success"})
+	rangeSec = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "etcd",
+		Subsystem: "server",
+		Name:      "range_duration_seconds",
+		Help:      "The latency distributions of txn.Range",
+
+		// lowest bucket start of upper bound 0.0001 sec (0.1 ms) with factor 2
+		// highest bucket start of 0.0001 sec * 2^19 == 52.4288 sec
+		Buckets: prometheus.ExponentialBuckets(0.0001, 2, 20),
+	},
+		[]string{"success"})
 )
 
 func ApplySecObserve(version, op string, success bool, latency time.Duration) {
 	applySec.WithLabelValues(version, op, strconv.FormatBool(success)).Observe(float64(latency.Microseconds()) / 1000000.0)
 }
 
+func RangeSecObserve(success bool, latency time.Duration) {
+	rangeSec.WithLabelValues(strconv.FormatBool(success)).Observe(float64(latency.Microseconds()) / 1000000.0)
+}
+
 func init() {
 	prometheus.MustRegister(applySec)
+	prometheus.MustRegister(rangeSec)
 	prometheus.MustRegister(slowApplies)
 }

server/etcdserver/txn/metrics_test.go

@@ -0,0 +1,62 @@
+// Copyright 2022 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package txn
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRangeSecObserve(t *testing.T) {
+	// Simulate a range operation taking 500 milliseconds.
+	latency := 500 * time.Millisecond
+	RangeSecObserve(true, latency)
+
+	// Use testutil to collect the results and check against expected value
+	expected := `
+# HELP etcd_server_range_duration_seconds The latency distributions of txn.Range
+# TYPE etcd_server_range_duration_seconds histogram
+etcd_server_range_duration_seconds_bucket{success="true",le="0.0001"} 0
+etcd_server_range_duration_seconds_bucket{success="true",le="0.0002"} 0
+etcd_server_range_duration_seconds_bucket{success="true",le="0.0004"} 0
+etcd_server_range_duration_seconds_bucket{success="true",le="0.0008"} 0
+etcd_server_range_duration_seconds_bucket{success="true",le="0.0016"} 0
+etcd_server_range_duration_seconds_bucket{success="true",le="0.0032"} 0
+etcd_server_range_duration_seconds_bucket{success="true",le="0.0064"} 0
+etcd_server_range_duration_seconds_bucket{success="true",le="0.0128"} 0
+etcd_server_range_duration_seconds_bucket{success="true",le="0.0256"} 0
+etcd_server_range_duration_seconds_bucket{success="true",le="0.0512"} 0
+etcd_server_range_duration_seconds_bucket{success="true",le="0.1024"} 0
+etcd_server_range_duration_seconds_bucket{success="true",le="0.2048"} 0
+etcd_server_range_duration_seconds_bucket{success="true",le="0.4096"} 0
+etcd_server_range_duration_seconds_bucket{success="true",le="0.8192"} 1
+etcd_server_range_duration_seconds_bucket{success="true",le="1.6384"} 1
+etcd_server_range_duration_seconds_bucket{success="true",le="3.2768"} 1
+etcd_server_range_duration_seconds_bucket{success="true",le="6.5536"} 1
+etcd_server_range_duration_seconds_bucket{success="true",le="13.1072"} 1
+etcd_server_range_duration_seconds_bucket{success="true",le="26.2144"} 1
+etcd_server_range_duration_seconds_bucket{success="true",le="52.4288"} 1
+etcd_server_range_duration_seconds_bucket{success="true",le="+Inf"} 1
+etcd_server_range_duration_seconds_sum{success="true"} 0.5
+etcd_server_range_duration_seconds_count{success="true"} 1
+`
+
+	err := testutil.CollectAndCompare(rangeSec, strings.NewReader(expected))
+	require.NoError(t, err, "Collected metrics did not match expected metrics: %v", err)
+}

server/etcdserver/txn/txn.go

@@ -19,6 +19,7 @@ import (
 	"context"
 	"fmt"
 	"sort"
+	"time"
 
 	"go.uber.org/zap"
 
@@ -138,6 +139,10 @@ func Range(ctx context.Context, lg *zap.Logger, kv mvcc.KV, r *pb.RangeRequest)
 		trace = traceutil.New("range", lg)
 		ctx = context.WithValue(ctx, traceutil.TraceKey{}, trace)
 	}
+	defer func(start time.Time) {
+		success := err == nil
+		RangeSecObserve(success, time.Since(start))
+	}(time.Now())
 	txnRead := kv.Read(mvcc.ConcurrentReadTxMode, trace)
 	defer txnRead.End()
 	resp, err = executeRange(ctx, lg, txnRead, r)

tests/integration/metrics_test.go

@@ -22,10 +22,14 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
+
 	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
 	"go.etcd.io/etcd/client/pkg/v3/transport"
 	"go.etcd.io/etcd/server/v3/storage"
 	"go.etcd.io/etcd/tests/v3/framework/integration"
+
+	clientv3 "go.etcd.io/etcd/client/v3"
 )
 
 // TestMetricDbSizeBoot checks that the db size metric is set on boot.
@@ -211,3 +215,33 @@ func TestMetricsHealth(t *testing.T) {
 		t.Fatalf("expected '0' from etcd_server_health_failures, got %q", hv)
 	}
 }
+
+func TestMetricsRangeDurationSeconds(t *testing.T) {
+	integration.BeforeTest(t)
+	clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1})
+	defer clus.Terminate(t)
+
+	client := clus.RandClient()
+
+	keys := []string{
+		"my-namespace/foobar", "my-namespace/foobar1", "namespace/foobar1"}
+	for _, key := range keys {
+		_, err := client.Put(context.Background(), key, "data")
+		require.NoError(t, err)
+	}
+
+	_, err := client.Get(context.Background(), "", clientv3.WithFromKey())
+	require.NoError(t, err)
+
+	rangeDurationSeconds, err := clus.Members[0].Metric("etcd_server_range_duration_seconds")
+	require.NoError(t, err)
+
+	require.NotEmpty(t, rangeDurationSeconds, "expected a number from etcd_server_range_duration_seconds")
+
+	rangeDuration, err := strconv.ParseFloat(rangeDurationSeconds, 64)
+	require.NoError(t, err, "failed to parse duration: %s", err)
+
+	maxRangeDuration := 600.0
+	require.GreaterOrEqual(t, rangeDuration, 0.0, "expected etcd_server_range_duration_seconds to be between 0 and %f, got %f", maxRangeDuration, rangeDuration)
+	require.LessOrEqual(t, rangeDuration, maxRangeDuration, "expected etcd_server_range_duration_seconds to be between 0 and %f, got %f", maxRangeDuration, rangeDuration)
+}