Merge pull request #12266 from chaitanyaenr/etcd_fsync_alert

Add alerts for etcd fsync duration

Documentation/etcd-mixin/mixin.libsonnet

@@ -173,6 +173,20 @@
               summary: 'etcd cluster 99th percentile fsync durations are too high.',
             },
           },
+          {
+            alert: 'etcdHighFsyncDurations',
+            expr: |||
+              histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{%(etcd_selector)s}[5m]))
+              > 1
+            ||| % $._config,
+            'for': '10m',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              message: 'etcd cluster "{{ $labels.job }}": 99th percentile fync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.',
+            },
+          },
           {
             alert: 'etcdHighCommitDurations',
             expr: |||