Documentation/etcd-mixin/mixin.libsonnet: Add alerts for etcd fsync duration

This commit adds support to check the 99th percentile of the etcd
members fsync duration and fires a critical alert when it is greater
than 1 sec. The recommended fsync for etcd is 20 ms but there might
be scenarios where a user might be using bad disks for reasons. This
will make sure to let the user/admin know that it is critical for
etcd performance.

Documentation/etcd-mixin/mixin.libsonnet

@@ -173,6 +173,20 @@
               summary: 'etcd cluster 99th percentile fsync durations are too high.',
             },
           },
+          {
+            alert: 'etcdHighFsyncDurations',
+            expr: |||
+              histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{%(etcd_selector)s}[5m]))
+              > 1
+            ||| % $._config,
+            'for': '10m',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              message: 'etcd cluster "{{ $labels.job }}": 99th percentile fync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.',
+            },
+          },
           {
             alert: 'etcdHighCommitDurations',
             expr: |||