From 13d4e1509bc3f65fbb81684776dac0432114c0a4 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Thu, 26 Apr 2018 16:17:14 +0100 Subject: [PATCH] Documentation: add Prometheus monitoring-mixin Signed-off-by: Tom Wilkie --- Documentation/etcd-mixin/README.md | 11 ++ Documentation/etcd-mixin/mixin.libsonnet | 231 +++++++++++++++++++++++ 2 files changed, 242 insertions(+) create mode 100644 Documentation/etcd-mixin/README.md create mode 100644 Documentation/etcd-mixin/mixin.libsonnet diff --git a/Documentation/etcd-mixin/README.md b/Documentation/etcd-mixin/README.md new file mode 100644 index 000000000..bd92d3bd2 --- /dev/null +++ b/Documentation/etcd-mixin/README.md @@ -0,0 +1,11 @@ +# Promethues Monitoring Mixin for etcd + +> NOTE: This project is *alpha* stage. Flags, configuration, behaviour and design may change significantly in following releases. + +A set of customisable Prometheus alerts for etcd. + +Instructions for use are the same as the [kubernetes-mixin](https://github.com/kubernetes-monitoring/kubernetes-mixin). + +## Background + +* For more information about monitoring mixins, see this [design doc](https://docs.google.com/document/d/1A9xvzwqnFVSOZ5fD3blKODXfsat5fg6ZhnKu9LK3lB4/edit#). diff --git a/Documentation/etcd-mixin/mixin.libsonnet b/Documentation/etcd-mixin/mixin.libsonnet new file mode 100644 index 000000000..ba16dfca8 --- /dev/null +++ b/Documentation/etcd-mixin/mixin.libsonnet @@ -0,0 +1,231 @@ +{ + _config+:: { + etcd_selector: 'job=~".*etcd.*"', + }, + + prometheusAlerts+:: { + groups+: [ + { + name: "etcd", + rules: [ + { + alert: "EtcdInsufficientMembers", + expr: ||| + count(up{%(etcd_selector)s} == 0) by (job) > (count(up{%(etcd_selector)s}) by (job) / 2 - 1) + ||| % $._config, + "for": "3m", + labels: { + severity: "critical", + }, + annotations: { + message: 'Etcd cluster "{{ $labels.job }}": insufficient members ({{ $value }}).', + }, + }, + { + alert: "EtcdNoLeader", + expr: ||| + etcd_server_has_leader{%(etcd_selector)s} == 0 + ||| % $._config, + "for": "1m", + labels: { + severity: "critical", + }, + annotations: { + message: 'Etcd cluster "{{ $labels.job }}": member {{ $labels.instance }} has no leader.', + }, + }, + { + alert: "EtcdHighNumberOfLeaderChanges", + expr: ||| + rate(etcd_server_leader_changes_seen_total{%(etcd_selector)s}[15m]) > 3 + ||| % $._config, + "for": "15m", + labels: { + severity: "warning", + }, + annotations: { + message: 'Etcd cluster "{{ $labels.job }}": instance {{ $labels.instance }} has seen {{ $value }} leader changes within the last hour.', + }, + }, + { + alert: "EtcdHighNumberOfFailedGRPCRequests", + expr: ||| + 100 * sum(rate(grpc_server_handled_total{%(etcd_selector)s, grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method) + / + sum(rate(grpc_server_handled_total{%(etcd_selector)s}[5m])) BY (job, instance, grpc_service, grpc_method) + > 1 + ||| % $._config, + "for": "10m", + labels: { + severity: "warning", + }, + annotations: { + message: 'Etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.', + }, + }, + { + alert: "EtcdHighNumberOfFailedGRPCRequests", + expr: ||| + 100 * sum(rate(grpc_server_handled_total{%(etcd_selector)s, grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method) + / + sum(rate(grpc_server_handled_total{%(etcd_selector)s}[5m])) BY (job, instance, grpc_service, grpc_method) + > 5 + ||| % $._config, + "for": "5m", + labels: { + severity: "critical", + }, + annotations: { + message: 'Etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.', + }, + }, + { + alert: "EtcdGRPCRequestsSlow", + expr: ||| + histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{%(etcd_selector)s, grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le)) + > 0.15 + ||| % $._config, + "for": "10m", + labels: { + severity: "critical", + }, + annotations: { + message: 'Etcd cluster "{{ $labels.job }}": gRPC requests to {{ $labels.grpc_method }} are taking {{ $value }}s on etcd instance {{ $labels.instance }}.', + }, + }, + { + alert: "EtcdHighNumberOfFailedHTTPRequests", + expr: ||| + 100 * sum(rate(etcd_http_failed_total{%(etcd_selector)s}[5m])) BY (job, instance, method) + / + sum(rate(etcd_http_received_total{%(etcd_selector)s}[5m])) BY (job, instance, method) + > 1 + ||| % $._config, + "for": "10m", + labels: { + severity: "warning", + }, + annotations: { + message: 'Etcd cluster "{{ $labels.job }}": {{ $value }}%% of requests for {{ $labels.method }} failed on etcd instance {{ $labels.instance }}.', + }, + }, + { + alert: "EtcdHighNumberOfFailedHTTPRequests", + expr: ||| + 100 * sum(rate(etcd_http_failed_total{%(etcd_selector)s}[5m])) BY (job, instance, method) + / + sum(rate(etcd_http_received_total{%(etcd_selector)s}[5m])) BY (job, instance, method) + > 5 + ||| % $._config, + "for": "5m", + labels: { + severity: "critical", + }, + annotations: { + message: 'Etcd cluster "{{ $labels.job }}": {{ $value }}%% of requests for {{ $labels.method }} failed on etcd instance {{ $labels.instance }}.', + }, + }, + { + alert: "EtcdHTTPRequestsSlow", + expr: ||| + histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket{%(etcd_selector)s}[5m])) + > 0.15 + ||| % $._config, + "for": "10m", + labels: { + severity: "warning", + }, + annotations: { + message: 'Etcd cluster "{{ $labels.job }}": HTTP requests to {{ $labels.method }} are taking {{ $value }} on etcd instance {{ $labels.instance }}.', + }, + }, + { + alert: "EtcdMemberCommunicationSlow", + expr: ||| + histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{%(etcd_selector)s}[5m])) + > 0.15 + ||| % $._config, + "for": "10m", + labels: { + severity: "warning", + }, + annotations: { + message: 'Etcd cluster "{{ $labels.job }}": member communication with {{ $labels.To }} is taking {{ $value }}s on etcd instance {{ $labels.instance }}.', + }, + }, + { + alert: "EtcdHighNumberOfFailedProposals", + expr: ||| + rate(etcd_server_proposals_failed_total{%(etcd_selector)s}[15m]) > 5 + ||| % $._config, + "for": "15m", + labels: { + severity: "warning", + }, + annotations: { + message: 'Etcd cluster "{{ $labels.job }}": {{ $value }} proposal failures within the last hour on etcd instance {{ $labels.instance }}.', + }, + }, + { + alert: "EtcdHighFsyncDurations", + expr: ||| + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{%(etcd_selector)s}[5m])) + > 0.5 + ||| % $._config, + "for": "10m", + labels: { + severity: "warning", + }, + annotations: { + message: 'Etcd cluster "{{ $labels.job }}": 99th percentile fync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.', + }, + }, + { + alert: "EtcdHighCommitDurations", + expr: ||| + histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{%(etcd_selector)s}[5m])) + > 0.25 + ||| % $._config, + "for": "10m", + labels: { + severity: "warning", + }, + annotations: { + message: 'Etcd cluster "{{ $labels.job }}": 99th percentile commit durations {{ $value }}s on etcd instance {{ $labels.instance }}.', + }, + }, + { + record: "instance:fd_utilization", + expr: "process_open_fds / process_max_fds", + }, + { + alert: "FdExhaustionClose", + expr: ||| + predict_linear(instance:fd_utilization{%(etcd_selector)s}[1h], 3600 * 4) > 1 + ||| % $._config, + "for": "10m", + labels: { + severity: "warning", + }, + annotations: { + message: '{{ $labels.job }} instance {{ $labels.instance }} will exhaust its file descriptors soon', + }, + }, + { + alert: "FdExhaustionClose", + expr: ||| + predict_linear(instance:fd_utilization{%(etcd_selector)s}[10m], 3600) > 1 + ||| % $._config, + "for": "10m", + labels: { + severity: "critical", + }, + annotations: { + description: '{{ $labels.job }} instance {{ $labels.instance }} will exhaust its file descriptors soon', + }, + } + ], + }, + ], + }, +}