etcd/contrib/mixin/test.yaml

---
rule_files: [manifests/etcd-prometheusRules.yaml]
evaluation_interval: 1m
tests:
  - interval: 1m
    input_series:
      - series: up{job="etcd",instance="10.10.10.0"}
        values: 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0
      - series: up{job="etcd",instance="10.10.10.1"}
        values: 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0
      - series: up{job="etcd",instance="10.10.10.2"}
        values: 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0
    alert_rule_test:
      - eval_time: 3m
        alertname: etcdInsufficientMembers
      - eval_time: 5m
        alertname: etcdInsufficientMembers
      - eval_time: 12m
        alertname: etcdMembersDown
      - eval_time: 14m
        alertname: etcdMembersDown
        exp_alerts:
          - exp_labels:
              job: etcd
              severity: critical
            exp_annotations:
              description: 'etcd cluster "etcd": members are down (3).'
              summary: etcd cluster members are down.
      - eval_time: 7m
        alertname: etcdInsufficientMembers
      - eval_time: 11m
        alertname: etcdInsufficientMembers
        exp_alerts:
          - exp_labels:
              job: etcd
              severity: critical
            exp_annotations:
              description: 'etcd cluster "etcd": insufficient members (1).'
              summary: etcd cluster has insufficient number of members.
      - eval_time: 15m
        alertname: etcdInsufficientMembers
        exp_alerts:
          - exp_labels:
              job: etcd
              severity: critical
            exp_annotations:
              description: 'etcd cluster "etcd": insufficient members (0).'
              summary: etcd cluster has insufficient number of members.
  - interval: 1m
    input_series:
      - series: up{job="etcd",instance="10.10.10.0"}
        values: 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0
      - series: up{job="etcd",instance="10.10.10.1"}
        values: 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0
      - series: up{job="etcd",instance="10.10.10.2"}
        values: 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
    alert_rule_test:
      - eval_time: 14m
        alertname: etcdMembersDown
        exp_alerts:
          - exp_labels:
              job: etcd
              severity: critical
            exp_annotations:
              description: 'etcd cluster "etcd": members are down (3).'
              summary: etcd cluster members are down.
  - interval: 1m
    input_series:
      - series: up{job="etcd",instance="10.10.10.0"}
        values: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0
      - series: up{job="etcd",instance="10.10.10.1"}
        values: 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0
      - series: etcd_network_peer_sent_failures_total{To="member-1",job="etcd",endpoint="test"}
        values: 0 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
    alert_rule_test:
      - eval_time: 13m
        alertname: etcdMembersDown
        exp_alerts:
          - exp_labels:
              job: etcd
              severity: critical
            exp_annotations:
              description: 'etcd cluster "etcd": members are down (1).'
              summary: etcd cluster members are down.
  - interval: 1m
    input_series:
      - series: etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.0"}
        values: 0 0 2 0 0 1 0 0 0 0 0 0 0 0 0 0
      - series: etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.1"}
        values: 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0
      - series: etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.2"}
        values: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
    alert_rule_test:
      - eval_time: 10m
        alertname: etcdHighNumberOfLeaderChanges
        exp_alerts:
          - exp_labels:
              job: etcd
              severity: warning
            exp_annotations:
              description: 'etcd cluster "etcd": 4 leader changes within the last
                15 minutes. Frequent elections may be a sign of insufficient resources,
                high network latency, or disruptions by other components and should
                be investigated.'
              summary: etcd cluster has high number of leader changes.
  - interval: 1m
    input_series:
      - series: etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.0"}
        values: 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0
      - series: etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.1"}
        values: 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0
      - series: etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.2"}
        values: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
    alert_rule_test:
      - eval_time: 10m
        alertname: etcdHighNumberOfLeaderChanges
        exp_alerts:
  - interval: 1m
    input_series:
      - series: etcd_mvcc_db_total_size_in_bytes{job="etcd",instance="10.10.10.0"}
        values: 0+8192x240
      - series: etcd_server_quota_backend_bytes{job="etcd",instance="10.10.10.0"}
        values: 524288+0x240
      - series: etcd_mvcc_db_total_size_in_bytes{job="etcd",instance="10.10.10.1"}
        values: 0+1024x240
      - series: etcd_server_quota_backend_bytes{job="etcd",instance="10.10.10.1"}
        values: 524288+0x240
    alert_rule_test:
      - eval_time: 11m
        alertname: etcdExcessiveDatabaseGrowth
        exp_alerts:
          - exp_labels:
              instance: 10.10.10.0
              job: etcd
              severity: warning
            exp_annotations:
              description: 'etcd cluster "etcd": Predicting running out of disk space
                in the next four hours, based on write observations within the past
                four hours on etcd instance 10.10.10.0, please check as it might be
                disruptive.'
              summary: etcd cluster database growing very fast.
  - interval: 1m
    input_series:
      - series: etcd_mvcc_db_total_size_in_use_in_bytes{job="etcd",instance="10.10.10.0"}
        values: 300000000+0x10
      - series: etcd_mvcc_db_total_size_in_bytes{job="etcd",instance="10.10.10.0"}
        values: 1000000000+0x10
      - series: etcd_mvcc_db_total_size_in_use_in_bytes{job="etcd",instance="10.10.10.1"}
        values: 700000000+0x10
      - series: etcd_mvcc_db_total_size_in_bytes{job="etcd",instance="10.10.10.1"}
        values: 1000000000+0x10
    alert_rule_test:
      - eval_time: 11m
        alertname: etcdDatabaseHighFragmentationRatio
        exp_alerts:
          - exp_labels:
              instance: 10.10.10.0
              job: etcd
              severity: warning
            exp_annotations:
              description: 'etcd cluster "etcd": database size in use on instance
                10.10.10.0 is 30% of the actual allocated disk space, please run defragmentation
                (e.g. etcdctl defrag) to retrieve the unused fragmented disk space.'
              runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation
              summary: etcd database size in use is less than 50% of the actual allocated
                storage.