# Alertmanager for the llmgw cluster.
#
# Nothing is sent anywhere until a receiver is chosen, on purpose. Where an alert should go — an
# address, a Slack workspace, a webhook — is a decision with credentials
# attached, and a monitoring system that quietly starts mailing people is worse
# than one that is visibly not wired up. Alerts group and fire here and are
# visible at :9093 and in Prometheus; uncomment a receiver template below and
# name it on the route to make them travel.
global:
  resolve_timeout: 5m

route:
  # By alert and by what it is about, so five nodes going quiet in the same
  # network blip arrive as one notification rather than five.
  group_by: ['alertname', 'node', 'model']
  group_wait: 30s
  group_interval: 5m
  # Long, because a repeat is a reminder and not news. A cluster with one thing
  # wrong should not produce a message an hour.
  repeat_interval: 12h
  receiver: none

  routes:
    # Critical gets its own branch so it can be pointed somewhere else later
    # without touching the rest: the difference between "a node is draining"
    # and "nothing is serving" is who should be woken.
    - matchers: [severity="critical"]
      receiver: none
      group_wait: 10s
      repeat_interval: 4h

receivers:
  - name: none

  # Receiver templates. Uncomment one, fill in where it goes, and name it in
  # `receiver:` above — on the route, the critical branch, or both. Secrets go
  # in files Alertmanager reads (the *_file settings), mode 0640 owned by
  # root:prometheus, not in this file. `amtool check-config` before a restart.
  #
  # Slack (an incoming webhook):
  # - name: slack
  #   slack_configs:
  #     - api_url_file: /etc/prometheus/slack-webhook-url
  #       channel: '#llmgw'
  #       send_resolved: true
  #       title: '{{ .Status | toUpper }}: {{ .CommonLabels.alertname }}'
  #       text: >-
  #         {{ range .Alerts }}{{ .Annotations.summary }}
  #         {{ .Annotations.description }}
  #         {{ end }}
  #
  # E-mail (through a relay that takes authenticated submission):
  # - name: email
  #   email_configs:
  #     - to: ops@example.com
  #       from: llmgw-alerts@example.com
  #       smarthost: smtp.example.com:587
  #       auth_username: llmgw-alerts@example.com
  #       auth_password_file: /etc/prometheus/smtp-password
  #       send_resolved: true
  #
  # PagerDuty (Events API v2), for the critical branch — what should wake
  # someone:
  # - name: pagerduty
  #   pagerduty_configs:
  #     - routing_key_file: /etc/prometheus/pagerduty-routing-key
  #       severity: '{{ .CommonLabels.severity }}'
  #       description: '{{ .CommonAnnotations.summary }}'
  #
  # A webhook of your own (Alertmanager's JSON, version 4):
  # - name: webhook
  #   webhook_configs:
  #     - url_file: /etc/prometheus/alert-webhook-url
  #       send_resolved: true
  #       max_alerts: 20
  #
  # The gateway itself also sends a few events straight to webhooks, with no
  # Prometheus in between — a model parked, a node lost, a budget spent, new
  # weights published: `notify.webhooks` in server.yaml (docs/install.md,
  # "Being told").

inhibit_rules:
  # A node that is down explains every replica on it. Reporting both says the
  # same thing twice and buries whichever is the actual cause.
  - source_matchers: [alertname="NodeNotReady"]
    target_matchers: [alertname="EndpointMetricsStale"]
    equal: ['node']
  - source_matchers: [alertname="ControlPlaneUnreachable"]
    target_matchers: [severity="warning"]