Ilker ManapandClaude Opus 5.5
1b71c38f89
notify.webhooks sends signed JSON or Slack-compatible messages, queued, retried, de-duplicated hourly; ctl notify test checks each webhook. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
90 lines
3.4 KiB
YAML
90 lines
3.4 KiB
YAML
# Alertmanager for the llmgw cluster.
|
|
#
|
|
# Nothing is sent anywhere until a receiver is chosen, on purpose. Where an alert should go — an
|
|
# address, a Slack workspace, a webhook — is a decision with credentials
|
|
# attached, and a monitoring system that quietly starts mailing people is worse
|
|
# than one that is visibly not wired up. Alerts group and fire here and are
|
|
# visible at :9093 and in Prometheus; uncomment a receiver template below and
|
|
# name it on the route to make them travel.
|
|
global:
|
|
resolve_timeout: 5m
|
|
|
|
route:
|
|
# By alert and by what it is about, so five nodes going quiet in the same
|
|
# network blip arrive as one notification rather than five.
|
|
group_by: ['alertname', 'node', 'model']
|
|
group_wait: 30s
|
|
group_interval: 5m
|
|
# Long, because a repeat is a reminder and not news. A cluster with one thing
|
|
# wrong should not produce a message an hour.
|
|
repeat_interval: 12h
|
|
receiver: none
|
|
|
|
routes:
|
|
# Critical gets its own branch so it can be pointed somewhere else later
|
|
# without touching the rest: the difference between "a node is draining"
|
|
# and "nothing is serving" is who should be woken.
|
|
- matchers: [severity="critical"]
|
|
receiver: none
|
|
group_wait: 10s
|
|
repeat_interval: 4h
|
|
|
|
receivers:
|
|
- name: none
|
|
|
|
# Receiver templates. Uncomment one, fill in where it goes, and name it in
|
|
# `receiver:` above — on the route, the critical branch, or both. Secrets go
|
|
# in files Alertmanager reads (the *_file settings), mode 0640 owned by
|
|
# root:prometheus, not in this file. `amtool check-config` before a restart.
|
|
#
|
|
# Slack (an incoming webhook):
|
|
# - name: slack
|
|
# slack_configs:
|
|
# - api_url_file: /etc/prometheus/slack-webhook-url
|
|
# channel: '#llmgw'
|
|
# send_resolved: true
|
|
# title: '{{ .Status | toUpper }}: {{ .CommonLabels.alertname }}'
|
|
# text: >-
|
|
# {{ range .Alerts }}{{ .Annotations.summary }}
|
|
# {{ .Annotations.description }}
|
|
# {{ end }}
|
|
#
|
|
# E-mail (through a relay that takes authenticated submission):
|
|
# - name: email
|
|
# email_configs:
|
|
# - to: ops@example.com
|
|
# from: llmgw-alerts@example.com
|
|
# smarthost: smtp.example.com:587
|
|
# auth_username: llmgw-alerts@example.com
|
|
# auth_password_file: /etc/prometheus/smtp-password
|
|
# send_resolved: true
|
|
#
|
|
# PagerDuty (Events API v2), for the critical branch — what should wake
|
|
# someone:
|
|
# - name: pagerduty
|
|
# pagerduty_configs:
|
|
# - routing_key_file: /etc/prometheus/pagerduty-routing-key
|
|
# severity: '{{ .CommonLabels.severity }}'
|
|
# description: '{{ .CommonAnnotations.summary }}'
|
|
#
|
|
# A webhook of your own (Alertmanager's JSON, version 4):
|
|
# - name: webhook
|
|
# webhook_configs:
|
|
# - url_file: /etc/prometheus/alert-webhook-url
|
|
# send_resolved: true
|
|
# max_alerts: 20
|
|
#
|
|
# The gateway itself also sends a few events straight to webhooks, with no
|
|
# Prometheus in between — a model parked, a node lost, a budget spent, new
|
|
# weights published: `notify.webhooks` in server.yaml (docs/install.md,
|
|
# "Being told").
|
|
|
|
inhibit_rules:
|
|
# A node that is down explains every replica on it. Reporting both says the
|
|
# same thing twice and buries whichever is the actual cause.
|
|
- source_matchers: [alertname="NodeNotReady"]
|
|
target_matchers: [alertname="EndpointMetricsStale"]
|
|
equal: ['node']
|
|
- source_matchers: [alertname="ControlPlaneUnreachable"]
|
|
target_matchers: [severity="warning"]
|