# Alertmanager for the llmgw cluster. # # Nothing is sent anywhere yet, on purpose. Where an alert should go — an # address, a Slack workspace, a webhook — is a decision with credentials # attached, and a monitoring system that quietly starts mailing people is worse # than one that is visibly not wired up. Alerts group and fire here and are # visible at :9093 and in Prometheus; add a receiver below to make them travel. global: resolve_timeout: 5m route: # By alert and by what it is about, so five nodes going quiet in the same # network blip arrive as one notification rather than five. group_by: ['alertname', 'node', 'model'] group_wait: 30s group_interval: 5m # Long, because a repeat is a reminder and not news. A cluster with one thing # wrong should not produce a message an hour. repeat_interval: 12h receiver: none routes: # Critical gets its own branch so it can be pointed somewhere else later # without touching the rest: the difference between "a node is draining" # and "nothing is serving" is who should be woken. - matchers: [severity="critical"] receiver: none group_wait: 10s repeat_interval: 4h receivers: - name: none inhibit_rules: # A node that is down explains every replica on it. Reporting both says the # same thing twice and buries whichever is the actual cause. - source_matchers: [alertname="NodeNotReady"] target_matchers: [alertname="EndpointMetricsStale"] equal: ['node'] - source_matchers: [alertname="ControlPlaneUnreachable"] target_matchers: [severity="warning"]