llmgw/deploy/monitoring/llmgw-rules.yml

# Alerts for llmgw. Every expression here is against a series this cluster
# actually publishes; nothing is written against a metric that might exist.
groups:
  - name: llmgw-fleet
    rules:
      - alert: ControlPlaneUnreachable
        expr: up{job="llmgw"} == 0
        for: 1m
        labels: { severity: critical }
        annotations:
          summary: "The control plane is not answering"
          description: >-
            Nothing has scraped 10.0.0.140:9091 for a minute. Engines already
            running keep serving — the router is in the same process, so if
            this is down, so is the gateway.

      - alert: NodeNotReady
        expr: llmgw_node_up == 0
        for: 2m
        labels: { severity: warning }
        annotations:
          summary: "{{ $labels.node }} is {{ $labels.state }}"
          description: >-
            The node is known but not taking work. Two minutes, because a node
            restarting its agent is briefly not ready and is not a problem.

      - alert: AgentGoneQuiet
        expr: llmgw_node_last_seen_seconds > 60
        for: 1m
        labels: { severity: warning }
        annotations:
          summary: "{{ $labels.node }} has not reported for {{ $value | humanizeDuration }}"
          description: >-
            Heartbeats are every 5s. A minute of silence is the network or the
            agent, and the control plane stages expiry at 15s unreachable and
            60s lost — so this fires around the point it stops placing there.

      - alert: ModelHasNothingServing
        expr: llmgw_model_replicas_ready == 0
        for: 10m
        labels: { severity: critical }
        annotations:
          summary: "{{ $labels.model }} has no replica ready"
          description: >-
            The model is placed and nothing is serving it. Ten minutes, because
            a large model on CPU legitimately takes minutes to load. A model
            that is deliberately stopped publishes no series at all, so this
            cannot fire for one nobody asked for.

      - alert: PoolSaturated
        expr: llmgw_pool_saturation > 1
        for: 5m
        labels: { severity: warning }
        annotations:
          summary: "{{ $labels.model }} is over its healthy limit"
          description: >-
            Saturation is the worse of queue depth and KV occupancy; 1 is the
            limit measured as healthy, not the point of failure. Sustained above
            it means requests are queueing — add a replica or lower the load.

      - alert: EndpointBreakerOpen
        expr: llmgw_endpoint_breaker_open == 1
        for: 2m
        labels: { severity: warning }
        annotations:
          summary: "{{ $labels.replica }} has been taken out of rotation"
          description: >-
            The router stopped sending to this replica after repeated failures.
            Traffic is going elsewhere; if it is the only replica, the model is
            down.

      - alert: EndpointMetricsStale
        expr: llmgw_endpoint_metrics_stale == 1
        for: 5m
        labels: { severity: warning }
        annotations:
          summary: "{{ $labels.replica }} is being routed to blind"
          description: >-
            Its queue depth and KV usage are old, so the scheduler is choosing
            with stale numbers. The replica may be serving perfectly well; the
            decisions about it are the problem.

      - alert: NodeDiskFilling
        expr: llmgw_node_disk_free_bytes / llmgw_node_disk_total_bytes < 0.10
        for: 15m
        labels: { severity: warning }
        annotations:
          summary: "{{ $labels.node }} has {{ $value | humanizePercentage }} of its model disk free"
          description: >-
            Weights are tens of gigabytes. A node that cannot hold the next one
            is skipped by placement rather than failing, so this is quiet until
            somebody wonders why a model will not land there.

  - name: monitoring-host
    rules:
      - alert: MonitoringDiskFilling
        expr: >-
          node_filesystem_avail_bytes{job="monitoring-host",fstype!~"tmpfs|overlay|squashfs"}
          / node_filesystem_size_bytes{job="monitoring-host",fstype!~"tmpfs|overlay|squashfs"} < 0.15
        for: 10m
        labels: { severity: warning }
        annotations:
          summary: "{{ $labels.mountpoint }} on the monitoring host is {{ $value | humanizePercentage }} free"
          description: >-
            This filesystem holds the model mirror and the metrics database at
            once. Filling it stops new weights arriving and stops Prometheus
            writing — so monitoring goes quiet at the moment it is most needed,
            which is why this fires at fifteen percent rather than five.

      - alert: MonitoringDiskAlmostFull
        expr: >-
          node_filesystem_avail_bytes{job="monitoring-host",fstype!~"tmpfs|overlay|squashfs"}
          / node_filesystem_size_bytes{job="monitoring-host",fstype!~"tmpfs|overlay|squashfs"} < 0.05
        for: 5m
        labels: { severity: critical }
        annotations:
          summary: "{{ $labels.mountpoint }} on the monitoring host has {{ $value | humanizePercentage }} left"

  - name: llmgw-traffic
    rules:
      # These need traffic through the gateway to have any series at all. A
      # counter with no observations publishes nothing, so a quiet cluster
      # simply has no data here rather than a rule that is always false.
      - alert: HighErrorRate
        expr: >-
          sum by (model) (rate(llmgw_requests_total{status=~"5.."}[5m]))
          / sum by (model) (rate(llmgw_requests_total[5m])) > 0.05
        for: 5m
        labels: { severity: critical }
        annotations:
          summary: "{{ $labels.model }} is failing {{ $value | humanizePercentage }} of requests"

      - alert: SlowFirstToken
        expr: >-
          histogram_quantile(0.99,
            sum by (model, le) (rate(llmgw_ttft_seconds_bucket[10m]))) > 30
        for: 10m
        labels: { severity: warning }
        annotations:
          summary: "{{ $labels.model }} takes {{ $value | humanizeDuration }} to first token at p99"
          description: >-
            On CPU this is expected under load and is why the threshold is
            thirty seconds rather than one. Compare with what ctl bench measured
            before treating it as a fault.

      - alert: RequestsBeingRejected
        expr: sum by (band) (rate(llmgw_admission_rejected_total[5m])) > 0
        for: 5m
        labels: { severity: warning }
        annotations:
          summary: "Admission is turning away {{ $labels.band }} traffic"
          description: >-
            Working as designed when the cluster is full: the lowest band is
            held first. Worth knowing about because the callers being refused
            do not know why.