# Prometheus for the llmgw cluster.
#
# One scrape target for the whole fleet. Agents publish nothing of their own —
# the control plane gathers what they report and republishes it, so node health,
# disk and endpoint state all arrive here from 10.0.0.140.
global:
  scrape_interval: 15s
  evaluation_interval: 15s
  external_labels:
    cluster: mynodes

alerting:
  alertmanagers:
    - static_configs:
        - targets: ['localhost:9093']

rule_files:
  - /etc/prometheus/rules/llmgw.yml

scrape_configs:
  # The control plane's dedicated metrics port. Not its admin listener: that one
  # is loopback because it also serves pprof, and pprof can dump the heap where
  # API keys sit in plaintext.
  #
  # Every control plane, one line each: the leader moves on restarts and the
  # router's metrics are the leading one's, so scraping one alone leaves gaps
  # — and alerts on them — whenever another leads.
  - job_name: llmgw
    static_configs:
      - targets: ['10.0.0.140:9091']
        labels:
          instance: control-plane-140
      # - targets: ['10.0.0.150:9091']
      #   labels:
      #     instance: control-plane-150

  # Every machine of the cluster, for its node exporter: network and disk
  # traffic, filesystems, CPU time. The control plane lists them — nodes,
  # itself, the store's members — so a node added is a node scraped with
  # nothing edited here, and each carries its llmgw id as the node label.
  # `llmgw node add` installs the exporter; on a machine added before that,
  # apt-get install prometheus-node-exporter.
  # Asked of every control plane too: a standby lists only the control
  # planes and the store, and they label every machine alike, so the same
  # target from two of them is scraped once.
  - job_name: nodes
    http_sd_configs:
      - url: http://10.0.0.140:9091/prometheus/targets
        refresh_interval: 1m
      # - url: http://10.0.0.150:9091/prometheus/targets
      #   refresh_interval: 1m

  - job_name: prometheus
    static_configs:
      - targets: ['localhost:9090']

  # The machine this runs on. It came with the Prometheus package and was not
  # being scraped, which left the disk holding both the model mirror and the
  # metrics themselves watched by nothing — so a mirror that filled it would
  # take monitoring down and the alert that would have warned did not exist.
  - job_name: monitoring-host
    static_configs:
      - targets: ['localhost:9100']
        labels:
          instance: monitoring