# Scraping llmgw.
#
# Point this at metrics_listen, not admin_listen. They are different ports for
# a reason: the admin listener also carries pprof, which can dump the process
# heap, and API keys are in that heap in plaintext.
#
# In the control plane's configuration:
#
#   admin_listen:   "127.0.0.1:9090"   # dashboard, management API, pprof
#   metrics_listen: "0.0.0.0:9091"     # metrics only, safe to reach
#
# One target for the whole fleet. Node and GPU figures are exported by the
# control plane, which already holds them — agents report hardware at
# registration and state on every heartbeat — so there is nothing to scrape on
# the nodes and no Prometheus configuration per machine.
global:
  scrape_interval: 15s

scrape_configs:
  - job_name: llmgw
    static_configs:
      - targets: ["llmgw-ctl.internal:9091"]
        labels:
          cluster: primary

# Worth alerting on, in rough order of how often it matters.
#
#   A node that has stopped reporting:
#     llmgw_node_last_seen_seconds > 60
#
#   A model that should be serving and is not. Covers a replica that will never
#   start as well as one that has died, which look the same from a distance:
#     llmgw_model_replicas_ready == 0
#
#   A card kept out of placement by a fault, which is otherwise invisible until
#   a model will not fit:
#     llmgw_gpu_healthy == 0
#
#   A fleet holding models nobody is asking for. Memory used says a model is
#   loaded; utilisation says whether it is doing anything:
#     avg_over_time(llmgw_gpu_utilization_percent[1h]) < 5
#
#   Requests queueing because the cluster is full, which is the pair of figures
#   that needed two systems to correlate before these were on one endpoint:
#     llmgw_admission_queue_depth > 0