From fc60481e40ce909e854f45a6eb71bc4e7ba8bf86 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 15 Oct 2026 07:53:22 +0200 Subject: [PATCH] Every control plane lists the others alike to Prometheus 140 named 150 a store member, 150 named itself a control plane: Prometheus asking both scraped 150 twice and counted its every series twice, and asked of 140 alone while 150 led it lost every node. Each control plane now lists all of them as control planes before the store's members, so two answers agree and a target is scraped once. The shipped prometheus.yml says to scrape and ask every control plane. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 5 +++++ deploy/monitoring/prometheus.yml | 14 ++++++++++++- internal/control/adminapi/promsd.go | 13 ++++++++++++ internal/control/adminapi/promsd_test.go | 26 ++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df11630f..b7bfaa9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,11 @@ APIs change between minors without a deprecation period. ### Fixed +- Node discovery for Prometheus listed only the control plane asked and the + store, so a machine that is both was a control plane to one and a store + member to the other — scraped twice — and a standby asked alone lost every + node. Every control plane lists all of them alike; deploy/monitoring's + prometheus.yml says to scrape and ask each. - A model whose weights and context pool did not fit its card was added and then failed to make its context on every start, taking the card from others each time (Apertus 70B with a 65536-token pool, 8B with 131072, on diff --git a/deploy/monitoring/prometheus.yml b/deploy/monitoring/prometheus.yml index d47be64b..c150e893 100644 --- a/deploy/monitoring/prometheus.yml +++ b/deploy/monitoring/prometheus.yml @@ -21,11 +21,18 @@ scrape_configs: # The control plane's dedicated metrics port. Not its admin listener: that one # is loopback because it also serves pprof, and pprof can dump the heap where # API keys sit in plaintext. + # + # Every control plane, one line each: the leader moves on restarts and the + # router's metrics are the leading one's, so scraping one alone leaves gaps + # — and alerts on them — whenever another leads. - job_name: llmgw static_configs: - targets: ['10.0.0.140:9091'] labels: - instance: control-plane + instance: control-plane-140 + # - targets: ['10.0.0.150:9091'] + # labels: + # instance: control-plane-150 # Every machine of the cluster, for its node exporter: network and disk # traffic, filesystems, CPU time. The control plane lists them — nodes, @@ -33,10 +40,15 @@ scrape_configs: # nothing edited here, and each carries its llmgw id as the node label. # `llmgw node add` installs the exporter; on a machine added before that, # apt-get install prometheus-node-exporter. + # Asked of every control plane too: a standby lists only the control + # planes and the store, and they label every machine alike, so the same + # target from two of them is scraped once. - job_name: nodes http_sd_configs: - url: http://10.0.0.140:9091/prometheus/targets refresh_interval: 1m + # - url: http://10.0.0.150:9091/prometheus/targets + # refresh_interval: 1m - job_name: prometheus static_configs: diff --git a/internal/control/adminapi/promsd.go b/internal/control/adminapi/promsd.go index 4edfdaf3..86a766e0 100644 --- a/internal/control/adminapi/promsd.go +++ b/internal/control/adminapi/promsd.go @@ -45,6 +45,19 @@ func (a *API) nodeExporterTargetsWith(r *http.Request, nodes []NodeView) []SDGro name, _ := os.Hostname() add(host, name, "control-plane") } + // Every control plane, not this one only, before the store's members: + // Prometheus asks each of them, and a machine that one called a control + // plane and another a store member was two targets, its every series + // counted twice. + if a.controlPlanes != nil { + for _, cp := range a.controlPlanes().Members { + host := cp.Address + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + add(strings.TrimSpace(host), cp.Name, "control-plane") + } + } for _, n := range nodes { host := n.MgmtAddr if h, _, err := net.SplitHostPort(host); err == nil { diff --git a/internal/control/adminapi/promsd_test.go b/internal/control/adminapi/promsd_test.go index 9ac6e799..7999c747 100644 --- a/internal/control/adminapi/promsd_test.go +++ b/internal/control/adminapi/promsd_test.go @@ -21,3 +21,29 @@ func TestNodeExporterTargetsAreTheControlPlaneAndTheStore(t *testing.T) { } } } + +func TestEveryControlPlaneListsTheOthersAlike(t *testing.T) { + // 140 called 150 a store member and 150 called itself a control plane: + // Prometheus, asking both, scraped 150 twice and counted it twice. + targets := func(self string) map[string]string { + a := &API{} + a.SetStoreStatus(func() ([]StoreMember, error) { + return []StoreMember{{Name: "llmgw-pg01", Host: "10.0.0.149"}, {Name: "llmgw-ctl02", Host: "10.0.0.150"}}, nil + }) + a.SetControlPlanes(func() ControlPlanes { + return ControlPlanes{Members: []ControlPlane{{Name: "llmgw-ctl", Address: "10.0.0.140:7443"}, + {Name: "llmgw-ctl02", Address: "10.0.0.150:7443"}}} + }) + r := httptest.NewRequest("GET", "http://"+self+":9091/prometheus/targets", nil) + out := map[string]string{} + for _, g := range a.nodeExporterTargetsWith(r, nil) { + out[g.Targets[0]] = g.Labels["role"] + } + return out + } + from140, from150 := targets("10.0.0.140"), targets("10.0.0.150") + if from140["10.0.0.150:9100"] != "control-plane" || from150["10.0.0.140:9100"] != "control-plane" || + from140["10.0.0.149:9100"] != "store" { + t.Errorf("from 140 %v, from 150 %v", from140, from150) + } +}