Every control plane lists the others alike to Prometheus

140 named 150 a store member, 150 named itself a control plane: Prometheus
asking both scraped 150 twice and counted its every series twice, and asked
of 140 alone while 150 led it lost every node. Each control plane now lists
all of them as control planes before the store's members, so two answers
agree and a target is scraped once. The shipped prometheus.yml says to
scrape and ask every control plane.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

CHANGELOG.md

@@ -67,6 +67,11 @@ APIs change between minors without a deprecation period.
 
 ### Fixed
 
+- Node discovery for Prometheus listed only the control plane asked and the
+  store, so a machine that is both was a control plane to one and a store
+  member to the other — scraped twice — and a standby asked alone lost every
+  node. Every control plane lists all of them alike; deploy/monitoring's
+  prometheus.yml says to scrape and ask each.
 - A model whose weights and context pool did not fit its card was added
   and then failed to make its context on every start, taking the card from
   others each time (Apertus 70B with a 65536-token pool, 8B with 131072, on

deploy/monitoring/prometheus.yml

@@ -21,11 +21,18 @@ scrape_configs:
   # The control plane's dedicated metrics port. Not its admin listener: that one
   # is loopback because it also serves pprof, and pprof can dump the heap where
   # API keys sit in plaintext.
+  #
+  # Every control plane, one line each: the leader moves on restarts and the
+  # router's metrics are the leading one's, so scraping one alone leaves gaps
+  # — and alerts on them — whenever another leads.
   - job_name: llmgw
     static_configs:
       - targets: ['10.0.0.140:9091']
         labels:
-          instance: control-plane
+          instance: control-plane-140
+      # - targets: ['10.0.0.150:9091']
+      #   labels:
+      #     instance: control-plane-150
 
   # Every machine of the cluster, for its node exporter: network and disk
   # traffic, filesystems, CPU time. The control plane lists them — nodes,
@@ -33,10 +40,15 @@ scrape_configs:
   # nothing edited here, and each carries its llmgw id as the node label.
   # `llmgw node add` installs the exporter; on a machine added before that,
   # apt-get install prometheus-node-exporter.
+  # Asked of every control plane too: a standby lists only the control
+  # planes and the store, and they label every machine alike, so the same
+  # target from two of them is scraped once.
   - job_name: nodes
     http_sd_configs:
       - url: http://10.0.0.140:9091/prometheus/targets
         refresh_interval: 1m
+      # - url: http://10.0.0.150:9091/prometheus/targets
+      #   refresh_interval: 1m
 
   - job_name: prometheus
     static_configs:

internal/control/adminapi/promsd.go

@@ -45,6 +45,19 @@ func (a *API) nodeExporterTargetsWith(r *http.Request, nodes []NodeView) []SDGro
 		name, _ := os.Hostname()
 		add(host, name, "control-plane")
 	}
+	// Every control plane, not this one only, before the store's members:
+	// Prometheus asks each of them, and a machine that one called a control
+	// plane and another a store member was two targets, its every series
+	// counted twice.
+	if a.controlPlanes != nil {
+		for _, cp := range a.controlPlanes().Members {
+			host := cp.Address
+			if h, _, err := net.SplitHostPort(host); err == nil {
+				host = h
+			}
+			add(strings.TrimSpace(host), cp.Name, "control-plane")
+		}
+	}
 	for _, n := range nodes {
 		host := n.MgmtAddr
 		if h, _, err := net.SplitHostPort(host); err == nil {

internal/control/adminapi/promsd_test.go

@@ -21,3 +21,29 @@ func TestNodeExporterTargetsAreTheControlPlaneAndTheStore(t *testing.T) {
 		}
 	}
 }
+
+func TestEveryControlPlaneListsTheOthersAlike(t *testing.T) {
+	// 140 called 150 a store member and 150 called itself a control plane:
+	// Prometheus, asking both, scraped 150 twice and counted it twice.
+	targets := func(self string) map[string]string {
+		a := &API{}
+		a.SetStoreStatus(func() ([]StoreMember, error) {
+			return []StoreMember{{Name: "llmgw-pg01", Host: "10.0.0.149"}, {Name: "llmgw-ctl02", Host: "10.0.0.150"}}, nil
+		})
+		a.SetControlPlanes(func() ControlPlanes {
+			return ControlPlanes{Members: []ControlPlane{{Name: "llmgw-ctl", Address: "10.0.0.140:7443"},
+				{Name: "llmgw-ctl02", Address: "10.0.0.150:7443"}}}
+		})
+		r := httptest.NewRequest("GET", "http://"+self+":9091/prometheus/targets", nil)
+		out := map[string]string{}
+		for _, g := range a.nodeExporterTargetsWith(r, nil) {
+			out[g.Targets[0]] = g.Labels["role"]
+		}
+		return out
+	}
+	from140, from150 := targets("10.0.0.140"), targets("10.0.0.150")
+	if from140["10.0.0.150:9100"] != "control-plane" || from150["10.0.0.140:9100"] != "control-plane" ||
+		from140["10.0.0.149:9100"] != "store" {
+		t.Errorf("from 140 %v, from 150 %v", from140, from150)
+	}
+}