Every control plane lists the others alike to Prometheus

140 named 150 a store member, 150 named itself a control plane: Prometheus
asking both scraped 150 twice and counted its every series twice, and asked
of 140 alone while 150 led it lost every node. Each control plane now lists
all of them as control planes before the store's members, so two answers
agree and a target is scraped once. The shipped prometheus.yml says to
scrape and ask every control plane.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-15 07:53:22 +02:00
1 parent c22528c26e
commit fc60481e40
4 files changed
+57 -1

No files matched your search

+13
View File
@@ -45,6 +45,19 @@ func (a *API) nodeExporterTargetsWith(r *http.Request, nodes []NodeView) []SDGro
name, _ := os.Hostname()
add(host, name, "control-plane")
}
// Every control plane, not this one only, before the store's members:
// Prometheus asks each of them, and a machine that one called a control
// plane and another a store member was two targets, its every series
// counted twice.
if a.controlPlanes != nil {
for _, cp := range a.controlPlanes().Members {
host := cp.Address
if h, _, err := net.SplitHostPort(host); err == nil {
host = h
}
add(strings.TrimSpace(host), cp.Name, "control-plane")
}
}
for _, n := range nodes {
host := n.MgmtAddr
if h, _, err := net.SplitHostPort(host); err == nil {
+26
View File
@@ -21,3 +21,29 @@ func TestNodeExporterTargetsAreTheControlPlaneAndTheStore(t *testing.T) {
}
}
}
func TestEveryControlPlaneListsTheOthersAlike(t *testing.T) {
// 140 called 150 a store member and 150 called itself a control plane:
// Prometheus, asking both, scraped 150 twice and counted it twice.
targets := func(self string) map[string]string {
a := &API{}
a.SetStoreStatus(func() ([]StoreMember, error) {
return []StoreMember{{Name: "llmgw-pg01", Host: "10.0.0.149"}, {Name: "llmgw-ctl02", Host: "10.0.0.150"}}, nil
})
a.SetControlPlanes(func() ControlPlanes {
return ControlPlanes{Members: []ControlPlane{{Name: "llmgw-ctl", Address: "10.0.0.140:7443"},
{Name: "llmgw-ctl02", Address: "10.0.0.150:7443"}}}
})
r := httptest.NewRequest("GET", "http://"+self+":9091/prometheus/targets", nil)
out := map[string]string{}
for _, g := range a.nodeExporterTargetsWith(r, nil) {
out[g.Targets[0]] = g.Labels["role"]
}
return out
}
from140, from150 := targets("10.0.0.140"), targets("10.0.0.150")
if from140["10.0.0.150:9100"] != "control-plane" || from150["10.0.0.140:9100"] != "control-plane" ||
from140["10.0.0.149:9100"] != "store" {
t.Errorf("from 140 %v, from 150 %v", from140, from150)
}
}