Every control plane lists the others alike to Prometheus
140 named 150 a store member, 150 named itself a control plane: Prometheus asking both scraped 150 twice and counted its every series twice, and asked of 140 alone while 150 led it lost every node. Each control plane now lists all of them as control planes before the store's members, so two answers agree and a target is scraped once. The shipped prometheus.yml says to scrape and ask every control plane. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
c22528c26e
commit
fc60481e40
4 files changed
+57
-1
No files matched your search
@@ -45,6 +45,19 @@ func (a *API) nodeExporterTargetsWith(r *http.Request, nodes []NodeView) []SDGro
|
||||
name, _ := os.Hostname()
|
||||
add(host, name, "control-plane")
|
||||
}
|
||||
// Every control plane, not this one only, before the store's members:
|
||||
// Prometheus asks each of them, and a machine that one called a control
|
||||
// plane and another a store member was two targets, its every series
|
||||
// counted twice.
|
||||
if a.controlPlanes != nil {
|
||||
for _, cp := range a.controlPlanes().Members {
|
||||
host := cp.Address
|
||||
if h, _, err := net.SplitHostPort(host); err == nil {
|
||||
host = h
|
||||
}
|
||||
add(strings.TrimSpace(host), cp.Name, "control-plane")
|
||||
}
|
||||
}
|
||||
for _, n := range nodes {
|
||||
host := n.MgmtAddr
|
||||
if h, _, err := net.SplitHostPort(host); err == nil {
|
||||
|
||||
@@ -21,3 +21,29 @@ func TestNodeExporterTargetsAreTheControlPlaneAndTheStore(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryControlPlaneListsTheOthersAlike(t *testing.T) {
|
||||
// 140 called 150 a store member and 150 called itself a control plane:
|
||||
// Prometheus, asking both, scraped 150 twice and counted it twice.
|
||||
targets := func(self string) map[string]string {
|
||||
a := &API{}
|
||||
a.SetStoreStatus(func() ([]StoreMember, error) {
|
||||
return []StoreMember{{Name: "llmgw-pg01", Host: "10.0.0.149"}, {Name: "llmgw-ctl02", Host: "10.0.0.150"}}, nil
|
||||
})
|
||||
a.SetControlPlanes(func() ControlPlanes {
|
||||
return ControlPlanes{Members: []ControlPlane{{Name: "llmgw-ctl", Address: "10.0.0.140:7443"},
|
||||
{Name: "llmgw-ctl02", Address: "10.0.0.150:7443"}}}
|
||||
})
|
||||
r := httptest.NewRequest("GET", "http://"+self+":9091/prometheus/targets", nil)
|
||||
out := map[string]string{}
|
||||
for _, g := range a.nodeExporterTargetsWith(r, nil) {
|
||||
out[g.Targets[0]] = g.Labels["role"]
|
||||
}
|
||||
return out
|
||||
}
|
||||
from140, from150 := targets("10.0.0.140"), targets("10.0.0.150")
|
||||
if from140["10.0.0.150:9100"] != "control-plane" || from150["10.0.0.140:9100"] != "control-plane" ||
|
||||
from140["10.0.0.149:9100"] != "store" {
|
||||
t.Errorf("from 140 %v, from 150 %v", from140, from150)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user