Ilker ManapandClaude Opus 5.5
fc60481e40
140 named 150 a store member, 150 named itself a control plane: Prometheus asking both scraped 150 twice and counted its every series twice, and asked of 140 alone while 150 led it lost every node. Each control plane now lists all of them as control planes before the store's members, so two answers agree and a target is scraped once. The shipped prometheus.yml says to scrape and ask every control plane. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
87 lines
2.7 KiB
Go
87 lines
2.7 KiB
Go
package adminapi
|
|
|
|
import (
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
)
|
|
|
|
// Prometheus service discovery: the machines of the cluster as scrape targets
|
|
// for their node exporters, which say what llmgw does not — network and disk
|
|
// traffic, filesystems, every CPU's time. Served by the control plane, which
|
|
// knows the fleet, so a node added is a node scraped with nothing edited; and
|
|
// labelled with the node's id in llmgw, so its charts can be put beside the
|
|
// cluster's own.
|
|
|
|
// NodeExporterPort is where prometheus-node-exporter listens.
|
|
const NodeExporterPort = "9100"
|
|
|
|
// SDGroup is one target group of Prometheus' HTTP service discovery.
|
|
type SDGroup struct {
|
|
Targets []string `json:"targets"`
|
|
Labels map[string]string `json:"labels"`
|
|
}
|
|
|
|
// NodeExporterTargets is every machine of the cluster: the nodes, this
|
|
// control plane, as the request reached it, and the store's members.
|
|
func (a *API) NodeExporterTargets(r *http.Request) []SDGroup {
|
|
return a.nodeExporterTargetsWith(r, a.Nodes())
|
|
}
|
|
|
|
func (a *API) nodeExporterTargetsWith(r *http.Request, nodes []NodeView) []SDGroup {
|
|
seen := map[string]bool{}
|
|
var out []SDGroup
|
|
add := func(host, node, role string) {
|
|
if host == "" || seen[host] {
|
|
return
|
|
}
|
|
seen[host] = true
|
|
out = append(out, SDGroup{Targets: []string{net.JoinHostPort(host, NodeExporterPort)},
|
|
Labels: map[string]string{"node": node, "role": role}})
|
|
}
|
|
if host, _, err := net.SplitHostPort(r.Host); err == nil {
|
|
name, _ := os.Hostname()
|
|
add(host, name, "control-plane")
|
|
}
|
|
// Every control plane, not this one only, before the store's members:
|
|
// Prometheus asks each of them, and a machine that one called a control
|
|
// plane and another a store member was two targets, its every series
|
|
// counted twice.
|
|
if a.controlPlanes != nil {
|
|
for _, cp := range a.controlPlanes().Members {
|
|
host := cp.Address
|
|
if h, _, err := net.SplitHostPort(host); err == nil {
|
|
host = h
|
|
}
|
|
add(strings.TrimSpace(host), cp.Name, "control-plane")
|
|
}
|
|
}
|
|
for _, n := range nodes {
|
|
host := n.MgmtAddr
|
|
if h, _, err := net.SplitHostPort(host); err == nil {
|
|
host = h
|
|
}
|
|
add(host, n.ID, "node")
|
|
}
|
|
if a.storeStatus != nil {
|
|
if members, err := a.storeStatus(); err == nil {
|
|
for _, m := range members {
|
|
host := m.Host
|
|
if h, _, err := net.SplitHostPort(host); err == nil {
|
|
host = h
|
|
}
|
|
add(strings.TrimSpace(host), m.Name, "store")
|
|
}
|
|
}
|
|
}
|
|
sort.Slice(out, func(i, j int) bool { return out[i].Labels["node"] < out[j].Labels["node"] })
|
|
return out
|
|
}
|
|
|
|
// PromSD serves the targets as Prometheus' http_sd_configs reads them.
|
|
func (a *API) PromSD(w http.ResponseWriter, r *http.Request) {
|
|
writeJSON(w, http.StatusOK, a.NodeExporterTargets(r))
|
|
}
|