package adminapi import ( "net" "net/http" "os" "sort" "strings" ) // Prometheus service discovery: the machines of the cluster as scrape targets // for their node exporters, which say what llmgw does not — network and disk // traffic, filesystems, every CPU's time. Served by the control plane, which // knows the fleet, so a node added is a node scraped with nothing edited; and // labelled with the node's id in llmgw, so its charts can be put beside the // cluster's own. // NodeExporterPort is where prometheus-node-exporter listens. const NodeExporterPort = "9100" // SDGroup is one target group of Prometheus' HTTP service discovery. type SDGroup struct { Targets []string `json:"targets"` Labels map[string]string `json:"labels"` } // NodeExporterTargets is every machine of the cluster: the nodes, this // control plane, as the request reached it, and the store's members. func (a *API) NodeExporterTargets(r *http.Request) []SDGroup { return a.nodeExporterTargetsWith(r, a.Nodes()) } func (a *API) nodeExporterTargetsWith(r *http.Request, nodes []NodeView) []SDGroup { seen := map[string]bool{} var out []SDGroup add := func(host, node, role string) { if host == "" || seen[host] { return } seen[host] = true out = append(out, SDGroup{Targets: []string{net.JoinHostPort(host, NodeExporterPort)}, Labels: map[string]string{"node": node, "role": role}}) } if host, _, err := net.SplitHostPort(r.Host); err == nil { name, _ := os.Hostname() add(host, name, "control-plane") } // Every control plane, not this one only, before the store's members: // Prometheus asks each of them, and a machine that one called a control // plane and another a store member was two targets, its every series // counted twice. if a.controlPlanes != nil { for _, cp := range a.controlPlanes().Members { host := cp.Address if h, _, err := net.SplitHostPort(host); err == nil { host = h } add(strings.TrimSpace(host), cp.Name, "control-plane") } } for _, n := range nodes { host := n.MgmtAddr if h, _, err := net.SplitHostPort(host); err == nil { host = h } add(host, n.ID, "node") } if a.storeStatus != nil { if members, err := a.storeStatus(); err == nil { for _, m := range members { host := m.Host if h, _, err := net.SplitHostPort(host); err == nil { host = h } add(strings.TrimSpace(host), m.Name, "store") } } } sort.Slice(out, func(i, j int) bool { return out[i].Labels["node"] < out[j].Labels["node"] }) return out } // PromSD serves the targets as Prometheus' http_sd_configs reads them. func (a *API) PromSD(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, a.NodeExporterTargets(r)) }