Ilker ManapandClaude Opus 5.5
a1b1663e9d
Monitor draws CPU, memory, GPUs, requests, tokens, latency, network and disk as Braille charts over 5m/1h/24h/7d, then per node, card and model sparklines; node and model pages have their own. History is read from monitoring.prometheus through POST /api/v1/metrics/range, or without it from what the console saw. The control plane serves the cluster's machines for Prometheus http_sd at /prometheus/targets on its metrics port, and node add installs prometheus-node-exporter. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
74 lines
2.2 KiB
Go
74 lines
2.2 KiB
Go
package adminapi
|
|
|
|
import (
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
)
|
|
|
|
// Prometheus service discovery: the machines of the cluster as scrape targets
|
|
// for their node exporters, which say what llmgw does not — network and disk
|
|
// traffic, filesystems, every CPU's time. Served by the control plane, which
|
|
// knows the fleet, so a node added is a node scraped with nothing edited; and
|
|
// labelled with the node's id in llmgw, so its charts can be put beside the
|
|
// cluster's own.
|
|
|
|
// NodeExporterPort is where prometheus-node-exporter listens.
|
|
const NodeExporterPort = "9100"
|
|
|
|
// SDGroup is one target group of Prometheus' HTTP service discovery.
|
|
type SDGroup struct {
|
|
Targets []string `json:"targets"`
|
|
Labels map[string]string `json:"labels"`
|
|
}
|
|
|
|
// NodeExporterTargets is every machine of the cluster: the nodes, this
|
|
// control plane, as the request reached it, and the store's members.
|
|
func (a *API) NodeExporterTargets(r *http.Request) []SDGroup {
|
|
return a.nodeExporterTargetsWith(r, a.Nodes())
|
|
}
|
|
|
|
func (a *API) nodeExporterTargetsWith(r *http.Request, nodes []NodeView) []SDGroup {
|
|
seen := map[string]bool{}
|
|
var out []SDGroup
|
|
add := func(host, node, role string) {
|
|
if host == "" || seen[host] {
|
|
return
|
|
}
|
|
seen[host] = true
|
|
out = append(out, SDGroup{Targets: []string{net.JoinHostPort(host, NodeExporterPort)},
|
|
Labels: map[string]string{"node": node, "role": role}})
|
|
}
|
|
if host, _, err := net.SplitHostPort(r.Host); err == nil {
|
|
name, _ := os.Hostname()
|
|
add(host, name, "control-plane")
|
|
}
|
|
for _, n := range nodes {
|
|
host := n.MgmtAddr
|
|
if h, _, err := net.SplitHostPort(host); err == nil {
|
|
host = h
|
|
}
|
|
add(host, n.ID, "node")
|
|
}
|
|
if a.storeStatus != nil {
|
|
if members, err := a.storeStatus(); err == nil {
|
|
for _, m := range members {
|
|
host := m.Host
|
|
if h, _, err := net.SplitHostPort(host); err == nil {
|
|
host = h
|
|
}
|
|
add(strings.TrimSpace(host), m.Name, "store")
|
|
}
|
|
}
|
|
}
|
|
sort.Slice(out, func(i, j int) bool { return out[i].Labels["node"] < out[j].Labels["node"] })
|
|
return out
|
|
}
|
|
|
|
// PromSD serves the targets as Prometheus' http_sd_configs reads them.
|
|
func (a *API) PromSD(w http.ResponseWriter, r *http.Request) {
|
|
writeJSON(w, http.StatusOK, a.NodeExporterTargets(r))
|
|
}
|