A btop-like monitor in the console, from Prometheus; node exporters discovered and installed
Monitor draws CPU, memory, GPUs, requests, tokens, latency, network and disk as Braille charts over 5m/1h/24h/7d, then per node, card and model sparklines; node and model pages have their own. History is read from monitoring.prometheus through POST /api/v1/metrics/range, or without it from what the console saw. The control plane serves the cluster's machines for Prometheus http_sd at /prometheus/targets on its metrics port, and node add installs prometheus-node-exporter. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
2e55487eca
commit
a1b1663e9d
16 files changed
+1217
-4
No files matched your search
@@ -0,0 +1,73 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Prometheus service discovery: the machines of the cluster as scrape targets
|
||||
// for their node exporters, which say what llmgw does not — network and disk
|
||||
// traffic, filesystems, every CPU's time. Served by the control plane, which
|
||||
// knows the fleet, so a node added is a node scraped with nothing edited; and
|
||||
// labelled with the node's id in llmgw, so its charts can be put beside the
|
||||
// cluster's own.
|
||||
|
||||
// NodeExporterPort is where prometheus-node-exporter listens.
|
||||
const NodeExporterPort = "9100"
|
||||
|
||||
// SDGroup is one target group of Prometheus' HTTP service discovery.
|
||||
type SDGroup struct {
|
||||
Targets []string `json:"targets"`
|
||||
Labels map[string]string `json:"labels"`
|
||||
}
|
||||
|
||||
// NodeExporterTargets is every machine of the cluster: the nodes, this
|
||||
// control plane, as the request reached it, and the store's members.
|
||||
func (a *API) NodeExporterTargets(r *http.Request) []SDGroup {
|
||||
return a.nodeExporterTargetsWith(r, a.Nodes())
|
||||
}
|
||||
|
||||
func (a *API) nodeExporterTargetsWith(r *http.Request, nodes []NodeView) []SDGroup {
|
||||
seen := map[string]bool{}
|
||||
var out []SDGroup
|
||||
add := func(host, node, role string) {
|
||||
if host == "" || seen[host] {
|
||||
return
|
||||
}
|
||||
seen[host] = true
|
||||
out = append(out, SDGroup{Targets: []string{net.JoinHostPort(host, NodeExporterPort)},
|
||||
Labels: map[string]string{"node": node, "role": role}})
|
||||
}
|
||||
if host, _, err := net.SplitHostPort(r.Host); err == nil {
|
||||
name, _ := os.Hostname()
|
||||
add(host, name, "control-plane")
|
||||
}
|
||||
for _, n := range nodes {
|
||||
host := n.MgmtAddr
|
||||
if h, _, err := net.SplitHostPort(host); err == nil {
|
||||
host = h
|
||||
}
|
||||
add(host, n.ID, "node")
|
||||
}
|
||||
if a.storeStatus != nil {
|
||||
if members, err := a.storeStatus(); err == nil {
|
||||
for _, m := range members {
|
||||
host := m.Host
|
||||
if h, _, err := net.SplitHostPort(host); err == nil {
|
||||
host = h
|
||||
}
|
||||
add(strings.TrimSpace(host), m.Name, "store")
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Labels["node"] < out[j].Labels["node"] })
|
||||
return out
|
||||
}
|
||||
|
||||
// PromSD serves the targets as Prometheus' http_sd_configs reads them.
|
||||
func (a *API) PromSD(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, a.NodeExporterTargets(r))
|
||||
}
|
||||
Reference in new issue
Block a user