A btop-like monitor in the console, from Prometheus; node exporters discovered and installed

Monitor draws CPU, memory, GPUs, requests, tokens, latency, network and disk
as Braille charts over 5m/1h/24h/7d, then per node, card and model
sparklines; node and model pages have their own. History is read from
monitoring.prometheus through POST /api/v1/metrics/range, or without it
from what the console saw. The control plane serves the cluster's machines
for Prometheus http_sd at /prometheus/targets on its metrics port, and node
add installs prometheus-node-exporter.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

internal/control/adminapi/api.go

@@ -45,6 +45,8 @@ type API struct {
 	storeStatus func() ([]StoreMember, error)
 	// serve runs registry-to-replica jobs; nil without a mirror and a store.
 	serve *modelserve.Server
+	// history is the Prometheus the console's charts read from.
+	history History
 
 	models   Models
 	capacity Capacity
@@ -135,6 +137,7 @@ func (a *API) Register(mux *http.ServeMux) {
 	mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel)
 	mux.HandleFunc("POST /api/v1/models/{name}/origin", a.setOrigin)
 	mux.HandleFunc("GET /api/v1/plan", a.planModel)
+	mux.HandleFunc("POST /api/v1/metrics/range", a.metricsBatch)
 	mux.HandleFunc("GET /api/v1/requests/{id}", a.request)
 	mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel)
 	mux.HandleFunc("GET /api/v1/models/{name}/headroom", func(w http.ResponseWriter, r *http.Request) {

internal/control/adminapi/metrics.go

@@ -0,0 +1,89 @@
+package adminapi
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"sync"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
+)
+
+// History reads the cluster's metrics back from the Prometheus that scraped
+// them: what the console's charts draw.
+type History interface {
+	RangeQuery(ctx context.Context, expr, nameBy string, window, step time.Duration) ([]promq.Series, error)
+}
+
+// SetHistory supplies the Prometheus to read from; nil leaves the charts to
+// what the console saw itself.
+func (a *API) SetHistory(h History) { a.history = h }
+
+// MetricsQuery is one line set of a chart: a PromQL expression, and the label
+// naming each line.
+type MetricsQuery struct {
+	ID   string `json:"id"`
+	Expr string `json:"expr"`
+	By   string `json:"by,omitempty"`
+}
+
+// MetricsRequest asks for several queries over one window at once.
+type MetricsRequest struct {
+	Window  time.Duration  `json:"window"`
+	Step    time.Duration  `json:"step"`
+	Queries []MetricsQuery `json:"queries"`
+}
+
+// MetricsResult is each query's lines, by id; Configured is false when no
+// Prometheus is set, and Errors says which queries failed and why.
+type MetricsResult struct {
+	Configured bool                      `json:"configured"`
+	Series     map[string][]promq.Series `json:"series"`
+	Errors     map[string]string         `json:"errors,omitempty"`
+}
+
+// metricsBatch runs the queries side by side: a screen of charts is a dozen
+// of them, and one after another they would take as long as the slowest
+// dozen times.
+func (a *API) metricsBatch(w http.ResponseWriter, r *http.Request) {
+	var req MetricsRequest
+	if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
+		return
+	}
+	res := MetricsResult{Series: map[string][]promq.Series{}}
+	if a.history == nil || len(req.Queries) == 0 {
+		writeJSON(w, http.StatusOK, res)
+		return
+	}
+	res.Configured = true
+	if req.Window <= 0 {
+		req.Window = time.Hour
+	}
+	if len(req.Queries) > 40 {
+		req.Queries = req.Queries[:40]
+	}
+	var mu sync.Mutex
+	var wg sync.WaitGroup
+	for _, q := range req.Queries {
+		wg.Add(1)
+		go func(q MetricsQuery) {
+			defer wg.Done()
+			s, err := a.history.RangeQuery(r.Context(), q.Expr, q.By, req.Window, req.Step)
+			mu.Lock()
+			defer mu.Unlock()
+			if err != nil {
+				if res.Errors == nil {
+					res.Errors = map[string]string{}
+				}
+				res.Errors[q.ID] = err.Error()
+				return
+			}
+			res.Series[q.ID] = s
+		}(q)
+	}
+	wg.Wait()
+	writeJSON(w, http.StatusOK, res)
+}

internal/control/adminapi/promsd.go

@@ -0,0 +1,73 @@
+package adminapi
+
+import (
+	"net"
+	"net/http"
+	"os"
+	"sort"
+	"strings"
+)
+
+// Prometheus service discovery: the machines of the cluster as scrape targets
+// for their node exporters, which say what llmgw does not — network and disk
+// traffic, filesystems, every CPU's time. Served by the control plane, which
+// knows the fleet, so a node added is a node scraped with nothing edited; and
+// labelled with the node's id in llmgw, so its charts can be put beside the
+// cluster's own.
+
+// NodeExporterPort is where prometheus-node-exporter listens.
+const NodeExporterPort = "9100"
+
+// SDGroup is one target group of Prometheus' HTTP service discovery.
+type SDGroup struct {
+	Targets []string          `json:"targets"`
+	Labels  map[string]string `json:"labels"`
+}
+
+// NodeExporterTargets is every machine of the cluster: the nodes, this
+// control plane, as the request reached it, and the store's members.
+func (a *API) NodeExporterTargets(r *http.Request) []SDGroup {
+	return a.nodeExporterTargetsWith(r, a.Nodes())
+}
+
+func (a *API) nodeExporterTargetsWith(r *http.Request, nodes []NodeView) []SDGroup {
+	seen := map[string]bool{}
+	var out []SDGroup
+	add := func(host, node, role string) {
+		if host == "" || seen[host] {
+			return
+		}
+		seen[host] = true
+		out = append(out, SDGroup{Targets: []string{net.JoinHostPort(host, NodeExporterPort)},
+			Labels: map[string]string{"node": node, "role": role}})
+	}
+	if host, _, err := net.SplitHostPort(r.Host); err == nil {
+		name, _ := os.Hostname()
+		add(host, name, "control-plane")
+	}
+	for _, n := range nodes {
+		host := n.MgmtAddr
+		if h, _, err := net.SplitHostPort(host); err == nil {
+			host = h
+		}
+		add(host, n.ID, "node")
+	}
+	if a.storeStatus != nil {
+		if members, err := a.storeStatus(); err == nil {
+			for _, m := range members {
+				host := m.Host
+				if h, _, err := net.SplitHostPort(host); err == nil {
+					host = h
+				}
+				add(strings.TrimSpace(host), m.Name, "store")
+			}
+		}
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Labels["node"] < out[j].Labels["node"] })
+	return out
+}
+
+// PromSD serves the targets as Prometheus' http_sd_configs reads them.
+func (a *API) PromSD(w http.ResponseWriter, r *http.Request) {
+	writeJSON(w, http.StatusOK, a.NodeExporterTargets(r))
+}

internal/control/adminapi/promsd_test.go

@@ -0,0 +1,23 @@
+package adminapi
+
+import (
+	"net/http/httptest"
+	"testing"
+)
+
+func TestNodeExporterTargetsAreTheControlPlaneAndTheStore(t *testing.T) {
+	a := &API{}
+	a.SetStoreStatus(func() ([]StoreMember, error) {
+		return []StoreMember{{Name: "pg01", Host: "10.0.0.149"}, {Name: "ctl", Host: "10.0.0.140"}}, nil
+	})
+	r := httptest.NewRequest("GET", "http://10.0.0.140:9091/prometheus/targets", nil)
+	got := a.nodeExporterTargetsWith(r, nil)
+	if len(got) != 2 {
+		t.Fatalf("%+v", got)
+	}
+	for _, g := range got {
+		if g.Targets[0] == "10.0.0.140:9100" && g.Labels["role"] != "control-plane" {
+			t.Errorf("the control plane twice: %+v", got)
+		}
+	}
+}