A btop-like monitor in the console, from Prometheus; node exporters discovered and installed

Monitor draws CPU, memory, GPUs, requests, tokens, latency, network and disk
as Braille charts over 5m/1h/24h/7d, then per node, card and model
sparklines; node and model pages have their own. History is read from
monitoring.prometheus through POST /api/v1/metrics/range, or without it
from what the console saw. The control plane serves the cluster's machines
for Prometheus http_sd at /prometheus/targets on its metrics port, and node
add installs prometheus-node-exporter.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 01:51:05 +02:00
1 parent 2e55487eca
commit a1b1663e9d
16 files changed
+1217 -4

No files matched your search

+35
View File
@@ -60,6 +60,9 @@ func ParseFabric(s string) (Fabric, error) {
// Options say which machine, and what to tell its agent.
type Options struct {
// NoNodeExporter leaves Prometheus' node exporter off the machine.
NoNodeExporter bool
// Addr is how to reach the machine over SSH, and User who to log in as.
Addr, User string
// Node is its id in the cluster. Empty means its hostname, which is what
@@ -673,9 +676,41 @@ systemctl is-active llmgw-agent`, nil); err != nil {
return fmt.Errorf("the agent did not start: %s; see journalctl -u llmgw-agent on the node", lastLine(out))
}
r.emit(StepStart, Done, "llmgw-agent is running", "")
if !r.Opts.NoNodeExporter {
r.nodeExporter(ctx)
}
return nil
}
// nodeExporter installs Prometheus' node exporter from the distribution, for
// what llmgw does not measure: network and disk traffic, filesystems. Best
// effort: a node without it serves as well, so a failure is a warning.
func (r *Runner) nodeExporter(ctx context.Context) {
out, err := r.Remote.Run(ctx, `set -e
if systemctl is-active --quiet prometheus-node-exporter 2>/dev/null || systemctl is-active --quiet node_exporter 2>/dev/null; then
echo already
elif command -v apt-get >/dev/null 2>&1; then
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq prometheus-node-exporter >/dev/null
sudo systemctl enable --now prometheus-node-exporter >/dev/null 2>&1
echo installed
elif command -v dnf >/dev/null 2>&1; then
sudo dnf install -y -q golang-github-prometheus-node-exporter >/dev/null
sudo systemctl enable --now node_exporter >/dev/null 2>&1
echo installed
else
echo "no apt-get or dnf"; exit 1
fi`, nil)
switch {
case err != nil:
r.emit(StepStart, Warn, "the Prometheus node exporter was not installed: "+lastLine(out)+
"; the node serves without it, and its network and disk are not charted", "")
case strings.Contains(string(out), "already"):
r.emit(StepStart, Info, "the Prometheus node exporter was already running", "")
default:
r.emit(StepStart, Info, "the Prometheus node exporter is installed, on port 9100", "")
}
}
// --- helpers ---
func lastLine(b []byte) string {