A btop-like monitor in the console, from Prometheus; node exporters discovered and installed
Monitor draws CPU, memory, GPUs, requests, tokens, latency, network and disk as Braille charts over 5m/1h/24h/7d, then per node, card and model sparklines; node and model pages have their own. History is read from monitoring.prometheus through POST /api/v1/metrics/range, or without it from what the console saw. The control plane serves the cluster's machines for Prometheus http_sd at /prometheus/targets on its metrics port, and node add installs prometheus-node-exporter. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
2e55487eca
commit
a1b1663e9d
16 files changed
+1217
-4
No files matched your search
@@ -60,6 +60,9 @@ func ParseFabric(s string) (Fabric, error) {
|
||||
|
||||
// Options say which machine, and what to tell its agent.
|
||||
type Options struct {
|
||||
// NoNodeExporter leaves Prometheus' node exporter off the machine.
|
||||
NoNodeExporter bool
|
||||
|
||||
// Addr is how to reach the machine over SSH, and User who to log in as.
|
||||
Addr, User string
|
||||
// Node is its id in the cluster. Empty means its hostname, which is what
|
||||
@@ -673,9 +676,41 @@ systemctl is-active llmgw-agent`, nil); err != nil {
|
||||
return fmt.Errorf("the agent did not start: %s; see journalctl -u llmgw-agent on the node", lastLine(out))
|
||||
}
|
||||
r.emit(StepStart, Done, "llmgw-agent is running", "")
|
||||
if !r.Opts.NoNodeExporter {
|
||||
r.nodeExporter(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// nodeExporter installs Prometheus' node exporter from the distribution, for
|
||||
// what llmgw does not measure: network and disk traffic, filesystems. Best
|
||||
// effort: a node without it serves as well, so a failure is a warning.
|
||||
func (r *Runner) nodeExporter(ctx context.Context) {
|
||||
out, err := r.Remote.Run(ctx, `set -e
|
||||
if systemctl is-active --quiet prometheus-node-exporter 2>/dev/null || systemctl is-active --quiet node_exporter 2>/dev/null; then
|
||||
echo already
|
||||
elif command -v apt-get >/dev/null 2>&1; then
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq prometheus-node-exporter >/dev/null
|
||||
sudo systemctl enable --now prometheus-node-exporter >/dev/null 2>&1
|
||||
echo installed
|
||||
elif command -v dnf >/dev/null 2>&1; then
|
||||
sudo dnf install -y -q golang-github-prometheus-node-exporter >/dev/null
|
||||
sudo systemctl enable --now node_exporter >/dev/null 2>&1
|
||||
echo installed
|
||||
else
|
||||
echo "no apt-get or dnf"; exit 1
|
||||
fi`, nil)
|
||||
switch {
|
||||
case err != nil:
|
||||
r.emit(StepStart, Warn, "the Prometheus node exporter was not installed: "+lastLine(out)+
|
||||
"; the node serves without it, and its network and disk are not charted", "")
|
||||
case strings.Contains(string(out), "already"):
|
||||
r.emit(StepStart, Info, "the Prometheus node exporter was already running", "")
|
||||
default:
|
||||
r.emit(StepStart, Info, "the Prometheus node exporter is installed, on port 9100", "")
|
||||
}
|
||||
}
|
||||
|
||||
// --- helpers ---
|
||||
|
||||
func lastLine(b []byte) string {
|
||||
|
||||
Reference in new issue
Block a user