A btop-like monitor in the console, from Prometheus; node exporters discovered and installed
Monitor draws CPU, memory, GPUs, requests, tokens, latency, network and disk as Braille charts over 5m/1h/24h/7d, then per node, card and model sparklines; node and model pages have their own. History is read from monitoring.prometheus through POST /api/v1/metrics/range, or without it from what the console saw. The control plane serves the cluster's machines for Prometheus http_sd at /prometheus/targets on its metrics port, and node add installs prometheus-node-exporter. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
2e55487eca
commit
a1b1663e9d
16 files changed
+1217
-4
No files matched your search
@@ -287,6 +287,10 @@ func newServerCmd() *cobra.Command {
|
||||
var history web.History
|
||||
if h := promq.New(cfg.Monitoring.Prometheus, cfg.Monitoring.Timeout); h != nil {
|
||||
history = h
|
||||
// And the console's charts, over the admin API.
|
||||
if adminAPI != nil {
|
||||
adminAPI.SetHistory(h)
|
||||
}
|
||||
log.Info("dashboard history configured",
|
||||
"prometheus", cfg.Monitoring.Prometheus)
|
||||
}
|
||||
@@ -382,6 +386,15 @@ func newServerCmd() *cobra.Command {
|
||||
_ = adminLn.Close()
|
||||
return fmt.Errorf("listening on %s: %w", cfg.MetricsListen, err)
|
||||
}
|
||||
// Beside the metrics, the machines to scrape for their node
|
||||
// exporters, for Prometheus' http_sd_configs: on this port
|
||||
// because Prometheus already reaches it.
|
||||
if adminAPI != nil {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /prometheus/targets", adminAPI.PromSD)
|
||||
mux.Handle("/", h)
|
||||
h = mux
|
||||
}
|
||||
metricsSrv = &http.Server{Handler: h, ReadHeaderTimeout: 10 * time.Second}
|
||||
go func() {
|
||||
if err := metricsSrv.Serve(metricsLn); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
|
||||
Reference in new issue
Block a user