A btop-like monitor in the console, from Prometheus; node exporters discovered and installed
Monitor draws CPU, memory, GPUs, requests, tokens, latency, network and disk as Braille charts over 5m/1h/24h/7d, then per node, card and model sparklines; node and model pages have their own. History is read from monitoring.prometheus through POST /api/v1/metrics/range, or without it from what the console saw. The control plane serves the cluster's machines for Prometheus http_sd at /prometheus/targets on its metrics port, and node add installs prometheus-node-exporter. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
2e55487eca
commit
a1b1663e9d
16 files changed
+1217
-4
No files matched your search
@@ -0,0 +1,89 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
||||
)
|
||||
|
||||
// History reads the cluster's metrics back from the Prometheus that scraped
|
||||
// them: what the console's charts draw.
|
||||
type History interface {
|
||||
RangeQuery(ctx context.Context, expr, nameBy string, window, step time.Duration) ([]promq.Series, error)
|
||||
}
|
||||
|
||||
// SetHistory supplies the Prometheus to read from; nil leaves the charts to
|
||||
// what the console saw itself.
|
||||
func (a *API) SetHistory(h History) { a.history = h }
|
||||
|
||||
// MetricsQuery is one line set of a chart: a PromQL expression, and the label
|
||||
// naming each line.
|
||||
type MetricsQuery struct {
|
||||
ID string `json:"id"`
|
||||
Expr string `json:"expr"`
|
||||
By string `json:"by,omitempty"`
|
||||
}
|
||||
|
||||
// MetricsRequest asks for several queries over one window at once.
|
||||
type MetricsRequest struct {
|
||||
Window time.Duration `json:"window"`
|
||||
Step time.Duration `json:"step"`
|
||||
Queries []MetricsQuery `json:"queries"`
|
||||
}
|
||||
|
||||
// MetricsResult is each query's lines, by id; Configured is false when no
|
||||
// Prometheus is set, and Errors says which queries failed and why.
|
||||
type MetricsResult struct {
|
||||
Configured bool `json:"configured"`
|
||||
Series map[string][]promq.Series `json:"series"`
|
||||
Errors map[string]string `json:"errors,omitempty"`
|
||||
}
|
||||
|
||||
// metricsBatch runs the queries side by side: a screen of charts is a dozen
|
||||
// of them, and one after another they would take as long as the slowest
|
||||
// dozen times.
|
||||
func (a *API) metricsBatch(w http.ResponseWriter, r *http.Request) {
|
||||
var req MetricsRequest
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
res := MetricsResult{Series: map[string][]promq.Series{}}
|
||||
if a.history == nil || len(req.Queries) == 0 {
|
||||
writeJSON(w, http.StatusOK, res)
|
||||
return
|
||||
}
|
||||
res.Configured = true
|
||||
if req.Window <= 0 {
|
||||
req.Window = time.Hour
|
||||
}
|
||||
if len(req.Queries) > 40 {
|
||||
req.Queries = req.Queries[:40]
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
for _, q := range req.Queries {
|
||||
wg.Add(1)
|
||||
go func(q MetricsQuery) {
|
||||
defer wg.Done()
|
||||
s, err := a.history.RangeQuery(r.Context(), q.Expr, q.By, req.Window, req.Step)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if err != nil {
|
||||
if res.Errors == nil {
|
||||
res.Errors = map[string]string{}
|
||||
}
|
||||
res.Errors[q.ID] = err.Error()
|
||||
return
|
||||
}
|
||||
res.Series[q.ID] = s
|
||||
}(q)
|
||||
}
|
||||
wg.Wait()
|
||||
writeJSON(w, http.StatusOK, res)
|
||||
}
|
||||
Reference in new issue
Block a user