Ilker ManapandClaude Opus 5.5
a1b1663e9d
Monitor draws CPU, memory, GPUs, requests, tokens, latency, network and disk as Braille charts over 5m/1h/24h/7d, then per node, card and model sparklines; node and model pages have their own. History is read from monitoring.prometheus through POST /api/v1/metrics/range, or without it from what the console saw. The control plane serves the cluster's machines for Prometheus http_sd at /prometheus/targets on its metrics port, and node add installs prometheus-node-exporter. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
90 lines
2.5 KiB
Go
90 lines
2.5 KiB
Go
package adminapi
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"io"
|
|
"net/http"
|
|
"sync"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
|
)
|
|
|
|
// History reads the cluster's metrics back from the Prometheus that scraped
|
|
// them: what the console's charts draw.
|
|
type History interface {
|
|
RangeQuery(ctx context.Context, expr, nameBy string, window, step time.Duration) ([]promq.Series, error)
|
|
}
|
|
|
|
// SetHistory supplies the Prometheus to read from; nil leaves the charts to
|
|
// what the console saw itself.
|
|
func (a *API) SetHistory(h History) { a.history = h }
|
|
|
|
// MetricsQuery is one line set of a chart: a PromQL expression, and the label
|
|
// naming each line.
|
|
type MetricsQuery struct {
|
|
ID string `json:"id"`
|
|
Expr string `json:"expr"`
|
|
By string `json:"by,omitempty"`
|
|
}
|
|
|
|
// MetricsRequest asks for several queries over one window at once.
|
|
type MetricsRequest struct {
|
|
Window time.Duration `json:"window"`
|
|
Step time.Duration `json:"step"`
|
|
Queries []MetricsQuery `json:"queries"`
|
|
}
|
|
|
|
// MetricsResult is each query's lines, by id; Configured is false when no
|
|
// Prometheus is set, and Errors says which queries failed and why.
|
|
type MetricsResult struct {
|
|
Configured bool `json:"configured"`
|
|
Series map[string][]promq.Series `json:"series"`
|
|
Errors map[string]string `json:"errors,omitempty"`
|
|
}
|
|
|
|
// metricsBatch runs the queries side by side: a screen of charts is a dozen
|
|
// of them, and one after another they would take as long as the slowest
|
|
// dozen times.
|
|
func (a *API) metricsBatch(w http.ResponseWriter, r *http.Request) {
|
|
var req MetricsRequest
|
|
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
|
|
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
|
return
|
|
}
|
|
res := MetricsResult{Series: map[string][]promq.Series{}}
|
|
if a.history == nil || len(req.Queries) == 0 {
|
|
writeJSON(w, http.StatusOK, res)
|
|
return
|
|
}
|
|
res.Configured = true
|
|
if req.Window <= 0 {
|
|
req.Window = time.Hour
|
|
}
|
|
if len(req.Queries) > 40 {
|
|
req.Queries = req.Queries[:40]
|
|
}
|
|
var mu sync.Mutex
|
|
var wg sync.WaitGroup
|
|
for _, q := range req.Queries {
|
|
wg.Add(1)
|
|
go func(q MetricsQuery) {
|
|
defer wg.Done()
|
|
s, err := a.history.RangeQuery(r.Context(), q.Expr, q.By, req.Window, req.Step)
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if err != nil {
|
|
if res.Errors == nil {
|
|
res.Errors = map[string]string{}
|
|
}
|
|
res.Errors[q.ID] = err.Error()
|
|
return
|
|
}
|
|
res.Series[q.ID] = s
|
|
}(q)
|
|
}
|
|
wg.Wait()
|
|
writeJSON(w, http.StatusOK, res)
|
|
}
|