Measure each node's CPU and memory, and give every node a page

The nodes are CPU machines, and nothing measured how busy one was: the
heartbeat carried GPU memory and disk, and llmgw_node_memory_bytes is the
total. The agent now reads /proc between heartbeats - CPU across all
cores, load averages, memory used and available, and each engine's
process group's cores and resident memory - and sends it with the
heartbeat. Nothing new runs on the nodes and Prometheus needs no new
target: the control plane exports it with the rest of the fleet, as
llmgw_node_cpu_utilization_percent, llmgw_node_cpus, llmgw_node_load,
llmgw_node_memory_used_bytes, llmgw_node_memory_available_bytes,
llmgw_shard_cpu_cores and llmgw_shard_memory_bytes. A measurement more
than a minute old is left out rather than repeated.

The cluster page's node table shows CPU and RAM as bars, and each node
links to a page of its own: current CPU, RAM, load and disk, the engines
on it with their cores and memory, and an hour of CPU, RAM, load,
per-model cores and memory and queue depth from Prometheus. It refreshes
in place.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-28 23:21:23 +02:00
1 parent b06fb9f51f
commit daed835d88
19 files changed
+1289 -102

No files matched your search

+53
View File
@@ -115,6 +115,43 @@ type NodeView struct {
LastSeenAgo string `json:"last_seen_ago"`
AgentVersion string `json:"agent_version,omitempty"`
Replicas int `json:"replicas"`
// Load is how busy the machine is, absent when its agent has not
// measured it lately — which is unknown, not idle.
Load *LoadView `json:"load,omitempty"`
}
// LoadView is a node's CPU and memory in use, at its last heartbeat.
type LoadView struct {
CPUPercent float64 `json:"cpu_percent"`
CPUs int `json:"cpus,omitempty"`
Load1 float64 `json:"load1"`
Load5 float64 `json:"load5"`
Load15 float64 `json:"load15"`
MemUsed int64 `json:"memory_used_bytes"`
MemTotal int64 `json:"memory_total_bytes"`
// MemPercent is used over used plus available.
MemPercent float64 `json:"memory_percent"`
AgeSec int64 `json:"age_seconds"`
}
// loadFresh is how old a measurement may be and still be shown as current.
const loadFresh = time.Minute
func loadView(l *inventory.Load, now time.Time) *LoadView {
if l == nil || now.Sub(l.At) > loadFresh {
return nil
}
v := &LoadView{
CPUPercent: l.CPUPercent, CPUs: l.CPUCount,
Load1: l.Load1, Load5: l.Load5, Load15: l.Load15,
MemUsed: l.MemUsedBytes, MemTotal: l.MemUsedBytes + l.MemAvailBytes,
AgeSec: int64(now.Sub(l.At).Seconds()),
}
if v.MemTotal > 0 {
v.MemPercent = float64(v.MemUsed) * 100 / float64(v.MemTotal)
}
return v
}
// ModelView is a model as `ctl models` shows it.
@@ -170,6 +207,10 @@ type ShardView struct {
// waiting on one. It is what tells a replica pulling a hundred gigabytes
// apart from one that is wedged — which look identical without it.
Fetch *FetchView `json:"fetch,omitempty"`
// CPUCores and MemoryBytes are the engine's use of its node, from the
// node's last heartbeat; zero when not measured.
CPUCores float64 `json:"cpu_cores,omitempty"`
MemoryBytes int64 `json:"memory_bytes,omitempty"`
}
// FetchView is a download in flight.
@@ -266,6 +307,7 @@ func (a *API) Nodes() []NodeView {
if !n.LastSeen.IsZero() {
v.LastSeenAgo = time.Since(n.LastSeen).Round(time.Second).String()
}
v.Load = loadView(n.Load, time.Now())
out = append(out, v)
}
return out
@@ -308,6 +350,12 @@ func (a *API) listReplicas(w http.ResponseWriter, r *http.Request) {
// Replicas builds the replica view, optionally filtered to one model.
func (a *API) Replicas(model string) []ReplicaView {
out := make([]ReplicaView, 0)
loads := map[string]*inventory.Load{}
for _, n := range a.ctl.Inventory().Nodes() {
if n.Load != nil && time.Since(n.Load.At) <= loadFresh {
loads[n.ID] = n.Load
}
}
for _, rep := range a.ctl.Reconciler().Replicas() {
if model != "" && rep.Model != model {
continue
@@ -338,6 +386,11 @@ func (a *API) Replicas(model string) []ReplicaView {
sv.Fetch.Percent = &p
}
}
if l := loads[s.NodeID]; l != nil {
if use, ok := l.Shards[s.ID]; ok {
sv.CPUCores, sv.MemoryBytes = use.CPUCores, use.RSSBytes
}
}
v.Shards = append(v.Shards, sv)
}
out = append(out, v)