Measure each node's CPU and memory, and give every node a page
The nodes are CPU machines, and nothing measured how busy one was: the heartbeat carried GPU memory and disk, and llmgw_node_memory_bytes is the total. The agent now reads /proc between heartbeats - CPU across all cores, load averages, memory used and available, and each engine's process group's cores and resident memory - and sends it with the heartbeat. Nothing new runs on the nodes and Prometheus needs no new target: the control plane exports it with the rest of the fleet, as llmgw_node_cpu_utilization_percent, llmgw_node_cpus, llmgw_node_load, llmgw_node_memory_used_bytes, llmgw_node_memory_available_bytes, llmgw_shard_cpu_cores and llmgw_shard_memory_bytes. A measurement more than a minute old is left out rather than repeated. The cluster page's node table shows CPU and RAM as bars, and each node links to a page of its own: current CPU, RAM, load and disk, the engines on it with their cores and memory, and an hour of CPU, RAM, load, per-model cores and memory and queue depth from Prometheus. It refreshes in place. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b06fb9f51f
commit
daed835d88
19 files changed
+1289
-102
No files matched your search
@@ -115,6 +115,43 @@ type NodeView struct {
|
||||
LastSeenAgo string `json:"last_seen_ago"`
|
||||
AgentVersion string `json:"agent_version,omitempty"`
|
||||
Replicas int `json:"replicas"`
|
||||
|
||||
// Load is how busy the machine is, absent when its agent has not
|
||||
// measured it lately — which is unknown, not idle.
|
||||
Load *LoadView `json:"load,omitempty"`
|
||||
}
|
||||
|
||||
// LoadView is a node's CPU and memory in use, at its last heartbeat.
|
||||
type LoadView struct {
|
||||
CPUPercent float64 `json:"cpu_percent"`
|
||||
CPUs int `json:"cpus,omitempty"`
|
||||
Load1 float64 `json:"load1"`
|
||||
Load5 float64 `json:"load5"`
|
||||
Load15 float64 `json:"load15"`
|
||||
MemUsed int64 `json:"memory_used_bytes"`
|
||||
MemTotal int64 `json:"memory_total_bytes"`
|
||||
// MemPercent is used over used plus available.
|
||||
MemPercent float64 `json:"memory_percent"`
|
||||
AgeSec int64 `json:"age_seconds"`
|
||||
}
|
||||
|
||||
// loadFresh is how old a measurement may be and still be shown as current.
|
||||
const loadFresh = time.Minute
|
||||
|
||||
func loadView(l *inventory.Load, now time.Time) *LoadView {
|
||||
if l == nil || now.Sub(l.At) > loadFresh {
|
||||
return nil
|
||||
}
|
||||
v := &LoadView{
|
||||
CPUPercent: l.CPUPercent, CPUs: l.CPUCount,
|
||||
Load1: l.Load1, Load5: l.Load5, Load15: l.Load15,
|
||||
MemUsed: l.MemUsedBytes, MemTotal: l.MemUsedBytes + l.MemAvailBytes,
|
||||
AgeSec: int64(now.Sub(l.At).Seconds()),
|
||||
}
|
||||
if v.MemTotal > 0 {
|
||||
v.MemPercent = float64(v.MemUsed) * 100 / float64(v.MemTotal)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// ModelView is a model as `ctl models` shows it.
|
||||
@@ -170,6 +207,10 @@ type ShardView struct {
|
||||
// waiting on one. It is what tells a replica pulling a hundred gigabytes
|
||||
// apart from one that is wedged — which look identical without it.
|
||||
Fetch *FetchView `json:"fetch,omitempty"`
|
||||
// CPUCores and MemoryBytes are the engine's use of its node, from the
|
||||
// node's last heartbeat; zero when not measured.
|
||||
CPUCores float64 `json:"cpu_cores,omitempty"`
|
||||
MemoryBytes int64 `json:"memory_bytes,omitempty"`
|
||||
}
|
||||
|
||||
// FetchView is a download in flight.
|
||||
@@ -266,6 +307,7 @@ func (a *API) Nodes() []NodeView {
|
||||
if !n.LastSeen.IsZero() {
|
||||
v.LastSeenAgo = time.Since(n.LastSeen).Round(time.Second).String()
|
||||
}
|
||||
v.Load = loadView(n.Load, time.Now())
|
||||
out = append(out, v)
|
||||
}
|
||||
return out
|
||||
@@ -308,6 +350,12 @@ func (a *API) listReplicas(w http.ResponseWriter, r *http.Request) {
|
||||
// Replicas builds the replica view, optionally filtered to one model.
|
||||
func (a *API) Replicas(model string) []ReplicaView {
|
||||
out := make([]ReplicaView, 0)
|
||||
loads := map[string]*inventory.Load{}
|
||||
for _, n := range a.ctl.Inventory().Nodes() {
|
||||
if n.Load != nil && time.Since(n.Load.At) <= loadFresh {
|
||||
loads[n.ID] = n.Load
|
||||
}
|
||||
}
|
||||
for _, rep := range a.ctl.Reconciler().Replicas() {
|
||||
if model != "" && rep.Model != model {
|
||||
continue
|
||||
@@ -338,6 +386,11 @@ func (a *API) Replicas(model string) []ReplicaView {
|
||||
sv.Fetch.Percent = &p
|
||||
}
|
||||
}
|
||||
if l := loads[s.NodeID]; l != nil {
|
||||
if use, ok := l.Shards[s.ID]; ok {
|
||||
sv.CPUCores, sv.MemoryBytes = use.CPUCores, use.RSSBytes
|
||||
}
|
||||
}
|
||||
v.Shards = append(v.Shards, sv)
|
||||
}
|
||||
out = append(out, v)
|
||||
|
||||
Reference in new issue
Block a user