How fast a model answers on the CPU, before it is served
CI / test (push) Successful in 9m31s

Each agent measures one core's memory rate beside the whole machine's; the
estimate takes a replica's threads at about one core's rate each, up to the
machine's, calibrated by llama-bench on the PoC. The serve form shows it
beside a card's, warns when slow, and asks for the CPU on purpose when a
reply would take minutes; ctl model plan, the console and serve jobs say
it too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-03 08:52:15 +01:00
1 parent 220110ed0d
commit 6403e61f4b
28 files changed
+657 -62

No files matched your search

+13 -1
View File
@@ -259,6 +259,9 @@ type NodeView struct {
// MemBandwidthGBs is how fast its memory reads, in GB/s, measured when
// its agent started; 0 when not reported. A model's speed on the CPU.
MemBandwidthGBs float64 `json:"mem_bandwidth_gbs,omitempty"`
// MemBandwidthCoreGBs is what one core reads alone, in GB/s; 0 when not
// reported. A CPU replica's threads read at about this each.
MemBandwidthCoreGBs float64 `json:"mem_bandwidth_core_gbs,omitempty"`
// Engines are the inference engines installed here, as the agent found
// them. Absent means the agent is older than the probe and said nothing,
// which is not the same as having none — placement treats it that way too,
@@ -723,7 +726,7 @@ func (a *API) Nodes() []NodeView {
v.EngineBuilds[e] = b
}
}
v.MemBandwidthGBs = n.Host.MemBandwidthGBs
v.MemBandwidthGBs, v.MemBandwidthCoreGBs = n.Host.MemBandwidthGBs, n.Host.MemBandwidthCoreGBs
for _, g := range n.GPUs {
if g.MIG != nil {
continue // said of the card it is carved from
@@ -1966,6 +1969,15 @@ func PoolNodesOf(nodes []NodeView) []modelplan.PoolNode {
if l := n.Load; l != nil && l.MemTotal > 0 {
p.HostFreeGB = int((l.MemTotal - l.MemUsed) >> 30)
}
p.CPUs, p.MemBWGBs, p.MemBWCoreGBs = n.CPUCount, n.MemBandwidthGBs, n.MemBandwidthCoreGBs
for _, g := range n.GPUs {
if bw := modelplan.GPUBandwidthGBs(g.Name); bw > 0 {
if p.CardBWGBs == nil {
p.CardBWGBs = map[string]float64{}
}
p.CardBWGBs[g.Class] = bw
}
}
out = append(out, p)
}
return out