How fast a model answers on the CPU, before it is served
Each agent measures one core's memory rate beside the whole machine's; the estimate takes a replica's threads at about one core's rate each, up to the machine's, calibrated by llama-bench on the PoC. The serve form shows it beside a card's, warns when slow, and asks for the CPU on purpose when a reply would take minutes; ctl model plan, the console and serve jobs say it too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
220110ed0d
commit
6403e61f4b
28 files changed
+657
-62
No files matched your search
@@ -259,6 +259,9 @@ type NodeView struct {
|
||||
// MemBandwidthGBs is how fast its memory reads, in GB/s, measured when
|
||||
// its agent started; 0 when not reported. A model's speed on the CPU.
|
||||
MemBandwidthGBs float64 `json:"mem_bandwidth_gbs,omitempty"`
|
||||
// MemBandwidthCoreGBs is what one core reads alone, in GB/s; 0 when not
|
||||
// reported. A CPU replica's threads read at about this each.
|
||||
MemBandwidthCoreGBs float64 `json:"mem_bandwidth_core_gbs,omitempty"`
|
||||
// Engines are the inference engines installed here, as the agent found
|
||||
// them. Absent means the agent is older than the probe and said nothing,
|
||||
// which is not the same as having none — placement treats it that way too,
|
||||
@@ -723,7 +726,7 @@ func (a *API) Nodes() []NodeView {
|
||||
v.EngineBuilds[e] = b
|
||||
}
|
||||
}
|
||||
v.MemBandwidthGBs = n.Host.MemBandwidthGBs
|
||||
v.MemBandwidthGBs, v.MemBandwidthCoreGBs = n.Host.MemBandwidthGBs, n.Host.MemBandwidthCoreGBs
|
||||
for _, g := range n.GPUs {
|
||||
if g.MIG != nil {
|
||||
continue // said of the card it is carved from
|
||||
@@ -1966,6 +1969,15 @@ func PoolNodesOf(nodes []NodeView) []modelplan.PoolNode {
|
||||
if l := n.Load; l != nil && l.MemTotal > 0 {
|
||||
p.HostFreeGB = int((l.MemTotal - l.MemUsed) >> 30)
|
||||
}
|
||||
p.CPUs, p.MemBWGBs, p.MemBWCoreGBs = n.CPUCount, n.MemBandwidthGBs, n.MemBandwidthCoreGBs
|
||||
for _, g := range n.GPUs {
|
||||
if bw := modelplan.GPUBandwidthGBs(g.Name); bw > 0 {
|
||||
if p.CardBWGBs == nil {
|
||||
p.CardBWGBs = map[string]float64{}
|
||||
}
|
||||
p.CardBWGBs[g.Class] = bw
|
||||
}
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out
|
||||
|
||||
Reference in new issue
Block a user