diff --git a/CHANGELOG.md b/CHANGELOG.md index da31fca..9fc4ff8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,27 @@ is only released once its **exit criterion** is met, not when its code lands. Until v1.0.0 the minor version is the breaking one: configuration formats and Go APIs change between minors without a deprecation period. +## Unreleased + +### Added + +- **How fast a model answers on the CPU, before it is served.** The serve + form says, as the cores are changed, about how many tokens a second one + reply gets on the CPU and on one card of the fleet's fastest class, and + warns when it is slow (under 15) or very slow (under 5, a reply of 500 + tokens taking minutes). Very slow on the CPU is refused until "run on the + CPU anyway" is ticked, GPUs free or not. `llmgw ctl model plan` and the + console's Hugging Face screen show the same figure for every way a model + can run, and `ctl model serve --cpu` says it in its log. +- **The speed follows the threads a replica is given.** Each agent now + measures what one core reads beside what all of them read: a replica of + four threads on a large server reads at about four cores' rate, not the + server's, which the estimate used to assume and overstated several times. + Calibrated on the PoC by llama-bench at 1, 2, 4 and 8 threads (within 15%), + and a model of experts reads its experts' share, from the GGUF header. + Agents need upgrading to report one core's rate; until then the old + estimate is used. + ## v0.16.2 — 2026-11-02 Standby control planes that answer clients themselves, with rate limits and budgets kept across them; FreeToken, an diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 9c2814a..f8aad72 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -1210,8 +1210,12 @@ type Host struct { // mem_bandwidth_gbs is how fast the memory reads, in GB/s, measured when // the agent started: what decides a model's speed on the CPU. 0 unknown. MemBandwidthGbs float64 `protobuf:"fixed64,7,opt,name=mem_bandwidth_gbs,json=memBandwidthGbs,proto3" json:"mem_bandwidth_gbs,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // mem_bandwidth_core_gbs is what one core reads alone, in GB/s: a replica + // given a few threads of a large machine reads at a few times this, not at + // the whole machine's rate. 0 unknown. + MemBandwidthCoreGbs float64 `protobuf:"fixed64,8,opt,name=mem_bandwidth_core_gbs,json=memBandwidthCoreGbs,proto3" json:"mem_bandwidth_core_gbs,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Host) Reset() { @@ -1293,6 +1297,13 @@ func (x *Host) GetMemBandwidthGbs() float64 { return 0 } +func (x *Host) GetMemBandwidthCoreGbs() float64 { + if x != nil { + return x.MemBandwidthCoreGbs + } + return 0 +} + // Storage is one filesystem a node keeps weights on. // // Per filesystem rather than per configured directory: free space belongs to @@ -3049,8 +3060,10 @@ type Heartbeat struct { // the agent has measured it — in the background after it starts, so a // registration usually has none yet. 0 means not measured. MemBandwidthGbs float64 `protobuf:"fixed64,20,opt,name=mem_bandwidth_gbs,json=memBandwidthGbs,proto3" json:"mem_bandwidth_gbs,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // mem_bandwidth_core_gbs is one core's read rate, as in Host. + MemBandwidthCoreGbs float64 `protobuf:"fixed64,21,opt,name=mem_bandwidth_core_gbs,json=memBandwidthCoreGbs,proto3" json:"mem_bandwidth_core_gbs,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Heartbeat) Reset() { @@ -3223,6 +3236,13 @@ func (x *Heartbeat) GetMemBandwidthGbs() float64 { return 0 } +func (x *Heartbeat) GetMemBandwidthCoreGbs() float64 { + if x != nil { + return x.MemBandwidthCoreGbs + } + return 0 +} + // Provision is the software a node should have: which engines, at which // versions. Nothing more. // @@ -5324,7 +5344,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" + "\x13EngineVersionsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xdd\x01\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x92\x02\n" + "\x04Host\x12\x13\n" + "\x05os_id\x18\x01 \x01(\tR\x04osId\x12\x1d\n" + "\n" + @@ -5334,7 +5354,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x04arch\x18\x04 \x01(\tR\x04arch\x12\x1b\n" + "\tcpu_flags\x18\x05 \x03(\tR\bcpuFlags\x12#\n" + "\rnvidia_driver\x18\x06 \x01(\tR\fnvidiaDriver\x12*\n" + - "\x11mem_bandwidth_gbs\x18\a \x01(\x01R\x0fmemBandwidthGbs\"u\n" + + "\x11mem_bandwidth_gbs\x18\a \x01(\x01R\x0fmemBandwidthGbs\x123\n" + + "\x16mem_bandwidth_core_gbs\x18\b \x01(\x01R\x13memBandwidthCoreGbs\"u\n" + "\aStorage\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" + "\n" + @@ -5520,7 +5541,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x06status\x18\x03 \x01(\tR\x06status\x12!\n" + "\fcluster_uuid\x18\x04 \x01(\tR\vclusterUuid\x12\x1b\n" + "\tclique_id\x18\x05 \x01(\tR\bcliqueId\x12\x1c\n" + - "\tbandwidth\x18\x06 \x01(\tR\tbandwidth\"\xd5\a\n" + + "\tbandwidth\x18\x06 \x01(\tR\tbandwidth\"\x8a\b\n" + "\tHeartbeat\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" + "\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" + @@ -5543,7 +5564,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x04imex\x18\x11 \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x12\x1b\n" + "\tcpu_count\x18\x12 \x01(\x05R\bcpuCount\x121\n" + "\x15gpu_foreign_memory_mb\x18\x13 \x03(\x05R\x12gpuForeignMemoryMb\x12*\n" + - "\x11mem_bandwidth_gbs\x18\x14 \x01(\x01R\x0fmemBandwidthGbs\x1aA\n" + + "\x11mem_bandwidth_gbs\x18\x14 \x01(\x01R\x0fmemBandwidthGbs\x123\n" + + "\x16mem_bandwidth_core_gbs\x18\x15 \x01(\x01R\x13memBandwidthCoreGbs\x1aA\n" + "\x13EngineVersionsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\";\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index e605e85..5140e35 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -241,6 +241,10 @@ message Host { // mem_bandwidth_gbs is how fast the memory reads, in GB/s, measured when // the agent started: what decides a model's speed on the CPU. 0 unknown. double mem_bandwidth_gbs = 7; + // mem_bandwidth_core_gbs is what one core reads alone, in GB/s: a replica + // given a few threads of a large machine reads at a few times this, not at + // the whole machine's rate. 0 unknown. + double mem_bandwidth_core_gbs = 8; } // Storage is one filesystem a node keeps weights on. @@ -618,6 +622,8 @@ message Heartbeat { // the agent has measured it — in the background after it starts, so a // registration usually has none yet. 0 means not measured. double mem_bandwidth_gbs = 20; + // mem_bandwidth_core_gbs is one core's read rate, as in Host. + double mem_bandwidth_core_gbs = 21; } // Provision is the software a node should have: which engines, at which diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index 8680d4b..9985f19 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -8305,6 +8305,10 @@ "type": "number", "description": "MemBandwidthGBs is how fast its memory reads, in GB/s, measured when its agent started; 0 when not reported. A model's speed on the CPU." }, + "mem_bandwidth_core_gbs": { + "type": "number", + "description": "MemBandwidthCoreGBs is what one core reads alone, in GB/s; 0 when not reported. A CPU replica's threads read at about this each." + }, "engines": { "type": "array", "items": { @@ -11406,6 +11410,10 @@ }, "strata_size": { "type": "string" + }, + "tokens_per_second": { + "type": "number", + "description": "TokensPerSecond is about how many tokens a second one reply gets this way, on the fastest host it fits; 0 when not known. On the CPU with the threads a replica is given by default: half the node's." } } }, diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 1248328..7a90337 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -519,7 +519,8 @@ func (a *Agent) scan(ctx context.Context) error { inv, err := discovery.Discover(ctx, opts, a.log) host := discovery.DiscoverHost() host.MinComputeCapability = discovery.MinComputeCapability(inv.GPUs) - host.MemBandwidthGBs = a.memBandwidth() + bw := a.memBandwidth() + host.MemBandwidthGBs, host.MemBandwidthCoreGBs = bw.All, bw.Core a.mu.Lock() a.inv = inv a.prov.Host = host @@ -578,7 +579,8 @@ func (a *Agent) session(ctx context.Context, server string) error { Arch: h.Arch, CpuFlags: h.CPUFlags, NvidiaDriver: h.NVIDIADriver, // What is measured by now, not what the startup scan found before // the measurement ended: a reconnect would otherwise say 0. - MemBandwidthGbs: a.memBandwidth(), + MemBandwidthGbs: a.memBandwidth().All, + MemBandwidthCoreGbs: a.memBandwidth().Core, } if err := stream.Send(&llmgwv1.AgentMessage{ @@ -676,7 +678,8 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage { // without anybody restarting anything. hb.MemoryTotalMb = int64(discovery.MemoryTotalMB()) hb.CpuCount = int32(runtime.NumCPU()) - hb.MemBandwidthGbs = a.memBandwidth() + bw := a.memBandwidth() + hb.MemBandwidthGbs, hb.MemBandwidthCoreGbs = bw.All, bw.Core a.reprobeEngines() a.mu.Lock() @@ -1573,20 +1576,32 @@ func (a *Agent) reportCompute(ctx context.Context, out chan<- *llmgwv1.AgentMess // VMs on one host that all restart together measure each other, and read a // third less than any of them gets alone. Once a process rather than an // agent, so tests running many agents in one read it once. -func (a *Agent) memBandwidth() float64 { +func (a *Agent) memBandwidth() discovery.Bandwidth { bwOnce.Do(func() { go func() { time.Sleep(bandwidthDelay()) for { - if bw := measureBandwidth(); bw > math.Float64frombits(bwBits.Load()) { - bwBits.Store(math.Float64bits(bw)) - a.log.Info("this node's memory", "read_gbs", math.Round(bw*10)/10) + // The best of each seen so far: an hourly pass on a node + // busy serving reads less, and that is not the memory. + bw := measureBandwidth() + cur := a.memBandwidthNow() + if bw.All > cur.All || bw.Core > cur.Core { + bwBits.Store(math.Float64bits(max(bw.All, cur.All))) + coreBits.Store(math.Float64bits(max(bw.Core, cur.Core))) + a.log.Info("this node's memory", "read_gbs", math.Round(bw.All*10)/10, + "one_core_gbs", math.Round(bw.Core*10)/10) } time.Sleep(time.Hour) } }() }) - return math.Float64frombits(bwBits.Load()) + return a.memBandwidthNow() +} + +// memBandwidthNow is what has been measured so far, without starting a +// measurement. +func (a *Agent) memBandwidthNow() discovery.Bandwidth { + return discovery.Bandwidth{All: math.Float64frombits(bwBits.Load()), Core: math.Float64frombits(coreBits.Load())} } // bandwidthDelay is how long an agent waits before measuring, so agents @@ -1594,8 +1609,9 @@ func (a *Agent) memBandwidth() float64 { var bandwidthDelay = func() time.Duration { return 10*time.Second + rand.N(110*time.Second) } var ( - bwOnce sync.Once - bwBits atomic.Uint64 + bwOnce sync.Once + bwBits atomic.Uint64 + coreBits atomic.Uint64 // measureBandwidth is replaced in tests, which have no use for it. measureBandwidth = discovery.MeasureMemoryBandwidth ) diff --git a/internal/agent/discovery/host.go b/internal/agent/discovery/host.go index f7923e5..a9a840e 100644 --- a/internal/agent/discovery/host.go +++ b/internal/agent/discovery/host.go @@ -35,6 +35,10 @@ type Host struct { // measured: what decides a model's speed on the CPU. Not read here but // measured by the agent when it starts (MeasureMemoryBandwidth). MemBandwidthGBs float64 `json:"mem_bandwidth_gbs,omitempty"` + // MemBandwidthCoreGBs is what one core reads alone, in GB/s, 0 when not + // measured: a replica given n threads reads at about n times this, up to + // MemBandwidthGBs. + MemBandwidthCoreGBs float64 `json:"mem_bandwidth_core_gbs,omitempty"` } // Family is the packaging family: "deb", "rpm", or "" when neither is known. diff --git a/internal/agent/discovery/membw.go b/internal/agent/discovery/membw.go index 9d66ba7..ff7b5aa 100644 --- a/internal/agent/discovery/membw.go +++ b/internal/agent/discovery/membw.go @@ -28,9 +28,22 @@ const ( minBuffer = 256 << 20 ) -// MeasureMemoryBandwidth is the memory's read rate in GB/s, the best of a -// few passes; 0 when the machine has too little free memory to measure. -func MeasureMemoryBandwidth() float64 { +// Bandwidth is how fast this machine's memory reads, in GB/s: with every +// CPU reading, and with one alone. A replica given a few threads of a large +// machine reads at a few times the second, not at the first: on a server of +// many memory channels one core cannot keep more than a fraction of them +// busy. +type Bandwidth struct { + All, Core float64 +} + +// coreBuffer is what the one-core passes read: past any cache, and short +// enough that one core reads it in a fraction of a second. +const coreBuffer = 512 << 20 + +// MeasureMemoryBandwidth is the memory's read rate, the best of a few passes +// each way; zero when the machine has too little free memory to measure. +func MeasureMemoryBandwidth() Bandwidth { size := int64(bandwidthBuffer) // Never more than a quarter of what is free: a node already holding // models must not be pushed into swap by a measurement. Without room @@ -40,7 +53,7 @@ func MeasureMemoryBandwidth() float64 { size = min(size, avail/4) } if size < max(minBuffer, 8*lastCacheBytes("/sys/devices/system/cpu")) { - return 0 + return Bandwidth{} } buf := make([]uint64, size/8) for i := range buf { @@ -68,7 +81,17 @@ func MeasureMemoryBandwidth() float64 { best = max(best, float64(size)/s/1e9) } } - return best + // And one core alone, over a part of the same buffer. + part := buf[:min(len(buf), coreBuffer/8)] + core := 0.0 + for pass := 0; pass < 3; pass++ { + start := time.Now() + sink(sum(part)) + if s := time.Since(start).Seconds(); s > 0 { + core = max(core, float64(len(part)*8)/s/1e9) + } + } + return Bandwidth{All: best, Core: min(core, best)} } // sum reads every word, eight at a time so the loop is not what limits it. diff --git a/internal/agent/discovery/membw_test.go b/internal/agent/discovery/membw_test.go index 0af2b53..cdf0f5d 100644 --- a/internal/agent/discovery/membw_test.go +++ b/internal/agent/discovery/membw_test.go @@ -12,11 +12,11 @@ func TestMemoryBandwidthIsMeasured(t *testing.T) { } bw := MeasureMemoryBandwidth() // Any machine running this reads more than a gigabyte a second and less - // than a few terabytes. - if bw < 1 || bw > 5000 { - t.Errorf("measured %.1f GB/s", bw) + // than a few terabytes, and one core no more than all of them. + if bw.All < 1 || bw.All > 5000 || bw.Core < 0.5 || bw.Core > bw.All { + t.Errorf("measured %.1f GB/s, %.1f on one core", bw.All, bw.Core) } - t.Logf("%.1f GB/s", bw) + t.Logf("%.1f GB/s, %.1f on one core", bw.All, bw.Core) } func TestTheLastCacheIsReadFromSys(t *testing.T) { diff --git a/internal/agent/main_test.go b/internal/agent/main_test.go index 1e62bbe..4e491f4 100644 --- a/internal/agent/main_test.go +++ b/internal/agent/main_test.go @@ -4,12 +4,14 @@ import ( "os" "testing" "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" ) func TestMain(m *testing.M) { // Half a gigabyte read per test binary, and contending with the tests // for the CPUs: these never look at the figure. - measureBandwidth = func() float64 { return 0 } + measureBandwidth = func() discovery.Bandwidth { return discovery.Bandwidth{} } bandwidthDelay = func() time.Duration { return 0 } os.Exit(m.Run()) } diff --git a/internal/cli/modelplan.go b/internal/cli/modelplan.go index 92092cf..7c7e3d7 100644 --- a/internal/cli/modelplan.go +++ b/internal/cli/modelplan.go @@ -321,7 +321,10 @@ func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string fmt.Fprintln(out, "\nNo file fits this cluster as it is: a smaller context (--context), or more memory.") return errDoesNotFit } - fmt.Fprintf(out, "\nBest: %s on %s: %s.\n", rec.Artifact, rec.Label(), rec.MemoryText()) + fmt.Fprintf(out, "\nBest: %s on %s: %s%s.\n", rec.Artifact, rec.Label(), rec.MemoryText(), speedNote(rec)) + if w := slowWarning(rec); w != "" { + fmt.Fprintln(out, w) + } if p.KVBytesPerToken > 0 { fmt.Fprintf(out, "The context costs %.2f GB per 1000 tokens, the whole model (--context to plan another).\n", modelplan.GB(p.KVBytesPerToken*1000)) @@ -334,7 +337,8 @@ func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string if o.FreeNow { now = "free" } - fmt.Fprintf(w, " %s\t%s\t%s\t%d host(s)\t%s\n", o.Artifact, o.Label(), o.MemoryText(), o.Hosts, now) + fmt.Fprintf(w, " %s\t%s\t%s\t%d host(s)\t%s\t%s\n", o.Artifact, o.Label(), o.MemoryText(), o.Hosts, now, + speedCell(o.TokensPerSecond)) } _ = w.Flush() } @@ -349,3 +353,37 @@ func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string fmt.Fprintln(out, " --fallbacks") return nil } + +// speedCell is a way's speed in a table: about how many tokens a second one +// reply gets, and whether that is slow. +func speedCell(tps float64) string { + switch { + case tps <= 0: + return "" + case tps < modelplan.CPUTooSlowTPS: + return fmt.Sprintf("~%.1f tok/s, very slow", tps) + case tps < modelplan.CPUSlowTPS: + return fmt.Sprintf("~%.0f tok/s, slow", tps) + } + return fmt.Sprintf("~%.0f tok/s", tps) +} + +// speedNote is the same for a sentence, empty when not known. +func speedNote(o modelplan.Option) string { + if o.TokensPerSecond <= 0 { + return "" + } + return fmt.Sprintf(", about %s tokens a second for one reply", strconv.FormatFloat(o.TokensPerSecond, 'f', 1, 64)) +} + +// slowWarning says what a slow way means for somebody waiting on a reply. +func slowWarning(o modelplan.Option) string { + tps := o.TokensPerSecond + switch { + case tps <= 0 || tps >= modelplan.CPUSlowTPS: + return "" + case tps < modelplan.CPUTooSlowTPS: + return fmt.Sprintf("Very slow: a reply of 500 tokens takes about %.0f minutes.", max(1, 500/tps/60)) + } + return fmt.Sprintf("Slow: a reply of 500 tokens takes about %.0f seconds.", 500/tps) +} diff --git a/internal/cli/tui_hf.go b/internal/cli/tui_hf.go index a5d3629..57312bf 100644 --- a/internal/cli/tui_hf.go +++ b/internal/cli/tui_hf.go @@ -107,7 +107,15 @@ func optionLabel(o modelplan.Option) string { if o.FreeNow { now = sOK.Render("free") } - return fmt.Sprintf("%s · %s · %s", o.Label(), o.MemoryText(), now) + label := fmt.Sprintf("%s · %s · %s", o.Label(), o.MemoryText(), now) + if c := speedCell(o.TokensPerSecond); c != "" { + style := sDim + if o.TokensPerSecond < modelplan.CPUTooSlowTPS { + style = sWarn + } + label += " · " + style.Render(c) + } + return label } // repoMenu offers a repository's files, each with the best way it runs here; @@ -196,6 +204,9 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact note += fmt.Sprintf("\nMemory for a %d-token context, each way it runs:", p.Context) for _, o := range opts { note += "\n " + o.Label() + ": " + o.MemoryText() + if w := slowWarning(o); w != "" && o.CPU { + note += "\n " + sWarn.Render(w) + } } if p.KVBytesPerToken > 0 { note += fmt.Sprintf("\nThe context costs %.2f GB per 1000 tokens, the whole model; a longer one is checked\n"+ diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 638271d..54b0971 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -259,6 +259,9 @@ type NodeView struct { // MemBandwidthGBs is how fast its memory reads, in GB/s, measured when // its agent started; 0 when not reported. A model's speed on the CPU. MemBandwidthGBs float64 `json:"mem_bandwidth_gbs,omitempty"` + // MemBandwidthCoreGBs is what one core reads alone, in GB/s; 0 when not + // reported. A CPU replica's threads read at about this each. + MemBandwidthCoreGBs float64 `json:"mem_bandwidth_core_gbs,omitempty"` // Engines are the inference engines installed here, as the agent found // them. Absent means the agent is older than the probe and said nothing, // which is not the same as having none — placement treats it that way too, @@ -723,7 +726,7 @@ func (a *API) Nodes() []NodeView { v.EngineBuilds[e] = b } } - v.MemBandwidthGBs = n.Host.MemBandwidthGBs + v.MemBandwidthGBs, v.MemBandwidthCoreGBs = n.Host.MemBandwidthGBs, n.Host.MemBandwidthCoreGBs for _, g := range n.GPUs { if g.MIG != nil { continue // said of the card it is carved from @@ -1966,6 +1969,15 @@ func PoolNodesOf(nodes []NodeView) []modelplan.PoolNode { if l := n.Load; l != nil && l.MemTotal > 0 { p.HostFreeGB = int((l.MemTotal - l.MemUsed) >> 30) } + p.CPUs, p.MemBWGBs, p.MemBWCoreGBs = n.CPUCount, n.MemBandwidthGBs, n.MemBandwidthCoreGBs + for _, g := range n.GPUs { + if bw := modelplan.GPUBandwidthGBs(g.Name); bw > 0 { + if p.CardBWGBs == nil { + p.CardBWGBs = map[string]float64{} + } + p.CardBWGBs[g.Class] = bw + } + } out = append(out, p) } return out diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go index 5c32539..0227be9 100644 --- a/internal/control/agentsvc/convert.go +++ b/internal/control/agentsvc/convert.go @@ -36,7 +36,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node { n.Host = discovery.Host{ OSID: h.GetOsId(), OSVersion: h.GetOsVersion(), OSCodename: h.GetOsCodename(), Arch: h.GetArch(), CPUFlags: h.GetCpuFlags(), NVIDIADriver: h.GetNvidiaDriver(), - MemBandwidthGBs: h.GetMemBandwidthGbs(), + MemBandwidthGBs: h.GetMemBandwidthGbs(), MemBandwidthCoreGBs: h.GetMemBandwidthCoreGbs(), } } for _, w := range r.GetWeights() { diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index d741ad4..3658487 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -547,7 +547,7 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { storage := hb.GetStorage() memMB := int(hb.GetMemoryTotalMb()) cpus := int(hb.GetCpuCount()) - bw := hb.GetMemBandwidthGbs() + bw, coreBW := hb.GetMemBandwidthGbs(), hb.GetMemBandwidthCoreGbs() if len(used) == 0 && len(foreign) == 0 && len(problems) == 0 && len(storage) == 0 && !hb.GetWeightsChanged() && memMB == 0 && cpus == 0 && !hb.GetEnginesChanged() && !hb.GetProvisionChanged() && hb.GetLoad() == nil && len(hb.GetIbPorts()) == 0 && @@ -626,6 +626,9 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { if bw > 0 { n.Host.MemBandwidthGBs = bw } + if coreBW > 0 { + n.Host.MemBandwidthCoreGBs = coreBW + } if cpus > 0 && cpus != n.CPUCount { if n.CPUCount > 0 { s.log.Info("node CPUs changed", "node", nodeID, "from", n.CPUCount, "to", cpus) diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index a9bc963..074e62f 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -572,6 +572,14 @@ func (s *Server) run(j *Job) { // On one card, the weights and the context's cache have to fit it: a // 70B with a 65536-token pool on a 24 GB card loaded its weights and then // failed to make its context, every time, as did an 8B with 131072. + // Asked for the CPU, and slow there: said, not refused — the request + // named it — so the job's log says why its replies take minutes. + if len(layout) > 0 && layout[0].CPU && layout[0].TokensPerSecond > 0 && + layout[0].TokensPerSecond < modelplan.CPUSlowTPS { + tps := layout[0].TokensPerSecond + s.step(j, "check", Running, fmt.Sprintf("on the CPU about %.1f tokens a second for one reply: "+ + "a reply of 500 tokens takes about %.0f seconds; a card would be many times faster", tps, 500/tps)) + } if len(layout) > 0 && layout[0].Class != "" && !layout[0].CPU { fitted, why, err := fitPoolToCard(req, header, set.Bytes, layout[0].Class, max(layout[0].GPUsPerNode, 1)) if err != nil { diff --git a/internal/control/modelserve/plan.go b/internal/control/modelserve/plan.go index 9767fbc..a81d797 100644 --- a/internal/control/modelserve/plan.go +++ b/internal/control/modelserve/plan.go @@ -150,6 +150,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan, } shape.KVBytesPerToken = v.Header.KVBytesPerToken() shape.KVHeads = int(v.Header.HeadCountKV) //nolint:gosec // a header's head count + shape.ActiveShare = v.Header.ActiveShare() if p.Arch == "" { p.Arch = v.Header.Architecture } @@ -181,6 +182,9 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan, p.Note = "the model's shape was not readable, so the context's memory is not counted" } shape.Params = p.Params + if shape.ActiveShare == 0 || shape.ActiveShare == 1 { + shape.ActiveShare = modelplan.ActiveShare(repo.ID, p.Params) + } p.KVBytesPerToken, p.Experts = shape.KVBytesPerToken, shape.Experts p.Options = modelplan.Options(arts, shape, s.deps.Pool(), p.Context) if !p.Servable { diff --git a/internal/gguf/gguf.go b/internal/gguf/gguf.go index 02b4671..de2b7e1 100644 --- a/internal/gguf/gguf.go +++ b/internal/gguf/gguf.go @@ -48,6 +48,22 @@ type Header struct { // (.pooling_type set and not none): an embedding model, which // llama.cpp serves with --embedding and does not chat. Embedding bool + // ExpertCount and ExpertUsedCount are a model of experts' experts and + // how many each token uses (.expert_count, expert_used_count); + // zero for a dense model. + ExpertCount, ExpertUsedCount uint64 +} + +// ActiveShare is the part of the weights a token reads: for a model of +// experts its experts' share plus a tenth for what every token reads +// whatever the experts — attention, embeddings, routing, shared experts. +// Qwen3-Coder-30B-A3B, eight of 128, read 0.16 of its weights a token on +// cuda01's card, which is 0.0625 and the tenth. 1 for a dense model. +func (h Header) ActiveShare() float64 { + if h.ExpertCount == 0 || h.ExpertUsedCount == 0 || h.ExpertUsedCount >= h.ExpertCount { + return 1 + } + return min(1, float64(h.ExpertUsedCount)/float64(h.ExpertCount)+0.1) } // KVBytesPerToken is what one token of context costs in llama.cpp's default @@ -276,6 +292,10 @@ func Read(r io.Reader) (Header, error) { h.Embedding = num > 0 case "context_length": h.ContextLength = num + case "expert_count": + h.ExpertCount = num + case "expert_used_count": + h.ExpertUsedCount = num } } // The tokenizer's arrays — megabytes of vocabulary — come after the diff --git a/internal/gguf/gguf_test.go b/internal/gguf/gguf_test.go index 5f006fc..1481a29 100644 --- a/internal/gguf/gguf_test.go +++ b/internal/gguf/gguf_test.go @@ -210,3 +210,14 @@ func TestAPoolIsSizedToTheCard(t *testing.T) { t.Error("class memory misread") } } + +// A model of experts reads its experts' share a token and what every token +// reads besides; a dense one reads it all. +func TestAModelOfExpertsReadsAShareOfItsWeights(t *testing.T) { + if got := (Header{ExpertCount: 128, ExpertUsedCount: 8}).ActiveShare(); got < 0.16 || got > 0.17 { + t.Errorf("Qwen3-30B-A3B's shape reads %.3f of its weights a token, measured 0.16", got) + } + if (Header{}).ActiveShare() != 1 || (Header{ExpertCount: 8, ExpertUsedCount: 8}).ActiveShare() != 1 { + t.Error("a dense model reads all its weights") + } +} diff --git a/internal/modelplan/cpuspeed.go b/internal/modelplan/cpuspeed.go index 5c89fdd..4b3bed3 100644 --- a/internal/modelplan/cpuspeed.go +++ b/internal/modelplan/cpuspeed.go @@ -1,6 +1,7 @@ package modelplan import ( + "math" "regexp" "strconv" ) @@ -9,28 +10,45 @@ import ( // // Writing a token reads every weight it uses once, so on a CPU the memory // decides the speed: a token takes the bytes read over the rate llama.cpp -// reads them at, plus a little that does not shrink with the model. Measured -// on the PoC on 2026-10-26, each node's bandwidth by MeasureMemoryBandwidth -// against llama.cpp's own predicted_per_second for 128 tokens: +// reads them at, plus a little that does not shrink with the model. // -// node01 4 vCPU 24.9 GB/s qwen2.5-1.5b Q4_K_M 1.1 GB 14.5 tok/s 0.65 of the bandwidth -// node05 8 vCPU 41.9 GB/s qwen2.5-14b Q4_K_M 9.0 GB 3.3 tok/s 0.71 -// cuda01 12 vCPU 42.4 GB/s qwen2.5-0.5b Q4_K_M 0.5 GB 36.0 tok/s (the fixed cost shows) +// The rate is the threads': each reads at about what one core reads alone, +// a little less with every one added, up to what the whole machine reads. +// On a server of many memory channels a replica given four threads reads at +// a fraction of the machine's rate, and the machine's rate alone overstated +// it several times. Measured on the PoC's node05 on 2026-11-03 (8 vCPU, +// 43.9 GB/s with every CPU, 9.7 with one; llama-bench, 64 tokens): // -// So 0.68 of the measured rate, and 5 ms a token besides. One reply at a -// time; several at once share the same reads and each is slower. The PoC's -// host has far fewer cores than its VMs, so these are what a busy host -// gives, and a machine of its own does better. +// threads 1 2 4 8 +// qwen2.5-coder-7b 1.9 3.6 6.3 9.3 tok/s 4.7 GB: 8.9 17.0 30.7 45.6 GB/s read +// qwen2.5-1.5b 8.3 15.2 25.4 37.0 tok/s 1.1 GB: 9.7 18.3 32.6 — +// +// so 0.95 of one core's rate times threads^0.9, up to the machine's, and +// 5 ms a token besides. Against what was measured before (2026-10-26): +// node01's 1.5b on two threads, 14.5 tok/s, is taken for 15.0; node05's 14b +// on four, 3.3, for 3.5. One reply at a time; several at once share the same +// reads and each is slower. The PoC's host has far fewer cores than its VMs, +// so these are what a busy host gives, and a machine of its own does better. -// cpuReadShare is the part of the measured bandwidth llama.cpp reads at. +// cpuReadShare is the part of the whole machine's rate llama.cpp reads at, +// for a node whose one-core rate is not known: an agent older than its +// measurement. const cpuReadShare = 0.68 +// coreShare and threadExp are how threads add up: each about one core's +// rate, a little less with every one added. +const ( + coreShare = 0.95 + threadExp = 0.9 +) + // cpuTokenFixed is a token's cost that does not grow with the model. const cpuTokenFixed = 0.005 // CPUTokensPerSecond is about how many tokens a second one reply gets on a // CPU whose memory reads bwGBs, from a model whose tokens each read bytes. -// 0 when either is unknown. +// 0 when either is unknown. Use CPUTokensPerSecondOn where the threads and +// one core's rate are known. func CPUTokensPerSecond(bytes int64, bwGBs float64) float64 { if bytes <= 0 || bwGBs <= 0 { return 0 @@ -38,6 +56,48 @@ func CPUTokensPerSecond(bytes int64, bwGBs float64) float64 { return 1 / (float64(bytes)/(bwGBs*1e9*cpuReadShare) + cpuTokenFixed) } +// CPUReadGBs is the rate a replica given threads reads its weights at, on a +// node whose memory reads allGBs with every CPU and coreGBs with one. Without +// one core's rate or the threads, the share of the whole that small machines +// gave. +func CPUReadGBs(allGBs, coreGBs float64, threads int) float64 { + if allGBs <= 0 { + return 0 + } + if coreGBs <= 0 || threads <= 0 { + return allGBs * cpuReadShare + } + return min(coreShare*coreGBs*math.Pow(float64(threads), threadExp), allGBs) +} + +// CPUTokensPerSecondOn is about how many tokens a second one reply gets +// from a model whose tokens each read bytes, on threads of a node whose +// memory reads allGBs with every CPU and coreGBs with one. 0 when unknown. +func CPUTokensPerSecondOn(bytes int64, allGBs, coreGBs float64, threads int) float64 { + read := CPUReadGBs(allGBs, coreGBs, threads) + if bytes <= 0 || read <= 0 { + return 0 + } + return 1 / (float64(bytes)/(read*1e9) + cpuTokenFixed) +} + +// CPUThreads is how many threads a CPU replica runs on: the cores its +// profile reserves, else half the node's, which is what the agent gives +// llama.cpp then. +func CPUThreads(cpuCores, nodeCPUs int) int { + if cpuCores > 0 { + return cpuCores + } + return max(nodeCPUs/2, 1) +} + +// Slow speeds: under the first a reply is a wait somebody notices, under the +// second one of a few hundred tokens takes minutes. +const ( + CPUSlowTPS = 15 + CPUTooSlowTPS = 5 +) + // moeRead is what a token of a model of experts reads over its active // share. const moeRead = 1.6 diff --git a/internal/modelplan/cpuspeed_test.go b/internal/modelplan/cpuspeed_test.go index d7aa8cf..15ef7eb 100644 --- a/internal/modelplan/cpuspeed_test.go +++ b/internal/modelplan/cpuspeed_test.go @@ -23,6 +23,44 @@ func TestCPUSpeedMatchesWhatThePoCMeasured(t *testing.T) { } } +// Each thread reads at about one core's rate, up to the machine's: what +// llama-bench measured on node05 by thread count, and what was measured on +// node01 and node05 before, all within 15%. +func TestCPUSpeedFollowsTheThreads(t *testing.T) { + for _, c := range []struct { + bytes int64 + all, core float64 + threads int + want float64 + }{ + {4683073536, 43.9, 9.7, 1, 1.88}, // node05, qwen2.5-coder-7b + {4683073536, 43.9, 9.7, 2, 3.56}, + {4683073536, 43.9, 9.7, 4, 6.35}, + {4683073536, 43.9, 9.7, 8, 9.29}, + {1117320736, 43.9, 9.7, 1, 8.31}, // node05, qwen2.5-1.5b + {1117320736, 43.9, 9.7, 4, 25.4}, + {1117320736, 34.2, 10.1, 2, 14.5}, // node01 on its two cores + {9_000_000_000, 43.9, 9.7, 4, 3.3}, // node05, qwen2.5-14b on four + } { + got := CPUTokensPerSecondOn(c.bytes, c.all, c.core, c.threads) + if math.Abs(got-c.want)/c.want > 0.15 { + t.Errorf("%d bytes, %d threads: %.1f tok/s, measured %.1f", c.bytes, c.threads, got, c.want) + } + } + // A large machine, a few threads: the threads decide, not the machine. + few := CPUTokensPerSecondOn(18e9, 300, 10, 4) + if few > 2.5 { + t.Errorf("four threads of a 300 GB/s machine: %.1f tok/s, as though they read at the whole machine's rate", few) + } + // An agent that did not measure one core: the old reckoning. + if got, want := CPUTokensPerSecondOn(1e9, 40, 0, 4), CPUTokensPerSecond(1e9, 40); got != want { + t.Errorf("without one core's rate %.2f, want %.2f", got, want) + } + if CPUThreads(0, 64) != 32 || CPUThreads(6, 64) != 6 || CPUThreads(0, 1) != 1 { + t.Error("threads: the profile's cores, else half the node's") + } +} + func TestAModelOfExpertsReadsItsActiveShare(t *testing.T) { for name, want := range map[string]float64{ "Qwen/Qwen3-30B-A3B-Instruct-2507-GGUF": 0.16, @@ -35,3 +73,26 @@ func TestAModelOfExpertsReadsItsActiveShare(t *testing.T) { } } } + +// A plan's ways say how fast they answer: on the CPU by the node's memory +// and half its threads, on a card by its bandwidth. +func TestAPlansWaysSayHowFastTheyAnswer(t *testing.T) { + nodes := []PoolNode{{ID: "big", Class: "l40s-48g", GPUMemGB: 48, GPUs: 2, FreeGPUs: 2, HostMemGB: 512, HostFreeGB: 400, + CPUs: 64, MemBWGBs: 300, MemBWCoreGBs: 10, CardBWGBs: map[string]float64{"l40s-48g": 864}}} + opts := Options([]Artifact{{Name: "m-Q4_K_M.gguf", Format: "gguf", Bytes: 18e9, Quant: "Q4_K_M"}}, + Shape{Params: 32e9}, nodes, 4096) + var cpu, gpu float64 + for _, o := range opts { + if o.CPU { + cpu = o.TokensPerSecond + } else { + gpu = o.TokensPerSecond + } + } + if want := CPUTokensPerSecondOn(18e9, 300, 10, 32); math.Abs(cpu-want) > 0.01 { + t.Errorf("on the CPU %.2f tok/s, want %.2f", cpu, want) + } + if gpu < 3*cpu { + t.Errorf("on a card %.1f tok/s against %.1f on the CPU", gpu, cpu) + } +} diff --git a/internal/modelplan/options.go b/internal/modelplan/options.go index bf84953..2f583f1 100644 --- a/internal/modelplan/options.go +++ b/internal/modelplan/options.go @@ -35,6 +35,17 @@ type Shape struct { Params int64 // Experts is a mixture-of-experts model's routed experts; 0 for dense. Experts int + // ActiveShare is the part of the weights a token reads; 0 is all of + // them. + ActiveShare float64 +} + +// share is ActiveShare, all of the weights when not known. +func (sh Shape) share() float64 { + if sh.ActiveShare <= 0 || sh.ActiveShare > 1 { + return 1 + } + return sh.ActiveShare } // PoolNode is one ready node, as a plan sees it. @@ -57,6 +68,13 @@ type PoolNode struct { Engines []string // CC is its newest card's compute capability, 0 when not known. CC float64 + // CPUs, MemBWGBs and MemBWCoreGBs are its CPUs and how fast its memory + // reads, with every CPU and with one: a model's speed on its CPU. Zero + // when not measured. + CPUs int + MemBWGBs, MemBWCoreGBs float64 + // CardBWGBs is a card's memory bandwidth by class, for its speed there. + CardBWGBs map[string]float64 } // PoolKind is one class of a node's devices. @@ -119,6 +137,10 @@ type Option struct { // NeedGB and HaveGB are then about. StrataFamily string `json:"strata_family,omitempty"` StrataSize string `json:"strata_size,omitempty"` + // TokensPerSecond is about how many tokens a second one reply gets this + // way, on the fastest host it fits; 0 when not known. On the CPU with + // the threads a replica is given by default: half the node's. + TokensPerSecond float64 `json:"tokens_per_second,omitempty"` } // GPUs is every card one replica takes. @@ -222,6 +244,9 @@ func Options(arts []Artifact, sh Shape, nodes []PoolNode, context int) []Option out = append(out, vllmOptions(a, kv, sh, nodes, bpw)...) } } + for i := range out { + out[i].TokensPerSecond = speedOf(out[i], bytesOf(arts, out[i].Artifact), sh, nodes) + } Rank(out) return out } @@ -539,3 +564,37 @@ func CardBudget(class string) float64 { return float64(MemFromClass(class)) * gp func SplitsHeads(kvHeads, width int) bool { return kvHeads == 0 || width <= 1 || kvHeads%width == 0 || width%kvHeads == 0 } + +func bytesOf(arts []Artifact, name string) int64 { + for _, a := range arts { + if a.Name == name { + return a.Bytes + } + } + return 0 +} + +// speedOf is about how many tokens a second one reply gets in o, on the +// fastest node it fits: on the CPU by the node's memory and the threads a +// replica is given, on cards by their bandwidth. +func speedOf(o Option, bytes int64, sh Shape, nodes []PoolNode) float64 { + best := 0.0 + for _, n := range nodes { + var tps float64 + switch { + case o.CPU: + if float64(n.HostMemGB)*hostShare < o.NeedGB { + continue + } + tps = CPUTokensPerSecondOn(int64(float64(bytes)*sh.share()), n.MemBWGBs, n.MemBWCoreGBs, CPUThreads(0, n.CPUs)) + case o.Engine != "strata": + bw := n.CardBWGBs[o.Class] + if bw <= 0 { + continue + } + tps = GPUTokensPerSecond(o, bytes, sh.share(), sh.Params, bw, Links{}) + } + best = max(best, tps) + } + return best +} diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go index 3cc5e9f..bac5b35 100644 --- a/internal/web/admin_mirror.go +++ b/internal/web/admin_mirror.go @@ -370,6 +370,17 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) { if err != nil { return "", err } + // Slow enough that a reply is minutes is a choice to make on + // purpose too, GPUs free or not: on a fleet whose every node + // has cards, the CPU is theirs, and some are slow. + if prof := model.Profiles[0]; !judge && prof.CPUOnly() && r.FormValue("cpu_ok") == "" { + share := p.checkSource(ctx, model.Source).Header.ActiveShare() + hosts := cpuHosts(p.fleet, prof.MinMemoryGB) + if tps, node := bestCPUSpeed(hosts, int64(float64(size)*share), prof.CPUCores); tps > 0 && tps < modelplan.CPUTooSlowTPS { + return "", errors.New(l.T("serve.err.cpu_too_slow", model.Name, l.Num(tps, 1), node, + l.Int(int64(500/tps/60+0.5)))) + } + } // The weights and the context's cache have to fit the card: // an 8B asked for a 131072-token pool on 24 GB loaded its // weights and failed to make its context, again and again. diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go index f703dab..6310b08 100644 --- a/internal/web/admin_serve.go +++ b/internal/web/admin_serve.go @@ -15,6 +15,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/guard" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" + "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" "gitea.mynodes.xyz/ilker/llmgw/internal/strata" ) @@ -86,6 +87,20 @@ type serveForm struct { // version of a family Strata runs, served on it. StrataFamily, StrataSize string + // CPUHosts are the nodes a CPU replica of it could run on, with their + // memory's rates, and ReadBytes what a token of it reads: what the page + // reckons its speed on the CPU from, live, as the cores change. + CPUHosts []speedHost + ReadBytes int64 + // CPUSpeed is about how many tokens a second one reply gets on the CPU, + // on CPUSpeedNode with the cores the form starts with; 0 when unknown. + CPUSpeed float64 + CPUSpeedNode string + // GPUSpeed is the same on one card of the fleet's fastest class, + // GPUSpeedClass; 0 when the fleet has none it knows. + GPUSpeed float64 + GPUSpeedClass string + // Refusal is set when this version cannot be served as it stands, and says // why. Offered and then failing at placement is worse than not offered. Refusal string @@ -226,6 +241,13 @@ func planServe(l *Lang, st mirror.Status, files []mirror.Status, base string, } f.CPUCores = catalog.SuggestCores(cpus) } + // How fast it would answer: on the CPU, and on a card for comparison. + if f.Engine == "llamacpp" { + f.ReadBytes = int64(float64(weights) * header.ActiveShare()) + f.CPUHosts = cpuHosts(fleet, f.MinMemoryGB) + f.CPUSpeed, f.CPUSpeedNode = bestCPUSpeed(f.CPUHosts, f.ReadBytes, f.CPUCores) + f.GPUSpeed, f.GPUSpeedClass = bestGPUSpeed(fleet, weights, header.ActiveShare()) + } if biggest := biggestNode(fleet); biggest > 0 && f.MinMemoryGB > biggest { big := l.T("serve.warn.too_big", biggest, f.MinMemoryGB) if f.Warning != "" { @@ -675,3 +697,63 @@ func (p *AccountPages) asJudge(m *catalog.Model) (judge bool, renamed string) { } return true, renamed } + +// speedHost is a node a CPU replica could run on, as its speed is reckoned: +// its memory's rate with every CPU and with one, and its CPUs. +type speedHost struct { + ID string `json:"id"` + All float64 `json:"all"` + Core float64 `json:"core"` + CPUs int `json:"cpus"` +} + +// cpuHosts are the ready nodes whose RAM holds memGB and whose agents +// measured their memory. +func cpuHosts(fleet Source, memGB int) []speedHost { + if fleet == nil { + return nil + } + var out []speedHost + for _, n := range fleet.Nodes() { + if n.State != "ready" || n.MemBandwidthGBs <= 0 || (n.MemoryGB > 0 && n.MemoryGB < memGB) { + continue + } + out = append(out, speedHost{ID: n.ID, All: n.MemBandwidthGBs, Core: n.MemBandwidthCoreGBs, CPUs: n.CPUCount}) + } + return out +} + +// bestCPUSpeed is about how many tokens a second one reply gets on the +// fastest of hosts, reading read bytes a token on cores threads (zero takes +// what the agent gives: half the node's), and that node. +func bestCPUSpeed(hosts []speedHost, read int64, cores int) (float64, string) { + best, node := 0.0, "" + for _, h := range hosts { + if tps := modelplan.CPUTokensPerSecondOn(read, h.All, h.Core, modelplan.CPUThreads(cores, h.CPUs)); tps > best { + best, node = tps, h.ID + } + } + return best, node +} + +// bestGPUSpeed is the same for one card of the fastest class the fleet has +// whose memory holds the weights, by its bandwidth, and that class. +func bestGPUSpeed(fleet Source, weights int64, share float64) (float64, string) { + if fleet == nil || weights <= 0 { + return 0, "" + } + best, class := 0.0, "" + for _, n := range fleet.Nodes() { + for _, g := range n.GPUs { + bw := modelplan.GPUBandwidthGBs(g.Name) + if bw <= 0 || g.MemoryTotalMB > 0 && int64(g.MemoryTotalMB)<<20 < weights+weights/10 { + continue + } + o := modelplan.Option{Engine: "llamacpp", Class: g.Class, GPUsPerNode: 1} + if tps := modelplan.GPUTokensPerSecond(o, weights, share, 0, bw, modelplan.Links{}); tps > best { + best, class = tps, g.Class + } + } + } + return best, class +} diff --git a/internal/web/admin_serve_test.go b/internal/web/admin_serve_test.go index cdf75a8..3eda1a4 100644 --- a/internal/web/admin_serve_test.go +++ b/internal/web/admin_serve_test.go @@ -590,3 +590,64 @@ func TestAStrataFamilysVersionIsServedOnStrata(t *testing.T) { t.Error("an unknown size was taken") } } + +// speedFleet is two nodes with GPUs whose memory rates were measured: a +// large server and a small one, as on a fleet where every node has cards. +type speedFleet struct{ fakeFleet } + +func (speedFleet) Nodes() []adminapi.NodeView { + return []adminapi.NodeView{ + {ID: "big", State: "ready", MemoryGB: 512, CPUCount: 64, MemBandwidthGBs: 300, MemBandwidthCoreGBs: 10, + Engines: []string{"llamacpp"}}, + {ID: "small", State: "ready", MemoryGB: 64, CPUCount: 8, MemBandwidthGBs: 40, MemBandwidthCoreGBs: 10, + Engines: []string{"llamacpp"}}, + } +} + +// The speed on the CPU follows the threads a replica is given: four of a +// large server read at four cores' rate, not the server's. +func TestTheCPUsSpeedFollowsTheCoresGiven(t *testing.T) { + hosts := cpuHosts(speedFleet{}, 32) + if len(hosts) != 2 { + t.Fatalf("hosts %+v", hosts) + } + few, _ := bestCPUSpeed(hosts, 18e9, 4) + half, node := bestCPUSpeed(hosts, 18e9, 0) + if few > 2.5 || half < 8 || node != "big" { + t.Errorf("four cores %.1f tok/s, half the big node's %.1f on %s", few, half, node) + } + if hosts := cpuHosts(speedFleet{}, 100); len(hosts) != 1 || hosts[0].ID != "big" { + t.Errorf("a model of 100 GB fits the big node only: %+v", hosts) + } +} + +// Slow enough that a reply is minutes, the CPU is a choice made on purpose, +// GPUs free or not. +func TestAVerySlowModelOnTheCPUIsAChoiceMadeOnPurpose(t *testing.T) { + big := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", "18000000000") + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(big.Close) + values := func(extra ...string) url.Values { + v := url.Values{ + "csrf": {"tok"}, "name": {"big-dense"}, "source": {big.URL + "/m/v1/m.gguf"}, "engine": {"llamacpp"}, + "replicas": {"1"}, "context_length": {"4096"}, "placement": {"cpu"}, "min_memory_gb": {"24"}, + "cpu_cores": {"4"}, + } + for i := 0; i+1 < len(extra); i += 2 { + v.Set(extra[i], extra[i+1]) + } + return v + } + cat := &fakeCatalog{} + page := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{}, Catalog: cat, Fleet: speedFleet{}}, true) + body := mirrorPost(t, page, "/admin/mirror/serve", values()).Body.String() + if len(cat.added) != 0 || !strings.Contains(body, "tokens a second on the CPU") { + t.Fatalf("an 18 GB model on four cores went through unasked: %s", body) + } + mirrorPost(t, page, "/admin/mirror/serve", values("cpu_ok", "yes")) + if len(cat.added) != 1 { + t.Fatal("ticking the box should add it") + } +} diff --git a/internal/web/explore_fit.go b/internal/web/explore_fit.go index 87b3fd4..e9efa0e 100644 --- a/internal/web/explore_fit.go +++ b/internal/web/explore_fit.go @@ -43,9 +43,17 @@ type clusterFit struct { // cpuHost is a node as a model on its CPU sees it. type cpuHost struct { - id string - ramGB int - bwGBs float64 + id string + ramGB int + cpus int + bwGBs float64 + coreBW float64 // one core's rate, 0 from an older agent +} + +// tps is about how many tokens a second one reply of a model whose tokens +// each read bytes gets here, with the threads a replica is given by default. +func (h cpuHost) tps(bytes int64) float64 { + return modelplan.CPUTokensPerSecondOn(bytes, h.bwGBs, h.coreBW, modelplan.CPUThreads(0, h.cpus)) } // fp8Capability is the oldest card vLLM runs an FP8 checkpoint on: Ampere, @@ -100,7 +108,8 @@ func (p *AccountPages) fitOf() *clusterFit { } } if n.State == "ready" && n.MemBandwidthGBs > 0 { - c.cpus = append(c.cpus, cpuHost{id: n.ID, ramGB: n.MemoryGB, bwGBs: n.MemBandwidthGBs}) + c.cpus = append(c.cpus, cpuHost{id: n.ID, ramGB: n.MemoryGB, cpus: n.CPUCount, + bwGBs: n.MemBandwidthGBs, coreBW: n.MemBandwidthCoreGBs}) } for _, s := range n.ComputeCapabilities { if v, err := strconv.ParseFloat(s, 64); err == nil && v > newest[n.ID] { @@ -268,17 +277,16 @@ func (c *clusterFit) holds(l *Lang, name string, bytes, params int64) (gguf.Verd // bytes on the fastest node whose RAM holds it, and that node; share is the // part a token reads (ActiveShare). 0 when no measured node holds it. func (c *clusterFit) cpuSpeed(bytes int64, share float64) (float64, string) { + read := int64(float64(bytes) * share) var best cpuHost + bestTPS := 0.0 for _, n := range c.cpus { need := float64(bytes) + float64(1<<30) + float64(bytes)/20 - if need <= float64(n.ramGB)*0.85*(1<<30) && n.bwGBs > best.bwGBs { - best = n + if tps := n.tps(read); need <= float64(n.ramGB)*0.85*(1<<30) && tps > bestTPS { + best, bestTPS = n, tps } } - if best.id == "" { - return 0, "" - } - return modelplan.CPUTokensPerSecond(int64(float64(bytes)*share), best.bwGBs), best.id + return bestTPS, best.id } // speedLabel is a rate as the page shows it: whole tokens from ten up, one diff --git a/internal/web/locales/en/serve.json b/internal/web/locales/en/serve.json index b8a1d23..cff1207 100644 --- a/internal/web/locales/en/serve.json +++ b/internal/web/locales/en/serve.json @@ -46,6 +46,12 @@ "serve.pool": "context pool %s", "serve.will_run_as": "Will run as: %s.", "serve.free_gpus_warning": "%s GPUs are free; on the CPU this runs many times slower.", + "serve.speed_cpu": "about %s tokens a second on %s", + "serve.speed_gpu": "on one %s card about %s", + "serve.speed_slow": "Slow: a reply of 500 tokens takes about %s seconds.", + "serve.speed_too_slow": "Very slow: a reply of 500 tokens takes about %s minutes.", + "serve.cpu_ok_slow": "Run on the CPU anyway, knowing it will be slow", + "serve.err.cpu_too_slow": "%s would answer at about %s tokens a second on the CPU, at best on %s: a reply of 500 tokens in about %s minutes. Choose a GPU placement, or tick \"run on the CPU anyway\"", "serve.pool_note": "The context pool is what the engine is given; llama.cpp divides it across its parallel slots, so the number one request may use is the smaller one above.", "serve.memory_note": "Memory: %s. Change the context pool and this changes with it, by the KV cache's share — an estimate, and editable. It is filled in rather than left blank because a CPU profile has no card to be short of: with no figure, every ready node is a candidate and the first one picked is whichever the scheduler reached, not whichever can hold the model. A node label does the same job by name if you would rather pin it.", diff --git a/internal/web/locales/tr/serve.json b/internal/web/locales/tr/serve.json index 86ad350..048f78f 100644 --- a/internal/web/locales/tr/serve.json +++ b/internal/web/locales/tr/serve.json @@ -45,6 +45,12 @@ "serve.pool": "%s token bağlam havuzu", "serve.will_run_as": "Şöyle çalışacak: %s.", "serve.free_gpus_warning": "%s GPU boşta; CPU üzerinde bu model kat kat yavaş çalışır.", + "serve.speed_cpu": "%[2]s üzerinde yaklaşık %[1]s token/sn", + "serve.speed_gpu": "tek bir %s kartında yaklaşık %s token/sn", + "serve.speed_slow": "Yavaş: 500 token'lık bir cevap yaklaşık %s saniye sürer.", + "serve.speed_too_slow": "Çok yavaş: 500 token'lık bir cevap yaklaşık %s dakika sürer.", + "serve.cpu_ok_slow": "Yine de CPU üzerinde çalıştır: yavaş olacağını biliyorum", + "serve.err.cpu_too_slow": "%s CPU üzerinde yaklaşık %s token/sn ile cevap verir (en iyi %s üzerinde): 500 token'lık bir cevap yaklaşık %s dakika. Bir GPU yerleşimi seçin ya da \"yine de CPU üzerinde çalıştır\" kutusunu işaretleyin", "serve.pool_note": "Bağlam havuzu motora verilen miktardır; llama.cpp onu paralel yuvalarına böler, bu yüzden tek bir isteğin kullanabileceği sayı yukarıdaki küçük olandır.", "serve.memory_note": "Bellek: %s. Bağlam havuzunu değiştirirseniz bu da KV önbelleğinin payı kadar değişir — bir tahmindir ve düzenlenebilir. Boş bırakılmak yerine doldurulur, çünkü bir CPU profilinin yetersiz kalacak bir kartı yoktur: bir değer olmadan hazır her düğüm adaydır ve ilk seçilen, modeli taşıyabilen değil, zamanlayıcının ilk ulaştığı düğüm olur. Sabitlemeyi tercih ederseniz bir düğüm etiketi aynı işi adla görür.", diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html index ee09abe..680091e 100644 --- a/internal/web/templates/admin_mirror.html +++ b/internal/web/templates/admin_mirror.html @@ -177,8 +177,8 @@ {{if eq .Engine "llamacpp"}}{{end}} - {{if .FreeGPUs}}{{end}} + {{if and (eq .Engine "llamacpp") (or .FreeGPUs .CPUHosts)}}{{end}}
@@ -204,7 +204,29 @@ mem: {{t "serve.host_memory"}}, cores: {{t "serve.cores.other"}}, poolPar: {{t "serve.pool_parallel"}}, pool: {{t "serve.pool"}}, runAs: {{t "serve.will_run_as"}}, free: {{t "serve.free_gpus_warning"}}, + speedCPU: {{t "serve.speed_cpu"}}, speedGPU: {{t "serve.speed_gpu"}}, + slow: {{t "serve.speed_slow"}}, tooSlow: {{t "serve.speed_too_slow"}}, }; + // How fast one reply comes on the CPU: what a token reads over the rate a + // replica's threads read at — each about one core's, up to the machine's — + // and 5 ms a token besides. modelplan.CPUTokensPerSecondOn, in the page so + // it follows the cores as they are typed. + const hosts = {{.CPUHosts}} || [], readBytes = {{.ReadBytes}}; + const gpuTPS = {{.GPUSpeed}}, gpuClass = {{.GPUSpeedClass}}; + const slowTPS = 15, tooSlowTPS = 5; + function cpuTPS(cores) { + let best = 0, node = ""; + for (const h of hosts) { + const threads = cores > 0 ? cores : Math.max(Math.floor(h.cpus / 2), 1); + let read = h.core > 0 && threads > 0 ? Math.min(0.95 * h.core * Math.pow(threads, 0.9), h.all) : h.all * 0.68; + if (read <= 0 || readBytes <= 0) continue; + const tps = 1 / (readBytes / (read * 1e9) + 0.005); + if (tps > best) { best = tps; node = h.id; } + } + return [best, node]; + } + const nf = new Intl.NumberFormat(document.documentElement.lang || undefined, { maximumFractionDigits: 1 }); + const rate = (v) => nf.format(v < 10 ? Math.round(v * 10) / 10 : Math.round(v)); const fmt = (s, ...a) => { let i = 0; return s.replace(/%(?:\[(\d+)\])?[sd]/g, (_, n) => { const k = n ? +n - 1 : i; i = k + 1; return String(a[k]); }); @@ -226,11 +248,21 @@ const pool = parseInt(v("ctx_size"), 10), par = parseInt(v("parallel"), 10); if (pool && par) parts.push(fmt(T.poolPar, pool, par, Math.floor(pool / par))); else if (pool) parts.push(fmt(T.pool, pool)); + let tps = 0; + if (onCPU && f.dataset.engine === "llamacpp") { + const r = cpuTPS(parseInt(v("cpu_cores"), 10) || 0); + tps = r[0]; + if (tps > 0) parts.push(fmt(T.speedCPU, rate(tps), r[1])); + if (gpuTPS > 0) parts.push(fmt(T.speedGPU, gpuClass, rate(gpuTPS))); + } out.textContent = fmt(T.runAs, parts.join(", ")); - const warn = onCPU && +f.dataset.freeGpus > 0; - out.classList.toggle("error", warn); - if (warn) out.textContent += " " + fmt(T.free, f.dataset.freeGpus); - if (ok) ok.hidden = !onCPU; + const tooSlow = tps > 0 && tps < tooSlowTPS, slow = tps > 0 && tps < slowTPS; + const free = onCPU && +f.dataset.freeGpus > 0; + out.classList.toggle("error", free || tooSlow); + if (tooSlow) out.textContent += " " + fmt(T.tooSlow, Math.max(1, Math.round(500 / tps / 60))); + else if (slow) out.textContent += " " + fmt(T.slow, Math.round(500 / tps)); + if (free) out.textContent += " " + fmt(T.free, f.dataset.freeGpus); + if (ok) ok.hidden = !(onCPU && (free || tooSlow)); out.hidden = false; } f.addEventListener("input", draw);