How fast a model answers on the CPU, before it is served

Each agent measures one core's memory rate beside the whole machine's; the
estimate takes a replica's threads at about one core's rate each, up to the
machine's, calibrated by llama-bench on the PoC. The serve form shows it
beside a card's, warns when slow, and asks for the CPU on purpose when a
reply would take minutes; ctl model plan, the console and serve jobs say
it too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -26,6 +26,27 @@ is only released once its **exit criterion** is met, not when its code lands.
 Until v1.0.0 the minor version is the breaking one: configuration formats and Go
 APIs change between minors without a deprecation period.
 
+## Unreleased
+
+### Added
+
+- **How fast a model answers on the CPU, before it is served.** The serve
+  form says, as the cores are changed, about how many tokens a second one
+  reply gets on the CPU and on one card of the fleet's fastest class, and
+  warns when it is slow (under 15) or very slow (under 5, a reply of 500
+  tokens taking minutes). Very slow on the CPU is refused until "run on the
+  CPU anyway" is ticked, GPUs free or not. `llmgw ctl model plan` and the
+  console's Hugging Face screen show the same figure for every way a model
+  can run, and `ctl model serve --cpu` says it in its log.
+- **The speed follows the threads a replica is given.** Each agent now
+  measures what one core reads beside what all of them read: a replica of
+  four threads on a large server reads at about four cores' rate, not the
+  server's, which the estimate used to assume and overstated several times.
+  Calibrated on the PoC by llama-bench at 1, 2, 4 and 8 threads (within 15%),
+  and a model of experts reads its experts' share, from the GGUF header.
+  Agents need upgrading to report one core's rate; until then the old
+  estimate is used.
+
 ## v0.16.2 — 2026-11-02
 
 Standby control planes that answer clients themselves, with rate limits and budgets kept across them; FreeToken, an

api/gen/llmgw/v1/agent.pb.go

@@ -1210,8 +1210,12 @@ type Host struct {
 	// mem_bandwidth_gbs is how fast the memory reads, in GB/s, measured when
 	// the agent started: what decides a model's speed on the CPU. 0 unknown.
 	MemBandwidthGbs float64 `protobuf:"fixed64,7,opt,name=mem_bandwidth_gbs,json=memBandwidthGbs,proto3" json:"mem_bandwidth_gbs,omitempty"`
-	unknownFields   protoimpl.UnknownFields
-	sizeCache       protoimpl.SizeCache
+	// mem_bandwidth_core_gbs is what one core reads alone, in GB/s: a replica
+	// given a few threads of a large machine reads at a few times this, not at
+	// the whole machine's rate. 0 unknown.
+	MemBandwidthCoreGbs float64 `protobuf:"fixed64,8,opt,name=mem_bandwidth_core_gbs,json=memBandwidthCoreGbs,proto3" json:"mem_bandwidth_core_gbs,omitempty"`
+	unknownFields       protoimpl.UnknownFields
+	sizeCache           protoimpl.SizeCache
 }
 
 func (x *Host) Reset() {
@@ -1293,6 +1297,13 @@ func (x *Host) GetMemBandwidthGbs() float64 {
 	return 0
 }
 
+func (x *Host) GetMemBandwidthCoreGbs() float64 {
+	if x != nil {
+		return x.MemBandwidthCoreGbs
+	}
+	return 0
+}
+
 // Storage is one filesystem a node keeps weights on.
 //
 // Per filesystem rather than per configured directory: free space belongs to
@@ -3049,8 +3060,10 @@ type Heartbeat struct {
 	// the agent has measured it — in the background after it starts, so a
 	// registration usually has none yet. 0 means not measured.
 	MemBandwidthGbs float64 `protobuf:"fixed64,20,opt,name=mem_bandwidth_gbs,json=memBandwidthGbs,proto3" json:"mem_bandwidth_gbs,omitempty"`
-	unknownFields   protoimpl.UnknownFields
-	sizeCache       protoimpl.SizeCache
+	// mem_bandwidth_core_gbs is one core's read rate, as in Host.
+	MemBandwidthCoreGbs float64 `protobuf:"fixed64,21,opt,name=mem_bandwidth_core_gbs,json=memBandwidthCoreGbs,proto3" json:"mem_bandwidth_core_gbs,omitempty"`
+	unknownFields       protoimpl.UnknownFields
+	sizeCache           protoimpl.SizeCache
 }
 
 func (x *Heartbeat) Reset() {
@@ -3223,6 +3236,13 @@ func (x *Heartbeat) GetMemBandwidthGbs() float64 {
 	return 0
 }
 
+func (x *Heartbeat) GetMemBandwidthCoreGbs() float64 {
+	if x != nil {
+		return x.MemBandwidthCoreGbs
+	}
+	return 0
+}
+
 // Provision is the software a node should have: which engines, at which
 // versions. Nothing more.
 //
@@ -5324,7 +5344,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
 	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" +
 	"\x13EngineVersionsEntry\x12\x10\n" +
 	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
-	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xdd\x01\n" +
+	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x92\x02\n" +
 	"\x04Host\x12\x13\n" +
 	"\x05os_id\x18\x01 \x01(\tR\x04osId\x12\x1d\n" +
 	"\n" +
@@ -5334,7 +5354,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
 	"\x04arch\x18\x04 \x01(\tR\x04arch\x12\x1b\n" +
 	"\tcpu_flags\x18\x05 \x03(\tR\bcpuFlags\x12#\n" +
 	"\rnvidia_driver\x18\x06 \x01(\tR\fnvidiaDriver\x12*\n" +
-	"\x11mem_bandwidth_gbs\x18\a \x01(\x01R\x0fmemBandwidthGbs\"u\n" +
+	"\x11mem_bandwidth_gbs\x18\a \x01(\x01R\x0fmemBandwidthGbs\x123\n" +
+	"\x16mem_bandwidth_core_gbs\x18\b \x01(\x01R\x13memBandwidthCoreGbs\"u\n" +
 	"\aStorage\x12\x12\n" +
 	"\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" +
 	"\n" +
@@ -5520,7 +5541,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
 	"\x06status\x18\x03 \x01(\tR\x06status\x12!\n" +
 	"\fcluster_uuid\x18\x04 \x01(\tR\vclusterUuid\x12\x1b\n" +
 	"\tclique_id\x18\x05 \x01(\tR\bcliqueId\x12\x1c\n" +
-	"\tbandwidth\x18\x06 \x01(\tR\tbandwidth\"\xd5\a\n" +
+	"\tbandwidth\x18\x06 \x01(\tR\tbandwidth\"\x8a\b\n" +
 	"\tHeartbeat\x12\x17\n" +
 	"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" +
 	"\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" +
@@ -5543,7 +5564,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
 	"\x04imex\x18\x11 \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x12\x1b\n" +
 	"\tcpu_count\x18\x12 \x01(\x05R\bcpuCount\x121\n" +
 	"\x15gpu_foreign_memory_mb\x18\x13 \x03(\x05R\x12gpuForeignMemoryMb\x12*\n" +
-	"\x11mem_bandwidth_gbs\x18\x14 \x01(\x01R\x0fmemBandwidthGbs\x1aA\n" +
+	"\x11mem_bandwidth_gbs\x18\x14 \x01(\x01R\x0fmemBandwidthGbs\x123\n" +
+	"\x16mem_bandwidth_core_gbs\x18\x15 \x01(\x01R\x13memBandwidthCoreGbs\x1aA\n" +
 	"\x13EngineVersionsEntry\x12\x10\n" +
 	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
 	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\";\n" +

api/proto/llmgw/v1/agent.proto

@@ -241,6 +241,10 @@ message Host {
   // mem_bandwidth_gbs is how fast the memory reads, in GB/s, measured when
   // the agent started: what decides a model's speed on the CPU. 0 unknown.
   double mem_bandwidth_gbs = 7;
+  // mem_bandwidth_core_gbs is what one core reads alone, in GB/s: a replica
+  // given a few threads of a large machine reads at a few times this, not at
+  // the whole machine's rate. 0 unknown.
+  double mem_bandwidth_core_gbs = 8;
 }
 
 // Storage is one filesystem a node keeps weights on.
@@ -618,6 +622,8 @@ message Heartbeat {
   // the agent has measured it — in the background after it starts, so a
   // registration usually has none yet. 0 means not measured.
   double mem_bandwidth_gbs = 20;
+  // mem_bandwidth_core_gbs is one core's read rate, as in Host.
+  double mem_bandwidth_core_gbs = 21;
 }
 
 // Provision is the software a node should have: which engines, at which

docs/admin-api.openapi.json

@@ -8305,6 +8305,10 @@
             "type": "number",
             "description": "MemBandwidthGBs is how fast its memory reads, in GB/s, measured when its agent started; 0 when not reported. A model's speed on the CPU."
           },
+          "mem_bandwidth_core_gbs": {
+            "type": "number",
+            "description": "MemBandwidthCoreGBs is what one core reads alone, in GB/s; 0 when not reported. A CPU replica's threads read at about this each."
+          },
           "engines": {
             "type": "array",
             "items": {
@@ -11406,6 +11410,10 @@
           },
           "strata_size": {
             "type": "string"
+          },
+          "tokens_per_second": {
+            "type": "number",
+            "description": "TokensPerSecond is about how many tokens a second one reply gets this way, on the fastest host it fits; 0 when not known. On the CPU with the threads a replica is given by default: half the node's."
           }
         }
       },

internal/agent/agent.go

@@ -519,7 +519,8 @@ func (a *Agent) scan(ctx context.Context) error {
 	inv, err := discovery.Discover(ctx, opts, a.log)
 	host := discovery.DiscoverHost()
 	host.MinComputeCapability = discovery.MinComputeCapability(inv.GPUs)
-	host.MemBandwidthGBs = a.memBandwidth()
+	bw := a.memBandwidth()
+	host.MemBandwidthGBs, host.MemBandwidthCoreGBs = bw.All, bw.Core
 	a.mu.Lock()
 	a.inv = inv
 	a.prov.Host = host
@@ -578,7 +579,8 @@ func (a *Agent) session(ctx context.Context, server string) error {
 		Arch: h.Arch, CpuFlags: h.CPUFlags, NvidiaDriver: h.NVIDIADriver,
 		// What is measured by now, not what the startup scan found before
 		// the measurement ended: a reconnect would otherwise say 0.
-		MemBandwidthGbs: a.memBandwidth(),
+		MemBandwidthGbs:     a.memBandwidth().All,
+		MemBandwidthCoreGbs: a.memBandwidth().Core,
 	}
 
 	if err := stream.Send(&llmgwv1.AgentMessage{
@@ -676,7 +678,8 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
 	// without anybody restarting anything.
 	hb.MemoryTotalMb = int64(discovery.MemoryTotalMB())
 	hb.CpuCount = int32(runtime.NumCPU())
-	hb.MemBandwidthGbs = a.memBandwidth()
+	bw := a.memBandwidth()
+	hb.MemBandwidthGbs, hb.MemBandwidthCoreGbs = bw.All, bw.Core
 
 	a.reprobeEngines()
 	a.mu.Lock()
@@ -1573,20 +1576,32 @@ func (a *Agent) reportCompute(ctx context.Context, out chan<- *llmgwv1.AgentMess
 // VMs on one host that all restart together measure each other, and read a
 // third less than any of them gets alone. Once a process rather than an
 // agent, so tests running many agents in one read it once.
-func (a *Agent) memBandwidth() float64 {
+func (a *Agent) memBandwidth() discovery.Bandwidth {
 	bwOnce.Do(func() {
 		go func() {
 			time.Sleep(bandwidthDelay())
 			for {
-				if bw := measureBandwidth(); bw > math.Float64frombits(bwBits.Load()) {
-					bwBits.Store(math.Float64bits(bw))
-					a.log.Info("this node's memory", "read_gbs", math.Round(bw*10)/10)
+				// The best of each seen so far: an hourly pass on a node
+				// busy serving reads less, and that is not the memory.
+				bw := measureBandwidth()
+				cur := a.memBandwidthNow()
+				if bw.All > cur.All || bw.Core > cur.Core {
+					bwBits.Store(math.Float64bits(max(bw.All, cur.All)))
+					coreBits.Store(math.Float64bits(max(bw.Core, cur.Core)))
+					a.log.Info("this node's memory", "read_gbs", math.Round(bw.All*10)/10,
+						"one_core_gbs", math.Round(bw.Core*10)/10)
 				}
 				time.Sleep(time.Hour)
 			}
 		}()
 	})
-	return math.Float64frombits(bwBits.Load())
+	return a.memBandwidthNow()
+}
+
+// memBandwidthNow is what has been measured so far, without starting a
+// measurement.
+func (a *Agent) memBandwidthNow() discovery.Bandwidth {
+	return discovery.Bandwidth{All: math.Float64frombits(bwBits.Load()), Core: math.Float64frombits(coreBits.Load())}
 }
 
 // bandwidthDelay is how long an agent waits before measuring, so agents
@@ -1594,8 +1609,9 @@ func (a *Agent) memBandwidth() float64 {
 var bandwidthDelay = func() time.Duration { return 10*time.Second + rand.N(110*time.Second) }
 
 var (
-	bwOnce sync.Once
-	bwBits atomic.Uint64
+	bwOnce   sync.Once
+	bwBits   atomic.Uint64
+	coreBits atomic.Uint64
 	// measureBandwidth is replaced in tests, which have no use for it.
 	measureBandwidth = discovery.MeasureMemoryBandwidth
 )

internal/agent/discovery/host.go

@@ -35,6 +35,10 @@ type Host struct {
 	// measured: what decides a model's speed on the CPU. Not read here but
 	// measured by the agent when it starts (MeasureMemoryBandwidth).
 	MemBandwidthGBs float64 `json:"mem_bandwidth_gbs,omitempty"`
+	// MemBandwidthCoreGBs is what one core reads alone, in GB/s, 0 when not
+	// measured: a replica given n threads reads at about n times this, up to
+	// MemBandwidthGBs.
+	MemBandwidthCoreGBs float64 `json:"mem_bandwidth_core_gbs,omitempty"`
 }
 
 // Family is the packaging family: "deb", "rpm", or "" when neither is known.

internal/agent/discovery/membw.go

@@ -28,9 +28,22 @@ const (
 	minBuffer       = 256 << 20
 )
 
-// MeasureMemoryBandwidth is the memory's read rate in GB/s, the best of a
-// few passes; 0 when the machine has too little free memory to measure.
-func MeasureMemoryBandwidth() float64 {
+// Bandwidth is how fast this machine's memory reads, in GB/s: with every
+// CPU reading, and with one alone. A replica given a few threads of a large
+// machine reads at a few times the second, not at the first: on a server of
+// many memory channels one core cannot keep more than a fraction of them
+// busy.
+type Bandwidth struct {
+	All, Core float64
+}
+
+// coreBuffer is what the one-core passes read: past any cache, and short
+// enough that one core reads it in a fraction of a second.
+const coreBuffer = 512 << 20
+
+// MeasureMemoryBandwidth is the memory's read rate, the best of a few passes
+// each way; zero when the machine has too little free memory to measure.
+func MeasureMemoryBandwidth() Bandwidth {
 	size := int64(bandwidthBuffer)
 	// Never more than a quarter of what is free: a node already holding
 	// models must not be pushed into swap by a measurement. Without room
@@ -40,7 +53,7 @@ func MeasureMemoryBandwidth() float64 {
 		size = min(size, avail/4)
 	}
 	if size < max(minBuffer, 8*lastCacheBytes("/sys/devices/system/cpu")) {
-		return 0
+		return Bandwidth{}
 	}
 	buf := make([]uint64, size/8)
 	for i := range buf {
@@ -68,7 +81,17 @@ func MeasureMemoryBandwidth() float64 {
 			best = max(best, float64(size)/s/1e9)
 		}
 	}
-	return best
+	// And one core alone, over a part of the same buffer.
+	part := buf[:min(len(buf), coreBuffer/8)]
+	core := 0.0
+	for pass := 0; pass < 3; pass++ {
+		start := time.Now()
+		sink(sum(part))
+		if s := time.Since(start).Seconds(); s > 0 {
+			core = max(core, float64(len(part)*8)/s/1e9)
+		}
+	}
+	return Bandwidth{All: best, Core: min(core, best)}
 }
 
 // sum reads every word, eight at a time so the loop is not what limits it.

internal/agent/discovery/membw_test.go

@@ -12,11 +12,11 @@ func TestMemoryBandwidthIsMeasured(t *testing.T) {
 	}
 	bw := MeasureMemoryBandwidth()
 	// Any machine running this reads more than a gigabyte a second and less
-	// than a few terabytes.
-	if bw < 1 || bw > 5000 {
-		t.Errorf("measured %.1f GB/s", bw)
+	// than a few terabytes, and one core no more than all of them.
+	if bw.All < 1 || bw.All > 5000 || bw.Core < 0.5 || bw.Core > bw.All {
+		t.Errorf("measured %.1f GB/s, %.1f on one core", bw.All, bw.Core)
 	}
-	t.Logf("%.1f GB/s", bw)
+	t.Logf("%.1f GB/s, %.1f on one core", bw.All, bw.Core)
 }
 
 func TestTheLastCacheIsReadFromSys(t *testing.T) {

internal/agent/main_test.go

@@ -4,12 +4,14 @@ import (
 	"os"
 	"testing"
 	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
 )
 
 func TestMain(m *testing.M) {
 	// Half a gigabyte read per test binary, and contending with the tests
 	// for the CPUs: these never look at the figure.
-	measureBandwidth = func() float64 { return 0 }
+	measureBandwidth = func() discovery.Bandwidth { return discovery.Bandwidth{} }
 	bandwidthDelay = func() time.Duration { return 0 }
 	os.Exit(m.Run())
 }

internal/cli/modelplan.go

@@ -321,7 +321,10 @@ func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string
 		fmt.Fprintln(out, "\nNo file fits this cluster as it is: a smaller context (--context), or more memory.")
 		return errDoesNotFit
 	}
-	fmt.Fprintf(out, "\nBest: %s on %s: %s.\n", rec.Artifact, rec.Label(), rec.MemoryText())
+	fmt.Fprintf(out, "\nBest: %s on %s: %s%s.\n", rec.Artifact, rec.Label(), rec.MemoryText(), speedNote(rec))
+	if w := slowWarning(rec); w != "" {
+		fmt.Fprintln(out, w)
+	}
 	if p.KVBytesPerToken > 0 {
 		fmt.Fprintf(out, "The context costs %.2f GB per 1000 tokens, the whole model (--context to plan another).\n",
 			modelplan.GB(p.KVBytesPerToken*1000))
@@ -334,7 +337,8 @@ func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string
 			if o.FreeNow {
 				now = "free"
 			}
-			fmt.Fprintf(w, "  %s\t%s\t%s\t%d host(s)\t%s\n", o.Artifact, o.Label(), o.MemoryText(), o.Hosts, now)
+			fmt.Fprintf(w, "  %s\t%s\t%s\t%d host(s)\t%s\t%s\n", o.Artifact, o.Label(), o.MemoryText(), o.Hosts, now,
+				speedCell(o.TokensPerSecond))
 		}
 		_ = w.Flush()
 	}
@@ -349,3 +353,37 @@ func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string
 	fmt.Fprintln(out, " --fallbacks")
 	return nil
 }
+
+// speedCell is a way's speed in a table: about how many tokens a second one
+// reply gets, and whether that is slow.
+func speedCell(tps float64) string {
+	switch {
+	case tps <= 0:
+		return ""
+	case tps < modelplan.CPUTooSlowTPS:
+		return fmt.Sprintf("~%.1f tok/s, very slow", tps)
+	case tps < modelplan.CPUSlowTPS:
+		return fmt.Sprintf("~%.0f tok/s, slow", tps)
+	}
+	return fmt.Sprintf("~%.0f tok/s", tps)
+}
+
+// speedNote is the same for a sentence, empty when not known.
+func speedNote(o modelplan.Option) string {
+	if o.TokensPerSecond <= 0 {
+		return ""
+	}
+	return fmt.Sprintf(", about %s tokens a second for one reply", strconv.FormatFloat(o.TokensPerSecond, 'f', 1, 64))
+}
+
+// slowWarning says what a slow way means for somebody waiting on a reply.
+func slowWarning(o modelplan.Option) string {
+	tps := o.TokensPerSecond
+	switch {
+	case tps <= 0 || tps >= modelplan.CPUSlowTPS:
+		return ""
+	case tps < modelplan.CPUTooSlowTPS:
+		return fmt.Sprintf("Very slow: a reply of 500 tokens takes about %.0f minutes.", max(1, 500/tps/60))
+	}
+	return fmt.Sprintf("Slow: a reply of 500 tokens takes about %.0f seconds.", 500/tps)
+}

internal/cli/tui_hf.go

@@ -107,7 +107,15 @@ func optionLabel(o modelplan.Option) string {
 	if o.FreeNow {
 		now = sOK.Render("free")
 	}
-	return fmt.Sprintf("%s · %s · %s", o.Label(), o.MemoryText(), now)
+	label := fmt.Sprintf("%s · %s · %s", o.Label(), o.MemoryText(), now)
+	if c := speedCell(o.TokensPerSecond); c != "" {
+		style := sDim
+		if o.TokensPerSecond < modelplan.CPUTooSlowTPS {
+			style = sWarn
+		}
+		label += " · " + style.Render(c)
+	}
+	return label
 }
 
 // repoMenu offers a repository's files, each with the best way it runs here;
@@ -196,6 +204,9 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact
 	note += fmt.Sprintf("\nMemory for a %d-token context, each way it runs:", p.Context)
 	for _, o := range opts {
 		note += "\n  " + o.Label() + ": " + o.MemoryText()
+		if w := slowWarning(o); w != "" && o.CPU {
+			note += "\n    " + sWarn.Render(w)
+		}
 	}
 	if p.KVBytesPerToken > 0 {
 		note += fmt.Sprintf("\nThe context costs %.2f GB per 1000 tokens, the whole model; a longer one is checked\n"+

internal/control/adminapi/api.go

@@ -259,6 +259,9 @@ type NodeView struct {
 	// MemBandwidthGBs is how fast its memory reads, in GB/s, measured when
 	// its agent started; 0 when not reported. A model's speed on the CPU.
 	MemBandwidthGBs float64 `json:"mem_bandwidth_gbs,omitempty"`
+	// MemBandwidthCoreGBs is what one core reads alone, in GB/s; 0 when not
+	// reported. A CPU replica's threads read at about this each.
+	MemBandwidthCoreGBs float64 `json:"mem_bandwidth_core_gbs,omitempty"`
 	// Engines are the inference engines installed here, as the agent found
 	// them. Absent means the agent is older than the probe and said nothing,
 	// which is not the same as having none — placement treats it that way too,
@@ -723,7 +726,7 @@ func (a *API) Nodes() []NodeView {
 				v.EngineBuilds[e] = b
 			}
 		}
-		v.MemBandwidthGBs = n.Host.MemBandwidthGBs
+		v.MemBandwidthGBs, v.MemBandwidthCoreGBs = n.Host.MemBandwidthGBs, n.Host.MemBandwidthCoreGBs
 		for _, g := range n.GPUs {
 			if g.MIG != nil {
 				continue // said of the card it is carved from
@@ -1966,6 +1969,15 @@ func PoolNodesOf(nodes []NodeView) []modelplan.PoolNode {
 		if l := n.Load; l != nil && l.MemTotal > 0 {
 			p.HostFreeGB = int((l.MemTotal - l.MemUsed) >> 30)
 		}
+		p.CPUs, p.MemBWGBs, p.MemBWCoreGBs = n.CPUCount, n.MemBandwidthGBs, n.MemBandwidthCoreGBs
+		for _, g := range n.GPUs {
+			if bw := modelplan.GPUBandwidthGBs(g.Name); bw > 0 {
+				if p.CardBWGBs == nil {
+					p.CardBWGBs = map[string]float64{}
+				}
+				p.CardBWGBs[g.Class] = bw
+			}
+		}
 		out = append(out, p)
 	}
 	return out

internal/control/agentsvc/convert.go

@@ -36,7 +36,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
 		n.Host = discovery.Host{
 			OSID: h.GetOsId(), OSVersion: h.GetOsVersion(), OSCodename: h.GetOsCodename(),
 			Arch: h.GetArch(), CPUFlags: h.GetCpuFlags(), NVIDIADriver: h.GetNvidiaDriver(),
-			MemBandwidthGBs: h.GetMemBandwidthGbs(),
+			MemBandwidthGBs: h.GetMemBandwidthGbs(), MemBandwidthCoreGBs: h.GetMemBandwidthCoreGbs(),
 		}
 	}
 	for _, w := range r.GetWeights() {

internal/control/agentsvc/service.go

@@ -547,7 +547,7 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
 	storage := hb.GetStorage()
 	memMB := int(hb.GetMemoryTotalMb())
 	cpus := int(hb.GetCpuCount())
-	bw := hb.GetMemBandwidthGbs()
+	bw, coreBW := hb.GetMemBandwidthGbs(), hb.GetMemBandwidthCoreGbs()
 	if len(used) == 0 && len(foreign) == 0 && len(problems) == 0 && len(storage) == 0 &&
 		!hb.GetWeightsChanged() && memMB == 0 && cpus == 0 && !hb.GetEnginesChanged() &&
 		!hb.GetProvisionChanged() && hb.GetLoad() == nil && len(hb.GetIbPorts()) == 0 &&
@@ -626,6 +626,9 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
 	if bw > 0 {
 		n.Host.MemBandwidthGBs = bw
 	}
+	if coreBW > 0 {
+		n.Host.MemBandwidthCoreGBs = coreBW
+	}
 	if cpus > 0 && cpus != n.CPUCount {
 		if n.CPUCount > 0 {
 			s.log.Info("node CPUs changed", "node", nodeID, "from", n.CPUCount, "to", cpus)

internal/control/modelserve/modelserve.go

@@ -572,6 +572,14 @@ func (s *Server) run(j *Job) {
 	// On one card, the weights and the context's cache have to fit it: a
 	// 70B with a 65536-token pool on a 24 GB card loaded its weights and then
 	// failed to make its context, every time, as did an 8B with 131072.
+	// Asked for the CPU, and slow there: said, not refused — the request
+	// named it — so the job's log says why its replies take minutes.
+	if len(layout) > 0 && layout[0].CPU && layout[0].TokensPerSecond > 0 &&
+		layout[0].TokensPerSecond < modelplan.CPUSlowTPS {
+		tps := layout[0].TokensPerSecond
+		s.step(j, "check", Running, fmt.Sprintf("on the CPU about %.1f tokens a second for one reply: "+
+			"a reply of 500 tokens takes about %.0f seconds; a card would be many times faster", tps, 500/tps))
+	}
 	if len(layout) > 0 && layout[0].Class != "" && !layout[0].CPU {
 		fitted, why, err := fitPoolToCard(req, header, set.Bytes, layout[0].Class, max(layout[0].GPUsPerNode, 1))
 		if err != nil {

internal/control/modelserve/plan.go

@@ -150,6 +150,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan,
 		}
 		shape.KVBytesPerToken = v.Header.KVBytesPerToken()
 		shape.KVHeads = int(v.Header.HeadCountKV) //nolint:gosec // a header's head count
+		shape.ActiveShare = v.Header.ActiveShare()
 		if p.Arch == "" {
 			p.Arch = v.Header.Architecture
 		}
@@ -181,6 +182,9 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan,
 		p.Note = "the model's shape was not readable, so the context's memory is not counted"
 	}
 	shape.Params = p.Params
+	if shape.ActiveShare == 0 || shape.ActiveShare == 1 {
+		shape.ActiveShare = modelplan.ActiveShare(repo.ID, p.Params)
+	}
 	p.KVBytesPerToken, p.Experts = shape.KVBytesPerToken, shape.Experts
 	p.Options = modelplan.Options(arts, shape, s.deps.Pool(), p.Context)
 	if !p.Servable {

internal/gguf/gguf.go

@@ -48,6 +48,22 @@ type Header struct {
 	// (<arch>.pooling_type set and not none): an embedding model, which
 	// llama.cpp serves with --embedding and does not chat.
 	Embedding bool
+	// ExpertCount and ExpertUsedCount are a model of experts' experts and
+	// how many each token uses (<arch>.expert_count, expert_used_count);
+	// zero for a dense model.
+	ExpertCount, ExpertUsedCount uint64
+}
+
+// ActiveShare is the part of the weights a token reads: for a model of
+// experts its experts' share plus a tenth for what every token reads
+// whatever the experts — attention, embeddings, routing, shared experts.
+// Qwen3-Coder-30B-A3B, eight of 128, read 0.16 of its weights a token on
+// cuda01's card, which is 0.0625 and the tenth. 1 for a dense model.
+func (h Header) ActiveShare() float64 {
+	if h.ExpertCount == 0 || h.ExpertUsedCount == 0 || h.ExpertUsedCount >= h.ExpertCount {
+		return 1
+	}
+	return min(1, float64(h.ExpertUsedCount)/float64(h.ExpertCount)+0.1)
 }
 
 // KVBytesPerToken is what one token of context costs in llama.cpp's default
@@ -276,6 +292,10 @@ func Read(r io.Reader) (Header, error) {
 				h.Embedding = num > 0
 			case "context_length":
 				h.ContextLength = num
+			case "expert_count":
+				h.ExpertCount = num
+			case "expert_used_count":
+				h.ExpertUsedCount = num
 			}
 		}
 		// The tokenizer's arrays — megabytes of vocabulary — come after the

internal/gguf/gguf_test.go

@@ -210,3 +210,14 @@ func TestAPoolIsSizedToTheCard(t *testing.T) {
 		t.Error("class memory misread")
 	}
 }
+
+// A model of experts reads its experts' share a token and what every token
+// reads besides; a dense one reads it all.
+func TestAModelOfExpertsReadsAShareOfItsWeights(t *testing.T) {
+	if got := (Header{ExpertCount: 128, ExpertUsedCount: 8}).ActiveShare(); got < 0.16 || got > 0.17 {
+		t.Errorf("Qwen3-30B-A3B's shape reads %.3f of its weights a token, measured 0.16", got)
+	}
+	if (Header{}).ActiveShare() != 1 || (Header{ExpertCount: 8, ExpertUsedCount: 8}).ActiveShare() != 1 {
+		t.Error("a dense model reads all its weights")
+	}
+}

internal/modelplan/cpuspeed.go

@@ -1,6 +1,7 @@
 package modelplan
 
 import (
+	"math"
 	"regexp"
 	"strconv"
 )
@@ -9,28 +10,45 @@ import (
 //
 // Writing a token reads every weight it uses once, so on a CPU the memory
 // decides the speed: a token takes the bytes read over the rate llama.cpp
-// reads them at, plus a little that does not shrink with the model. Measured
-// on the PoC on 2026-10-26, each node's bandwidth by MeasureMemoryBandwidth
-// against llama.cpp's own predicted_per_second for 128 tokens:
+// reads them at, plus a little that does not shrink with the model.
 //
-//	node01  4 vCPU 24.9 GB/s  qwen2.5-1.5b Q4_K_M 1.1 GB  14.5 tok/s  0.65 of the bandwidth
-//	node05  8 vCPU 41.9 GB/s  qwen2.5-14b  Q4_K_M 9.0 GB   3.3 tok/s  0.71
-//	cuda01 12 vCPU 42.4 GB/s  qwen2.5-0.5b Q4_K_M 0.5 GB  36.0 tok/s  (the fixed cost shows)
+// The rate is the threads': each reads at about what one core reads alone,
+// a little less with every one added, up to what the whole machine reads.
+// On a server of many memory channels a replica given four threads reads at
+// a fraction of the machine's rate, and the machine's rate alone overstated
+// it several times. Measured on the PoC's node05 on 2026-11-03 (8 vCPU,
+// 43.9 GB/s with every CPU, 9.7 with one; llama-bench, 64 tokens):
 //
-// So 0.68 of the measured rate, and 5 ms a token besides. One reply at a
-// time; several at once share the same reads and each is slower. The PoC's
-// host has far fewer cores than its VMs, so these are what a busy host
-// gives, and a machine of its own does better.
+//	threads           1     2     4     8
+//	qwen2.5-coder-7b  1.9   3.6   6.3   9.3 tok/s   4.7 GB: 8.9 17.0 30.7 45.6 GB/s read
+//	qwen2.5-1.5b      8.3  15.2  25.4  37.0 tok/s   1.1 GB: 9.7 18.3 32.6 —
+//
+// so 0.95 of one core's rate times threads^0.9, up to the machine's, and
+// 5 ms a token besides. Against what was measured before (2026-10-26):
+// node01's 1.5b on two threads, 14.5 tok/s, is taken for 15.0; node05's 14b
+// on four, 3.3, for 3.5. One reply at a time; several at once share the same
+// reads and each is slower. The PoC's host has far fewer cores than its VMs,
+// so these are what a busy host gives, and a machine of its own does better.
 
-// cpuReadShare is the part of the measured bandwidth llama.cpp reads at.
+// cpuReadShare is the part of the whole machine's rate llama.cpp reads at,
+// for a node whose one-core rate is not known: an agent older than its
+// measurement.
 const cpuReadShare = 0.68
 
+// coreShare and threadExp are how threads add up: each about one core's
+// rate, a little less with every one added.
+const (
+	coreShare = 0.95
+	threadExp = 0.9
+)
+
 // cpuTokenFixed is a token's cost that does not grow with the model.
 const cpuTokenFixed = 0.005
 
 // CPUTokensPerSecond is about how many tokens a second one reply gets on a
 // CPU whose memory reads bwGBs, from a model whose tokens each read bytes.
-// 0 when either is unknown.
+// 0 when either is unknown. Use CPUTokensPerSecondOn where the threads and
+// one core's rate are known.
 func CPUTokensPerSecond(bytes int64, bwGBs float64) float64 {
 	if bytes <= 0 || bwGBs <= 0 {
 		return 0
@@ -38,6 +56,48 @@ func CPUTokensPerSecond(bytes int64, bwGBs float64) float64 {
 	return 1 / (float64(bytes)/(bwGBs*1e9*cpuReadShare) + cpuTokenFixed)
 }
 
+// CPUReadGBs is the rate a replica given threads reads its weights at, on a
+// node whose memory reads allGBs with every CPU and coreGBs with one. Without
+// one core's rate or the threads, the share of the whole that small machines
+// gave.
+func CPUReadGBs(allGBs, coreGBs float64, threads int) float64 {
+	if allGBs <= 0 {
+		return 0
+	}
+	if coreGBs <= 0 || threads <= 0 {
+		return allGBs * cpuReadShare
+	}
+	return min(coreShare*coreGBs*math.Pow(float64(threads), threadExp), allGBs)
+}
+
+// CPUTokensPerSecondOn is about how many tokens a second one reply gets
+// from a model whose tokens each read bytes, on threads of a node whose
+// memory reads allGBs with every CPU and coreGBs with one. 0 when unknown.
+func CPUTokensPerSecondOn(bytes int64, allGBs, coreGBs float64, threads int) float64 {
+	read := CPUReadGBs(allGBs, coreGBs, threads)
+	if bytes <= 0 || read <= 0 {
+		return 0
+	}
+	return 1 / (float64(bytes)/(read*1e9) + cpuTokenFixed)
+}
+
+// CPUThreads is how many threads a CPU replica runs on: the cores its
+// profile reserves, else half the node's, which is what the agent gives
+// llama.cpp then.
+func CPUThreads(cpuCores, nodeCPUs int) int {
+	if cpuCores > 0 {
+		return cpuCores
+	}
+	return max(nodeCPUs/2, 1)
+}
+
+// Slow speeds: under the first a reply is a wait somebody notices, under the
+// second one of a few hundred tokens takes minutes.
+const (
+	CPUSlowTPS    = 15
+	CPUTooSlowTPS = 5
+)
+
 // moeRead is what a token of a model of experts reads over its active
 // share.
 const moeRead = 1.6

internal/modelplan/cpuspeed_test.go

@@ -23,6 +23,44 @@ func TestCPUSpeedMatchesWhatThePoCMeasured(t *testing.T) {
 	}
 }
 
+// Each thread reads at about one core's rate, up to the machine's: what
+// llama-bench measured on node05 by thread count, and what was measured on
+// node01 and node05 before, all within 15%.
+func TestCPUSpeedFollowsTheThreads(t *testing.T) {
+	for _, c := range []struct {
+		bytes     int64
+		all, core float64
+		threads   int
+		want      float64
+	}{
+		{4683073536, 43.9, 9.7, 1, 1.88}, // node05, qwen2.5-coder-7b
+		{4683073536, 43.9, 9.7, 2, 3.56},
+		{4683073536, 43.9, 9.7, 4, 6.35},
+		{4683073536, 43.9, 9.7, 8, 9.29},
+		{1117320736, 43.9, 9.7, 1, 8.31}, // node05, qwen2.5-1.5b
+		{1117320736, 43.9, 9.7, 4, 25.4},
+		{1117320736, 34.2, 10.1, 2, 14.5},  // node01 on its two cores
+		{9_000_000_000, 43.9, 9.7, 4, 3.3}, // node05, qwen2.5-14b on four
+	} {
+		got := CPUTokensPerSecondOn(c.bytes, c.all, c.core, c.threads)
+		if math.Abs(got-c.want)/c.want > 0.15 {
+			t.Errorf("%d bytes, %d threads: %.1f tok/s, measured %.1f", c.bytes, c.threads, got, c.want)
+		}
+	}
+	// A large machine, a few threads: the threads decide, not the machine.
+	few := CPUTokensPerSecondOn(18e9, 300, 10, 4)
+	if few > 2.5 {
+		t.Errorf("four threads of a 300 GB/s machine: %.1f tok/s, as though they read at the whole machine's rate", few)
+	}
+	// An agent that did not measure one core: the old reckoning.
+	if got, want := CPUTokensPerSecondOn(1e9, 40, 0, 4), CPUTokensPerSecond(1e9, 40); got != want {
+		t.Errorf("without one core's rate %.2f, want %.2f", got, want)
+	}
+	if CPUThreads(0, 64) != 32 || CPUThreads(6, 64) != 6 || CPUThreads(0, 1) != 1 {
+		t.Error("threads: the profile's cores, else half the node's")
+	}
+}
+
 func TestAModelOfExpertsReadsItsActiveShare(t *testing.T) {
 	for name, want := range map[string]float64{
 		"Qwen/Qwen3-30B-A3B-Instruct-2507-GGUF": 0.16,
@@ -35,3 +73,26 @@ func TestAModelOfExpertsReadsItsActiveShare(t *testing.T) {
 		}
 	}
 }
+
+// A plan's ways say how fast they answer: on the CPU by the node's memory
+// and half its threads, on a card by its bandwidth.
+func TestAPlansWaysSayHowFastTheyAnswer(t *testing.T) {
+	nodes := []PoolNode{{ID: "big", Class: "l40s-48g", GPUMemGB: 48, GPUs: 2, FreeGPUs: 2, HostMemGB: 512, HostFreeGB: 400,
+		CPUs: 64, MemBWGBs: 300, MemBWCoreGBs: 10, CardBWGBs: map[string]float64{"l40s-48g": 864}}}
+	opts := Options([]Artifact{{Name: "m-Q4_K_M.gguf", Format: "gguf", Bytes: 18e9, Quant: "Q4_K_M"}},
+		Shape{Params: 32e9}, nodes, 4096)
+	var cpu, gpu float64
+	for _, o := range opts {
+		if o.CPU {
+			cpu = o.TokensPerSecond
+		} else {
+			gpu = o.TokensPerSecond
+		}
+	}
+	if want := CPUTokensPerSecondOn(18e9, 300, 10, 32); math.Abs(cpu-want) > 0.01 {
+		t.Errorf("on the CPU %.2f tok/s, want %.2f", cpu, want)
+	}
+	if gpu < 3*cpu {
+		t.Errorf("on a card %.1f tok/s against %.1f on the CPU", gpu, cpu)
+	}
+}

internal/modelplan/options.go

@@ -35,6 +35,17 @@ type Shape struct {
 	Params int64
 	// Experts is a mixture-of-experts model's routed experts; 0 for dense.
 	Experts int
+	// ActiveShare is the part of the weights a token reads; 0 is all of
+	// them.
+	ActiveShare float64
+}
+
+// share is ActiveShare, all of the weights when not known.
+func (sh Shape) share() float64 {
+	if sh.ActiveShare <= 0 || sh.ActiveShare > 1 {
+		return 1
+	}
+	return sh.ActiveShare
 }
 
 // PoolNode is one ready node, as a plan sees it.
@@ -57,6 +68,13 @@ type PoolNode struct {
 	Engines []string
 	// CC is its newest card's compute capability, 0 when not known.
 	CC float64
+	// CPUs, MemBWGBs and MemBWCoreGBs are its CPUs and how fast its memory
+	// reads, with every CPU and with one: a model's speed on its CPU. Zero
+	// when not measured.
+	CPUs                   int
+	MemBWGBs, MemBWCoreGBs float64
+	// CardBWGBs is a card's memory bandwidth by class, for its speed there.
+	CardBWGBs map[string]float64
 }
 
 // PoolKind is one class of a node's devices.
@@ -119,6 +137,10 @@ type Option struct {
 	// NeedGB and HaveGB are then about.
 	StrataFamily string `json:"strata_family,omitempty"`
 	StrataSize   string `json:"strata_size,omitempty"`
+	// TokensPerSecond is about how many tokens a second one reply gets this
+	// way, on the fastest host it fits; 0 when not known. On the CPU with
+	// the threads a replica is given by default: half the node's.
+	TokensPerSecond float64 `json:"tokens_per_second,omitempty"`
 }
 
 // GPUs is every card one replica takes.
@@ -222,6 +244,9 @@ func Options(arts []Artifact, sh Shape, nodes []PoolNode, context int) []Option
 			out = append(out, vllmOptions(a, kv, sh, nodes, bpw)...)
 		}
 	}
+	for i := range out {
+		out[i].TokensPerSecond = speedOf(out[i], bytesOf(arts, out[i].Artifact), sh, nodes)
+	}
 	Rank(out)
 	return out
 }
@@ -539,3 +564,37 @@ func CardBudget(class string) float64 { return float64(MemFromClass(class)) * gp
 func SplitsHeads(kvHeads, width int) bool {
 	return kvHeads == 0 || width <= 1 || kvHeads%width == 0 || width%kvHeads == 0
 }
+
+func bytesOf(arts []Artifact, name string) int64 {
+	for _, a := range arts {
+		if a.Name == name {
+			return a.Bytes
+		}
+	}
+	return 0
+}
+
+// speedOf is about how many tokens a second one reply gets in o, on the
+// fastest node it fits: on the CPU by the node's memory and the threads a
+// replica is given, on cards by their bandwidth.
+func speedOf(o Option, bytes int64, sh Shape, nodes []PoolNode) float64 {
+	best := 0.0
+	for _, n := range nodes {
+		var tps float64
+		switch {
+		case o.CPU:
+			if float64(n.HostMemGB)*hostShare < o.NeedGB {
+				continue
+			}
+			tps = CPUTokensPerSecondOn(int64(float64(bytes)*sh.share()), n.MemBWGBs, n.MemBWCoreGBs, CPUThreads(0, n.CPUs))
+		case o.Engine != "strata":
+			bw := n.CardBWGBs[o.Class]
+			if bw <= 0 {
+				continue
+			}
+			tps = GPUTokensPerSecond(o, bytes, sh.share(), sh.Params, bw, Links{})
+		}
+		best = max(best, tps)
+	}
+	return best
+}

internal/web/admin_mirror.go

@@ -370,6 +370,17 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
 				if err != nil {
 					return "", err
 				}
+				// Slow enough that a reply is minutes is a choice to make on
+				// purpose too, GPUs free or not: on a fleet whose every node
+				// has cards, the CPU is theirs, and some are slow.
+				if prof := model.Profiles[0]; !judge && prof.CPUOnly() && r.FormValue("cpu_ok") == "" {
+					share := p.checkSource(ctx, model.Source).Header.ActiveShare()
+					hosts := cpuHosts(p.fleet, prof.MinMemoryGB)
+					if tps, node := bestCPUSpeed(hosts, int64(float64(size)*share), prof.CPUCores); tps > 0 && tps < modelplan.CPUTooSlowTPS {
+						return "", errors.New(l.T("serve.err.cpu_too_slow", model.Name, l.Num(tps, 1), node,
+							l.Int(int64(500/tps/60+0.5))))
+					}
+				}
 				// The weights and the context's cache have to fit the card:
 				// an 8B asked for a 131072-token pool on 24 GB loaded its
 				// weights and failed to make its context, again and again.

internal/web/admin_serve.go

@@ -15,6 +15,7 @@ import (
 	"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/strata"
 )
 
@@ -86,6 +87,20 @@ type serveForm struct {
 	// version of a family Strata runs, served on it.
 	StrataFamily, StrataSize string
 
+	// CPUHosts are the nodes a CPU replica of it could run on, with their
+	// memory's rates, and ReadBytes what a token of it reads: what the page
+	// reckons its speed on the CPU from, live, as the cores change.
+	CPUHosts  []speedHost
+	ReadBytes int64
+	// CPUSpeed is about how many tokens a second one reply gets on the CPU,
+	// on CPUSpeedNode with the cores the form starts with; 0 when unknown.
+	CPUSpeed     float64
+	CPUSpeedNode string
+	// GPUSpeed is the same on one card of the fleet's fastest class,
+	// GPUSpeedClass; 0 when the fleet has none it knows.
+	GPUSpeed      float64
+	GPUSpeedClass string
+
 	// Refusal is set when this version cannot be served as it stands, and says
 	// why. Offered and then failing at placement is worse than not offered.
 	Refusal string
@@ -226,6 +241,13 @@ func planServe(l *Lang, st mirror.Status, files []mirror.Status, base string,
 		}
 		f.CPUCores = catalog.SuggestCores(cpus)
 	}
+	// How fast it would answer: on the CPU, and on a card for comparison.
+	if f.Engine == "llamacpp" {
+		f.ReadBytes = int64(float64(weights) * header.ActiveShare())
+		f.CPUHosts = cpuHosts(fleet, f.MinMemoryGB)
+		f.CPUSpeed, f.CPUSpeedNode = bestCPUSpeed(f.CPUHosts, f.ReadBytes, f.CPUCores)
+		f.GPUSpeed, f.GPUSpeedClass = bestGPUSpeed(fleet, weights, header.ActiveShare())
+	}
 	if biggest := biggestNode(fleet); biggest > 0 && f.MinMemoryGB > biggest {
 		big := l.T("serve.warn.too_big", biggest, f.MinMemoryGB)
 		if f.Warning != "" {
@@ -675,3 +697,63 @@ func (p *AccountPages) asJudge(m *catalog.Model) (judge bool, renamed string) {
 	}
 	return true, renamed
 }
+
+// speedHost is a node a CPU replica could run on, as its speed is reckoned:
+// its memory's rate with every CPU and with one, and its CPUs.
+type speedHost struct {
+	ID   string  `json:"id"`
+	All  float64 `json:"all"`
+	Core float64 `json:"core"`
+	CPUs int     `json:"cpus"`
+}
+
+// cpuHosts are the ready nodes whose RAM holds memGB and whose agents
+// measured their memory.
+func cpuHosts(fleet Source, memGB int) []speedHost {
+	if fleet == nil {
+		return nil
+	}
+	var out []speedHost
+	for _, n := range fleet.Nodes() {
+		if n.State != "ready" || n.MemBandwidthGBs <= 0 || (n.MemoryGB > 0 && n.MemoryGB < memGB) {
+			continue
+		}
+		out = append(out, speedHost{ID: n.ID, All: n.MemBandwidthGBs, Core: n.MemBandwidthCoreGBs, CPUs: n.CPUCount})
+	}
+	return out
+}
+
+// bestCPUSpeed is about how many tokens a second one reply gets on the
+// fastest of hosts, reading read bytes a token on cores threads (zero takes
+// what the agent gives: half the node's), and that node.
+func bestCPUSpeed(hosts []speedHost, read int64, cores int) (float64, string) {
+	best, node := 0.0, ""
+	for _, h := range hosts {
+		if tps := modelplan.CPUTokensPerSecondOn(read, h.All, h.Core, modelplan.CPUThreads(cores, h.CPUs)); tps > best {
+			best, node = tps, h.ID
+		}
+	}
+	return best, node
+}
+
+// bestGPUSpeed is the same for one card of the fastest class the fleet has
+// whose memory holds the weights, by its bandwidth, and that class.
+func bestGPUSpeed(fleet Source, weights int64, share float64) (float64, string) {
+	if fleet == nil || weights <= 0 {
+		return 0, ""
+	}
+	best, class := 0.0, ""
+	for _, n := range fleet.Nodes() {
+		for _, g := range n.GPUs {
+			bw := modelplan.GPUBandwidthGBs(g.Name)
+			if bw <= 0 || g.MemoryTotalMB > 0 && int64(g.MemoryTotalMB)<<20 < weights+weights/10 {
+				continue
+			}
+			o := modelplan.Option{Engine: "llamacpp", Class: g.Class, GPUsPerNode: 1}
+			if tps := modelplan.GPUTokensPerSecond(o, weights, share, 0, bw, modelplan.Links{}); tps > best {
+				best, class = tps, g.Class
+			}
+		}
+	}
+	return best, class
+}

internal/web/admin_serve_test.go

@@ -590,3 +590,64 @@ func TestAStrataFamilysVersionIsServedOnStrata(t *testing.T) {
 		t.Error("an unknown size was taken")
 	}
 }
+
+// speedFleet is two nodes with GPUs whose memory rates were measured: a
+// large server and a small one, as on a fleet where every node has cards.
+type speedFleet struct{ fakeFleet }
+
+func (speedFleet) Nodes() []adminapi.NodeView {
+	return []adminapi.NodeView{
+		{ID: "big", State: "ready", MemoryGB: 512, CPUCount: 64, MemBandwidthGBs: 300, MemBandwidthCoreGBs: 10,
+			Engines: []string{"llamacpp"}},
+		{ID: "small", State: "ready", MemoryGB: 64, CPUCount: 8, MemBandwidthGBs: 40, MemBandwidthCoreGBs: 10,
+			Engines: []string{"llamacpp"}},
+	}
+}
+
+// The speed on the CPU follows the threads a replica is given: four of a
+// large server read at four cores' rate, not the server's.
+func TestTheCPUsSpeedFollowsTheCoresGiven(t *testing.T) {
+	hosts := cpuHosts(speedFleet{}, 32)
+	if len(hosts) != 2 {
+		t.Fatalf("hosts %+v", hosts)
+	}
+	few, _ := bestCPUSpeed(hosts, 18e9, 4)
+	half, node := bestCPUSpeed(hosts, 18e9, 0)
+	if few > 2.5 || half < 8 || node != "big" {
+		t.Errorf("four cores %.1f tok/s, half the big node's %.1f on %s", few, half, node)
+	}
+	if hosts := cpuHosts(speedFleet{}, 100); len(hosts) != 1 || hosts[0].ID != "big" {
+		t.Errorf("a model of 100 GB fits the big node only: %+v", hosts)
+	}
+}
+
+// Slow enough that a reply is minutes, the CPU is a choice made on purpose,
+// GPUs free or not.
+func TestAVerySlowModelOnTheCPUIsAChoiceMadeOnPurpose(t *testing.T) {
+	big := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Length", "18000000000")
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(big.Close)
+	values := func(extra ...string) url.Values {
+		v := url.Values{
+			"csrf": {"tok"}, "name": {"big-dense"}, "source": {big.URL + "/m/v1/m.gguf"}, "engine": {"llamacpp"},
+			"replicas": {"1"}, "context_length": {"4096"}, "placement": {"cpu"}, "min_memory_gb": {"24"},
+			"cpu_cores": {"4"},
+		}
+		for i := 0; i+1 < len(extra); i += 2 {
+			v.Set(extra[i], extra[i+1])
+		}
+		return v
+	}
+	cat := &fakeCatalog{}
+	page := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{}, Catalog: cat, Fleet: speedFleet{}}, true)
+	body := mirrorPost(t, page, "/admin/mirror/serve", values()).Body.String()
+	if len(cat.added) != 0 || !strings.Contains(body, "tokens a second on the CPU") {
+		t.Fatalf("an 18 GB model on four cores went through unasked: %s", body)
+	}
+	mirrorPost(t, page, "/admin/mirror/serve", values("cpu_ok", "yes"))
+	if len(cat.added) != 1 {
+		t.Fatal("ticking the box should add it")
+	}
+}

internal/web/explore_fit.go

@@ -43,9 +43,17 @@ type clusterFit struct {
 
 // cpuHost is a node as a model on its CPU sees it.
 type cpuHost struct {
-	id    string
-	ramGB int
-	bwGBs float64
+	id     string
+	ramGB  int
+	cpus   int
+	bwGBs  float64
+	coreBW float64 // one core's rate, 0 from an older agent
+}
+
+// tps is about how many tokens a second one reply of a model whose tokens
+// each read bytes gets here, with the threads a replica is given by default.
+func (h cpuHost) tps(bytes int64) float64 {
+	return modelplan.CPUTokensPerSecondOn(bytes, h.bwGBs, h.coreBW, modelplan.CPUThreads(0, h.cpus))
 }
 
 // fp8Capability is the oldest card vLLM runs an FP8 checkpoint on: Ampere,
@@ -100,7 +108,8 @@ func (p *AccountPages) fitOf() *clusterFit {
 			}
 		}
 		if n.State == "ready" && n.MemBandwidthGBs > 0 {
-			c.cpus = append(c.cpus, cpuHost{id: n.ID, ramGB: n.MemoryGB, bwGBs: n.MemBandwidthGBs})
+			c.cpus = append(c.cpus, cpuHost{id: n.ID, ramGB: n.MemoryGB, cpus: n.CPUCount,
+				bwGBs: n.MemBandwidthGBs, coreBW: n.MemBandwidthCoreGBs})
 		}
 		for _, s := range n.ComputeCapabilities {
 			if v, err := strconv.ParseFloat(s, 64); err == nil && v > newest[n.ID] {
@@ -268,17 +277,16 @@ func (c *clusterFit) holds(l *Lang, name string, bytes, params int64) (gguf.Verd
 // bytes on the fastest node whose RAM holds it, and that node; share is the
 // part a token reads (ActiveShare). 0 when no measured node holds it.
 func (c *clusterFit) cpuSpeed(bytes int64, share float64) (float64, string) {
+	read := int64(float64(bytes) * share)
 	var best cpuHost
+	bestTPS := 0.0
 	for _, n := range c.cpus {
 		need := float64(bytes) + float64(1<<30) + float64(bytes)/20
-		if need <= float64(n.ramGB)*0.85*(1<<30) && n.bwGBs > best.bwGBs {
-			best = n
+		if tps := n.tps(read); need <= float64(n.ramGB)*0.85*(1<<30) && tps > bestTPS {
+			best, bestTPS = n, tps
 		}
 	}
-	if best.id == "" {
-		return 0, ""
-	}
-	return modelplan.CPUTokensPerSecond(int64(float64(bytes)*share), best.bwGBs), best.id
+	return bestTPS, best.id
 }
 
 // speedLabel is a rate as the page shows it: whole tokens from ten up, one

internal/web/locales/en/serve.json

@@ -46,6 +46,12 @@
   "serve.pool": "context pool %s",
   "serve.will_run_as": "Will run as: %s.",
   "serve.free_gpus_warning": "%s GPUs are free; on the CPU this runs many times slower.",
+  "serve.speed_cpu": "about %s tokens a second on %s",
+  "serve.speed_gpu": "on one %s card about %s",
+  "serve.speed_slow": "Slow: a reply of 500 tokens takes about %s seconds.",
+  "serve.speed_too_slow": "Very slow: a reply of 500 tokens takes about %s minutes.",
+  "serve.cpu_ok_slow": "Run on the CPU anyway, knowing it will be slow",
+  "serve.err.cpu_too_slow": "%s would answer at about %s tokens a second on the CPU, at best on %s: a reply of 500 tokens in about %s minutes. Choose a GPU placement, or tick \"run on the CPU anyway\"",
 
   "serve.pool_note": "The context pool is what the engine is given; llama.cpp divides it across its parallel slots, so the number one request may use is the smaller one above.",
   "serve.memory_note": "Memory: %s. Change the context pool and this changes with it, by the KV cache's share — an estimate, and editable. It is filled in rather than left blank because a CPU profile has no card to be short of: with no figure, every ready node is a candidate and the first one picked is whichever the scheduler reached, not whichever can hold the model. A node label does the same job by name if you would rather pin it.",

internal/web/locales/tr/serve.json

@@ -45,6 +45,12 @@
   "serve.pool": "%s token bağlam havuzu",
   "serve.will_run_as": "Şöyle çalışacak: %s.",
   "serve.free_gpus_warning": "%s GPU boşta; CPU üzerinde bu model kat kat yavaş çalışır.",
+  "serve.speed_cpu": "%[2]s üzerinde yaklaşık %[1]s token/sn",
+  "serve.speed_gpu": "tek bir %s kartında yaklaşık %s token/sn",
+  "serve.speed_slow": "Yavaş: 500 token'lık bir cevap yaklaşık %s saniye sürer.",
+  "serve.speed_too_slow": "Çok yavaş: 500 token'lık bir cevap yaklaşık %s dakika sürer.",
+  "serve.cpu_ok_slow": "Yine de CPU üzerinde çalıştır: yavaş olacağını biliyorum",
+  "serve.err.cpu_too_slow": "%s CPU üzerinde yaklaşık %s token/sn ile cevap verir (en iyi %s üzerinde): 500 token'lık bir cevap yaklaşık %s dakika. Bir GPU yerleşimi seçin ya da \"yine de CPU üzerinde çalıştır\" kutusunu işaretleyin",
 
   "serve.pool_note": "Bağlam havuzu motora verilen miktardır; llama.cpp onu paralel yuvalarına böler, bu yüzden tek bir isteğin kullanabileceği sayı yukarıdaki küçük olandır.",
   "serve.memory_note": "Bellek: %s. Bağlam havuzunu değiştirirseniz bu da KV önbelleğinin payı kadar değişir — bir tahmindir ve düzenlenebilir. Boş bırakılmak yerine doldurulur, çünkü bir CPU profilinin yetersiz kalacak bir kartı yoktur: bir değer olmadan hazır her düğüm adaydır ve ilk seçilen, modeli taşıyabilen değil, zamanlayıcının ilk ulaştığı düğüm olur. Sabitlemeyi tercih ederseniz bir düğüm etiketi aynı işi adla görür.",

internal/web/templates/admin_mirror.html

@@ -177,8 +177,8 @@
     {{if eq .Engine "llamacpp"}}<label>{{t "serve.parallel"}} <span class="opt">{{t "serve.parallel_hint"}}</span>
       <input type="text" name="parallel" value="{{.Parallel}}" size="3"></label>{{end}}
   </div>
-  {{if .FreeGPUs}}<label class="cpu-ok"><input type="checkbox" name="cpu_ok" value="yes">
-    {{t "serve.cpu_ok" .FreeGPUs}}</label>{{end}}
+  {{if and (eq .Engine "llamacpp") (or .FreeGPUs .CPUHosts)}}<label class="cpu-ok"><input type="checkbox" name="cpu_ok" value="yes">
+    {{if .FreeGPUs}}{{t "serve.cpu_ok" .FreeGPUs}}{{else}}{{t "serve.cpu_ok_slow"}}{{end}}</label>{{end}}
   <div class="field-row">
     <label>{{t "serve.memory"}} <span class="opt">{{t "serve.memory_hint"}}</span>
       <input type="text" name="min_memory_gb" value="{{.MinMemoryGB}}" required></label>
@@ -204,7 +204,29 @@
     mem: {{t "serve.host_memory"}}, cores: {{t "serve.cores.other"}},
     poolPar: {{t "serve.pool_parallel"}}, pool: {{t "serve.pool"}},
     runAs: {{t "serve.will_run_as"}}, free: {{t "serve.free_gpus_warning"}},
+    speedCPU: {{t "serve.speed_cpu"}}, speedGPU: {{t "serve.speed_gpu"}},
+    slow: {{t "serve.speed_slow"}}, tooSlow: {{t "serve.speed_too_slow"}},
   };
+  // How fast one reply comes on the CPU: what a token reads over the rate a
+  // replica's threads read at — each about one core's, up to the machine's —
+  // and 5 ms a token besides. modelplan.CPUTokensPerSecondOn, in the page so
+  // it follows the cores as they are typed.
+  const hosts = {{.CPUHosts}} || [], readBytes = {{.ReadBytes}};
+  const gpuTPS = {{.GPUSpeed}}, gpuClass = {{.GPUSpeedClass}};
+  const slowTPS = 15, tooSlowTPS = 5;
+  function cpuTPS(cores) {
+    let best = 0, node = "";
+    for (const h of hosts) {
+      const threads = cores > 0 ? cores : Math.max(Math.floor(h.cpus / 2), 1);
+      let read = h.core > 0 && threads > 0 ? Math.min(0.95 * h.core * Math.pow(threads, 0.9), h.all) : h.all * 0.68;
+      if (read <= 0 || readBytes <= 0) continue;
+      const tps = 1 / (readBytes / (read * 1e9) + 0.005);
+      if (tps > best) { best = tps; node = h.id; }
+    }
+    return [best, node];
+  }
+  const nf = new Intl.NumberFormat(document.documentElement.lang || undefined, { maximumFractionDigits: 1 });
+  const rate = (v) => nf.format(v < 10 ? Math.round(v * 10) / 10 : Math.round(v));
   const fmt = (s, ...a) => {
     let i = 0;
     return s.replace(/%(?:\[(\d+)\])?[sd]/g, (_, n) => { const k = n ? +n - 1 : i; i = k + 1; return String(a[k]); });
@@ -226,11 +248,21 @@
     const pool = parseInt(v("ctx_size"), 10), par = parseInt(v("parallel"), 10);
     if (pool && par) parts.push(fmt(T.poolPar, pool, par, Math.floor(pool / par)));
     else if (pool) parts.push(fmt(T.pool, pool));
+    let tps = 0;
+    if (onCPU && f.dataset.engine === "llamacpp") {
+      const r = cpuTPS(parseInt(v("cpu_cores"), 10) || 0);
+      tps = r[0];
+      if (tps > 0) parts.push(fmt(T.speedCPU, rate(tps), r[1]));
+      if (gpuTPS > 0) parts.push(fmt(T.speedGPU, gpuClass, rate(gpuTPS)));
+    }
     out.textContent = fmt(T.runAs, parts.join(", "));
-    const warn = onCPU && +f.dataset.freeGpus > 0;
-    out.classList.toggle("error", warn);
-    if (warn) out.textContent += " " + fmt(T.free, f.dataset.freeGpus);
-    if (ok) ok.hidden = !onCPU;
+    const tooSlow = tps > 0 && tps < tooSlowTPS, slow = tps > 0 && tps < slowTPS;
+    const free = onCPU && +f.dataset.freeGpus > 0;
+    out.classList.toggle("error", free || tooSlow);
+    if (tooSlow) out.textContent += " " + fmt(T.tooSlow, Math.max(1, Math.round(500 / tps / 60)));
+    else if (slow) out.textContent += " " + fmt(T.slow, Math.round(500 / tps));
+    if (free) out.textContent += " " + fmt(T.free, f.dataset.freeGpus);
+    if (ok) ok.hidden = !(onCPU && (free || tooSlow));
     out.hidden = false;
   }
   f.addEventListener("input", draw);