diff --git a/CHANGELOG.md b/CHANGELOG.md index d307a1e..aec9e64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,14 @@ APIs change between minors without a deprecation period. ### Added +- **How fast a model would answer on a CPU, before it is fetched.** Each + agent measures how fast its node's memory reads when it starts (`Host` + gets `mem_bandwidth_gbs`, the node's view `mem_bandwidth_gbs`); the + explorer's **CPU tok/s** column and a repository's GGUF files say about + how many tokens a second one reply gets on the fastest node that holds + the model — the bandwidth over the bytes a token reads, its active part + for a model of experts. Calibrated on the PoC against llama.cpp's own + timings: within 15% from 1.5B to 14B. - **The explorer lists what this cluster can run.** Judged by the nodes as they are now — every card's memory and compute capability, every node's RAM — through the same planner as serving a model, rather than a diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 5f25bc7..30b1fd5 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -1206,9 +1206,12 @@ type Host struct { CpuFlags []string `protobuf:"bytes,5,rep,name=cpu_flags,json=cpuFlags,proto3" json:"cpu_flags,omitempty"` // nvidia_driver is the driver version, empty when there is none. A card // without a driver is not a GPU anything can use. - NvidiaDriver string `protobuf:"bytes,6,opt,name=nvidia_driver,json=nvidiaDriver,proto3" json:"nvidia_driver,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + NvidiaDriver string `protobuf:"bytes,6,opt,name=nvidia_driver,json=nvidiaDriver,proto3" json:"nvidia_driver,omitempty"` + // mem_bandwidth_gbs is how fast the memory reads, in GB/s, measured when + // the agent started: what decides a model's speed on the CPU. 0 unknown. + MemBandwidthGbs float64 `protobuf:"fixed64,7,opt,name=mem_bandwidth_gbs,json=memBandwidthGbs,proto3" json:"mem_bandwidth_gbs,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Host) Reset() { @@ -1283,6 +1286,13 @@ func (x *Host) GetNvidiaDriver() string { return "" } +func (x *Host) GetMemBandwidthGbs() float64 { + if x != nil { + return x.MemBandwidthGbs + } + return 0 +} + // Storage is one filesystem a node keeps weights on. // // Per filesystem rather than per configured directory: free space belongs to @@ -5190,7 +5200,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" + "\x13EngineVersionsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xb1\x01\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xdd\x01\n" + "\x04Host\x12\x13\n" + "\x05os_id\x18\x01 \x01(\tR\x04osId\x12\x1d\n" + "\n" + @@ -5199,7 +5209,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "osCodename\x12\x12\n" + "\x04arch\x18\x04 \x01(\tR\x04arch\x12\x1b\n" + "\tcpu_flags\x18\x05 \x03(\tR\bcpuFlags\x12#\n" + - "\rnvidia_driver\x18\x06 \x01(\tR\fnvidiaDriver\"u\n" + + "\rnvidia_driver\x18\x06 \x01(\tR\fnvidiaDriver\x12*\n" + + "\x11mem_bandwidth_gbs\x18\a \x01(\x01R\x0fmemBandwidthGbs\"u\n" + "\aStorage\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" + "\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 4afe00d..cf7ac19 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -238,6 +238,9 @@ message Host { // nvidia_driver is the driver version, empty when there is none. A card // without a driver is not a GPU anything can use. string nvidia_driver = 6; + // mem_bandwidth_gbs is how fast the memory reads, in GB/s, measured when + // the agent started: what decides a model's speed on the CPU. 0 unknown. + double mem_bandwidth_gbs = 7; } // Storage is one filesystem a node keeps weights on. diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index 2b5fb86..76d36f4 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -8011,6 +8011,10 @@ "type": "integer", "description": "CPUCount is the machine's CPUs, 0 when the agent did not report them." }, + "mem_bandwidth_gbs": { + "type": "number", + "description": "MemBandwidthGBs is how fast its memory reads, in GB/s, measured when its agent started; 0 when not reported. A model's speed on the CPU." + }, "engines": { "type": "array", "items": { diff --git a/docs/quick-install.md b/docs/quick-install.md index c3a5561..c474380 100644 --- a/docs/quick-install.md +++ b/docs/quick-install.md @@ -154,6 +154,15 @@ Without GPUs it lists GGUF only. Untick **only what these nodes can run** to see everything. On a model's page a file too big for the cluster is shown without a **Fetch** button, with what it would need. +The **CPU tok/s** column is about how fast one reply would come on the CPU of +the fastest node whose memory holds the model at Q4_K_M; a model's page gives +it for each GGUF file by its own size. Every node's agent measures how fast its +memory reads when it starts (it is in the agent's log as `memory_gbs`), and a +token on a CPU reads every weight once: the estimate is that speed over the +model's size — for a model of experts named like `30B-A3B`, its active part. +On the test cluster it came within 15% of what llama.cpp measured. Several +replies at once share the reads and each is slower. + 1. Click a model to read its card. 2. Pick one of its files, usually a `q4_k_m` for a first try, and click **Fetch**. It is downloaded into the mirror once, for every node. diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 107e39a..25759e0 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -12,6 +12,7 @@ import ( "io" "log/slog" "maps" + "math" "math/rand/v2" "net" "net/http" @@ -513,12 +514,14 @@ func (a *Agent) scan(ctx context.Context) error { inv, err := discovery.Discover(ctx, opts, a.log) host := discovery.DiscoverHost() host.MinComputeCapability = discovery.MinComputeCapability(inv.GPUs) + host.MemBandwidthGBs = discovery.MeasureMemoryBandwidth() a.mu.Lock() a.inv = inv a.prov.Host = host a.mu.Unlock() a.log.Info("this node", "os", host.OSID+" "+host.OSVersion, "arch", host.Arch, - "cpu_flags", host.CPUFlags, "nvidia_driver", host.NVIDIADriver) + "cpu_flags", host.CPUFlags, "nvidia_driver", host.NVIDIADriver, + "memory_gbs", math.Round(host.MemBandwidthGBs*10)/10) return err } @@ -569,6 +572,7 @@ func (a *Agent) session(ctx context.Context, server string) error { reg.Host = &llmgwv1.Host{ OsId: h.OSID, OsVersion: h.OSVersion, OsCodename: h.OSCodename, Arch: h.Arch, CpuFlags: h.CPUFlags, NvidiaDriver: h.NVIDIADriver, + MemBandwidthGbs: h.MemBandwidthGBs, } if err := stream.Send(&llmgwv1.AgentMessage{ diff --git a/internal/agent/discovery/host.go b/internal/agent/discovery/host.go index 2e99e7c..f7923e5 100644 --- a/internal/agent/discovery/host.go +++ b/internal/agent/discovery/host.go @@ -31,6 +31,10 @@ type Host struct { // MinComputeCapability is the oldest GPU's, 0 when there are none or it // is not known. Not read here but from the GPU inventory, by the agent. MinComputeCapability float64 `json:"min_compute_capability,omitempty"` + // MemBandwidthGBs is how fast the memory reads, in GB/s, 0 when not + // measured: what decides a model's speed on the CPU. Not read here but + // measured by the agent when it starts (MeasureMemoryBandwidth). + MemBandwidthGBs float64 `json:"mem_bandwidth_gbs,omitempty"` } // Family is the packaging family: "deb", "rpm", or "" when neither is known. diff --git a/internal/agent/discovery/membw.go b/internal/agent/discovery/membw.go new file mode 100644 index 0000000..866064b --- /dev/null +++ b/internal/agent/discovery/membw.go @@ -0,0 +1,113 @@ +package discovery + +import ( + "bufio" + "os" + "runtime" + "strconv" + "strings" + "sync" + "time" +) + +// How fast this machine reads its memory, which on a CPU is what decides how +// fast a model answers: every token of a reply reads all the weights it uses +// once, so a node that reads 40 GB/s writes a 4 GB model's tokens about ten +// times a second, whatever its core count. Measured once when the agent +// starts, in a fraction of a second, by every CPU summing a share of a buffer +// too large for any cache — a VM sharing its host's memory with others gets +// its share, which is the number that matters for it. + +// bandwidthBuffer is what is read: well past any processor's cache. +const bandwidthBuffer = 512 << 20 + +// MeasureMemoryBandwidth is the memory's read rate in GB/s, the best of a +// few passes; 0 when the machine has too little free memory to measure. +func MeasureMemoryBandwidth() float64 { + size := int64(bandwidthBuffer) + // Never more than a tenth of what is free: a node already holding models + // must not be pushed into swap by a measurement. + if avail := memAvailable("/proc/meminfo"); avail > 0 { + size = min(size, avail/10) + } + if size < 64<<20 { + return 0 + } + buf := make([]uint64, size/8) + for i := range buf { + buf[i] = uint64(i) + } + workers := runtime.NumCPU() + chunk := (len(buf) + workers - 1) / workers + best := 0.0 + for pass := 0; pass < 4; pass++ { + var wg sync.WaitGroup + start := time.Now() + for w := 0; w < workers; w++ { + lo, hi := w*chunk, min((w+1)*chunk, len(buf)) + if lo >= hi { + continue + } + wg.Add(1) + go func(part []uint64) { + defer wg.Done() + sink(sum(part)) + }(buf[lo:hi]) + } + wg.Wait() + if s := time.Since(start).Seconds(); s > 0 { + best = max(best, float64(size)/s/1e9) + } + } + return best +} + +// sum reads every word, eight at a time so the loop is not what limits it. +func sum(p []uint64) uint64 { + var a, b, c, d, e, f, g, h uint64 + i := 0 + for ; i+8 <= len(p); i += 8 { + a += p[i] + b += p[i+1] + c += p[i+2] + d += p[i+3] + e += p[i+4] + f += p[i+5] + g += p[i+6] + h += p[i+7] + } + for ; i < len(p); i++ { + a += p[i] + } + return a + b + c + d + e + f + g + h +} + +// sinkValue keeps the sums alive, so the reads are not optimised away. +var sinkValue struct { + sync.Mutex + v uint64 +} + +func sink(v uint64) { + sinkValue.Lock() + sinkValue.v += v + sinkValue.Unlock() +} + +// memAvailable is MemAvailable in bytes, 0 when unknown. +func memAvailable(path string) int64 { + f, err := os.Open(path) //nolint:gosec // a fixed system path + if err != nil { + return 0 + } + defer f.Close() + sc := bufio.NewScanner(f) + for sc.Scan() { + fields := strings.Fields(sc.Text()) + if len(fields) >= 2 && fields[0] == "MemAvailable:" { + kb, _ := strconv.ParseInt(fields[1], 10, 64) + return kb << 10 + } + } + return 0 +} diff --git a/internal/agent/discovery/membw_test.go b/internal/agent/discovery/membw_test.go new file mode 100644 index 0000000..ec31862 --- /dev/null +++ b/internal/agent/discovery/membw_test.go @@ -0,0 +1,16 @@ +package discovery + +import "testing" + +func TestMemoryBandwidthIsMeasured(t *testing.T) { + if testing.Short() { + t.Skip("reads half a gigabyte") + } + bw := MeasureMemoryBandwidth() + // Any machine running this reads more than a gigabyte a second and less + // than a few terabytes. + if bw < 1 || bw > 5000 { + t.Errorf("measured %.1f GB/s", bw) + } + t.Logf("%.1f GB/s", bw) +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 5c6a7e4..4aeb1fe 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -248,6 +248,9 @@ type NodeView struct { MemoryGB int `json:"memory_gb,omitempty"` // CPUCount is the machine's CPUs, 0 when the agent did not report them. CPUCount int `json:"cpu_count,omitempty"` + // MemBandwidthGBs is how fast its memory reads, in GB/s, measured when + // its agent started; 0 when not reported. A model's speed on the CPU. + MemBandwidthGBs float64 `json:"mem_bandwidth_gbs,omitempty"` // Engines are the inference engines installed here, as the agent found // them. Absent means the agent is older than the probe and said nothing, // which is not the same as having none — placement treats it that way too, @@ -703,6 +706,7 @@ func (a *API) Nodes() []NodeView { v.EngineBuilds[e] = b } } + v.MemBandwidthGBs = n.Host.MemBandwidthGBs if h := n.Host; h.OSID != "" { v.OS = strings.TrimSpace(h.OSID + " " + h.OSVersion + " " + h.Arch) if h.Arch == "amd64" && !h.HasCPUFlag("avx2") { diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go index a045bcd..6639f54 100644 --- a/internal/control/agentsvc/convert.go +++ b/internal/control/agentsvc/convert.go @@ -34,6 +34,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node { n.Host = discovery.Host{ OSID: h.GetOsId(), OSVersion: h.GetOsVersion(), OSCodename: h.GetOsCodename(), Arch: h.GetArch(), CPUFlags: h.GetCpuFlags(), NVIDIADriver: h.GetNvidiaDriver(), + MemBandwidthGBs: h.GetMemBandwidthGbs(), } } for _, w := range r.GetWeights() { diff --git a/internal/modelplan/cpuspeed.go b/internal/modelplan/cpuspeed.go new file mode 100644 index 0000000..4e2fe29 --- /dev/null +++ b/internal/modelplan/cpuspeed.go @@ -0,0 +1,62 @@ +package modelplan + +import ( + "regexp" + "strconv" +) + +// How fast a model answers on a CPU, from its size and the node's memory. +// +// Writing a token reads every weight it uses once, so on a CPU the memory +// decides the speed: a token takes the bytes read over the rate llama.cpp +// reads them at, plus a little that does not shrink with the model. Measured +// on the PoC on 2026-10-26, each node's bandwidth by MeasureMemoryBandwidth +// against llama.cpp's own predicted_per_second for 128 tokens: +// +// node01 4 vCPU 24.9 GB/s qwen2.5-1.5b Q4_K_M 1.1 GB 14.5 tok/s 0.65 of the bandwidth +// node05 8 vCPU 41.9 GB/s qwen2.5-14b Q4_K_M 9.0 GB 3.3 tok/s 0.71 +// cuda01 12 vCPU 42.4 GB/s qwen2.5-0.5b Q4_K_M 0.5 GB 36.0 tok/s (the fixed cost shows) +// +// So 0.68 of the measured rate, and 5 ms a token besides. One reply at a +// time; several at once share the same reads and each is slower. The PoC's +// host has far fewer cores than its VMs, so these are what a busy host +// gives, and a machine of its own does better. + +// cpuReadShare is the part of the measured bandwidth llama.cpp reads at. +const cpuReadShare = 0.68 + +// cpuTokenFixed is a token's cost that does not grow with the model. +const cpuTokenFixed = 0.005 + +// CPUTokensPerSecond is about how many tokens a second one reply gets on a +// CPU whose memory reads bwGBs, from a model whose tokens each read bytes. +// 0 when either is unknown. +func CPUTokensPerSecond(bytes int64, bwGBs float64) float64 { + if bytes <= 0 || bwGBs <= 0 { + return 0 + } + return 1 / (float64(bytes)/(bwGBs*1e9*cpuReadShare) + cpuTokenFixed) +} + +// activeRE finds a model of experts' active parameters in its name, the +// way they are written: Qwen3-30B-A3B, Ling-flash-A2.5B. +var activeRE = regexp.MustCompile(`(?i)(?:^|[-_./])a(\d+(?:\.\d+)?)b(?:$|[-_.])`) + +// ActiveShare is the part of a model's weights one token reads: for a model +// of experts named with its active parameters (…-30B-A3B) that over its +// total, otherwise 1. Rough — the shared layers are read for every token — +// and what a name says is all a listing has. +func ActiveShare(name string, params int64) float64 { + m := activeRE.FindStringSubmatch(name) + if m == nil || params <= 0 { + return 1 + } + b, err := strconv.ParseFloat(m[1], 64) + if err != nil || b <= 0 { + return 1 + } + if s := b * 1e9 / float64(params); s < 1 { + return s + } + return 1 +} diff --git a/internal/modelplan/cpuspeed_test.go b/internal/modelplan/cpuspeed_test.go new file mode 100644 index 0000000..f60a54a --- /dev/null +++ b/internal/modelplan/cpuspeed_test.go @@ -0,0 +1,37 @@ +package modelplan + +import ( + "math" + "testing" +) + +func TestCPUSpeedMatchesWhatThePoCMeasured(t *testing.T) { + for _, c := range []struct { + bytes int64 + bw, want float64 + }{ + {1117320736, 24.9, 14.5}, // node01, qwen2.5-1.5b + {9_000_000_000, 41.9, 3.3}, // node05, qwen2.5-coder-14b + } { + got := CPUTokensPerSecond(c.bytes, c.bw) + if math.Abs(got-c.want)/c.want > 0.15 { + t.Errorf("%d bytes at %.1f GB/s: %.1f tok/s, measured %.1f", c.bytes, c.bw, got, c.want) + } + } + if CPUTokensPerSecond(0, 40) != 0 || CPUTokensPerSecond(1e9, 0) != 0 { + t.Error("an unknown size or bandwidth should give no figure") + } +} + +func TestAModelOfExpertsReadsItsActiveShare(t *testing.T) { + for name, want := range map[string]float64{ + "Qwen/Qwen3-30B-A3B-Instruct-2507-GGUF": 0.1, + "inclusionAI/Ling-flash-A2.5B": 2.5 / 30, + "Qwen/Qwen3-8B": 1, + "meta-llama/Llama-3.1-8B-Instruct": 1, + } { + if got := ActiveShare(name, 30e9); math.Abs(got-want) > 0.001 && !(want == 1 && got == 1) { + t.Errorf("%s: %.3f, want %.3f", name, got, want) + } + } +} diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go index e8ef983..dc6d2bb 100644 --- a/internal/web/admin_mirror.go +++ b/internal/web/admin_mirror.go @@ -20,6 +20,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" + "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" "gitea.mynodes.xyz/ilker/llmgw/internal/safedial" "gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck" "gitea.mynodes.xyz/ilker/llmgw/internal/store" @@ -57,6 +58,9 @@ type repoSet struct { // architecture the registry reports; a set that cannot be served is // shown with the reason and no fetch button. Verdict gguf.Verdict + // CPUSpeed is about how many tokens a second one reply gets from a GGUF + // on the CPU of CPUNode, the fastest node that holds it; empty otherwise. + CPUSpeed, CPUNode string hf.FileSet // Sources are every file in the set, in the order the registry listed them. // A shard set goes as one action — a fifteen-shard model is not fifteen @@ -820,6 +824,9 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string if fit != nil && rs.Verdict.Servable { if v, ok := fit.holds(langOf(r), set.Name, set.Bytes, got.Params); !ok { rs.Verdict = v + } else if gguf.IsGGUF(set.Name) { + tps, node := fit.cpuSpeed(set.Bytes, modelplan.ActiveShare(got.ID, got.Params)) + rs.CPUSpeed, rs.CPUNode = speedLabel(tps), node } } for _, f := range set.Files { diff --git a/internal/web/explore_fit.go b/internal/web/explore_fit.go index d95cd44..dced133 100644 --- a/internal/web/explore_fit.go +++ b/internal/web/explore_fit.go @@ -29,6 +29,16 @@ type clusterFit struct { card int cc string host int + // cpus are the nodes whose memory's speed their agent measured, for + // how fast a model would answer on one. + cpus []cpuHost +} + +// cpuHost is a node as a model on its CPU sees it. +type cpuHost struct { + id string + ramGB int + bwGBs float64 } // fp8Capability is the oldest card vLLM runs FP8 weights on. @@ -47,6 +57,9 @@ func (p *AccountPages) fitOf() *clusterFit { } newest := map[string]float64{} for _, n := range nodes { + if n.State == "ready" && n.MemBandwidthGBs > 0 { + c.cpus = append(c.cpus, cpuHost{id: n.ID, ramGB: n.MemoryGB, bwGBs: n.MemBandwidthGBs}) + } for _, s := range n.ComputeCapabilities { if v, err := strconv.ParseFloat(s, 64); err == nil && v > newest[n.ID] { newest[n.ID] = v @@ -179,3 +192,45 @@ func (c *clusterFit) holds(l *Lang, name string, bytes, params int64) (gguf.Verd } return gguf.Verdict{Checked: true, Reason: l.T("card.too_big", best.NeedGB, best.HaveGB)}, false } + +// cpuSpeed is about how many tokens a second one reply gets from a GGUF of +// bytes on the fastest node whose RAM holds it, and that node; share is the +// part a token reads (ActiveShare). 0 when no measured node holds it. +func (c *clusterFit) cpuSpeed(bytes int64, share float64) (float64, string) { + var best cpuHost + for _, n := range c.cpus { + need := float64(bytes) + float64(1<<30) + float64(bytes)/20 + if need <= float64(n.ramGB)*0.85*(1<<30) && n.bwGBs > best.bwGBs { + best = n + } + } + if best.id == "" { + return 0, "" + } + return modelplan.CPUTokensPerSecond(int64(float64(bytes)*share), best.bwGBs), best.id +} + +// speedLabel is a rate as the page shows it: whole tokens from ten up, one +// decimal below, and under one said so. +func speedLabel(tps float64) string { + switch { + case tps <= 0: + return "" + case tps < 1: + return "<1" + case tps < 10: + return "~" + strconv.FormatFloat(tps, 'f', 1, 64) + } + return "~" + strconv.FormatFloat(tps, 'f', 0, 64) +} + +// rowSpeed is a listed repository's speed on the CPU, judged at Q4_K_M: +// GGUF only, which is what runs there. +func (c *clusterFit) rowSpeed(m hf.Model) (string, string) { + if m.GGUF == nil && !m.HasTag("gguf") || m.Params() <= 0 { + return "", "" + } + b := int64(float64(m.Params()) * ggufBitsPerWeight / 8) + tps, node := c.cpuSpeed(b, modelplan.ActiveShare(m.ID, m.Params())) + return speedLabel(tps), node +} diff --git a/internal/web/explore_fit_test.go b/internal/web/explore_fit_test.go index 96cdf41..adc58bd 100644 --- a/internal/web/explore_fit_test.go +++ b/internal/web/explore_fit_test.go @@ -1,6 +1,7 @@ package web import ( + "html" "strings" "testing" "time" @@ -143,3 +144,28 @@ func TestPackedWeightsAreSizedAsTheyAreStored(t *testing.T) { t.Errorf("both run on cuda01: %s", body) } } + +func TestTheExplorerSaysHowFastAModelAnswersOnACPU(t *testing.T) { + slow, fast := cpuNode("node01", 7), cpuNode("node05", 31) + slow.MemBandwidthGBs, fast.MemBandwidthGBs = 24.9, 41.9 + body, _ := listing(t, []adminapi.NodeView{slow, fast}, + ggufModel("Qwen/Qwen3-8B-GGUF", "qwen3", 8e9, time.Now()), + ggufModel("Qwen/Qwen3-30B-A3B-GGUF", "qwen3moe", 30e9, time.Now()), + ) + // 8B at Q4_K_M is 4.8 GB, on node05's 41.9 GB/s: about 5.8 a second. + // The 30B model of experts reads its 3B active part: about 15. + for _, want := range []string{">~5.8<", ">~15<", "on node05's CPU"} { + if !strings.Contains(body, want) { + t.Errorf("the listing lacks %q: %s", want, body) + } + } + + reg := &fakeRegistry{repo: hf.Repo{ID: "Qwen/Qwen3-8B-GGUF", Revision: "abc123def456", Params: 8e9, + Files: []hf.File{{Path: "Qwen3-8B-Q4_K_M.gguf", Bytes: 5 << 30}, {Path: "Qwen3-8B-Q8_0.gguf", Bytes: 8 << 30}}}} + mux := mirrorPageLlama(t, MirrorDeps{Mirror: &fakeMirror{}, Registry: reg, + Fleet: nodesFleet{nodes: []adminapi.NodeView{slow, fast}}}) + page := html.UnescapeString(mirrorGet(t, mux, "/admin/mirror?repo=Qwen/Qwen3-8B-GGUF").Body.String()) + if !strings.Contains(page, "~5.2 tok/s on node05's CPU") || !strings.Contains(page, "~3.3 tok/s on node05's CPU") { + t.Errorf("each file should say its speed on the CPU, by its own size: %s", page) + } +} diff --git a/internal/web/locales/en/mirror.json b/internal/web/locales/en/mirror.json index 8ee7188..4eb8aad 100644 --- a/internal/web/locales/en/mirror.json +++ b/internal/web/locales/en/mirror.json @@ -82,6 +82,9 @@ "explore.col.downloads": "Downloads", "explore.col.likes": "Likes", "explore.col.updated": "Updated", + "explore.col.cpu": "CPU tok/s", + "explore.col.cpu_title": "About how many tokens a second one reply gets on the CPU of the fastest node whose memory holds it, at Q4_K_M: from the node's measured memory speed and the model's size (for a model of experts, its active part).", + "explore.cpu_on": "on %s's CPU", "explore.new": "new", "explore.gated": "gated", "explore.from": "from %s", @@ -139,6 +142,7 @@ "card.model": "model", "card.cannot_serve": "cannot be served", "card.imatrix": "it is an importance matrix for quantising, not a model", + "card.cpu_speed": "%s tok/s on %s's CPU", "card.too_big": "too big for this cluster: it needs %.0f GB where the most a node here offers it is %.0f GB", "card.no_gpu": "safetensors run on a GPU, and this cluster has none; take a GGUF", "card.pickle_hidden.one": "%d file is not shown: it is in a pickle format, which runs code when it is loaded, and nodes refuse it.", diff --git a/internal/web/locales/tr/mirror.json b/internal/web/locales/tr/mirror.json index d35fae9..803c271 100644 --- a/internal/web/locales/tr/mirror.json +++ b/internal/web/locales/tr/mirror.json @@ -80,6 +80,9 @@ "explore.col.downloads": "İndirme", "explore.col.likes": "Beğeni", "explore.col.updated": "Güncellendi", + "explore.col.cpu": "CPU token/sn", + "explore.col.cpu_title": "Belleği modeli alan en hızlı düğümün CPU'sunda tek bir yanıtın saniyede aşağı yukarı kaç token alacağı, Q4_K_M ile: düğümün ölçülen bellek hızından ve modelin boyutundan (uzmanlar karışımı modelde etkin kısmından).", + "explore.cpu_on": "%s düğümünün CPU'sunda", "explore.new": "yeni", "explore.gated": "erişimi kısıtlı", "explore.from": "temeli: %s", @@ -134,6 +137,7 @@ "card.model": "model", "card.cannot_serve": "sunulamaz", "card.imatrix": "bu bir model değil, nicemleme için bir önem matrisi", + "card.cpu_speed": "%[2]s CPU'sunda %[1]s token/sn", "card.too_big": "bu küme için fazla büyük: %.0f GB gerekiyor, buradaki bir düğümün ona verebileceği en çok %.0f GB", "card.no_gpu": "safetensors GPU'da çalışır ve bu kümede GPU yok; GGUF olanı seçin", "card.pickle_hidden.other": "%d dosya gösterilmiyor: pickle biçiminde, yüklenirken kod çalıştırır ve düğümler onu almaz.", diff --git a/internal/web/mirror_explore.go b/internal/web/mirror_explore.go index 9b80cea..13c3d60 100644 --- a/internal/web/mirror_explore.go +++ b/internal/web/mirror_explore.go @@ -93,6 +93,9 @@ type exploreRow struct { New bool // Updated is how long ago the repository changed, in the page's language. Updated string + // CPUSpeed is about how many tokens a second one reply would get on the + // CPU of CPUNode, the fastest node that holds it; empty when unknown. + CPUSpeed, CPUNode string } // ParamsLabel is the parameter count as people say it: 7.6B, 494M. @@ -311,6 +314,9 @@ func (p *AccountPages) exploreList(r *http.Request, d *adminMirrorData) { e.Hidden++ continue } + if fit != nil { + row.CPUSpeed, row.CPUNode = fit.rowSpeed(m) + } if e.Here && row.Verdict.Checked && !row.Verdict.Servable { e.Hidden++ continue diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html index 25bccb5..c2aba7d 100644 --- a/internal/web/templates/admin_mirror.html +++ b/internal/web/templates/admin_mirror.html @@ -302,6 +302,7 @@ {{if .Rows}} + {{range .Rows}} + @@ -467,7 +469,8 @@ document.querySelector("table.explore").addEventListener("submit", async functio {{if .Shards}}{{t "card.shards" .Shards}}{{end}} {{with .Companions}}{{t "card.companions" (len .)}}{{end}} {{if eq .Kind "adapter"}}{{t "card.adapter" .Arch}} - {{else if eq .Kind "model"}}{{if .Arch}}{{t "card.model_arch" .Arch}}{{else}}{{t "card.model"}}{{end}}{{end}} + {{else if eq .Kind "model"}}{{if .Arch}}{{t "card.model_arch" .Arch}}{{else}}{{t "card.model"}}{{end}}{{end}} + {{if .CPUSpeed}}{{t "card.cpu_speed" .CPUSpeed .CPUNode}}{{end}}
{{t "explore.col.repo"}}{{t "explore.col.size"}}{{t "explore.col.arch"}}{{t "explore.col.context"}}{{t "explore.col.cpu"}} {{t "explore.col.downloads"}}{{t "explore.col.likes"}}{{t "explore.col.updated"}}
@@ -310,6 +311,7 @@ {{.ParamsLabel}} {{if .Architecture}}{{.Architecture}}{{if .Verdict.Checked}}{{if .Verdict.Servable}} ✓{{else}} ✗{{end}}{{end}}{{else}}—{{end}} {{.ContextLabel}}{{if .CPUSpeed}}{{.CPUSpeed}}{{else}}—{{end}} {{compact .Downloads}} {{compact .Likes}} {{.Updated}} {{bytes .Bytes}} {{with .SHA256}}{{slice . 0 12}}{{else}}—{{end}}