package modelplan import ( "regexp" "strconv" ) // How fast a model answers on a CPU, from its size and the node's memory. // // Writing a token reads every weight it uses once, so on a CPU the memory // decides the speed: a token takes the bytes read over the rate llama.cpp // reads them at, plus a little that does not shrink with the model. Measured // on the PoC on 2026-10-26, each node's bandwidth by MeasureMemoryBandwidth // against llama.cpp's own predicted_per_second for 128 tokens: // // node01 4 vCPU 24.9 GB/s qwen2.5-1.5b Q4_K_M 1.1 GB 14.5 tok/s 0.65 of the bandwidth // node05 8 vCPU 41.9 GB/s qwen2.5-14b Q4_K_M 9.0 GB 3.3 tok/s 0.71 // cuda01 12 vCPU 42.4 GB/s qwen2.5-0.5b Q4_K_M 0.5 GB 36.0 tok/s (the fixed cost shows) // // So 0.68 of the measured rate, and 5 ms a token besides. One reply at a // time; several at once share the same reads and each is slower. The PoC's // host has far fewer cores than its VMs, so these are what a busy host // gives, and a machine of its own does better. // cpuReadShare is the part of the measured bandwidth llama.cpp reads at. const cpuReadShare = 0.68 // cpuTokenFixed is a token's cost that does not grow with the model. const cpuTokenFixed = 0.005 // CPUTokensPerSecond is about how many tokens a second one reply gets on a // CPU whose memory reads bwGBs, from a model whose tokens each read bytes. // 0 when either is unknown. func CPUTokensPerSecond(bytes int64, bwGBs float64) float64 { if bytes <= 0 || bwGBs <= 0 { return 0 } return 1 / (float64(bytes)/(bwGBs*1e9*cpuReadShare) + cpuTokenFixed) } // moeRead is what a token of a model of experts reads over its active // share. const moeRead = 1.6 // activeRE finds a model of experts' active parameters in its name, the // way they are written: Qwen3-30B-A3B, Ling-flash-A2.5B. var activeRE = regexp.MustCompile(`(?i)(?:^|[-_./])a(\d+(?:\.\d+)?)b(?:$|[-_.])`) // ActiveShare is the part of a model's weights one token reads: for a model // of experts named with its active parameters (…-30B-A3B) that over its // total, otherwise 1 — and 1.6 times that, which is what Qwen3-Coder-30B-A3B // read on cuda01's card (144 tok/s from 18.6 GB): the shared layers, the // embedding and the routing are read for every token whatever the experts. func ActiveShare(name string, params int64) float64 { m := activeRE.FindStringSubmatch(name) if m == nil || params <= 0 { return 1 } b, err := strconv.ParseFloat(m[1], 64) if err != nil || b <= 0 { return 1 } if s := moeRead * b * 1e9 / float64(params); s < 1 { return s } return 1 }