package modelplan import ( "math" "testing" ) func TestCPUSpeedMatchesWhatThePoCMeasured(t *testing.T) { for _, c := range []struct { bytes int64 bw, want float64 }{ {1117320736, 24.9, 14.5}, // node01, qwen2.5-1.5b {9_000_000_000, 41.9, 3.3}, // node05, qwen2.5-coder-14b } { got := CPUTokensPerSecond(c.bytes, c.bw) if math.Abs(got-c.want)/c.want > 0.15 { t.Errorf("%d bytes at %.1f GB/s: %.1f tok/s, measured %.1f", c.bytes, c.bw, got, c.want) } } if CPUTokensPerSecond(0, 40) != 0 || CPUTokensPerSecond(1e9, 0) != 0 { t.Error("an unknown size or bandwidth should give no figure") } } // Each thread reads at about one core's rate, up to the machine's: what // llama-bench measured on node05 by thread count, and what was measured on // node01 and node05 before, all within 15%. func TestCPUSpeedFollowsTheThreads(t *testing.T) { for _, c := range []struct { bytes int64 all, core float64 threads int want float64 }{ {4683073536, 43.9, 9.7, 1, 1.88}, // node05, qwen2.5-coder-7b {4683073536, 43.9, 9.7, 2, 3.56}, {4683073536, 43.9, 9.7, 4, 6.35}, {4683073536, 43.9, 9.7, 8, 9.29}, {1117320736, 43.9, 9.7, 1, 8.31}, // node05, qwen2.5-1.5b {1117320736, 43.9, 9.7, 4, 25.4}, {1117320736, 34.2, 10.1, 2, 14.5}, // node01 on its two cores {9_000_000_000, 43.9, 9.7, 4, 3.3}, // node05, qwen2.5-14b on four } { got := CPUTokensPerSecondOn(c.bytes, c.all, c.core, c.threads) if math.Abs(got-c.want)/c.want > 0.15 { t.Errorf("%d bytes, %d threads: %.1f tok/s, measured %.1f", c.bytes, c.threads, got, c.want) } } // A large machine, a few threads: the threads decide, not the machine. few := CPUTokensPerSecondOn(18e9, 300, 10, 4) if few > 2.5 { t.Errorf("four threads of a 300 GB/s machine: %.1f tok/s, as though they read at the whole machine's rate", few) } // An agent that did not measure one core: the old reckoning. if got, want := CPUTokensPerSecondOn(1e9, 40, 0, 4), CPUTokensPerSecond(1e9, 40); got != want { t.Errorf("without one core's rate %.2f, want %.2f", got, want) } if CPUThreads(0, 64) != 32 || CPUThreads(6, 64) != 6 || CPUThreads(0, 1) != 1 { t.Error("threads: the profile's cores, else half the node's") } } func TestAModelOfExpertsReadsItsActiveShare(t *testing.T) { for name, want := range map[string]float64{ "Qwen/Qwen3-30B-A3B-Instruct-2507-GGUF": 0.16, "inclusionAI/Ling-flash-A2.5B": 1.6 * 2.5 / 30, "Qwen/Qwen3-8B": 1, "meta-llama/Llama-3.1-8B-Instruct": 1, } { if got := ActiveShare(name, 30e9); math.Abs(got-want) > 0.001 && !(want == 1 && got == 1) { t.Errorf("%s: %.3f, want %.3f", name, got, want) } } } // A plan's ways say how fast they answer: on the CPU by the node's memory // and half its threads, on a card by its bandwidth. func TestAPlansWaysSayHowFastTheyAnswer(t *testing.T) { nodes := []PoolNode{{ID: "big", Class: "l40s-48g", GPUMemGB: 48, GPUs: 2, FreeGPUs: 2, HostMemGB: 512, HostFreeGB: 400, CPUs: 64, MemBWGBs: 300, MemBWCoreGBs: 10, CardBWGBs: map[string]float64{"l40s-48g": 864}}} opts := Options([]Artifact{{Name: "m-Q4_K_M.gguf", Format: "gguf", Bytes: 18e9, Quant: "Q4_K_M"}}, Shape{Params: 32e9}, nodes, 4096) var cpu, gpu float64 for _, o := range opts { if o.CPU { cpu = o.TokensPerSecond } else { gpu = o.TokensPerSecond } } if want := CPUTokensPerSecondOn(18e9, 300, 10, 32); math.Abs(cpu-want) > 0.01 { t.Errorf("on the CPU %.2f tok/s, want %.2f", cpu, want) } if gpu < 3*cpu { t.Errorf("on a card %.1f tok/s against %.1f on the CPU", gpu, cpu) } }