Ilker ManapandClaude Opus 5.5
6403e61f4b
CI / test (push) Successful in 9m31s
Each agent measures one core's memory rate beside the whole machine's; the estimate takes a replica's threads at about one core's rate each, up to the machine's, calibrated by llama-bench on the PoC. The serve form shows it beside a card's, warns when slow, and asks for the CPU on purpose when a reply would take minutes; ctl model plan, the console and serve jobs say it too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
99 lines
3.5 KiB
Go
99 lines
3.5 KiB
Go
package modelplan
|
|
|
|
import (
|
|
"math"
|
|
"testing"
|
|
)
|
|
|
|
func TestCPUSpeedMatchesWhatThePoCMeasured(t *testing.T) {
|
|
for _, c := range []struct {
|
|
bytes int64
|
|
bw, want float64
|
|
}{
|
|
{1117320736, 24.9, 14.5}, // node01, qwen2.5-1.5b
|
|
{9_000_000_000, 41.9, 3.3}, // node05, qwen2.5-coder-14b
|
|
} {
|
|
got := CPUTokensPerSecond(c.bytes, c.bw)
|
|
if math.Abs(got-c.want)/c.want > 0.15 {
|
|
t.Errorf("%d bytes at %.1f GB/s: %.1f tok/s, measured %.1f", c.bytes, c.bw, got, c.want)
|
|
}
|
|
}
|
|
if CPUTokensPerSecond(0, 40) != 0 || CPUTokensPerSecond(1e9, 0) != 0 {
|
|
t.Error("an unknown size or bandwidth should give no figure")
|
|
}
|
|
}
|
|
|
|
// Each thread reads at about one core's rate, up to the machine's: what
|
|
// llama-bench measured on node05 by thread count, and what was measured on
|
|
// node01 and node05 before, all within 15%.
|
|
func TestCPUSpeedFollowsTheThreads(t *testing.T) {
|
|
for _, c := range []struct {
|
|
bytes int64
|
|
all, core float64
|
|
threads int
|
|
want float64
|
|
}{
|
|
{4683073536, 43.9, 9.7, 1, 1.88}, // node05, qwen2.5-coder-7b
|
|
{4683073536, 43.9, 9.7, 2, 3.56},
|
|
{4683073536, 43.9, 9.7, 4, 6.35},
|
|
{4683073536, 43.9, 9.7, 8, 9.29},
|
|
{1117320736, 43.9, 9.7, 1, 8.31}, // node05, qwen2.5-1.5b
|
|
{1117320736, 43.9, 9.7, 4, 25.4},
|
|
{1117320736, 34.2, 10.1, 2, 14.5}, // node01 on its two cores
|
|
{9_000_000_000, 43.9, 9.7, 4, 3.3}, // node05, qwen2.5-14b on four
|
|
} {
|
|
got := CPUTokensPerSecondOn(c.bytes, c.all, c.core, c.threads)
|
|
if math.Abs(got-c.want)/c.want > 0.15 {
|
|
t.Errorf("%d bytes, %d threads: %.1f tok/s, measured %.1f", c.bytes, c.threads, got, c.want)
|
|
}
|
|
}
|
|
// A large machine, a few threads: the threads decide, not the machine.
|
|
few := CPUTokensPerSecondOn(18e9, 300, 10, 4)
|
|
if few > 2.5 {
|
|
t.Errorf("four threads of a 300 GB/s machine: %.1f tok/s, as though they read at the whole machine's rate", few)
|
|
}
|
|
// An agent that did not measure one core: the old reckoning.
|
|
if got, want := CPUTokensPerSecondOn(1e9, 40, 0, 4), CPUTokensPerSecond(1e9, 40); got != want {
|
|
t.Errorf("without one core's rate %.2f, want %.2f", got, want)
|
|
}
|
|
if CPUThreads(0, 64) != 32 || CPUThreads(6, 64) != 6 || CPUThreads(0, 1) != 1 {
|
|
t.Error("threads: the profile's cores, else half the node's")
|
|
}
|
|
}
|
|
|
|
func TestAModelOfExpertsReadsItsActiveShare(t *testing.T) {
|
|
for name, want := range map[string]float64{
|
|
"Qwen/Qwen3-30B-A3B-Instruct-2507-GGUF": 0.16,
|
|
"inclusionAI/Ling-flash-A2.5B": 1.6 * 2.5 / 30,
|
|
"Qwen/Qwen3-8B": 1,
|
|
"meta-llama/Llama-3.1-8B-Instruct": 1,
|
|
} {
|
|
if got := ActiveShare(name, 30e9); math.Abs(got-want) > 0.001 && !(want == 1 && got == 1) {
|
|
t.Errorf("%s: %.3f, want %.3f", name, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A plan's ways say how fast they answer: on the CPU by the node's memory
|
|
// and half its threads, on a card by its bandwidth.
|
|
func TestAPlansWaysSayHowFastTheyAnswer(t *testing.T) {
|
|
nodes := []PoolNode{{ID: "big", Class: "l40s-48g", GPUMemGB: 48, GPUs: 2, FreeGPUs: 2, HostMemGB: 512, HostFreeGB: 400,
|
|
CPUs: 64, MemBWGBs: 300, MemBWCoreGBs: 10, CardBWGBs: map[string]float64{"l40s-48g": 864}}}
|
|
opts := Options([]Artifact{{Name: "m-Q4_K_M.gguf", Format: "gguf", Bytes: 18e9, Quant: "Q4_K_M"}},
|
|
Shape{Params: 32e9}, nodes, 4096)
|
|
var cpu, gpu float64
|
|
for _, o := range opts {
|
|
if o.CPU {
|
|
cpu = o.TokensPerSecond
|
|
} else {
|
|
gpu = o.TokensPerSecond
|
|
}
|
|
}
|
|
if want := CPUTokensPerSecondOn(18e9, 300, 10, 32); math.Abs(cpu-want) > 0.01 {
|
|
t.Errorf("on the CPU %.2f tok/s, want %.2f", cpu, want)
|
|
}
|
|
if gpu < 3*cpu {
|
|
t.Errorf("on a card %.1f tok/s against %.1f on the CPU", gpu, cpu)
|
|
}
|
|
}
|