Files
llmgw/internal/modelplan/options_test.go
T
Ilker ManapandClaude Opus 5.5 54db7aecb7 Plan a repository's files against the fleet, and serve the one that suits it the way it fits
The control plane reads the model's shape from one GGUF header or its
config.json and lists, for every file, the best way it runs here: one card,
several of a node's (by layer for llama.cpp, tensor-parallel for vLLM),
several nodes' over one fabric (vLLM, pipeline-parallel between them), or a
CPU node; what each card holds and whether it is free now. ctl model plan
prints it; model serve takes its file and way unless told, gives a GGUF on
GPUs a GPU profile, and with --fallbacks adds the next best ways. The
console shows it per file and asks for the way and the fallback.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-04 01:03:48 +02:00

100 lines
3.6 KiB
Go

package modelplan
import (
"strings"
"testing"
)
const gib = int64(1) << 30
// A 14B GGUF: 48 layers, 8 KV heads of 128 — 192 KB a token at f16.
var shape14B = Shape{KVBytesPerToken: 48 * 8 * 256 * 2, KVHeads: 8, Params: 14_800_000_000}
func labels(opts []Option) []string {
var out []string
for _, o := range opts {
out = append(out, o.Artifact+" "+o.Label())
}
return out
}
func TestEachFileIsMatchedToTheCardsThatHoldIt(t *testing.T) {
nodes := []PoolNode{
{ID: "gpu1", Class: "6000-24g", GPUMemGB: 24, GPUs: 1, FreeGPUs: 1, HostMemGB: 64, HostFreeGB: 40},
{ID: "cpu1", HostMemGB: 32, HostFreeGB: 30},
}
arts := []Artifact{
{Name: "q4.gguf", Format: "gguf", Bytes: 9 * gib},
{Name: "q8.gguf", Format: "gguf", Bytes: 15 * gib},
{Name: "f16.gguf", Format: "gguf", Bytes: 29 * gib},
}
opts := Options(arts, shape14B, nodes, 16384)
got := strings.Join(labels(opts), "; ")
// The best: the better precision on the one card, free now.
if opts[0].Artifact != "q8.gguf" || opts[0].CPU || !opts[0].FreeNow {
t.Errorf("first: %+v (all: %s)", opts[0], got)
}
// f16 fits no card, only a CPU node with the RAM.
for _, o := range opts {
if o.Artifact == "f16.gguf" && !o.CPU {
t.Errorf("f16 placed on a card: %+v", o)
}
}
if !strings.Contains(got, "f16.gguf CPU") {
t.Errorf("f16 not offered the CPU: %s", got)
}
// The card busy: what is free comes first, which is the CPU.
nodes[0].FreeGPUs = 0
opts = Options(arts, shape14B, nodes, 16384)
if !opts[0].CPU || !opts[0].FreeNow {
t.Errorf("with the card busy: %+v", opts[0])
}
}
func TestALlamaCppModelSpreadsOverANodesCardsByLayer(t *testing.T) {
nodes := []PoolNode{{ID: "a", Class: "l4-24g", GPUMemGB: 24, GPUs: 4, FreeGPUs: 4}}
opts := Options([]Artifact{{Name: "70b-q4.gguf", Format: "gguf", Bytes: 40 * gib}},
Shape{KVBytesPerToken: 80 * 8 * 256 * 2}, nodes, 8192)
if len(opts) != 1 || opts[0].GPUsPerNode != 3 || opts[0].Nodes != 1 {
t.Fatalf("%+v", opts)
}
}
func TestOriginalWeightsSpanNodesOnlyOverAFabric(t *testing.T) {
var nodes []PoolNode
for _, id := range []string{"a", "b"} {
nodes = append(nodes, PoolNode{ID: id, Class: "h100-80g", GPUMemGB: 80, GPUs: 8, FreeGPUs: 8, Fabric: "ib-0"})
}
llama70 := Shape{KVBytesPerToken: 80 * 8 * 256 * 2, KVHeads: 8, Params: 70e9}
opts := Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 140 * gib}}, llama70, nodes, 8192)
if len(opts) != 1 || opts[0].GPUsPerNode != 4 || opts[0].Nodes != 1 || opts[0].Engine != "vllm" {
t.Fatalf("70B: %+v", opts)
}
big := Shape{KVBytesPerToken: 126 * 8 * 256 * 2, KVHeads: 8, Params: 405e9}
opts = Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 810 * gib}}, big, nodes, 8192)
if len(opts) != 1 || opts[0].Nodes != 2 || opts[0].GPUsPerNode != 8 || opts[0].Fabric != "ib-0" || !opts[0].FreeNow {
t.Fatalf("405B: %+v", opts)
}
// No fabric, no replica across the two.
for i := range nodes {
nodes[i].Fabric = ""
}
if opts = Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 810 * gib}}, big, nodes, 8192); len(opts) != 0 {
t.Fatalf("spanned nodes with no fabric: %+v", opts)
}
}
func TestQuantisationsAreReadFromNames(t *testing.T) {
for name, want := range map[string]string{
"Qwen2.5-7B-Instruct-Q4_K_M.gguf": "Q4_K_M",
"qwen2.5-coder-14b-instruct-q4_k_m-00001-of-00002.gguf": "Q4_K_M",
"phi-4-IQ3_XS.gguf": "IQ3_XS",
"model-f16.gguf": "F16",
"JiRackUltra_1b.Q6_K.gguf": "Q6_K",
} {
if got := QuantOf(name); got != want {
t.Errorf("%s: %s", name, got)
}
}
}