Files
llmgw/internal/cli/status_test.go
T
Ilker ManapandClaude Opus 5 300e298053
CI / test (push) Has been cancelled
Work out whether a model fits the cluster, and how to split it
llmgw ctl model plan Qwen/Qwen2.5-Coder-32B-Instruct

The arithmetic this replaces is the last thing between having GPUs and
serving a model on them. Choosing tp, gpu_class and a context that will
actually fit means knowing the layer count, the KV head count, what the
weights weigh at a chosen quantisation and how much of a card the runtime
keeps — and getting it wrong is not an error message, it is an
out-of-memory twenty minutes into a load. I computed those by hand for
every model on this cluster.

Both halves are read rather than retyped: the architecture from the model's
own config.json and parameter count, the hardware from the running fleet.
Attention heads in place of KV heads is the classic slip and is worth a
factor of five on a model with grouped-query attention, so it is not a
number anyone should be entering.

Every candidate is listed, not one answer. With cards to spare you want the
full-precision fit; with one card free you want to know a quantised one
exists. Choosing between those is not the planner's job.

The tests are calibrated against files actually downloaded here — llama.cpp
reported 14.62 GiB and 8.37 GiB for the 14B at q8_0 and q4_K_M, and the
model reproduces both — and they caught two errors in my own arithmetic.
The overhead was charged per rank from the whole model's weights rather
than each rank's share, and it reserved a tenth of a card that
usableFraction had already reserved. Between them, Llama-3.3-70B at bf16
across two 80 GB cards came out as not fitting, which is a configuration
people run every day. Slack reserved in two places, each unaware of the
other, reads as caution and behaves as a bug.

Pools are grouped by class and fabric together, because a replica may never
span two fabrics: eight free cards split across two of them are not eight
cards for a model that needs eight, and merging them would plan something
that starts, passes its health checks and hangs on its first all-reduce.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 07:46:53 +02:00

146 lines
5.1 KiB
Go

package cli
import (
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)
func TestBarStaysWithinItsWidth(t *testing.T) {
// It is drawn from counts that can disagree briefly — a GPU freed between
// two reads — and a bar that overruns its column wrecks the table it is in.
for _, tc := range [][2]int{{0, 0}, {0, 8}, {8, 8}, {9, 8}, {100, 8}} {
got := bar(tc[0], tc[1])
if tc[1] == 0 {
if got != "" {
t.Errorf("bar(%d,%d) = %q, want empty for an empty class", tc[0], tc[1], got)
}
continue
}
if n := strings.Count(got, "#") + strings.Count(got, "."); n != 20 {
t.Errorf("bar(%d,%d) drew %d cells: %q", tc[0], tc[1], n, got)
}
}
}
func TestIdleColumnDistinguishesUnknownFromIdle(t *testing.T) {
// Every replica looks unreported before its first request and after a
// router restart. Showing that as a very large duration would make an
// untouched fleet read as an abandoned one.
if got := idleColumn(-1); got != "-" {
t.Errorf("idleColumn(-1) = %q, want a dash", got)
}
if got := idleColumn(90); got != "1m30s" {
t.Errorf("idleColumn(90) = %q", got)
}
}
func TestModelStatusSeparatesWaitingFromBroken(t *testing.T) {
// These used to differ by a parenthetical — "0/1 ready (1 total)" against
// "0/1 ready" — and they need opposite responses: one means wait, the other
// means look at explain-placement. On forty models that difference is
// invisible.
for _, tc := range []struct {
name string
view adminapi.ModelView
want string
}{
{"serving what it asked for", adminapi.ModelView{Min: 2, Ready: 2, Total: 2}, "ready"},
{"a replica is coming up", adminapi.ModelView{Min: 1, Ready: 0, Total: 1}, "starting"},
{"nothing could be placed", adminapi.ModelView{Min: 1, Ready: 0, Total: 0}, "not placed"},
{"deliberately not running", adminapi.ModelView{Min: 0, Ready: 0, Total: 0}, "stopped"},
{"short but rebuilding", adminapi.ModelView{Min: 2, Ready: 1, Total: 2}, "starting"},
} {
t.Run(tc.name, func(t *testing.T) {
if got := modelStatus(tc.view); got != tc.want {
t.Errorf("modelStatus(%+v) = %q, want %q", tc.view, got, tc.want)
}
})
}
}
func TestHardwarePoolsNeverMergeAcrossFabrics(t *testing.T) {
// A replica may never span two fabrics, so eight free cards split across
// two of them are not eight cards for a model that needs eight. Merging
// them would plan something that starts, passes health checks and hangs on
// its first all-reduce.
nodes := []adminapi.NodeView{
{ID: "a", State: "ready", Classes: []string{"h100-80g"}, Fabrics: []string{"ib-a"}, GPUFree: 4, GPUTotal: 4},
{ID: "b", State: "ready", Classes: []string{"h100-80g"}, Fabrics: []string{"ib-a"}, GPUFree: 4, GPUTotal: 4},
{ID: "c", State: "ready", Classes: []string{"h100-80g"}, Fabrics: []string{"ib-b"}, GPUFree: 4, GPUTotal: 4},
}
hw := hardwareFrom(nodes)
if len(hw) != 2 {
t.Fatalf("expected one pool per fabric, got %d: %+v", len(hw), hw)
}
for _, p := range hw {
switch p.Fabric {
case "ib-a":
if p.Nodes != 2 {
t.Errorf("ib-a has %d nodes, want 2", p.Nodes)
}
case "ib-b":
if p.Nodes != 1 {
t.Errorf("ib-b has %d nodes, want 1", p.Nodes)
}
default:
t.Errorf("unexpected fabric %q", p.Fabric)
}
}
}
func TestUnreadyNodesAreNotCountedAsCapacity(t *testing.T) {
// Planning against a node that is draining or unreachable produces a fit
// that cannot be placed, which is worse than saying nothing fits.
nodes := []adminapi.NodeView{
{ID: "a", State: "ready", Classes: []string{"h100-80g"}, GPUFree: 8, GPUTotal: 8},
{ID: "b", State: "draining", Classes: []string{"h100-80g"}, GPUFree: 8, GPUTotal: 8},
{ID: "c", State: "unreachable", Classes: []string{"h100-80g"}, GPUFree: 8, GPUTotal: 8},
}
hw := hardwareFrom(nodes)
if len(hw) != 1 || hw[0].Nodes != 1 {
t.Fatalf("expected only the ready node, got %+v", hw)
}
}
func TestCardMemoryComesFromTheClassLabel(t *testing.T) {
// The label carries it by construction — "h100-80g" is eighty gigabytes and
// says so — which is why the planner needs no table of card models.
for label, want := range map[string]int{
"h100-80g": 80, "a100-40g": 40, "l40s-48g": 48, "rtx4090-24g": 24,
"nonsense": 0, "": 0, "h100-": 0,
} {
if got := memFromClass(label); got != want {
t.Errorf("memFromClass(%q) = %d, want %d", label, got, want)
}
}
}
func TestACPUOnlyNodeBecomesAMemoryPool(t *testing.T) {
// How this cluster began: a node with no cards is not an empty pool, it is
// a pool measured in RAM.
hw := hardwareFrom([]adminapi.NodeView{
{ID: "cpu1", State: "ready", MemoryGB: 31},
})
if len(hw) != 1 {
t.Fatalf("the CPU node produced %d pools", len(hw))
}
if hw[0].Class != "" || hw[0].HostMemGB != 31 {
t.Errorf("pool = %+v, want a classless pool with 31 GB", hw[0])
}
}
func TestAGPUNodeWithNoFreeCardsIsNotAPool(t *testing.T) {
// Fully allocated hardware is not capacity, and offering it would plan a
// replica that cannot be placed.
hw := hardwareFrom([]adminapi.NodeView{
{ID: "full", State: "ready", Classes: []string{"h100-80g"}, GPUFree: 0, GPUTotal: 8},
})
for _, p := range hw {
if p.Class != "" {
t.Errorf("a node with no free cards became a pool: %+v", p)
}
}
}