llmgw ctl model plan Qwen/Qwen2.5-Coder-32B-Instruct The arithmetic this replaces is the last thing between having GPUs and serving a model on them. Choosing tp, gpu_class and a context that will actually fit means knowing the layer count, the KV head count, what the weights weigh at a chosen quantisation and how much of a card the runtime keeps — and getting it wrong is not an error message, it is an out-of-memory twenty minutes into a load. I computed those by hand for every model on this cluster. Both halves are read rather than retyped: the architecture from the model's own config.json and parameter count, the hardware from the running fleet. Attention heads in place of KV heads is the classic slip and is worth a factor of five on a model with grouped-query attention, so it is not a number anyone should be entering. Every candidate is listed, not one answer. With cards to spare you want the full-precision fit; with one card free you want to know a quantised one exists. Choosing between those is not the planner's job. The tests are calibrated against files actually downloaded here — llama.cpp reported 14.62 GiB and 8.37 GiB for the 14B at q8_0 and q4_K_M, and the model reproduces both — and they caught two errors in my own arithmetic. The overhead was charged per rank from the whole model's weights rather than each rank's share, and it reserved a tenth of a card that usableFraction had already reserved. Between them, Llama-3.3-70B at bf16 across two 80 GB cards came out as not fitting, which is a configuration people run every day. Slack reserved in two places, each unaware of the other, reads as caution and behaves as a bug. Pools are grouped by class and fabric together, because a replica may never span two fabrics: eight free cards split across two of them are not eight cards for a model that needs eight, and merging them would plan something that starts, passes its health checks and hangs on its first all-reduce. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
ea4dd37f80
commit
300e298053
9 files changed
+1029
-16
No files matched your search
@@ -59,3 +59,87 @@ func TestModelStatusSeparatesWaitingFromBroken(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardwarePoolsNeverMergeAcrossFabrics(t *testing.T) {
|
||||
// A replica may never span two fabrics, so eight free cards split across
|
||||
// two of them are not eight cards for a model that needs eight. Merging
|
||||
// them would plan something that starts, passes health checks and hangs on
|
||||
// its first all-reduce.
|
||||
nodes := []adminapi.NodeView{
|
||||
{ID: "a", State: "ready", Classes: []string{"h100-80g"}, Fabrics: []string{"ib-a"}, GPUFree: 4, GPUTotal: 4},
|
||||
{ID: "b", State: "ready", Classes: []string{"h100-80g"}, Fabrics: []string{"ib-a"}, GPUFree: 4, GPUTotal: 4},
|
||||
{ID: "c", State: "ready", Classes: []string{"h100-80g"}, Fabrics: []string{"ib-b"}, GPUFree: 4, GPUTotal: 4},
|
||||
}
|
||||
hw := hardwareFrom(nodes)
|
||||
if len(hw) != 2 {
|
||||
t.Fatalf("expected one pool per fabric, got %d: %+v", len(hw), hw)
|
||||
}
|
||||
for _, p := range hw {
|
||||
switch p.Fabric {
|
||||
case "ib-a":
|
||||
if p.Nodes != 2 {
|
||||
t.Errorf("ib-a has %d nodes, want 2", p.Nodes)
|
||||
}
|
||||
case "ib-b":
|
||||
if p.Nodes != 1 {
|
||||
t.Errorf("ib-b has %d nodes, want 1", p.Nodes)
|
||||
}
|
||||
default:
|
||||
t.Errorf("unexpected fabric %q", p.Fabric)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnreadyNodesAreNotCountedAsCapacity(t *testing.T) {
|
||||
// Planning against a node that is draining or unreachable produces a fit
|
||||
// that cannot be placed, which is worse than saying nothing fits.
|
||||
nodes := []adminapi.NodeView{
|
||||
{ID: "a", State: "ready", Classes: []string{"h100-80g"}, GPUFree: 8, GPUTotal: 8},
|
||||
{ID: "b", State: "draining", Classes: []string{"h100-80g"}, GPUFree: 8, GPUTotal: 8},
|
||||
{ID: "c", State: "unreachable", Classes: []string{"h100-80g"}, GPUFree: 8, GPUTotal: 8},
|
||||
}
|
||||
hw := hardwareFrom(nodes)
|
||||
if len(hw) != 1 || hw[0].Nodes != 1 {
|
||||
t.Fatalf("expected only the ready node, got %+v", hw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCardMemoryComesFromTheClassLabel(t *testing.T) {
|
||||
// The label carries it by construction — "h100-80g" is eighty gigabytes and
|
||||
// says so — which is why the planner needs no table of card models.
|
||||
for label, want := range map[string]int{
|
||||
"h100-80g": 80, "a100-40g": 40, "l40s-48g": 48, "rtx4090-24g": 24,
|
||||
"nonsense": 0, "": 0, "h100-": 0,
|
||||
} {
|
||||
if got := memFromClass(label); got != want {
|
||||
t.Errorf("memFromClass(%q) = %d, want %d", label, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestACPUOnlyNodeBecomesAMemoryPool(t *testing.T) {
|
||||
// How this cluster began: a node with no cards is not an empty pool, it is
|
||||
// a pool measured in RAM.
|
||||
hw := hardwareFrom([]adminapi.NodeView{
|
||||
{ID: "cpu1", State: "ready", MemoryGB: 31},
|
||||
})
|
||||
if len(hw) != 1 {
|
||||
t.Fatalf("the CPU node produced %d pools", len(hw))
|
||||
}
|
||||
if hw[0].Class != "" || hw[0].HostMemGB != 31 {
|
||||
t.Errorf("pool = %+v, want a classless pool with 31 GB", hw[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGPUNodeWithNoFreeCardsIsNotAPool(t *testing.T) {
|
||||
// Fully allocated hardware is not capacity, and offering it would plan a
|
||||
// replica that cannot be placed.
|
||||
hw := hardwareFrom([]adminapi.NodeView{
|
||||
{ID: "full", State: "ready", Classes: []string{"h100-80g"}, GPUFree: 0, GPUTotal: 8},
|
||||
})
|
||||
for _, p := range hw {
|
||||
if p.Class != "" {
|
||||
t.Errorf("a node with no free cards became a pool: %+v", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user