Make the planned entry use an engine that can serve the precision
Reading the tool's own output against the real fleet: it chose q8_0 as the best fit and emitted a vLLM profile with no quantisation. q8_0 is a GGUF quantisation that only llama.cpp serves, and from a file rather than a repository — so following that entry loads full-precision weights, which the same output had just listed as not fitting one card. The planner would have caused the out-of-memory it exists to prevent, from a recommendation that looked right. Choosing a precision and choosing an engine turn out to be one decision, so a Precision now carries the engine that serves it, the quantisation flag where one is needed, and whether it lives in a file. A GGUF fit prints a path and says to download it; a vLLM fit prints the repository and passes --quantization so the engine does not quietly load bf16. The arithmetic was right and the recommendation was wrong, which is the failure mode worth having tests for: everything reported a number, and the numbers agreed with each other. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
88a5949f7c
commit
bd134a7ab9
3 files changed
+75
-10
No files matched your search
@@ -5,6 +5,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
)
|
||||
|
||||
func TestBarStaysWithinItsWidth(t *testing.T) {
|
||||
@@ -143,3 +144,37 @@ func TestAGPUNodeWithNoFreeCardsIsNotAPool(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheEntryNamesAnEngineThatCanServeThePrecision(t *testing.T) {
|
||||
// The defect this prevents, found by reading the tool's own output: it
|
||||
// chose q8_0 — a GGUF quantisation — and emitted a vLLM profile with no
|
||||
// quantisation at all. Following that loads full-precision weights, which
|
||||
// the same plan had just shown do not fit on one card. The tool would have
|
||||
// caused the out-of-memory it exists to prevent.
|
||||
m := modelplan.Model{Name: "Qwen2.5-Coder-32B", Params: 32_800_000_000, Layers: 64, KVHeads: 8, HeadDim: 128}
|
||||
|
||||
gguf := modelplan.Fit{Precision: modelplan.Q8GGUF, Class: "h100-80g", TP: 1, Nodes: 1}
|
||||
entry := catalogEntry(m, gguf, "Qwen/Qwen2.5-Coder-32B-Instruct", 32768)
|
||||
if !strings.Contains(entry, "engine: llamacpp") {
|
||||
t.Errorf("a GGUF precision produced a non-llamacpp profile:\n%s", entry)
|
||||
}
|
||||
if strings.Contains(entry, "source: Qwen/") {
|
||||
t.Errorf("a GGUF entry points at a repository rather than a file:\n%s", entry)
|
||||
}
|
||||
if !strings.Contains(entry, ".gguf") {
|
||||
t.Errorf("a GGUF entry has no file path:\n%s", entry)
|
||||
}
|
||||
|
||||
fp8 := modelplan.Fit{Precision: modelplan.FP8, Class: "h100-80g", TP: 2, Nodes: 1}
|
||||
entry = catalogEntry(m, fp8, "Qwen/Qwen2.5-Coder-32B-Instruct", 32768)
|
||||
if !strings.Contains(entry, "engine: vllm") {
|
||||
t.Errorf("fp8 did not produce a vLLM profile:\n%s", entry)
|
||||
}
|
||||
// And the quantisation has to reach the engine, or vLLM loads bf16.
|
||||
if !strings.Contains(entry, "--quantization") || !strings.Contains(entry, "fp8") {
|
||||
t.Errorf("the fp8 quantisation is not passed to the engine:\n%s", entry)
|
||||
}
|
||||
if !strings.Contains(entry, "source: Qwen/") {
|
||||
t.Errorf("a vLLM entry should load from the repository:\n%s", entry)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user