Make the planned entry use an engine that can serve the precision
CI / test (push) Successful in 1m39s

Reading the tool's own output against the real fleet: it chose q8_0 as the
best fit and emitted a vLLM profile with no quantisation. q8_0 is a GGUF
quantisation that only llama.cpp serves, and from a file rather than a
repository — so following that entry loads full-precision weights, which
the same output had just listed as not fitting one card. The planner would
have caused the out-of-memory it exists to prevent, from a recommendation
that looked right.

Choosing a precision and choosing an engine turn out to be one decision, so
a Precision now carries the engine that serves it, the quantisation flag
where one is needed, and whether it lives in a file. A GGUF fit prints a
path and says to download it; a vLLM fit prints the repository and passes
--quantization so the engine does not quietly load bf16.

The arithmetic was right and the recommendation was wrong, which is the
failure mode worth having tests for: everything reported a number, and the
numbers agreed with each other.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-17 07:48:20 +02:00
1 parent 88a5949f7c
commit bd134a7ab9
3 files changed
+75 -10

No files matched your search

+16 -3
View File
@@ -190,7 +190,17 @@ func catalogEntry(m modelplan.Model, f modelplan.Fit, source string, context int
var b strings.Builder
name := strings.ToLower(m.Name)
fmt.Fprintf(&b, "name: %s\n", name)
fmt.Fprintf(&b, "source: %s\n", source)
if f.Precision.GGUF {
// A GGUF quantisation lives in a file, not a repository, and llama.cpp
// is given a path. Printing the repository id here would produce an
// entry that looks right and cannot load — which is the failure this
// whole command exists to prevent.
fmt.Fprintf(&b, "# %s is a GGUF quantisation: download the %s file and give its path.\n",
f.Precision.Name, f.Precision.Name)
fmt.Fprintf(&b, "source: /var/lib/llmgw-models/%s-%s.gguf\n", name, f.Precision.Name)
} else {
fmt.Fprintf(&b, "source: %s\n", source)
}
fmt.Fprintf(&b, "routing_profile: chat\n")
fmt.Fprintf(&b, "context_length: %d\n", context)
fmt.Fprintf(&b, "scaling: { min: 1, max: 1 }\n")
@@ -198,14 +208,17 @@ func catalogEntry(m modelplan.Model, f modelplan.Fit, source string, context int
switch {
case f.CPUOnly:
fmt.Fprintf(&b, " - name: cpu\n engine: llamacpp\n")
fmt.Fprintf(&b, " - name: cpu\n engine: %s\n", f.Precision.Engine)
fmt.Fprintf(&b, " min_memory_gb: %d\n", int(f.PerGPUGB)+2)
default:
fmt.Fprintf(&b, " - name: tp%d\n engine: vllm\n", f.TP)
fmt.Fprintf(&b, " - name: tp%d\n engine: %s\n", f.TP, f.Precision.Engine)
fmt.Fprintf(&b, " gpu_class: %s\n tp: %d\n", f.Class, f.TP)
if f.Nodes > 1 {
fmt.Fprintf(&b, " nodes: %d\n gpus_per_node: %d\n", f.Nodes, f.GPUsPerNode)
}
if q := f.Precision.Quantization; q != "" {
fmt.Fprintf(&b, " args: [\"--quantization\", \"%s\"]\n", q)
}
}
fmt.Fprintf(&b, " priority: 100\n weight: 1\n")
// Measured rather than guessed is the rule everywhere else in this system,
+35
View File
@@ -5,6 +5,7 @@ import (
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
)
func TestBarStaysWithinItsWidth(t *testing.T) {
@@ -143,3 +144,37 @@ func TestAGPUNodeWithNoFreeCardsIsNotAPool(t *testing.T) {
}
}
}
func TestTheEntryNamesAnEngineThatCanServeThePrecision(t *testing.T) {
// The defect this prevents, found by reading the tool's own output: it
// chose q8_0 — a GGUF quantisation — and emitted a vLLM profile with no
// quantisation at all. Following that loads full-precision weights, which
// the same plan had just shown do not fit on one card. The tool would have
// caused the out-of-memory it exists to prevent.
m := modelplan.Model{Name: "Qwen2.5-Coder-32B", Params: 32_800_000_000, Layers: 64, KVHeads: 8, HeadDim: 128}
gguf := modelplan.Fit{Precision: modelplan.Q8GGUF, Class: "h100-80g", TP: 1, Nodes: 1}
entry := catalogEntry(m, gguf, "Qwen/Qwen2.5-Coder-32B-Instruct", 32768)
if !strings.Contains(entry, "engine: llamacpp") {
t.Errorf("a GGUF precision produced a non-llamacpp profile:\n%s", entry)
}
if strings.Contains(entry, "source: Qwen/") {
t.Errorf("a GGUF entry points at a repository rather than a file:\n%s", entry)
}
if !strings.Contains(entry, ".gguf") {
t.Errorf("a GGUF entry has no file path:\n%s", entry)
}
fp8 := modelplan.Fit{Precision: modelplan.FP8, Class: "h100-80g", TP: 2, Nodes: 1}
entry = catalogEntry(m, fp8, "Qwen/Qwen2.5-Coder-32B-Instruct", 32768)
if !strings.Contains(entry, "engine: vllm") {
t.Errorf("fp8 did not produce a vLLM profile:\n%s", entry)
}
// And the quantisation has to reach the engine, or vLLM loads bf16.
if !strings.Contains(entry, "--quantization") || !strings.Contains(entry, "fp8") {
t.Errorf("the fp8 quantisation is not passed to the engine:\n%s", entry)
}
if !strings.Contains(entry, "source: Qwen/") {
t.Errorf("a vLLM entry should load from the repository:\n%s", entry)
}
}