From bd134a7ab9af7a4384b60d8bb6b72602be548af0 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 17 Sep 2026 07:48:20 +0200 Subject: [PATCH] Make the planned entry use an engine that can serve the precision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reading the tool's own output against the real fleet: it chose q8_0 as the best fit and emitted a vLLM profile with no quantisation. q8_0 is a GGUF quantisation that only llama.cpp serves, and from a file rather than a repository — so following that entry loads full-precision weights, which the same output had just listed as not fitting one card. The planner would have caused the out-of-memory it exists to prevent, from a recommendation that looked right. Choosing a precision and choosing an engine turn out to be one decision, so a Precision now carries the engine that serves it, the quantisation flag where one is needed, and whether it lives in a file. A GGUF fit prints a path and says to download it; a vLLM fit prints the repository and passes --quantization so the engine does not quietly load bf16. The arithmetic was right and the recommendation was wrong, which is the failure mode worth having tests for: everything reported a number, and the numbers agreed with each other. Co-Authored-By: Claude Opus 5 (1M context) --- internal/cli/modelplan.go | 19 ++++++++++++++++--- internal/cli/status_test.go | 35 +++++++++++++++++++++++++++++++++++ internal/modelplan/size.go | 31 ++++++++++++++++++++++++------- 3 files changed, 75 insertions(+), 10 deletions(-) diff --git a/internal/cli/modelplan.go b/internal/cli/modelplan.go index 3caedbad..eba0746e 100644 --- a/internal/cli/modelplan.go +++ b/internal/cli/modelplan.go @@ -190,7 +190,17 @@ func catalogEntry(m modelplan.Model, f modelplan.Fit, source string, context int var b strings.Builder name := strings.ToLower(m.Name) fmt.Fprintf(&b, "name: %s\n", name) - fmt.Fprintf(&b, "source: %s\n", source) + if f.Precision.GGUF { + // A GGUF quantisation lives in a file, not a repository, and llama.cpp + // is given a path. Printing the repository id here would produce an + // entry that looks right and cannot load — which is the failure this + // whole command exists to prevent. + fmt.Fprintf(&b, "# %s is a GGUF quantisation: download the %s file and give its path.\n", + f.Precision.Name, f.Precision.Name) + fmt.Fprintf(&b, "source: /var/lib/llmgw-models/%s-%s.gguf\n", name, f.Precision.Name) + } else { + fmt.Fprintf(&b, "source: %s\n", source) + } fmt.Fprintf(&b, "routing_profile: chat\n") fmt.Fprintf(&b, "context_length: %d\n", context) fmt.Fprintf(&b, "scaling: { min: 1, max: 1 }\n") @@ -198,14 +208,17 @@ func catalogEntry(m modelplan.Model, f modelplan.Fit, source string, context int switch { case f.CPUOnly: - fmt.Fprintf(&b, " - name: cpu\n engine: llamacpp\n") + fmt.Fprintf(&b, " - name: cpu\n engine: %s\n", f.Precision.Engine) fmt.Fprintf(&b, " min_memory_gb: %d\n", int(f.PerGPUGB)+2) default: - fmt.Fprintf(&b, " - name: tp%d\n engine: vllm\n", f.TP) + fmt.Fprintf(&b, " - name: tp%d\n engine: %s\n", f.TP, f.Precision.Engine) fmt.Fprintf(&b, " gpu_class: %s\n tp: %d\n", f.Class, f.TP) if f.Nodes > 1 { fmt.Fprintf(&b, " nodes: %d\n gpus_per_node: %d\n", f.Nodes, f.GPUsPerNode) } + if q := f.Precision.Quantization; q != "" { + fmt.Fprintf(&b, " args: [\"--quantization\", \"%s\"]\n", q) + } } fmt.Fprintf(&b, " priority: 100\n weight: 1\n") // Measured rather than guessed is the rule everywhere else in this system, diff --git a/internal/cli/status_test.go b/internal/cli/status_test.go index 7bf538d9..ea77edb7 100644 --- a/internal/cli/status_test.go +++ b/internal/cli/status_test.go @@ -5,6 +5,7 @@ import ( "testing" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" ) func TestBarStaysWithinItsWidth(t *testing.T) { @@ -143,3 +144,37 @@ func TestAGPUNodeWithNoFreeCardsIsNotAPool(t *testing.T) { } } } + +func TestTheEntryNamesAnEngineThatCanServeThePrecision(t *testing.T) { + // The defect this prevents, found by reading the tool's own output: it + // chose q8_0 — a GGUF quantisation — and emitted a vLLM profile with no + // quantisation at all. Following that loads full-precision weights, which + // the same plan had just shown do not fit on one card. The tool would have + // caused the out-of-memory it exists to prevent. + m := modelplan.Model{Name: "Qwen2.5-Coder-32B", Params: 32_800_000_000, Layers: 64, KVHeads: 8, HeadDim: 128} + + gguf := modelplan.Fit{Precision: modelplan.Q8GGUF, Class: "h100-80g", TP: 1, Nodes: 1} + entry := catalogEntry(m, gguf, "Qwen/Qwen2.5-Coder-32B-Instruct", 32768) + if !strings.Contains(entry, "engine: llamacpp") { + t.Errorf("a GGUF precision produced a non-llamacpp profile:\n%s", entry) + } + if strings.Contains(entry, "source: Qwen/") { + t.Errorf("a GGUF entry points at a repository rather than a file:\n%s", entry) + } + if !strings.Contains(entry, ".gguf") { + t.Errorf("a GGUF entry has no file path:\n%s", entry) + } + + fp8 := modelplan.Fit{Precision: modelplan.FP8, Class: "h100-80g", TP: 2, Nodes: 1} + entry = catalogEntry(m, fp8, "Qwen/Qwen2.5-Coder-32B-Instruct", 32768) + if !strings.Contains(entry, "engine: vllm") { + t.Errorf("fp8 did not produce a vLLM profile:\n%s", entry) + } + // And the quantisation has to reach the engine, or vLLM loads bf16. + if !strings.Contains(entry, "--quantization") || !strings.Contains(entry, "fp8") { + t.Errorf("the fp8 quantisation is not passed to the engine:\n%s", entry) + } + if !strings.Contains(entry, "source: Qwen/") { + t.Errorf("a vLLM entry should load from the repository:\n%s", entry) + } +} diff --git a/internal/modelplan/size.go b/internal/modelplan/size.go index 83f7323e..4d1b0a65 100644 --- a/internal/modelplan/size.go +++ b/internal/modelplan/size.go @@ -40,25 +40,42 @@ type Precision struct { Name string BytesPerParam float64 BytesPerKVElem float64 + // Engine is the one that can serve this precision. Carried here because + // choosing a precision and choosing an engine are the same decision, and + // separating them produced a plan recommending a GGUF quantisation with a + // vLLM profile — which would have loaded full-precision weights that the + // same plan had just shown do not fit. + Engine string + // Quantization is the engine's own flag value, where it needs one. + Quantization string + // GGUF marks a precision that lives in a file rather than a repository, so + // the source is a path and not a model id. + GGUF bool } // Known precisions, from full to the quantisations worth serving. +// +// Engine matters as much as the number of bytes. q8_0 and q4_k_m are GGUF +// quantisations and only llama.cpp serves them, from a .gguf file rather than a +// repository; bf16 and fp8 are what vLLM and SGLang load from the weights as +// published. A plan that picks a precision without saying which engine can +// serve it is a plan that does not work — and worse, one that looks like it +// should. var ( - BF16 = Precision{"bf16", 2, 2} - FP8 = Precision{"fp8", 1, 1} - INT4 = Precision{"int4", 0.5, 2} // weights quantised, KV usually is not - Q8GGUF = Precision{"q8_0", 1.06, 2} - Q4GGUF = Precision{"q4_k_m", 0.61, 2} + BF16 = Precision{Name: "bf16", BytesPerParam: 2, BytesPerKVElem: 2, Engine: "vllm"} + FP8 = Precision{Name: "fp8", BytesPerParam: 1, BytesPerKVElem: 1, Engine: "vllm", Quantization: "fp8"} + Q8GGUF = Precision{Name: "q8_0", BytesPerParam: 1.06, BytesPerKVElem: 2, Engine: "llamacpp", GGUF: true} + Q4GGUF = Precision{Name: "q4_k_m", BytesPerParam: 0.61, BytesPerKVElem: 2, Engine: "llamacpp", GGUF: true} ) // PrecisionByName resolves a precision, reporting what it knows when it cannot. func PrecisionByName(s string) (Precision, error) { - for _, p := range []Precision{BF16, FP8, INT4, Q8GGUF, Q4GGUF} { + for _, p := range []Precision{BF16, FP8, Q8GGUF, Q4GGUF} { if p.Name == s { return p, nil } } - return Precision{}, fmt.Errorf("unknown precision %q; known: bf16, fp8, int4, q8_0, q4_k_m", s) + return Precision{}, fmt.Errorf("unknown precision %q; known: bf16, fp8, q8_0, q4_k_m", s) } // Demand is what one replica needs, in bytes.