From e40b9ac202499af8fbe437b088bc50025b1f7047 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 1 Oct 2026 07:39:12 +0200 Subject: [PATCH] Load LoRA adapters at scale zero --lora with --lora-init-without-apply leaves an adapter's scale at 1, and llama-server applies it at that scale to any request without a "lora" list of its own: every plain request for the model was served with its adapters on. Found live on qwen2.5-coder-7b-instruct. Each adapter is now loaded with --lora-scaled FILE:0. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 2 +- internal/agent/engines/llamacpp.go | 12 ++++++------ internal/agent/engines/llamacpp_test.go | 4 ++-- internal/router/handlers.go | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 546b495a..d50e4417 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,7 +46,7 @@ What v0.10.5 made settable from the command line is now on the pages too: A llama.cpp model can carry `adapters`, each a name and a GGUF LoRA file. The nodes fetch them with the model's weights and llama.cpp loads them all at start, -applied to nothing (`--lora-init-without-apply`); a request that names an +applied to nothing (`--lora-scaled FILE:0`); a request that names an adapter goes to the model's replicas with `"lora": [{"id": i, "scale": s}]`, so one replica serves the model and all its adapters. Adapter names share the namespace of models, are listed in `/v1/models`, and follow key allowlists on diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go index eb14d635..16fee3e1 100644 --- a/internal/agent/engines/llamacpp.go +++ b/internal/agent/engines/llamacpp.go @@ -146,12 +146,12 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { } // Loaded at start, applied per request: a request that names an adapter // carries "lora": [{"id": i, "scale": s}], and one that names none is the - // base model, since every adapter starts at scale zero. - if len(p.Adapters) > 0 { - args = append(args, "--lora-init-without-apply") - for _, a := range p.Adapters { - args = append(args, "--lora", a.Path) - } + // base model. Each is loaded at scale zero, not with --lora and + // --lora-init-without-apply: that leaves an adapter's scale at 1, and + // llama-server applies it at that scale to every request with no "lora" + // of its own — every request for the model itself. + for _, a := range p.Adapters { + args = append(args, "--lora-scaled", a.Path+":0") } args = append(args, p.Args...) diff --git a/internal/agent/engines/llamacpp_test.go b/internal/agent/engines/llamacpp_test.go index c3133c34..186aa41c 100644 --- a/internal/agent/engines/llamacpp_test.go +++ b/internal/agent/engines/llamacpp_test.go @@ -315,8 +315,8 @@ func TestAdaptersAreLoadedAtStartAndAppliedToNoneByDefault(t *testing.T) { t.Fatal(err) } args := strings.Join(cmds[0].Args, " ") - if !strings.Contains(args, "--lora-init-without-apply --lora /w/sql.gguf --lora /w/docs.gguf") { - t.Errorf("args %s: both adapters, in order, loaded unapplied", args) + if !strings.Contains(args, "--lora-scaled /w/sql.gguf:0 --lora-scaled /w/docs.gguf:0") { + t.Errorf("args %s: both adapters, in order, loaded at scale zero", args) } plain, _ := (&LlamaCpp{}).Commands(llamaSpec(nil)) if strings.Contains(strings.Join(plain[0].Args, " "), "--lora") { diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 774ce1d2..c47c60ff 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -1013,7 +1013,7 @@ func withModel(body []byte, name string) []byte { // withAdapter is body asking for model with a LoRA adapter applied, in // llama.cpp's form: "lora": [{"id": index, "scale": scale}]. Adapters not -// listed are applied at zero, so the list is the whole of it. +// listed stay at zero, the scale the engine loads each one at. func withAdapter(body []byte, model string, index int, scale float64) []byte { var m map[string]json.RawMessage if err := json.Unmarshal(body, &m); err != nil {