diff --git a/CHANGELOG.md b/CHANGELOG.md index 546b495..d50e441 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,7 +46,7 @@ What v0.10.5 made settable from the command line is now on the pages too: A llama.cpp model can carry `adapters`, each a name and a GGUF LoRA file. The nodes fetch them with the model's weights and llama.cpp loads them all at start, -applied to nothing (`--lora-init-without-apply`); a request that names an +applied to nothing (`--lora-scaled FILE:0`); a request that names an adapter goes to the model's replicas with `"lora": [{"id": i, "scale": s}]`, so one replica serves the model and all its adapters. Adapter names share the namespace of models, are listed in `/v1/models`, and follow key allowlists on diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go index eb14d63..16fee3e 100644 --- a/internal/agent/engines/llamacpp.go +++ b/internal/agent/engines/llamacpp.go @@ -146,12 +146,12 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { } // Loaded at start, applied per request: a request that names an adapter // carries "lora": [{"id": i, "scale": s}], and one that names none is the - // base model, since every adapter starts at scale zero. - if len(p.Adapters) > 0 { - args = append(args, "--lora-init-without-apply") - for _, a := range p.Adapters { - args = append(args, "--lora", a.Path) - } + // base model. Each is loaded at scale zero, not with --lora and + // --lora-init-without-apply: that leaves an adapter's scale at 1, and + // llama-server applies it at that scale to every request with no "lora" + // of its own — every request for the model itself. + for _, a := range p.Adapters { + args = append(args, "--lora-scaled", a.Path+":0") } args = append(args, p.Args...) diff --git a/internal/agent/engines/llamacpp_test.go b/internal/agent/engines/llamacpp_test.go index c3133c3..186aa41 100644 --- a/internal/agent/engines/llamacpp_test.go +++ b/internal/agent/engines/llamacpp_test.go @@ -315,8 +315,8 @@ func TestAdaptersAreLoadedAtStartAndAppliedToNoneByDefault(t *testing.T) { t.Fatal(err) } args := strings.Join(cmds[0].Args, " ") - if !strings.Contains(args, "--lora-init-without-apply --lora /w/sql.gguf --lora /w/docs.gguf") { - t.Errorf("args %s: both adapters, in order, loaded unapplied", args) + if !strings.Contains(args, "--lora-scaled /w/sql.gguf:0 --lora-scaled /w/docs.gguf:0") { + t.Errorf("args %s: both adapters, in order, loaded at scale zero", args) } plain, _ := (&LlamaCpp{}).Commands(llamaSpec(nil)) if strings.Contains(strings.Join(plain[0].Args, " "), "--lora") { diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 774ce1d..c47c60f 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -1013,7 +1013,7 @@ func withModel(body []byte, name string) []byte { // withAdapter is body asking for model with a LoRA adapter applied, in // llama.cpp's form: "lora": [{"id": index, "scale": scale}]. Adapters not -// listed are applied at zero, so the list is the whole of it. +// listed stay at zero, the scale the engine loads each one at. func withAdapter(body []byte, model string, index int, scale float64) []byte { var m map[string]json.RawMessage if err := json.Unmarshal(body, &m); err != nil {