Merge C24 LoRA adapters on demand for vLLM, with affinity

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:15:12 +02:00
commit bd3eca4092
24 files changed
+1773 -35

No files matched your search

+5 -1
View File
@@ -1846,8 +1846,12 @@ func (a *API) setAdapters(w http.ResponseWriter, r *http.Request) {
for _, ad := range m.Adapters {
have[ad.Source] = true
}
// The file is read only where llama.cpp will load it: a vLLM adapter is
// a PEFT directory, which has no header to read, and vLLM itself refuses
// one that does not fit the model when it is loaded.
ggufs := slices.ContainsFunc(m.Profiles, func(p catalog.Profile) bool { return p.Engine == "llamacpp" })
for _, ad := range req.Adapters {
if have[ad.Source] {
if have[ad.Source] || !ggufs {
continue // already checked when it was attached
}
h, err := gguf.KindFromURL(r.Context(), safedial.Client(safedial.ControlPlane, 30*time.Second), ad.Source)
+45 -9
View File
@@ -288,9 +288,18 @@ type Model struct {
External *External `json:"external,omitempty" koanf:"external"`
// Adapters are LoRA adapters served on this model's replicas, each asked
// for by its own name as though it were a model. Every replica loads
// every adapter, applying one only to a request that names it.
Adapters []Adapter `json:"adapters,omitempty" koanf:"adapters"`
Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"`
// every adapter, applying one only to a request that names it — unless
// MaxLoadedAdapters is set.
Adapters []Adapter `json:"adapters,omitempty" koanf:"adapters"`
// MaxLoadedAdapters, above zero, has each replica hold at most this many
// adapters at once, loaded when a request first asks for one and unloaded
// least recently used first to make room. The router sends a request for
// an adapter to a replica that already holds it while that one has room
// to take it. vLLM profiles only: llama-server loads adapters at start
// and cannot load one afterwards. Zero loads every adapter on every
// replica at start.
MaxLoadedAdapters int `json:"max_loaded_adapters,omitempty" koanf:"max_loaded_adapters"`
Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"`
// Canary sends a share of this model's requests to another catalog
// model — the same model at new weights — to compare the two before
// moving to it (`llmgw ctl model canary`).
@@ -871,7 +880,7 @@ func (m Model) EngineProfile(name string) (engines.Profile, bool) {
ModelPath: m.Source,
ServedModelName: m.Name,
Adapters: m.engineAdapters(),
}, true
}.WithRuntimeLoRA(m.MaxLoadedAdapters), true
}
return engines.Profile{}, false
}
@@ -1040,8 +1049,10 @@ func (m Model) IsExternal() bool { return m.External != nil }
type Adapter struct {
// Name is what clients ask for, like a model name.
Name string `json:"name" koanf:"name"`
// Source is the adapter file (GGUF for llama.cpp), fetched by each node
// as the model's weights are.
// Source is the adapter: a GGUF file for llama.cpp; for vLLM a PEFT
// adapter directory (a mirror version ending in "/", a path on the node,
// or a Hugging Face repository id). Fetched by each node as the model's
// weights are.
Source string `json:"source" koanf:"source"`
// Scale is how strongly it is applied; zero takes 1.
Scale float64 `json:"scale,omitempty" koanf:"scale"`
@@ -1056,6 +1067,20 @@ func (a Adapter) EffectiveScale() float64 {
}
func validateAdapters(m Model) error {
if m.MaxLoadedAdapters < 0 {
return fmt.Errorf("model %q: max_loaded_adapters cannot be negative", m.Name)
}
if m.MaxLoadedAdapters > 0 {
// Loading on demand is vLLM's runtime LoRA API. llama-server can
// only change the scale of adapters it loaded at start, so a cap
// there would be a cap on nothing.
for _, p := range m.Profiles {
if p.Engine != "vllm" {
return fmt.Errorf("model %q profile %q: max_loaded_adapters needs vllm profiles; "+
"llama.cpp loads every adapter at start", m.Name, p.Name)
}
}
}
if len(m.Adapters) == 0 {
return nil
}
@@ -1064,9 +1089,17 @@ func validateAdapters(m Model) error {
}
for _, p := range m.Profiles {
// llama.cpp loads adapters at start and applies them per request;
// the other engines are not driven that way yet.
if p.Engine != "llamacpp" && p.Engine != "sim" {
return fmt.Errorf("model %q profile %q: adapters are served by llamacpp profiles only", m.Name, p.Name)
// vLLM loads them at start or on demand. The other engines are not
// driven either way yet.
if p.Engine != "llamacpp" && p.Engine != "vllm" && p.Engine != "sim" {
return fmt.Errorf("model %q profile %q: adapters are served by llamacpp and vllm profiles only", m.Name, p.Name)
}
// The variable is what switches vLLM to loading on demand, and the
// router loads adapters only for a model with a cap: set by hand, it
// would start replicas holding no adapter that nothing ever loads.
if _, set := p.Env[engines.RuntimeLoRAEnv]; set && m.MaxLoadedAdapters == 0 {
return fmt.Errorf("model %q profile %q: set max_loaded_adapters rather than %s",
m.Name, p.Name, engines.RuntimeLoRAEnv)
}
}
seen := map[string]bool{}
@@ -1080,6 +1113,9 @@ func validateAdapters(m Model) error {
return fmt.Errorf("model %q: adapter %q is declared twice", m.Name, a.Name)
case a.Scale < 0:
return fmt.Errorf("model %q: adapter %q has a negative scale", m.Name, a.Name)
case m.MaxLoadedAdapters > 0 && (strings.ContainsAny(a.Name, `/\`) || a.Name == "." || a.Name == ".."):
// Loaded on demand, the name is a file's name on the node.
return fmt.Errorf("model %q: adapter %q cannot contain a slash when adapters load on demand", m.Name, a.Name)
}
seen[a.Name] = true
}
+65
View File
@@ -0,0 +1,65 @@
package catalog
import (
"slices"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
)
func loraModel(engine string, limit int) Model {
m := model("coder", Profile{Name: "p", Engine: engine, GPUClass: "h100-80g", TP: 1})
m.Adapters = []Adapter{{Name: "coder-sql", Source: "http://mirror/coder-sql/v1/"}}
m.MaxLoadedAdapters = limit
return m
}
func TestAdaptersLoadOnDemandOnVLLMOnly(t *testing.T) {
if _, err := New([]Model{loraModel("vllm", 4)}); err != nil {
t.Errorf("vLLM with a cap: %v", err)
}
if _, err := New([]Model{loraModel("vllm", 0)}); err != nil {
t.Errorf("vLLM loading every adapter at start: %v", err)
}
_, err := New([]Model{loraModel("llamacpp", 4)})
if err == nil || !strings.Contains(err.Error(), "max_loaded_adapters needs vllm") {
t.Errorf("llama.cpp with a cap: %v", err)
}
if _, err := New([]Model{loraModel("vllm", -1)}); err == nil {
t.Error("a negative cap was accepted")
}
}
func TestRuntimeLoadingCannotBeTurnedOnByHand(t *testing.T) {
m := loraModel("vllm", 0)
m.Profiles[0].Env = map[string]string{engines.RuntimeLoRAEnv: "True"}
if _, err := New([]Model{m}); err == nil || !strings.Contains(err.Error(), "max_loaded_adapters") {
t.Errorf("want a pointer to max_loaded_adapters, got %v", err)
}
}
func TestAnAdapterThatLoadsOnDemandIsNamedLikeAFile(t *testing.T) {
m := loraModel("vllm", 2)
m.Adapters[0].Name = "team/sql"
if _, err := New([]Model{m}); err == nil {
t.Error("an adapter with a slash in its name was accepted")
}
m.MaxLoadedAdapters = 0
if _, err := New([]Model{m}); err != nil {
t.Errorf("loaded at start, the name names no file: %v", err)
}
}
func TestTheEngineProfileOfAModelWithACapStartsVLLMReadyToLoad(t *testing.T) {
p, ok := loraModel("vllm", 3).EngineProfile("p")
if !ok {
t.Fatal("no profile")
}
if !p.RuntimeLoRA() || !slices.Equal(p.Args, []string{"--max-loras", "3"}) {
t.Errorf("env %v args %v", p.Env, p.Args)
}
if p, _ := loraModel("vllm", 0).EngineProfile("p"); p.RuntimeLoRA() || len(p.Args) != 0 {
t.Errorf("without a cap: env %v args %v", p.Env, p.Args)
}
}