LoRA adapters on vLLM, loaded on demand with replica affinity: max_loaded_adapters

A vLLM model can carry PEFT adapters, registered at start, or with
max_loaded_adapters held at most N per replica: the router loads an adapter
through vLLM's runtime LoRA API when a request first wants it, sends the
adapter's requests to a replica that holds it while that one is not
saturated, loads each cold adapter once, and unloads the least recently used
idle one to make room. llama.cpp keeps loading every adapter at start.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:14:38 +02:00
1 parent d41104a452
commit 2a1e0e6c4c
24 files changed
+1773 -35

No files matched your search

+65
View File
@@ -0,0 +1,65 @@
package catalog
import (
"slices"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
)
func loraModel(engine string, limit int) Model {
m := model("coder", Profile{Name: "p", Engine: engine, GPUClass: "h100-80g", TP: 1})
m.Adapters = []Adapter{{Name: "coder-sql", Source: "http://mirror/coder-sql/v1/"}}
m.MaxLoadedAdapters = limit
return m
}
func TestAdaptersLoadOnDemandOnVLLMOnly(t *testing.T) {
if _, err := New([]Model{loraModel("vllm", 4)}); err != nil {
t.Errorf("vLLM with a cap: %v", err)
}
if _, err := New([]Model{loraModel("vllm", 0)}); err != nil {
t.Errorf("vLLM loading every adapter at start: %v", err)
}
_, err := New([]Model{loraModel("llamacpp", 4)})
if err == nil || !strings.Contains(err.Error(), "max_loaded_adapters needs vllm") {
t.Errorf("llama.cpp with a cap: %v", err)
}
if _, err := New([]Model{loraModel("vllm", -1)}); err == nil {
t.Error("a negative cap was accepted")
}
}
func TestRuntimeLoadingCannotBeTurnedOnByHand(t *testing.T) {
m := loraModel("vllm", 0)
m.Profiles[0].Env = map[string]string{engines.RuntimeLoRAEnv: "True"}
if _, err := New([]Model{m}); err == nil || !strings.Contains(err.Error(), "max_loaded_adapters") {
t.Errorf("want a pointer to max_loaded_adapters, got %v", err)
}
}
func TestAnAdapterThatLoadsOnDemandIsNamedLikeAFile(t *testing.T) {
m := loraModel("vllm", 2)
m.Adapters[0].Name = "team/sql"
if _, err := New([]Model{m}); err == nil {
t.Error("an adapter with a slash in its name was accepted")
}
m.MaxLoadedAdapters = 0
if _, err := New([]Model{m}); err != nil {
t.Errorf("loaded at start, the name names no file: %v", err)
}
}
func TestTheEngineProfileOfAModelWithACapStartsVLLMReadyToLoad(t *testing.T) {
p, ok := loraModel("vllm", 3).EngineProfile("p")
if !ok {
t.Fatal("no profile")
}
if !p.RuntimeLoRA() || !slices.Equal(p.Args, []string{"--max-loras", "3"}) {
t.Errorf("env %v args %v", p.Env, p.Args)
}
if p, _ := loraModel("vllm", 0).EngineProfile("p"); p.RuntimeLoRA() || len(p.Args) != 0 {
t.Errorf("without a cap: env %v args %v", p.Env, p.Args)
}
}