LoRA adapters on vLLM, loaded on demand with replica affinity: max_loaded_adapters
A vLLM model can carry PEFT adapters, registered at start, or with max_loaded_adapters held at most N per replica: the router loads an adapter through vLLM's runtime LoRA API when a request first wants it, sends the adapter's requests to a replica that holds it while that one is not saturated, loads each cold adapter once, and unloads the least recently used idle one to make room. llama.cpp keeps loading every adapter at start. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
2a1e0e6c4c
24 files changed
+1773
-35
No files matched your search
@@ -288,9 +288,18 @@ type Model struct {
|
||||
External *External `json:"external,omitempty" koanf:"external"`
|
||||
// Adapters are LoRA adapters served on this model's replicas, each asked
|
||||
// for by its own name as though it were a model. Every replica loads
|
||||
// every adapter, applying one only to a request that names it.
|
||||
Adapters []Adapter `json:"adapters,omitempty" koanf:"adapters"`
|
||||
Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"`
|
||||
// every adapter, applying one only to a request that names it — unless
|
||||
// MaxLoadedAdapters is set.
|
||||
Adapters []Adapter `json:"adapters,omitempty" koanf:"adapters"`
|
||||
// MaxLoadedAdapters, above zero, has each replica hold at most this many
|
||||
// adapters at once, loaded when a request first asks for one and unloaded
|
||||
// least recently used first to make room. The router sends a request for
|
||||
// an adapter to a replica that already holds it while that one has room
|
||||
// to take it. vLLM profiles only: llama-server loads adapters at start
|
||||
// and cannot load one afterwards. Zero loads every adapter on every
|
||||
// replica at start.
|
||||
MaxLoadedAdapters int `json:"max_loaded_adapters,omitempty" koanf:"max_loaded_adapters"`
|
||||
Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"`
|
||||
// Canary sends a share of this model's requests to another catalog
|
||||
// model — the same model at new weights — to compare the two before
|
||||
// moving to it (`llmgw ctl model canary`).
|
||||
@@ -871,7 +880,7 @@ func (m Model) EngineProfile(name string) (engines.Profile, bool) {
|
||||
ModelPath: m.Source,
|
||||
ServedModelName: m.Name,
|
||||
Adapters: m.engineAdapters(),
|
||||
}, true
|
||||
}.WithRuntimeLoRA(m.MaxLoadedAdapters), true
|
||||
}
|
||||
return engines.Profile{}, false
|
||||
}
|
||||
@@ -1040,8 +1049,10 @@ func (m Model) IsExternal() bool { return m.External != nil }
|
||||
type Adapter struct {
|
||||
// Name is what clients ask for, like a model name.
|
||||
Name string `json:"name" koanf:"name"`
|
||||
// Source is the adapter file (GGUF for llama.cpp), fetched by each node
|
||||
// as the model's weights are.
|
||||
// Source is the adapter: a GGUF file for llama.cpp; for vLLM a PEFT
|
||||
// adapter directory (a mirror version ending in "/", a path on the node,
|
||||
// or a Hugging Face repository id). Fetched by each node as the model's
|
||||
// weights are.
|
||||
Source string `json:"source" koanf:"source"`
|
||||
// Scale is how strongly it is applied; zero takes 1.
|
||||
Scale float64 `json:"scale,omitempty" koanf:"scale"`
|
||||
@@ -1056,6 +1067,20 @@ func (a Adapter) EffectiveScale() float64 {
|
||||
}
|
||||
|
||||
func validateAdapters(m Model) error {
|
||||
if m.MaxLoadedAdapters < 0 {
|
||||
return fmt.Errorf("model %q: max_loaded_adapters cannot be negative", m.Name)
|
||||
}
|
||||
if m.MaxLoadedAdapters > 0 {
|
||||
// Loading on demand is vLLM's runtime LoRA API. llama-server can
|
||||
// only change the scale of adapters it loaded at start, so a cap
|
||||
// there would be a cap on nothing.
|
||||
for _, p := range m.Profiles {
|
||||
if p.Engine != "vllm" {
|
||||
return fmt.Errorf("model %q profile %q: max_loaded_adapters needs vllm profiles; "+
|
||||
"llama.cpp loads every adapter at start", m.Name, p.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(m.Adapters) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -1064,9 +1089,17 @@ func validateAdapters(m Model) error {
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
// llama.cpp loads adapters at start and applies them per request;
|
||||
// the other engines are not driven that way yet.
|
||||
if p.Engine != "llamacpp" && p.Engine != "sim" {
|
||||
return fmt.Errorf("model %q profile %q: adapters are served by llamacpp profiles only", m.Name, p.Name)
|
||||
// vLLM loads them at start or on demand. The other engines are not
|
||||
// driven either way yet.
|
||||
if p.Engine != "llamacpp" && p.Engine != "vllm" && p.Engine != "sim" {
|
||||
return fmt.Errorf("model %q profile %q: adapters are served by llamacpp and vllm profiles only", m.Name, p.Name)
|
||||
}
|
||||
// The variable is what switches vLLM to loading on demand, and the
|
||||
// router loads adapters only for a model with a cap: set by hand, it
|
||||
// would start replicas holding no adapter that nothing ever loads.
|
||||
if _, set := p.Env[engines.RuntimeLoRAEnv]; set && m.MaxLoadedAdapters == 0 {
|
||||
return fmt.Errorf("model %q profile %q: set max_loaded_adapters rather than %s",
|
||||
m.Name, p.Name, engines.RuntimeLoRAEnv)
|
||||
}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
@@ -1080,6 +1113,9 @@ func validateAdapters(m Model) error {
|
||||
return fmt.Errorf("model %q: adapter %q is declared twice", m.Name, a.Name)
|
||||
case a.Scale < 0:
|
||||
return fmt.Errorf("model %q: adapter %q has a negative scale", m.Name, a.Name)
|
||||
case m.MaxLoadedAdapters > 0 && (strings.ContainsAny(a.Name, `/\`) || a.Name == "." || a.Name == ".."):
|
||||
// Loaded on demand, the name is a file's name on the node.
|
||||
return fmt.Errorf("model %q: adapter %q cannot contain a slash when adapters load on demand", m.Name, a.Name)
|
||||
}
|
||||
seen[a.Name] = true
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
)
|
||||
|
||||
func loraModel(engine string, limit int) Model {
|
||||
m := model("coder", Profile{Name: "p", Engine: engine, GPUClass: "h100-80g", TP: 1})
|
||||
m.Adapters = []Adapter{{Name: "coder-sql", Source: "http://mirror/coder-sql/v1/"}}
|
||||
m.MaxLoadedAdapters = limit
|
||||
return m
|
||||
}
|
||||
|
||||
func TestAdaptersLoadOnDemandOnVLLMOnly(t *testing.T) {
|
||||
if _, err := New([]Model{loraModel("vllm", 4)}); err != nil {
|
||||
t.Errorf("vLLM with a cap: %v", err)
|
||||
}
|
||||
if _, err := New([]Model{loraModel("vllm", 0)}); err != nil {
|
||||
t.Errorf("vLLM loading every adapter at start: %v", err)
|
||||
}
|
||||
_, err := New([]Model{loraModel("llamacpp", 4)})
|
||||
if err == nil || !strings.Contains(err.Error(), "max_loaded_adapters needs vllm") {
|
||||
t.Errorf("llama.cpp with a cap: %v", err)
|
||||
}
|
||||
if _, err := New([]Model{loraModel("vllm", -1)}); err == nil {
|
||||
t.Error("a negative cap was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeLoadingCannotBeTurnedOnByHand(t *testing.T) {
|
||||
m := loraModel("vllm", 0)
|
||||
m.Profiles[0].Env = map[string]string{engines.RuntimeLoRAEnv: "True"}
|
||||
if _, err := New([]Model{m}); err == nil || !strings.Contains(err.Error(), "max_loaded_adapters") {
|
||||
t.Errorf("want a pointer to max_loaded_adapters, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAdapterThatLoadsOnDemandIsNamedLikeAFile(t *testing.T) {
|
||||
m := loraModel("vllm", 2)
|
||||
m.Adapters[0].Name = "team/sql"
|
||||
if _, err := New([]Model{m}); err == nil {
|
||||
t.Error("an adapter with a slash in its name was accepted")
|
||||
}
|
||||
m.MaxLoadedAdapters = 0
|
||||
if _, err := New([]Model{m}); err != nil {
|
||||
t.Errorf("loaded at start, the name names no file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheEngineProfileOfAModelWithACapStartsVLLMReadyToLoad(t *testing.T) {
|
||||
p, ok := loraModel("vllm", 3).EngineProfile("p")
|
||||
if !ok {
|
||||
t.Fatal("no profile")
|
||||
}
|
||||
if !p.RuntimeLoRA() || !slices.Equal(p.Args, []string{"--max-loras", "3"}) {
|
||||
t.Errorf("env %v args %v", p.Env, p.Args)
|
||||
}
|
||||
if p, _ := loraModel("vllm", 0).EngineProfile("p"); p.RuntimeLoRA() || len(p.Args) != 0 {
|
||||
t.Errorf("without a cap: env %v args %v", p.Env, p.Args)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user