Merge C24 LoRA adapters on demand for vLLM, with affinity
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
bd3eca4092
24 files changed
+1773
-35
No files matched your search
@@ -1846,8 +1846,12 @@ func (a *API) setAdapters(w http.ResponseWriter, r *http.Request) {
|
||||
for _, ad := range m.Adapters {
|
||||
have[ad.Source] = true
|
||||
}
|
||||
// The file is read only where llama.cpp will load it: a vLLM adapter is
|
||||
// a PEFT directory, which has no header to read, and vLLM itself refuses
|
||||
// one that does not fit the model when it is loaded.
|
||||
ggufs := slices.ContainsFunc(m.Profiles, func(p catalog.Profile) bool { return p.Engine == "llamacpp" })
|
||||
for _, ad := range req.Adapters {
|
||||
if have[ad.Source] {
|
||||
if have[ad.Source] || !ggufs {
|
||||
continue // already checked when it was attached
|
||||
}
|
||||
h, err := gguf.KindFromURL(r.Context(), safedial.Client(safedial.ControlPlane, 30*time.Second), ad.Source)
|
||||
|
||||
@@ -288,9 +288,18 @@ type Model struct {
|
||||
External *External `json:"external,omitempty" koanf:"external"`
|
||||
// Adapters are LoRA adapters served on this model's replicas, each asked
|
||||
// for by its own name as though it were a model. Every replica loads
|
||||
// every adapter, applying one only to a request that names it.
|
||||
Adapters []Adapter `json:"adapters,omitempty" koanf:"adapters"`
|
||||
Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"`
|
||||
// every adapter, applying one only to a request that names it — unless
|
||||
// MaxLoadedAdapters is set.
|
||||
Adapters []Adapter `json:"adapters,omitempty" koanf:"adapters"`
|
||||
// MaxLoadedAdapters, above zero, has each replica hold at most this many
|
||||
// adapters at once, loaded when a request first asks for one and unloaded
|
||||
// least recently used first to make room. The router sends a request for
|
||||
// an adapter to a replica that already holds it while that one has room
|
||||
// to take it. vLLM profiles only: llama-server loads adapters at start
|
||||
// and cannot load one afterwards. Zero loads every adapter on every
|
||||
// replica at start.
|
||||
MaxLoadedAdapters int `json:"max_loaded_adapters,omitempty" koanf:"max_loaded_adapters"`
|
||||
Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"`
|
||||
// Canary sends a share of this model's requests to another catalog
|
||||
// model — the same model at new weights — to compare the two before
|
||||
// moving to it (`llmgw ctl model canary`).
|
||||
@@ -871,7 +880,7 @@ func (m Model) EngineProfile(name string) (engines.Profile, bool) {
|
||||
ModelPath: m.Source,
|
||||
ServedModelName: m.Name,
|
||||
Adapters: m.engineAdapters(),
|
||||
}, true
|
||||
}.WithRuntimeLoRA(m.MaxLoadedAdapters), true
|
||||
}
|
||||
return engines.Profile{}, false
|
||||
}
|
||||
@@ -1040,8 +1049,10 @@ func (m Model) IsExternal() bool { return m.External != nil }
|
||||
type Adapter struct {
|
||||
// Name is what clients ask for, like a model name.
|
||||
Name string `json:"name" koanf:"name"`
|
||||
// Source is the adapter file (GGUF for llama.cpp), fetched by each node
|
||||
// as the model's weights are.
|
||||
// Source is the adapter: a GGUF file for llama.cpp; for vLLM a PEFT
|
||||
// adapter directory (a mirror version ending in "/", a path on the node,
|
||||
// or a Hugging Face repository id). Fetched by each node as the model's
|
||||
// weights are.
|
||||
Source string `json:"source" koanf:"source"`
|
||||
// Scale is how strongly it is applied; zero takes 1.
|
||||
Scale float64 `json:"scale,omitempty" koanf:"scale"`
|
||||
@@ -1056,6 +1067,20 @@ func (a Adapter) EffectiveScale() float64 {
|
||||
}
|
||||
|
||||
func validateAdapters(m Model) error {
|
||||
if m.MaxLoadedAdapters < 0 {
|
||||
return fmt.Errorf("model %q: max_loaded_adapters cannot be negative", m.Name)
|
||||
}
|
||||
if m.MaxLoadedAdapters > 0 {
|
||||
// Loading on demand is vLLM's runtime LoRA API. llama-server can
|
||||
// only change the scale of adapters it loaded at start, so a cap
|
||||
// there would be a cap on nothing.
|
||||
for _, p := range m.Profiles {
|
||||
if p.Engine != "vllm" {
|
||||
return fmt.Errorf("model %q profile %q: max_loaded_adapters needs vllm profiles; "+
|
||||
"llama.cpp loads every adapter at start", m.Name, p.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(m.Adapters) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -1064,9 +1089,17 @@ func validateAdapters(m Model) error {
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
// llama.cpp loads adapters at start and applies them per request;
|
||||
// the other engines are not driven that way yet.
|
||||
if p.Engine != "llamacpp" && p.Engine != "sim" {
|
||||
return fmt.Errorf("model %q profile %q: adapters are served by llamacpp profiles only", m.Name, p.Name)
|
||||
// vLLM loads them at start or on demand. The other engines are not
|
||||
// driven either way yet.
|
||||
if p.Engine != "llamacpp" && p.Engine != "vllm" && p.Engine != "sim" {
|
||||
return fmt.Errorf("model %q profile %q: adapters are served by llamacpp and vllm profiles only", m.Name, p.Name)
|
||||
}
|
||||
// The variable is what switches vLLM to loading on demand, and the
|
||||
// router loads adapters only for a model with a cap: set by hand, it
|
||||
// would start replicas holding no adapter that nothing ever loads.
|
||||
if _, set := p.Env[engines.RuntimeLoRAEnv]; set && m.MaxLoadedAdapters == 0 {
|
||||
return fmt.Errorf("model %q profile %q: set max_loaded_adapters rather than %s",
|
||||
m.Name, p.Name, engines.RuntimeLoRAEnv)
|
||||
}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
@@ -1080,6 +1113,9 @@ func validateAdapters(m Model) error {
|
||||
return fmt.Errorf("model %q: adapter %q is declared twice", m.Name, a.Name)
|
||||
case a.Scale < 0:
|
||||
return fmt.Errorf("model %q: adapter %q has a negative scale", m.Name, a.Name)
|
||||
case m.MaxLoadedAdapters > 0 && (strings.ContainsAny(a.Name, `/\`) || a.Name == "." || a.Name == ".."):
|
||||
// Loaded on demand, the name is a file's name on the node.
|
||||
return fmt.Errorf("model %q: adapter %q cannot contain a slash when adapters load on demand", m.Name, a.Name)
|
||||
}
|
||||
seen[a.Name] = true
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
)
|
||||
|
||||
func loraModel(engine string, limit int) Model {
|
||||
m := model("coder", Profile{Name: "p", Engine: engine, GPUClass: "h100-80g", TP: 1})
|
||||
m.Adapters = []Adapter{{Name: "coder-sql", Source: "http://mirror/coder-sql/v1/"}}
|
||||
m.MaxLoadedAdapters = limit
|
||||
return m
|
||||
}
|
||||
|
||||
func TestAdaptersLoadOnDemandOnVLLMOnly(t *testing.T) {
|
||||
if _, err := New([]Model{loraModel("vllm", 4)}); err != nil {
|
||||
t.Errorf("vLLM with a cap: %v", err)
|
||||
}
|
||||
if _, err := New([]Model{loraModel("vllm", 0)}); err != nil {
|
||||
t.Errorf("vLLM loading every adapter at start: %v", err)
|
||||
}
|
||||
_, err := New([]Model{loraModel("llamacpp", 4)})
|
||||
if err == nil || !strings.Contains(err.Error(), "max_loaded_adapters needs vllm") {
|
||||
t.Errorf("llama.cpp with a cap: %v", err)
|
||||
}
|
||||
if _, err := New([]Model{loraModel("vllm", -1)}); err == nil {
|
||||
t.Error("a negative cap was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeLoadingCannotBeTurnedOnByHand(t *testing.T) {
|
||||
m := loraModel("vllm", 0)
|
||||
m.Profiles[0].Env = map[string]string{engines.RuntimeLoRAEnv: "True"}
|
||||
if _, err := New([]Model{m}); err == nil || !strings.Contains(err.Error(), "max_loaded_adapters") {
|
||||
t.Errorf("want a pointer to max_loaded_adapters, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAdapterThatLoadsOnDemandIsNamedLikeAFile(t *testing.T) {
|
||||
m := loraModel("vllm", 2)
|
||||
m.Adapters[0].Name = "team/sql"
|
||||
if _, err := New([]Model{m}); err == nil {
|
||||
t.Error("an adapter with a slash in its name was accepted")
|
||||
}
|
||||
m.MaxLoadedAdapters = 0
|
||||
if _, err := New([]Model{m}); err != nil {
|
||||
t.Errorf("loaded at start, the name names no file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheEngineProfileOfAModelWithACapStartsVLLMReadyToLoad(t *testing.T) {
|
||||
p, ok := loraModel("vllm", 3).EngineProfile("p")
|
||||
if !ok {
|
||||
t.Fatal("no profile")
|
||||
}
|
||||
if !p.RuntimeLoRA() || !slices.Equal(p.Args, []string{"--max-loras", "3"}) {
|
||||
t.Errorf("env %v args %v", p.Env, p.Args)
|
||||
}
|
||||
if p, _ := loraModel("vllm", 0).EngineProfile("p"); p.RuntimeLoRA() || len(p.Args) != 0 {
|
||||
t.Errorf("without a cap: env %v args %v", p.Env, p.Args)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user