LoRA adapters on vLLM, loaded on demand with replica affinity: max_loaded_adapters
A vLLM model can carry PEFT adapters, registered at start, or with max_loaded_adapters held at most N per replica: the router loads an adapter through vLLM's runtime LoRA API when a request first wants it, sends the adapter's requests to a replica that holds it while that one is not saturated, loads each cold adapter once, and unloads the least recently used idle one to make room. llama.cpp keeps loading every adapter at start. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
2a1e0e6c4c
24 files changed
+1773
-35
No files matched your search
@@ -1053,6 +1053,15 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
}
|
||||
spec.Profile.Adapters[i].Path = path
|
||||
}
|
||||
// Adapters that load on demand are loaded by the router, by a path that
|
||||
// has to mean the same file on every node: a link of the adapter's name
|
||||
// in the directory engines run in.
|
||||
if spec.Profile.RuntimeLoRA() {
|
||||
if err := engines.LinkRuntimeAdapters(a.sup.EngineDir(), spec.Profile.Adapters); err != nil {
|
||||
report(supervisor.Failed, err.Error(), nil)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
cmds, err := eng.Commands(spec)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
package engines
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// RuntimeLoRAEnv is the variable that lets vLLM load and unload adapters
|
||||
// while it runs, through POST /v1/load_lora_adapter and
|
||||
// /v1/unload_lora_adapter. Without it those routes do not exist.
|
||||
//
|
||||
// It doubles as the mark that a profile's adapters load on demand: the
|
||||
// catalog sets it for a model with max_loaded_adapters and refuses it set by
|
||||
// hand anywhere else, so it cannot disagree with what the router does.
|
||||
const RuntimeLoRAEnv = "VLLM_ALLOW_RUNTIME_LORA_UPDATING"
|
||||
|
||||
// RuntimeLoRADir is where an agent links each adapter it fetched for a
|
||||
// replica whose adapters load on demand, under the adapter's name, relative
|
||||
// to the directory engines run in.
|
||||
//
|
||||
// Relative on purpose. The router asks the engine to load an adapter by a
|
||||
// path, and does not know where any node keeps its cache — each node may
|
||||
// keep it somewhere else, and a replica spanning nodes reads the adapter on
|
||||
// every one of them. vLLM resolves a relative path that exists against its
|
||||
// working directory, which on every node is the one the agent starts
|
||||
// engines in, and the agent links the adapter there before the engine
|
||||
// starts. So one path, the same everywhere, names the right file on each.
|
||||
const RuntimeLoRADir = "llmgw-lora"
|
||||
|
||||
// WithRuntimeLoRA has a vLLM profile with adapters start holding none of
|
||||
// them, ready to load up to limit at a time on demand: the engine is allowed
|
||||
// runtime loading, and --max-loras — how many adapters one batch may mix —
|
||||
// is the cap, unless the operator's arguments already say. Anything else is
|
||||
// returned as it is.
|
||||
func (p Profile) WithRuntimeLoRA(limit int) Profile {
|
||||
if limit <= 0 || p.Engine != "vllm" || len(p.Adapters) == 0 {
|
||||
return p
|
||||
}
|
||||
env := maps.Clone(p.Env)
|
||||
if env == nil {
|
||||
env = map[string]string{}
|
||||
}
|
||||
env[RuntimeLoRAEnv] = "True"
|
||||
p.Env = env
|
||||
if !hasFlag(p.Args, "--max-loras") {
|
||||
p.Args = append([]string{"--max-loras", strconv.Itoa(limit)}, p.Args...)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// RuntimeLoRA reports whether a profile's adapters load on demand rather
|
||||
// than at start.
|
||||
func (p Profile) RuntimeLoRA() bool {
|
||||
v, ok := p.Env[RuntimeLoRAEnv]
|
||||
return ok && v != "" && v != "0" && !strings.EqualFold(v, "false")
|
||||
}
|
||||
|
||||
// RuntimeLoRAPath is the lora_path the router gives vLLM to load an adapter
|
||||
// from: the link the agent made, or, for a Hugging Face repository id, the
|
||||
// id, which vLLM downloads itself and the agent leaves alone.
|
||||
func RuntimeLoRAPath(name, source string) string {
|
||||
if IsHubID(source) {
|
||||
return source
|
||||
}
|
||||
return RuntimeLoRADir + "/" + name
|
||||
}
|
||||
|
||||
// IsHubID reports whether a source is a bare Hugging Face repository id
|
||||
// ("org/repo") rather than a URL or a path: what the agent passes to an
|
||||
// engine untouched.
|
||||
func IsHubID(source string) bool {
|
||||
switch {
|
||||
case source == "",
|
||||
strings.HasPrefix(source, "/"), strings.HasPrefix(source, "./"), strings.HasPrefix(source, "../"),
|
||||
strings.HasPrefix(source, "~"),
|
||||
strings.HasPrefix(source, "http://"), strings.HasPrefix(source, "https://"),
|
||||
strings.HasPrefix(source, "hf:"):
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// LinkRuntimeAdapters makes RuntimeLoRADir under dir hold a link to each
|
||||
// fetched adapter, named as the adapter is. A link already there is
|
||||
// replaced, so a replica started after an adapter's source changed loads
|
||||
// the new one. Adapters still named by a repository id have nothing local
|
||||
// to link.
|
||||
func LinkRuntimeAdapters(dir string, ads []Adapter) error {
|
||||
if len(ads) == 0 {
|
||||
return nil
|
||||
}
|
||||
root := filepath.Join(dir, RuntimeLoRADir)
|
||||
if err := os.MkdirAll(root, 0o755); err != nil {
|
||||
return fmt.Errorf("making the adapters' directory: %w", err)
|
||||
}
|
||||
for _, a := range ads {
|
||||
if IsHubID(a.Path) {
|
||||
continue
|
||||
}
|
||||
if a.Name == "" || strings.ContainsAny(a.Name, `/\`) || a.Name == "." || a.Name == ".." {
|
||||
return fmt.Errorf("adapter %q: not a name a file can have", a.Name)
|
||||
}
|
||||
target, err := filepath.Abs(a.Path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("adapter %s: %w", a.Name, err)
|
||||
}
|
||||
link := filepath.Join(root, a.Name)
|
||||
// Made beside the old one and renamed over it, so an engine loading
|
||||
// the adapter at this moment sees one or the other and never neither.
|
||||
tmp := link + ".new"
|
||||
_ = os.Remove(tmp)
|
||||
if err := os.Symlink(target, tmp); err != nil {
|
||||
return fmt.Errorf("adapter %s: %w", a.Name, err)
|
||||
}
|
||||
if err := os.Rename(tmp, link); err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return fmt.Errorf("adapter %s: %w", a.Name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// loraModules is vLLM's --lora-modules for adapters loaded at start: one
|
||||
// flag followed by every name=path, since vLLM keeps only the last of a
|
||||
// repeated flag.
|
||||
func loraModules(ads []Adapter) []string {
|
||||
out := []string{"--lora-modules"}
|
||||
for _, a := range ads {
|
||||
out = append(out, a.Name+"="+a.Path)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package engines
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func adapterSpec() ShardSpec {
|
||||
s := singleNodeSpec()
|
||||
s.Profile.Engine = "vllm"
|
||||
s.Profile.Adapters = []Adapter{
|
||||
{Name: "sql", Path: "/cache/ab-sql"},
|
||||
{Name: "docs", Path: "/cache/cd-docs"},
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func TestVLLMLoadsEveryAdapterAtStartUnderOneFlag(t *testing.T) {
|
||||
cmd := buildOne(t, &VLLM{}, adapterSpec())
|
||||
if !slices.Contains(cmd.Args, "--enable-lora") {
|
||||
t.Errorf("LoRA not enabled: %v", cmd.Args)
|
||||
}
|
||||
// vLLM keeps only the last of a repeated --lora-modules, so both
|
||||
// adapters must follow one flag.
|
||||
i := slices.Index(cmd.Args, "--lora-modules")
|
||||
if i < 0 || !slices.Equal(cmd.Args[i+1:i+3], []string{"sql=/cache/ab-sql", "docs=/cache/cd-docs"}) {
|
||||
t.Errorf("adapters not registered at start: %v", cmd.Args)
|
||||
}
|
||||
if n := countFlag(cmd.Args, "--lora-modules"); n != 1 {
|
||||
t.Errorf("--lora-modules given %d times", n)
|
||||
}
|
||||
if _, set := envMap(cmd.Env)[RuntimeLoRAEnv]; set {
|
||||
t.Error("runtime loading was allowed for adapters loaded at start")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVLLMWithAdaptersOnDemandStartsHoldingNone(t *testing.T) {
|
||||
s := adapterSpec()
|
||||
s.Profile = s.Profile.WithRuntimeLoRA(3)
|
||||
cmd := buildOne(t, &VLLM{}, s)
|
||||
if slices.Contains(cmd.Args, "--lora-modules") {
|
||||
t.Errorf("adapters registered at start: %v", cmd.Args)
|
||||
}
|
||||
if !slices.Contains(cmd.Args, "--enable-lora") || argValue(cmd.Args, "--max-loras") != "3" {
|
||||
t.Errorf("want LoRA enabled with the cap as --max-loras: %v", cmd.Args)
|
||||
}
|
||||
if envMap(cmd.Env)[RuntimeLoRAEnv] != "True" {
|
||||
t.Errorf("the load and unload routes were not turned on: %v", cmd.Env)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheOperatorsMaxLorasWinsOverTheCap(t *testing.T) {
|
||||
p := adapterSpec().Profile
|
||||
p.Args = []string{"--max-loras", "8"}
|
||||
got := p.WithRuntimeLoRA(3)
|
||||
if argValue(got.Args, "--max-loras") != "8" || countFlag(got.Args, "--max-loras") != 1 {
|
||||
t.Errorf("args %v", got.Args)
|
||||
}
|
||||
if p.Env != nil {
|
||||
t.Error("the profile's own env map was written to")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeLoRALeavesOtherProfilesAlone(t *testing.T) {
|
||||
llama := adapterSpec().Profile
|
||||
llama.Engine = "llamacpp"
|
||||
none := adapterSpec().Profile
|
||||
none.Adapters = nil
|
||||
for _, p := range []Profile{llama, none} {
|
||||
if got := p.WithRuntimeLoRA(2); got.RuntimeLoRA() || len(got.Args) != 0 {
|
||||
t.Errorf("%s with %d adapters: %+v", p.Engine, len(p.Adapters), got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheRouterLoadsAnAdapterByItsLinkOrItsRepository(t *testing.T) {
|
||||
for source, want := range map[string]string{
|
||||
"http://mirror:8081/sql-lora/v1/": "llmgw-lora/sql",
|
||||
"/srv/adapters/sql": "llmgw-lora/sql",
|
||||
"hf:org/repo/adapter.gguf": "llmgw-lora/sql",
|
||||
"someone/sql-lora": "someone/sql-lora",
|
||||
"https://huggingface.co/someone/sql": "llmgw-lora/sql",
|
||||
} {
|
||||
if got := RuntimeLoRAPath("sql", source); got != want {
|
||||
t.Errorf("%s: %s, want %s", source, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdaptersAreLinkedUnderTheirNamesAndRelinkedWhenTheyChange(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
old := filepath.Join(dir, "old-sql")
|
||||
cur := filepath.Join(dir, "new-sql")
|
||||
for _, d := range []string{old, cur} {
|
||||
if err := os.Mkdir(d, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ads := []Adapter{{Name: "sql", Path: old}, {Name: "hub", Path: "someone/hub-lora"}}
|
||||
if err := LinkRuntimeAdapters(dir, ads); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ads[0].Path = cur
|
||||
if err := LinkRuntimeAdapters(dir, ads); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := os.Readlink(filepath.Join(dir, RuntimeLoRADir, "sql"))
|
||||
if err != nil || got != cur {
|
||||
t.Errorf("link to %q (%v), want %q", got, err, cur)
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(dir, RuntimeLoRADir, "hub")); !os.IsNotExist(err) {
|
||||
t.Errorf("a repository id was linked: %v", err)
|
||||
}
|
||||
if err := LinkRuntimeAdapters(dir, []Adapter{{Name: "../escape", Path: cur}}); err == nil {
|
||||
t.Error("an adapter named like a path was linked")
|
||||
}
|
||||
}
|
||||
|
||||
func countFlag(args []string, flag string) int {
|
||||
n := 0
|
||||
for _, a := range args {
|
||||
if a == flag {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -147,6 +147,20 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Adapters loaded at start are registered under their names, and a
|
||||
// request names one as its model. Adapters loaded on demand start
|
||||
// unregistered: the router loads each when a request first wants it
|
||||
// here, by the link the agent made (RuntimeLoRADir). --max-loras is
|
||||
// vLLM's default of one at start — raise it in the profile's arguments
|
||||
// to mix more adapters in a batch — and the cap on demand, which
|
||||
// WithRuntimeLoRA put in the arguments.
|
||||
if len(p.Adapters) > 0 {
|
||||
args = append(args, "--enable-lora")
|
||||
if !p.RuntimeLoRA() {
|
||||
args = append(args, loraModules(p.Adapters)...)
|
||||
}
|
||||
}
|
||||
|
||||
// Profile arguments last, so an operator can override anything above.
|
||||
args = append(args, p.Args...)
|
||||
|
||||
|
||||
@@ -870,6 +870,16 @@ func (s *Supervisor) KillOrphan(o Orphan) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// EngineDir is the working directory engines run in: the engines' user's
|
||||
// home when they run as one, the work directory otherwise. A relative path
|
||||
// an engine is given resolves against it.
|
||||
func (s *Supervisor) EngineDir() string {
|
||||
if s.as != nil {
|
||||
return s.as.home
|
||||
}
|
||||
return s.cfg.WorkDir
|
||||
}
|
||||
|
||||
// dropTo makes a command run as the engines' user, when there is one: its
|
||||
// credential, its home as the working directory, and its caches there.
|
||||
func (s *Supervisor) dropTo(c *exec.Cmd) {
|
||||
|
||||
Reference in new issue
Block a user