Ilker ManapandClaude Opus 5.5
2a1e0e6c4c
A vLLM model can carry PEFT adapters, registered at start, or with max_loaded_adapters held at most N per replica: the router loads an adapter through vLLM's runtime LoRA API when a request first wants it, sends the adapter's requests to a replica that holds it while that one is not saturated, loads each cold adapter once, and unloads the least recently used idle one to make room. llama.cpp keeps loading every adapter at start. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
69 lines
2.0 KiB
Go
69 lines
2.0 KiB
Go
package scheduler
|
|
|
|
import (
|
|
"slices"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
)
|
|
|
|
func ids(eps []endpoint.Snapshot) []string {
|
|
var out []string
|
|
for _, e := range eps {
|
|
out = append(out, e.ID)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func adapterCtx(holders, free []string) *Ctx {
|
|
c := newCtx(Request{Model: "m", Adapter: "m-sql"})
|
|
c.Adapters = &AdapterView{Holders: map[string]bool{}, Free: map[string]bool{}}
|
|
for _, h := range holders {
|
|
c.Adapters.Holders[h] = true
|
|
}
|
|
for _, f := range free {
|
|
c.Adapters.Free[f] = true
|
|
}
|
|
return c
|
|
}
|
|
|
|
func TestAdapterAffinityKeepsTheHoldersFirstThenTheReplicasWithRoom(t *testing.T) {
|
|
f := AdapterAffinity(4, 0.9)
|
|
eps := []endpoint.Snapshot{ep("a"), ep("b"), ep("c")}
|
|
for _, c := range []struct {
|
|
name string
|
|
holders, free []string
|
|
want []string
|
|
}{
|
|
{"held", []string{"b"}, []string{"a", "c"}, []string{"b"}},
|
|
{"held nowhere, room on one", nil, []string{"c"}, []string{"c"}},
|
|
{"no room anywhere", nil, nil, []string{"a", "b", "c"}},
|
|
} {
|
|
if got := ids(f.Filter(adapterCtx(c.holders, c.free), eps)); !slices.Equal(got, c.want) {
|
|
t.Errorf("%s: kept %v, want %v", c.name, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestASaturatedHolderDoesNotKeepItsAdaptersTraffic(t *testing.T) {
|
|
f := AdapterAffinity(4, 0.9)
|
|
eps := []endpoint.Snapshot{ep("a", waiting(8)), ep("b"), ep("c", kv(0.95))}
|
|
got := ids(f.Filter(adapterCtx([]string{"a"}, []string{"b", "c"}), eps))
|
|
if !slices.Equal(got, []string{"b"}) {
|
|
t.Errorf("kept %v; the holder is saturated, and so is the other with room", got)
|
|
}
|
|
}
|
|
|
|
func TestAdapterAffinityLeavesOtherRequestsAlone(t *testing.T) {
|
|
f := AdapterAffinity(4, 0.9)
|
|
eps := []endpoint.Snapshot{ep("a"), ep("b")}
|
|
c := adapterCtx([]string{"a"}, nil)
|
|
c.Req.Adapter = ""
|
|
if got := f.Filter(c, eps); len(got) != 2 {
|
|
t.Errorf("a request for no adapter was filtered: %v", ids(got))
|
|
}
|
|
if got := f.Filter(newCtx(Request{Model: "m"}), eps); len(got) != 2 {
|
|
t.Errorf("a request with no view was filtered: %v", ids(got))
|
|
}
|
|
}
|