Ilker ManapandClaude Opus 5.5
2a1e0e6c4c
A vLLM model can carry PEFT adapters, registered at start, or with max_loaded_adapters held at most N per replica: the router loads an adapter through vLLM's runtime LoRA API when a request first wants it, sends the adapter's requests to a replica that holds it while that one is not saturated, loads each cold adapter once, and unloads the least recently used idle one to make room. llama.cpp keeps loading every adapter at start. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
99 lines
3.3 KiB
Go
99 lines
3.3 KiB
Go
package router
|
|
|
|
import (
|
|
"encoding/json"
|
|
"net/http"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
func adapterModel() catalog.Model {
|
|
m := catModel("coder", nil)
|
|
m.Profiles = []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 4}}
|
|
m.Adapters = []catalog.Adapter{
|
|
{Name: "coder-sql", Source: "http://mirror/sql.gguf"},
|
|
{Name: "coder-docs", Source: "http://mirror/docs.gguf", Scale: 0.5},
|
|
}
|
|
return m
|
|
}
|
|
|
|
func TestAnAdapterIsAskedForByNameAndAppliedOnItsModel(t *testing.T) {
|
|
e := newFakeEngine(t, "coder")
|
|
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"coder"}})
|
|
cfg.Models = []catalog.Model{adapterModel()}
|
|
rt := newTestRouter(t, cfg)
|
|
|
|
sent := func(model string) map[string]any {
|
|
t.Helper()
|
|
rec := post(t, rt.Handler(), map[string]any{"model": model, "messages": []any{}}, nil)
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("%s: status %d: %s", model, rec.Code, rec.Body.String())
|
|
}
|
|
var body map[string]any
|
|
_ = json.Unmarshal(e.last.Load().([]byte), &body)
|
|
return body
|
|
}
|
|
body := sent("coder-docs")
|
|
if body["model"] != "coder" {
|
|
t.Errorf("the engine was asked for %v, not the model the adapter runs on", body["model"])
|
|
}
|
|
lora, _ := json.Marshal(body["lora"])
|
|
if string(lora) != `[{"id":1,"scale":0.5}]` {
|
|
t.Errorf("lora %s, expected the second adapter at its scale", lora)
|
|
}
|
|
if body := sent("coder-sql"); func() string { b, _ := json.Marshal(body["lora"]); return string(b) }() != `[{"id":0,"scale":1}]` {
|
|
t.Errorf("the first adapter, at the default scale: %v", body["lora"])
|
|
}
|
|
if body := sent("coder"); body["lora"] != nil {
|
|
t.Errorf("the base model was sent an adapter: %v", body["lora"])
|
|
}
|
|
}
|
|
|
|
func TestAKeyAllowedAnAdapterReachesThatAdapterOnly(t *testing.T) {
|
|
cfg := baseConfig()
|
|
cfg.Models = []catalog.Model{adapterModel()}
|
|
rt := newTestRouter(t, cfg)
|
|
key := func(allowed ...string) auth.Principal {
|
|
return auth.Principal{Limits: store.Effective{AllowedModels: allowed}}
|
|
}
|
|
for _, c := range []struct {
|
|
allowed []string
|
|
requested string
|
|
want bool
|
|
}{
|
|
{[]string{"coder-sql"}, "coder-sql", true},
|
|
{[]string{"coder-sql"}, "coder", false},
|
|
{[]string{"coder-sql"}, "coder-docs", false},
|
|
{[]string{"coder"}, "coder-sql", true}, // the model allows its adapters
|
|
{[]string{"coder"}, "coder", true},
|
|
} {
|
|
if got := rt.permits(key(c.allowed...), c.requested); got != c.want {
|
|
t.Errorf("allowed %v, asked for %s: %v", c.allowed, c.requested, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAdaptersShareTheNamespaceOfModels(t *testing.T) {
|
|
a := adapterModel()
|
|
b := catModel("coder-sql", nil)
|
|
if _, err := catalog.New([]catalog.Model{a, b}); err == nil {
|
|
t.Error("an adapter named like another model was accepted")
|
|
}
|
|
bad := adapterModel()
|
|
bad.Profiles = []catalog.Profile{{Name: "gpu", Engine: "sglang", GPUClass: "h100-80g", TP: 1}}
|
|
if _, err := catalog.New([]catalog.Model{bad}); err == nil {
|
|
t.Error("adapters on an SGLang profile were accepted")
|
|
}
|
|
c, err := catalog.New([]catalog.Model{adapterModel()})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if m, ad, i, ok := c.ResolveAdapter("coder-docs"); !ok || m.Name != "coder" || i != 1 || ad.EffectiveScale() != 0.5 {
|
|
t.Errorf("resolve: %v %+v %d %v", m.Name, ad, i, ok)
|
|
}
|
|
}
|