Ilker ManapandClaude Opus 5.5
cf53904cc5
A model's adapters are fetched by each node with its weights and loaded unapplied at start; a request naming an adapter reaches the model's replicas with llama.cpp's per-request lora field. Adapter names share the model namespace, are listed, and follow allowlists on their own. The router's config seed and pushed catalog now build the name map in one place. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
99 lines
3.3 KiB
Go
99 lines
3.3 KiB
Go
package router
|
|
|
|
import (
|
|
"encoding/json"
|
|
"net/http"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
func adapterModel() catalog.Model {
|
|
m := catModel("coder", nil)
|
|
m.Profiles = []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 4}}
|
|
m.Adapters = []catalog.Adapter{
|
|
{Name: "coder-sql", Source: "http://mirror/sql.gguf"},
|
|
{Name: "coder-docs", Source: "http://mirror/docs.gguf", Scale: 0.5},
|
|
}
|
|
return m
|
|
}
|
|
|
|
func TestAnAdapterIsAskedForByNameAndAppliedOnItsModel(t *testing.T) {
|
|
e := newFakeEngine(t, "coder")
|
|
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"coder"}})
|
|
cfg.Models = []catalog.Model{adapterModel()}
|
|
rt := newTestRouter(t, cfg)
|
|
|
|
sent := func(model string) map[string]any {
|
|
t.Helper()
|
|
rec := post(t, rt.Handler(), map[string]any{"model": model, "messages": []any{}}, nil)
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("%s: status %d: %s", model, rec.Code, rec.Body.String())
|
|
}
|
|
var body map[string]any
|
|
_ = json.Unmarshal(e.last.Load().([]byte), &body)
|
|
return body
|
|
}
|
|
body := sent("coder-docs")
|
|
if body["model"] != "coder" {
|
|
t.Errorf("the engine was asked for %v, not the model the adapter runs on", body["model"])
|
|
}
|
|
lora, _ := json.Marshal(body["lora"])
|
|
if string(lora) != `[{"id":1,"scale":0.5}]` {
|
|
t.Errorf("lora %s, expected the second adapter at its scale", lora)
|
|
}
|
|
if body := sent("coder-sql"); func() string { b, _ := json.Marshal(body["lora"]); return string(b) }() != `[{"id":0,"scale":1}]` {
|
|
t.Errorf("the first adapter, at the default scale: %v", body["lora"])
|
|
}
|
|
if body := sent("coder"); body["lora"] != nil {
|
|
t.Errorf("the base model was sent an adapter: %v", body["lora"])
|
|
}
|
|
}
|
|
|
|
func TestAKeyAllowedAnAdapterReachesThatAdapterOnly(t *testing.T) {
|
|
cfg := baseConfig()
|
|
cfg.Models = []catalog.Model{adapterModel()}
|
|
rt := newTestRouter(t, cfg)
|
|
key := func(allowed ...string) auth.Principal {
|
|
return auth.Principal{Limits: store.Effective{AllowedModels: allowed}}
|
|
}
|
|
for _, c := range []struct {
|
|
allowed []string
|
|
requested string
|
|
want bool
|
|
}{
|
|
{[]string{"coder-sql"}, "coder-sql", true},
|
|
{[]string{"coder-sql"}, "coder", false},
|
|
{[]string{"coder-sql"}, "coder-docs", false},
|
|
{[]string{"coder"}, "coder-sql", true}, // the model allows its adapters
|
|
{[]string{"coder"}, "coder", true},
|
|
} {
|
|
if got := rt.permits(key(c.allowed...), c.requested); got != c.want {
|
|
t.Errorf("allowed %v, asked for %s: %v", c.allowed, c.requested, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAdaptersShareTheNamespaceOfModels(t *testing.T) {
|
|
a := adapterModel()
|
|
b := catModel("coder-sql", nil)
|
|
if _, err := catalog.New([]catalog.Model{a, b}); err == nil {
|
|
t.Error("an adapter named like another model was accepted")
|
|
}
|
|
bad := adapterModel()
|
|
bad.Profiles = []catalog.Profile{{Name: "gpu", Engine: "vllm", GPUClass: "h100-80g", TP: 1}}
|
|
if _, err := catalog.New([]catalog.Model{bad}); err == nil {
|
|
t.Error("adapters on a vLLM profile were accepted")
|
|
}
|
|
c, err := catalog.New([]catalog.Model{adapterModel()})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if m, ad, i, ok := c.ResolveAdapter("coder-docs"); !ok || m.Name != "coder" || i != 1 || ad.EffectiveScale() != 0.5 {
|
|
t.Errorf("resolve: %v %+v %d %v", m.Name, ad, i, ok)
|
|
}
|
|
}
|