Compare commits

...
Author SHA1 Message Date
Ilker ManapandClaude Opus 5.5 cf53904cc5 Serve LoRA adapters on llama.cpp models, asked for by name
Release / build (push) Successful in 1m11s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m47s
A model's adapters are fetched by each node with its weights and loaded
unapplied at start; a request naming an adapter reaches the model's
replicas with llama.cpp's per-request lora field. Adapter names share the
model namespace, are listed, and follow allowlists on their own. The
router's config seed and pushed catalog now build the name map in one place.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 23:40:47 +02:00
18 changed files with 542 additions and 41 deletions

No files matched your search

+12
View File
@@ -42,6 +42,18 @@ What v0.10.5 made settable from the command line is now on the pages too:
critical model's card no longer refuses a replica that preemption would make
room for.
### LoRA adapters
A llama.cpp model can carry `adapters`, each a name and a GGUF LoRA file. The
nodes fetch them with the model's weights and llama.cpp loads them all at start,
applied to nothing (`--lora-init-without-apply`); a request that names an
adapter goes to the model's replicas with `"lora": [{"id": i, "scale": s}]`,
so one replica serves the model and all its adapters. Adapter names share the
namespace of models, are listed in `/v1/models`, and follow key allowlists on
their own — a key allowed an adapter does not reach the model under it. Adapter
files are kept from weight cleanup like the model's own. Not yet for vLLM or
SGLang profiles, whose adapters are loaded differently.
### Models served outside the cluster
A catalog model can be an OpenAI-compatible API elsewhere: `external: {url,
+96 -26
View File
@@ -2873,8 +2873,11 @@ type EngineProfile struct {
Env map[string]string `protobuf:"bytes,9,rep,name=env,proto3" json:"env,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
ModelPath string `protobuf:"bytes,10,opt,name=model_path,json=modelPath,proto3" json:"model_path,omitempty"`
ServedModelName string `protobuf:"bytes,11,opt,name=served_model_name,json=servedModelName,proto3" json:"served_model_name,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
// adapters are LoRA adapters loaded beside the model, in order: the index
// is the id a request applies one by.
Adapters []*LoraAdapter `protobuf:"bytes,12,rep,name=adapters,proto3" json:"adapters,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *EngineProfile) Reset() {
@@ -2984,6 +2987,13 @@ func (x *EngineProfile) GetServedModelName() string {
return ""
}
func (x *EngineProfile) GetAdapters() []*LoraAdapter {
if x != nil {
return x.Adapters
}
return nil
}
// RemoveWeights deletes cached model files a node no longer needs.
//
// The paths are computed by the control plane from the catalog, but the agent
@@ -3777,6 +3787,60 @@ func (x *UpgradeReply) GetError() string {
return ""
}
// LoraAdapter is one adapter: the name clients ask for it by, and its source,
// which the agent fetches as it does the model's weights.
type LoraAdapter struct {
state protoimpl.MessageState `protogen:"open.v1"`
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
Source string `protobuf:"bytes,2,opt,name=source,proto3" json:"source,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *LoraAdapter) Reset() {
*x = LoraAdapter{}
mi := &file_llmgw_v1_agent_proto_msgTypes[36]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *LoraAdapter) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*LoraAdapter) ProtoMessage() {}
func (x *LoraAdapter) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[36]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use LoraAdapter.ProtoReflect.Descriptor instead.
func (*LoraAdapter) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{36}
}
func (x *LoraAdapter) GetName() string {
if x != nil {
return x.Name
}
return ""
}
func (x *LoraAdapter) GetSource() string {
if x != nil {
return x.Source
}
return ""
}
var File_llmgw_v1_agent_proto protoreflect.FileDescriptor
const file_llmgw_v1_agent_proto_rawDesc = "" +
@@ -4066,7 +4130,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"profile_fp\x18\x0f \x01(\tR\tprofileFp\x12\x17\n" +
"\aapi_key\x18\x10 \x01(\tR\x06apiKey\x12\x1f\n" +
"\vcpu_threads\x18\x11 \x01(\x05R\n" +
"cpuThreads\"\xfd\x02\n" +
"cpuThreads\"\xb0\x03\n" +
"\rEngineProfile\x12\x12\n" +
"\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" +
"\x06engine\x18\x02 \x01(\tR\x06engine\x12\x1b\n" +
@@ -4080,7 +4144,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\n" +
"model_path\x18\n" +
" \x01(\tR\tmodelPath\x12*\n" +
"\x11served_model_name\x18\v \x01(\tR\x0fservedModelName\x1a6\n" +
"\x11served_model_name\x18\v \x01(\tR\x0fservedModelName\x121\n" +
"\badapters\x18\f \x03(\v2\x15.llmgw.v1.LoraAdapterR\badapters\x1a6\n" +
"\bEnvEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"%\n" +
@@ -4143,7 +4208,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\n" +
"request_id\x18\x01 \x01(\tR\trequestId\x12\x18\n" +
"\aversion\x18\x02 \x01(\tR\aversion\x12\x14\n" +
"\x05error\x18\x03 \x01(\tR\x05error2O\n" +
"\x05error\x18\x03 \x01(\tR\x05error\"9\n" +
"\vLoraAdapter\x12\x12\n" +
"\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" +
"\x06source\x18\x02 \x01(\tR\x06source2O\n" +
"\fAgentService\x12?\n" +
"\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" +
"\fcom.llmgw.v1B\n" +
@@ -4162,7 +4230,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
}
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 42)
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 43)
var file_llmgw_v1_agent_proto_goTypes = []any{
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
@@ -4201,12 +4269,13 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
(*FabricTestReply)(nil), // 34: llmgw.v1.FabricTestReply
(*UpgradeAgent)(nil), // 35: llmgw.v1.UpgradeAgent
(*UpgradeReply)(nil), // 36: llmgw.v1.UpgradeReply
nil, // 37: llmgw.v1.Register.LabelsEntry
nil, // 38: llmgw.v1.Register.EngineVersionsEntry
nil, // 39: llmgw.v1.GPU.NicPathsEntry
nil, // 40: llmgw.v1.IBPort.CountersEntry
nil, // 41: llmgw.v1.Heartbeat.EngineVersionsEntry
nil, // 42: llmgw.v1.EngineProfile.EnvEntry
(*LoraAdapter)(nil), // 37: llmgw.v1.LoraAdapter
nil, // 38: llmgw.v1.Register.LabelsEntry
nil, // 39: llmgw.v1.Register.EngineVersionsEntry
nil, // 40: llmgw.v1.GPU.NicPathsEntry
nil, // 41: llmgw.v1.IBPort.CountersEntry
nil, // 42: llmgw.v1.Heartbeat.EngineVersionsEntry
nil, // 43: llmgw.v1.EngineProfile.EnvEntry
}
var file_llmgw_v1_agent_proto_depIdxs = []int32{
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
@@ -4226,24 +4295,24 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
31, // 14: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
33, // 15: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest
35, // 16: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent
37, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
38, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
7, // 18: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
8, // 19: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
38, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
39, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
6, // 21: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
9, // 22: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
5, // 23: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
4, // 24: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
15, // 25: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
0, // 26: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
39, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
40, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
12, // 28: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
11, // 29: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor
40, // 30: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
41, // 30: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
14, // 31: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
5, // 32: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
9, // 33: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
41, // 34: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
42, // 34: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
20, // 35: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
10, // 36: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
13, // 37: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
@@ -4254,14 +4323,15 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
0, // 42: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
22, // 43: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
24, // 44: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
42, // 45: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
1, // 46: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
2, // 47: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
47, // [47:48] is the sub-list for method output_type
46, // [46:47] is the sub-list for method input_type
46, // [46:46] is the sub-list for extension type_name
46, // [46:46] is the sub-list for extension extendee
0, // [0:46] is the sub-list for field type_name
43, // 45: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
37, // 46: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter
1, // 47: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
2, // 48: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
48, // [48:49] is the sub-list for method output_type
47, // [47:48] is the sub-list for method input_type
47, // [47:47] is the sub-list for extension type_name
47, // [47:47] is the sub-list for extension extendee
0, // [0:47] is the sub-list for field type_name
}
func init() { file_llmgw_v1_agent_proto_init() }
@@ -4296,7 +4366,7 @@ func file_llmgw_v1_agent_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
NumEnums: 1,
NumMessages: 42,
NumMessages: 43,
NumExtensions: 0,
NumServices: 1,
},
+10
View File
@@ -572,6 +572,9 @@ message EngineProfile {
map<string, string> env = 9;
string model_path = 10;
string served_model_name = 11;
// adapters are LoRA adapters loaded beside the model, in order: the index
// is the id a request applies one by.
repeated LoraAdapter adapters = 12;
}
// RemoveWeights deletes cached model files a node no longer needs.
@@ -704,3 +707,10 @@ message UpgradeReply {
string version = 2;
string error = 3;
}
// LoraAdapter is one adapter: the name clients ask for it by, and its source,
// which the agent fetches as it does the model's weights.
message LoraAdapter {
string name = 1;
string source = 2;
}
+28
View File
@@ -671,6 +671,34 @@ llmgw ctl explain-placement qwen2.5-coder-3b-instruct
which names every candidate and why each was rejected, rather than leaving you
to infer it.
### LoRA adapters
A llama.cpp model can carry LoRA adapters, each asked for by its own name as
though it were a model:
```yaml
- name: qwen2.5-coder-7b-instruct
source: http://mirror:8081/qwen2.5-coder-7b-instruct/5d567d26/model.gguf
adapters:
- name: coder-sql
source: http://mirror:8081/coder-sql/v1/adapter.gguf
scale: 1.0 # optional; 1 when left out
profiles: [...]
```
Every replica of the model fetches every adapter and loads it at start, applied
to nothing. A request for `coder-sql` goes to the model's replicas with the
adapter applied to it alone; a request for the model gets the model. Adapter
names share the namespace of models and aliases, appear in `/v1/models`, and
follow key allowlists on their own: a key allowed `coder-sql` reaches that
adapter and not the model under it, while a key allowed the model reaches its
adapters too. Adding or removing an adapter is a change to the model; running
replicas load the new set when restarted (`llmgw ctl restart --model M`).
The adapter file must be a GGUF LoRA made for this model — llama.cpp's
`convert_lora_to_gguf.py` makes one from a PEFT adapter — and not a model with
the adapter already merged in, which is what most "LoRA GGUF" repositories hold.
### Models served outside the cluster
A model can be an OpenAI-compatible API elsewhere — a hosted provider, or
+33
View File
@@ -15,6 +15,9 @@ rather than repeated.
| Key | Type | In |
|---|---|---|
| `name` | `string` | [Adapter](#adapter) |
| `source` | `string` | [Adapter](#adapter) |
| `scale` | `float64` | [Adapter](#adapter) |
| `enabled` | `bool` | [Admission](#admission) |
| `bands` | `[]Band` | [Admission](#admission) |
| `default_band` | `string` | [Admission](#admission) |
@@ -99,6 +102,7 @@ rather than repeated.
| `class` | `string` | [Model](#model) |
| `owner` | `string` | [Model](#model) |
| `external` | `*External` | [Model](#model) |
| `adapters` | `[]Adapter` | [Model](#model) |
| `fallbacks` | `[]string` | [Model](#model) |
| `routing_profile` | `string` | [Model](#model) |
| `slo` | `SLO` | [Model](#model) |
@@ -200,6 +204,27 @@ rather than repeated.
| `launch_mode` | `string` | [VLLMConfig](#vllmconfig) |
| `ray_bin` | `string` | [VLLMConfig](#vllmconfig) |
## Adapter
### `name`
`string`
Name is what clients ask for, like a model name.
### `source`
`string`
Source is the adapter file (GGUF for llama.cpp), fetched by each node
as the model's weights are.
### `scale`
`float64`
Scale is how strongly it is applied; zero takes 1.
## Admission
### `enabled`
@@ -805,6 +830,14 @@ allowlist.
Takes a [External](#external) block.
### `adapters`
`[]Adapter`
Adapters are LoRA adapters served on this model's replicas, each asked
for by its own name as though it were a model. Every replica loads
every adapter, applying one only to a request that names it.
### `fallbacks`
`[]string`
+11
View File
@@ -891,6 +891,17 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
stop()
spec.Profile.ModelPath = path
}
// Adapters are fetched like the model, and before the engine starts: it
// loads them at start, and one missing is a replica that answers for the
// base model under the adapter's name.
for i, ad := range spec.Profile.Adapters {
path, err := a.weights().Ensure(ctx, ad.Path)
if err != nil {
report(supervisor.Failed, fmt.Sprintf("adapter %s: %v", ad.Name, err), nil)
return
}
spec.Profile.Adapters[i].Path = path
}
cmds, err := eng.Commands(spec)
if err != nil {
+10
View File
@@ -40,6 +40,9 @@ type Profile struct {
ModelPath string
// ServedModelName is the name clients use, which need not match the path.
ServedModelName string
// Adapters are LoRA adapters to load beside the model, in order: an
// adapter's index is the id requests apply it by.
Adapters []Adapter
}
// ShardSpec is one shard of one replica on this node.
@@ -373,3 +376,10 @@ func quantizationOf(args []string) string {
}
return ""
}
// Adapter is a LoRA adapter an engine loads beside its model. Path is a
// source until the agent has fetched it, a local file after.
type Adapter struct {
Name string `json:"name"`
Path string `json:"path"`
}
+9
View File
@@ -144,6 +144,15 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
}
args = append(args, "--threads", strconv.Itoa(n))
}
// Loaded at start, applied per request: a request that names an adapter
// carries "lora": [{"id": i, "scale": s}], and one that names none is the
// base model, since every adapter starts at scale zero.
if len(p.Adapters) > 0 {
args = append(args, "--lora-init-without-apply")
for _, a := range p.Adapters {
args = append(args, "--lora", a.Path)
}
}
args = append(args, p.Args...)
env := baseEnv(s)
+17
View File
@@ -306,3 +306,20 @@ func TestTheCoresPlacementReservedAreTheThreads(t *testing.T) {
t.Errorf("--threads %q; what placement reserved should win over the node's setting", v)
}
}
func TestAdaptersAreLoadedAtStartAndAppliedToNoneByDefault(t *testing.T) {
s := llamaSpec(nil)
s.Profile.Adapters = []Adapter{{Name: "sql", Path: "/w/sql.gguf"}, {Name: "docs", Path: "/w/docs.gguf"}}
cmds, err := (&LlamaCpp{}).Commands(s)
if err != nil {
t.Fatal(err)
}
args := strings.Join(cmds[0].Args, " ")
if !strings.Contains(args, "--lora-init-without-apply --lora /w/sql.gguf --lora /w/docs.gguf") {
t.Errorf("args %s: both adapters, in order, loaded unapplied", args)
}
plain, _ := (&LlamaCpp{}).Commands(llamaSpec(nil))
if strings.Contains(strings.Join(plain[0].Args, " "), "--lora") {
t.Error("a model with no adapters was given lora flags")
}
}
+5
View File
@@ -31,6 +31,11 @@ func (a *API) Orphans() []OrphanView {
var wanted []string
for _, m := range a.ctl.Catalog().Models() {
wanted = append(wanted, m.Source)
// A model's adapters are its weights too; collecting them would
// leave replicas that cannot restart.
for _, ad := range m.Adapters {
wanted = append(wanted, ad.Source)
}
}
out := make([]OrphanView, 0)
+18
View File
@@ -189,6 +189,7 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard {
GpusPerNode: int32(p.GPUsPerNode), Nodes: int32(p.Nodes),
Args: p.Args, Env: p.Env,
ModelPath: p.ModelPath, ServedModelName: p.ServedModelName,
Adapters: adaptersToWire(p.Adapters),
},
StartupTimeoutMs: cmd.StartupTimeout.Milliseconds(),
Pool: cmd.Pool,
@@ -224,6 +225,7 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration
GPUsPerNode: int(p.GetGpusPerNode()), Nodes: int(p.GetNodes()),
Args: p.GetArgs(), Env: p.GetEnv(),
ModelPath: p.GetModelPath(), ServedModelName: p.GetServedModelName(),
Adapters: adaptersFromWire(p.GetAdapters()),
},
}
return spec, time.Duration(m.GetStartupTimeoutMs()) * time.Millisecond
@@ -292,3 +294,19 @@ func fabricFrom(state, status, cluster, clique, bandwidth string) *discovery.GPU
}
return &discovery.GPUFabric{State: state, Status: status, ClusterUUID: cluster, CliqueID: clique, Bandwidth: bandwidth}
}
func adaptersToWire(as []engines.Adapter) []*llmgwv1.LoraAdapter {
var out []*llmgwv1.LoraAdapter
for _, a := range as {
out = append(out, &llmgwv1.LoraAdapter{Name: a.Name, Source: a.Path})
}
return out
}
func adaptersFromWire(as []*llmgwv1.LoraAdapter) []engines.Adapter {
var out []engines.Adapter
for _, a := range as {
out = append(out, engines.Adapter{Name: a.GetName(), Path: a.GetSource()})
}
return out
}
+20
View File
@@ -0,0 +1,20 @@
package agentsvc
import (
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
)
func TestAdaptersTravelToTheAgentInOrder(t *testing.T) {
cmd := reconciler.ShardCommand{ReplicaID: "r", ShardID: "s", Spec: engines.ShardSpec{
Profile: engines.Profile{Name: "cpu", Engine: "llamacpp", ModelPath: "/m.gguf",
Adapters: []engines.Adapter{{Name: "sql", Path: "http://mirror/sql.gguf"}, {Name: "docs", Path: "http://mirror/docs.gguf"}}},
}}
spec, _ := ShardSpecFromStart(StartShardFromCommand(cmd))
got := spec.Profile.Adapters
if len(got) != 2 || got[0].Name != "sql" || got[1].Path != "http://mirror/docs.gguf" {
t.Errorf("adapters %+v", got)
}
}
+102 -4
View File
@@ -169,7 +169,11 @@ type Model struct {
// fallbacks: used only when the cluster has no capacity left. Its
// prompts leave the cluster, so which keys may name it is worth an
// allowlist.
External *External `json:"external,omitempty" koanf:"external"`
External *External `json:"external,omitempty" koanf:"external"`
// Adapters are LoRA adapters served on this model's replicas, each asked
// for by its own name as though it were a model. Every replica loads
// every adapter, applying one only to a request that names it.
Adapters []Adapter `json:"adapters,omitempty" koanf:"adapters"`
Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"`
// RoutingProfile selects how the router scores endpoints for this model.
RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"`
@@ -213,14 +217,17 @@ type Capabilities struct {
type Catalog struct {
models map[string]Model
aliases map[string]string
order []string
// adapters maps an adapter's name to the model it runs on.
adapters map[string]string
order []string
}
// New builds a catalog, rejecting definitions that would fail confusingly later.
func New(models []Model) (*Catalog, error) {
c := &Catalog{
models: make(map[string]Model, len(models)),
aliases: map[string]string{},
models: make(map[string]Model, len(models)),
aliases: map[string]string{},
adapters: map[string]string{},
}
for i := range models {
@@ -249,6 +256,18 @@ func New(models []Model) (*Catalog, error) {
}
}
// Adapters are named like models, so they share the namespace too.
for _, name := range c.order {
for _, a := range c.models[name].Adapters {
_, isModel := c.models[a.Name]
_, isAlias := c.aliases[a.Name]
if _, isAdapter := c.adapters[a.Name]; isModel || isAlias || isAdapter {
return nil, fmt.Errorf("model %q: adapter %q collides with a model, an alias or another adapter", name, a.Name)
}
c.adapters[a.Name] = name
}
}
// Fallbacks are resolved now rather than at request time: a typo in a
// fallback should stop a deployment, not surface during an outage when the
// fallback is finally needed.
@@ -314,6 +333,9 @@ func (m Model) validate() error {
if m.Source == "" {
return fmt.Errorf("model %q: source is required", m.Name)
}
if err := validateAdapters(m); err != nil {
return err
}
if m.External != nil {
if err := m.External.validate(m.Name, len(m.Profiles)); err != nil {
return err
@@ -370,6 +392,23 @@ func (m Model) validate() error {
}
// Resolve maps a client-supplied name or alias onto a model.
// ResolveAdapter finds an adapter by its name: the model it runs on, the
// adapter, and its index among the model's adapters — which is the id the
// engine knows it by.
func (c *Catalog) ResolveAdapter(name string) (Model, Adapter, int, bool) {
base, ok := c.adapters[name]
if !ok {
return Model{}, Adapter{}, 0, false
}
m := c.models[base]
for i, a := range m.Adapters {
if a.Name == name {
return m, a, i, true
}
}
return Model{}, Adapter{}, 0, false
}
func (c *Catalog) Resolve(name string) (Model, bool) {
if m, ok := c.models[name]; ok {
return m, true
@@ -446,6 +485,7 @@ func (m Model) EngineProfile(name string) (engines.Profile, bool) {
Env: p.Env,
ModelPath: m.Source,
ServedModelName: m.Name,
Adapters: m.engineAdapters(),
}, true
}
return engines.Profile{}, false
@@ -606,3 +646,61 @@ func (e *External) validate(model string, profiles int) error {
// IsExternal reports whether a model is served outside the cluster.
func (m Model) IsExternal() bool { return m.External != nil }
// Adapter is a LoRA adapter served on a model's replicas.
type Adapter struct {
// Name is what clients ask for, like a model name.
Name string `json:"name" koanf:"name"`
// Source is the adapter file (GGUF for llama.cpp), fetched by each node
// as the model's weights are.
Source string `json:"source" koanf:"source"`
// Scale is how strongly it is applied; zero takes 1.
Scale float64 `json:"scale,omitempty" koanf:"scale"`
}
// EffectiveScale is the scale an adapter is applied at.
func (a Adapter) EffectiveScale() float64 {
if a.Scale <= 0 {
return 1
}
return a.Scale
}
func validateAdapters(m Model) error {
if len(m.Adapters) == 0 {
return nil
}
if m.External != nil {
return fmt.Errorf("model %q: an external model cannot carry adapters", m.Name)
}
for _, p := range m.Profiles {
// llama.cpp loads adapters at start and applies them per request;
// the other engines are not driven that way yet.
if p.Engine != "llamacpp" && p.Engine != "sim" {
return fmt.Errorf("model %q profile %q: adapters are served by llamacpp profiles only", m.Name, p.Name)
}
}
seen := map[string]bool{}
for _, a := range m.Adapters {
switch {
case a.Name == "" || a.Source == "":
return fmt.Errorf("model %q: an adapter needs a name and a source", m.Name)
case a.Name == m.Name:
return fmt.Errorf("model %q: an adapter cannot have the model's own name", m.Name)
case seen[a.Name]:
return fmt.Errorf("model %q: adapter %q is declared twice", m.Name, a.Name)
case a.Scale < 0:
return fmt.Errorf("model %q: adapter %q has a negative scale", m.Name, a.Name)
}
seen[a.Name] = true
}
return nil
}
func (m Model) engineAdapters() []engines.Adapter {
var out []engines.Adapter
for _, a := range m.Adapters {
out = append(out, engines.Adapter{Name: a.Name, Path: a.Source})
}
return out
}
+3
View File
@@ -87,6 +87,9 @@ func (s *Service) Sources(ctx context.Context) (map[string][]catalog.Model, erro
if m.Source != "" {
out[m.Source] = append(out[m.Source], m)
}
for _, ad := range m.Adapters {
out[ad.Source] = append(out[ad.Source], m)
}
}
return out, nil
}
+98
View File
@@ -0,0 +1,98 @@
package router
import (
"encoding/json"
"net/http"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func adapterModel() catalog.Model {
m := catModel("coder", nil)
m.Profiles = []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 4}}
m.Adapters = []catalog.Adapter{
{Name: "coder-sql", Source: "http://mirror/sql.gguf"},
{Name: "coder-docs", Source: "http://mirror/docs.gguf", Scale: 0.5},
}
return m
}
func TestAnAdapterIsAskedForByNameAndAppliedOnItsModel(t *testing.T) {
e := newFakeEngine(t, "coder")
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"coder"}})
cfg.Models = []catalog.Model{adapterModel()}
rt := newTestRouter(t, cfg)
sent := func(model string) map[string]any {
t.Helper()
rec := post(t, rt.Handler(), map[string]any{"model": model, "messages": []any{}}, nil)
if rec.Code != http.StatusOK {
t.Fatalf("%s: status %d: %s", model, rec.Code, rec.Body.String())
}
var body map[string]any
_ = json.Unmarshal(e.last.Load().([]byte), &body)
return body
}
body := sent("coder-docs")
if body["model"] != "coder" {
t.Errorf("the engine was asked for %v, not the model the adapter runs on", body["model"])
}
lora, _ := json.Marshal(body["lora"])
if string(lora) != `[{"id":1,"scale":0.5}]` {
t.Errorf("lora %s, expected the second adapter at its scale", lora)
}
if body := sent("coder-sql"); func() string { b, _ := json.Marshal(body["lora"]); return string(b) }() != `[{"id":0,"scale":1}]` {
t.Errorf("the first adapter, at the default scale: %v", body["lora"])
}
if body := sent("coder"); body["lora"] != nil {
t.Errorf("the base model was sent an adapter: %v", body["lora"])
}
}
func TestAKeyAllowedAnAdapterReachesThatAdapterOnly(t *testing.T) {
cfg := baseConfig()
cfg.Models = []catalog.Model{adapterModel()}
rt := newTestRouter(t, cfg)
key := func(allowed ...string) auth.Principal {
return auth.Principal{Limits: store.Effective{AllowedModels: allowed}}
}
for _, c := range []struct {
allowed []string
requested string
want bool
}{
{[]string{"coder-sql"}, "coder-sql", true},
{[]string{"coder-sql"}, "coder", false},
{[]string{"coder-sql"}, "coder-docs", false},
{[]string{"coder"}, "coder-sql", true}, // the model allows its adapters
{[]string{"coder"}, "coder", true},
} {
if got := rt.permits(key(c.allowed...), c.requested); got != c.want {
t.Errorf("allowed %v, asked for %s: %v", c.allowed, c.requested, got)
}
}
}
func TestAdaptersShareTheNamespaceOfModels(t *testing.T) {
a := adapterModel()
b := catModel("coder-sql", nil)
if _, err := catalog.New([]catalog.Model{a, b}); err == nil {
t.Error("an adapter named like another model was accepted")
}
bad := adapterModel()
bad.Profiles = []catalog.Profile{{Name: "gpu", Engine: "vllm", GPUClass: "h100-80g", TP: 1}}
if _, err := catalog.New([]catalog.Model{bad}); err == nil {
t.Error("adapters on a vLLM profile were accepted")
}
c, err := catalog.New([]catalog.Model{adapterModel()})
if err != nil {
t.Fatal(err)
}
if m, ad, i, ok := c.ResolveAdapter("coder-docs"); !ok || m.Name != "coder" || i != 1 || ad.EffectiveScale() != 0.5 {
t.Errorf("resolve: %v %+v %d %v", m.Name, ad, i, ok)
}
}
+27
View File
@@ -426,6 +426,12 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
// around the allowlist; matching only the canonical name would silently ignore
// an allowlist written in the names clients actually use.
func (r *Router) permits(p auth.Principal, requested string) bool {
// An adapter named in the allowlist allows that adapter: not its model,
// and not the model's other adapters, which the checks below would let
// through if the adapter's own name were not asked about first.
if _, ok := r.adapter(requested); ok && p.Allows(requested) {
return true
}
canonical, m := r.resolve(requested)
if p.Allows(canonical) {
return true
@@ -1004,3 +1010,24 @@ func withModel(body []byte, name string) []byte {
}
return out
}
// withAdapter is body asking for model with a LoRA adapter applied, in
// llama.cpp's form: "lora": [{"id": index, "scale": scale}]. Adapters not
// listed are applied at zero, so the list is the whole of it.
func withAdapter(body []byte, model string, index int, scale float64) []byte {
var m map[string]json.RawMessage
if err := json.Unmarshal(body, &m); err != nil {
return body
}
name, err1 := json.Marshal(model)
lora, err2 := json.Marshal([]map[string]any{{"id": index, "scale": scale}})
if err1 != nil || err2 != nil {
return body
}
m["model"], m["lora"] = name, lora
out, err := json.Marshal(m)
if err != nil {
return body
}
return out
}
+40 -10
View File
@@ -61,6 +61,8 @@ type Router struct {
// router's half of not having to. A request takes the map once and works
// from it, so a swap mid-request is invisible rather than a race.
models atomic.Pointer[map[string]catalog.Model]
// adapters are the LoRA adapters by the name clients ask for them by.
adapters atomic.Pointer[map[string]adapterRef]
// extraMetrics are collectors other components publish on /metrics.
extraMetrics []prometheus.Collector
@@ -315,14 +317,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) {
// all. A control plane pushes over the top of this at startup and on every
// change after.
if len(cfg.Models) > 0 {
seed := map[string]catalog.Model{}
for _, m := range cfg.Models {
seed[m.Name] = m
for _, a := range m.Aliases {
seed[a] = m
}
}
r.models.Store(&seed)
r.storeModels(cfg.Models)
}
return r, nil
}
@@ -505,15 +500,44 @@ func (r *Router) modelMap() map[string]catalog.Model {
// The catalog's counterpart to SetEndpoints, and pushed for the same reason:
// the control plane owns what should exist, the router needs to know, and
// neither has to be in the other's process for that to work.
func (r *Router) SetCatalog(c *catalog.Catalog) {
func (r *Router) SetCatalog(c *catalog.Catalog) { r.storeModels(c.Models()) }
// storeModels makes models what names resolve to: each model by its name, its
// aliases and its adapters. One place for it, whether the models came from
// configuration at start or from the control plane since.
func (r *Router) storeModels(models []catalog.Model) {
m := map[string]catalog.Model{}
for _, entry := range c.Models() {
ads := map[string]adapterRef{}
for _, entry := range models {
m[entry.Name] = entry
for _, a := range entry.Aliases {
m[a] = entry
}
// An adapter resolves to the model it runs on, as an alias does, and
// is applied to the request on the way there.
for i, a := range entry.Adapters {
m[a.Name] = entry
ads[a.Name] = adapterRef{base: entry.Name, index: i, scale: a.EffectiveScale()}
}
}
r.models.Store(&m)
r.adapters.Store(&ads)
}
// adapterRef is where an adapter runs and how it is applied.
type adapterRef struct {
base string
index int
scale float64
}
// adapter finds one by the name a client asked for.
func (r *Router) adapter(name string) (adapterRef, bool) {
if p := r.adapters.Load(); p != nil {
a, ok := (*p)[name]
return a, ok
}
return adapterRef{}, false
}
// LastUsed reports when each endpoint was last sent a request, satisfying
@@ -719,6 +743,12 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
if dec.Endpoint.UpstreamModel != "" {
body = withModel(body, dec.Endpoint.UpstreamModel)
}
// An adapter asked for by name, served by its model's replicas: the
// engine is asked for the model, with the adapter applied. Not on a
// fallback to another model, which does not have it.
if a, ok := r.adapter(req.Model); ok && a.base == model {
body = withAdapter(body, model, a.index, a.scale)
}
out, perr := r.px.Do(uctx, w, proxy.Request{
URL: dec.Endpoint.URL + req.Path,
APIKey: dec.Endpoint.APIKey,
+3 -1
View File
@@ -1497,7 +1497,9 @@ The launch path is now built too:
- [x] Weight downloading and cache management: the mirror, agent fetches
checked against their digest, and reclaiming what nothing uses
- [ ] Prefill/decode disaggregation (§12)
- [ ] LoRA adapter routing (affinity to a replica that already has the adapter)
- [x] LoRA adapters on llama.cpp: every replica loads a model's adapters and
applies one per request, so any replica serves any adapter
- [ ] LoRA affinity for engines that load adapters on demand (vLLM)
- [x] External provider support: a catalog model served by an OpenAI-compatible
API elsewhere, usable as a fallback, under the same allowlists and budgets
- [x] Read-only dashboard on the admin listener (nodes, replicas, models,