From cf53904cc52431da6dfcc003b4db10577b1f618c Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 30 Sep 2026 23:40:47 +0200 Subject: [PATCH] Serve LoRA adapters on llama.cpp models, asked for by name A model's adapters are fetched by each node with its weights and loaded unapplied at start; a request naming an adapter reaches the model's replicas with llama.cpp's per-request lora field. Adapter names share the model namespace, are listed, and follow allowlists on their own. The router's config seed and pushed catalog now build the name map in one place. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 12 +++ api/gen/llmgw/v1/agent.pb.go | 122 +++++++++++++++++----- api/proto/llmgw/v1/agent.proto | 10 ++ docs/install.md | 28 +++++ docs/reference.md | 33 ++++++ internal/agent/agent.go | 11 ++ internal/agent/engines/engine.go | 10 ++ internal/agent/engines/llamacpp.go | 9 ++ internal/agent/engines/llamacpp_test.go | 17 +++ internal/control/adminapi/weights.go | 5 + internal/control/agentsvc/convert.go | 18 ++++ internal/control/agentsvc/convert_test.go | 20 ++++ internal/control/catalog/catalog.go | 106 ++++++++++++++++++- internal/control/modelsvc/modelsvc.go | 3 + internal/router/adapter_test.go | 98 +++++++++++++++++ internal/router/handlers.go | 27 +++++ internal/router/router.go | 50 +++++++-- llmgw-plan.md | 4 +- 18 files changed, 542 insertions(+), 41 deletions(-) create mode 100644 internal/control/agentsvc/convert_test.go create mode 100644 internal/router/adapter_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index fa21d92..341c596 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,18 @@ What v0.10.5 made settable from the command line is now on the pages too: critical model's card no longer refuses a replica that preemption would make room for. +### LoRA adapters + +A llama.cpp model can carry `adapters`, each a name and a GGUF LoRA file. The +nodes fetch them with the model's weights and llama.cpp loads them all at start, +applied to nothing (`--lora-init-without-apply`); a request that names an +adapter goes to the model's replicas with `"lora": [{"id": i, "scale": s}]`, +so one replica serves the model and all its adapters. Adapter names share the +namespace of models, are listed in `/v1/models`, and follow key allowlists on +their own — a key allowed an adapter does not reach the model under it. Adapter +files are kept from weight cleanup like the model's own. Not yet for vLLM or +SGLang profiles, whose adapters are loaded differently. + ### Models served outside the cluster A catalog model can be an OpenAI-compatible API elsewhere: `external: {url, diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index d2c353e..04ad5c0 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -2873,8 +2873,11 @@ type EngineProfile struct { Env map[string]string `protobuf:"bytes,9,rep,name=env,proto3" json:"env,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` ModelPath string `protobuf:"bytes,10,opt,name=model_path,json=modelPath,proto3" json:"model_path,omitempty"` ServedModelName string `protobuf:"bytes,11,opt,name=served_model_name,json=servedModelName,proto3" json:"served_model_name,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // adapters are LoRA adapters loaded beside the model, in order: the index + // is the id a request applies one by. + Adapters []*LoraAdapter `protobuf:"bytes,12,rep,name=adapters,proto3" json:"adapters,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *EngineProfile) Reset() { @@ -2984,6 +2987,13 @@ func (x *EngineProfile) GetServedModelName() string { return "" } +func (x *EngineProfile) GetAdapters() []*LoraAdapter { + if x != nil { + return x.Adapters + } + return nil +} + // RemoveWeights deletes cached model files a node no longer needs. // // The paths are computed by the control plane from the catalog, but the agent @@ -3777,6 +3787,60 @@ func (x *UpgradeReply) GetError() string { return "" } +// LoraAdapter is one adapter: the name clients ask for it by, and its source, +// which the agent fetches as it does the model's weights. +type LoraAdapter struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Source string `protobuf:"bytes,2,opt,name=source,proto3" json:"source,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LoraAdapter) Reset() { + *x = LoraAdapter{} + mi := &file_llmgw_v1_agent_proto_msgTypes[36] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LoraAdapter) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoraAdapter) ProtoMessage() {} + +func (x *LoraAdapter) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[36] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoraAdapter.ProtoReflect.Descriptor instead. +func (*LoraAdapter) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{36} +} + +func (x *LoraAdapter) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *LoraAdapter) GetSource() string { + if x != nil { + return x.Source + } + return "" +} + var File_llmgw_v1_agent_proto protoreflect.FileDescriptor const file_llmgw_v1_agent_proto_rawDesc = "" + @@ -4066,7 +4130,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "profile_fp\x18\x0f \x01(\tR\tprofileFp\x12\x17\n" + "\aapi_key\x18\x10 \x01(\tR\x06apiKey\x12\x1f\n" + "\vcpu_threads\x18\x11 \x01(\x05R\n" + - "cpuThreads\"\xfd\x02\n" + + "cpuThreads\"\xb0\x03\n" + "\rEngineProfile\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" + "\x06engine\x18\x02 \x01(\tR\x06engine\x12\x1b\n" + @@ -4080,7 +4144,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\n" + "model_path\x18\n" + " \x01(\tR\tmodelPath\x12*\n" + - "\x11served_model_name\x18\v \x01(\tR\x0fservedModelName\x1a6\n" + + "\x11served_model_name\x18\v \x01(\tR\x0fservedModelName\x121\n" + + "\badapters\x18\f \x03(\v2\x15.llmgw.v1.LoraAdapterR\badapters\x1a6\n" + "\bEnvEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"%\n" + @@ -4143,7 +4208,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\n" + "request_id\x18\x01 \x01(\tR\trequestId\x12\x18\n" + "\aversion\x18\x02 \x01(\tR\aversion\x12\x14\n" + - "\x05error\x18\x03 \x01(\tR\x05error2O\n" + + "\x05error\x18\x03 \x01(\tR\x05error\"9\n" + + "\vLoraAdapter\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" + + "\x06source\x18\x02 \x01(\tR\x06source2O\n" + "\fAgentService\x12?\n" + "\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" + "\fcom.llmgw.v1B\n" + @@ -4162,7 +4230,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 42) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 43) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage @@ -4201,12 +4269,13 @@ var file_llmgw_v1_agent_proto_goTypes = []any{ (*FabricTestReply)(nil), // 34: llmgw.v1.FabricTestReply (*UpgradeAgent)(nil), // 35: llmgw.v1.UpgradeAgent (*UpgradeReply)(nil), // 36: llmgw.v1.UpgradeReply - nil, // 37: llmgw.v1.Register.LabelsEntry - nil, // 38: llmgw.v1.Register.EngineVersionsEntry - nil, // 39: llmgw.v1.GPU.NicPathsEntry - nil, // 40: llmgw.v1.IBPort.CountersEntry - nil, // 41: llmgw.v1.Heartbeat.EngineVersionsEntry - nil, // 42: llmgw.v1.EngineProfile.EnvEntry + (*LoraAdapter)(nil), // 37: llmgw.v1.LoraAdapter + nil, // 38: llmgw.v1.Register.LabelsEntry + nil, // 39: llmgw.v1.Register.EngineVersionsEntry + nil, // 40: llmgw.v1.GPU.NicPathsEntry + nil, // 41: llmgw.v1.IBPort.CountersEntry + nil, // 42: llmgw.v1.Heartbeat.EngineVersionsEntry + nil, // 43: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register @@ -4226,24 +4295,24 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{ 31, // 14: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest 33, // 15: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest 35, // 16: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent - 37, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 38, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry 7, // 18: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU 8, // 19: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 38, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 39, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry 6, // 21: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard 9, // 22: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights 5, // 23: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage 4, // 24: llmgw.v1.Register.host:type_name -> llmgw.v1.Host 15, // 25: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX 0, // 26: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State - 39, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry + 40, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry 12, // 28: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad 11, // 29: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor - 40, // 30: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry + 41, // 30: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry 14, // 31: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager 5, // 32: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage 9, // 33: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights - 41, // 34: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry + 42, // 34: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry 20, // 35: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus 10, // 36: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad 13, // 37: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort @@ -4254,14 +4323,15 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{ 0, // 42: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State 22, // 43: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress 24, // 44: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 42, // 45: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 1, // 46: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 47: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 47, // [47:48] is the sub-list for method output_type - 46, // [46:47] is the sub-list for method input_type - 46, // [46:46] is the sub-list for extension type_name - 46, // [46:46] is the sub-list for extension extendee - 0, // [0:46] is the sub-list for field type_name + 43, // 45: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 37, // 46: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter + 1, // 47: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 48: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 48, // [48:49] is the sub-list for method output_type + 47, // [47:48] is the sub-list for method input_type + 47, // [47:47] is the sub-list for extension type_name + 47, // [47:47] is the sub-list for extension extendee + 0, // [0:47] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -4296,7 +4366,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 42, + NumMessages: 43, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 3283981..45818fb 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -572,6 +572,9 @@ message EngineProfile { map env = 9; string model_path = 10; string served_model_name = 11; + // adapters are LoRA adapters loaded beside the model, in order: the index + // is the id a request applies one by. + repeated LoraAdapter adapters = 12; } // RemoveWeights deletes cached model files a node no longer needs. @@ -704,3 +707,10 @@ message UpgradeReply { string version = 2; string error = 3; } + +// LoraAdapter is one adapter: the name clients ask for it by, and its source, +// which the agent fetches as it does the model's weights. +message LoraAdapter { + string name = 1; + string source = 2; +} diff --git a/docs/install.md b/docs/install.md index b57be64..0989b0a 100644 --- a/docs/install.md +++ b/docs/install.md @@ -671,6 +671,34 @@ llmgw ctl explain-placement qwen2.5-coder-3b-instruct which names every candidate and why each was rejected, rather than leaving you to infer it. +### LoRA adapters + +A llama.cpp model can carry LoRA adapters, each asked for by its own name as +though it were a model: + +```yaml +- name: qwen2.5-coder-7b-instruct + source: http://mirror:8081/qwen2.5-coder-7b-instruct/5d567d26/model.gguf + adapters: + - name: coder-sql + source: http://mirror:8081/coder-sql/v1/adapter.gguf + scale: 1.0 # optional; 1 when left out + profiles: [...] +``` + +Every replica of the model fetches every adapter and loads it at start, applied +to nothing. A request for `coder-sql` goes to the model's replicas with the +adapter applied to it alone; a request for the model gets the model. Adapter +names share the namespace of models and aliases, appear in `/v1/models`, and +follow key allowlists on their own: a key allowed `coder-sql` reaches that +adapter and not the model under it, while a key allowed the model reaches its +adapters too. Adding or removing an adapter is a change to the model; running +replicas load the new set when restarted (`llmgw ctl restart --model M`). + +The adapter file must be a GGUF LoRA made for this model — llama.cpp's +`convert_lora_to_gguf.py` makes one from a PEFT adapter — and not a model with +the adapter already merged in, which is what most "LoRA GGUF" repositories hold. + ### Models served outside the cluster A model can be an OpenAI-compatible API elsewhere — a hosted provider, or diff --git a/docs/reference.md b/docs/reference.md index 1c9b5a0..225ec8b 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -15,6 +15,9 @@ rather than repeated. | Key | Type | In | |---|---|---| +| `name` | `string` | [Adapter](#adapter) | +| `source` | `string` | [Adapter](#adapter) | +| `scale` | `float64` | [Adapter](#adapter) | | `enabled` | `bool` | [Admission](#admission) | | `bands` | `[]Band` | [Admission](#admission) | | `default_band` | `string` | [Admission](#admission) | @@ -99,6 +102,7 @@ rather than repeated. | `class` | `string` | [Model](#model) | | `owner` | `string` | [Model](#model) | | `external` | `*External` | [Model](#model) | +| `adapters` | `[]Adapter` | [Model](#model) | | `fallbacks` | `[]string` | [Model](#model) | | `routing_profile` | `string` | [Model](#model) | | `slo` | `SLO` | [Model](#model) | @@ -200,6 +204,27 @@ rather than repeated. | `launch_mode` | `string` | [VLLMConfig](#vllmconfig) | | `ray_bin` | `string` | [VLLMConfig](#vllmconfig) | +## Adapter + +### `name` + +`string` + +Name is what clients ask for, like a model name. + +### `source` + +`string` + +Source is the adapter file (GGUF for llama.cpp), fetched by each node +as the model's weights are. + +### `scale` + +`float64` + +Scale is how strongly it is applied; zero takes 1. + ## Admission ### `enabled` @@ -805,6 +830,14 @@ allowlist. Takes a [External](#external) block. +### `adapters` + +`[]Adapter` + +Adapters are LoRA adapters served on this model's replicas, each asked +for by its own name as though it were a model. Every replica loads +every adapter, applying one only to a request that names it. + ### `fallbacks` `[]string` diff --git a/internal/agent/agent.go b/internal/agent/agent.go index e5d6993..469e437 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -891,6 +891,17 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan< stop() spec.Profile.ModelPath = path } + // Adapters are fetched like the model, and before the engine starts: it + // loads them at start, and one missing is a replica that answers for the + // base model under the adapter's name. + for i, ad := range spec.Profile.Adapters { + path, err := a.weights().Ensure(ctx, ad.Path) + if err != nil { + report(supervisor.Failed, fmt.Sprintf("adapter %s: %v", ad.Name, err), nil) + return + } + spec.Profile.Adapters[i].Path = path + } cmds, err := eng.Commands(spec) if err != nil { diff --git a/internal/agent/engines/engine.go b/internal/agent/engines/engine.go index 52569bf..831d7f9 100644 --- a/internal/agent/engines/engine.go +++ b/internal/agent/engines/engine.go @@ -40,6 +40,9 @@ type Profile struct { ModelPath string // ServedModelName is the name clients use, which need not match the path. ServedModelName string + // Adapters are LoRA adapters to load beside the model, in order: an + // adapter's index is the id requests apply it by. + Adapters []Adapter } // ShardSpec is one shard of one replica on this node. @@ -373,3 +376,10 @@ func quantizationOf(args []string) string { } return "" } + +// Adapter is a LoRA adapter an engine loads beside its model. Path is a +// source until the agent has fetched it, a local file after. +type Adapter struct { + Name string `json:"name"` + Path string `json:"path"` +} diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go index 4de8060..eb14d63 100644 --- a/internal/agent/engines/llamacpp.go +++ b/internal/agent/engines/llamacpp.go @@ -144,6 +144,15 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { } args = append(args, "--threads", strconv.Itoa(n)) } + // Loaded at start, applied per request: a request that names an adapter + // carries "lora": [{"id": i, "scale": s}], and one that names none is the + // base model, since every adapter starts at scale zero. + if len(p.Adapters) > 0 { + args = append(args, "--lora-init-without-apply") + for _, a := range p.Adapters { + args = append(args, "--lora", a.Path) + } + } args = append(args, p.Args...) env := baseEnv(s) diff --git a/internal/agent/engines/llamacpp_test.go b/internal/agent/engines/llamacpp_test.go index 3cafd35..c3133c3 100644 --- a/internal/agent/engines/llamacpp_test.go +++ b/internal/agent/engines/llamacpp_test.go @@ -306,3 +306,20 @@ func TestTheCoresPlacementReservedAreTheThreads(t *testing.T) { t.Errorf("--threads %q; what placement reserved should win over the node's setting", v) } } + +func TestAdaptersAreLoadedAtStartAndAppliedToNoneByDefault(t *testing.T) { + s := llamaSpec(nil) + s.Profile.Adapters = []Adapter{{Name: "sql", Path: "/w/sql.gguf"}, {Name: "docs", Path: "/w/docs.gguf"}} + cmds, err := (&LlamaCpp{}).Commands(s) + if err != nil { + t.Fatal(err) + } + args := strings.Join(cmds[0].Args, " ") + if !strings.Contains(args, "--lora-init-without-apply --lora /w/sql.gguf --lora /w/docs.gguf") { + t.Errorf("args %s: both adapters, in order, loaded unapplied", args) + } + plain, _ := (&LlamaCpp{}).Commands(llamaSpec(nil)) + if strings.Contains(strings.Join(plain[0].Args, " "), "--lora") { + t.Error("a model with no adapters was given lora flags") + } +} diff --git a/internal/control/adminapi/weights.go b/internal/control/adminapi/weights.go index 2f7927b..ac8e904 100644 --- a/internal/control/adminapi/weights.go +++ b/internal/control/adminapi/weights.go @@ -31,6 +31,11 @@ func (a *API) Orphans() []OrphanView { var wanted []string for _, m := range a.ctl.Catalog().Models() { wanted = append(wanted, m.Source) + // A model's adapters are its weights too; collecting them would + // leave replicas that cannot restart. + for _, ad := range m.Adapters { + wanted = append(wanted, ad.Source) + } } out := make([]OrphanView, 0) diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go index 9af2a31..294ee3f 100644 --- a/internal/control/agentsvc/convert.go +++ b/internal/control/agentsvc/convert.go @@ -189,6 +189,7 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard { GpusPerNode: int32(p.GPUsPerNode), Nodes: int32(p.Nodes), Args: p.Args, Env: p.Env, ModelPath: p.ModelPath, ServedModelName: p.ServedModelName, + Adapters: adaptersToWire(p.Adapters), }, StartupTimeoutMs: cmd.StartupTimeout.Milliseconds(), Pool: cmd.Pool, @@ -224,6 +225,7 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration GPUsPerNode: int(p.GetGpusPerNode()), Nodes: int(p.GetNodes()), Args: p.GetArgs(), Env: p.GetEnv(), ModelPath: p.GetModelPath(), ServedModelName: p.GetServedModelName(), + Adapters: adaptersFromWire(p.GetAdapters()), }, } return spec, time.Duration(m.GetStartupTimeoutMs()) * time.Millisecond @@ -292,3 +294,19 @@ func fabricFrom(state, status, cluster, clique, bandwidth string) *discovery.GPU } return &discovery.GPUFabric{State: state, Status: status, ClusterUUID: cluster, CliqueID: clique, Bandwidth: bandwidth} } + +func adaptersToWire(as []engines.Adapter) []*llmgwv1.LoraAdapter { + var out []*llmgwv1.LoraAdapter + for _, a := range as { + out = append(out, &llmgwv1.LoraAdapter{Name: a.Name, Source: a.Path}) + } + return out +} + +func adaptersFromWire(as []*llmgwv1.LoraAdapter) []engines.Adapter { + var out []engines.Adapter + for _, a := range as { + out = append(out, engines.Adapter{Name: a.GetName(), Path: a.GetSource()}) + } + return out +} diff --git a/internal/control/agentsvc/convert_test.go b/internal/control/agentsvc/convert_test.go new file mode 100644 index 0000000..6888006 --- /dev/null +++ b/internal/control/agentsvc/convert_test.go @@ -0,0 +1,20 @@ +package agentsvc + +import ( + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" +) + +func TestAdaptersTravelToTheAgentInOrder(t *testing.T) { + cmd := reconciler.ShardCommand{ReplicaID: "r", ShardID: "s", Spec: engines.ShardSpec{ + Profile: engines.Profile{Name: "cpu", Engine: "llamacpp", ModelPath: "/m.gguf", + Adapters: []engines.Adapter{{Name: "sql", Path: "http://mirror/sql.gguf"}, {Name: "docs", Path: "http://mirror/docs.gguf"}}}, + }} + spec, _ := ShardSpecFromStart(StartShardFromCommand(cmd)) + got := spec.Profile.Adapters + if len(got) != 2 || got[0].Name != "sql" || got[1].Path != "http://mirror/docs.gguf" { + t.Errorf("adapters %+v", got) + } +} diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index e4f3871..9fa2713 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -169,7 +169,11 @@ type Model struct { // fallbacks: used only when the cluster has no capacity left. Its // prompts leave the cluster, so which keys may name it is worth an // allowlist. - External *External `json:"external,omitempty" koanf:"external"` + External *External `json:"external,omitempty" koanf:"external"` + // Adapters are LoRA adapters served on this model's replicas, each asked + // for by its own name as though it were a model. Every replica loads + // every adapter, applying one only to a request that names it. + Adapters []Adapter `json:"adapters,omitempty" koanf:"adapters"` Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"` // RoutingProfile selects how the router scores endpoints for this model. RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"` @@ -213,14 +217,17 @@ type Capabilities struct { type Catalog struct { models map[string]Model aliases map[string]string - order []string + // adapters maps an adapter's name to the model it runs on. + adapters map[string]string + order []string } // New builds a catalog, rejecting definitions that would fail confusingly later. func New(models []Model) (*Catalog, error) { c := &Catalog{ - models: make(map[string]Model, len(models)), - aliases: map[string]string{}, + models: make(map[string]Model, len(models)), + aliases: map[string]string{}, + adapters: map[string]string{}, } for i := range models { @@ -249,6 +256,18 @@ func New(models []Model) (*Catalog, error) { } } + // Adapters are named like models, so they share the namespace too. + for _, name := range c.order { + for _, a := range c.models[name].Adapters { + _, isModel := c.models[a.Name] + _, isAlias := c.aliases[a.Name] + if _, isAdapter := c.adapters[a.Name]; isModel || isAlias || isAdapter { + return nil, fmt.Errorf("model %q: adapter %q collides with a model, an alias or another adapter", name, a.Name) + } + c.adapters[a.Name] = name + } + } + // Fallbacks are resolved now rather than at request time: a typo in a // fallback should stop a deployment, not surface during an outage when the // fallback is finally needed. @@ -314,6 +333,9 @@ func (m Model) validate() error { if m.Source == "" { return fmt.Errorf("model %q: source is required", m.Name) } + if err := validateAdapters(m); err != nil { + return err + } if m.External != nil { if err := m.External.validate(m.Name, len(m.Profiles)); err != nil { return err @@ -370,6 +392,23 @@ func (m Model) validate() error { } // Resolve maps a client-supplied name or alias onto a model. +// ResolveAdapter finds an adapter by its name: the model it runs on, the +// adapter, and its index among the model's adapters — which is the id the +// engine knows it by. +func (c *Catalog) ResolveAdapter(name string) (Model, Adapter, int, bool) { + base, ok := c.adapters[name] + if !ok { + return Model{}, Adapter{}, 0, false + } + m := c.models[base] + for i, a := range m.Adapters { + if a.Name == name { + return m, a, i, true + } + } + return Model{}, Adapter{}, 0, false +} + func (c *Catalog) Resolve(name string) (Model, bool) { if m, ok := c.models[name]; ok { return m, true @@ -446,6 +485,7 @@ func (m Model) EngineProfile(name string) (engines.Profile, bool) { Env: p.Env, ModelPath: m.Source, ServedModelName: m.Name, + Adapters: m.engineAdapters(), }, true } return engines.Profile{}, false @@ -606,3 +646,61 @@ func (e *External) validate(model string, profiles int) error { // IsExternal reports whether a model is served outside the cluster. func (m Model) IsExternal() bool { return m.External != nil } + +// Adapter is a LoRA adapter served on a model's replicas. +type Adapter struct { + // Name is what clients ask for, like a model name. + Name string `json:"name" koanf:"name"` + // Source is the adapter file (GGUF for llama.cpp), fetched by each node + // as the model's weights are. + Source string `json:"source" koanf:"source"` + // Scale is how strongly it is applied; zero takes 1. + Scale float64 `json:"scale,omitempty" koanf:"scale"` +} + +// EffectiveScale is the scale an adapter is applied at. +func (a Adapter) EffectiveScale() float64 { + if a.Scale <= 0 { + return 1 + } + return a.Scale +} + +func validateAdapters(m Model) error { + if len(m.Adapters) == 0 { + return nil + } + if m.External != nil { + return fmt.Errorf("model %q: an external model cannot carry adapters", m.Name) + } + for _, p := range m.Profiles { + // llama.cpp loads adapters at start and applies them per request; + // the other engines are not driven that way yet. + if p.Engine != "llamacpp" && p.Engine != "sim" { + return fmt.Errorf("model %q profile %q: adapters are served by llamacpp profiles only", m.Name, p.Name) + } + } + seen := map[string]bool{} + for _, a := range m.Adapters { + switch { + case a.Name == "" || a.Source == "": + return fmt.Errorf("model %q: an adapter needs a name and a source", m.Name) + case a.Name == m.Name: + return fmt.Errorf("model %q: an adapter cannot have the model's own name", m.Name) + case seen[a.Name]: + return fmt.Errorf("model %q: adapter %q is declared twice", m.Name, a.Name) + case a.Scale < 0: + return fmt.Errorf("model %q: adapter %q has a negative scale", m.Name, a.Name) + } + seen[a.Name] = true + } + return nil +} + +func (m Model) engineAdapters() []engines.Adapter { + var out []engines.Adapter + for _, a := range m.Adapters { + out = append(out, engines.Adapter{Name: a.Name, Path: a.Source}) + } + return out +} diff --git a/internal/control/modelsvc/modelsvc.go b/internal/control/modelsvc/modelsvc.go index 6ffb3e5..984b94c 100644 --- a/internal/control/modelsvc/modelsvc.go +++ b/internal/control/modelsvc/modelsvc.go @@ -87,6 +87,9 @@ func (s *Service) Sources(ctx context.Context) (map[string][]catalog.Model, erro if m.Source != "" { out[m.Source] = append(out[m.Source], m) } + for _, ad := range m.Adapters { + out[ad.Source] = append(out[ad.Source], m) + } } return out, nil } diff --git a/internal/router/adapter_test.go b/internal/router/adapter_test.go new file mode 100644 index 0000000..fea6f3c --- /dev/null +++ b/internal/router/adapter_test.go @@ -0,0 +1,98 @@ +package router + +import ( + "encoding/json" + "net/http" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/auth" + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +func adapterModel() catalog.Model { + m := catModel("coder", nil) + m.Profiles = []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 4}} + m.Adapters = []catalog.Adapter{ + {Name: "coder-sql", Source: "http://mirror/sql.gguf"}, + {Name: "coder-docs", Source: "http://mirror/docs.gguf", Scale: 0.5}, + } + return m +} + +func TestAnAdapterIsAskedForByNameAndAppliedOnItsModel(t *testing.T) { + e := newFakeEngine(t, "coder") + cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"coder"}}) + cfg.Models = []catalog.Model{adapterModel()} + rt := newTestRouter(t, cfg) + + sent := func(model string) map[string]any { + t.Helper() + rec := post(t, rt.Handler(), map[string]any{"model": model, "messages": []any{}}, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: status %d: %s", model, rec.Code, rec.Body.String()) + } + var body map[string]any + _ = json.Unmarshal(e.last.Load().([]byte), &body) + return body + } + body := sent("coder-docs") + if body["model"] != "coder" { + t.Errorf("the engine was asked for %v, not the model the adapter runs on", body["model"]) + } + lora, _ := json.Marshal(body["lora"]) + if string(lora) != `[{"id":1,"scale":0.5}]` { + t.Errorf("lora %s, expected the second adapter at its scale", lora) + } + if body := sent("coder-sql"); func() string { b, _ := json.Marshal(body["lora"]); return string(b) }() != `[{"id":0,"scale":1}]` { + t.Errorf("the first adapter, at the default scale: %v", body["lora"]) + } + if body := sent("coder"); body["lora"] != nil { + t.Errorf("the base model was sent an adapter: %v", body["lora"]) + } +} + +func TestAKeyAllowedAnAdapterReachesThatAdapterOnly(t *testing.T) { + cfg := baseConfig() + cfg.Models = []catalog.Model{adapterModel()} + rt := newTestRouter(t, cfg) + key := func(allowed ...string) auth.Principal { + return auth.Principal{Limits: store.Effective{AllowedModels: allowed}} + } + for _, c := range []struct { + allowed []string + requested string + want bool + }{ + {[]string{"coder-sql"}, "coder-sql", true}, + {[]string{"coder-sql"}, "coder", false}, + {[]string{"coder-sql"}, "coder-docs", false}, + {[]string{"coder"}, "coder-sql", true}, // the model allows its adapters + {[]string{"coder"}, "coder", true}, + } { + if got := rt.permits(key(c.allowed...), c.requested); got != c.want { + t.Errorf("allowed %v, asked for %s: %v", c.allowed, c.requested, got) + } + } +} + +func TestAdaptersShareTheNamespaceOfModels(t *testing.T) { + a := adapterModel() + b := catModel("coder-sql", nil) + if _, err := catalog.New([]catalog.Model{a, b}); err == nil { + t.Error("an adapter named like another model was accepted") + } + bad := adapterModel() + bad.Profiles = []catalog.Profile{{Name: "gpu", Engine: "vllm", GPUClass: "h100-80g", TP: 1}} + if _, err := catalog.New([]catalog.Model{bad}); err == nil { + t.Error("adapters on a vLLM profile were accepted") + } + c, err := catalog.New([]catalog.Model{adapterModel()}) + if err != nil { + t.Fatal(err) + } + if m, ad, i, ok := c.ResolveAdapter("coder-docs"); !ok || m.Name != "coder" || i != 1 || ad.EffectiveScale() != 0.5 { + t.Errorf("resolve: %v %+v %d %v", m.Name, ad, i, ok) + } +} diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 63ed615..774ce1d 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -426,6 +426,12 @@ func (r *Router) handleInference(path string) http.HandlerFunc { // around the allowlist; matching only the canonical name would silently ignore // an allowlist written in the names clients actually use. func (r *Router) permits(p auth.Principal, requested string) bool { + // An adapter named in the allowlist allows that adapter: not its model, + // and not the model's other adapters, which the checks below would let + // through if the adapter's own name were not asked about first. + if _, ok := r.adapter(requested); ok && p.Allows(requested) { + return true + } canonical, m := r.resolve(requested) if p.Allows(canonical) { return true @@ -1004,3 +1010,24 @@ func withModel(body []byte, name string) []byte { } return out } + +// withAdapter is body asking for model with a LoRA adapter applied, in +// llama.cpp's form: "lora": [{"id": index, "scale": scale}]. Adapters not +// listed are applied at zero, so the list is the whole of it. +func withAdapter(body []byte, model string, index int, scale float64) []byte { + var m map[string]json.RawMessage + if err := json.Unmarshal(body, &m); err != nil { + return body + } + name, err1 := json.Marshal(model) + lora, err2 := json.Marshal([]map[string]any{{"id": index, "scale": scale}}) + if err1 != nil || err2 != nil { + return body + } + m["model"], m["lora"] = name, lora + out, err := json.Marshal(m) + if err != nil { + return body + } + return out +} diff --git a/internal/router/router.go b/internal/router/router.go index 89cc3fb..b4a6f0d 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -61,6 +61,8 @@ type Router struct { // router's half of not having to. A request takes the map once and works // from it, so a swap mid-request is invisible rather than a race. models atomic.Pointer[map[string]catalog.Model] + // adapters are the LoRA adapters by the name clients ask for them by. + adapters atomic.Pointer[map[string]adapterRef] // extraMetrics are collectors other components publish on /metrics. extraMetrics []prometheus.Collector @@ -315,14 +317,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) { // all. A control plane pushes over the top of this at startup and on every // change after. if len(cfg.Models) > 0 { - seed := map[string]catalog.Model{} - for _, m := range cfg.Models { - seed[m.Name] = m - for _, a := range m.Aliases { - seed[a] = m - } - } - r.models.Store(&seed) + r.storeModels(cfg.Models) } return r, nil } @@ -505,15 +500,44 @@ func (r *Router) modelMap() map[string]catalog.Model { // The catalog's counterpart to SetEndpoints, and pushed for the same reason: // the control plane owns what should exist, the router needs to know, and // neither has to be in the other's process for that to work. -func (r *Router) SetCatalog(c *catalog.Catalog) { +func (r *Router) SetCatalog(c *catalog.Catalog) { r.storeModels(c.Models()) } + +// storeModels makes models what names resolve to: each model by its name, its +// aliases and its adapters. One place for it, whether the models came from +// configuration at start or from the control plane since. +func (r *Router) storeModels(models []catalog.Model) { m := map[string]catalog.Model{} - for _, entry := range c.Models() { + ads := map[string]adapterRef{} + for _, entry := range models { m[entry.Name] = entry for _, a := range entry.Aliases { m[a] = entry } + // An adapter resolves to the model it runs on, as an alias does, and + // is applied to the request on the way there. + for i, a := range entry.Adapters { + m[a.Name] = entry + ads[a.Name] = adapterRef{base: entry.Name, index: i, scale: a.EffectiveScale()} + } } r.models.Store(&m) + r.adapters.Store(&ads) +} + +// adapterRef is where an adapter runs and how it is applied. +type adapterRef struct { + base string + index int + scale float64 +} + +// adapter finds one by the name a client asked for. +func (r *Router) adapter(name string) (adapterRef, bool) { + if p := r.adapters.Load(); p != nil { + a, ok := (*p)[name] + return a, ok + } + return adapterRef{}, false } // LastUsed reports when each endpoint was last sent a request, satisfying @@ -719,6 +743,12 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa if dec.Endpoint.UpstreamModel != "" { body = withModel(body, dec.Endpoint.UpstreamModel) } + // An adapter asked for by name, served by its model's replicas: the + // engine is asked for the model, with the adapter applied. Not on a + // fallback to another model, which does not have it. + if a, ok := r.adapter(req.Model); ok && a.base == model { + body = withAdapter(body, model, a.index, a.scale) + } out, perr := r.px.Do(uctx, w, proxy.Request{ URL: dec.Endpoint.URL + req.Path, APIKey: dec.Endpoint.APIKey, diff --git a/llmgw-plan.md b/llmgw-plan.md index 1ac9b95..4165b01 100644 --- a/llmgw-plan.md +++ b/llmgw-plan.md @@ -1497,7 +1497,9 @@ The launch path is now built too: - [x] Weight downloading and cache management: the mirror, agent fetches checked against their digest, and reclaiming what nothing uses - [ ] Prefill/decode disaggregation (§12) -- [ ] LoRA adapter routing (affinity to a replica that already has the adapter) +- [x] LoRA adapters on llama.cpp: every replica loads a model's adapters and + applies one per request, so any replica serves any adapter +- [ ] LoRA affinity for engines that load adapters on demand (vLLM) - [x] External provider support: a catalog model served by an OpenAI-compatible API elsewhere, usable as a fallback, under the same allowlists and budgets - [x] Read-only dashboard on the admin listener (nodes, replicas, models,