diff --git a/CHANGELOG.md b/CHANGELOG.md index fa21d92..341c596 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,18 @@ What v0.10.5 made settable from the command line is now on the pages too: critical model's card no longer refuses a replica that preemption would make room for. +### LoRA adapters + +A llama.cpp model can carry `adapters`, each a name and a GGUF LoRA file. The +nodes fetch them with the model's weights and llama.cpp loads them all at start, +applied to nothing (`--lora-init-without-apply`); a request that names an +adapter goes to the model's replicas with `"lora": [{"id": i, "scale": s}]`, +so one replica serves the model and all its adapters. Adapter names share the +namespace of models, are listed in `/v1/models`, and follow key allowlists on +their own — a key allowed an adapter does not reach the model under it. Adapter +files are kept from weight cleanup like the model's own. Not yet for vLLM or +SGLang profiles, whose adapters are loaded differently. + ### Models served outside the cluster A catalog model can be an OpenAI-compatible API elsewhere: `external: {url, diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index d2c353e..04ad5c0 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -2873,8 +2873,11 @@ type EngineProfile struct { Env map[string]string `protobuf:"bytes,9,rep,name=env,proto3" json:"env,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` ModelPath string `protobuf:"bytes,10,opt,name=model_path,json=modelPath,proto3" json:"model_path,omitempty"` ServedModelName string `protobuf:"bytes,11,opt,name=served_model_name,json=servedModelName,proto3" json:"served_model_name,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // adapters are LoRA adapters loaded beside the model, in order: the index + // is the id a request applies one by. + Adapters []*LoraAdapter `protobuf:"bytes,12,rep,name=adapters,proto3" json:"adapters,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *EngineProfile) Reset() { @@ -2984,6 +2987,13 @@ func (x *EngineProfile) GetServedModelName() string { return "" } +func (x *EngineProfile) GetAdapters() []*LoraAdapter { + if x != nil { + return x.Adapters + } + return nil +} + // RemoveWeights deletes cached model files a node no longer needs. // // The paths are computed by the control plane from the catalog, but the agent @@ -3777,6 +3787,60 @@ func (x *UpgradeReply) GetError() string { return "" } +// LoraAdapter is one adapter: the name clients ask for it by, and its source, +// which the agent fetches as it does the model's weights. +type LoraAdapter struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Source string `protobuf:"bytes,2,opt,name=source,proto3" json:"source,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LoraAdapter) Reset() { + *x = LoraAdapter{} + mi := &file_llmgw_v1_agent_proto_msgTypes[36] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LoraAdapter) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoraAdapter) ProtoMessage() {} + +func (x *LoraAdapter) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[36] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoraAdapter.ProtoReflect.Descriptor instead. +func (*LoraAdapter) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{36} +} + +func (x *LoraAdapter) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *LoraAdapter) GetSource() string { + if x != nil { + return x.Source + } + return "" +} + var File_llmgw_v1_agent_proto protoreflect.FileDescriptor const file_llmgw_v1_agent_proto_rawDesc = "" + @@ -4066,7 +4130,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "profile_fp\x18\x0f \x01(\tR\tprofileFp\x12\x17\n" + "\aapi_key\x18\x10 \x01(\tR\x06apiKey\x12\x1f\n" + "\vcpu_threads\x18\x11 \x01(\x05R\n" + - "cpuThreads\"\xfd\x02\n" + + "cpuThreads\"\xb0\x03\n" + "\rEngineProfile\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" + "\x06engine\x18\x02 \x01(\tR\x06engine\x12\x1b\n" + @@ -4080,7 +4144,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\n" + "model_path\x18\n" + " \x01(\tR\tmodelPath\x12*\n" + - "\x11served_model_name\x18\v \x01(\tR\x0fservedModelName\x1a6\n" + + "\x11served_model_name\x18\v \x01(\tR\x0fservedModelName\x121\n" + + "\badapters\x18\f \x03(\v2\x15.llmgw.v1.LoraAdapterR\badapters\x1a6\n" + "\bEnvEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"%\n" + @@ -4143,7 +4208,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\n" + "request_id\x18\x01 \x01(\tR\trequestId\x12\x18\n" + "\aversion\x18\x02 \x01(\tR\aversion\x12\x14\n" + - "\x05error\x18\x03 \x01(\tR\x05error2O\n" + + "\x05error\x18\x03 \x01(\tR\x05error\"9\n" + + "\vLoraAdapter\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" + + "\x06source\x18\x02 \x01(\tR\x06source2O\n" + "\fAgentService\x12?\n" + "\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" + "\fcom.llmgw.v1B\n" + @@ -4162,7 +4230,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 42) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 43) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage @@ -4201,12 +4269,13 @@ var file_llmgw_v1_agent_proto_goTypes = []any{ (*FabricTestReply)(nil), // 34: llmgw.v1.FabricTestReply (*UpgradeAgent)(nil), // 35: llmgw.v1.UpgradeAgent (*UpgradeReply)(nil), // 36: llmgw.v1.UpgradeReply - nil, // 37: llmgw.v1.Register.LabelsEntry - nil, // 38: llmgw.v1.Register.EngineVersionsEntry - nil, // 39: llmgw.v1.GPU.NicPathsEntry - nil, // 40: llmgw.v1.IBPort.CountersEntry - nil, // 41: llmgw.v1.Heartbeat.EngineVersionsEntry - nil, // 42: llmgw.v1.EngineProfile.EnvEntry + (*LoraAdapter)(nil), // 37: llmgw.v1.LoraAdapter + nil, // 38: llmgw.v1.Register.LabelsEntry + nil, // 39: llmgw.v1.Register.EngineVersionsEntry + nil, // 40: llmgw.v1.GPU.NicPathsEntry + nil, // 41: llmgw.v1.IBPort.CountersEntry + nil, // 42: llmgw.v1.Heartbeat.EngineVersionsEntry + nil, // 43: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register @@ -4226,24 +4295,24 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{ 31, // 14: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest 33, // 15: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest 35, // 16: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent - 37, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 38, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry 7, // 18: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU 8, // 19: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 38, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 39, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry 6, // 21: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard 9, // 22: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights 5, // 23: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage 4, // 24: llmgw.v1.Register.host:type_name -> llmgw.v1.Host 15, // 25: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX 0, // 26: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State - 39, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry + 40, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry 12, // 28: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad 11, // 29: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor - 40, // 30: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry + 41, // 30: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry 14, // 31: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager 5, // 32: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage 9, // 33: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights - 41, // 34: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry + 42, // 34: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry 20, // 35: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus 10, // 36: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad 13, // 37: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort @@ -4254,14 +4323,15 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{ 0, // 42: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State 22, // 43: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress 24, // 44: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 42, // 45: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 1, // 46: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 47: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 47, // [47:48] is the sub-list for method output_type - 46, // [46:47] is the sub-list for method input_type - 46, // [46:46] is the sub-list for extension type_name - 46, // [46:46] is the sub-list for extension extendee - 0, // [0:46] is the sub-list for field type_name + 43, // 45: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 37, // 46: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter + 1, // 47: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 48: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 48, // [48:49] is the sub-list for method output_type + 47, // [47:48] is the sub-list for method input_type + 47, // [47:47] is the sub-list for extension type_name + 47, // [47:47] is the sub-list for extension extendee + 0, // [0:47] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -4296,7 +4366,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 42, + NumMessages: 43, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 3283981..45818fb 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -572,6 +572,9 @@ message EngineProfile { map env = 9; string model_path = 10; string served_model_name = 11; + // adapters are LoRA adapters loaded beside the model, in order: the index + // is the id a request applies one by. + repeated LoraAdapter adapters = 12; } // RemoveWeights deletes cached model files a node no longer needs. @@ -704,3 +707,10 @@ message UpgradeReply { string version = 2; string error = 3; } + +// LoraAdapter is one adapter: the name clients ask for it by, and its source, +// which the agent fetches as it does the model's weights. +message LoraAdapter { + string name = 1; + string source = 2; +} diff --git a/docs/install.md b/docs/install.md index b57be64..0989b0a 100644 --- a/docs/install.md +++ b/docs/install.md @@ -671,6 +671,34 @@ llmgw ctl explain-placement qwen2.5-coder-3b-instruct which names every candidate and why each was rejected, rather than leaving you to infer it. +### LoRA adapters + +A llama.cpp model can carry LoRA adapters, each asked for by its own name as +though it were a model: + +```yaml +- name: qwen2.5-coder-7b-instruct + source: http://mirror:8081/qwen2.5-coder-7b-instruct/5d567d26/model.gguf + adapters: + - name: coder-sql + source: http://mirror:8081/coder-sql/v1/adapter.gguf + scale: 1.0 # optional; 1 when left out + profiles: [...] +``` + +Every replica of the model fetches every adapter and loads it at start, applied +to nothing. A request for `coder-sql` goes to the model's replicas with the +adapter applied to it alone; a request for the model gets the model. Adapter +names share the namespace of models and aliases, appear in `/v1/models`, and +follow key allowlists on their own: a key allowed `coder-sql` reaches that +adapter and not the model under it, while a key allowed the model reaches its +adapters too. Adding or removing an adapter is a change to the model; running +replicas load the new set when restarted (`llmgw ctl restart --model M`). + +The adapter file must be a GGUF LoRA made for this model — llama.cpp's +`convert_lora_to_gguf.py` makes one from a PEFT adapter — and not a model with +the adapter already merged in, which is what most "LoRA GGUF" repositories hold. + ### Models served outside the cluster A model can be an OpenAI-compatible API elsewhere — a hosted provider, or diff --git a/docs/reference.md b/docs/reference.md index 1c9b5a0..225ec8b 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -15,6 +15,9 @@ rather than repeated. | Key | Type | In | |---|---|---| +| `name` | `string` | [Adapter](#adapter) | +| `source` | `string` | [Adapter](#adapter) | +| `scale` | `float64` | [Adapter](#adapter) | | `enabled` | `bool` | [Admission](#admission) | | `bands` | `[]Band` | [Admission](#admission) | | `default_band` | `string` | [Admission](#admission) | @@ -99,6 +102,7 @@ rather than repeated. | `class` | `string` | [Model](#model) | | `owner` | `string` | [Model](#model) | | `external` | `*External` | [Model](#model) | +| `adapters` | `[]Adapter` | [Model](#model) | | `fallbacks` | `[]string` | [Model](#model) | | `routing_profile` | `string` | [Model](#model) | | `slo` | `SLO` | [Model](#model) | @@ -200,6 +204,27 @@ rather than repeated. | `launch_mode` | `string` | [VLLMConfig](#vllmconfig) | | `ray_bin` | `string` | [VLLMConfig](#vllmconfig) | +## Adapter + +### `name` + +`string` + +Name is what clients ask for, like a model name. + +### `source` + +`string` + +Source is the adapter file (GGUF for llama.cpp), fetched by each node +as the model's weights are. + +### `scale` + +`float64` + +Scale is how strongly it is applied; zero takes 1. + ## Admission ### `enabled` @@ -805,6 +830,14 @@ allowlist. Takes a [External](#external) block. +### `adapters` + +`[]Adapter` + +Adapters are LoRA adapters served on this model's replicas, each asked +for by its own name as though it were a model. Every replica loads +every adapter, applying one only to a request that names it. + ### `fallbacks` `[]string` diff --git a/internal/agent/agent.go b/internal/agent/agent.go index e5d6993..469e437 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -891,6 +891,17 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan< stop() spec.Profile.ModelPath = path } + // Adapters are fetched like the model, and before the engine starts: it + // loads them at start, and one missing is a replica that answers for the + // base model under the adapter's name. + for i, ad := range spec.Profile.Adapters { + path, err := a.weights().Ensure(ctx, ad.Path) + if err != nil { + report(supervisor.Failed, fmt.Sprintf("adapter %s: %v", ad.Name, err), nil) + return + } + spec.Profile.Adapters[i].Path = path + } cmds, err := eng.Commands(spec) if err != nil { diff --git a/internal/agent/engines/engine.go b/internal/agent/engines/engine.go index 52569bf..831d7f9 100644 --- a/internal/agent/engines/engine.go +++ b/internal/agent/engines/engine.go @@ -40,6 +40,9 @@ type Profile struct { ModelPath string // ServedModelName is the name clients use, which need not match the path. ServedModelName string + // Adapters are LoRA adapters to load beside the model, in order: an + // adapter's index is the id requests apply it by. + Adapters []Adapter } // ShardSpec is one shard of one replica on this node. @@ -373,3 +376,10 @@ func quantizationOf(args []string) string { } return "" } + +// Adapter is a LoRA adapter an engine loads beside its model. Path is a +// source until the agent has fetched it, a local file after. +type Adapter struct { + Name string `json:"name"` + Path string `json:"path"` +} diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go index 4de8060..eb14d63 100644 --- a/internal/agent/engines/llamacpp.go +++ b/internal/agent/engines/llamacpp.go @@ -144,6 +144,15 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { } args = append(args, "--threads", strconv.Itoa(n)) } + // Loaded at start, applied per request: a request that names an adapter + // carries "lora": [{"id": i, "scale": s}], and one that names none is the + // base model, since every adapter starts at scale zero. + if len(p.Adapters) > 0 { + args = append(args, "--lora-init-without-apply") + for _, a := range p.Adapters { + args = append(args, "--lora", a.Path) + } + } args = append(args, p.Args...) env := baseEnv(s) diff --git a/internal/agent/engines/llamacpp_test.go b/internal/agent/engines/llamacpp_test.go index 3cafd35..c3133c3 100644 --- a/internal/agent/engines/llamacpp_test.go +++ b/internal/agent/engines/llamacpp_test.go @@ -306,3 +306,20 @@ func TestTheCoresPlacementReservedAreTheThreads(t *testing.T) { t.Errorf("--threads %q; what placement reserved should win over the node's setting", v) } } + +func TestAdaptersAreLoadedAtStartAndAppliedToNoneByDefault(t *testing.T) { + s := llamaSpec(nil) + s.Profile.Adapters = []Adapter{{Name: "sql", Path: "/w/sql.gguf"}, {Name: "docs", Path: "/w/docs.gguf"}} + cmds, err := (&LlamaCpp{}).Commands(s) + if err != nil { + t.Fatal(err) + } + args := strings.Join(cmds[0].Args, " ") + if !strings.Contains(args, "--lora-init-without-apply --lora /w/sql.gguf --lora /w/docs.gguf") { + t.Errorf("args %s: both adapters, in order, loaded unapplied", args) + } + plain, _ := (&LlamaCpp{}).Commands(llamaSpec(nil)) + if strings.Contains(strings.Join(plain[0].Args, " "), "--lora") { + t.Error("a model with no adapters was given lora flags") + } +} diff --git a/internal/control/adminapi/weights.go b/internal/control/adminapi/weights.go index 2f7927b..ac8e904 100644 --- a/internal/control/adminapi/weights.go +++ b/internal/control/adminapi/weights.go @@ -31,6 +31,11 @@ func (a *API) Orphans() []OrphanView { var wanted []string for _, m := range a.ctl.Catalog().Models() { wanted = append(wanted, m.Source) + // A model's adapters are its weights too; collecting them would + // leave replicas that cannot restart. + for _, ad := range m.Adapters { + wanted = append(wanted, ad.Source) + } } out := make([]OrphanView, 0) diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go index 9af2a31..294ee3f 100644 --- a/internal/control/agentsvc/convert.go +++ b/internal/control/agentsvc/convert.go @@ -189,6 +189,7 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard { GpusPerNode: int32(p.GPUsPerNode), Nodes: int32(p.Nodes), Args: p.Args, Env: p.Env, ModelPath: p.ModelPath, ServedModelName: p.ServedModelName, + Adapters: adaptersToWire(p.Adapters), }, StartupTimeoutMs: cmd.StartupTimeout.Milliseconds(), Pool: cmd.Pool, @@ -224,6 +225,7 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration GPUsPerNode: int(p.GetGpusPerNode()), Nodes: int(p.GetNodes()), Args: p.GetArgs(), Env: p.GetEnv(), ModelPath: p.GetModelPath(), ServedModelName: p.GetServedModelName(), + Adapters: adaptersFromWire(p.GetAdapters()), }, } return spec, time.Duration(m.GetStartupTimeoutMs()) * time.Millisecond @@ -292,3 +294,19 @@ func fabricFrom(state, status, cluster, clique, bandwidth string) *discovery.GPU } return &discovery.GPUFabric{State: state, Status: status, ClusterUUID: cluster, CliqueID: clique, Bandwidth: bandwidth} } + +func adaptersToWire(as []engines.Adapter) []*llmgwv1.LoraAdapter { + var out []*llmgwv1.LoraAdapter + for _, a := range as { + out = append(out, &llmgwv1.LoraAdapter{Name: a.Name, Source: a.Path}) + } + return out +} + +func adaptersFromWire(as []*llmgwv1.LoraAdapter) []engines.Adapter { + var out []engines.Adapter + for _, a := range as { + out = append(out, engines.Adapter{Name: a.GetName(), Path: a.GetSource()}) + } + return out +} diff --git a/internal/control/agentsvc/convert_test.go b/internal/control/agentsvc/convert_test.go new file mode 100644 index 0000000..6888006 --- /dev/null +++ b/internal/control/agentsvc/convert_test.go @@ -0,0 +1,20 @@ +package agentsvc + +import ( + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" +) + +func TestAdaptersTravelToTheAgentInOrder(t *testing.T) { + cmd := reconciler.ShardCommand{ReplicaID: "r", ShardID: "s", Spec: engines.ShardSpec{ + Profile: engines.Profile{Name: "cpu", Engine: "llamacpp", ModelPath: "/m.gguf", + Adapters: []engines.Adapter{{Name: "sql", Path: "http://mirror/sql.gguf"}, {Name: "docs", Path: "http://mirror/docs.gguf"}}}, + }} + spec, _ := ShardSpecFromStart(StartShardFromCommand(cmd)) + got := spec.Profile.Adapters + if len(got) != 2 || got[0].Name != "sql" || got[1].Path != "http://mirror/docs.gguf" { + t.Errorf("adapters %+v", got) + } +} diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index e4f3871..9fa2713 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -169,7 +169,11 @@ type Model struct { // fallbacks: used only when the cluster has no capacity left. Its // prompts leave the cluster, so which keys may name it is worth an // allowlist. - External *External `json:"external,omitempty" koanf:"external"` + External *External `json:"external,omitempty" koanf:"external"` + // Adapters are LoRA adapters served on this model's replicas, each asked + // for by its own name as though it were a model. Every replica loads + // every adapter, applying one only to a request that names it. + Adapters []Adapter `json:"adapters,omitempty" koanf:"adapters"` Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"` // RoutingProfile selects how the router scores endpoints for this model. RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"` @@ -213,14 +217,17 @@ type Capabilities struct { type Catalog struct { models map[string]Model aliases map[string]string - order []string + // adapters maps an adapter's name to the model it runs on. + adapters map[string]string + order []string } // New builds a catalog, rejecting definitions that would fail confusingly later. func New(models []Model) (*Catalog, error) { c := &Catalog{ - models: make(map[string]Model, len(models)), - aliases: map[string]string{}, + models: make(map[string]Model, len(models)), + aliases: map[string]string{}, + adapters: map[string]string{}, } for i := range models { @@ -249,6 +256,18 @@ func New(models []Model) (*Catalog, error) { } } + // Adapters are named like models, so they share the namespace too. + for _, name := range c.order { + for _, a := range c.models[name].Adapters { + _, isModel := c.models[a.Name] + _, isAlias := c.aliases[a.Name] + if _, isAdapter := c.adapters[a.Name]; isModel || isAlias || isAdapter { + return nil, fmt.Errorf("model %q: adapter %q collides with a model, an alias or another adapter", name, a.Name) + } + c.adapters[a.Name] = name + } + } + // Fallbacks are resolved now rather than at request time: a typo in a // fallback should stop a deployment, not surface during an outage when the // fallback is finally needed. @@ -314,6 +333,9 @@ func (m Model) validate() error { if m.Source == "" { return fmt.Errorf("model %q: source is required", m.Name) } + if err := validateAdapters(m); err != nil { + return err + } if m.External != nil { if err := m.External.validate(m.Name, len(m.Profiles)); err != nil { return err @@ -370,6 +392,23 @@ func (m Model) validate() error { } // Resolve maps a client-supplied name or alias onto a model. +// ResolveAdapter finds an adapter by its name: the model it runs on, the +// adapter, and its index among the model's adapters — which is the id the +// engine knows it by. +func (c *Catalog) ResolveAdapter(name string) (Model, Adapter, int, bool) { + base, ok := c.adapters[name] + if !ok { + return Model{}, Adapter{}, 0, false + } + m := c.models[base] + for i, a := range m.Adapters { + if a.Name == name { + return m, a, i, true + } + } + return Model{}, Adapter{}, 0, false +} + func (c *Catalog) Resolve(name string) (Model, bool) { if m, ok := c.models[name]; ok { return m, true @@ -446,6 +485,7 @@ func (m Model) EngineProfile(name string) (engines.Profile, bool) { Env: p.Env, ModelPath: m.Source, ServedModelName: m.Name, + Adapters: m.engineAdapters(), }, true } return engines.Profile{}, false @@ -606,3 +646,61 @@ func (e *External) validate(model string, profiles int) error { // IsExternal reports whether a model is served outside the cluster. func (m Model) IsExternal() bool { return m.External != nil } + +// Adapter is a LoRA adapter served on a model's replicas. +type Adapter struct { + // Name is what clients ask for, like a model name. + Name string `json:"name" koanf:"name"` + // Source is the adapter file (GGUF for llama.cpp), fetched by each node + // as the model's weights are. + Source string `json:"source" koanf:"source"` + // Scale is how strongly it is applied; zero takes 1. + Scale float64 `json:"scale,omitempty" koanf:"scale"` +} + +// EffectiveScale is the scale an adapter is applied at. +func (a Adapter) EffectiveScale() float64 { + if a.Scale <= 0 { + return 1 + } + return a.Scale +} + +func validateAdapters(m Model) error { + if len(m.Adapters) == 0 { + return nil + } + if m.External != nil { + return fmt.Errorf("model %q: an external model cannot carry adapters", m.Name) + } + for _, p := range m.Profiles { + // llama.cpp loads adapters at start and applies them per request; + // the other engines are not driven that way yet. + if p.Engine != "llamacpp" && p.Engine != "sim" { + return fmt.Errorf("model %q profile %q: adapters are served by llamacpp profiles only", m.Name, p.Name) + } + } + seen := map[string]bool{} + for _, a := range m.Adapters { + switch { + case a.Name == "" || a.Source == "": + return fmt.Errorf("model %q: an adapter needs a name and a source", m.Name) + case a.Name == m.Name: + return fmt.Errorf("model %q: an adapter cannot have the model's own name", m.Name) + case seen[a.Name]: + return fmt.Errorf("model %q: adapter %q is declared twice", m.Name, a.Name) + case a.Scale < 0: + return fmt.Errorf("model %q: adapter %q has a negative scale", m.Name, a.Name) + } + seen[a.Name] = true + } + return nil +} + +func (m Model) engineAdapters() []engines.Adapter { + var out []engines.Adapter + for _, a := range m.Adapters { + out = append(out, engines.Adapter{Name: a.Name, Path: a.Source}) + } + return out +} diff --git a/internal/control/modelsvc/modelsvc.go b/internal/control/modelsvc/modelsvc.go index 6ffb3e5..984b94c 100644 --- a/internal/control/modelsvc/modelsvc.go +++ b/internal/control/modelsvc/modelsvc.go @@ -87,6 +87,9 @@ func (s *Service) Sources(ctx context.Context) (map[string][]catalog.Model, erro if m.Source != "" { out[m.Source] = append(out[m.Source], m) } + for _, ad := range m.Adapters { + out[ad.Source] = append(out[ad.Source], m) + } } return out, nil } diff --git a/internal/router/adapter_test.go b/internal/router/adapter_test.go new file mode 100644 index 0000000..fea6f3c --- /dev/null +++ b/internal/router/adapter_test.go @@ -0,0 +1,98 @@ +package router + +import ( + "encoding/json" + "net/http" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/auth" + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +func adapterModel() catalog.Model { + m := catModel("coder", nil) + m.Profiles = []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 4}} + m.Adapters = []catalog.Adapter{ + {Name: "coder-sql", Source: "http://mirror/sql.gguf"}, + {Name: "coder-docs", Source: "http://mirror/docs.gguf", Scale: 0.5}, + } + return m +} + +func TestAnAdapterIsAskedForByNameAndAppliedOnItsModel(t *testing.T) { + e := newFakeEngine(t, "coder") + cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"coder"}}) + cfg.Models = []catalog.Model{adapterModel()} + rt := newTestRouter(t, cfg) + + sent := func(model string) map[string]any { + t.Helper() + rec := post(t, rt.Handler(), map[string]any{"model": model, "messages": []any{}}, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: status %d: %s", model, rec.Code, rec.Body.String()) + } + var body map[string]any + _ = json.Unmarshal(e.last.Load().([]byte), &body) + return body + } + body := sent("coder-docs") + if body["model"] != "coder" { + t.Errorf("the engine was asked for %v, not the model the adapter runs on", body["model"]) + } + lora, _ := json.Marshal(body["lora"]) + if string(lora) != `[{"id":1,"scale":0.5}]` { + t.Errorf("lora %s, expected the second adapter at its scale", lora) + } + if body := sent("coder-sql"); func() string { b, _ := json.Marshal(body["lora"]); return string(b) }() != `[{"id":0,"scale":1}]` { + t.Errorf("the first adapter, at the default scale: %v", body["lora"]) + } + if body := sent("coder"); body["lora"] != nil { + t.Errorf("the base model was sent an adapter: %v", body["lora"]) + } +} + +func TestAKeyAllowedAnAdapterReachesThatAdapterOnly(t *testing.T) { + cfg := baseConfig() + cfg.Models = []catalog.Model{adapterModel()} + rt := newTestRouter(t, cfg) + key := func(allowed ...string) auth.Principal { + return auth.Principal{Limits: store.Effective{AllowedModels: allowed}} + } + for _, c := range []struct { + allowed []string + requested string + want bool + }{ + {[]string{"coder-sql"}, "coder-sql", true}, + {[]string{"coder-sql"}, "coder", false}, + {[]string{"coder-sql"}, "coder-docs", false}, + {[]string{"coder"}, "coder-sql", true}, // the model allows its adapters + {[]string{"coder"}, "coder", true}, + } { + if got := rt.permits(key(c.allowed...), c.requested); got != c.want { + t.Errorf("allowed %v, asked for %s: %v", c.allowed, c.requested, got) + } + } +} + +func TestAdaptersShareTheNamespaceOfModels(t *testing.T) { + a := adapterModel() + b := catModel("coder-sql", nil) + if _, err := catalog.New([]catalog.Model{a, b}); err == nil { + t.Error("an adapter named like another model was accepted") + } + bad := adapterModel() + bad.Profiles = []catalog.Profile{{Name: "gpu", Engine: "vllm", GPUClass: "h100-80g", TP: 1}} + if _, err := catalog.New([]catalog.Model{bad}); err == nil { + t.Error("adapters on a vLLM profile were accepted") + } + c, err := catalog.New([]catalog.Model{adapterModel()}) + if err != nil { + t.Fatal(err) + } + if m, ad, i, ok := c.ResolveAdapter("coder-docs"); !ok || m.Name != "coder" || i != 1 || ad.EffectiveScale() != 0.5 { + t.Errorf("resolve: %v %+v %d %v", m.Name, ad, i, ok) + } +} diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 63ed615..774ce1d 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -426,6 +426,12 @@ func (r *Router) handleInference(path string) http.HandlerFunc { // around the allowlist; matching only the canonical name would silently ignore // an allowlist written in the names clients actually use. func (r *Router) permits(p auth.Principal, requested string) bool { + // An adapter named in the allowlist allows that adapter: not its model, + // and not the model's other adapters, which the checks below would let + // through if the adapter's own name were not asked about first. + if _, ok := r.adapter(requested); ok && p.Allows(requested) { + return true + } canonical, m := r.resolve(requested) if p.Allows(canonical) { return true @@ -1004,3 +1010,24 @@ func withModel(body []byte, name string) []byte { } return out } + +// withAdapter is body asking for model with a LoRA adapter applied, in +// llama.cpp's form: "lora": [{"id": index, "scale": scale}]. Adapters not +// listed are applied at zero, so the list is the whole of it. +func withAdapter(body []byte, model string, index int, scale float64) []byte { + var m map[string]json.RawMessage + if err := json.Unmarshal(body, &m); err != nil { + return body + } + name, err1 := json.Marshal(model) + lora, err2 := json.Marshal([]map[string]any{{"id": index, "scale": scale}}) + if err1 != nil || err2 != nil { + return body + } + m["model"], m["lora"] = name, lora + out, err := json.Marshal(m) + if err != nil { + return body + } + return out +} diff --git a/internal/router/router.go b/internal/router/router.go index 89cc3fb..b4a6f0d 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -61,6 +61,8 @@ type Router struct { // router's half of not having to. A request takes the map once and works // from it, so a swap mid-request is invisible rather than a race. models atomic.Pointer[map[string]catalog.Model] + // adapters are the LoRA adapters by the name clients ask for them by. + adapters atomic.Pointer[map[string]adapterRef] // extraMetrics are collectors other components publish on /metrics. extraMetrics []prometheus.Collector @@ -315,14 +317,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) { // all. A control plane pushes over the top of this at startup and on every // change after. if len(cfg.Models) > 0 { - seed := map[string]catalog.Model{} - for _, m := range cfg.Models { - seed[m.Name] = m - for _, a := range m.Aliases { - seed[a] = m - } - } - r.models.Store(&seed) + r.storeModels(cfg.Models) } return r, nil } @@ -505,15 +500,44 @@ func (r *Router) modelMap() map[string]catalog.Model { // The catalog's counterpart to SetEndpoints, and pushed for the same reason: // the control plane owns what should exist, the router needs to know, and // neither has to be in the other's process for that to work. -func (r *Router) SetCatalog(c *catalog.Catalog) { +func (r *Router) SetCatalog(c *catalog.Catalog) { r.storeModels(c.Models()) } + +// storeModels makes models what names resolve to: each model by its name, its +// aliases and its adapters. One place for it, whether the models came from +// configuration at start or from the control plane since. +func (r *Router) storeModels(models []catalog.Model) { m := map[string]catalog.Model{} - for _, entry := range c.Models() { + ads := map[string]adapterRef{} + for _, entry := range models { m[entry.Name] = entry for _, a := range entry.Aliases { m[a] = entry } + // An adapter resolves to the model it runs on, as an alias does, and + // is applied to the request on the way there. + for i, a := range entry.Adapters { + m[a.Name] = entry + ads[a.Name] = adapterRef{base: entry.Name, index: i, scale: a.EffectiveScale()} + } } r.models.Store(&m) + r.adapters.Store(&ads) +} + +// adapterRef is where an adapter runs and how it is applied. +type adapterRef struct { + base string + index int + scale float64 +} + +// adapter finds one by the name a client asked for. +func (r *Router) adapter(name string) (adapterRef, bool) { + if p := r.adapters.Load(); p != nil { + a, ok := (*p)[name] + return a, ok + } + return adapterRef{}, false } // LastUsed reports when each endpoint was last sent a request, satisfying @@ -719,6 +743,12 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa if dec.Endpoint.UpstreamModel != "" { body = withModel(body, dec.Endpoint.UpstreamModel) } + // An adapter asked for by name, served by its model's replicas: the + // engine is asked for the model, with the adapter applied. Not on a + // fallback to another model, which does not have it. + if a, ok := r.adapter(req.Model); ok && a.base == model { + body = withAdapter(body, model, a.index, a.scale) + } out, perr := r.px.Do(uctx, w, proxy.Request{ URL: dec.Endpoint.URL + req.Path, APIKey: dec.Endpoint.APIKey, diff --git a/llmgw-plan.md b/llmgw-plan.md index 1ac9b95..4165b01 100644 --- a/llmgw-plan.md +++ b/llmgw-plan.md @@ -1497,7 +1497,9 @@ The launch path is now built too: - [x] Weight downloading and cache management: the mirror, agent fetches checked against their digest, and reclaiming what nothing uses - [ ] Prefill/decode disaggregation (§12) -- [ ] LoRA adapter routing (affinity to a replica that already has the adapter) +- [x] LoRA adapters on llama.cpp: every replica loads a model's adapters and + applies one per request, so any replica serves any adapter +- [ ] LoRA affinity for engines that load adapters on demand (vLLM) - [x] External provider support: a catalog model served by an OpenAI-compatible API elsewhere, usable as a fallback, under the same allowlists and budgets - [x] Read-only dashboard on the admin listener (nodes, replicas, models,