Serve LoRA adapters on llama.cpp models, asked for by name
A model's adapters are fetched by each node with its weights and loaded unapplied at start; a request naming an adapter reaches the model's replicas with llama.cpp's per-request lora field. Adapter names share the model namespace, are listed, and follow allowlists on their own. The router's config seed and pushed catalog now build the name map in one place. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
94eec557ad
commit
cf53904cc5
18 files changed
+542
-41
No files matched your search
@@ -2873,8 +2873,11 @@ type EngineProfile struct {
|
||||
Env map[string]string `protobuf:"bytes,9,rep,name=env,proto3" json:"env,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
|
||||
ModelPath string `protobuf:"bytes,10,opt,name=model_path,json=modelPath,proto3" json:"model_path,omitempty"`
|
||||
ServedModelName string `protobuf:"bytes,11,opt,name=served_model_name,json=servedModelName,proto3" json:"served_model_name,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
// adapters are LoRA adapters loaded beside the model, in order: the index
|
||||
// is the id a request applies one by.
|
||||
Adapters []*LoraAdapter `protobuf:"bytes,12,rep,name=adapters,proto3" json:"adapters,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *EngineProfile) Reset() {
|
||||
@@ -2984,6 +2987,13 @@ func (x *EngineProfile) GetServedModelName() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *EngineProfile) GetAdapters() []*LoraAdapter {
|
||||
if x != nil {
|
||||
return x.Adapters
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveWeights deletes cached model files a node no longer needs.
|
||||
//
|
||||
// The paths are computed by the control plane from the catalog, but the agent
|
||||
@@ -3777,6 +3787,60 @@ func (x *UpgradeReply) GetError() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// LoraAdapter is one adapter: the name clients ask for it by, and its source,
|
||||
// which the agent fetches as it does the model's weights.
|
||||
type LoraAdapter struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
|
||||
Source string `protobuf:"bytes,2,opt,name=source,proto3" json:"source,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *LoraAdapter) Reset() {
|
||||
*x = LoraAdapter{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[36]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *LoraAdapter) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*LoraAdapter) ProtoMessage() {}
|
||||
|
||||
func (x *LoraAdapter) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[36]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use LoraAdapter.ProtoReflect.Descriptor instead.
|
||||
func (*LoraAdapter) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{36}
|
||||
}
|
||||
|
||||
func (x *LoraAdapter) GetName() string {
|
||||
if x != nil {
|
||||
return x.Name
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *LoraAdapter) GetSource() string {
|
||||
if x != nil {
|
||||
return x.Source
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var File_llmgw_v1_agent_proto protoreflect.FileDescriptor
|
||||
|
||||
const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
@@ -4066,7 +4130,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"profile_fp\x18\x0f \x01(\tR\tprofileFp\x12\x17\n" +
|
||||
"\aapi_key\x18\x10 \x01(\tR\x06apiKey\x12\x1f\n" +
|
||||
"\vcpu_threads\x18\x11 \x01(\x05R\n" +
|
||||
"cpuThreads\"\xfd\x02\n" +
|
||||
"cpuThreads\"\xb0\x03\n" +
|
||||
"\rEngineProfile\x12\x12\n" +
|
||||
"\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" +
|
||||
"\x06engine\x18\x02 \x01(\tR\x06engine\x12\x1b\n" +
|
||||
@@ -4080,7 +4144,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\n" +
|
||||
"model_path\x18\n" +
|
||||
" \x01(\tR\tmodelPath\x12*\n" +
|
||||
"\x11served_model_name\x18\v \x01(\tR\x0fservedModelName\x1a6\n" +
|
||||
"\x11served_model_name\x18\v \x01(\tR\x0fservedModelName\x121\n" +
|
||||
"\badapters\x18\f \x03(\v2\x15.llmgw.v1.LoraAdapterR\badapters\x1a6\n" +
|
||||
"\bEnvEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"%\n" +
|
||||
@@ -4143,7 +4208,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\n" +
|
||||
"request_id\x18\x01 \x01(\tR\trequestId\x12\x18\n" +
|
||||
"\aversion\x18\x02 \x01(\tR\aversion\x12\x14\n" +
|
||||
"\x05error\x18\x03 \x01(\tR\x05error2O\n" +
|
||||
"\x05error\x18\x03 \x01(\tR\x05error\"9\n" +
|
||||
"\vLoraAdapter\x12\x12\n" +
|
||||
"\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" +
|
||||
"\x06source\x18\x02 \x01(\tR\x06source2O\n" +
|
||||
"\fAgentService\x12?\n" +
|
||||
"\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" +
|
||||
"\fcom.llmgw.v1B\n" +
|
||||
@@ -4162,7 +4230,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
|
||||
}
|
||||
|
||||
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 42)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 43)
|
||||
var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
|
||||
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
|
||||
@@ -4201,12 +4269,13 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(*FabricTestReply)(nil), // 34: llmgw.v1.FabricTestReply
|
||||
(*UpgradeAgent)(nil), // 35: llmgw.v1.UpgradeAgent
|
||||
(*UpgradeReply)(nil), // 36: llmgw.v1.UpgradeReply
|
||||
nil, // 37: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 38: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 39: llmgw.v1.GPU.NicPathsEntry
|
||||
nil, // 40: llmgw.v1.IBPort.CountersEntry
|
||||
nil, // 41: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 42: llmgw.v1.EngineProfile.EnvEntry
|
||||
(*LoraAdapter)(nil), // 37: llmgw.v1.LoraAdapter
|
||||
nil, // 38: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 39: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 40: llmgw.v1.GPU.NicPathsEntry
|
||||
nil, // 41: llmgw.v1.IBPort.CountersEntry
|
||||
nil, // 42: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 43: llmgw.v1.EngineProfile.EnvEntry
|
||||
}
|
||||
var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
|
||||
@@ -4226,24 +4295,24 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
31, // 14: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
|
||||
33, // 15: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest
|
||||
35, // 16: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent
|
||||
37, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
38, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
7, // 18: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
|
||||
8, // 19: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
38, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
39, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
6, // 21: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
|
||||
9, // 22: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
|
||||
5, // 23: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
|
||||
4, // 24: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
|
||||
15, // 25: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
|
||||
0, // 26: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
39, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
|
||||
40, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
|
||||
12, // 28: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
|
||||
11, // 29: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor
|
||||
40, // 30: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
|
||||
41, // 30: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
|
||||
14, // 31: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
|
||||
5, // 32: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
9, // 33: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
41, // 34: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
42, // 34: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
20, // 35: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
|
||||
10, // 36: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
|
||||
13, // 37: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
|
||||
@@ -4254,14 +4323,15 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
0, // 42: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
22, // 43: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
24, // 44: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
42, // 45: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
1, // 46: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 47: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
47, // [47:48] is the sub-list for method output_type
|
||||
46, // [46:47] is the sub-list for method input_type
|
||||
46, // [46:46] is the sub-list for extension type_name
|
||||
46, // [46:46] is the sub-list for extension extendee
|
||||
0, // [0:46] is the sub-list for field type_name
|
||||
43, // 45: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
37, // 46: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter
|
||||
1, // 47: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 48: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
48, // [48:49] is the sub-list for method output_type
|
||||
47, // [47:48] is the sub-list for method input_type
|
||||
47, // [47:47] is the sub-list for extension type_name
|
||||
47, // [47:47] is the sub-list for extension extendee
|
||||
0, // [0:47] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_llmgw_v1_agent_proto_init() }
|
||||
@@ -4296,7 +4366,7 @@ func file_llmgw_v1_agent_proto_init() {
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
|
||||
NumEnums: 1,
|
||||
NumMessages: 42,
|
||||
NumMessages: 43,
|
||||
NumExtensions: 0,
|
||||
NumServices: 1,
|
||||
},
|
||||
|
||||
@@ -572,6 +572,9 @@ message EngineProfile {
|
||||
map<string, string> env = 9;
|
||||
string model_path = 10;
|
||||
string served_model_name = 11;
|
||||
// adapters are LoRA adapters loaded beside the model, in order: the index
|
||||
// is the id a request applies one by.
|
||||
repeated LoraAdapter adapters = 12;
|
||||
}
|
||||
|
||||
// RemoveWeights deletes cached model files a node no longer needs.
|
||||
@@ -704,3 +707,10 @@ message UpgradeReply {
|
||||
string version = 2;
|
||||
string error = 3;
|
||||
}
|
||||
|
||||
// LoraAdapter is one adapter: the name clients ask for it by, and its source,
|
||||
// which the agent fetches as it does the model's weights.
|
||||
message LoraAdapter {
|
||||
string name = 1;
|
||||
string source = 2;
|
||||
}
|
||||
Reference in new issue
Block a user