Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cf53904cc5 |
No files matched your search
@@ -42,6 +42,18 @@ What v0.10.5 made settable from the command line is now on the pages too:
|
||||
critical model's card no longer refuses a replica that preemption would make
|
||||
room for.
|
||||
|
||||
### LoRA adapters
|
||||
|
||||
A llama.cpp model can carry `adapters`, each a name and a GGUF LoRA file. The
|
||||
nodes fetch them with the model's weights and llama.cpp loads them all at start,
|
||||
applied to nothing (`--lora-init-without-apply`); a request that names an
|
||||
adapter goes to the model's replicas with `"lora": [{"id": i, "scale": s}]`,
|
||||
so one replica serves the model and all its adapters. Adapter names share the
|
||||
namespace of models, are listed in `/v1/models`, and follow key allowlists on
|
||||
their own — a key allowed an adapter does not reach the model under it. Adapter
|
||||
files are kept from weight cleanup like the model's own. Not yet for vLLM or
|
||||
SGLang profiles, whose adapters are loaded differently.
|
||||
|
||||
### Models served outside the cluster
|
||||
|
||||
A catalog model can be an OpenAI-compatible API elsewhere: `external: {url,
|
||||
|
||||
@@ -2873,8 +2873,11 @@ type EngineProfile struct {
|
||||
Env map[string]string `protobuf:"bytes,9,rep,name=env,proto3" json:"env,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
|
||||
ModelPath string `protobuf:"bytes,10,opt,name=model_path,json=modelPath,proto3" json:"model_path,omitempty"`
|
||||
ServedModelName string `protobuf:"bytes,11,opt,name=served_model_name,json=servedModelName,proto3" json:"served_model_name,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
// adapters are LoRA adapters loaded beside the model, in order: the index
|
||||
// is the id a request applies one by.
|
||||
Adapters []*LoraAdapter `protobuf:"bytes,12,rep,name=adapters,proto3" json:"adapters,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *EngineProfile) Reset() {
|
||||
@@ -2984,6 +2987,13 @@ func (x *EngineProfile) GetServedModelName() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *EngineProfile) GetAdapters() []*LoraAdapter {
|
||||
if x != nil {
|
||||
return x.Adapters
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveWeights deletes cached model files a node no longer needs.
|
||||
//
|
||||
// The paths are computed by the control plane from the catalog, but the agent
|
||||
@@ -3777,6 +3787,60 @@ func (x *UpgradeReply) GetError() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// LoraAdapter is one adapter: the name clients ask for it by, and its source,
|
||||
// which the agent fetches as it does the model's weights.
|
||||
type LoraAdapter struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
|
||||
Source string `protobuf:"bytes,2,opt,name=source,proto3" json:"source,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *LoraAdapter) Reset() {
|
||||
*x = LoraAdapter{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[36]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *LoraAdapter) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*LoraAdapter) ProtoMessage() {}
|
||||
|
||||
func (x *LoraAdapter) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[36]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use LoraAdapter.ProtoReflect.Descriptor instead.
|
||||
func (*LoraAdapter) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{36}
|
||||
}
|
||||
|
||||
func (x *LoraAdapter) GetName() string {
|
||||
if x != nil {
|
||||
return x.Name
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *LoraAdapter) GetSource() string {
|
||||
if x != nil {
|
||||
return x.Source
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var File_llmgw_v1_agent_proto protoreflect.FileDescriptor
|
||||
|
||||
const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
@@ -4066,7 +4130,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"profile_fp\x18\x0f \x01(\tR\tprofileFp\x12\x17\n" +
|
||||
"\aapi_key\x18\x10 \x01(\tR\x06apiKey\x12\x1f\n" +
|
||||
"\vcpu_threads\x18\x11 \x01(\x05R\n" +
|
||||
"cpuThreads\"\xfd\x02\n" +
|
||||
"cpuThreads\"\xb0\x03\n" +
|
||||
"\rEngineProfile\x12\x12\n" +
|
||||
"\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" +
|
||||
"\x06engine\x18\x02 \x01(\tR\x06engine\x12\x1b\n" +
|
||||
@@ -4080,7 +4144,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\n" +
|
||||
"model_path\x18\n" +
|
||||
" \x01(\tR\tmodelPath\x12*\n" +
|
||||
"\x11served_model_name\x18\v \x01(\tR\x0fservedModelName\x1a6\n" +
|
||||
"\x11served_model_name\x18\v \x01(\tR\x0fservedModelName\x121\n" +
|
||||
"\badapters\x18\f \x03(\v2\x15.llmgw.v1.LoraAdapterR\badapters\x1a6\n" +
|
||||
"\bEnvEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"%\n" +
|
||||
@@ -4143,7 +4208,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\n" +
|
||||
"request_id\x18\x01 \x01(\tR\trequestId\x12\x18\n" +
|
||||
"\aversion\x18\x02 \x01(\tR\aversion\x12\x14\n" +
|
||||
"\x05error\x18\x03 \x01(\tR\x05error2O\n" +
|
||||
"\x05error\x18\x03 \x01(\tR\x05error\"9\n" +
|
||||
"\vLoraAdapter\x12\x12\n" +
|
||||
"\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" +
|
||||
"\x06source\x18\x02 \x01(\tR\x06source2O\n" +
|
||||
"\fAgentService\x12?\n" +
|
||||
"\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" +
|
||||
"\fcom.llmgw.v1B\n" +
|
||||
@@ -4162,7 +4230,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
|
||||
}
|
||||
|
||||
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 42)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 43)
|
||||
var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
|
||||
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
|
||||
@@ -4201,12 +4269,13 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(*FabricTestReply)(nil), // 34: llmgw.v1.FabricTestReply
|
||||
(*UpgradeAgent)(nil), // 35: llmgw.v1.UpgradeAgent
|
||||
(*UpgradeReply)(nil), // 36: llmgw.v1.UpgradeReply
|
||||
nil, // 37: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 38: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 39: llmgw.v1.GPU.NicPathsEntry
|
||||
nil, // 40: llmgw.v1.IBPort.CountersEntry
|
||||
nil, // 41: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 42: llmgw.v1.EngineProfile.EnvEntry
|
||||
(*LoraAdapter)(nil), // 37: llmgw.v1.LoraAdapter
|
||||
nil, // 38: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 39: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 40: llmgw.v1.GPU.NicPathsEntry
|
||||
nil, // 41: llmgw.v1.IBPort.CountersEntry
|
||||
nil, // 42: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 43: llmgw.v1.EngineProfile.EnvEntry
|
||||
}
|
||||
var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
|
||||
@@ -4226,24 +4295,24 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
31, // 14: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
|
||||
33, // 15: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest
|
||||
35, // 16: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent
|
||||
37, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
38, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
7, // 18: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
|
||||
8, // 19: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
38, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
39, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
6, // 21: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
|
||||
9, // 22: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
|
||||
5, // 23: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
|
||||
4, // 24: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
|
||||
15, // 25: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
|
||||
0, // 26: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
39, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
|
||||
40, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
|
||||
12, // 28: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
|
||||
11, // 29: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor
|
||||
40, // 30: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
|
||||
41, // 30: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
|
||||
14, // 31: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
|
||||
5, // 32: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
9, // 33: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
41, // 34: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
42, // 34: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
20, // 35: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
|
||||
10, // 36: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
|
||||
13, // 37: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
|
||||
@@ -4254,14 +4323,15 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
0, // 42: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
22, // 43: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
24, // 44: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
42, // 45: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
1, // 46: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 47: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
47, // [47:48] is the sub-list for method output_type
|
||||
46, // [46:47] is the sub-list for method input_type
|
||||
46, // [46:46] is the sub-list for extension type_name
|
||||
46, // [46:46] is the sub-list for extension extendee
|
||||
0, // [0:46] is the sub-list for field type_name
|
||||
43, // 45: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
37, // 46: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter
|
||||
1, // 47: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 48: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
48, // [48:49] is the sub-list for method output_type
|
||||
47, // [47:48] is the sub-list for method input_type
|
||||
47, // [47:47] is the sub-list for extension type_name
|
||||
47, // [47:47] is the sub-list for extension extendee
|
||||
0, // [0:47] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_llmgw_v1_agent_proto_init() }
|
||||
@@ -4296,7 +4366,7 @@ func file_llmgw_v1_agent_proto_init() {
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
|
||||
NumEnums: 1,
|
||||
NumMessages: 42,
|
||||
NumMessages: 43,
|
||||
NumExtensions: 0,
|
||||
NumServices: 1,
|
||||
},
|
||||
|
||||
@@ -572,6 +572,9 @@ message EngineProfile {
|
||||
map<string, string> env = 9;
|
||||
string model_path = 10;
|
||||
string served_model_name = 11;
|
||||
// adapters are LoRA adapters loaded beside the model, in order: the index
|
||||
// is the id a request applies one by.
|
||||
repeated LoraAdapter adapters = 12;
|
||||
}
|
||||
|
||||
// RemoveWeights deletes cached model files a node no longer needs.
|
||||
@@ -704,3 +707,10 @@ message UpgradeReply {
|
||||
string version = 2;
|
||||
string error = 3;
|
||||
}
|
||||
|
||||
// LoraAdapter is one adapter: the name clients ask for it by, and its source,
|
||||
// which the agent fetches as it does the model's weights.
|
||||
message LoraAdapter {
|
||||
string name = 1;
|
||||
string source = 2;
|
||||
}
|
||||
@@ -671,6 +671,34 @@ llmgw ctl explain-placement qwen2.5-coder-3b-instruct
|
||||
which names every candidate and why each was rejected, rather than leaving you
|
||||
to infer it.
|
||||
|
||||
### LoRA adapters
|
||||
|
||||
A llama.cpp model can carry LoRA adapters, each asked for by its own name as
|
||||
though it were a model:
|
||||
|
||||
```yaml
|
||||
- name: qwen2.5-coder-7b-instruct
|
||||
source: http://mirror:8081/qwen2.5-coder-7b-instruct/5d567d26/model.gguf
|
||||
adapters:
|
||||
- name: coder-sql
|
||||
source: http://mirror:8081/coder-sql/v1/adapter.gguf
|
||||
scale: 1.0 # optional; 1 when left out
|
||||
profiles: [...]
|
||||
```
|
||||
|
||||
Every replica of the model fetches every adapter and loads it at start, applied
|
||||
to nothing. A request for `coder-sql` goes to the model's replicas with the
|
||||
adapter applied to it alone; a request for the model gets the model. Adapter
|
||||
names share the namespace of models and aliases, appear in `/v1/models`, and
|
||||
follow key allowlists on their own: a key allowed `coder-sql` reaches that
|
||||
adapter and not the model under it, while a key allowed the model reaches its
|
||||
adapters too. Adding or removing an adapter is a change to the model; running
|
||||
replicas load the new set when restarted (`llmgw ctl restart --model M`).
|
||||
|
||||
The adapter file must be a GGUF LoRA made for this model — llama.cpp's
|
||||
`convert_lora_to_gguf.py` makes one from a PEFT adapter — and not a model with
|
||||
the adapter already merged in, which is what most "LoRA GGUF" repositories hold.
|
||||
|
||||
### Models served outside the cluster
|
||||
|
||||
A model can be an OpenAI-compatible API elsewhere — a hosted provider, or
|
||||
|
||||
@@ -15,6 +15,9 @@ rather than repeated.
|
||||
|
||||
| Key | Type | In |
|
||||
|---|---|---|
|
||||
| `name` | `string` | [Adapter](#adapter) |
|
||||
| `source` | `string` | [Adapter](#adapter) |
|
||||
| `scale` | `float64` | [Adapter](#adapter) |
|
||||
| `enabled` | `bool` | [Admission](#admission) |
|
||||
| `bands` | `[]Band` | [Admission](#admission) |
|
||||
| `default_band` | `string` | [Admission](#admission) |
|
||||
@@ -99,6 +102,7 @@ rather than repeated.
|
||||
| `class` | `string` | [Model](#model) |
|
||||
| `owner` | `string` | [Model](#model) |
|
||||
| `external` | `*External` | [Model](#model) |
|
||||
| `adapters` | `[]Adapter` | [Model](#model) |
|
||||
| `fallbacks` | `[]string` | [Model](#model) |
|
||||
| `routing_profile` | `string` | [Model](#model) |
|
||||
| `slo` | `SLO` | [Model](#model) |
|
||||
@@ -200,6 +204,27 @@ rather than repeated.
|
||||
| `launch_mode` | `string` | [VLLMConfig](#vllmconfig) |
|
||||
| `ray_bin` | `string` | [VLLMConfig](#vllmconfig) |
|
||||
|
||||
## Adapter
|
||||
|
||||
### `name`
|
||||
|
||||
`string`
|
||||
|
||||
Name is what clients ask for, like a model name.
|
||||
|
||||
### `source`
|
||||
|
||||
`string`
|
||||
|
||||
Source is the adapter file (GGUF for llama.cpp), fetched by each node
|
||||
as the model's weights are.
|
||||
|
||||
### `scale`
|
||||
|
||||
`float64`
|
||||
|
||||
Scale is how strongly it is applied; zero takes 1.
|
||||
|
||||
## Admission
|
||||
|
||||
### `enabled`
|
||||
@@ -805,6 +830,14 @@ allowlist.
|
||||
|
||||
Takes a [External](#external) block.
|
||||
|
||||
### `adapters`
|
||||
|
||||
`[]Adapter`
|
||||
|
||||
Adapters are LoRA adapters served on this model's replicas, each asked
|
||||
for by its own name as though it were a model. Every replica loads
|
||||
every adapter, applying one only to a request that names it.
|
||||
|
||||
### `fallbacks`
|
||||
|
||||
`[]string`
|
||||
|
||||
@@ -891,6 +891,17 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
stop()
|
||||
spec.Profile.ModelPath = path
|
||||
}
|
||||
// Adapters are fetched like the model, and before the engine starts: it
|
||||
// loads them at start, and one missing is a replica that answers for the
|
||||
// base model under the adapter's name.
|
||||
for i, ad := range spec.Profile.Adapters {
|
||||
path, err := a.weights().Ensure(ctx, ad.Path)
|
||||
if err != nil {
|
||||
report(supervisor.Failed, fmt.Sprintf("adapter %s: %v", ad.Name, err), nil)
|
||||
return
|
||||
}
|
||||
spec.Profile.Adapters[i].Path = path
|
||||
}
|
||||
|
||||
cmds, err := eng.Commands(spec)
|
||||
if err != nil {
|
||||
|
||||
@@ -40,6 +40,9 @@ type Profile struct {
|
||||
ModelPath string
|
||||
// ServedModelName is the name clients use, which need not match the path.
|
||||
ServedModelName string
|
||||
// Adapters are LoRA adapters to load beside the model, in order: an
|
||||
// adapter's index is the id requests apply it by.
|
||||
Adapters []Adapter
|
||||
}
|
||||
|
||||
// ShardSpec is one shard of one replica on this node.
|
||||
@@ -373,3 +376,10 @@ func quantizationOf(args []string) string {
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Adapter is a LoRA adapter an engine loads beside its model. Path is a
|
||||
// source until the agent has fetched it, a local file after.
|
||||
type Adapter struct {
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
}
|
||||
@@ -144,6 +144,15 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
|
||||
}
|
||||
args = append(args, "--threads", strconv.Itoa(n))
|
||||
}
|
||||
// Loaded at start, applied per request: a request that names an adapter
|
||||
// carries "lora": [{"id": i, "scale": s}], and one that names none is the
|
||||
// base model, since every adapter starts at scale zero.
|
||||
if len(p.Adapters) > 0 {
|
||||
args = append(args, "--lora-init-without-apply")
|
||||
for _, a := range p.Adapters {
|
||||
args = append(args, "--lora", a.Path)
|
||||
}
|
||||
}
|
||||
args = append(args, p.Args...)
|
||||
|
||||
env := baseEnv(s)
|
||||
|
||||
@@ -306,3 +306,20 @@ func TestTheCoresPlacementReservedAreTheThreads(t *testing.T) {
|
||||
t.Errorf("--threads %q; what placement reserved should win over the node's setting", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdaptersAreLoadedAtStartAndAppliedToNoneByDefault(t *testing.T) {
|
||||
s := llamaSpec(nil)
|
||||
s.Profile.Adapters = []Adapter{{Name: "sql", Path: "/w/sql.gguf"}, {Name: "docs", Path: "/w/docs.gguf"}}
|
||||
cmds, err := (&LlamaCpp{}).Commands(s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
args := strings.Join(cmds[0].Args, " ")
|
||||
if !strings.Contains(args, "--lora-init-without-apply --lora /w/sql.gguf --lora /w/docs.gguf") {
|
||||
t.Errorf("args %s: both adapters, in order, loaded unapplied", args)
|
||||
}
|
||||
plain, _ := (&LlamaCpp{}).Commands(llamaSpec(nil))
|
||||
if strings.Contains(strings.Join(plain[0].Args, " "), "--lora") {
|
||||
t.Error("a model with no adapters was given lora flags")
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,11 @@ func (a *API) Orphans() []OrphanView {
|
||||
var wanted []string
|
||||
for _, m := range a.ctl.Catalog().Models() {
|
||||
wanted = append(wanted, m.Source)
|
||||
// A model's adapters are its weights too; collecting them would
|
||||
// leave replicas that cannot restart.
|
||||
for _, ad := range m.Adapters {
|
||||
wanted = append(wanted, ad.Source)
|
||||
}
|
||||
}
|
||||
|
||||
out := make([]OrphanView, 0)
|
||||
|
||||
@@ -189,6 +189,7 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard {
|
||||
GpusPerNode: int32(p.GPUsPerNode), Nodes: int32(p.Nodes),
|
||||
Args: p.Args, Env: p.Env,
|
||||
ModelPath: p.ModelPath, ServedModelName: p.ServedModelName,
|
||||
Adapters: adaptersToWire(p.Adapters),
|
||||
},
|
||||
StartupTimeoutMs: cmd.StartupTimeout.Milliseconds(),
|
||||
Pool: cmd.Pool,
|
||||
@@ -224,6 +225,7 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration
|
||||
GPUsPerNode: int(p.GetGpusPerNode()), Nodes: int(p.GetNodes()),
|
||||
Args: p.GetArgs(), Env: p.GetEnv(),
|
||||
ModelPath: p.GetModelPath(), ServedModelName: p.GetServedModelName(),
|
||||
Adapters: adaptersFromWire(p.GetAdapters()),
|
||||
},
|
||||
}
|
||||
return spec, time.Duration(m.GetStartupTimeoutMs()) * time.Millisecond
|
||||
@@ -292,3 +294,19 @@ func fabricFrom(state, status, cluster, clique, bandwidth string) *discovery.GPU
|
||||
}
|
||||
return &discovery.GPUFabric{State: state, Status: status, ClusterUUID: cluster, CliqueID: clique, Bandwidth: bandwidth}
|
||||
}
|
||||
|
||||
func adaptersToWire(as []engines.Adapter) []*llmgwv1.LoraAdapter {
|
||||
var out []*llmgwv1.LoraAdapter
|
||||
for _, a := range as {
|
||||
out = append(out, &llmgwv1.LoraAdapter{Name: a.Name, Source: a.Path})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func adaptersFromWire(as []*llmgwv1.LoraAdapter) []engines.Adapter {
|
||||
var out []engines.Adapter
|
||||
for _, a := range as {
|
||||
out = append(out, engines.Adapter{Name: a.GetName(), Path: a.GetSource()})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package agentsvc
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
)
|
||||
|
||||
func TestAdaptersTravelToTheAgentInOrder(t *testing.T) {
|
||||
cmd := reconciler.ShardCommand{ReplicaID: "r", ShardID: "s", Spec: engines.ShardSpec{
|
||||
Profile: engines.Profile{Name: "cpu", Engine: "llamacpp", ModelPath: "/m.gguf",
|
||||
Adapters: []engines.Adapter{{Name: "sql", Path: "http://mirror/sql.gguf"}, {Name: "docs", Path: "http://mirror/docs.gguf"}}},
|
||||
}}
|
||||
spec, _ := ShardSpecFromStart(StartShardFromCommand(cmd))
|
||||
got := spec.Profile.Adapters
|
||||
if len(got) != 2 || got[0].Name != "sql" || got[1].Path != "http://mirror/docs.gguf" {
|
||||
t.Errorf("adapters %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -169,7 +169,11 @@ type Model struct {
|
||||
// fallbacks: used only when the cluster has no capacity left. Its
|
||||
// prompts leave the cluster, so which keys may name it is worth an
|
||||
// allowlist.
|
||||
External *External `json:"external,omitempty" koanf:"external"`
|
||||
External *External `json:"external,omitempty" koanf:"external"`
|
||||
// Adapters are LoRA adapters served on this model's replicas, each asked
|
||||
// for by its own name as though it were a model. Every replica loads
|
||||
// every adapter, applying one only to a request that names it.
|
||||
Adapters []Adapter `json:"adapters,omitempty" koanf:"adapters"`
|
||||
Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"`
|
||||
// RoutingProfile selects how the router scores endpoints for this model.
|
||||
RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"`
|
||||
@@ -213,14 +217,17 @@ type Capabilities struct {
|
||||
type Catalog struct {
|
||||
models map[string]Model
|
||||
aliases map[string]string
|
||||
order []string
|
||||
// adapters maps an adapter's name to the model it runs on.
|
||||
adapters map[string]string
|
||||
order []string
|
||||
}
|
||||
|
||||
// New builds a catalog, rejecting definitions that would fail confusingly later.
|
||||
func New(models []Model) (*Catalog, error) {
|
||||
c := &Catalog{
|
||||
models: make(map[string]Model, len(models)),
|
||||
aliases: map[string]string{},
|
||||
models: make(map[string]Model, len(models)),
|
||||
aliases: map[string]string{},
|
||||
adapters: map[string]string{},
|
||||
}
|
||||
|
||||
for i := range models {
|
||||
@@ -249,6 +256,18 @@ func New(models []Model) (*Catalog, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Adapters are named like models, so they share the namespace too.
|
||||
for _, name := range c.order {
|
||||
for _, a := range c.models[name].Adapters {
|
||||
_, isModel := c.models[a.Name]
|
||||
_, isAlias := c.aliases[a.Name]
|
||||
if _, isAdapter := c.adapters[a.Name]; isModel || isAlias || isAdapter {
|
||||
return nil, fmt.Errorf("model %q: adapter %q collides with a model, an alias or another adapter", name, a.Name)
|
||||
}
|
||||
c.adapters[a.Name] = name
|
||||
}
|
||||
}
|
||||
|
||||
// Fallbacks are resolved now rather than at request time: a typo in a
|
||||
// fallback should stop a deployment, not surface during an outage when the
|
||||
// fallback is finally needed.
|
||||
@@ -314,6 +333,9 @@ func (m Model) validate() error {
|
||||
if m.Source == "" {
|
||||
return fmt.Errorf("model %q: source is required", m.Name)
|
||||
}
|
||||
if err := validateAdapters(m); err != nil {
|
||||
return err
|
||||
}
|
||||
if m.External != nil {
|
||||
if err := m.External.validate(m.Name, len(m.Profiles)); err != nil {
|
||||
return err
|
||||
@@ -370,6 +392,23 @@ func (m Model) validate() error {
|
||||
}
|
||||
|
||||
// Resolve maps a client-supplied name or alias onto a model.
|
||||
// ResolveAdapter finds an adapter by its name: the model it runs on, the
|
||||
// adapter, and its index among the model's adapters — which is the id the
|
||||
// engine knows it by.
|
||||
func (c *Catalog) ResolveAdapter(name string) (Model, Adapter, int, bool) {
|
||||
base, ok := c.adapters[name]
|
||||
if !ok {
|
||||
return Model{}, Adapter{}, 0, false
|
||||
}
|
||||
m := c.models[base]
|
||||
for i, a := range m.Adapters {
|
||||
if a.Name == name {
|
||||
return m, a, i, true
|
||||
}
|
||||
}
|
||||
return Model{}, Adapter{}, 0, false
|
||||
}
|
||||
|
||||
func (c *Catalog) Resolve(name string) (Model, bool) {
|
||||
if m, ok := c.models[name]; ok {
|
||||
return m, true
|
||||
@@ -446,6 +485,7 @@ func (m Model) EngineProfile(name string) (engines.Profile, bool) {
|
||||
Env: p.Env,
|
||||
ModelPath: m.Source,
|
||||
ServedModelName: m.Name,
|
||||
Adapters: m.engineAdapters(),
|
||||
}, true
|
||||
}
|
||||
return engines.Profile{}, false
|
||||
@@ -606,3 +646,61 @@ func (e *External) validate(model string, profiles int) error {
|
||||
|
||||
// IsExternal reports whether a model is served outside the cluster.
|
||||
func (m Model) IsExternal() bool { return m.External != nil }
|
||||
|
||||
// Adapter is a LoRA adapter served on a model's replicas.
|
||||
type Adapter struct {
|
||||
// Name is what clients ask for, like a model name.
|
||||
Name string `json:"name" koanf:"name"`
|
||||
// Source is the adapter file (GGUF for llama.cpp), fetched by each node
|
||||
// as the model's weights are.
|
||||
Source string `json:"source" koanf:"source"`
|
||||
// Scale is how strongly it is applied; zero takes 1.
|
||||
Scale float64 `json:"scale,omitempty" koanf:"scale"`
|
||||
}
|
||||
|
||||
// EffectiveScale is the scale an adapter is applied at.
|
||||
func (a Adapter) EffectiveScale() float64 {
|
||||
if a.Scale <= 0 {
|
||||
return 1
|
||||
}
|
||||
return a.Scale
|
||||
}
|
||||
|
||||
func validateAdapters(m Model) error {
|
||||
if len(m.Adapters) == 0 {
|
||||
return nil
|
||||
}
|
||||
if m.External != nil {
|
||||
return fmt.Errorf("model %q: an external model cannot carry adapters", m.Name)
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
// llama.cpp loads adapters at start and applies them per request;
|
||||
// the other engines are not driven that way yet.
|
||||
if p.Engine != "llamacpp" && p.Engine != "sim" {
|
||||
return fmt.Errorf("model %q profile %q: adapters are served by llamacpp profiles only", m.Name, p.Name)
|
||||
}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, a := range m.Adapters {
|
||||
switch {
|
||||
case a.Name == "" || a.Source == "":
|
||||
return fmt.Errorf("model %q: an adapter needs a name and a source", m.Name)
|
||||
case a.Name == m.Name:
|
||||
return fmt.Errorf("model %q: an adapter cannot have the model's own name", m.Name)
|
||||
case seen[a.Name]:
|
||||
return fmt.Errorf("model %q: adapter %q is declared twice", m.Name, a.Name)
|
||||
case a.Scale < 0:
|
||||
return fmt.Errorf("model %q: adapter %q has a negative scale", m.Name, a.Name)
|
||||
}
|
||||
seen[a.Name] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m Model) engineAdapters() []engines.Adapter {
|
||||
var out []engines.Adapter
|
||||
for _, a := range m.Adapters {
|
||||
out = append(out, engines.Adapter{Name: a.Name, Path: a.Source})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -87,6 +87,9 @@ func (s *Service) Sources(ctx context.Context) (map[string][]catalog.Model, erro
|
||||
if m.Source != "" {
|
||||
out[m.Source] = append(out[m.Source], m)
|
||||
}
|
||||
for _, ad := range m.Adapters {
|
||||
out[ad.Source] = append(out[ad.Source], m)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
func adapterModel() catalog.Model {
|
||||
m := catModel("coder", nil)
|
||||
m.Profiles = []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 4}}
|
||||
m.Adapters = []catalog.Adapter{
|
||||
{Name: "coder-sql", Source: "http://mirror/sql.gguf"},
|
||||
{Name: "coder-docs", Source: "http://mirror/docs.gguf", Scale: 0.5},
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestAnAdapterIsAskedForByNameAndAppliedOnItsModel(t *testing.T) {
|
||||
e := newFakeEngine(t, "coder")
|
||||
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"coder"}})
|
||||
cfg.Models = []catalog.Model{adapterModel()}
|
||||
rt := newTestRouter(t, cfg)
|
||||
|
||||
sent := func(model string) map[string]any {
|
||||
t.Helper()
|
||||
rec := post(t, rt.Handler(), map[string]any{"model": model, "messages": []any{}}, nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: status %d: %s", model, rec.Code, rec.Body.String())
|
||||
}
|
||||
var body map[string]any
|
||||
_ = json.Unmarshal(e.last.Load().([]byte), &body)
|
||||
return body
|
||||
}
|
||||
body := sent("coder-docs")
|
||||
if body["model"] != "coder" {
|
||||
t.Errorf("the engine was asked for %v, not the model the adapter runs on", body["model"])
|
||||
}
|
||||
lora, _ := json.Marshal(body["lora"])
|
||||
if string(lora) != `[{"id":1,"scale":0.5}]` {
|
||||
t.Errorf("lora %s, expected the second adapter at its scale", lora)
|
||||
}
|
||||
if body := sent("coder-sql"); func() string { b, _ := json.Marshal(body["lora"]); return string(b) }() != `[{"id":0,"scale":1}]` {
|
||||
t.Errorf("the first adapter, at the default scale: %v", body["lora"])
|
||||
}
|
||||
if body := sent("coder"); body["lora"] != nil {
|
||||
t.Errorf("the base model was sent an adapter: %v", body["lora"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAKeyAllowedAnAdapterReachesThatAdapterOnly(t *testing.T) {
|
||||
cfg := baseConfig()
|
||||
cfg.Models = []catalog.Model{adapterModel()}
|
||||
rt := newTestRouter(t, cfg)
|
||||
key := func(allowed ...string) auth.Principal {
|
||||
return auth.Principal{Limits: store.Effective{AllowedModels: allowed}}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
allowed []string
|
||||
requested string
|
||||
want bool
|
||||
}{
|
||||
{[]string{"coder-sql"}, "coder-sql", true},
|
||||
{[]string{"coder-sql"}, "coder", false},
|
||||
{[]string{"coder-sql"}, "coder-docs", false},
|
||||
{[]string{"coder"}, "coder-sql", true}, // the model allows its adapters
|
||||
{[]string{"coder"}, "coder", true},
|
||||
} {
|
||||
if got := rt.permits(key(c.allowed...), c.requested); got != c.want {
|
||||
t.Errorf("allowed %v, asked for %s: %v", c.allowed, c.requested, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdaptersShareTheNamespaceOfModels(t *testing.T) {
|
||||
a := adapterModel()
|
||||
b := catModel("coder-sql", nil)
|
||||
if _, err := catalog.New([]catalog.Model{a, b}); err == nil {
|
||||
t.Error("an adapter named like another model was accepted")
|
||||
}
|
||||
bad := adapterModel()
|
||||
bad.Profiles = []catalog.Profile{{Name: "gpu", Engine: "vllm", GPUClass: "h100-80g", TP: 1}}
|
||||
if _, err := catalog.New([]catalog.Model{bad}); err == nil {
|
||||
t.Error("adapters on a vLLM profile were accepted")
|
||||
}
|
||||
c, err := catalog.New([]catalog.Model{adapterModel()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, ad, i, ok := c.ResolveAdapter("coder-docs"); !ok || m.Name != "coder" || i != 1 || ad.EffectiveScale() != 0.5 {
|
||||
t.Errorf("resolve: %v %+v %d %v", m.Name, ad, i, ok)
|
||||
}
|
||||
}
|
||||
@@ -426,6 +426,12 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
|
||||
// around the allowlist; matching only the canonical name would silently ignore
|
||||
// an allowlist written in the names clients actually use.
|
||||
func (r *Router) permits(p auth.Principal, requested string) bool {
|
||||
// An adapter named in the allowlist allows that adapter: not its model,
|
||||
// and not the model's other adapters, which the checks below would let
|
||||
// through if the adapter's own name were not asked about first.
|
||||
if _, ok := r.adapter(requested); ok && p.Allows(requested) {
|
||||
return true
|
||||
}
|
||||
canonical, m := r.resolve(requested)
|
||||
if p.Allows(canonical) {
|
||||
return true
|
||||
@@ -1004,3 +1010,24 @@ func withModel(body []byte, name string) []byte {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// withAdapter is body asking for model with a LoRA adapter applied, in
|
||||
// llama.cpp's form: "lora": [{"id": index, "scale": scale}]. Adapters not
|
||||
// listed are applied at zero, so the list is the whole of it.
|
||||
func withAdapter(body []byte, model string, index int, scale float64) []byte {
|
||||
var m map[string]json.RawMessage
|
||||
if err := json.Unmarshal(body, &m); err != nil {
|
||||
return body
|
||||
}
|
||||
name, err1 := json.Marshal(model)
|
||||
lora, err2 := json.Marshal([]map[string]any{{"id": index, "scale": scale}})
|
||||
if err1 != nil || err2 != nil {
|
||||
return body
|
||||
}
|
||||
m["model"], m["lora"] = name, lora
|
||||
out, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return body
|
||||
}
|
||||
return out
|
||||
}
|
||||
+40
-10
@@ -61,6 +61,8 @@ type Router struct {
|
||||
// router's half of not having to. A request takes the map once and works
|
||||
// from it, so a swap mid-request is invisible rather than a race.
|
||||
models atomic.Pointer[map[string]catalog.Model]
|
||||
// adapters are the LoRA adapters by the name clients ask for them by.
|
||||
adapters atomic.Pointer[map[string]adapterRef]
|
||||
|
||||
// extraMetrics are collectors other components publish on /metrics.
|
||||
extraMetrics []prometheus.Collector
|
||||
@@ -315,14 +317,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) {
|
||||
// all. A control plane pushes over the top of this at startup and on every
|
||||
// change after.
|
||||
if len(cfg.Models) > 0 {
|
||||
seed := map[string]catalog.Model{}
|
||||
for _, m := range cfg.Models {
|
||||
seed[m.Name] = m
|
||||
for _, a := range m.Aliases {
|
||||
seed[a] = m
|
||||
}
|
||||
}
|
||||
r.models.Store(&seed)
|
||||
r.storeModels(cfg.Models)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
@@ -505,15 +500,44 @@ func (r *Router) modelMap() map[string]catalog.Model {
|
||||
// The catalog's counterpart to SetEndpoints, and pushed for the same reason:
|
||||
// the control plane owns what should exist, the router needs to know, and
|
||||
// neither has to be in the other's process for that to work.
|
||||
func (r *Router) SetCatalog(c *catalog.Catalog) {
|
||||
func (r *Router) SetCatalog(c *catalog.Catalog) { r.storeModels(c.Models()) }
|
||||
|
||||
// storeModels makes models what names resolve to: each model by its name, its
|
||||
// aliases and its adapters. One place for it, whether the models came from
|
||||
// configuration at start or from the control plane since.
|
||||
func (r *Router) storeModels(models []catalog.Model) {
|
||||
m := map[string]catalog.Model{}
|
||||
for _, entry := range c.Models() {
|
||||
ads := map[string]adapterRef{}
|
||||
for _, entry := range models {
|
||||
m[entry.Name] = entry
|
||||
for _, a := range entry.Aliases {
|
||||
m[a] = entry
|
||||
}
|
||||
// An adapter resolves to the model it runs on, as an alias does, and
|
||||
// is applied to the request on the way there.
|
||||
for i, a := range entry.Adapters {
|
||||
m[a.Name] = entry
|
||||
ads[a.Name] = adapterRef{base: entry.Name, index: i, scale: a.EffectiveScale()}
|
||||
}
|
||||
}
|
||||
r.models.Store(&m)
|
||||
r.adapters.Store(&ads)
|
||||
}
|
||||
|
||||
// adapterRef is where an adapter runs and how it is applied.
|
||||
type adapterRef struct {
|
||||
base string
|
||||
index int
|
||||
scale float64
|
||||
}
|
||||
|
||||
// adapter finds one by the name a client asked for.
|
||||
func (r *Router) adapter(name string) (adapterRef, bool) {
|
||||
if p := r.adapters.Load(); p != nil {
|
||||
a, ok := (*p)[name]
|
||||
return a, ok
|
||||
}
|
||||
return adapterRef{}, false
|
||||
}
|
||||
|
||||
// LastUsed reports when each endpoint was last sent a request, satisfying
|
||||
@@ -719,6 +743,12 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
if dec.Endpoint.UpstreamModel != "" {
|
||||
body = withModel(body, dec.Endpoint.UpstreamModel)
|
||||
}
|
||||
// An adapter asked for by name, served by its model's replicas: the
|
||||
// engine is asked for the model, with the adapter applied. Not on a
|
||||
// fallback to another model, which does not have it.
|
||||
if a, ok := r.adapter(req.Model); ok && a.base == model {
|
||||
body = withAdapter(body, model, a.index, a.scale)
|
||||
}
|
||||
out, perr := r.px.Do(uctx, w, proxy.Request{
|
||||
URL: dec.Endpoint.URL + req.Path,
|
||||
APIKey: dec.Endpoint.APIKey,
|
||||
|
||||
+3
-1
@@ -1497,7 +1497,9 @@ The launch path is now built too:
|
||||
- [x] Weight downloading and cache management: the mirror, agent fetches
|
||||
checked against their digest, and reclaiming what nothing uses
|
||||
- [ ] Prefill/decode disaggregation (§12)
|
||||
- [ ] LoRA adapter routing (affinity to a replica that already has the adapter)
|
||||
- [x] LoRA adapters on llama.cpp: every replica loads a model's adapters and
|
||||
applies one per request, so any replica serves any adapter
|
||||
- [ ] LoRA affinity for engines that load adapters on demand (vLLM)
|
||||
- [x] External provider support: a catalog model served by an OpenAI-compatible
|
||||
API elsewhere, usable as a fallback, under the same allowlists and budgets
|
||||
- [x] Read-only dashboard on the admin listener (nodes, replicas, models,
|
||||
|
||||
Reference in new issue
Block a user