Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cf53904cc5 | ||
|
|
94eec557ad | ||
|
|
d5f02c0a4b |
No files matched your search
@@ -42,6 +42,40 @@ What v0.10.5 made settable from the command line is now on the pages too:
|
||||
critical model's card no longer refuses a replica that preemption would make
|
||||
room for.
|
||||
|
||||
### LoRA adapters
|
||||
|
||||
A llama.cpp model can carry `adapters`, each a name and a GGUF LoRA file. The
|
||||
nodes fetch them with the model's weights and llama.cpp loads them all at start,
|
||||
applied to nothing (`--lora-init-without-apply`); a request that names an
|
||||
adapter goes to the model's replicas with `"lora": [{"id": i, "scale": s}]`,
|
||||
so one replica serves the model and all its adapters. Adapter names share the
|
||||
namespace of models, are listed in `/v1/models`, and follow key allowlists on
|
||||
their own — a key allowed an adapter does not reach the model under it. Adapter
|
||||
files are kept from weight cleanup like the model's own. Not yet for vLLM or
|
||||
SGLang profiles, whose adapters are loaded differently.
|
||||
|
||||
### Models served outside the cluster
|
||||
|
||||
A catalog model can be an OpenAI-compatible API elsewhere: `external: {url,
|
||||
model, api_key_file}` and no profiles. Nothing is placed for it; the router
|
||||
sends its requests to the provider, rewritten to the provider's name for the
|
||||
model and with its key, read from the file on every publish so a rotated key is
|
||||
picked up. Named in another model's `fallbacks`, it is used only when the
|
||||
cluster has no capacity left. It is never scraped for metrics, and never reads
|
||||
as stale. Prices, budgets, limits, usage and key allowlists apply as to any
|
||||
model — and since its prompts leave the cluster, the allowlist is how to decide
|
||||
who may send them (docs/install.md). `ctl models` and the cluster page mark it
|
||||
external.
|
||||
|
||||
### Pool spillover
|
||||
|
||||
A model served from pools of different priority — H100 before A100, say — was
|
||||
routed to the preferred pool however full it was. Now a pool whose replicas are
|
||||
saturated, by the same measure admission uses (queue past its healthy depth, or
|
||||
KV cache past its ceiling), passes requests to the next pool that is not. When
|
||||
every pool is saturated the preferred one is kept: a pool that is just as full
|
||||
gains nothing.
|
||||
|
||||
### Displacement: a higher band is not refused while a lower one waits
|
||||
|
||||
Each band's queue was bounded, and nothing bounded them together, so a full
|
||||
|
||||
@@ -2873,8 +2873,11 @@ type EngineProfile struct {
|
||||
Env map[string]string `protobuf:"bytes,9,rep,name=env,proto3" json:"env,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
|
||||
ModelPath string `protobuf:"bytes,10,opt,name=model_path,json=modelPath,proto3" json:"model_path,omitempty"`
|
||||
ServedModelName string `protobuf:"bytes,11,opt,name=served_model_name,json=servedModelName,proto3" json:"served_model_name,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
// adapters are LoRA adapters loaded beside the model, in order: the index
|
||||
// is the id a request applies one by.
|
||||
Adapters []*LoraAdapter `protobuf:"bytes,12,rep,name=adapters,proto3" json:"adapters,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *EngineProfile) Reset() {
|
||||
@@ -2984,6 +2987,13 @@ func (x *EngineProfile) GetServedModelName() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *EngineProfile) GetAdapters() []*LoraAdapter {
|
||||
if x != nil {
|
||||
return x.Adapters
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveWeights deletes cached model files a node no longer needs.
|
||||
//
|
||||
// The paths are computed by the control plane from the catalog, but the agent
|
||||
@@ -3777,6 +3787,60 @@ func (x *UpgradeReply) GetError() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// LoraAdapter is one adapter: the name clients ask for it by, and its source,
|
||||
// which the agent fetches as it does the model's weights.
|
||||
type LoraAdapter struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
|
||||
Source string `protobuf:"bytes,2,opt,name=source,proto3" json:"source,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *LoraAdapter) Reset() {
|
||||
*x = LoraAdapter{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[36]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *LoraAdapter) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*LoraAdapter) ProtoMessage() {}
|
||||
|
||||
func (x *LoraAdapter) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[36]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use LoraAdapter.ProtoReflect.Descriptor instead.
|
||||
func (*LoraAdapter) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{36}
|
||||
}
|
||||
|
||||
func (x *LoraAdapter) GetName() string {
|
||||
if x != nil {
|
||||
return x.Name
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *LoraAdapter) GetSource() string {
|
||||
if x != nil {
|
||||
return x.Source
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var File_llmgw_v1_agent_proto protoreflect.FileDescriptor
|
||||
|
||||
const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
@@ -4066,7 +4130,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"profile_fp\x18\x0f \x01(\tR\tprofileFp\x12\x17\n" +
|
||||
"\aapi_key\x18\x10 \x01(\tR\x06apiKey\x12\x1f\n" +
|
||||
"\vcpu_threads\x18\x11 \x01(\x05R\n" +
|
||||
"cpuThreads\"\xfd\x02\n" +
|
||||
"cpuThreads\"\xb0\x03\n" +
|
||||
"\rEngineProfile\x12\x12\n" +
|
||||
"\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" +
|
||||
"\x06engine\x18\x02 \x01(\tR\x06engine\x12\x1b\n" +
|
||||
@@ -4080,7 +4144,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\n" +
|
||||
"model_path\x18\n" +
|
||||
" \x01(\tR\tmodelPath\x12*\n" +
|
||||
"\x11served_model_name\x18\v \x01(\tR\x0fservedModelName\x1a6\n" +
|
||||
"\x11served_model_name\x18\v \x01(\tR\x0fservedModelName\x121\n" +
|
||||
"\badapters\x18\f \x03(\v2\x15.llmgw.v1.LoraAdapterR\badapters\x1a6\n" +
|
||||
"\bEnvEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"%\n" +
|
||||
@@ -4143,7 +4208,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\n" +
|
||||
"request_id\x18\x01 \x01(\tR\trequestId\x12\x18\n" +
|
||||
"\aversion\x18\x02 \x01(\tR\aversion\x12\x14\n" +
|
||||
"\x05error\x18\x03 \x01(\tR\x05error2O\n" +
|
||||
"\x05error\x18\x03 \x01(\tR\x05error\"9\n" +
|
||||
"\vLoraAdapter\x12\x12\n" +
|
||||
"\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" +
|
||||
"\x06source\x18\x02 \x01(\tR\x06source2O\n" +
|
||||
"\fAgentService\x12?\n" +
|
||||
"\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" +
|
||||
"\fcom.llmgw.v1B\n" +
|
||||
@@ -4162,7 +4230,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
|
||||
}
|
||||
|
||||
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 42)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 43)
|
||||
var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
|
||||
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
|
||||
@@ -4201,12 +4269,13 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(*FabricTestReply)(nil), // 34: llmgw.v1.FabricTestReply
|
||||
(*UpgradeAgent)(nil), // 35: llmgw.v1.UpgradeAgent
|
||||
(*UpgradeReply)(nil), // 36: llmgw.v1.UpgradeReply
|
||||
nil, // 37: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 38: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 39: llmgw.v1.GPU.NicPathsEntry
|
||||
nil, // 40: llmgw.v1.IBPort.CountersEntry
|
||||
nil, // 41: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 42: llmgw.v1.EngineProfile.EnvEntry
|
||||
(*LoraAdapter)(nil), // 37: llmgw.v1.LoraAdapter
|
||||
nil, // 38: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 39: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 40: llmgw.v1.GPU.NicPathsEntry
|
||||
nil, // 41: llmgw.v1.IBPort.CountersEntry
|
||||
nil, // 42: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 43: llmgw.v1.EngineProfile.EnvEntry
|
||||
}
|
||||
var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
|
||||
@@ -4226,24 +4295,24 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
31, // 14: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
|
||||
33, // 15: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest
|
||||
35, // 16: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent
|
||||
37, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
38, // 17: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
7, // 18: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
|
||||
8, // 19: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
38, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
39, // 20: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
6, // 21: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
|
||||
9, // 22: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
|
||||
5, // 23: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
|
||||
4, // 24: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
|
||||
15, // 25: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
|
||||
0, // 26: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
39, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
|
||||
40, // 27: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
|
||||
12, // 28: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
|
||||
11, // 29: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor
|
||||
40, // 30: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
|
||||
41, // 30: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
|
||||
14, // 31: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
|
||||
5, // 32: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
9, // 33: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
41, // 34: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
42, // 34: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
20, // 35: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
|
||||
10, // 36: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
|
||||
13, // 37: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
|
||||
@@ -4254,14 +4323,15 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
0, // 42: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
22, // 43: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
24, // 44: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
42, // 45: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
1, // 46: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 47: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
47, // [47:48] is the sub-list for method output_type
|
||||
46, // [46:47] is the sub-list for method input_type
|
||||
46, // [46:46] is the sub-list for extension type_name
|
||||
46, // [46:46] is the sub-list for extension extendee
|
||||
0, // [0:46] is the sub-list for field type_name
|
||||
43, // 45: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
37, // 46: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter
|
||||
1, // 47: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 48: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
48, // [48:49] is the sub-list for method output_type
|
||||
47, // [47:48] is the sub-list for method input_type
|
||||
47, // [47:47] is the sub-list for extension type_name
|
||||
47, // [47:47] is the sub-list for extension extendee
|
||||
0, // [0:47] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_llmgw_v1_agent_proto_init() }
|
||||
@@ -4296,7 +4366,7 @@ func file_llmgw_v1_agent_proto_init() {
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
|
||||
NumEnums: 1,
|
||||
NumMessages: 42,
|
||||
NumMessages: 43,
|
||||
NumExtensions: 0,
|
||||
NumServices: 1,
|
||||
},
|
||||
|
||||
@@ -572,6 +572,9 @@ message EngineProfile {
|
||||
map<string, string> env = 9;
|
||||
string model_path = 10;
|
||||
string served_model_name = 11;
|
||||
// adapters are LoRA adapters loaded beside the model, in order: the index
|
||||
// is the id a request applies one by.
|
||||
repeated LoraAdapter adapters = 12;
|
||||
}
|
||||
|
||||
// RemoveWeights deletes cached model files a node no longer needs.
|
||||
@@ -704,3 +707,10 @@ message UpgradeReply {
|
||||
string version = 2;
|
||||
string error = 3;
|
||||
}
|
||||
|
||||
// LoraAdapter is one adapter: the name clients ask for it by, and its source,
|
||||
// which the agent fetches as it does the model's weights.
|
||||
message LoraAdapter {
|
||||
string name = 1;
|
||||
string source = 2;
|
||||
}
|
||||
@@ -671,6 +671,68 @@ llmgw ctl explain-placement qwen2.5-coder-3b-instruct
|
||||
which names every candidate and why each was rejected, rather than leaving you
|
||||
to infer it.
|
||||
|
||||
### LoRA adapters
|
||||
|
||||
A llama.cpp model can carry LoRA adapters, each asked for by its own name as
|
||||
though it were a model:
|
||||
|
||||
```yaml
|
||||
- name: qwen2.5-coder-7b-instruct
|
||||
source: http://mirror:8081/qwen2.5-coder-7b-instruct/5d567d26/model.gguf
|
||||
adapters:
|
||||
- name: coder-sql
|
||||
source: http://mirror:8081/coder-sql/v1/adapter.gguf
|
||||
scale: 1.0 # optional; 1 when left out
|
||||
profiles: [...]
|
||||
```
|
||||
|
||||
Every replica of the model fetches every adapter and loads it at start, applied
|
||||
to nothing. A request for `coder-sql` goes to the model's replicas with the
|
||||
adapter applied to it alone; a request for the model gets the model. Adapter
|
||||
names share the namespace of models and aliases, appear in `/v1/models`, and
|
||||
follow key allowlists on their own: a key allowed `coder-sql` reaches that
|
||||
adapter and not the model under it, while a key allowed the model reaches its
|
||||
adapters too. Adding or removing an adapter is a change to the model; running
|
||||
replicas load the new set when restarted (`llmgw ctl restart --model M`).
|
||||
|
||||
The adapter file must be a GGUF LoRA made for this model — llama.cpp's
|
||||
`convert_lora_to_gguf.py` makes one from a PEFT adapter — and not a model with
|
||||
the adapter already merged in, which is what most "LoRA GGUF" repositories hold.
|
||||
|
||||
### Models served outside the cluster
|
||||
|
||||
A model can be an OpenAI-compatible API elsewhere — a hosted provider, or
|
||||
another gateway — rather than replicas here. Nothing is placed for it; the
|
||||
router sends its requests on, with the provider's key and the provider's name
|
||||
for the model:
|
||||
|
||||
```yaml
|
||||
- name: cloud-mini
|
||||
source: external
|
||||
external:
|
||||
url: https://api.openai.com # without /v1
|
||||
model: gpt-4o-mini # what the provider calls it
|
||||
api_key_file: /etc/llmgw/secrets/openai.key
|
||||
```
|
||||
|
||||
Named in another model's `fallbacks`, it is used only when the cluster has no
|
||||
capacity left for that model:
|
||||
|
||||
```yaml
|
||||
- name: qwen2.5-coder-7b-instruct
|
||||
fallbacks: [cloud-mini]
|
||||
...
|
||||
```
|
||||
|
||||
The key is read from its file on the control plane, never stored in the
|
||||
catalog, and a key rotated in the file is used within seconds. A model whose key
|
||||
file cannot be read is not offered. Everything else applies as to any model:
|
||||
prices and budgets, rate limits, usage records, and key allowlists.
|
||||
|
||||
**Its prompts leave the cluster.** Decide who may send them: a key or project
|
||||
whose allowlist leaves the external model out cannot reach it, as a fallback or
|
||||
by name. The cluster page and `ctl models` mark it `external`.
|
||||
|
||||
### How many replicas, and how much each one carries
|
||||
|
||||
Replica count is changed while the cluster runs, without editing the
|
||||
|
||||
@@ -15,6 +15,9 @@ rather than repeated.
|
||||
|
||||
| Key | Type | In |
|
||||
|---|---|---|
|
||||
| `name` | `string` | [Adapter](#adapter) |
|
||||
| `source` | `string` | [Adapter](#adapter) |
|
||||
| `scale` | `float64` | [Adapter](#adapter) |
|
||||
| `enabled` | `bool` | [Admission](#admission) |
|
||||
| `bands` | `[]Band` | [Admission](#admission) |
|
||||
| `default_band` | `string` | [Admission](#admission) |
|
||||
@@ -78,6 +81,9 @@ rather than repeated.
|
||||
| `pool` | `string` | [Endpoint](#endpoint) |
|
||||
| `weight` | `float64` | [Endpoint](#endpoint) |
|
||||
| `priority` | `int` | [Endpoint](#endpoint) |
|
||||
| `url` | `string` | [External](#external) |
|
||||
| `model` | `string` | [External](#external) |
|
||||
| `api_key_file` | `string` | [External](#external) |
|
||||
| `contact` | `string` | [Landing](#landing) |
|
||||
| `notice` | `string` | [Landing](#landing) |
|
||||
| `bin` | `string` | [LlamaCppConfig](#llamacppconfig) |
|
||||
@@ -95,6 +101,8 @@ rather than repeated.
|
||||
| `scaling` | `Scaling` | [Model](#model) |
|
||||
| `class` | `string` | [Model](#model) |
|
||||
| `owner` | `string` | [Model](#model) |
|
||||
| `external` | `*External` | [Model](#model) |
|
||||
| `adapters` | `[]Adapter` | [Model](#model) |
|
||||
| `fallbacks` | `[]string` | [Model](#model) |
|
||||
| `routing_profile` | `string` | [Model](#model) |
|
||||
| `slo` | `SLO` | [Model](#model) |
|
||||
@@ -196,6 +204,27 @@ rather than repeated.
|
||||
| `launch_mode` | `string` | [VLLMConfig](#vllmconfig) |
|
||||
| `ray_bin` | `string` | [VLLMConfig](#vllmconfig) |
|
||||
|
||||
## Adapter
|
||||
|
||||
### `name`
|
||||
|
||||
`string`
|
||||
|
||||
Name is what clients ask for, like a model name.
|
||||
|
||||
### `source`
|
||||
|
||||
`string`
|
||||
|
||||
Source is the adapter file (GGUF for llama.cpp), fetched by each node
|
||||
as the model's weights are.
|
||||
|
||||
### `scale`
|
||||
|
||||
`float64`
|
||||
|
||||
Scale is how strongly it is applied; zero takes 1.
|
||||
|
||||
## Admission
|
||||
|
||||
### `enabled`
|
||||
@@ -616,6 +645,28 @@ called. Empty means vLLM's names.
|
||||
|
||||
`int`
|
||||
|
||||
## External
|
||||
|
||||
### `url`
|
||||
|
||||
`string`
|
||||
|
||||
URL is the provider's base address, without /v1: https://api.openai.com.
|
||||
|
||||
### `model`
|
||||
|
||||
`string`
|
||||
|
||||
Model is what the provider calls it; requests are rewritten to it.
|
||||
|
||||
### `api_key_file`
|
||||
|
||||
`string`
|
||||
|
||||
APIKeyFile holds the provider's key, on the control plane. A file
|
||||
rather than the key itself, so the key is in neither the catalog, the
|
||||
store nor the audit log; empty for a provider that asks for none.
|
||||
|
||||
## Landing
|
||||
|
||||
### `contact`
|
||||
@@ -765,6 +816,28 @@ Owner is the part of the tenancy tree whose capacity quota this model
|
||||
counts against, written org:ID, team:ID or project:ID. Empty is shared:
|
||||
held by the cluster, counted against nobody.
|
||||
|
||||
### `external`
|
||||
|
||||
`*External`
|
||||
|
||||
External is a model served by someone else — an OpenAI-compatible API
|
||||
outside the cluster — rather than by replicas placed here. Such a model
|
||||
has no profiles and nothing is placed for it; the router sends its
|
||||
requests to the provider. Most useful named in another model's
|
||||
fallbacks: used only when the cluster has no capacity left. Its
|
||||
prompts leave the cluster, so which keys may name it is worth an
|
||||
allowlist.
|
||||
|
||||
Takes a [External](#external) block.
|
||||
|
||||
### `adapters`
|
||||
|
||||
`[]Adapter`
|
||||
|
||||
Adapters are LoRA adapters served on this model's replicas, each asked
|
||||
for by its own name as though it were a model. Every replica loads
|
||||
every adapter, applying one only to a request that names it.
|
||||
|
||||
### `fallbacks`
|
||||
|
||||
`[]string`
|
||||
|
||||
@@ -891,6 +891,17 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
stop()
|
||||
spec.Profile.ModelPath = path
|
||||
}
|
||||
// Adapters are fetched like the model, and before the engine starts: it
|
||||
// loads them at start, and one missing is a replica that answers for the
|
||||
// base model under the adapter's name.
|
||||
for i, ad := range spec.Profile.Adapters {
|
||||
path, err := a.weights().Ensure(ctx, ad.Path)
|
||||
if err != nil {
|
||||
report(supervisor.Failed, fmt.Sprintf("adapter %s: %v", ad.Name, err), nil)
|
||||
return
|
||||
}
|
||||
spec.Profile.Adapters[i].Path = path
|
||||
}
|
||||
|
||||
cmds, err := eng.Commands(spec)
|
||||
if err != nil {
|
||||
|
||||
@@ -40,6 +40,9 @@ type Profile struct {
|
||||
ModelPath string
|
||||
// ServedModelName is the name clients use, which need not match the path.
|
||||
ServedModelName string
|
||||
// Adapters are LoRA adapters to load beside the model, in order: an
|
||||
// adapter's index is the id requests apply it by.
|
||||
Adapters []Adapter
|
||||
}
|
||||
|
||||
// ShardSpec is one shard of one replica on this node.
|
||||
@@ -373,3 +376,10 @@ func quantizationOf(args []string) string {
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Adapter is a LoRA adapter an engine loads beside its model. Path is a
|
||||
// source until the agent has fetched it, a local file after.
|
||||
type Adapter struct {
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
}
|
||||
@@ -144,6 +144,15 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
|
||||
}
|
||||
args = append(args, "--threads", strconv.Itoa(n))
|
||||
}
|
||||
// Loaded at start, applied per request: a request that names an adapter
|
||||
// carries "lora": [{"id": i, "scale": s}], and one that names none is the
|
||||
// base model, since every adapter starts at scale zero.
|
||||
if len(p.Adapters) > 0 {
|
||||
args = append(args, "--lora-init-without-apply")
|
||||
for _, a := range p.Adapters {
|
||||
args = append(args, "--lora", a.Path)
|
||||
}
|
||||
}
|
||||
args = append(args, p.Args...)
|
||||
|
||||
env := baseEnv(s)
|
||||
|
||||
@@ -306,3 +306,20 @@ func TestTheCoresPlacementReservedAreTheThreads(t *testing.T) {
|
||||
t.Errorf("--threads %q; what placement reserved should win over the node's setting", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdaptersAreLoadedAtStartAndAppliedToNoneByDefault(t *testing.T) {
|
||||
s := llamaSpec(nil)
|
||||
s.Profile.Adapters = []Adapter{{Name: "sql", Path: "/w/sql.gguf"}, {Name: "docs", Path: "/w/docs.gguf"}}
|
||||
cmds, err := (&LlamaCpp{}).Commands(s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
args := strings.Join(cmds[0].Args, " ")
|
||||
if !strings.Contains(args, "--lora-init-without-apply --lora /w/sql.gguf --lora /w/docs.gguf") {
|
||||
t.Errorf("args %s: both adapters, in order, loaded unapplied", args)
|
||||
}
|
||||
plain, _ := (&LlamaCpp{}).Commands(llamaSpec(nil))
|
||||
if strings.Contains(strings.Join(plain[0].Args, " "), "--lora") {
|
||||
t.Error("a model with no adapters was given lora flags")
|
||||
}
|
||||
}
|
||||
@@ -254,6 +254,8 @@ func modelStatus(m adminapi.ModelView) string {
|
||||
switch {
|
||||
case m.Parked:
|
||||
return "parked"
|
||||
case m.External != "":
|
||||
return "external: " + m.External
|
||||
case m.HeldFor != "":
|
||||
return "gave room to " + m.HeldFor
|
||||
case m.Unplaceable:
|
||||
|
||||
@@ -10,16 +10,18 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
@@ -312,8 +314,11 @@ type ModelView struct {
|
||||
Warnings []string `json:"warnings,omitempty"`
|
||||
// Class is its capacity class; Preempted the replicas it lost to a higher
|
||||
// class; HeldFor the model it is giving room to now, if any.
|
||||
Class string `json:"class"`
|
||||
Owner string `json:"owner,omitempty"`
|
||||
Class string `json:"class"`
|
||||
Owner string `json:"owner,omitempty"`
|
||||
// External is where a model served outside the cluster goes, provider
|
||||
// host and its name for the model; empty for one served here.
|
||||
External string `json:"external,omitempty"`
|
||||
Preempted int `json:"preempted,omitempty"`
|
||||
HeldFor string `json:"held_for,omitempty"`
|
||||
}
|
||||
@@ -569,6 +574,7 @@ func (a *API) Models() []ModelView {
|
||||
Desired: h.Desired, Failures: h.Failures,
|
||||
Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable,
|
||||
Class: m.EffectiveClass(), Owner: m.Owner, Preempted: h.Preempted, HeldFor: h.HeldFor,
|
||||
External: externalLabel(m),
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
v.Profiles = append(v.Profiles, p.Name)
|
||||
@@ -1443,3 +1449,15 @@ func reservationWarnings(m catalog.Model, measuredGB float64) []string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// externalLabel is "api.openai.com · gpt-4o-mini" for a model served elsewhere.
|
||||
func externalLabel(m catalog.Model) string {
|
||||
if !m.IsExternal() {
|
||||
return ""
|
||||
}
|
||||
host := m.External.URL
|
||||
if u, err := url.Parse(host); err == nil && u.Host != "" {
|
||||
host = u.Host
|
||||
}
|
||||
return host + " · " + m.External.Model
|
||||
}
|
||||
@@ -31,6 +31,11 @@ func (a *API) Orphans() []OrphanView {
|
||||
var wanted []string
|
||||
for _, m := range a.ctl.Catalog().Models() {
|
||||
wanted = append(wanted, m.Source)
|
||||
// A model's adapters are its weights too; collecting them would
|
||||
// leave replicas that cannot restart.
|
||||
for _, ad := range m.Adapters {
|
||||
wanted = append(wanted, ad.Source)
|
||||
}
|
||||
}
|
||||
|
||||
out := make([]OrphanView, 0)
|
||||
|
||||
@@ -189,6 +189,7 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard {
|
||||
GpusPerNode: int32(p.GPUsPerNode), Nodes: int32(p.Nodes),
|
||||
Args: p.Args, Env: p.Env,
|
||||
ModelPath: p.ModelPath, ServedModelName: p.ServedModelName,
|
||||
Adapters: adaptersToWire(p.Adapters),
|
||||
},
|
||||
StartupTimeoutMs: cmd.StartupTimeout.Milliseconds(),
|
||||
Pool: cmd.Pool,
|
||||
@@ -224,6 +225,7 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration
|
||||
GPUsPerNode: int(p.GetGpusPerNode()), Nodes: int(p.GetNodes()),
|
||||
Args: p.GetArgs(), Env: p.GetEnv(),
|
||||
ModelPath: p.GetModelPath(), ServedModelName: p.GetServedModelName(),
|
||||
Adapters: adaptersFromWire(p.GetAdapters()),
|
||||
},
|
||||
}
|
||||
return spec, time.Duration(m.GetStartupTimeoutMs()) * time.Millisecond
|
||||
@@ -292,3 +294,19 @@ func fabricFrom(state, status, cluster, clique, bandwidth string) *discovery.GPU
|
||||
}
|
||||
return &discovery.GPUFabric{State: state, Status: status, ClusterUUID: cluster, CliqueID: clique, Bandwidth: bandwidth}
|
||||
}
|
||||
|
||||
func adaptersToWire(as []engines.Adapter) []*llmgwv1.LoraAdapter {
|
||||
var out []*llmgwv1.LoraAdapter
|
||||
for _, a := range as {
|
||||
out = append(out, &llmgwv1.LoraAdapter{Name: a.Name, Source: a.Path})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func adaptersFromWire(as []*llmgwv1.LoraAdapter) []engines.Adapter {
|
||||
var out []engines.Adapter
|
||||
for _, a := range as {
|
||||
out = append(out, engines.Adapter{Name: a.GetName(), Path: a.GetSource()})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package agentsvc
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
)
|
||||
|
||||
func TestAdaptersTravelToTheAgentInOrder(t *testing.T) {
|
||||
cmd := reconciler.ShardCommand{ReplicaID: "r", ShardID: "s", Spec: engines.ShardSpec{
|
||||
Profile: engines.Profile{Name: "cpu", Engine: "llamacpp", ModelPath: "/m.gguf",
|
||||
Adapters: []engines.Adapter{{Name: "sql", Path: "http://mirror/sql.gguf"}, {Name: "docs", Path: "http://mirror/docs.gguf"}}},
|
||||
}}
|
||||
spec, _ := ShardSpecFromStart(StartShardFromCommand(cmd))
|
||||
got := spec.Profile.Adapters
|
||||
if len(got) != 2 || got[0].Name != "sql" || got[1].Path != "http://mirror/docs.gguf" {
|
||||
t.Errorf("adapters %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -160,8 +161,20 @@ type Model struct {
|
||||
// Owner is the part of the tenancy tree whose capacity quota this model
|
||||
// counts against, written org:ID, team:ID or project:ID. Empty is shared:
|
||||
// held by the cluster, counted against nobody.
|
||||
Owner string `json:"owner,omitempty" koanf:"owner"`
|
||||
Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"`
|
||||
Owner string `json:"owner,omitempty" koanf:"owner"`
|
||||
// External is a model served by someone else — an OpenAI-compatible API
|
||||
// outside the cluster — rather than by replicas placed here. Such a model
|
||||
// has no profiles and nothing is placed for it; the router sends its
|
||||
// requests to the provider. Most useful named in another model's
|
||||
// fallbacks: used only when the cluster has no capacity left. Its
|
||||
// prompts leave the cluster, so which keys may name it is worth an
|
||||
// allowlist.
|
||||
External *External `json:"external,omitempty" koanf:"external"`
|
||||
// Adapters are LoRA adapters served on this model's replicas, each asked
|
||||
// for by its own name as though it were a model. Every replica loads
|
||||
// every adapter, applying one only to a request that names it.
|
||||
Adapters []Adapter `json:"adapters,omitempty" koanf:"adapters"`
|
||||
Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"`
|
||||
// RoutingProfile selects how the router scores endpoints for this model.
|
||||
RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"`
|
||||
SLO SLO `json:"slo,omitempty" koanf:"slo"`
|
||||
@@ -204,14 +217,17 @@ type Capabilities struct {
|
||||
type Catalog struct {
|
||||
models map[string]Model
|
||||
aliases map[string]string
|
||||
order []string
|
||||
// adapters maps an adapter's name to the model it runs on.
|
||||
adapters map[string]string
|
||||
order []string
|
||||
}
|
||||
|
||||
// New builds a catalog, rejecting definitions that would fail confusingly later.
|
||||
func New(models []Model) (*Catalog, error) {
|
||||
c := &Catalog{
|
||||
models: make(map[string]Model, len(models)),
|
||||
aliases: map[string]string{},
|
||||
models: make(map[string]Model, len(models)),
|
||||
aliases: map[string]string{},
|
||||
adapters: map[string]string{},
|
||||
}
|
||||
|
||||
for i := range models {
|
||||
@@ -240,6 +256,18 @@ func New(models []Model) (*Catalog, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Adapters are named like models, so they share the namespace too.
|
||||
for _, name := range c.order {
|
||||
for _, a := range c.models[name].Adapters {
|
||||
_, isModel := c.models[a.Name]
|
||||
_, isAlias := c.aliases[a.Name]
|
||||
if _, isAdapter := c.adapters[a.Name]; isModel || isAlias || isAdapter {
|
||||
return nil, fmt.Errorf("model %q: adapter %q collides with a model, an alias or another adapter", name, a.Name)
|
||||
}
|
||||
c.adapters[a.Name] = name
|
||||
}
|
||||
}
|
||||
|
||||
// Fallbacks are resolved now rather than at request time: a typo in a
|
||||
// fallback should stop a deployment, not surface during an outage when the
|
||||
// fallback is finally needed.
|
||||
@@ -305,7 +333,14 @@ func (m Model) validate() error {
|
||||
if m.Source == "" {
|
||||
return fmt.Errorf("model %q: source is required", m.Name)
|
||||
}
|
||||
if len(m.Profiles) == 0 {
|
||||
if err := validateAdapters(m); err != nil {
|
||||
return err
|
||||
}
|
||||
if m.External != nil {
|
||||
if err := m.External.validate(m.Name, len(m.Profiles)); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if len(m.Profiles) == 0 {
|
||||
return fmt.Errorf("model %q: at least one profile is required", m.Name)
|
||||
}
|
||||
if m.Owner != "" && !validOwner(m.Owner) {
|
||||
@@ -357,6 +392,23 @@ func (m Model) validate() error {
|
||||
}
|
||||
|
||||
// Resolve maps a client-supplied name or alias onto a model.
|
||||
// ResolveAdapter finds an adapter by its name: the model it runs on, the
|
||||
// adapter, and its index among the model's adapters — which is the id the
|
||||
// engine knows it by.
|
||||
func (c *Catalog) ResolveAdapter(name string) (Model, Adapter, int, bool) {
|
||||
base, ok := c.adapters[name]
|
||||
if !ok {
|
||||
return Model{}, Adapter{}, 0, false
|
||||
}
|
||||
m := c.models[base]
|
||||
for i, a := range m.Adapters {
|
||||
if a.Name == name {
|
||||
return m, a, i, true
|
||||
}
|
||||
}
|
||||
return Model{}, Adapter{}, 0, false
|
||||
}
|
||||
|
||||
func (c *Catalog) Resolve(name string) (Model, bool) {
|
||||
if m, ok := c.models[name]; ok {
|
||||
return m, true
|
||||
@@ -433,6 +485,7 @@ func (m Model) EngineProfile(name string) (engines.Profile, bool) {
|
||||
Env: p.Env,
|
||||
ModelPath: m.Source,
|
||||
ServedModelName: m.Name,
|
||||
Adapters: m.engineAdapters(),
|
||||
}, true
|
||||
}
|
||||
return engines.Profile{}, false
|
||||
@@ -564,3 +617,90 @@ func validOwner(o string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// External is a provider outside the cluster that serves a model.
|
||||
type External struct {
|
||||
// URL is the provider's base address, without /v1: https://api.openai.com.
|
||||
URL string `json:"url" koanf:"url"`
|
||||
// Model is what the provider calls it; requests are rewritten to it.
|
||||
Model string `json:"model" koanf:"model"`
|
||||
// APIKeyFile holds the provider's key, on the control plane. A file
|
||||
// rather than the key itself, so the key is in neither the catalog, the
|
||||
// store nor the audit log; empty for a provider that asks for none.
|
||||
APIKeyFile string `json:"api_key_file,omitempty" koanf:"api_key_file"`
|
||||
}
|
||||
|
||||
func (e *External) validate(model string, profiles int) error {
|
||||
if profiles > 0 {
|
||||
return fmt.Errorf("model %q: an external model is served elsewhere and has no profiles", model)
|
||||
}
|
||||
u, err := url.Parse(e.URL)
|
||||
if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" {
|
||||
return fmt.Errorf("model %q: external url %q is not an http(s) address", model, e.URL)
|
||||
}
|
||||
if strings.TrimSpace(e.Model) == "" {
|
||||
return fmt.Errorf("model %q: external model needs the name the provider knows it by", model)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsExternal reports whether a model is served outside the cluster.
|
||||
func (m Model) IsExternal() bool { return m.External != nil }
|
||||
|
||||
// Adapter is a LoRA adapter served on a model's replicas.
|
||||
type Adapter struct {
|
||||
// Name is what clients ask for, like a model name.
|
||||
Name string `json:"name" koanf:"name"`
|
||||
// Source is the adapter file (GGUF for llama.cpp), fetched by each node
|
||||
// as the model's weights are.
|
||||
Source string `json:"source" koanf:"source"`
|
||||
// Scale is how strongly it is applied; zero takes 1.
|
||||
Scale float64 `json:"scale,omitempty" koanf:"scale"`
|
||||
}
|
||||
|
||||
// EffectiveScale is the scale an adapter is applied at.
|
||||
func (a Adapter) EffectiveScale() float64 {
|
||||
if a.Scale <= 0 {
|
||||
return 1
|
||||
}
|
||||
return a.Scale
|
||||
}
|
||||
|
||||
func validateAdapters(m Model) error {
|
||||
if len(m.Adapters) == 0 {
|
||||
return nil
|
||||
}
|
||||
if m.External != nil {
|
||||
return fmt.Errorf("model %q: an external model cannot carry adapters", m.Name)
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
// llama.cpp loads adapters at start and applies them per request;
|
||||
// the other engines are not driven that way yet.
|
||||
if p.Engine != "llamacpp" && p.Engine != "sim" {
|
||||
return fmt.Errorf("model %q profile %q: adapters are served by llamacpp profiles only", m.Name, p.Name)
|
||||
}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, a := range m.Adapters {
|
||||
switch {
|
||||
case a.Name == "" || a.Source == "":
|
||||
return fmt.Errorf("model %q: an adapter needs a name and a source", m.Name)
|
||||
case a.Name == m.Name:
|
||||
return fmt.Errorf("model %q: an adapter cannot have the model's own name", m.Name)
|
||||
case seen[a.Name]:
|
||||
return fmt.Errorf("model %q: adapter %q is declared twice", m.Name, a.Name)
|
||||
case a.Scale < 0:
|
||||
return fmt.Errorf("model %q: adapter %q has a negative scale", m.Name, a.Name)
|
||||
}
|
||||
seen[a.Name] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m Model) engineAdapters() []engines.Adapter {
|
||||
var out []engines.Adapter
|
||||
for _, a := range m.Adapters {
|
||||
out = append(out, engines.Adapter{Name: a.Name, Path: a.Source})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -338,6 +339,34 @@ func (c *Controller) endpoints() []endpoint.Info {
|
||||
Priority: prof.Priority,
|
||||
})
|
||||
}
|
||||
return append(out, c.externalEndpoints()...)
|
||||
}
|
||||
|
||||
// externalEndpoints are the models served outside the cluster, as endpoints:
|
||||
// one each, with the provider's key read from its file every time, so a key
|
||||
// rotated on disk is used from the next publish on.
|
||||
func (c *Controller) externalEndpoints() []endpoint.Info {
|
||||
var out []endpoint.Info
|
||||
for _, m := range c.cat.Load().Models() {
|
||||
if !m.IsExternal() {
|
||||
continue
|
||||
}
|
||||
key := ""
|
||||
if f := m.External.APIKeyFile; f != "" {
|
||||
b, err := os.ReadFile(f) //nolint:gosec // the operator's own path
|
||||
if err != nil {
|
||||
c.log.Warn("an external model's key could not be read; not offered", "model", m.Name, "err", err)
|
||||
continue
|
||||
}
|
||||
key = strings.TrimSpace(string(b))
|
||||
}
|
||||
out = append(out, endpoint.Info{
|
||||
ID: "external:" + m.Name, URL: strings.TrimRight(m.External.URL, "/"),
|
||||
Models: append([]string{m.Name}, m.Aliases...), APIKey: key,
|
||||
Engine: endpoint.EngineExternal, Pool: "external", Weight: 1,
|
||||
UpstreamModel: m.External.Model,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -345,7 +374,8 @@ func (c *Controller) endpoints() []endpoint.Info {
|
||||
func fingerprint(infos []endpoint.Info) string {
|
||||
var b strings.Builder
|
||||
for _, i := range infos {
|
||||
fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight, i.Priority)
|
||||
fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d|%s|%x;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight, i.Priority,
|
||||
i.UpstreamModel, sha256.Sum256([]byte(i.APIKey)))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
package control
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
||||
)
|
||||
|
||||
type recordSink struct {
|
||||
mu sync.Mutex
|
||||
infos []endpoint.Info
|
||||
times int
|
||||
}
|
||||
|
||||
func (s *recordSink) SetEndpoints(_ context.Context, infos []endpoint.Info) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.infos, s.times = infos, s.times+1
|
||||
}
|
||||
|
||||
type noDispatch struct{}
|
||||
|
||||
func (noDispatch) StartShard(context.Context, reconciler.ShardCommand) error { return nil }
|
||||
func (noDispatch) StopShard(context.Context, string, string) error { return nil }
|
||||
func (noDispatch) DrainShard(context.Context, string, string) error { return nil }
|
||||
|
||||
func TestAnExternalModelIsPublishedWithItsKeyAndNothingIsPlacedForIt(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
keyFile := filepath.Join(dir, "provider.key")
|
||||
if err := os.WriteFile(keyFile, []byte("sk-provider-1\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
models := []catalog.Model{
|
||||
{Name: "cloud", Source: "external", Aliases: []string{"gpt"}, Scaling: catalog.Scaling{Min: 3, Max: 3},
|
||||
External: &catalog.External{URL: "https://api.example.com/", Model: "gpt-4o-mini", APIKeyFile: keyFile}},
|
||||
{Name: "gone", Source: "external",
|
||||
External: &catalog.External{URL: "https://api.example.com", Model: "m", APIKeyFile: filepath.Join(dir, "missing")}},
|
||||
}
|
||||
cat, err := catalog.New(models)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sink := &recordSink{}
|
||||
c := New(DefaultConfig(), catalog.NewLive(cat), noDispatch{}, sink, slog.New(slog.DiscardHandler))
|
||||
c.Publish(t.Context())
|
||||
|
||||
if len(sink.infos) != 1 {
|
||||
t.Fatalf("published %+v; the model whose key is missing must not be offered", sink.infos)
|
||||
}
|
||||
e := sink.infos[0]
|
||||
if e.ID != "external:cloud" || e.URL != "https://api.example.com" || e.APIKey != "sk-provider-1" ||
|
||||
e.UpstreamModel != "gpt-4o-mini" || e.Engine != endpoint.EngineExternal || len(e.Models) != 2 {
|
||||
t.Errorf("endpoint %+v", e)
|
||||
}
|
||||
// A key rotated on disk is published on the next pass.
|
||||
_ = os.WriteFile(keyFile, []byte("sk-provider-2"), 0o600)
|
||||
c.Publish(t.Context())
|
||||
if sink.times != 2 || sink.infos[0].APIKey != "sk-provider-2" {
|
||||
t.Errorf("the rotated key was not published: %d pushes, key %q", sink.times, sink.infos[0].APIKey)
|
||||
}
|
||||
// Scaling of 3 or not, nothing is wanted of the cluster for a model
|
||||
// served elsewhere, and so it never reads as short or unplaceable.
|
||||
for _, h := range c.Reconciler().ModelHealth() {
|
||||
if h.Model == "cloud" && (h.Desired != 0 || h.Unplaceable) {
|
||||
t.Errorf("an external model wants replicas: %+v", h)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -87,6 +87,9 @@ func (s *Service) Sources(ctx context.Context) (map[string][]catalog.Model, erro
|
||||
if m.Source != "" {
|
||||
out[m.Source] = append(out[m.Source], m)
|
||||
}
|
||||
for _, ad := range m.Adapters {
|
||||
out[ad.Source] = append(out[ad.Source], m)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -94,6 +94,9 @@ func (r *Reconciler) desiredFor(m catalog.Model) int {
|
||||
// baseFor is what a model runs without load: its minimum, or for an on-demand
|
||||
// model one while somebody is asking and none once they stop.
|
||||
func (r *Reconciler) baseFor(m catalog.Model) int {
|
||||
if m.IsExternal() {
|
||||
return 0 // served elsewhere: nothing to place, whatever its scaling says
|
||||
}
|
||||
if !m.Scaling.OnDemand() {
|
||||
return m.Scaling.Min
|
||||
}
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
func adapterModel() catalog.Model {
|
||||
m := catModel("coder", nil)
|
||||
m.Profiles = []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 4}}
|
||||
m.Adapters = []catalog.Adapter{
|
||||
{Name: "coder-sql", Source: "http://mirror/sql.gguf"},
|
||||
{Name: "coder-docs", Source: "http://mirror/docs.gguf", Scale: 0.5},
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestAnAdapterIsAskedForByNameAndAppliedOnItsModel(t *testing.T) {
|
||||
e := newFakeEngine(t, "coder")
|
||||
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"coder"}})
|
||||
cfg.Models = []catalog.Model{adapterModel()}
|
||||
rt := newTestRouter(t, cfg)
|
||||
|
||||
sent := func(model string) map[string]any {
|
||||
t.Helper()
|
||||
rec := post(t, rt.Handler(), map[string]any{"model": model, "messages": []any{}}, nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: status %d: %s", model, rec.Code, rec.Body.String())
|
||||
}
|
||||
var body map[string]any
|
||||
_ = json.Unmarshal(e.last.Load().([]byte), &body)
|
||||
return body
|
||||
}
|
||||
body := sent("coder-docs")
|
||||
if body["model"] != "coder" {
|
||||
t.Errorf("the engine was asked for %v, not the model the adapter runs on", body["model"])
|
||||
}
|
||||
lora, _ := json.Marshal(body["lora"])
|
||||
if string(lora) != `[{"id":1,"scale":0.5}]` {
|
||||
t.Errorf("lora %s, expected the second adapter at its scale", lora)
|
||||
}
|
||||
if body := sent("coder-sql"); func() string { b, _ := json.Marshal(body["lora"]); return string(b) }() != `[{"id":0,"scale":1}]` {
|
||||
t.Errorf("the first adapter, at the default scale: %v", body["lora"])
|
||||
}
|
||||
if body := sent("coder"); body["lora"] != nil {
|
||||
t.Errorf("the base model was sent an adapter: %v", body["lora"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAKeyAllowedAnAdapterReachesThatAdapterOnly(t *testing.T) {
|
||||
cfg := baseConfig()
|
||||
cfg.Models = []catalog.Model{adapterModel()}
|
||||
rt := newTestRouter(t, cfg)
|
||||
key := func(allowed ...string) auth.Principal {
|
||||
return auth.Principal{Limits: store.Effective{AllowedModels: allowed}}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
allowed []string
|
||||
requested string
|
||||
want bool
|
||||
}{
|
||||
{[]string{"coder-sql"}, "coder-sql", true},
|
||||
{[]string{"coder-sql"}, "coder", false},
|
||||
{[]string{"coder-sql"}, "coder-docs", false},
|
||||
{[]string{"coder"}, "coder-sql", true}, // the model allows its adapters
|
||||
{[]string{"coder"}, "coder", true},
|
||||
} {
|
||||
if got := rt.permits(key(c.allowed...), c.requested); got != c.want {
|
||||
t.Errorf("allowed %v, asked for %s: %v", c.allowed, c.requested, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdaptersShareTheNamespaceOfModels(t *testing.T) {
|
||||
a := adapterModel()
|
||||
b := catModel("coder-sql", nil)
|
||||
if _, err := catalog.New([]catalog.Model{a, b}); err == nil {
|
||||
t.Error("an adapter named like another model was accepted")
|
||||
}
|
||||
bad := adapterModel()
|
||||
bad.Profiles = []catalog.Profile{{Name: "gpu", Engine: "vllm", GPUClass: "h100-80g", TP: 1}}
|
||||
if _, err := catalog.New([]catalog.Model{bad}); err == nil {
|
||||
t.Error("adapters on a vLLM profile were accepted")
|
||||
}
|
||||
c, err := catalog.New([]catalog.Model{adapterModel()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, ad, i, ok := c.ResolveAdapter("coder-docs"); !ok || m.Name != "coder" || i != 1 || ad.EffectiveScale() != 0.5 {
|
||||
t.Errorf("resolve: %v %+v %d %v", m.Name, ad, i, ok)
|
||||
}
|
||||
}
|
||||
@@ -37,8 +37,16 @@ type Info struct {
|
||||
Weight float64
|
||||
// Priority expresses pool preference; higher pools are tried first.
|
||||
Priority int
|
||||
// UpstreamModel is what the endpoint calls the model, when that differs
|
||||
// from the catalog's name: an external provider's. Requests are rewritten
|
||||
// to it. Empty sends the name as the client did.
|
||||
UpstreamModel string
|
||||
}
|
||||
|
||||
// EngineExternal marks an endpoint that is a provider outside the cluster.
|
||||
// It publishes no metrics, so it is never scraped and never reads as stale.
|
||||
const EngineExternal = "external"
|
||||
|
||||
// Serves reports whether the replica serves the given model.
|
||||
func (i Info) Serves(model string) bool {
|
||||
return slices.Contains(i.Models, model)
|
||||
@@ -242,7 +250,7 @@ func (r *Registry) Snapshots(model string) []Snapshot {
|
||||
continue
|
||||
}
|
||||
var age time.Duration
|
||||
stale := true
|
||||
stale := l.info.Engine != EngineExternal
|
||||
if !l.scrapedAt.IsZero() {
|
||||
age = now.Sub(l.scrapedAt)
|
||||
stale = age > r.cfg.StaleAfter
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
||||
)
|
||||
|
||||
func TestALocalModelWithNoCapacityFallsBackToAnExternalProvider(t *testing.T) {
|
||||
local := newFakeEngine(t, "local")
|
||||
local.status.Store(http.StatusServiceUnavailable)
|
||||
|
||||
var calls atomic.Int64
|
||||
var gotModel, gotAuth atomic.Value
|
||||
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/metrics" {
|
||||
t.Error("the provider was scraped for metrics")
|
||||
}
|
||||
calls.Add(1)
|
||||
gotAuth.Store(r.Header.Get("Authorization"))
|
||||
var body map[string]any
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
_ = json.Unmarshal(b, &body)
|
||||
gotModel.Store(fmt.Sprint(body["model"]))
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprint(w, `{"model":"gpt-4o-mini","choices":[{"message":{"content":"from outside"}}],"usage":{"prompt_tokens":3,"completion_tokens":2}}`)
|
||||
}))
|
||||
t.Cleanup(provider.Close)
|
||||
|
||||
cfg := baseConfig(config.Endpoint{ID: "l", URL: local.srv.URL, Models: []string{"local"}})
|
||||
cloud := catalog.Model{Name: "cloud", Source: "external",
|
||||
External: &catalog.External{URL: provider.URL, Model: "gpt-4o-mini"}}
|
||||
cfg.Models = []catalog.Model{catModel("local", nil, "cloud"), cloud}
|
||||
rt := newTestRouter(t, cfg)
|
||||
rt.SetEndpoints(t.Context(), []endpoint.Info{
|
||||
{ID: "l", URL: local.srv.URL, Models: []string{"local"}},
|
||||
{ID: "external:cloud", URL: provider.URL, Models: []string{"cloud"}, APIKey: "provider-key",
|
||||
Engine: endpoint.EngineExternal, Pool: "external", Weight: 1, UpstreamModel: "gpt-4o-mini"},
|
||||
})
|
||||
|
||||
rec := post(t, rt.Handler(), map[string]any{"model": "local", "messages": []any{}}, nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if calls.Load() != 1 {
|
||||
t.Fatalf("the provider was called %d times", calls.Load())
|
||||
}
|
||||
if gotModel.Load() != "gpt-4o-mini" {
|
||||
t.Errorf("the provider was asked for %v, not its own name for the model", gotModel.Load())
|
||||
}
|
||||
if gotAuth.Load() != "Bearer provider-key" {
|
||||
t.Errorf("the provider got %v, not its key", gotAuth.Load())
|
||||
}
|
||||
if got := rec.Header().Get("x-llmgw-served-model"); got != "cloud" {
|
||||
t.Errorf("the client should be told it was served by cloud, got %q", got)
|
||||
}
|
||||
// Never scraped, and never stale for it.
|
||||
for _, s := range rt.reg.Snapshots("cloud") {
|
||||
if s.Stale {
|
||||
t.Error("an external endpoint reads as stale")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnExternalModelIsNotAModelWithProfiles(t *testing.T) {
|
||||
good := catalog.Model{Name: "cloud", Source: "external",
|
||||
External: &catalog.External{URL: "https://api.example.com", Model: "gpt-4o-mini"}}
|
||||
if _, err := catalog.New([]catalog.Model{good}); err != nil {
|
||||
t.Fatalf("a valid external model was refused: %v", err)
|
||||
}
|
||||
for name, m := range map[string]catalog.Model{
|
||||
"profiles": {Name: "x", Source: "external", External: &catalog.External{URL: "https://a.b", Model: "m"},
|
||||
Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp"}}},
|
||||
"no url": {Name: "x", Source: "external", External: &catalog.External{URL: "api.example.com", Model: "m"}},
|
||||
"no model": {Name: "x", Source: "external", External: &catalog.External{URL: "https://a.b"}},
|
||||
} {
|
||||
if _, err := catalog.New([]catalog.Model{m}); err == nil {
|
||||
t.Errorf("%s: accepted", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -426,6 +426,12 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
|
||||
// around the allowlist; matching only the canonical name would silently ignore
|
||||
// an allowlist written in the names clients actually use.
|
||||
func (r *Router) permits(p auth.Principal, requested string) bool {
|
||||
// An adapter named in the allowlist allows that adapter: not its model,
|
||||
// and not the model's other adapters, which the checks below would let
|
||||
// through if the adapter's own name were not asked about first.
|
||||
if _, ok := r.adapter(requested); ok && p.Allows(requested) {
|
||||
return true
|
||||
}
|
||||
canonical, m := r.resolve(requested)
|
||||
if p.Allows(canonical) {
|
||||
return true
|
||||
@@ -984,3 +990,44 @@ func (r *Router) rateLimitHeaders(w http.ResponseWriter, p auth.Principal) {
|
||||
w.Header().Set("x-ratelimit-reset-tokens", reset)
|
||||
}
|
||||
}
|
||||
|
||||
// withModel is body with its model set to name, for an endpoint that knows the
|
||||
// model by another name — an external provider. The body unchanged when it is
|
||||
// not a JSON object, which the endpoint then refuses as it would have anyway.
|
||||
func withModel(body []byte, name string) []byte {
|
||||
var m map[string]json.RawMessage
|
||||
if err := json.Unmarshal(body, &m); err != nil {
|
||||
return body
|
||||
}
|
||||
encoded, err := json.Marshal(name)
|
||||
if err != nil {
|
||||
return body
|
||||
}
|
||||
m["model"] = encoded
|
||||
out, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return body
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// withAdapter is body asking for model with a LoRA adapter applied, in
|
||||
// llama.cpp's form: "lora": [{"id": index, "scale": scale}]. Adapters not
|
||||
// listed are applied at zero, so the list is the whole of it.
|
||||
func withAdapter(body []byte, model string, index int, scale float64) []byte {
|
||||
var m map[string]json.RawMessage
|
||||
if err := json.Unmarshal(body, &m); err != nil {
|
||||
return body
|
||||
}
|
||||
name, err1 := json.Marshal(model)
|
||||
lora, err2 := json.Marshal([]map[string]any{{"id": index, "scale": scale}})
|
||||
if err1 != nil || err2 != nil {
|
||||
return body
|
||||
}
|
||||
m["model"], m["lora"] = name, lora
|
||||
out, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return body
|
||||
}
|
||||
return out
|
||||
}
|
||||
+45
-11
@@ -61,6 +61,8 @@ type Router struct {
|
||||
// router's half of not having to. A request takes the map once and works
|
||||
// from it, so a swap mid-request is invisible rather than a race.
|
||||
models atomic.Pointer[map[string]catalog.Model]
|
||||
// adapters are the LoRA adapters by the name clients ask for them by.
|
||||
adapters atomic.Pointer[map[string]adapterRef]
|
||||
|
||||
// extraMetrics are collectors other components publish on /metrics.
|
||||
extraMetrics []prometheus.Collector
|
||||
@@ -315,14 +317,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) {
|
||||
// all. A control plane pushes over the top of this at startup and on every
|
||||
// change after.
|
||||
if len(cfg.Models) > 0 {
|
||||
seed := map[string]catalog.Model{}
|
||||
for _, m := range cfg.Models {
|
||||
seed[m.Name] = m
|
||||
for _, a := range m.Aliases {
|
||||
seed[a] = m
|
||||
}
|
||||
}
|
||||
r.models.Store(&seed)
|
||||
r.storeModels(cfg.Models)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
@@ -505,15 +500,44 @@ func (r *Router) modelMap() map[string]catalog.Model {
|
||||
// The catalog's counterpart to SetEndpoints, and pushed for the same reason:
|
||||
// the control plane owns what should exist, the router needs to know, and
|
||||
// neither has to be in the other's process for that to work.
|
||||
func (r *Router) SetCatalog(c *catalog.Catalog) {
|
||||
func (r *Router) SetCatalog(c *catalog.Catalog) { r.storeModels(c.Models()) }
|
||||
|
||||
// storeModels makes models what names resolve to: each model by its name, its
|
||||
// aliases and its adapters. One place for it, whether the models came from
|
||||
// configuration at start or from the control plane since.
|
||||
func (r *Router) storeModels(models []catalog.Model) {
|
||||
m := map[string]catalog.Model{}
|
||||
for _, entry := range c.Models() {
|
||||
ads := map[string]adapterRef{}
|
||||
for _, entry := range models {
|
||||
m[entry.Name] = entry
|
||||
for _, a := range entry.Aliases {
|
||||
m[a] = entry
|
||||
}
|
||||
// An adapter resolves to the model it runs on, as an alias does, and
|
||||
// is applied to the request on the way there.
|
||||
for i, a := range entry.Adapters {
|
||||
m[a.Name] = entry
|
||||
ads[a.Name] = adapterRef{base: entry.Name, index: i, scale: a.EffectiveScale()}
|
||||
}
|
||||
}
|
||||
r.models.Store(&m)
|
||||
r.adapters.Store(&ads)
|
||||
}
|
||||
|
||||
// adapterRef is where an adapter runs and how it is applied.
|
||||
type adapterRef struct {
|
||||
base string
|
||||
index int
|
||||
scale float64
|
||||
}
|
||||
|
||||
// adapter finds one by the name a client asked for.
|
||||
func (r *Router) adapter(name string) (adapterRef, bool) {
|
||||
if p := r.adapters.Load(); p != nil {
|
||||
a, ok := (*p)[name]
|
||||
return a, ok
|
||||
}
|
||||
return adapterRef{}, false
|
||||
}
|
||||
|
||||
// LastUsed reports when each endpoint was last sent a request, satisfying
|
||||
@@ -715,10 +739,20 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
uctx, uspan := tracer().Start(ctx, "upstream", trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("llmgw.endpoint", dec.Endpoint.ID),
|
||||
attribute.String("url.full", dec.Endpoint.URL+req.Path)))
|
||||
body := req.UpstreamBody
|
||||
if dec.Endpoint.UpstreamModel != "" {
|
||||
body = withModel(body, dec.Endpoint.UpstreamModel)
|
||||
}
|
||||
// An adapter asked for by name, served by its model's replicas: the
|
||||
// engine is asked for the model, with the adapter applied. Not on a
|
||||
// fallback to another model, which does not have it.
|
||||
if a, ok := r.adapter(req.Model); ok && a.base == model {
|
||||
body = withAdapter(body, model, a.index, a.scale)
|
||||
}
|
||||
out, perr := r.px.Do(uctx, w, proxy.Request{
|
||||
URL: dec.Endpoint.URL + req.Path,
|
||||
APIKey: dec.Endpoint.APIKey,
|
||||
Body: req.UpstreamBody,
|
||||
Body: body,
|
||||
Header: withTraceparent(uctx, req.Header),
|
||||
Stream: req.Stream,
|
||||
SuppressUsageChunk: req.SuppressUsageChunk,
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
||||
)
|
||||
@@ -51,34 +54,41 @@ func KVSaturated(ceiling float64) Filter {
|
||||
}
|
||||
}
|
||||
|
||||
// PoolPriority keeps only the endpoints of the highest-priority pool
|
||||
// (plan §7.3, the two-stage decision: pool first, then replica).
|
||||
// PoolPriority keeps the endpoints of one pool: the highest-priority one,
|
||||
// unless it is saturated and a lower one is not (plan §7.3, the two-stage
|
||||
// decision — pool first, then replica).
|
||||
//
|
||||
// Spillover arrives in phase 5, at which point this filter will fall through
|
||||
// to the next pool once the preferred one is saturated.
|
||||
func PoolPriority() Filter {
|
||||
return poolPriorityFilter{}
|
||||
// Saturated is what admission means by it, the same formula over the same
|
||||
// thresholds, so the two layers agree on when a pool is full: the H100 pool is
|
||||
// used until it is, and only then the A100 one — never the A100 because it
|
||||
// happens to be emptier. When every pool is saturated the preferred one is
|
||||
// kept: moving to a pool that is just as full buys nothing and loses whatever
|
||||
// made the first preferred.
|
||||
func PoolPriority(targetWaiting, kvCeiling float64) Filter {
|
||||
return poolPriorityFilter{t: admission.Thresholds{TargetWaitingPerReplica: targetWaiting, KVCeiling: kvCeiling}}
|
||||
}
|
||||
|
||||
type poolPriorityFilter struct{}
|
||||
type poolPriorityFilter struct{ t admission.Thresholds }
|
||||
|
||||
func (poolPriorityFilter) Name() string { return "pool-priority" }
|
||||
|
||||
func (poolPriorityFilter) Filter(_ *Ctx, eps []endpoint.Snapshot) []endpoint.Snapshot {
|
||||
func (f poolPriorityFilter) Filter(_ *Ctx, eps []endpoint.Snapshot) []endpoint.Snapshot {
|
||||
if len(eps) == 0 {
|
||||
return eps
|
||||
}
|
||||
best := eps[0].Priority
|
||||
for _, e := range eps[1:] {
|
||||
if e.Priority > best {
|
||||
best = e.Priority
|
||||
}
|
||||
}
|
||||
out := make([]endpoint.Snapshot, 0, len(eps))
|
||||
byPriority := map[int][]endpoint.Snapshot{}
|
||||
var order []int
|
||||
for _, e := range eps {
|
||||
if e.Priority == best {
|
||||
out = append(out, e)
|
||||
if _, ok := byPriority[e.Priority]; !ok {
|
||||
order = append(order, e.Priority)
|
||||
}
|
||||
byPriority[e.Priority] = append(byPriority[e.Priority], e)
|
||||
}
|
||||
sort.Sort(sort.Reverse(sort.IntSlice(order)))
|
||||
for _, p := range order {
|
||||
if admission.Saturation(byPriority[p], 0, f.t) < 1 {
|
||||
return byPriority[p]
|
||||
}
|
||||
}
|
||||
return out
|
||||
return byPriority[order[0]]
|
||||
}
|
||||
@@ -50,7 +50,7 @@ func Chat(c ProfileConfig) *Profile {
|
||||
c.normalize()
|
||||
return &Profile{
|
||||
Name: "chat",
|
||||
Filters: []Filter{BreakerOpen(), PoolPriority()},
|
||||
Filters: []Filter{BreakerOpen(), PoolPriority(c.TargetWaiting, c.KVCeiling)},
|
||||
Scorers: c.withPrefix([]WeightedScorer{
|
||||
{Session(c.SessionLoadFactor), 3.0},
|
||||
{Queue(c.TargetWaiting), 1.0},
|
||||
@@ -81,7 +81,7 @@ func Batch(c ProfileConfig) *Profile {
|
||||
c.normalize()
|
||||
return &Profile{
|
||||
Name: "batch",
|
||||
Filters: []Filter{BreakerOpen(), PoolPriority()},
|
||||
Filters: []Filter{BreakerOpen(), PoolPriority(c.TargetWaiting, c.KVCeiling)},
|
||||
Scorers: []WeightedScorer{
|
||||
{Queue(c.TargetWaiting), 3.0},
|
||||
{KV(c.KVCeiling), 1.0},
|
||||
@@ -99,7 +99,7 @@ func LongContext(c ProfileConfig) *Profile {
|
||||
c.normalize()
|
||||
return &Profile{
|
||||
Name: "long-context",
|
||||
Filters: []Filter{BreakerOpen(), PoolPriority()},
|
||||
Filters: []Filter{BreakerOpen(), PoolPriority(c.TargetWaiting, c.KVCeiling)},
|
||||
// Prefix matching earns more here than anywhere else: a long prompt is
|
||||
// mostly prefill, and prefill is what a cache hit skips.
|
||||
Scorers: c.withPrefix([]WeightedScorer{
|
||||
|
||||
@@ -177,7 +177,7 @@ func TestStaleEndpointIsPenalizedNotDropped(t *testing.T) {
|
||||
func TestPoolPriority(t *testing.T) {
|
||||
p := &Profile{
|
||||
Name: "t",
|
||||
Filters: []Filter{PoolPriority()},
|
||||
Filters: []Filter{PoolPriority(4, 0.9)},
|
||||
Scorers: []WeightedScorer{{Queue(4), 1}},
|
||||
Picker: MaxScore(),
|
||||
}
|
||||
@@ -189,7 +189,8 @@ func TestPoolPriority(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The lower-priority pool must not win despite being emptier (spillover is phase 5).
|
||||
// The lower-priority pool must not win for being emptier: the preferred
|
||||
// one is not saturated (3 waiting of a healthy 4).
|
||||
if d.Endpoint.ID != "h100" {
|
||||
t.Errorf("pool priority was ignored: %s", d.Endpoint.ID)
|
||||
}
|
||||
@@ -383,3 +384,36 @@ func TestBuiltinProfilesSchedule(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestASaturatedPoolSpillsToTheNext(t *testing.T) {
|
||||
p := &Profile{Name: "t", Filters: []Filter{PoolPriority(4, 0.9)},
|
||||
Scorers: []WeightedScorer{{Queue(4), 1}}, Picker: MaxScore()}
|
||||
h100 := func(w int) endpoint.Snapshot {
|
||||
return ep("h100", waiting(w), func(s *endpoint.Snapshot) { s.Priority = 100; s.Pool = "ib-a/h100" })
|
||||
}
|
||||
a100 := func(w int) endpoint.Snapshot {
|
||||
return ep("a100", waiting(w), func(s *endpoint.Snapshot) { s.Priority = 50; s.Pool = "ib-b/a100" })
|
||||
}
|
||||
pick := func(eps ...endpoint.Snapshot) string {
|
||||
d, err := p.Schedule(newCtx(Request{Model: "m"}), eps)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d.Endpoint.ID
|
||||
}
|
||||
if got := pick(h100(8), a100(0)); got != "a100" {
|
||||
t.Errorf("the preferred pool at twice its healthy queue did not spill: %s", got)
|
||||
}
|
||||
if got := pick(h100(8), a100(6)); got != "h100" {
|
||||
t.Errorf("both pools saturated: the preferred one should be kept, got %s", got)
|
||||
}
|
||||
if got := pick(h100(4), a100(0)); got != "a100" {
|
||||
t.Errorf("at exactly its healthy limit a pool is full: %s", got)
|
||||
}
|
||||
// KV pressure saturates a pool as queue depth does.
|
||||
full := h100(0)
|
||||
full.Stats.KVCacheUsage = 0.95
|
||||
if got := pick(full, a100(0)); got != "a100" {
|
||||
t.Errorf("a pool out of KV cache did not spill: %s", got)
|
||||
}
|
||||
}
|
||||
@@ -96,6 +96,9 @@ func (p *Poller) Sync(ctx context.Context) {
|
||||
|
||||
want := make(map[string]endpoint.Info, len(infos))
|
||||
for _, in := range infos {
|
||||
if in.Engine == endpoint.EngineExternal {
|
||||
continue // a provider elsewhere publishes no /metrics
|
||||
}
|
||||
want[in.ID] = in
|
||||
}
|
||||
|
||||
|
||||
@@ -89,7 +89,7 @@
|
||||
<td{{if .HeldFor}} class="warn" title="gave room to {{.HeldFor}}"{{end}}>{{.Class}}{{if .Preempted}} <span class="note" title="replicas lost to a higher class">−{{.Preempted}}</span>{{end}}</td>
|
||||
<td{{with .Warnings}} class="warn" title="{{join .}}"{{end}}>{{if or .MemoryGB .CPUCores}}{{.MemoryGB}}G{{if .CPUCores}} · {{.CPUCores}}c{{end}}{{else}}—{{end}}
|
||||
{{- if .MeasuredMemoryGB}} <span class="note">{{printf "%.1f" .MeasuredMemoryGB}}G used</span>{{end}}{{if .Warnings}} ⚠{{end}}</td>
|
||||
<td>{{join .Profiles}}</td>
|
||||
<td>{{if .External}}<span class="tag" title="served outside the cluster; its prompts leave it">external</span> {{.External}}{{else}}{{join .Profiles}}{{end}}</td>
|
||||
{{if $.AnyFallbacks}}<td>{{join .Fallbacks}}</td>{{end}}
|
||||
{{/* Cut to the column, the whole sentence on hover: a row of its own for
|
||||
each description made the table twice as long as the catalog. */}}
|
||||
|
||||
+7
-3
@@ -1459,7 +1459,8 @@ The launch path is now built too:
|
||||
hits. It does nothing when the working set fits in one replica's cache,
|
||||
which is worth knowing rather than assuming.
|
||||
- [x] Fallback models
|
||||
- [ ] Pool spillover
|
||||
- [x] Pool spillover: a saturated pool passes requests to the next one that
|
||||
is not, by admission's own measure of saturation
|
||||
- [x] `ctl bench`: a load ramp that reports capacity as the last concurrency
|
||||
level meeting the SLO, plus the queue depth and KV usage observed there,
|
||||
so `weight`, `target_waiting` and `kv_ceiling` stop being guesses
|
||||
@@ -1496,8 +1497,11 @@ The launch path is now built too:
|
||||
- [x] Weight downloading and cache management: the mirror, agent fetches
|
||||
checked against their digest, and reclaiming what nothing uses
|
||||
- [ ] Prefill/decode disaggregation (§12)
|
||||
- [ ] LoRA adapter routing (affinity to a replica that already has the adapter)
|
||||
- [ ] External provider support (as a fallback)
|
||||
- [x] LoRA adapters on llama.cpp: every replica loads a model's adapters and
|
||||
applies one per request, so any replica serves any adapter
|
||||
- [ ] LoRA affinity for engines that load adapters on demand (vLLM)
|
||||
- [x] External provider support: a catalog model served by an OpenAI-compatible
|
||||
API elsewhere, usable as a fallback, under the same allowlists and budgets
|
||||
- [x] Read-only dashboard on the admin listener (nodes, replicas, models,
|
||||
placement failures, usage) and a public landing page
|
||||
- [x] `/chat`: an in-browser playground for signed-in people, dispatched through
|
||||
|
||||
Reference in new issue
Block a user