Merge C24 federation: another llmgw cluster as a provider

Conflicts with facility-aware placement (config, control, admin API routes,
store test tables) all keep both; the generated reference and OpenAPI
document regenerated.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:29:35 +02:00
commit 33334c8f33
33 files changed
+2945 -24

No files matched your search

+351
View File
@@ -41,6 +41,9 @@
{
"name": "facility"
},
{
"name": "federation"
},
{
"name": "guard"
},
@@ -591,6 +594,157 @@
}
}
},
"/api/v1/federation": {
"get": {
"operationId": "getFederation",
"summary": "This cluster's id, and its federation peers with their health and models",
"tags": [
"federation"
],
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/FederationView"
}
}
}
},
"401": {
"$ref": "#/components/responses/Unauthorized"
},
"default": {
"$ref": "#/components/responses/Error"
}
}
},
"post": {
"operationId": "postFederation",
"summary": "Add another llmgw cluster as a peer, or change one",
"description": "The peer is checked first: its /ready, timed, and its /v1/models with the key. Its models join the catalog as PEER/MODEL.",
"tags": [
"federation"
],
"parameters": [
{
"$ref": "#/components/parameters/Actor"
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/federation.AddRequest"
}
}
}
},
"responses": {
"201": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/federation.Status"
}
}
}
},
"401": {
"$ref": "#/components/responses/Unauthorized"
},
"default": {
"$ref": "#/components/responses/Error"
}
}
}
},
"/api/v1/federation/{peer}": {
"delete": {
"operationId": "deleteFederationPeer",
"summary": "Remove a federation peer and its models",
"description": "Refused while a model here falls back to one of the peer's.",
"tags": [
"federation"
],
"parameters": [
{
"name": "peer",
"in": "path",
"required": true,
"description": "The federation peer's name, as `llmgw ctl federation list` lists it.",
"schema": {
"type": "string"
}
},
{
"$ref": "#/components/parameters/Actor"
}
],
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Removed"
}
}
}
},
"401": {
"$ref": "#/components/responses/Unauthorized"
},
"default": {
"$ref": "#/components/responses/Error"
}
}
}
},
"/api/v1/federation/{peer}/sync": {
"post": {
"operationId": "postFederationPeerSync",
"summary": "Check a peer now, rather than at the next interval",
"tags": [
"federation"
],
"parameters": [
{
"name": "peer",
"in": "path",
"required": true,
"description": "The federation peer's name, as `llmgw ctl federation list` lists it.",
"schema": {
"type": "string"
}
},
{
"$ref": "#/components/parameters/Actor"
}
],
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/federation.Status"
}
}
}
},
"401": {
"$ref": "#/components/responses/Unauthorized"
},
"default": {
"$ref": "#/components/responses/Error"
}
}
}
},
"/api/v1/guard": {
"get": {
"operationId": "getGuard",
@@ -1439,6 +1593,58 @@
}
}
},
"/api/v1/models/{name}/fallbacks": {
"put": {
"operationId": "putModelsNameFallbacks",
"summary": "Replace the models a model falls back to when it has no capacity",
"description": "A model added at runtime only. A federation peer's model, PEER/MODEL, sends the overflow to the other cluster.",
"tags": [
"models"
],
"parameters": [
{
"name": "name",
"in": "path",
"required": true,
"description": "The model's name in the catalog.",
"schema": {
"type": "string"
}
},
{
"$ref": "#/components/parameters/Actor"
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/FallbacksRequest"
}
}
}
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/FallbacksView"
}
}
}
},
"401": {
"$ref": "#/components/responses/Unauthorized"
},
"default": {
"$ref": "#/components/responses/Error"
}
}
}
},
"/api/v1/models/{name}/headroom": {
"get": {
"operationId": "getModelsNameHeadroom",
@@ -3445,6 +3651,49 @@
}
}
},
"FallbacksRequest": {
"type": "object",
"description": "FallbacksRequest replaces a model's fallbacks; empty takes them all away.",
"properties": {
"fallbacks": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
"FallbacksView": {
"type": "object",
"description": "FallbacksView is a model's fallbacks, as they are now.",
"properties": {
"model": {
"type": "string"
},
"fallbacks": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
"FederationView": {
"type": "object",
"description": "FederationView is this cluster's id and its peers.",
"properties": {
"cluster": {
"type": "string",
"description": "Cluster is this cluster's own id, which its peers see in the hop header."
},
"peers": {
"type": "array",
"items": {
"$ref": "#/components/schemas/federation.Status"
}
}
}
},
"FetchView": {
"type": "object",
"description": "FetchView is a download in flight.",
@@ -5363,6 +5612,10 @@
"api_key_file": {
"type": "string",
"description": "APIKeyFile holds the provider's key, on the control plane. A file rather than the key itself, so the key is in neither the catalog, the store nor the audit log; empty for a provider that asks for none."
},
"peer": {
"type": "string",
"description": "Peer names the federated llmgw cluster this model is offered by, for a model that federation added (`llmgw ctl federation add`) rather than one written by hand. Such a model is named PEER/MODEL, comes and goes with the peer's own list, and is sent only while the peer is up."
}
}
},
@@ -6027,6 +6280,104 @@
}
}
},
"federation.AddRequest": {
"type": "object",
"description": "AddRequest adds a peer.",
"properties": {
"name": {
"type": "string",
"description": "Name is what this cluster calls the peer, and the prefix of its models: NAME/MODEL."
},
"url": {
"type": "string",
"description": "URL is the peer gateway's client address, without /v1."
},
"key_file": {
"type": "string",
"description": "KeyFile is a file on the control plane holding a key the peer issued to this cluster (or env:NAME, vault:PATH#FIELD)."
},
"send_tags": {
"type": "boolean",
"description": "SendTags makes requests sent to the peer carry x-llmgw-tags naming this cluster and the sending project. Off unless asked: the peer learns who in this cluster is asking."
}
}
},
"federation.ModelStatus": {
"type": "object",
"description": "ModelStatus is one of a peer's models: its name here, the peer's own, and whether the peer has stopped offering it.",
"properties": {
"name": {
"type": "string"
},
"upstream": {
"type": "string"
},
"withdrawn": {
"type": "boolean"
}
}
},
"federation.Status": {
"type": "object",
"description": "Status is a peer as `llmgw ctl federation list` shows it.",
"properties": {
"name": {
"type": "string"
},
"url": {
"type": "string"
},
"key_file": {
"type": "string"
},
"send_tags": {
"type": "boolean"
},
"cluster": {
"type": "string",
"description": "Cluster is the peer's own cluster id."
},
"up": {
"type": "boolean"
},
"checked": {
"type": "string",
"format": "date-time",
"description": "Checked is when; zero before the first check."
},
"latency_ns": {
"type": "integer",
"format": "int64",
"description": "Latency is how long the peer took to answer /ready: the network to the other site and the peer's front, not a model's speed.\n\nA duration, in nanoseconds."
},
"error": {
"type": "string",
"description": "Error is why the peer is down, empty while it is up."
},
"latency_ms": {
"type": "number",
"description": "LatencyMs is Latency in milliseconds, for reading."
},
"synced_at": {
"type": "string",
"format": "date-time",
"description": "SyncedAt is when its models last changed here."
},
"models": {
"type": "array",
"items": {
"$ref": "#/components/schemas/federation.ModelStatus"
}
},
"created_at": {
"type": "string",
"format": "date-time"
},
"created_by": {
"type": "string"
}
}
},
"guard.Attachment": {
"type": "object",
"description": "Attachment ties a policy to a part of the tenancy tree: scope is org, team, project, user or key, and ScopeID which one.",
+57
View File
@@ -1196,6 +1196,63 @@ prices and budgets, rate limits, usage records, and key allowlists.
whose allowlist leaves the external model out cannot reach it, as a fallback or
by name. The cluster page and `ctl models` mark it `external`.
### Federating clusters
Another llmgw cluster — another site, another team's GPUs — can be a provider
of this one, and llmgw knows it is one. Ask the other cluster's operator for a
key in one of their projects (`llmgw keys create`), put it in a file on this
control plane (on every control plane, with HA), and add the peer:
```
llmgw ctl federation add east --url https://llm.east.example --key-file /etc/llmgw/secrets/east.key
llmgw ctl federation list
llmgw ctl model fallbacks qwen-7b east/qwen-7b # overflow goes to east
```
- **Its models, named PEER/MODEL.** What the peer's `/v1/models` offers that
key is offered here as `east/qwen-7b` and so on, always with the prefix —
never under the peer's own name, even when nothing here uses it, so
whether a request leaves the cluster never depends on what else is in the
catalog, and every usage record and `x-llmgw-served-model` header says which
site answered. A model here called `east/...` wins over the peer's of the
same name. The peer's aliases, and models the peer has from peers of its
own, are not taken: federation is one hop.
- **Kept in step.** Each peer is checked every `federation.interval` (10s):
its `/ready`, timed — the latency `ctl federation list` shows — and its
`/v1/models`. A model it adds appears here; one it drops goes, unless a
model here still falls back to it, when it is kept *withdrawn* (in the
catalog, sent nothing) until the fallback is changed. `ctl federation sync
east` checks now.
- **Taken out when down.** A peer whose `/ready` fails, which refuses the key,
or which does not answer is sent nothing until it answers again; its models
stay in the catalog. A peer's `/ready` counts only what it serves itself, so
two clusters federating each other are not each ready because of the other.
- **Overflow.** Named in a model's fallbacks (`ctl model fallbacks`, for
models added at runtime), a peer's model is used only when the model has
no capacity here — as for any external model.
- **What stays here, stays here.** A peer's model is an external model: a
project with `keys residency --in-cluster`, or under a guardrails policy with
`no_store`, is refused it by name and never sent to it as a fallback.
- **One key, and who is asking.** The peer sees this cluster's key, so its
usage, limits and budgets are one tenant's. The request's `X-Request-Id`
goes with it, so one id finds the request on both sides. With
`--send-tags` the request also carries `x-llmgw-tags:
federated_from=<federation.name>,org=…,team=…,project=…` (ids), for the
peer's usage records to attribute it; off unless given, because it tells
the other site who here is asking.
- **No loops.** Each cluster has an id (`ctl federation list` shows it), and a
request sent to a peer carries `x-llmgw-hop` with the ids it has passed
through. A cluster refuses a request that has already passed through it
(508, `federation_loop`), and does not send one to a peer it came from —
so two clusters falling back to each other try each once.
- `llmgw_federation_peer_up`, `_latency_seconds` and `_models` are on
`/metrics`; the console has a Federation view. `ctl federation remove east`
takes the peer and its models away, refused while a fallback names one.
Not yet: models declared in server.yaml cannot name a peer's model as a
fallback (the catalog in the file is checked before the peers are read), and
a peer behind a private CA needs that CA in the system's trust store.
### Trying new weights on some of the traffic first
When a model's repository publishes new weights, `llmgw ctl model update`
+41
View File
@@ -86,6 +86,7 @@ rather than repeated.
| `url` | `string` | [External](#external) |
| `model` | `string` | [External](#external) |
| `api_key_file` | `string` | [External](#external) |
| `peer` | `string` | [External](#external) |
| `budgets` | `[]FacilityBudget` | [Facility](#facility) |
| `margin` | `float64` | [Facility](#facility) |
| `gpu_watts` | `float64` | [Facility](#facility) |
@@ -95,6 +96,8 @@ rather than repeated.
| `domain` | `string` | [FacilityBudget](#facilitybudget) |
| `cap_watts` | `float64` | [FacilityBudget](#facilitybudget) |
| `other_watts` | `float64` | [FacilityBudget](#facilitybudget) |
| `name` | `string` | [Federation](#federation) |
| `interval` | `time.Duration` | [Federation](#federation) |
| `format` | `string` | [GuardModel](#guardmodel) |
| `categories` | `[]string` | [GuardModel](#guardmodel) |
| `model` | `string` | [JudgePolicy](#judgepolicy) |
@@ -293,6 +296,7 @@ rather than repeated.
| `control` | `Control` | [Server](#server) |
| `models` | `[]catalog.Model` | [Server](#server) |
| `facility` | `Facility` | [Server](#server) |
| `federation` | `Federation` | [Server](#server) |
| `endpoints` | `[]Endpoint` | [Server](#server) |
| `bin` | `string` | [SimConfig](#simconfig) |
| `python` | `string` | [TRTLLMConfig](#trtllmconfig) |
@@ -863,6 +867,15 @@ APIKeyFile holds the provider's key, on the control plane. A file
rather than the key itself, so the key is in neither the catalog, the
store nor the audit log; empty for a provider that asks for none.
### `peer`
`string`
Peer names the federated llmgw cluster this model is offered by, for
a model that federation added (`llmgw ctl federation add`) rather
than one written by hand. Such a model is named PEER/MODEL, comes and
goes with the peer's own list, and is sent only while the peer is up.
## Facility
### `budgets`
@@ -932,6 +945,24 @@ one feed carries alone, derated as the facility derates it (usually
OtherWatts is drawn in the domain by what llmgw does not place on:
switches, storage, machines outside the cluster.
## Federation
### `name`
`string`
Name is this cluster as a peer reads it, in the federated_from tag
sent with a request to a peer set to receive tags (`ctl federation add
--send-tags`). Empty is this machine's host name.
### `interval`
`time.Duration`
Interval is how often each peer is asked whether it is ready, timed,
and what it offers. A peer that does not answer is taken out of
routing until it does. Zero is 10s.
## GuardModel
### `format`
@@ -2482,6 +2513,16 @@ maintenance is `llmgw ctl facility hold`, not configuration.
Takes a [Facility](#facility) block.
### `federation`
`Federation`
Federation is how this cluster takes part in federation with other
llmgw clusters. The peers themselves are added at runtime, with
`llmgw ctl federation add`, and kept in the store.
Takes a [Federation](#federation) block.
### `endpoints`
`[]Endpoint`