Federation: another llmgw cluster as a provider, its models as PEER/MODEL
ctl federation add/list/remove/sync keeps peers in the store (migration 0029); their /v1/models are offered here as external models named PEER/MODEL and kept in step, a model still named by a fallback kept withdrawn. Peers are checked at /ready, timed, and taken out of routing while down. Requests to a peer carry x-llmgw-hop and are refused (508) when they come back to a cluster they passed through; residency and no_store keep requests off peers; X-Request-Id and, with --send-tags, attribution tags go along. ctl model fallbacks sets overflow to a peer. Console view, metrics, admin API routes and docs. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed
1 parent
d41104a452
commit
40e9fd2ab1
33 files changed
+2944
-23
No files matched your search
@@ -38,6 +38,9 @@
|
||||
{
|
||||
"name": "fabrics"
|
||||
},
|
||||
{
|
||||
"name": "federation"
|
||||
},
|
||||
{
|
||||
"name": "guard"
|
||||
},
|
||||
@@ -476,6 +479,157 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/federation": {
|
||||
"get": {
|
||||
"operationId": "getFederation",
|
||||
"summary": "This cluster's id, and its federation peers with their health and models",
|
||||
"tags": [
|
||||
"federation"
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/FederationView"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"$ref": "#/components/responses/Unauthorized"
|
||||
},
|
||||
"default": {
|
||||
"$ref": "#/components/responses/Error"
|
||||
}
|
||||
}
|
||||
},
|
||||
"post": {
|
||||
"operationId": "postFederation",
|
||||
"summary": "Add another llmgw cluster as a peer, or change one",
|
||||
"description": "The peer is checked first: its /ready, timed, and its /v1/models with the key. Its models join the catalog as PEER/MODEL.",
|
||||
"tags": [
|
||||
"federation"
|
||||
],
|
||||
"parameters": [
|
||||
{
|
||||
"$ref": "#/components/parameters/Actor"
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"required": true,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/federation.AddRequest"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "OK",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/federation.Status"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"$ref": "#/components/responses/Unauthorized"
|
||||
},
|
||||
"default": {
|
||||
"$ref": "#/components/responses/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/federation/{peer}": {
|
||||
"delete": {
|
||||
"operationId": "deleteFederationPeer",
|
||||
"summary": "Remove a federation peer and its models",
|
||||
"description": "Refused while a model here falls back to one of the peer's.",
|
||||
"tags": [
|
||||
"federation"
|
||||
],
|
||||
"parameters": [
|
||||
{
|
||||
"name": "peer",
|
||||
"in": "path",
|
||||
"required": true,
|
||||
"description": "The federation peer's name, as `llmgw ctl federation list` lists it.",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/parameters/Actor"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/Removed"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"$ref": "#/components/responses/Unauthorized"
|
||||
},
|
||||
"default": {
|
||||
"$ref": "#/components/responses/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/federation/{peer}/sync": {
|
||||
"post": {
|
||||
"operationId": "postFederationPeerSync",
|
||||
"summary": "Check a peer now, rather than at the next interval",
|
||||
"tags": [
|
||||
"federation"
|
||||
],
|
||||
"parameters": [
|
||||
{
|
||||
"name": "peer",
|
||||
"in": "path",
|
||||
"required": true,
|
||||
"description": "The federation peer's name, as `llmgw ctl federation list` lists it.",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/parameters/Actor"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/federation.Status"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"$ref": "#/components/responses/Unauthorized"
|
||||
},
|
||||
"default": {
|
||||
"$ref": "#/components/responses/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/guard": {
|
||||
"get": {
|
||||
"operationId": "getGuard",
|
||||
@@ -1324,6 +1478,58 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/models/{name}/fallbacks": {
|
||||
"put": {
|
||||
"operationId": "putModelsNameFallbacks",
|
||||
"summary": "Replace the models a model falls back to when it has no capacity",
|
||||
"description": "A model added at runtime only. A federation peer's model, PEER/MODEL, sends the overflow to the other cluster.",
|
||||
"tags": [
|
||||
"models"
|
||||
],
|
||||
"parameters": [
|
||||
{
|
||||
"name": "name",
|
||||
"in": "path",
|
||||
"required": true,
|
||||
"description": "The model's name in the catalog.",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/parameters/Actor"
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"required": true,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/FallbacksRequest"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/FallbacksView"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"$ref": "#/components/responses/Unauthorized"
|
||||
},
|
||||
"default": {
|
||||
"$ref": "#/components/responses/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/models/{name}/headroom": {
|
||||
"get": {
|
||||
"operationId": "getModelsNameHeadroom",
|
||||
@@ -3202,6 +3408,49 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"FallbacksRequest": {
|
||||
"type": "object",
|
||||
"description": "FallbacksRequest replaces a model's fallbacks; empty takes them all away.",
|
||||
"properties": {
|
||||
"fallbacks": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"FallbacksView": {
|
||||
"type": "object",
|
||||
"description": "FallbacksView is a model's fallbacks, as they are now.",
|
||||
"properties": {
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
"fallbacks": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"FederationView": {
|
||||
"type": "object",
|
||||
"description": "FederationView is this cluster's id and its peers.",
|
||||
"properties": {
|
||||
"cluster": {
|
||||
"type": "string",
|
||||
"description": "Cluster is this cluster's own id, which its peers see in the hop header."
|
||||
},
|
||||
"peers": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/federation.Status"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"FetchView": {
|
||||
"type": "object",
|
||||
"description": "FetchView is a download in flight.",
|
||||
@@ -5010,6 +5259,10 @@
|
||||
"api_key_file": {
|
||||
"type": "string",
|
||||
"description": "APIKeyFile holds the provider's key, on the control plane. A file rather than the key itself, so the key is in neither the catalog, the store nor the audit log; empty for a provider that asks for none."
|
||||
},
|
||||
"peer": {
|
||||
"type": "string",
|
||||
"description": "Peer names the federated llmgw cluster this model is offered by, for a model that federation added (`llmgw ctl federation add`) rather than one written by hand. Such a model is named PEER/MODEL, comes and goes with the peer's own list, and is sent only while the peer is up."
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -5634,6 +5887,104 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"federation.AddRequest": {
|
||||
"type": "object",
|
||||
"description": "AddRequest adds a peer.",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string",
|
||||
"description": "Name is what this cluster calls the peer, and the prefix of its models: NAME/MODEL."
|
||||
},
|
||||
"url": {
|
||||
"type": "string",
|
||||
"description": "URL is the peer gateway's client address, without /v1."
|
||||
},
|
||||
"key_file": {
|
||||
"type": "string",
|
||||
"description": "KeyFile is a file on the control plane holding a key the peer issued to this cluster (or env:NAME, vault:PATH#FIELD)."
|
||||
},
|
||||
"send_tags": {
|
||||
"type": "boolean",
|
||||
"description": "SendTags makes requests sent to the peer carry x-llmgw-tags naming this cluster and the sending project. Off unless asked: the peer learns who in this cluster is asking."
|
||||
}
|
||||
}
|
||||
},
|
||||
"federation.ModelStatus": {
|
||||
"type": "object",
|
||||
"description": "ModelStatus is one of a peer's models: its name here, the peer's own, and whether the peer has stopped offering it.",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"upstream": {
|
||||
"type": "string"
|
||||
},
|
||||
"withdrawn": {
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
},
|
||||
"federation.Status": {
|
||||
"type": "object",
|
||||
"description": "Status is a peer as `llmgw ctl federation list` shows it.",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
},
|
||||
"key_file": {
|
||||
"type": "string"
|
||||
},
|
||||
"send_tags": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"cluster": {
|
||||
"type": "string",
|
||||
"description": "Cluster is the peer's own cluster id."
|
||||
},
|
||||
"up": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"checked": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"description": "Checked is when; zero before the first check."
|
||||
},
|
||||
"latency_ns": {
|
||||
"type": "integer",
|
||||
"format": "int64",
|
||||
"description": "Latency is how long the peer took to answer /ready: the network to the other site and the peer's front, not a model's speed.\n\nA duration, in nanoseconds."
|
||||
},
|
||||
"error": {
|
||||
"type": "string",
|
||||
"description": "Error is why the peer is down, empty while it is up."
|
||||
},
|
||||
"latency_ms": {
|
||||
"type": "number",
|
||||
"description": "LatencyMs is Latency in milliseconds, for reading."
|
||||
},
|
||||
"synced_at": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"description": "SyncedAt is when its models last changed here."
|
||||
},
|
||||
"models": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/federation.ModelStatus"
|
||||
}
|
||||
},
|
||||
"created_at": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"created_by": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"guard.Attachment": {
|
||||
"type": "object",
|
||||
"description": "Attachment ties a policy to a part of the tenancy tree: scope is org, team, project, user or key, and ScopeID which one.",
|
||||
|
||||
Reference in new issue
Block a user