Federation: another llmgw cluster as a provider, its models as PEER/MODEL
ctl federation add/list/remove/sync keeps peers in the store (migration 0029); their /v1/models are offered here as external models named PEER/MODEL and kept in step, a model still named by a fallback kept withdrawn. Peers are checked at /ready, timed, and taken out of routing while down. Requests to a peer carry x-llmgw-hop and are refused (508) when they come back to a cluster they passed through; residency and no_store keep requests off peers; X-Request-Id and, with --send-tags, attribution tags go along. ctl model fallbacks sets overflow to a peer. Console view, metrics, admin API routes and docs. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
40e9fd2ab1
33 files changed
+2944
-23
No files matched your search
@@ -8,6 +8,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
@@ -56,13 +57,14 @@ type Param struct {
|
||||
// PathParams describes the parameters in Routes' paths, by the segment
|
||||
// before them and their name.
|
||||
var PathParams = map[string]string{
|
||||
"models/{name}": "The model's name in the catalog.",
|
||||
"nodes/{id}": "The node's ID, as `llmgw ctl nodes` lists it.",
|
||||
"jobs/{id}": "The job's ID, as starting it answered.",
|
||||
"runs/{id}": "The quality run's ID, as starting it answered.",
|
||||
"requests/{id}": "The request's ID: the x-request-id a client was answered with.",
|
||||
"fabrics/{id}": "The physical fabric's ID, as GET /api/v1/fabrics lists it.",
|
||||
"quotas/{scope}": "The scope: org:NAME, team:ORG/TEAM, project:ORG/TEAM/PROJECT, or their IDs after the colon.",
|
||||
"models/{name}": "The model's name in the catalog.",
|
||||
"nodes/{id}": "The node's ID, as `llmgw ctl nodes` lists it.",
|
||||
"jobs/{id}": "The job's ID, as starting it answered.",
|
||||
"runs/{id}": "The quality run's ID, as starting it answered.",
|
||||
"requests/{id}": "The request's ID: the x-request-id a client was answered with.",
|
||||
"fabrics/{id}": "The physical fabric's ID, as GET /api/v1/fabrics lists it.",
|
||||
"quotas/{scope}": "The scope: org:NAME, team:ORG/TEAM, project:ORG/TEAM/PROJECT, or their IDs after the colon.",
|
||||
"federation/{peer}": "The federation peer's name, as `llmgw ctl federation list` lists it.",
|
||||
}
|
||||
|
||||
// Routes is every endpoint Register mounts.
|
||||
@@ -210,6 +212,21 @@ var Routes = []Route{
|
||||
{Method: "DELETE", Path: "/api/v1/models/{name}/eval-gate", Summary: "Stop holding a model's updates to a suite",
|
||||
Response: EvalGateView{}},
|
||||
|
||||
// Federation: other llmgw clusters as providers.
|
||||
{Method: "GET", Path: "/api/v1/federation", Summary: "This cluster's id, and its federation peers with their health and models",
|
||||
Response: FederationView{}},
|
||||
{Method: "POST", Path: "/api/v1/federation", Summary: "Add another llmgw cluster as a peer, or change one",
|
||||
Description: "The peer is checked first: its /ready, timed, and its /v1/models with the key. Its models join the catalog as PEER/MODEL.",
|
||||
Request: federation.AddRequest{}, Response: federation.Status{}, Status: http.StatusCreated},
|
||||
{Method: "DELETE", Path: "/api/v1/federation/{peer}", Summary: "Remove a federation peer and its models",
|
||||
Description: "Refused while a model here falls back to one of the peer's.",
|
||||
Response: Removed{}},
|
||||
{Method: "POST", Path: "/api/v1/federation/{peer}/sync", Summary: "Check a peer now, rather than at the next interval",
|
||||
Response: federation.Status{}},
|
||||
{Method: "PUT", Path: "/api/v1/models/{name}/fallbacks", Summary: "Replace the models a model falls back to when it has no capacity",
|
||||
Description: "A model added at runtime only. A federation peer's model, PEER/MODEL, sends the overflow to the other cluster.",
|
||||
Request: FallbacksRequest{}, Response: FallbacksView{}},
|
||||
|
||||
// Tenancy: quotas and guardrails.
|
||||
{Method: "GET", Path: "/api/v1/quotas", Summary: "Every capacity quota, with what its scope holds now",
|
||||
Response: []QuotaView{}},
|
||||
|
||||
Reference in new issue
Block a user