Federation: another llmgw cluster as a provider, its models as PEER/MODEL
ctl federation add/list/remove/sync keeps peers in the store (migration 0029); their /v1/models are offered here as external models named PEER/MODEL and kept in step, a model still named by a fallback kept withdrawn. Peers are checked at /ready, timed, and taken out of routing while down. Requests to a peer carry x-llmgw-hop and are refused (508) when they come back to a cluster they passed through; residency and no_store keep requests off peers; X-Request-Id and, with --send-tags, attribution tags go along. ctl model fallbacks sets overflow to a peer. Console view, metrics, admin API routes and docs. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
40e9fd2ab1
33 files changed
+2944
-23
No files matched your search
@@ -149,6 +149,10 @@ type Request struct {
|
||||
Body []byte
|
||||
// Header carries headers to forward (authorization is deliberately not).
|
||||
Header http.Header
|
||||
// Extra are headers set on the upstream request beyond those forwarded
|
||||
// from the client: what the router itself says to this upstream, such
|
||||
// as the hop header a federated cluster is sent.
|
||||
Extra http.Header
|
||||
// APIKey is the engine's own key, sent in place of the client's.
|
||||
APIKey string
|
||||
// Stream says whether an SSE response is expected.
|
||||
@@ -250,6 +254,9 @@ func (p *Proxy) Do(ctx context.Context, w http.ResponseWriter, r Request) (Outco
|
||||
return Outcome{Retryable: false}, err
|
||||
}
|
||||
req.Header = cloneForUpstream(r.Header)
|
||||
for k, vs := range r.Extra {
|
||||
req.Header[http.CanonicalHeaderKey(k)] = vs
|
||||
}
|
||||
ct := r.ContentType
|
||||
if ct == "" {
|
||||
ct = "application/json"
|
||||
@@ -620,9 +627,15 @@ var hopByHop = map[string]bool{
|
||||
"Upgrade": true,
|
||||
}
|
||||
|
||||
// copyHeaders gives the client the upstream's response headers.
|
||||
//
|
||||
// A header the gateway has already set is its own answer and is kept: an
|
||||
// upstream that is itself an llmgw — a federated peer — sends X-Request-Id
|
||||
// and x-llmgw-served-model of its own, and the client would otherwise be
|
||||
// given both.
|
||||
func copyHeaders(dst, src http.Header) {
|
||||
for k, vs := range src {
|
||||
if hopByHop[http.CanonicalHeaderKey(k)] {
|
||||
if hopByHop[http.CanonicalHeaderKey(k)] || len(dst.Values(k)) > 0 {
|
||||
continue
|
||||
}
|
||||
for _, v := range vs {
|
||||
|
||||
Reference in new issue
Block a user