Federation: another llmgw cluster as a provider, its models as PEER/MODEL
ctl federation add/list/remove/sync keeps peers in the store (migration 0029); their /v1/models are offered here as external models named PEER/MODEL and kept in step, a model still named by a fallback kept withdrawn. Peers are checked at /ready, timed, and taken out of routing while down. Requests to a peer carry x-llmgw-hop and are refused (508) when they come back to a cluster they passed through; residency and no_store keep requests off peers; X-Request-Id and, with --send-tags, attribution tags go along. ctl model fallbacks sets overflow to a peer. Console view, metrics, admin API routes and docs. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
40e9fd2ab1
33 files changed
+2944
-23
No files matched your search
@@ -0,0 +1,155 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
||||
)
|
||||
|
||||
// Federation, as the router sees it: requests sent to another llmgw cluster
|
||||
// say which clusters they have passed through, and a request that comes back
|
||||
// to one of them is refused rather than sent round again.
|
||||
|
||||
const (
|
||||
// hopHeader lists the cluster ids a request has passed through, oldest
|
||||
// first: "3f9a…, 81c0…". Each gateway adds its own when it sends the
|
||||
// request to a peer.
|
||||
hopHeader = "x-llmgw-hop"
|
||||
// clusterHeader says this gateway's cluster id, on /v1/models, which is
|
||||
// where a peer learns it.
|
||||
clusterHeader = "x-llmgw-cluster"
|
||||
// maxHops bounds a chain of peers whatever their ids say: a backstop
|
||||
// for two clusters given the same id by a restore of one's store into
|
||||
// the other, where the loop check cannot tell them apart.
|
||||
maxHops = 4
|
||||
)
|
||||
|
||||
// federationID is this cluster as its peers know it: its id, in the hop
|
||||
// header, and its name, in the tags a peer may be sent.
|
||||
type federationID struct {
|
||||
id, name string
|
||||
}
|
||||
|
||||
// SetFederationIdentity says which cluster this gateway is. Without it,
|
||||
// requests are never refused as loops and peers are sent no hop of this one.
|
||||
func (r *Router) SetFederationIdentity(id, name string) {
|
||||
r.federation.Store(&federationID{id: id, name: name})
|
||||
}
|
||||
|
||||
func (r *Router) clusterID() federationID {
|
||||
if f := r.federation.Load(); f != nil {
|
||||
return *f
|
||||
}
|
||||
return federationID{}
|
||||
}
|
||||
|
||||
// hops reads the clusters a request has passed through.
|
||||
func hops(h http.Header) []string {
|
||||
var out []string
|
||||
for _, v := range h.Values(hopHeader) {
|
||||
for _, p := range strings.Split(v, ",") {
|
||||
if p = strings.TrimSpace(p); p != "" {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// refuseLoops answers 508 to a request that has already passed through this
|
||||
// cluster, or through more clusters than any sensible chain of peers has.
|
||||
//
|
||||
// Before anything else, authentication included: such a request was sent by
|
||||
// a peer that had no capacity, and answering it at once is what lets that
|
||||
// peer move on to its next fallback instead of waiting out a queue here.
|
||||
func (r *Router) refuseLoops(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
if req.Header.Get(hopHeader) == "" || !strings.HasPrefix(req.URL.Path, "/v1/") {
|
||||
next.ServeHTTP(w, req)
|
||||
return
|
||||
}
|
||||
seen := hops(req.Header)
|
||||
self := r.clusterID().id
|
||||
switch {
|
||||
case self != "" && slices.Contains(seen, self):
|
||||
writeErrorCode(w, http.StatusLoopDetected, "invalid_request_error", "federation_loop",
|
||||
"this request has already passed through this cluster; it is not sent round again")
|
||||
case len(seen) >= maxHops:
|
||||
writeErrorCode(w, http.StatusLoopDetected, "invalid_request_error", "federation_loop",
|
||||
"this request has passed through too many federated clusters")
|
||||
default:
|
||||
next.ServeHTTP(w, req)
|
||||
return
|
||||
}
|
||||
r.m.observeRequest("", "", http.StatusLoopDetected, 0, nil, 0)
|
||||
})
|
||||
}
|
||||
|
||||
// withoutVisited drops the endpoints of peers a request has already passed
|
||||
// through: sending it there would only be refused as a loop, after a round
|
||||
// trip to the other site.
|
||||
func withoutVisited(eps []endpoint.Snapshot, h http.Header) []endpoint.Snapshot {
|
||||
seen := hops(h)
|
||||
if len(seen) == 0 {
|
||||
return eps
|
||||
}
|
||||
return slices.DeleteFunc(slices.Clone(eps), func(e endpoint.Snapshot) bool {
|
||||
return e.Peer != "" && e.PeerCluster != "" && slices.Contains(seen, e.PeerCluster)
|
||||
})
|
||||
}
|
||||
|
||||
// peerHeaders are what a request sent to a peer carries beyond what any
|
||||
// upstream is sent: the hop header with this cluster added, and, when the
|
||||
// peer is set to receive them, tags naming this cluster and the project.
|
||||
//
|
||||
// The peer sees one key, this cluster's, so without the tags every request
|
||||
// from here is one tenant to it. They are off by default because they tell
|
||||
// another site who in this one is asking.
|
||||
func (r *Router) peerHeaders(ctx context.Context, ep endpoint.Snapshot, h http.Header) http.Header {
|
||||
out := http.Header{}
|
||||
self := r.clusterID()
|
||||
chain := hops(h)
|
||||
if self.id != "" {
|
||||
chain = append(chain, self.id)
|
||||
}
|
||||
if len(chain) > 0 {
|
||||
out.Set(hopHeader, strings.Join(chain, ", "))
|
||||
}
|
||||
if ep.PeerTags {
|
||||
tags := []string{"federated_from=" + tagValue(self.name)}
|
||||
if p, ok := auth.FromContext(ctx); ok {
|
||||
for _, kv := range [][2]string{{"org", p.OrgID}, {"team", p.TeamID}, {"project", p.ProjectID}} {
|
||||
if kv[1] != "" {
|
||||
tags = append(tags, kv[0]+"="+tagValue(kv[1]))
|
||||
}
|
||||
}
|
||||
}
|
||||
out.Set(tagsHeader, strings.Join(tags, ","))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// tagValue keeps a value inside one tag: x-llmgw-tags is comma-separated.
|
||||
func tagValue(v string) string {
|
||||
v = strings.NewReplacer(",", "_", "=", "_").Replace(v)
|
||||
if len(v) > maxTagValue {
|
||||
v = v[:maxTagValue]
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// localEndpoints counts the endpoints that are not a federated peer's: what
|
||||
// this cluster can serve with itself.
|
||||
func (r *Router) localEndpoints() int {
|
||||
n := 0
|
||||
for _, s := range r.reg.Snapshots("") {
|
||||
if s.Peer == "" {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
Reference in new issue
Block a user