Federation: another llmgw cluster as a provider, its models as PEER/MODEL

ctl federation add/list/remove/sync keeps peers in the store (migration
0029); their /v1/models are offered here as external models named
PEER/MODEL and kept in step, a model still named by a fallback kept
withdrawn. Peers are checked at /ready, timed, and taken out of routing
while down. Requests to a peer carry x-llmgw-hop and are refused (508) when
they come back to a cluster they passed through; residency and no_store
keep requests off peers; X-Request-Id and, with --send-tags, attribution
tags go along. ctl model fallbacks sets overflow to a peer. Console view,
metrics, admin API routes and docs.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:21:30 +02:00
1 parent d41104a452
commit 40e9fd2ab1
33 files changed
+2944 -23

No files matched your search

+7
View File
@@ -41,6 +41,13 @@ type Info struct {
// from the catalog's name: an external provider's. Requests are rewritten
// to it. Empty sends the name as the client did.
UpstreamModel string
// Peer names the federated llmgw cluster behind an external endpoint,
// empty for every other. PeerCluster is that cluster's own id, which a
// request that has already passed through it carries in its hop header,
// and PeerTags whether requests sent there name who sent them.
Peer string
PeerCluster string
PeerTags bool
}
// EngineExternal marks an endpoint that is a provider outside the cluster.
+155
View File
@@ -0,0 +1,155 @@
package router
import (
"context"
"net/http"
"slices"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
// Federation, as the router sees it: requests sent to another llmgw cluster
// say which clusters they have passed through, and a request that comes back
// to one of them is refused rather than sent round again.
const (
// hopHeader lists the cluster ids a request has passed through, oldest
// first: "3f9a…, 81c0…". Each gateway adds its own when it sends the
// request to a peer.
hopHeader = "x-llmgw-hop"
// clusterHeader says this gateway's cluster id, on /v1/models, which is
// where a peer learns it.
clusterHeader = "x-llmgw-cluster"
// maxHops bounds a chain of peers whatever their ids say: a backstop
// for two clusters given the same id by a restore of one's store into
// the other, where the loop check cannot tell them apart.
maxHops = 4
)
// federationID is this cluster as its peers know it: its id, in the hop
// header, and its name, in the tags a peer may be sent.
type federationID struct {
id, name string
}
// SetFederationIdentity says which cluster this gateway is. Without it,
// requests are never refused as loops and peers are sent no hop of this one.
func (r *Router) SetFederationIdentity(id, name string) {
r.federation.Store(&federationID{id: id, name: name})
}
func (r *Router) clusterID() federationID {
if f := r.federation.Load(); f != nil {
return *f
}
return federationID{}
}
// hops reads the clusters a request has passed through.
func hops(h http.Header) []string {
var out []string
for _, v := range h.Values(hopHeader) {
for _, p := range strings.Split(v, ",") {
if p = strings.TrimSpace(p); p != "" {
out = append(out, p)
}
}
}
return out
}
// refuseLoops answers 508 to a request that has already passed through this
// cluster, or through more clusters than any sensible chain of peers has.
//
// Before anything else, authentication included: such a request was sent by
// a peer that had no capacity, and answering it at once is what lets that
// peer move on to its next fallback instead of waiting out a queue here.
func (r *Router) refuseLoops(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
if req.Header.Get(hopHeader) == "" || !strings.HasPrefix(req.URL.Path, "/v1/") {
next.ServeHTTP(w, req)
return
}
seen := hops(req.Header)
self := r.clusterID().id
switch {
case self != "" && slices.Contains(seen, self):
writeErrorCode(w, http.StatusLoopDetected, "invalid_request_error", "federation_loop",
"this request has already passed through this cluster; it is not sent round again")
case len(seen) >= maxHops:
writeErrorCode(w, http.StatusLoopDetected, "invalid_request_error", "federation_loop",
"this request has passed through too many federated clusters")
default:
next.ServeHTTP(w, req)
return
}
r.m.observeRequest("", "", http.StatusLoopDetected, 0, nil, 0)
})
}
// withoutVisited drops the endpoints of peers a request has already passed
// through: sending it there would only be refused as a loop, after a round
// trip to the other site.
func withoutVisited(eps []endpoint.Snapshot, h http.Header) []endpoint.Snapshot {
seen := hops(h)
if len(seen) == 0 {
return eps
}
return slices.DeleteFunc(slices.Clone(eps), func(e endpoint.Snapshot) bool {
return e.Peer != "" && e.PeerCluster != "" && slices.Contains(seen, e.PeerCluster)
})
}
// peerHeaders are what a request sent to a peer carries beyond what any
// upstream is sent: the hop header with this cluster added, and, when the
// peer is set to receive them, tags naming this cluster and the project.
//
// The peer sees one key, this cluster's, so without the tags every request
// from here is one tenant to it. They are off by default because they tell
// another site who in this one is asking.
func (r *Router) peerHeaders(ctx context.Context, ep endpoint.Snapshot, h http.Header) http.Header {
out := http.Header{}
self := r.clusterID()
chain := hops(h)
if self.id != "" {
chain = append(chain, self.id)
}
if len(chain) > 0 {
out.Set(hopHeader, strings.Join(chain, ", "))
}
if ep.PeerTags {
tags := []string{"federated_from=" + tagValue(self.name)}
if p, ok := auth.FromContext(ctx); ok {
for _, kv := range [][2]string{{"org", p.OrgID}, {"team", p.TeamID}, {"project", p.ProjectID}} {
if kv[1] != "" {
tags = append(tags, kv[0]+"="+tagValue(kv[1]))
}
}
}
out.Set(tagsHeader, strings.Join(tags, ","))
}
return out
}
// tagValue keeps a value inside one tag: x-llmgw-tags is comma-separated.
func tagValue(v string) string {
v = strings.NewReplacer(",", "_", "=", "_").Replace(v)
if len(v) > maxTagValue {
v = v[:maxTagValue]
}
return v
}
// localEndpoints counts the endpoints that are not a federated peer's: what
// this cluster can serve with itself.
func (r *Router) localEndpoints() int {
n := 0
for _, s := range r.reg.Snapshots("") {
if s.Peer == "" {
n++
}
}
return n
}
+594
View File
@@ -0,0 +1,594 @@
package router
import (
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"log/slog"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// fedCluster is one llmgw cluster, whole but for its nodes: a router serving
// on a loopback port, the control plane that publishes to it, the model
// service and federation. Its one model is served by a fake engine standing
// in for the replica a node would run.
type fedCluster struct {
name, id string
model string
rt *Router
srv *httptest.Server
ctl *control.Controller
models *modelsvc.Service
fed *federation.Service
engine *fakeEngine
// key is a client's key here, in project.
key, project string
keys int
}
type fedNoDispatch struct{}
func (fedNoDispatch) StartShard(context.Context, reconciler.ShardCommand) error { return nil }
func (fedNoDispatch) StopShard(context.Context, string, string) error { return nil }
func (fedNoDispatch) DrainShard(context.Context, string, string) error { return nil }
// withLocal is the sink between a control plane with no nodes and its
// router: what the control plane publishes, and the cluster's own engine.
type withLocal struct {
rt *Router
local endpoint.Info
}
func (s withLocal) SetEndpoints(ctx context.Context, infos []endpoint.Info) {
s.rt.SetEndpoints(ctx, append(slices.Clone(infos), s.local))
}
func (s withLocal) SetCatalog(c *catalog.Catalog) { s.rt.SetCatalog(c) }
func newFedCluster(t *testing.T, name, model string) *fedCluster {
t.Helper()
c := &fedCluster{name: name, id: name + "-cluster-id", model: model}
c.engine = newFakeEngine(t, model)
cfg := baseConfig()
cfg.Auth.Enabled = true
cfg.Auth.Store = filepath.Join(t.TempDir(), "store.db")
cfg.Auth.FlushInterval = 20 * time.Millisecond
cfg.Auth.BatchSize = 1
cfg.Router.Retries = 0
c.rt = newTestRouter(t, cfg)
c.rt.SetFederationIdentity(c.id, name)
empty, err := catalog.New(nil)
if err != nil {
t.Fatal(err)
}
log := slog.New(slog.DiscardHandler)
sink := withLocal{rt: c.rt, local: endpoint.Info{ID: name + "-replica", URL: c.engine.srv.URL, Models: []string{model}}}
c.ctl = control.New(control.DefaultConfig(), catalog.NewLive(empty), fedNoDispatch{}, sink, log)
c.models = modelsvc.New(nil, c.rt.store, c.ctl, log)
c.fed = federation.New(federation.Config{Self: c.id, Interval: time.Hour}, c.rt.store, c.models, c.ctl.Catalog, log)
if err := c.fed.Load(t.Context()); err != nil {
t.Fatal(err)
}
c.models.SetFederated(c.fed.Models)
c.ctl.SetPeers(c.fed)
c.srv = httptest.NewServer(c.rt.Handler())
t.Cleanup(c.srv.Close)
if err := c.models.Add(t.Context(), catModel(model, nil), "test"); err != nil {
t.Fatal(err)
}
// The control plane has published nothing yet — an empty list is no
// change — so the engine is put in by hand the first time.
sink.SetEndpoints(t.Context(), nil)
org, err := c.rt.store.CreateOrg(t.Context(), "acme-"+name)
if err != nil {
t.Fatal(err)
}
team, err := c.rt.store.CreateTeam(t.Context(), org, "platform")
if err != nil {
t.Fatal(err)
}
if c.project, err = c.rt.store.CreateProject(t.Context(), team, "gateway"); err != nil {
t.Fatal(err)
}
c.key = c.issueKey(t)
return c
}
// issueKey makes a key in the cluster's project, as `llmgw keys create`
// would, and returns its secret.
func (c *fedCluster) issueKey(t *testing.T) string {
t.Helper()
c.keys++
secret := store.KeyPrefix + strings.Repeat(string(rune('a'+c.keys)), 43)
sum := sha256.Sum256([]byte(secret))
if err := c.rt.store.CreateKey(t.Context(), store.Key{
ID: fmt.Sprintf("%s-key-%d", c.name, c.keys), Hash: sum[:], Prefix: "pfx" + c.name, Name: "test",
ProjectID: c.project, CreatedAt: time.Now(),
}); err != nil {
t.Fatal(err)
}
return secret
}
// federate makes other a peer of c, with a key other issued to c.
func (c *fedCluster) federate(t *testing.T, other *fedCluster, sendTags bool) federation.Status {
t.Helper()
keyFile := filepath.Join(t.TempDir(), other.name+".key")
if err := os.WriteFile(keyFile, []byte(other.issueKey(t)+"\n"), 0o600); err != nil {
t.Fatal(err)
}
st, err := c.fed.Add(t.Context(), federation.AddRequest{Name: other.name, URL: other.srv.URL,
KeyFile: keyFile, SendTags: sendTags}, "test")
if err != nil {
t.Fatal(err)
}
c.ctl.Publish(t.Context())
return st
}
// ask sends a chat completion to the cluster as its client.
func (c *fedCluster) ask(t *testing.T, model string, hdr map[string]string) *httptest.ResponseRecorder {
t.Helper()
h := map[string]string{"Authorization": "Bearer " + c.key}
for k, v := range hdr {
h[k] = v
}
return post(t, c.rt.Handler(), map[string]any{"model": model,
"messages": []any{map[string]any{"role": "user", "content": "hello"}}}, h)
}
// usageOf waits for the cluster's record of a request.
func (c *fedCluster) usageOf(t *testing.T, requestID string) store.Usage {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
u, ok, err := c.rt.store.UsageByRequest(t.Context(), requestID)
if err != nil {
t.Fatal(err)
}
if ok {
return u
}
time.Sleep(20 * time.Millisecond)
}
t.Fatalf("%s has no record of request %s", c.name, requestID)
return store.Usage{}
}
func listedModels(t *testing.T, c *fedCluster) (map[string]map[string]any, http.Header) {
t.Helper()
req := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
req.Header.Set("Authorization", "Bearer "+c.key)
rec := httptest.NewRecorder()
c.rt.Handler().ServeHTTP(rec, req)
var list struct {
Data []map[string]any `json:"data"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &list); err != nil {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
out := map[string]map[string]any{}
for _, m := range list.Data {
out[fmt.Sprint(m["id"])] = m
}
return out, rec.Header()
}
func TestAPeersModelsAreOfferedHereUnderItsNameAndFollowItsList(t *testing.T) {
west := newFedCluster(t, "west", "qwen-west")
east := newFedCluster(t, "east", "qwen-east")
st := west.federate(t, east, false)
if !st.Up || st.Cluster != east.id || st.Latency <= 0 {
t.Errorf("east after adding: %+v", st)
}
listed, hdr := listedModels(t, west)
if listed["east/qwen-east"] == nil || listed["east/qwen-east"]["peer"] != "east" {
t.Fatalf("west offers %v; east's model should be there as east/qwen-east", slices.Collect(mapsKeys(listed)))
}
if hdr.Get(clusterHeader) != west.id {
t.Errorf("west's /v1/models says cluster %q", hdr.Get(clusterHeader))
}
if m, ok := west.ctl.Catalog().Resolve("east/qwen-east"); !ok || !m.IsPeer() || m.External.Model != "qwen-east" {
t.Errorf("west's catalog has %+v", m)
}
// Asked for by its name here, answered by east's engine under east's name.
rec := west.ask(t, "east/qwen-east", nil)
if rec.Code != http.StatusOK || east.engine.calls.Load() != 1 {
t.Fatalf("%d %s; east's engine was called %d times", rec.Code, rec.Body.String(), east.engine.calls.Load())
}
var sent map[string]any
_ = json.Unmarshal(east.engine.last.Load().([]byte), &sent)
if sent["model"] != "qwen-east" {
t.Errorf("east's engine was asked for %v", sent["model"])
}
// East adds a model, then drops its first: west follows at its next check.
if err := east.models.Add(t.Context(), catModel("qwen-east-2", nil), "test"); err != nil {
t.Fatal(err)
}
if err := east.models.Remove(t.Context(), "qwen-east"); err != nil {
t.Fatal(err)
}
east.rt.SetEndpoints(t.Context(), []endpoint.Info{{ID: "e2", URL: east.engine.srv.URL, Models: []string{"qwen-east-2"}}})
if _, err := west.fed.Sync(t.Context(), "east"); err != nil {
t.Fatal(err)
}
west.ctl.Publish(t.Context())
if _, ok := west.ctl.Catalog().Resolve("east/qwen-east-2"); !ok {
t.Error("a model east added is not offered here")
}
if _, ok := west.ctl.Catalog().Resolve("east/qwen-east"); ok {
t.Error("a model east dropped, which nothing here names, is still offered")
}
}
func mapsKeys(m map[string]map[string]any) func(func(string) bool) {
return func(yield func(string) bool) {
for k := range m {
if !yield(k) {
return
}
}
}
}
func TestOverflowGoesToThePeerWhichSeesOneKeyAndTheRequestID(t *testing.T) {
west := newFedCluster(t, "west", "qwen-west")
east := newFedCluster(t, "east", "qwen-east")
west.federate(t, east, false)
if _, err := west.models.SetFallbacks(t.Context(), "qwen-west", []string{"east/qwen-east"}, "test"); err != nil {
t.Fatal(err)
}
west.ctl.Publish(t.Context())
// West has capacity: east is not asked.
if rec := west.ask(t, "qwen-west", nil); rec.Code != http.StatusOK || east.engine.calls.Load() != 0 {
t.Fatalf("%d; east called %d times", rec.Code, east.engine.calls.Load())
}
// West's replica is full: the overflow goes east.
west.engine.status.Store(http.StatusServiceUnavailable)
rec := west.ask(t, "qwen-west", map[string]string{"X-Request-Id": "overflow-1"})
if rec.Code != http.StatusOK {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
if got := rec.Header().Values("x-llmgw-served-model"); !slices.Equal(got, []string{"east/qwen-east"}) {
t.Errorf("served by %q", got)
}
// East's own answer headers do not reach the client beside west's.
if got := rec.Header().Values("X-Request-Id"); len(got) != 1 {
t.Errorf("the client was given request ids %q", got)
}
if east.engine.calls.Load() != 1 {
t.Errorf("east's engine was called %d times", east.engine.calls.Load())
}
// West records which peer served; east, the one key west uses there,
// the request's id, and no tags, which are off unless asked for.
here := west.usageOf(t, "overflow-1")
if here.ServedModel != "east/qwen-east" || here.Replica != "peer:east/qwen-east" {
t.Errorf("west's record: served %q by %q", here.ServedModel, here.Replica)
}
there := east.usageOf(t, "overflow-1")
if there.KeyID != "east-key-2" || there.Model != "qwen-east" {
t.Errorf("east's record: key %q, model %q", there.KeyID, there.Model)
}
if len(there.Tags) != 0 {
t.Errorf("east was told %v without send-tags", there.Tags)
}
// With tags on, east can say which cluster and project asked.
west.federate(t, east, true)
rec = west.ask(t, "qwen-west", map[string]string{"X-Request-Id": "overflow-2"})
if rec.Code != http.StatusOK {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
there = east.usageOf(t, "overflow-2")
if there.Tags["federated_from"] != "west" || there.Tags["project"] != west.project {
t.Errorf("east's record carries tags %v", there.Tags)
}
}
func TestTwoClustersFederatingEachOtherDoNotSendARequestRoundAgain(t *testing.T) {
west := newFedCluster(t, "west", "qwen-west")
east := newFedCluster(t, "east", "qwen-east")
west.federate(t, east, false)
east.federate(t, west, false)
// Neither offers the other's peer models back: no east/west/... names.
if _, ok := west.ctl.Catalog().Resolve("east/west/qwen-west"); ok {
t.Error("west took back its own model through east")
}
if _, ok := east.ctl.Catalog().Resolve("west/east/qwen-east"); ok {
t.Error("east took back its own model through west")
}
// Each falls back to the other, and both are full.
if _, err := west.models.SetFallbacks(t.Context(), "qwen-west", []string{"east/qwen-east"}, "test"); err != nil {
t.Fatal(err)
}
if _, err := east.models.SetFallbacks(t.Context(), "qwen-east", []string{"west/qwen-west"}, "test"); err != nil {
t.Fatal(err)
}
west.ctl.Publish(t.Context())
east.ctl.Publish(t.Context())
west.engine.status.Store(http.StatusServiceUnavailable)
east.engine.status.Store(http.StatusServiceUnavailable)
rec := west.ask(t, "qwen-west", nil)
if rec.Code == http.StatusOK {
t.Fatalf("served: %s", rec.Body.String())
}
// West's replica once, east's once; east did not send it back west.
if w, e := west.engine.calls.Load(), east.engine.calls.Load(); w != 1 || e != 1 {
t.Errorf("west's engine was called %d times, east's %d; each should be once", w, e)
}
// And a request that says it has passed through a cluster already is
// refused there, before anything else.
rec = west.ask(t, "qwen-west", map[string]string{hopHeader: east.id + ", " + west.id})
if rec.Code != http.StatusLoopDetected {
t.Errorf("a request that came back: %d %s", rec.Code, rec.Body.String())
}
}
func TestAProjectThatStaysInTheClusterIsNeverSentToAPeer(t *testing.T) {
west := newFedCluster(t, "west", "qwen-west")
east := newFedCluster(t, "east", "qwen-east")
west.federate(t, east, false)
if _, err := west.models.SetFallbacks(t.Context(), "qwen-west", []string{"east/qwen-east"}, "test"); err != nil {
t.Fatal(err)
}
west.ctl.Publish(t.Context())
west.engine.status.Store(http.StatusServiceUnavailable)
if err := west.rt.store.SetInCluster(t.Context(), west.project, true); err != nil {
t.Fatal(err)
}
if rec := west.ask(t, "east/qwen-east", nil); rec.Code != http.StatusForbidden {
t.Errorf("a peer's model, asked for by a project that stays in the cluster: %d %s", rec.Code, rec.Body.String())
}
if rec := west.ask(t, "qwen-west", nil); rec.Code == http.StatusOK {
t.Errorf("served by %q", rec.Header().Get("x-llmgw-served-model"))
}
if east.engine.calls.Load() != 0 {
t.Errorf("east was sent %d requests", east.engine.calls.Load())
}
}
func TestAPolicyThatKeepsNothingIsNeverSentToAPeer(t *testing.T) {
west := newFedCluster(t, "west", "qwen-west")
east := newFedCluster(t, "east", "qwen-east")
west.federate(t, east, false)
if _, err := west.models.SetFallbacks(t.Context(), "qwen-west", []string{"east/qwen-east"}, "test"); err != nil {
t.Fatal(err)
}
west.ctl.Publish(t.Context())
west.engine.status.Store(http.StatusServiceUnavailable)
g := &switchGuard{}
g.noStore.Store(true)
west.rt.SetGuard(g)
if rec := west.ask(t, "east/qwen-east", nil); rec.Code != http.StatusBadRequest {
t.Errorf("a peer's model under no_store: %d %s", rec.Code, rec.Body.String())
}
if rec := west.ask(t, "qwen-west", nil); rec.Code == http.StatusOK {
t.Errorf("served by %q", rec.Header().Get("x-llmgw-served-model"))
}
if east.engine.calls.Load() != 0 {
t.Errorf("east was sent %d requests", east.engine.calls.Load())
}
}
func TestAPeerThatIsDownIsTakenOutOfRoutingAndBackWhenItAnswers(t *testing.T) {
west := newFedCluster(t, "west", "qwen-west")
east := newFedCluster(t, "east", "qwen-east")
west.federate(t, east, false)
if len(west.rt.reg.Snapshots("east/qwen-east")) != 1 {
t.Fatal("east's model is not routed to while east is up")
}
// East has nothing of its own to serve with: its /ready says so.
east.rt.SetEndpoints(t.Context(), nil)
if st, _ := west.fed.Sync(t.Context(), "east"); st.Up || st.Error == "" {
t.Errorf("east with nothing to serve: %+v", st)
}
west.ctl.Publish(t.Context())
if n := len(west.rt.reg.Snapshots("east/qwen-east")); n != 0 {
t.Errorf("east is down and still routed to: %d endpoints", n)
}
if _, ok := west.ctl.Catalog().Resolve("east/qwen-east"); !ok {
t.Error("east's model left the catalog when east went down; down is not gone")
}
if rec := west.ask(t, "east/qwen-east", nil); rec.Code == http.StatusOK {
t.Error("served by a peer that is down")
}
// Back.
east.ctl.Publish(t.Context())
east.rt.SetEndpoints(t.Context(), []endpoint.Info{{ID: "e", URL: east.engine.srv.URL, Models: []string{"qwen-east"}}})
if st, _ := west.fed.Sync(t.Context(), "east"); !st.Up {
t.Errorf("east is back: %+v", st)
}
west.ctl.Publish(t.Context())
if rec := west.ask(t, "east/qwen-east", nil); rec.Code != http.StatusOK {
t.Errorf("east is back: %d %s", rec.Code, rec.Body.String())
}
// Gone altogether.
east.srv.Close()
if st, _ := west.fed.Sync(t.Context(), "east"); st.Up {
t.Errorf("east is gone: %+v", st)
}
west.ctl.Publish(t.Context())
if n := len(west.rt.reg.Snapshots("east/qwen-east")); n != 0 {
t.Errorf("east is gone and still routed to: %d endpoints", n)
}
}
func TestAPeerModelAFallbackNamesIsKeptWithdrawnAndThePeerCannotBeRemoved(t *testing.T) {
west := newFedCluster(t, "west", "qwen-west")
east := newFedCluster(t, "east", "qwen-east")
west.federate(t, east, false)
if _, err := west.models.SetFallbacks(t.Context(), "qwen-west", []string{"east/qwen-east"}, "test"); err != nil {
t.Fatal(err)
}
if err := west.fed.Remove(t.Context(), "east"); err == nil || !strings.Contains(err.Error(), "qwen-west") {
t.Errorf("removing a peer a fallback names: %v", err)
}
// East stops offering it: kept, withdrawn, and not routed to.
if err := east.models.Remove(t.Context(), "qwen-east"); err != nil {
t.Fatal(err)
}
// Its replica goes too; east still serves something, so it is up.
east.rt.SetEndpoints(t.Context(), []endpoint.Info{{ID: "other", URL: east.engine.srv.URL, Models: []string{"other"}}})
if _, err := west.fed.Sync(t.Context(), "east"); err != nil {
t.Fatal(err)
}
west.ctl.Publish(t.Context())
if _, ok := west.ctl.Catalog().Resolve("qwen-west"); !ok {
t.Fatal("the model that falls back to east's lost its place in the catalog")
}
st, _ := west.fed.Status("east")
if !slices.ContainsFunc(st.Models, func(m federation.ModelStatus) bool { return m.Upstream == "qwen-east" && m.Withdrawn }) {
t.Errorf("east's models: %+v", st.Models)
}
if n := len(west.rt.reg.Snapshots("east/qwen-east")); n != 0 {
t.Errorf("a withdrawn model is routed to: %d endpoints", n)
}
// Once nothing names it, it goes, and so may the peer.
if _, err := west.models.SetFallbacks(t.Context(), "qwen-west", nil, "test"); err != nil {
t.Fatal(err)
}
if _, err := west.fed.Sync(t.Context(), "east"); err != nil {
t.Fatal(err)
}
if _, ok := west.ctl.Catalog().Resolve("east/qwen-east"); ok {
t.Error("a withdrawn model nothing names is still in the catalog")
}
if err := west.fed.Remove(t.Context(), "east"); err != nil {
t.Fatal(err)
}
if len(west.fed.List()) != 0 {
t.Error("east is still a peer")
}
}
func TestPeersAndTheirModelsSurviveARestart(t *testing.T) {
west := newFedCluster(t, "west", "qwen-west")
east := newFedCluster(t, "east", "qwen-east")
west.federate(t, east, false)
if _, err := west.models.SetFallbacks(t.Context(), "qwen-west", []string{"east/qwen-east"}, "test"); err != nil {
t.Fatal(err)
}
// A new control plane against the same store, before east is asked
// anything: the model falling back to east's is whole.
log := slog.New(slog.DiscardHandler)
empty, _ := catalog.New(nil)
ctl := control.New(control.DefaultConfig(), catalog.NewLive(empty), fedNoDispatch{}, withLocal{rt: west.rt}, log)
models := modelsvc.New(nil, west.rt.store, ctl, log)
fed := federation.New(federation.Config{Self: west.id}, west.rt.store, models, ctl.Catalog, log)
if err := fed.Load(t.Context()); err != nil {
t.Fatal(err)
}
models.SetFederated(fed.Models)
built, err := models.Load(t.Context())
if err != nil {
t.Fatal(err)
}
m, ok := built.Resolve("qwen-west")
if !ok || !slices.Equal(m.Fallbacks, []string{"east/qwen-east"}) {
t.Errorf("after a restart: %+v", m)
}
if _, ok := built.Resolve("east/qwen-east"); !ok {
t.Error("east's model is not in the catalog after a restart")
}
// Down until checked.
if _, _, ok := fed.PeerRoute("east", "qwen-east"); ok {
t.Error("a peer not yet checked is routed to")
}
}
func TestAClusterCannotBeItsOwnPeer(t *testing.T) {
west := newFedCluster(t, "west", "qwen-west")
keyFile := filepath.Join(t.TempDir(), "self.key")
if err := os.WriteFile(keyFile, []byte(west.issueKey(t)), 0o600); err != nil {
t.Fatal(err)
}
_, err := west.fed.Add(t.Context(), federation.AddRequest{Name: "self", URL: west.srv.URL, KeyFile: keyFile}, "test")
if err == nil || !strings.Contains(err.Error(), "this cluster") {
t.Errorf("adding itself: %v", err)
}
}
func TestAPeerIsCheckedBeforeItIsKept(t *testing.T) {
west := newFedCluster(t, "west", "qwen-west")
east := newFedCluster(t, "east", "qwen-east")
keyFile := filepath.Join(t.TempDir(), "wrong.key")
if err := os.WriteFile(keyFile, []byte(store.KeyPrefix+strings.Repeat("z", 43)), 0o600); err != nil {
t.Fatal(err)
}
for name, req := range map[string]federation.AddRequest{
"a key the peer refuses": {Name: "east", URL: east.srv.URL, KeyFile: keyFile},
"nothing at the address": {Name: "east", URL: "http://127.0.0.1:1", KeyFile: keyFile},
"a name with a slash": {Name: "ea/st", URL: east.srv.URL, KeyFile: keyFile},
"no key file": {Name: "east", URL: east.srv.URL},
} {
if _, err := west.fed.Add(t.Context(), req, "test"); err == nil {
t.Errorf("%s: accepted", name)
}
}
if len(west.fed.List()) != 0 {
t.Errorf("kept: %+v", west.fed.List())
}
}
// The federation's checks run beside requests and catalog rebuilds.
func TestFederationChecksAreSafeBesideTraffic(t *testing.T) {
west := newFedCluster(t, "west", "qwen-west")
east := newFedCluster(t, "east", "qwen-east")
west.federate(t, east, false)
var wg sync.WaitGroup
for range 4 {
wg.Go(func() {
for range 5 {
west.fed.SyncAll(t.Context())
west.ctl.Publish(t.Context())
}
})
wg.Go(func() {
for range 5 {
west.ask(t, "east/qwen-east", nil)
_ = west.fed.List()
}
})
}
wg.Wait()
}
+20 -3
View File
@@ -139,8 +139,13 @@ func (r *Router) Handler() http.Handler {
// Open, like /health: a load balancer does not have a key, and this leaks
// one bit — whether anything is servable — which the next request would
// reveal anyway.
//
// Only this cluster's own endpoints count, not a federated peer's: a
// peer reads this to decide whether to send here, and two clusters
// federating each other would otherwise each be ready because the other
// is, with nothing of their own to serve with.
mux.HandleFunc("GET /ready", func(w http.ResponseWriter, _ *http.Request) {
if len(r.reg.IDs()) == 0 {
if r.localEndpoints() == 0 {
http.Error(w, "no endpoints are available to serve", http.StatusServiceUnavailable)
return
}
@@ -295,7 +300,7 @@ func (r *Router) Handler() http.Handler {
}
if r.auth == nil {
return mux
return r.refuseLoops(mux)
}
exempt := []string{"/health", "/ready", "/leader", "/", "/about", "/_static/", "/claim/",
"/signup", "/login", "/login/", "/logout", "/profile", "/profile/", "/scim/",
@@ -309,7 +314,7 @@ func (r *Router) Handler() http.Handler {
// Wrapping the whole mux rather than checking inside each handler: adding a
// route must not be able to add an unauthenticated one by omission. Health
// stays open because a load balancer has no key.
return auth.Middleware(r.auth, r.log, exempt...)(mux)
return r.refuseLoops(auth.Middleware(r.auth, r.log, exempt...)(mux))
}
// parsedRequest is what the router needs from a request. Everything else in the
@@ -1014,6 +1019,15 @@ func (r *Router) handleModels(w http.ResponseWriter, req *http.Request) {
entry := map[string]any{
"id": n, "object": "model", "created": time.Now().Unix(), "owned_by": owner,
}
// Not OpenAI's either: what a federated peer reading this list needs
// to leave out — a second name for a model already listed, and a
// model this cluster has from a peer of its own.
if m := r.modelMap()[n]; m.Name != "" && m.Name != n {
entry["alias_of"] = m.Name
}
if m := r.modelMap()[n]; m.IsPeer() {
entry["peer"] = m.External.Peer
}
// Only when known. A context_length of 0 would read as "no context",
// which is a worse answer than saying nothing.
m := r.modelMap()[n]
@@ -1035,6 +1049,9 @@ func (r *Router) handleModels(w http.ResponseWriter, req *http.Request) {
}
data = append(data, entry)
}
if id := r.clusterID().id; id != "" {
w.Header().Set(clusterHeader, id)
}
writeJSON(w, http.StatusOK, map[string]any{"object": "list", "data": data})
}
+14 -1
View File
@@ -149,6 +149,10 @@ type Request struct {
Body []byte
// Header carries headers to forward (authorization is deliberately not).
Header http.Header
// Extra are headers set on the upstream request beyond those forwarded
// from the client: what the router itself says to this upstream, such
// as the hop header a federated cluster is sent.
Extra http.Header
// APIKey is the engine's own key, sent in place of the client's.
APIKey string
// Stream says whether an SSE response is expected.
@@ -250,6 +254,9 @@ func (p *Proxy) Do(ctx context.Context, w http.ResponseWriter, r Request) (Outco
return Outcome{Retryable: false}, err
}
req.Header = cloneForUpstream(r.Header)
for k, vs := range r.Extra {
req.Header[http.CanonicalHeaderKey(k)] = vs
}
ct := r.ContentType
if ct == "" {
ct = "application/json"
@@ -620,9 +627,15 @@ var hopByHop = map[string]bool{
"Upgrade": true,
}
// copyHeaders gives the client the upstream's response headers.
//
// A header the gateway has already set is its own answer and is kept: an
// upstream that is itself an llmgw — a federated peer — sends X-Request-Id
// and x-llmgw-served-model of its own, and the client would otherwise be
// given both.
func copyHeaders(dst, src http.Header) {
for k, vs := range src {
if hopByHop[http.CanonicalHeaderKey(k)] {
if hopByHop[http.CanonicalHeaderKey(k)] || len(dst.Values(k)) > 0 {
continue
}
for _, v := range vs {
+13 -1
View File
@@ -100,7 +100,9 @@ type Router struct {
usageRetention time.Duration
// isLeader says whether this process leads; nil is always.
isLeader atomic.Pointer[func() bool]
acct *accounting.Accountant
// federation is which cluster this is, to its peers; nil until said.
federation atomic.Pointer[federationID]
acct *accounting.Accountant
// guard judges requests and answers; nil judges nothing. guardLibrary
// is the policies it was built from, for the pages that show them.
guard guard.Guard
@@ -882,6 +884,13 @@ var errLoading = errors.New("model loading")
func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *parsedRequest, model string, entry catalog.Model) dispatchResult {
candidates := r.reg.Snapshots(model)
if visited := withoutVisited(candidates, req.Header); len(visited) < len(candidates) {
if len(visited) == 0 {
return dispatchResult{err: fmt.Errorf("%w: %s is served by a federated cluster this request has already passed through",
errNoCapacity, model)}
}
candidates = visited
}
if len(candidates) == 0 {
// Nothing is serving it. That may be because nothing ever will, or
// because it loads when asked and nobody has asked yet — and the second
@@ -978,6 +987,9 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
CaptureReply: req.captureBytes(),
ContentType: req.ContentType,
}
if dec.Endpoint.Peer != "" {
preq.Extra = r.peerHeaders(ctx, dec.Endpoint, req.Header)
}
if req.Stream {
gaps := r.m.token.gap.WithLabelValues(r.metricModel(model))
preq.OnEvent = func(d time.Duration) { gaps.Observe(d.Seconds()) }