Federation: another llmgw cluster as a provider, its models as PEER/MODEL

ctl federation add/list/remove/sync keeps peers in the store (migration
0029); their /v1/models are offered here as external models named
PEER/MODEL and kept in step, a model still named by a fallback kept
withdrawn. Peers are checked at /ready, timed, and taken out of routing
while down. Requests to a peer carry x-llmgw-hop and are refused (508) when
they come back to a cluster they passed through; residency and no_store
keep requests off peers; X-Request-Id and, with --send-tags, attribution
tags go along. ctl model fallbacks sets overflow to a peer. Console view,
metrics, admin API routes and docs.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:21:30 +02:00
1 parent d41104a452
commit 40e9fd2ab1
33 files changed
+2944 -23

No files matched your search

+1
View File
@@ -65,6 +65,7 @@ var copyOrder = []string{
// Runs before their cases, which point at them.
"eval_runs",
"eval_cases",
"federation_peers",
}
// notCopied are tables deliberately left behind.
+131
View File
@@ -0,0 +1,131 @@
package store
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"time"
)
// FederationPeer is another llmgw cluster whose models this one offers.
type FederationPeer struct {
Name string
URL string
// KeyFile holds the key this cluster uses at the peer, on the control
// plane: a path, never the key, for the reason an external model's is.
KeyFile string
// SendTags makes requests sent to the peer carry x-llmgw-tags naming
// this cluster and the sending project, so the peer can attribute them.
SendTags bool
// Cluster is the peer's own cluster id, as it last said it: what a
// request that has passed through the peer carries in its hop header.
Cluster string
// Models is what the peer last offered, as JSON.
Models json.RawMessage
SyncedAt time.Time
CreatedAt time.Time
CreatedBy string
}
// PutFederationPeer adds a peer, or replaces one whole: its address, key
// file, tags setting and what it offers. Who added it, and when, stay as
// they were first written.
func (s *DB) PutFederationPeer(ctx context.Context, p FederationPeer) error {
if p.Name == "" || p.URL == "" {
return fmt.Errorf("a federation peer needs a name and an address")
}
if p.CreatedAt.IsZero() {
p.CreatedAt = time.Now()
}
models := string(p.Models)
if models == "" {
models = "[]"
}
if !json.Valid([]byte(models)) {
return fmt.Errorf("peer %q: its models are not valid JSON", p.Name)
}
_, err := s.db.ExecContext(ctx,
`INSERT INTO federation_peers (name, url, key_file, send_tags, cluster, models, synced_at, created_at, created_by)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (name) DO UPDATE SET url = excluded.url, key_file = excluded.key_file,
send_tags = excluded.send_tags, cluster = excluded.cluster, models = excluded.models,
synced_at = excluded.synced_at`,
p.Name, p.URL, p.KeyFile, boolInt(p.SendTags), p.Cluster, models, syncedMillis(p.SyncedAt),
millis(p.CreatedAt), p.CreatedBy)
return err
}
// SetFederationPeerModels records what a peer offers now, and its cluster
// id, without touching how it is reached. ErrNotFound when the peer was
// removed meanwhile: a sync that finishes after a removal must not bring
// the peer back.
func (s *DB) SetFederationPeerModels(ctx context.Context, name, cluster string, models json.RawMessage, at time.Time) error {
if !json.Valid(models) {
return fmt.Errorf("peer %q: its models are not valid JSON", name)
}
res, err := s.db.ExecContext(ctx,
`UPDATE federation_peers SET cluster = ?, models = ?, synced_at = ? WHERE name = ?`,
cluster, string(models), syncedMillis(at), name)
if err != nil {
return err
}
if n, _ := res.RowsAffected(); n == 0 {
return ErrNotFound
}
return nil
}
// DeleteFederationPeer removes a peer; removing one that is not there is
// ErrNotFound, so a mistyped name is said rather than silently accepted.
func (s *DB) DeleteFederationPeer(ctx context.Context, name string) error {
res, err := s.db.ExecContext(ctx, `DELETE FROM federation_peers WHERE name = ?`, name)
if err != nil {
return err
}
if n, _ := res.RowsAffected(); n == 0 {
return ErrNotFound
}
return nil
}
// FederationPeers lists the peers by name.
func (s *DB) FederationPeers(ctx context.Context) ([]FederationPeer, error) {
return queryAll(ctx, s.db, `SELECT name, url, key_file, send_tags, cluster, models, synced_at, created_at, created_by
FROM federation_peers ORDER BY name`, nil, func(rows *sql.Rows) (FederationPeer, error) {
var p FederationPeer
var tags int
var models string
var synced, created int64
err := rows.Scan(&p.Name, &p.URL, &p.KeyFile, &tags, &p.Cluster, &models, &synced, &created, &p.CreatedBy)
p.SendTags, p.Models = tags == 1, json.RawMessage(models)
p.CreatedAt = fromMillis(created)
if synced > 0 {
p.SyncedAt = fromMillis(synced)
}
return p, err
})
}
// FederationPeer is one peer by name; ErrNotFound when there is none.
func (s *DB) FederationPeer(ctx context.Context, name string) (FederationPeer, error) {
peers, err := s.FederationPeers(ctx)
if err != nil {
return FederationPeer{}, err
}
for _, p := range peers {
if p.Name == name {
return p, nil
}
}
return FederationPeer{}, ErrNotFound
}
// syncedMillis is 0 for a peer never synced, which reads back as no time
// rather than as a date in 1754.
func syncedMillis(t time.Time) int64 {
if t.IsZero() {
return 0
}
return t.UnixMilli()
}
+50
View File
@@ -0,0 +1,50 @@
package store
import (
"encoding/json"
"errors"
"path/filepath"
"testing"
"time"
)
func TestAFederationPeerIsKeptWithWhatItLastOffered(t *testing.T) {
s, err := OpenSQLite(t.Context(), filepath.Join(t.TempDir(), "s.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = s.Close() })
ctx := t.Context()
if err := s.PutFederationPeer(ctx, FederationPeer{Name: "east", URL: "https://east", KeyFile: "/k",
SendTags: true, CreatedBy: "ops"}); err != nil {
t.Fatal(err)
}
peers, err := s.FederationPeers(ctx)
if err != nil || len(peers) != 1 {
t.Fatalf("%v %+v", err, peers)
}
if p := peers[0]; !p.SendTags || string(p.Models) != "[]" || !p.SyncedAt.IsZero() || p.CreatedBy != "ops" {
t.Errorf("as kept: %+v", p)
}
at := time.Now()
if err := s.SetFederationPeerModels(ctx, "east", "c1", json.RawMessage(`[{"id":"m"}]`), at); err != nil {
t.Fatal(err)
}
p, err := s.FederationPeer(ctx, "east")
if err != nil || p.Cluster != "c1" || string(p.Models) != `[{"id":"m"}]` || p.SyncedAt.UnixMilli() != at.UnixMilli() {
t.Errorf("after a sync: %v %+v", err, p)
}
// A sync finishing after the peer was removed does not bring it back.
if err := s.DeleteFederationPeer(ctx, "east"); err != nil {
t.Fatal(err)
}
if err := s.SetFederationPeerModels(ctx, "east", "c1", json.RawMessage(`[]`), at); !errors.Is(err, ErrNotFound) {
t.Errorf("syncing a removed peer: %v", err)
}
if err := s.DeleteFederationPeer(ctx, "east"); !errors.Is(err, ErrNotFound) {
t.Errorf("removing it twice: %v", err)
}
}
@@ -0,0 +1,23 @@
-- Ported from the SQLite migration of the same number.
-- Other llmgw clusters this one federates with (`llmgw ctl federation add`):
-- each peer's address, the file on the control plane holding the key this
-- cluster uses there, whether requests sent to it carry tags naming who sent
-- them, the peer's own cluster id, and the models it last offered.
--
-- The models are kept, not only fetched, so a catalog that names one of them
-- as a fallback is whole at startup, before the peer has been asked again:
-- a local model falling back to a peer's would otherwise be skipped as
-- naming a model that is not there. Stored as JSON for the same reason a
-- runtime model's definition is: a list of what the peer says about each,
-- which grows.
CREATE TABLE federation_peers (
name TEXT PRIMARY KEY,
url TEXT NOT NULL,
key_file TEXT NOT NULL DEFAULT '',
send_tags BIGINT NOT NULL DEFAULT 0,
cluster TEXT NOT NULL DEFAULT '',
models TEXT NOT NULL DEFAULT '[]',
synced_at BIGINT NOT NULL DEFAULT 0,
created_at BIGINT NOT NULL,
created_by TEXT NOT NULL DEFAULT ''
);
@@ -0,0 +1,22 @@
-- Other llmgw clusters this one federates with (`llmgw ctl federation add`):
-- each peer's address, the file on the control plane holding the key this
-- cluster uses there, whether requests sent to it carry tags naming who sent
-- them, the peer's own cluster id, and the models it last offered.
--
-- The models are kept, not only fetched, so a catalog that names one of them
-- as a fallback is whole at startup, before the peer has been asked again:
-- a local model falling back to a peer's would otherwise be skipped as
-- naming a model that is not there. Stored as JSON for the same reason a
-- runtime model's definition is: a list of what the peer says about each,
-- which grows.
CREATE TABLE federation_peers (
name TEXT PRIMARY KEY,
url TEXT NOT NULL,
key_file TEXT NOT NULL DEFAULT '',
send_tags INTEGER NOT NULL DEFAULT 0,
cluster TEXT NOT NULL DEFAULT '',
models TEXT NOT NULL DEFAULT '[]',
synced_at INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
created_by TEXT NOT NULL DEFAULT ''
);
+1 -1
View File
@@ -44,7 +44,7 @@ func openPG(t *testing.T) *DB {
`TRUNCATE thread_messages, threads, memory_notes, usage, usage_5m, usage_1h, usage_1d, api_keys, memberships,
key_claims, limits, models, sessions, users, projects, teams,
organizations, leadership, control_planes, files, batches, batch_results, request_bodies,
eval_runs, eval_cases RESTART IDENTITY CASCADE`)
eval_runs, eval_cases, federation_peers RESTART IDENTITY CASCADE`)
if err != nil {
t.Fatalf("clearing the test database: %v", err)
}