Federation: another llmgw cluster as a provider, its models as PEER/MODEL
ctl federation add/list/remove/sync keeps peers in the store (migration 0029); their /v1/models are offered here as external models named PEER/MODEL and kept in step, a model still named by a fallback kept withdrawn. Peers are checked at /ready, timed, and taken out of routing while down. Requests to a peer carry x-llmgw-hop and are refused (508) when they come back to a cluster they passed through; residency and no_store keep requests off peers; X-Request-Id and, with --send-tags, attribution tags go along. ctl model fallbacks sets overflow to a peer. Console view, metrics, admin API routes and docs. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
40e9fd2ab1
33 files changed
+2944
-23
No files matched your search
@@ -65,6 +65,7 @@ var copyOrder = []string{
|
||||
// Runs before their cases, which point at them.
|
||||
"eval_runs",
|
||||
"eval_cases",
|
||||
"federation_peers",
|
||||
}
|
||||
|
||||
// notCopied are tables deliberately left behind.
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// FederationPeer is another llmgw cluster whose models this one offers.
|
||||
type FederationPeer struct {
|
||||
Name string
|
||||
URL string
|
||||
// KeyFile holds the key this cluster uses at the peer, on the control
|
||||
// plane: a path, never the key, for the reason an external model's is.
|
||||
KeyFile string
|
||||
// SendTags makes requests sent to the peer carry x-llmgw-tags naming
|
||||
// this cluster and the sending project, so the peer can attribute them.
|
||||
SendTags bool
|
||||
// Cluster is the peer's own cluster id, as it last said it: what a
|
||||
// request that has passed through the peer carries in its hop header.
|
||||
Cluster string
|
||||
// Models is what the peer last offered, as JSON.
|
||||
Models json.RawMessage
|
||||
SyncedAt time.Time
|
||||
CreatedAt time.Time
|
||||
CreatedBy string
|
||||
}
|
||||
|
||||
// PutFederationPeer adds a peer, or replaces one whole: its address, key
|
||||
// file, tags setting and what it offers. Who added it, and when, stay as
|
||||
// they were first written.
|
||||
func (s *DB) PutFederationPeer(ctx context.Context, p FederationPeer) error {
|
||||
if p.Name == "" || p.URL == "" {
|
||||
return fmt.Errorf("a federation peer needs a name and an address")
|
||||
}
|
||||
if p.CreatedAt.IsZero() {
|
||||
p.CreatedAt = time.Now()
|
||||
}
|
||||
models := string(p.Models)
|
||||
if models == "" {
|
||||
models = "[]"
|
||||
}
|
||||
if !json.Valid([]byte(models)) {
|
||||
return fmt.Errorf("peer %q: its models are not valid JSON", p.Name)
|
||||
}
|
||||
_, err := s.db.ExecContext(ctx,
|
||||
`INSERT INTO federation_peers (name, url, key_file, send_tags, cluster, models, synced_at, created_at, created_by)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (name) DO UPDATE SET url = excluded.url, key_file = excluded.key_file,
|
||||
send_tags = excluded.send_tags, cluster = excluded.cluster, models = excluded.models,
|
||||
synced_at = excluded.synced_at`,
|
||||
p.Name, p.URL, p.KeyFile, boolInt(p.SendTags), p.Cluster, models, syncedMillis(p.SyncedAt),
|
||||
millis(p.CreatedAt), p.CreatedBy)
|
||||
return err
|
||||
}
|
||||
|
||||
// SetFederationPeerModels records what a peer offers now, and its cluster
|
||||
// id, without touching how it is reached. ErrNotFound when the peer was
|
||||
// removed meanwhile: a sync that finishes after a removal must not bring
|
||||
// the peer back.
|
||||
func (s *DB) SetFederationPeerModels(ctx context.Context, name, cluster string, models json.RawMessage, at time.Time) error {
|
||||
if !json.Valid(models) {
|
||||
return fmt.Errorf("peer %q: its models are not valid JSON", name)
|
||||
}
|
||||
res, err := s.db.ExecContext(ctx,
|
||||
`UPDATE federation_peers SET cluster = ?, models = ?, synced_at = ? WHERE name = ?`,
|
||||
cluster, string(models), syncedMillis(at), name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n, _ := res.RowsAffected(); n == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteFederationPeer removes a peer; removing one that is not there is
|
||||
// ErrNotFound, so a mistyped name is said rather than silently accepted.
|
||||
func (s *DB) DeleteFederationPeer(ctx context.Context, name string) error {
|
||||
res, err := s.db.ExecContext(ctx, `DELETE FROM federation_peers WHERE name = ?`, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n, _ := res.RowsAffected(); n == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FederationPeers lists the peers by name.
|
||||
func (s *DB) FederationPeers(ctx context.Context) ([]FederationPeer, error) {
|
||||
return queryAll(ctx, s.db, `SELECT name, url, key_file, send_tags, cluster, models, synced_at, created_at, created_by
|
||||
FROM federation_peers ORDER BY name`, nil, func(rows *sql.Rows) (FederationPeer, error) {
|
||||
var p FederationPeer
|
||||
var tags int
|
||||
var models string
|
||||
var synced, created int64
|
||||
err := rows.Scan(&p.Name, &p.URL, &p.KeyFile, &tags, &p.Cluster, &models, &synced, &created, &p.CreatedBy)
|
||||
p.SendTags, p.Models = tags == 1, json.RawMessage(models)
|
||||
p.CreatedAt = fromMillis(created)
|
||||
if synced > 0 {
|
||||
p.SyncedAt = fromMillis(synced)
|
||||
}
|
||||
return p, err
|
||||
})
|
||||
}
|
||||
|
||||
// FederationPeer is one peer by name; ErrNotFound when there is none.
|
||||
func (s *DB) FederationPeer(ctx context.Context, name string) (FederationPeer, error) {
|
||||
peers, err := s.FederationPeers(ctx)
|
||||
if err != nil {
|
||||
return FederationPeer{}, err
|
||||
}
|
||||
for _, p := range peers {
|
||||
if p.Name == name {
|
||||
return p, nil
|
||||
}
|
||||
}
|
||||
return FederationPeer{}, ErrNotFound
|
||||
}
|
||||
|
||||
// syncedMillis is 0 for a peer never synced, which reads back as no time
|
||||
// rather than as a date in 1754.
|
||||
func syncedMillis(t time.Time) int64 {
|
||||
if t.IsZero() {
|
||||
return 0
|
||||
}
|
||||
return t.UnixMilli()
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAFederationPeerIsKeptWithWhatItLastOffered(t *testing.T) {
|
||||
s, err := OpenSQLite(t.Context(), filepath.Join(t.TempDir(), "s.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
ctx := t.Context()
|
||||
|
||||
if err := s.PutFederationPeer(ctx, FederationPeer{Name: "east", URL: "https://east", KeyFile: "/k",
|
||||
SendTags: true, CreatedBy: "ops"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
peers, err := s.FederationPeers(ctx)
|
||||
if err != nil || len(peers) != 1 {
|
||||
t.Fatalf("%v %+v", err, peers)
|
||||
}
|
||||
if p := peers[0]; !p.SendTags || string(p.Models) != "[]" || !p.SyncedAt.IsZero() || p.CreatedBy != "ops" {
|
||||
t.Errorf("as kept: %+v", p)
|
||||
}
|
||||
|
||||
at := time.Now()
|
||||
if err := s.SetFederationPeerModels(ctx, "east", "c1", json.RawMessage(`[{"id":"m"}]`), at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, err := s.FederationPeer(ctx, "east")
|
||||
if err != nil || p.Cluster != "c1" || string(p.Models) != `[{"id":"m"}]` || p.SyncedAt.UnixMilli() != at.UnixMilli() {
|
||||
t.Errorf("after a sync: %v %+v", err, p)
|
||||
}
|
||||
|
||||
// A sync finishing after the peer was removed does not bring it back.
|
||||
if err := s.DeleteFederationPeer(ctx, "east"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.SetFederationPeerModels(ctx, "east", "c1", json.RawMessage(`[]`), at); !errors.Is(err, ErrNotFound) {
|
||||
t.Errorf("syncing a removed peer: %v", err)
|
||||
}
|
||||
if err := s.DeleteFederationPeer(ctx, "east"); !errors.Is(err, ErrNotFound) {
|
||||
t.Errorf("removing it twice: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
-- Ported from the SQLite migration of the same number.
|
||||
-- Other llmgw clusters this one federates with (`llmgw ctl federation add`):
|
||||
-- each peer's address, the file on the control plane holding the key this
|
||||
-- cluster uses there, whether requests sent to it carry tags naming who sent
|
||||
-- them, the peer's own cluster id, and the models it last offered.
|
||||
--
|
||||
-- The models are kept, not only fetched, so a catalog that names one of them
|
||||
-- as a fallback is whole at startup, before the peer has been asked again:
|
||||
-- a local model falling back to a peer's would otherwise be skipped as
|
||||
-- naming a model that is not there. Stored as JSON for the same reason a
|
||||
-- runtime model's definition is: a list of what the peer says about each,
|
||||
-- which grows.
|
||||
CREATE TABLE federation_peers (
|
||||
name TEXT PRIMARY KEY,
|
||||
url TEXT NOT NULL,
|
||||
key_file TEXT NOT NULL DEFAULT '',
|
||||
send_tags BIGINT NOT NULL DEFAULT 0,
|
||||
cluster TEXT NOT NULL DEFAULT '',
|
||||
models TEXT NOT NULL DEFAULT '[]',
|
||||
synced_at BIGINT NOT NULL DEFAULT 0,
|
||||
created_at BIGINT NOT NULL,
|
||||
created_by TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
@@ -0,0 +1,22 @@
|
||||
-- Other llmgw clusters this one federates with (`llmgw ctl federation add`):
|
||||
-- each peer's address, the file on the control plane holding the key this
|
||||
-- cluster uses there, whether requests sent to it carry tags naming who sent
|
||||
-- them, the peer's own cluster id, and the models it last offered.
|
||||
--
|
||||
-- The models are kept, not only fetched, so a catalog that names one of them
|
||||
-- as a fallback is whole at startup, before the peer has been asked again:
|
||||
-- a local model falling back to a peer's would otherwise be skipped as
|
||||
-- naming a model that is not there. Stored as JSON for the same reason a
|
||||
-- runtime model's definition is: a list of what the peer says about each,
|
||||
-- which grows.
|
||||
CREATE TABLE federation_peers (
|
||||
name TEXT PRIMARY KEY,
|
||||
url TEXT NOT NULL,
|
||||
key_file TEXT NOT NULL DEFAULT '',
|
||||
send_tags INTEGER NOT NULL DEFAULT 0,
|
||||
cluster TEXT NOT NULL DEFAULT '',
|
||||
models TEXT NOT NULL DEFAULT '[]',
|
||||
synced_at INTEGER NOT NULL DEFAULT 0,
|
||||
created_at INTEGER NOT NULL,
|
||||
created_by TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
@@ -44,7 +44,7 @@ func openPG(t *testing.T) *DB {
|
||||
`TRUNCATE thread_messages, threads, memory_notes, usage, usage_5m, usage_1h, usage_1d, api_keys, memberships,
|
||||
key_claims, limits, models, sessions, users, projects, teams,
|
||||
organizations, leadership, control_planes, files, batches, batch_results, request_bodies,
|
||||
eval_runs, eval_cases RESTART IDENTITY CASCADE`)
|
||||
eval_runs, eval_cases, federation_peers RESTART IDENTITY CASCADE`)
|
||||
if err != nil {
|
||||
t.Fatalf("clearing the test database: %v", err)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user