Merge C24 federation: another llmgw cluster as a provider

Conflicts with facility-aware placement (config, control, admin API routes,
store test tables) all keep both; the generated reference and OpenAPI
document regenerated.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:29:35 +02:00
commit 33334c8f33
33 files changed
+2945 -24

No files matched your search

+1
View File
@@ -168,6 +168,7 @@ func newCtlCmd() *cobra.Command {
newAuditCmd(client, &asJSON),
newNotifyCmd(client, &asJSON),
newUsageCmd(client, &asJSON),
newFederationCmd(client, &asJSON),
newCertCmd(),
)
return cmd
+211
View File
@@ -0,0 +1,211 @@
package cli
import (
"encoding/json"
"fmt"
"net/url"
"strings"
"text/tabwriter"
"time"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
)
func newFederationCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
c := &cobra.Command{
Use: "federation",
Short: "Offer another llmgw cluster's models here, as PEER/MODEL",
Long: "A peer is another llmgw gateway, reached with a key it issued to this\n" +
"cluster. Its models join this catalog as PEER/MODEL and follow its own\n" +
"list; each is sent requests only while the peer's /ready answers, which\n" +
"is checked every federation.interval (10s) and timed. Name one as a\n" +
"local model's fallback to send overflow to the other site:\n" +
" llmgw ctl model fallbacks qwen-7b east/qwen-7b\n\n" +
"A project whose requests stay in the cluster (keys residency\n" +
"--in-cluster), or a policy that keeps nothing, is never sent to a peer.\n" +
"Requests carry x-llmgw-hop, so two clusters federating each other do\n" +
"not send one back and forth.",
}
c.AddCommand(newFederationAddCmd(client), newFederationListCmd(client, asJSON),
newFederationRemoveCmd(client), newFederationSyncCmd(client, asJSON))
return c
}
func newFederationAddCmd(client func() *ctlClient) *cobra.Command {
var req federation.AddRequest
c := &cobra.Command{
Use: "add NAME --url URL --key-file FILE",
Short: "Add another llmgw cluster as a peer, or change one",
Long: "The peer is checked first — its /ready, and its /v1/models with the\n" +
"key — and refused if either fails. The key file is read on the control\n" +
"plane (or env:NAME, vault:PATH#FIELD), at every check, so a rotated\n" +
"key is used without a change here; every control plane needs it.\n\n" +
"--send-tags makes each request sent to the peer carry x-llmgw-tags\n" +
"federated_from=<this cluster>,org=,team=,project=<ids>, so the peer's\n" +
"usage can say who here asked. Off unless given: the peer otherwise sees\n" +
"one key, this cluster's, and nothing about who is behind it.",
Example: " llmgw ctl federation add east --url https://llm.east.example --key-file /etc/llmgw/east.key\n" +
" llmgw ctl federation add east --url https://llm.east.example --key-file /etc/llmgw/east.key --send-tags",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
req.Name = args[0]
body, _ := json.Marshal(req)
var st federation.Status
if err := client().post(cmd.Context(), "/api/v1/federation", body, &st); err != nil {
return err
}
fmt.Fprintf(cmd.OutOrStdout(), "%s added: %d models, %s\n", st.Name, len(st.Models), healthText(st))
for _, m := range st.Models {
fmt.Fprintln(cmd.OutOrStdout(), " "+m.Name)
}
return nil
},
}
f := c.Flags()
f.StringVar(&req.URL, "url", "", "the peer gateway's client address, without /v1")
f.StringVar(&req.KeyFile, "key-file", "", "file on the control plane holding the key the peer issued to this cluster")
f.BoolVar(&req.SendTags, "send-tags", false, "tell the peer which cluster and project each request is from")
_ = c.MarkFlagRequired("url")
_ = c.MarkFlagRequired("key-file")
return c
}
func newFederationListCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
return &cobra.Command{
Use: "list",
Short: "Peers, whether each is up, how far away, and the models it offers",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
var v adminapi.FederationView
if err := client().get(cmd.Context(), "/api/v1/federation", &v); err != nil {
return err
}
if *asJSON {
return emitJSON(v)
}
out := cmd.OutOrStdout()
fmt.Fprintf(out, "this cluster: %s\n", v.Cluster)
if len(v.Peers) == 0 {
fmt.Fprintln(out, "no peers: llmgw ctl federation add NAME --url URL --key-file FILE")
return nil
}
w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
fmt.Fprintln(w, "PEER\tURL\tHEALTH\tLATENCY\tTAGS\tMODELS")
for _, p := range v.Peers {
tags := "off"
if p.SendTags {
tags = "on"
}
lat := "-"
if p.Latency > 0 {
lat = fmt.Sprintf("%.1fms", p.LatencyMs)
}
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", p.Name, p.URL, healthText(p), lat, tags, modelList(p))
}
return w.Flush()
},
}
}
func newFederationRemoveCmd(client func() *ctlClient) *cobra.Command {
return &cobra.Command{
Use: "remove NAME",
Short: "Remove a peer and its models",
Long: "Refused while a model here falls back to one of the peer's: change its\n" +
"fallbacks first (llmgw ctl model fallbacks).",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
var out adminapi.Removed
if err := client().delete(cmd.Context(), "/api/v1/federation/"+url.PathEscape(args[0]), &out); err != nil {
return err
}
fmt.Fprintf(cmd.OutOrStdout(), "%s removed\n", out.Removed)
return nil
},
}
}
func newFederationSyncCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
return &cobra.Command{
Use: "sync NAME",
Short: "Check a peer now and take its current models, rather than at the next interval",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
var st federation.Status
if err := client().post(cmd.Context(), "/api/v1/federation/"+url.PathEscape(args[0])+"/sync", nil, &st); err != nil {
return err
}
if *asJSON {
return emitJSON(st)
}
fmt.Fprintf(cmd.OutOrStdout(), "%s: %s, %s\n", st.Name, healthText(st), modelList(st))
return nil
},
}
}
// healthText is "up" or "down: why", with how long ago it was checked.
func healthText(st federation.Status) string {
ago := ""
if !st.Checked.IsZero() {
ago = fmt.Sprintf(" (%s ago)", time.Since(st.Checked).Round(time.Second))
}
if st.Up {
return "up" + ago
}
return "down" + ago + ": " + st.Error
}
// modelList is the peer's models by their names here, a withdrawn one
// marked as such.
func modelList(st federation.Status) string {
if len(st.Models) == 0 {
return "-"
}
names := make([]string, 0, len(st.Models))
for _, m := range st.Models {
n := m.Name
if m.Withdrawn {
n += " (withdrawn)"
}
names = append(names, n)
}
return strings.Join(names, ", ")
}
func newModelFallbacksCmd(client func() *ctlClient) *cobra.Command {
var clear bool
c := &cobra.Command{
Use: "fallbacks MODEL [FALLBACK...]",
Short: "Set the models a model falls back to when it has no capacity",
Long: "Replaces the model's fallbacks, tried in order when nothing serving it\n" +
"has room. A federation peer's model, PEER/MODEL, sends the overflow to\n" +
"the other cluster. A model added at runtime only; one declared in\n" +
"server.yaml has its fallbacks there. --clear takes them all away.",
Example: " llmgw ctl model fallbacks qwen-7b east/qwen-7b\n" +
" llmgw ctl model fallbacks qwen-7b qwen-3b east/qwen-7b\n" +
" llmgw ctl model fallbacks qwen-7b --clear",
Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) == 1 && !clear {
return fmt.Errorf("name the fallbacks, or give --clear to take them away")
}
body, _ := json.Marshal(adminapi.FallbacksRequest{Fallbacks: args[1:]})
var v adminapi.FallbacksView
if err := client().put(cmd.Context(), "/api/v1/models/"+args[0]+"/fallbacks", body, &v); err != nil {
return err
}
if len(v.Fallbacks) == 0 {
fmt.Fprintf(cmd.OutOrStdout(), "%s has no fallbacks\n", v.Model)
return nil
}
fmt.Fprintf(cmd.OutOrStdout(), "%s falls back to %s\n", v.Model, strings.Join(v.Fallbacks, ", then "))
return nil
},
}
c.Flags().BoolVar(&clear, "clear", false, "take every fallback away")
return c
}
+1 -1
View File
@@ -38,7 +38,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
newModelRollbackCmd(client), newModelOriginCmd(client),
newModelCanaryCmd(client), newModelPromoteCmd(client), newModelCanariesCmd(client, asJSON),
newModelGateCmd(client))
newModelGateCmd(client), newModelFallbacksCmd(client))
return c
}
+55
View File
@@ -2,7 +2,9 @@ package cli
import (
"context"
"crypto/rand"
"crypto/tls"
"encoding/hex"
"errors"
"fmt"
"log/slog"
@@ -41,6 +43,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fleetmetrics"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
@@ -176,6 +179,21 @@ func newServerCmd() *cobra.Command {
rt.Start(ctx)
defer rt.Stop()
// Which cluster this is, to federated peers: the id in the hop
// header that stops a request going round two clusters that
// federate each other, and the name in the tags a peer may be
// sent. The id is the store's, so every control plane of one
// cluster says the same.
clusterID, err := federationID(ctx, rt.Store())
if err != nil {
return err
}
clusterName := cfg.Federation.Name
if clusterName == "" {
clusterName = host
}
rt.SetFederationIdentity(clusterID, clusterName)
// Webhooks for what needs a person, when server.yaml names any.
notifier, err := notify.New(ctx, cfg.Notify, log)
if err != nil {
@@ -333,6 +351,21 @@ func newServerCmd() *cobra.Command {
if st := rt.Store(); st != nil {
svc := modelsvc.New(cfg.Models, st, admin, log)
svc.SetCheck(servableCheck(cfg.WantedEngines()["llamacpp"]))
// Other llmgw clusters' models, offered as PEER/MODEL:
// read before the catalog is built, so a model that falls
// back to one is whole at startup.
fed := federation.New(federation.Config{Self: clusterID, Interval: cfg.Federation.Interval},
st, svc, admin.Catalog, log)
if err := fed.Load(ctx); err != nil {
log.Warn("could not read the federation peers; continuing without them", "err", err)
}
svc.SetFederated(fed.Models)
admin.SetPeers(fed)
api.SetFederation(fed)
if err := rt.RegisterMetrics(fed); err != nil {
return err
}
go fed.Run(ctx)
// Rebuild the catalog from configuration plus what was
// stored, and put that in force — otherwise models added
// before the last restart would be silently absent until
@@ -965,3 +998,25 @@ func withLimits(s *http.Server) *http.Server {
s.MaxHeaderBytes = 1 << 20
return s
}
// federationID is this cluster's id among federated peers: kept in the store,
// so every control plane of the cluster and every restart says the same. A
// gateway with no store makes one for its lifetime, which is all a router
// with no peers of its own needs to refuse a request that comes back to it.
func federationID(ctx context.Context, st *storepkg.DB) (string, error) {
if st == nil {
b := make([]byte, 8)
if _, err := rand.Read(b); err != nil {
return "", err
}
return hex.EncodeToString(b), nil
}
secret, err := st.ClusterSecret(ctx, "cluster-id")
if err != nil {
return "", fmt.Errorf("this cluster's id for federation: %w", err)
}
// A prefix is plenty to tell clusters apart, and the id is sent to
// peers: nothing is derived from it, but there is no reason to send
// more of a secret than needed.
return secret[:16], nil
}
+14 -2
View File
@@ -42,6 +42,7 @@ const (
cReplicas
cUsage
cQuotas
cFederation
cMirror
cJobs
cLogs
@@ -61,7 +62,7 @@ const (
cCount
)
var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Audit", "Manual",
var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Federation", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Audit", "Manual",
"server.yaml", "Add a node", "Replicate the store", "Serve a model", "Upgrade the cluster",
"Restart all replicas", "Back up now", "Leave"}
@@ -79,7 +80,10 @@ type consoleData struct {
replicas []adminapi.ReplicaView
jobs []modelserve.Job
audit []store.AuditEvent
err error
// federation is the other llmgw clusters whose models are offered
// here; empty on a control plane without a store.
federation adminapi.FederationView
err error
}
type (
@@ -238,6 +242,7 @@ func (m consoleModel) fetch() tea.Cmd {
_ = c.get(ctx, "/api/v1/replicas", &d.replicas)
_ = c.get(ctx, "/api/v1/jobs", &d.jobs)
_ = c.get(ctx, "/api/v1/audit?since=7d&limit=300", &d.audit)
_ = c.get(ctx, "/api/v1/federation", &d.federation)
return consoleDataMsg(d)
}
}
@@ -494,6 +499,9 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
case cAudit:
m.dialog = "error"
m.dialogTitle, m.dialogText = auditDialog(m.data.audit[m.row])
case cFederation:
m.dialog = "error"
m.dialogTitle, m.dialogText = peerDialog(m.data.federation.Peers[m.row])
case cServe:
return m, m.fetchRepo(m.hf.models[m.row].ID)
case cAccounts:
@@ -908,6 +916,8 @@ func (m consoleModel) rowCount() int {
return len(m.data.jobs)
case cAudit:
return len(m.data.audit)
case cFederation:
return len(m.data.federation.Peers)
case cServe:
return len(m.hf.models)
case cAccounts:
@@ -1149,6 +1159,8 @@ func (m consoleModel) panel() (title, body, keys string) {
return m.jobsPanel(pw)
case cAudit:
return m.auditPanel(pw)
case cFederation:
return m.federationPanel(pw)
case cMonitor:
return m.monitorPanel(pw)
case cServe:
+80
View File
@@ -0,0 +1,80 @@
package cli
import (
"fmt"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
)
// The console's federation view: the other llmgw clusters whose models are
// offered here, whether each is up and how far away, as `llmgw ctl
// federation list` shows them. Peers are added and removed with ctl, which
// needs a key file on this machine that a form here could not check.
func (m consoleModel) federationPanel(pw int) (string, string, string) {
keys := "→ or Enter open · ↑↓ choose · Q leave"
if m.inPanel {
keys = "↑↓ peer · Enter its models · ← back · Q leave"
}
v := m.data.federation
head := "\n " + sDim.Render("Other llmgw clusters whose models are offered here as PEER/MODEL.")
if v.Cluster != "" {
head += "\n " + sDim.Render("This cluster's id: "+v.Cluster)
}
head += "\n\n"
if len(v.Peers) == 0 {
return "Federation", head + " " + sDim.Render("No peers. Add one with: llmgw ctl federation add NAME --url URL --key-file FILE"), keys
}
var rows [][]string
for _, p := range v.Peers {
health := sOK.Render("up")
if !p.Up {
health = sBad.Render("down")
}
lat := "-"
if p.Latency > 0 {
lat = fmt.Sprintf("%.1fms", p.LatencyMs)
}
offered := 0
for _, md := range p.Models {
if !md.Withdrawn {
offered++
}
}
rows = append(rows, []string{p.Name, health, lat, fmt.Sprint(offered), p.URL})
}
return "Federation", head + table([]string{"PEER", "HEALTH", "LATENCY", "MODELS", "URL"}, rows, m.row, pw, m.inPanel), keys
}
// peerDialog is one peer whole: how it was last found, and its models.
func peerDialog(p federation.Status) (string, string) {
var b strings.Builder
fmt.Fprintf(&b, "%s\n", p.URL)
if p.Up {
fmt.Fprintf(&b, "%s, /ready in %.1fms", sOK.Render("up"), p.LatencyMs)
} else {
fmt.Fprintf(&b, "%s: %s", sBad.Render("down"), wrap(p.Error, 80, " "))
}
if !p.Checked.IsZero() {
fmt.Fprintf(&b, ", checked %s ago", time.Since(p.Checked).Round(time.Second))
}
tags := "not told who asks"
if p.SendTags {
tags = "told this cluster and the project of each request (x-llmgw-tags)"
}
fmt.Fprintf(&b, "\ncluster %s, key %s; %s\n\n", p.Cluster, p.KeyFile, tags)
if len(p.Models) == 0 {
b.WriteString(sDim.Render("offers this cluster's key no models") + "\n")
}
for _, md := range p.Models {
line := md.Name
if md.Withdrawn {
line += sDim.Render(" withdrawn by the peer; kept while a fallback here names it")
}
b.WriteString(" " + line + "\n")
}
fmt.Fprintf(&b, "\n%s", sDim.Render("llmgw ctl federation sync|remove "+p.Name+" · any key"))
return p.Name, b.String()
}
+37
View File
@@ -0,0 +1,37 @@
package cli
import (
"strings"
"testing"
"time"
tea "github.com/charmbracelet/bubbletea"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
)
func TestTheConsoleShowsEachPeerAndOpensItsModels(t *testing.T) {
m, _ := consoleWith(t, cFederation)
m.data.federation = adminapi.FederationView{Cluster: "c-west", Peers: []federation.Status{
{Name: "east", URL: "https://east.example", Cluster: "c-east",
Health: federation.Health{Up: true, Latency: 12 * time.Millisecond, Checked: time.Now()},
LatencyMs: 12, Models: []federation.ModelStatus{{Name: "east/qwen", Upstream: "qwen"},
{Name: "east/old", Upstream: "old", Withdrawn: true}}},
{Name: "north", URL: "https://north.example", Health: federation.Health{Error: "the peer did not answer"}},
}}
var tm tea.Model = m
view := tm.View()
for _, want := range []string{"east", "12.0ms", "north", "down", "c-west"} {
if !strings.Contains(view, want) {
t.Errorf("the federation view has no %q:\n%s", want, view)
}
}
tm = press(tm, "enter")
view = tm.View()
for _, want := range []string{"east/qwen", "east/old", "withdrawn", "c-east"} {
if !strings.Contains(view, want) {
t.Errorf("the peer's dialog has no %q:\n%s", want, view)
}
}
}