Standbys that answer the client API themselves, behind router.active_active

A standby follows the leader's Ready replicas over the agent port, every
two seconds, and reports back the replicas its router used and the
on-demand models it was asked for. No engine keys travel: every control
plane derives them from the cluster secret. Budgets are raised to the
store's every 30 s. Unheard from the leader for a minute, a standby
passes /v1 on as before. Off by default.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-01 13:47:47 +01:00
1 parent f664592948
commit 8b7eefef16
22 files changed
+936 -15

No files matched your search

+14 -1
View File
@@ -49,6 +49,9 @@ type leaderForward struct {
self string
tls tlsconf.Config
log *slog.Logger
// routes, with router.active_active, is the leader's replicas this
// standby routes to; nil when every request goes to the leader.
routes *standbyRoutes
}
// standby wraps the admin handler: /api/ goes to the leader while this
@@ -117,7 +120,8 @@ func (f *leaderForward) public(next http.Handler, port string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Once only: a request another control plane forwarded is answered
// here, leading or not, rather than sent on while the lease moves.
if publicStaysHere(r.URL.Path) || f.lead.IsLeader() || r.Header.Get(hdrForwardedBy) != "" {
if publicStaysHere(r.URL.Path) || f.lead.IsLeader() || r.Header.Get(hdrForwardedBy) != "" ||
f.answersHere(r.URL.Path) {
next.ServeHTTP(w, r)
return
}
@@ -154,6 +158,15 @@ func (f *leaderForward) public(next http.Handler, port string) http.Handler {
})
}
// answersHere is whether a standby answers this client request itself: the
// API under /v1, with router.active_active, while it has the leader's
// replicas to send it to. Not the portal, its admin pages or /mcp, whose
// llmgw:admin tools read the cluster from memory only the leader has; and
// not while the leader has gone unheard, when it is passed on as before.
func (f *leaderForward) answersHere(path string) bool {
return f.routes != nil && strings.HasPrefix(path, "/v1/") && f.routes.serving()
}
// trustControlPlanes keeps the client port trusting the other control
// planes' X-Forwarded-For, besides router.trusted_proxies: a standby passes
// requests on, and the client is the address it was given, not the standby.
+8 -1
View File
@@ -32,7 +32,11 @@ type forwardCluster struct {
leaderURL string
}
func newForwardCluster(t *testing.T) *forwardCluster {
func newForwardCluster(t *testing.T) *forwardCluster { return newForwardClusterWith(t, nil) }
// newForwardClusterWith answers forwarded requests on the leader with admin,
// or, given nil, with one that says what it was asked.
func newForwardClusterWith(t *testing.T, leaderAdmin http.Handler) *forwardCluster {
t.Helper()
dir := t.TempDir()
ca, err := tlsconf.NewCA("test CA")
@@ -81,6 +85,9 @@ func newForwardCluster(t *testing.T) *forwardCluster {
"auth": r.Header.Get("Authorization"), "actor": r.Header.Get("X-Llmgw-Actor")})
})
mux := http.NewServeMux()
if leaderAdmin != nil {
admin = leaderAdmin.ServeHTTP
}
mux.Handle(forwardPrefix+"/", c.leader.leaderSide(admin))
srv := httptest.NewUnstartedServer(mux)
tc, err := cfgFor("ctl-a").ServerConfig()
+15
View File
@@ -691,6 +691,21 @@ func newServerCmd() *cobra.Command {
if admin != nil && admin.Leadership() != nil && rt.Store() != nil && cfg.AgentTLS.Enabled() {
self, _ := os.Hostname()
forward = &leaderForward{st: rt.Store(), lead: admin.Leadership(), self: self, tls: cfg.AgentTLS, log: log}
// And with router.active_active, a standby answers the client
// API itself, to the leader's replicas.
if cfg.Router.ActiveActive {
var refresh func(context.Context) error
if catalogSvc != nil {
refresh = catalogSvc.Refresh
}
routes := newStandbyRoutes(forward, rt, refresh, admin.Publish, log)
forward.routes = routes
admin.SetReplicaFeed(routes)
rt.SetActivator(standbyActivator{Reconciler: admin.Reconciler(), routes: routes})
go routes.run(ctx)
go rt.ShareBudgets(ctx)
log.Info("standbys answer the client API themselves (router.active_active)")
}
}
served := adminHandler
if forward != nil {
+240
View File
@@ -0,0 +1,240 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"sync"
"time"
"golang.org/x/net/http2"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
)
// How often a standby answering requests itself hears from the leader, and
// how long it keeps routing to the replicas it last heard of when it does
// not. Long enough to ride out the leader changing hands — the engines are
// still where they were, and a dead one opens its breaker — short enough
// that a standby cut off from the cluster stops sending to a list it cannot
// check.
const (
standbyPoll = 2 * time.Second
standbyTrust = time.Minute
)
// standbyRoutes is a standby's copy of the leader's Ready replicas, when
// standbys answer requests themselves (router.active_active). Each tick it
// reports what its router used and is answered with the replicas, in one
// request over the agent port with its control-plane certificate.
type standbyRoutes struct {
fwd *leaderForward
usage control.UsageSource
refresh func(context.Context) error // reads the catalog again from the store
publish func(context.Context) // hands the router the new list now
log *slog.Logger
now func() time.Time
mu sync.Mutex
reps []control.RoutedReplica
heard time.Time
catalog string // the leader's catalog revision last seen
sent map[string]time.Time // the last use already reported, by replica
want map[string]bool // on-demand models asked for since the last report
kick chan struct{}
// The connection to the leader, kept between ticks: one per leader
// address, the previous one's closed when the lease moves.
client *http.Client
clientHost string
}
func newStandbyRoutes(fwd *leaderForward, usage control.UsageSource, refresh func(context.Context) error,
publish func(context.Context), log *slog.Logger,
) *standbyRoutes {
return &standbyRoutes{fwd: fwd, usage: usage, refresh: refresh, publish: publish, log: log, now: time.Now,
sent: map[string]time.Time{}, want: map[string]bool{}, kick: make(chan struct{}, 1)}
}
// Ready satisfies control.ReplicaFeed.
func (s *standbyRoutes) Ready() ([]control.RoutedReplica, bool) {
s.mu.Lock()
defer s.mu.Unlock()
if s.heard.IsZero() || s.now().Sub(s.heard) > standbyTrust {
return nil, false
}
return s.reps, true
}
// serving says whether this standby has replicas to send a request to, which
// is when it keeps the client API rather than passing it to the leader.
func (s *standbyRoutes) serving() bool {
reps, ok := s.Ready()
return ok && len(reps) > 0
}
// wanted queues an on-demand model for the leader, and reports at once
// rather than on the next tick: someone is waiting on it.
func (s *standbyRoutes) wanted(model string) {
s.mu.Lock()
s.want[model] = true
s.mu.Unlock()
select {
case s.kick <- struct{}{}:
default:
}
}
func (s *standbyRoutes) run(ctx context.Context) {
t := time.NewTicker(standbyPoll)
defer t.Stop()
for {
if s.fwd.lead.IsLeader() {
// Its own reconciler's replicas now. What it heard as a standby
// is forgotten, so becoming one again starts afresh.
s.mu.Lock()
s.reps, s.heard, s.catalog = nil, time.Time{}, ""
clear(s.sent)
clear(s.want)
s.mu.Unlock()
} else if err := s.exchange(ctx); err != nil {
s.log.Debug("could not hear from the leader", "err", err)
}
select {
case <-ctx.Done():
return
case <-t.C:
case <-s.kick:
}
}
}
// exchange reports activity and takes the replicas in return.
func (s *standbyRoutes) exchange(ctx context.Context) error {
act, sent := s.activity()
body, err := json.Marshal(act)
if err != nil {
return err
}
target, name, err := s.fwd.leader(ctx)
if err != nil {
return err
}
client, err := s.clientFor(target.Host, target.Hostname())
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
"https://"+target.Host+forwardPrefix+"/api/v1/routing/activity", bytes.NewReader(body))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set(hdrForwardedBy, s.fwd.self)
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
return fmt.Errorf("%s answered %s: %s", name, resp.Status, bytes.TrimSpace(b))
}
var rt control.Routing
if err := json.NewDecoder(resp.Body).Decode(&rt); err != nil {
return fmt.Errorf("%s's routing: %w", name, err)
}
s.mu.Lock()
s.reps, s.heard = rt.Replicas, s.now()
stale := rt.Catalog != s.catalog
s.catalog = rt.Catalog
for id, at := range sent {
s.sent[id] = at
}
for _, m := range act.Want {
delete(s.want, m)
}
s.mu.Unlock()
// A model added or changed through the leader is in the store: the
// catalog first, so the replicas published next resolve against it.
if stale && s.refresh != nil {
if err := s.refresh(ctx); err != nil {
s.log.Warn("could not read the catalog the leader changed", "err", err)
s.mu.Lock()
s.catalog = "" // tried again next tick
s.mu.Unlock()
}
}
if s.publish != nil {
s.publish(ctx)
}
return nil
}
// clientFor is the client for the leader at host, made once per leader.
// Only the run loop calls it.
func (s *standbyRoutes) clientFor(host, name string) (*http.Client, error) {
if s.client != nil && s.clientHost == host {
return s.client, nil
}
tc, err := s.fwd.tls.ClientConfig(name)
if err != nil {
return nil, err
}
if s.client != nil {
s.client.CloseIdleConnections()
}
s.client = &http.Client{Transport: &http2.Transport{TLSClientConfig: tc, ReadIdleTimeout: 30 * time.Second}}
s.clientHost = host
return s.client, nil
}
// activity is what to report: uses newer than the last reported, and the
// models asked for. sent is what to record as reported once the leader has
// it.
func (s *standbyRoutes) activity() (control.Activity, map[string]time.Time) {
var used map[string]time.Time
if s.usage != nil {
used = s.usage.LastUsed()
}
s.mu.Lock()
defer s.mu.Unlock()
act := control.Activity{LastUsed: map[string]time.Time{}}
for id, at := range used {
if at.After(s.sent[id]) {
act.LastUsed[id] = at
}
}
for m := range s.want {
act.Want = append(act.Want, m)
}
return act, act.LastUsed
}
// standbyActivator asks the leader for an on-demand model when a request for
// it reaches a standby: the standby's own reconciler places nothing. Which
// models load on demand, and why one cannot, are still the reconciler's to
// answer from the catalog.
type standbyActivator struct {
*reconciler.Reconciler
routes *standbyRoutes
}
func (a standbyActivator) Want(model string) bool {
if !a.Reconciler.Want(model) {
return false
}
if !a.routes.fwd.lead.IsLeader() {
a.routes.wanted(model)
}
return true
}
+153
View File
@@ -0,0 +1,153 @@
package cli
import (
"context"
"encoding/json"
"io"
"log/slog"
"net"
"net/http"
"net/http/httptest"
"slices"
"strings"
"sync"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
)
// fakeLeaderRouting is the leader's routing API: what it was told, and the
// replicas and catalog it answers with.
type fakeLeaderRouting struct {
mu sync.Mutex
got []control.Activity
routing control.Routing
}
func (f *fakeLeaderRouting) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/api/v1/routing/activity" {
http.NotFound(w, r)
return
}
var act control.Activity
if err := json.NewDecoder(r.Body).Decode(&act); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
f.mu.Lock()
defer f.mu.Unlock()
f.got = append(f.got, act)
_ = json.NewEncoder(w).Encode(f.routing)
}
func (f *fakeLeaderRouting) last() control.Activity {
f.mu.Lock()
defer f.mu.Unlock()
return f.got[len(f.got)-1]
}
type fixedUsage map[string]time.Time
func (u fixedUsage) LastUsed() map[string]time.Time { return u }
func TestAStandbyFollowsTheLeadersReplicas(t *testing.T) {
leader := &fakeLeaderRouting{routing: control.Routing{Catalog: "a/1",
Replicas: []control.RoutedReplica{{ID: "llama-r0", Model: "llama", Endpoint: "http://10.0.0.5:20001"}}}}
c := newForwardClusterWith(t, leader)
used := time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC)
refreshed := 0
s := newStandbyRoutes(c.standby, fixedUsage{"llama-r0": used}, func(_ context.Context) error {
refreshed++
return nil
}, nil, slog.New(slog.DiscardHandler))
if _, ok := s.Ready(); ok {
t.Fatal("ready before hearing from the leader")
}
if err := s.exchange(t.Context()); err != nil {
t.Fatal(err)
}
if reps, ok := s.Ready(); !ok || len(reps) != 1 || reps[0].ID != "llama-r0" {
t.Fatalf("replicas %+v, %v", reps, ok)
}
if got := leader.last().LastUsed["llama-r0"]; !got.Equal(used) {
t.Errorf("the leader was told the replica was last used %v", got)
}
if refreshed != 1 {
t.Errorf("the catalog was read %d times on first hearing from the leader", refreshed)
}
// Only what is new is reported, and the catalog is read again only when
// the leader's changed.
s.wanted("flux")
if err := s.exchange(t.Context()); err != nil {
t.Fatal(err)
}
if act := leader.last(); len(act.LastUsed) != 0 || !slices.Equal(act.Want, []string{"flux"}) {
t.Errorf("second report %+v", act)
}
if refreshed != 1 {
t.Errorf("an unchanged catalog was read again")
}
leader.mu.Lock()
leader.routing.Catalog = "a/2"
leader.mu.Unlock()
if err := s.exchange(t.Context()); err != nil {
t.Fatal(err)
}
if act := leader.last(); len(act.Want) != 0 {
t.Errorf("an on-demand model reported twice: %+v", act)
}
if refreshed != 2 {
t.Errorf("the changed catalog was not read: %d", refreshed)
}
// Unheard for longer than it is trusted, it routes nothing, and requests
// go to the leader again.
c.standby.routes = s
if !c.standby.answersHere("/v1/chat/completions") {
t.Error("a standby with replicas passed the API on")
}
s.now = func() time.Time { return time.Now().Add(standbyTrust + time.Second) }
if _, ok := s.Ready(); ok || c.standby.answersHere("/v1/chat/completions") {
t.Error("a standby the leader has not answered for a minute still routes")
}
}
// The client API stays on a standby with replicas; the portal, its admin
// pages and /mcp still go to the leader.
func TestAStandbyAnswersOnlyTheAPIItself(t *testing.T) {
leader := &fakeLeaderRouting{routing: control.Routing{
Replicas: []control.RoutedReplica{{ID: "llama-r0", Model: "llama", Endpoint: "http://10.0.0.5:20001"}}}}
c := newForwardClusterWith(t, leader)
s := newStandbyRoutes(c.standby, nil, nil, nil, slog.New(slog.DiscardHandler))
if err := s.exchange(t.Context()); err != nil {
t.Fatal(err)
}
c.standby.routes = s
port := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("leader " + r.URL.Path))
}))
defer port.Close()
_, p, _ := net.SplitHostPort(strings.TrimPrefix(port.URL, "http://"))
local := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("standby " + r.URL.Path)) })
srv := httptest.NewServer(c.standby.public(local, p))
defer srv.Close()
for path, want := range map[string]string{
"/v1/chat/completions": "standby", "/v1/models": "standby",
"/admin/replicas": "leader", "/mcp": "leader", "/": "leader",
} {
resp, err := http.Get(srv.URL + path) //nolint:noctx // a test
if err != nil {
t.Fatal(err)
}
b, _ := io.ReadAll(resp.Body)
resp.Body.Close()
if got, _, _ := strings.Cut(string(b), " "); got != want {
t.Errorf("%s answered by %q, want the %s", path, b, want)
}
}
}