Standbys that answer the client API themselves, behind router.active_active
CI / test (push) Successful in 9m35s
CI / test (push) Successful in 9m35s
A standby follows the leader's Ready replicas over the agent port, every two seconds, and reports back the replicas its router used and the on-demand models it was asked for. No engine keys travel: every control plane derives them from the cluster secret. Budgets are raised to the store's every 30 s. Unheard from the leader for a minute, a standby passes /v1 on as before. Off by default. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
f664592948
commit
8b7eefef16
22 files changed
+936
-15
No files matched your search
@@ -49,6 +49,9 @@ type leaderForward struct {
|
||||
self string
|
||||
tls tlsconf.Config
|
||||
log *slog.Logger
|
||||
// routes, with router.active_active, is the leader's replicas this
|
||||
// standby routes to; nil when every request goes to the leader.
|
||||
routes *standbyRoutes
|
||||
}
|
||||
|
||||
// standby wraps the admin handler: /api/ goes to the leader while this
|
||||
@@ -117,7 +120,8 @@ func (f *leaderForward) public(next http.Handler, port string) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Once only: a request another control plane forwarded is answered
|
||||
// here, leading or not, rather than sent on while the lease moves.
|
||||
if publicStaysHere(r.URL.Path) || f.lead.IsLeader() || r.Header.Get(hdrForwardedBy) != "" {
|
||||
if publicStaysHere(r.URL.Path) || f.lead.IsLeader() || r.Header.Get(hdrForwardedBy) != "" ||
|
||||
f.answersHere(r.URL.Path) {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
@@ -154,6 +158,15 @@ func (f *leaderForward) public(next http.Handler, port string) http.Handler {
|
||||
})
|
||||
}
|
||||
|
||||
// answersHere is whether a standby answers this client request itself: the
|
||||
// API under /v1, with router.active_active, while it has the leader's
|
||||
// replicas to send it to. Not the portal, its admin pages or /mcp, whose
|
||||
// llmgw:admin tools read the cluster from memory only the leader has; and
|
||||
// not while the leader has gone unheard, when it is passed on as before.
|
||||
func (f *leaderForward) answersHere(path string) bool {
|
||||
return f.routes != nil && strings.HasPrefix(path, "/v1/") && f.routes.serving()
|
||||
}
|
||||
|
||||
// trustControlPlanes keeps the client port trusting the other control
|
||||
// planes' X-Forwarded-For, besides router.trusted_proxies: a standby passes
|
||||
// requests on, and the client is the address it was given, not the standby.
|
||||
|
||||
@@ -32,7 +32,11 @@ type forwardCluster struct {
|
||||
leaderURL string
|
||||
}
|
||||
|
||||
func newForwardCluster(t *testing.T) *forwardCluster {
|
||||
func newForwardCluster(t *testing.T) *forwardCluster { return newForwardClusterWith(t, nil) }
|
||||
|
||||
// newForwardClusterWith answers forwarded requests on the leader with admin,
|
||||
// or, given nil, with one that says what it was asked.
|
||||
func newForwardClusterWith(t *testing.T, leaderAdmin http.Handler) *forwardCluster {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
ca, err := tlsconf.NewCA("test CA")
|
||||
@@ -81,6 +85,9 @@ func newForwardCluster(t *testing.T) *forwardCluster {
|
||||
"auth": r.Header.Get("Authorization"), "actor": r.Header.Get("X-Llmgw-Actor")})
|
||||
})
|
||||
mux := http.NewServeMux()
|
||||
if leaderAdmin != nil {
|
||||
admin = leaderAdmin.ServeHTTP
|
||||
}
|
||||
mux.Handle(forwardPrefix+"/", c.leader.leaderSide(admin))
|
||||
srv := httptest.NewUnstartedServer(mux)
|
||||
tc, err := cfgFor("ctl-a").ServerConfig()
|
||||
|
||||
@@ -691,6 +691,21 @@ func newServerCmd() *cobra.Command {
|
||||
if admin != nil && admin.Leadership() != nil && rt.Store() != nil && cfg.AgentTLS.Enabled() {
|
||||
self, _ := os.Hostname()
|
||||
forward = &leaderForward{st: rt.Store(), lead: admin.Leadership(), self: self, tls: cfg.AgentTLS, log: log}
|
||||
// And with router.active_active, a standby answers the client
|
||||
// API itself, to the leader's replicas.
|
||||
if cfg.Router.ActiveActive {
|
||||
var refresh func(context.Context) error
|
||||
if catalogSvc != nil {
|
||||
refresh = catalogSvc.Refresh
|
||||
}
|
||||
routes := newStandbyRoutes(forward, rt, refresh, admin.Publish, log)
|
||||
forward.routes = routes
|
||||
admin.SetReplicaFeed(routes)
|
||||
rt.SetActivator(standbyActivator{Reconciler: admin.Reconciler(), routes: routes})
|
||||
go routes.run(ctx)
|
||||
go rt.ShareBudgets(ctx)
|
||||
log.Info("standbys answer the client API themselves (router.active_active)")
|
||||
}
|
||||
}
|
||||
served := adminHandler
|
||||
if forward != nil {
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/http2"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
)
|
||||
|
||||
// How often a standby answering requests itself hears from the leader, and
|
||||
// how long it keeps routing to the replicas it last heard of when it does
|
||||
// not. Long enough to ride out the leader changing hands — the engines are
|
||||
// still where they were, and a dead one opens its breaker — short enough
|
||||
// that a standby cut off from the cluster stops sending to a list it cannot
|
||||
// check.
|
||||
const (
|
||||
standbyPoll = 2 * time.Second
|
||||
standbyTrust = time.Minute
|
||||
)
|
||||
|
||||
// standbyRoutes is a standby's copy of the leader's Ready replicas, when
|
||||
// standbys answer requests themselves (router.active_active). Each tick it
|
||||
// reports what its router used and is answered with the replicas, in one
|
||||
// request over the agent port with its control-plane certificate.
|
||||
type standbyRoutes struct {
|
||||
fwd *leaderForward
|
||||
usage control.UsageSource
|
||||
refresh func(context.Context) error // reads the catalog again from the store
|
||||
publish func(context.Context) // hands the router the new list now
|
||||
log *slog.Logger
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
reps []control.RoutedReplica
|
||||
heard time.Time
|
||||
catalog string // the leader's catalog revision last seen
|
||||
sent map[string]time.Time // the last use already reported, by replica
|
||||
want map[string]bool // on-demand models asked for since the last report
|
||||
kick chan struct{}
|
||||
|
||||
// The connection to the leader, kept between ticks: one per leader
|
||||
// address, the previous one's closed when the lease moves.
|
||||
client *http.Client
|
||||
clientHost string
|
||||
}
|
||||
|
||||
func newStandbyRoutes(fwd *leaderForward, usage control.UsageSource, refresh func(context.Context) error,
|
||||
publish func(context.Context), log *slog.Logger,
|
||||
) *standbyRoutes {
|
||||
return &standbyRoutes{fwd: fwd, usage: usage, refresh: refresh, publish: publish, log: log, now: time.Now,
|
||||
sent: map[string]time.Time{}, want: map[string]bool{}, kick: make(chan struct{}, 1)}
|
||||
}
|
||||
|
||||
// Ready satisfies control.ReplicaFeed.
|
||||
func (s *standbyRoutes) Ready() ([]control.RoutedReplica, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.heard.IsZero() || s.now().Sub(s.heard) > standbyTrust {
|
||||
return nil, false
|
||||
}
|
||||
return s.reps, true
|
||||
}
|
||||
|
||||
// serving says whether this standby has replicas to send a request to, which
|
||||
// is when it keeps the client API rather than passing it to the leader.
|
||||
func (s *standbyRoutes) serving() bool {
|
||||
reps, ok := s.Ready()
|
||||
return ok && len(reps) > 0
|
||||
}
|
||||
|
||||
// wanted queues an on-demand model for the leader, and reports at once
|
||||
// rather than on the next tick: someone is waiting on it.
|
||||
func (s *standbyRoutes) wanted(model string) {
|
||||
s.mu.Lock()
|
||||
s.want[model] = true
|
||||
s.mu.Unlock()
|
||||
select {
|
||||
case s.kick <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (s *standbyRoutes) run(ctx context.Context) {
|
||||
t := time.NewTicker(standbyPoll)
|
||||
defer t.Stop()
|
||||
for {
|
||||
if s.fwd.lead.IsLeader() {
|
||||
// Its own reconciler's replicas now. What it heard as a standby
|
||||
// is forgotten, so becoming one again starts afresh.
|
||||
s.mu.Lock()
|
||||
s.reps, s.heard, s.catalog = nil, time.Time{}, ""
|
||||
clear(s.sent)
|
||||
clear(s.want)
|
||||
s.mu.Unlock()
|
||||
} else if err := s.exchange(ctx); err != nil {
|
||||
s.log.Debug("could not hear from the leader", "err", err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
case <-s.kick:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// exchange reports activity and takes the replicas in return.
|
||||
func (s *standbyRoutes) exchange(ctx context.Context) error {
|
||||
act, sent := s.activity()
|
||||
body, err := json.Marshal(act)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
target, name, err := s.fwd.leader(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
client, err := s.clientFor(target.Host, target.Hostname())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
"https://"+target.Host+forwardPrefix+"/api/v1/routing/activity", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set(hdrForwardedBy, s.fwd.self)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
return fmt.Errorf("%s answered %s: %s", name, resp.Status, bytes.TrimSpace(b))
|
||||
}
|
||||
var rt control.Routing
|
||||
if err := json.NewDecoder(resp.Body).Decode(&rt); err != nil {
|
||||
return fmt.Errorf("%s's routing: %w", name, err)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.reps, s.heard = rt.Replicas, s.now()
|
||||
stale := rt.Catalog != s.catalog
|
||||
s.catalog = rt.Catalog
|
||||
for id, at := range sent {
|
||||
s.sent[id] = at
|
||||
}
|
||||
for _, m := range act.Want {
|
||||
delete(s.want, m)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
// A model added or changed through the leader is in the store: the
|
||||
// catalog first, so the replicas published next resolve against it.
|
||||
if stale && s.refresh != nil {
|
||||
if err := s.refresh(ctx); err != nil {
|
||||
s.log.Warn("could not read the catalog the leader changed", "err", err)
|
||||
s.mu.Lock()
|
||||
s.catalog = "" // tried again next tick
|
||||
s.mu.Unlock()
|
||||
}
|
||||
}
|
||||
if s.publish != nil {
|
||||
s.publish(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// clientFor is the client for the leader at host, made once per leader.
|
||||
// Only the run loop calls it.
|
||||
func (s *standbyRoutes) clientFor(host, name string) (*http.Client, error) {
|
||||
if s.client != nil && s.clientHost == host {
|
||||
return s.client, nil
|
||||
}
|
||||
tc, err := s.fwd.tls.ClientConfig(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.client != nil {
|
||||
s.client.CloseIdleConnections()
|
||||
}
|
||||
s.client = &http.Client{Transport: &http2.Transport{TLSClientConfig: tc, ReadIdleTimeout: 30 * time.Second}}
|
||||
s.clientHost = host
|
||||
return s.client, nil
|
||||
}
|
||||
|
||||
// activity is what to report: uses newer than the last reported, and the
|
||||
// models asked for. sent is what to record as reported once the leader has
|
||||
// it.
|
||||
func (s *standbyRoutes) activity() (control.Activity, map[string]time.Time) {
|
||||
var used map[string]time.Time
|
||||
if s.usage != nil {
|
||||
used = s.usage.LastUsed()
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
act := control.Activity{LastUsed: map[string]time.Time{}}
|
||||
for id, at := range used {
|
||||
if at.After(s.sent[id]) {
|
||||
act.LastUsed[id] = at
|
||||
}
|
||||
}
|
||||
for m := range s.want {
|
||||
act.Want = append(act.Want, m)
|
||||
}
|
||||
return act, act.LastUsed
|
||||
}
|
||||
|
||||
// standbyActivator asks the leader for an on-demand model when a request for
|
||||
// it reaches a standby: the standby's own reconciler places nothing. Which
|
||||
// models load on demand, and why one cannot, are still the reconciler's to
|
||||
// answer from the catalog.
|
||||
type standbyActivator struct {
|
||||
*reconciler.Reconciler
|
||||
routes *standbyRoutes
|
||||
}
|
||||
|
||||
func (a standbyActivator) Want(model string) bool {
|
||||
if !a.Reconciler.Want(model) {
|
||||
return false
|
||||
}
|
||||
if !a.routes.fwd.lead.IsLeader() {
|
||||
a.routes.wanted(model)
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
)
|
||||
|
||||
// fakeLeaderRouting is the leader's routing API: what it was told, and the
|
||||
// replicas and catalog it answers with.
|
||||
type fakeLeaderRouting struct {
|
||||
mu sync.Mutex
|
||||
got []control.Activity
|
||||
routing control.Routing
|
||||
}
|
||||
|
||||
func (f *fakeLeaderRouting) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost || r.URL.Path != "/api/v1/routing/activity" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
var act control.Activity
|
||||
if err := json.NewDecoder(r.Body).Decode(&act); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.got = append(f.got, act)
|
||||
_ = json.NewEncoder(w).Encode(f.routing)
|
||||
}
|
||||
|
||||
func (f *fakeLeaderRouting) last() control.Activity {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.got[len(f.got)-1]
|
||||
}
|
||||
|
||||
type fixedUsage map[string]time.Time
|
||||
|
||||
func (u fixedUsage) LastUsed() map[string]time.Time { return u }
|
||||
|
||||
func TestAStandbyFollowsTheLeadersReplicas(t *testing.T) {
|
||||
leader := &fakeLeaderRouting{routing: control.Routing{Catalog: "a/1",
|
||||
Replicas: []control.RoutedReplica{{ID: "llama-r0", Model: "llama", Endpoint: "http://10.0.0.5:20001"}}}}
|
||||
c := newForwardClusterWith(t, leader)
|
||||
used := time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC)
|
||||
refreshed := 0
|
||||
s := newStandbyRoutes(c.standby, fixedUsage{"llama-r0": used}, func(_ context.Context) error {
|
||||
refreshed++
|
||||
return nil
|
||||
}, nil, slog.New(slog.DiscardHandler))
|
||||
|
||||
if _, ok := s.Ready(); ok {
|
||||
t.Fatal("ready before hearing from the leader")
|
||||
}
|
||||
if err := s.exchange(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if reps, ok := s.Ready(); !ok || len(reps) != 1 || reps[0].ID != "llama-r0" {
|
||||
t.Fatalf("replicas %+v, %v", reps, ok)
|
||||
}
|
||||
if got := leader.last().LastUsed["llama-r0"]; !got.Equal(used) {
|
||||
t.Errorf("the leader was told the replica was last used %v", got)
|
||||
}
|
||||
if refreshed != 1 {
|
||||
t.Errorf("the catalog was read %d times on first hearing from the leader", refreshed)
|
||||
}
|
||||
|
||||
// Only what is new is reported, and the catalog is read again only when
|
||||
// the leader's changed.
|
||||
s.wanted("flux")
|
||||
if err := s.exchange(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if act := leader.last(); len(act.LastUsed) != 0 || !slices.Equal(act.Want, []string{"flux"}) {
|
||||
t.Errorf("second report %+v", act)
|
||||
}
|
||||
if refreshed != 1 {
|
||||
t.Errorf("an unchanged catalog was read again")
|
||||
}
|
||||
leader.mu.Lock()
|
||||
leader.routing.Catalog = "a/2"
|
||||
leader.mu.Unlock()
|
||||
if err := s.exchange(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if act := leader.last(); len(act.Want) != 0 {
|
||||
t.Errorf("an on-demand model reported twice: %+v", act)
|
||||
}
|
||||
if refreshed != 2 {
|
||||
t.Errorf("the changed catalog was not read: %d", refreshed)
|
||||
}
|
||||
|
||||
// Unheard for longer than it is trusted, it routes nothing, and requests
|
||||
// go to the leader again.
|
||||
c.standby.routes = s
|
||||
if !c.standby.answersHere("/v1/chat/completions") {
|
||||
t.Error("a standby with replicas passed the API on")
|
||||
}
|
||||
s.now = func() time.Time { return time.Now().Add(standbyTrust + time.Second) }
|
||||
if _, ok := s.Ready(); ok || c.standby.answersHere("/v1/chat/completions") {
|
||||
t.Error("a standby the leader has not answered for a minute still routes")
|
||||
}
|
||||
}
|
||||
|
||||
// The client API stays on a standby with replicas; the portal, its admin
|
||||
// pages and /mcp still go to the leader.
|
||||
func TestAStandbyAnswersOnlyTheAPIItself(t *testing.T) {
|
||||
leader := &fakeLeaderRouting{routing: control.Routing{
|
||||
Replicas: []control.RoutedReplica{{ID: "llama-r0", Model: "llama", Endpoint: "http://10.0.0.5:20001"}}}}
|
||||
c := newForwardClusterWith(t, leader)
|
||||
s := newStandbyRoutes(c.standby, nil, nil, nil, slog.New(slog.DiscardHandler))
|
||||
if err := s.exchange(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.standby.routes = s
|
||||
|
||||
port := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte("leader " + r.URL.Path))
|
||||
}))
|
||||
defer port.Close()
|
||||
_, p, _ := net.SplitHostPort(strings.TrimPrefix(port.URL, "http://"))
|
||||
local := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("standby " + r.URL.Path)) })
|
||||
srv := httptest.NewServer(c.standby.public(local, p))
|
||||
defer srv.Close()
|
||||
|
||||
for path, want := range map[string]string{
|
||||
"/v1/chat/completions": "standby", "/v1/models": "standby",
|
||||
"/admin/replicas": "leader", "/mcp": "leader", "/": "leader",
|
||||
} {
|
||||
resp, err := http.Get(srv.URL + path) //nolint:noctx // a test
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if got, _, _ := strings.Cut(string(b), " "); got != want {
|
||||
t.Errorf("%s answered by %q, want the %s", path, b, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user