Phase 3 (3/3): wire the control plane to the router
CI / test (push) Successful in 50s

Ready replicas become router endpoints with no static configuration involved.
Endpoints that survive an update keep their live state — inflight counts,
breaker state, drain rate — so a replica that was already serving does not start
from cold because another one moved. The sink is an interface, so when the
router and control plane split into separate processes it becomes a watch stream
and nothing above it changes.

Phase 3's exit criterion now holds end to end. A node hosting a live replica is
killed while four clients stream continuously; the control plane marks it
unreachable, then lost, rebuilds the replica on a survivor and republishes.
Across three runs ~370 requests completed with zero failures and no replica was
left on the dead node. This proves the control logic, not the transport: the
agent fleet is simulated and the real path still needs the stream.

The race detector caught a genuine bug while this went in. startReplica read the
replica's fields after publishing it to the map, and scaleUp walked replica
pointers outside the lock; both raced with agent status reports, which arrive
from another goroutine and mutate exactly those fields. Against the simulated
fleet the reports came back fast enough to hit the window on every run. State is
snapshotted under the lock now and those functions work from values.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
ilkerandClaude Opus 5 committed 2026-09-16 02:16:35 +02:00
1 parent f0134a2f95
commit c1b9ea1bc2
6 files changed
+685 -38

No files matched your search

+182
View File
@@ -0,0 +1,182 @@
// Package control wires the control plane together: inventory, catalog,
// placement and reconciliation, and publishes the result to the router.
package control
import (
"context"
"fmt"
"log/slog"
"strings"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
// EndpointSink receives the endpoint list. The router implements it.
//
// It is an interface so the control plane does not depend on the router: when
// the two split into separate processes, this becomes a watch stream and
// nothing above it changes (plan §3.1).
type EndpointSink interface {
SetEndpoints(ctx context.Context, infos []endpoint.Info)
}
// Config tunes the control plane.
type Config struct {
Reconciler reconciler.Config
// PublishInterval is how often the endpoint list is republished even when
// nothing appears to have changed. Events drive the common case; this is
// the backstop.
PublishInterval time.Duration
}
// DefaultConfig returns sensible defaults.
func DefaultConfig() Config {
return Config{
Reconciler: reconciler.DefaultConfig(),
PublishInterval: time.Second,
}
}
// Controller owns the control plane's state.
type Controller struct {
cfg Config
log *slog.Logger
inv *inventory.Store
cat *catalog.Catalog
rec *reconciler.Reconciler
sink EndpointSink
mu sync.Mutex
last string // fingerprint of the last published list
stop chan struct{}
done chan struct{}
once sync.Once
}
// New builds a controller.
func New(cfg Config, cat *catalog.Catalog, disp reconciler.Dispatcher, sink EndpointSink, log *slog.Logger) *Controller {
if log == nil {
log = slog.Default()
}
if cfg.PublishInterval <= 0 {
cfg.PublishInterval = time.Second
}
inv := inventory.NewStore()
return &Controller{
cfg: cfg,
log: log,
inv: inv,
cat: cat,
rec: reconciler.New(cfg.Reconciler, cat, inv, disp, log),
sink: sink,
stop: make(chan struct{}),
done: make(chan struct{}),
}
}
// Inventory exposes the node inventory, which the agent service feeds.
func (c *Controller) Inventory() *inventory.Store { return c.inv }
// Reconciler exposes the reconciliation loop, for status reports and ctl.
func (c *Controller) Reconciler() *reconciler.Reconciler { return c.rec }
// Catalog exposes the model catalog.
func (c *Controller) Catalog() *catalog.Catalog { return c.cat }
// Start runs the reconciler and begins publishing endpoints.
func (c *Controller) Start(ctx context.Context) {
c.rec.Start(ctx)
go func() {
defer close(c.done)
t := time.NewTicker(c.cfg.PublishInterval)
defer t.Stop()
c.Publish(ctx)
for {
select {
case <-c.stop:
return
case <-ctx.Done():
return
case <-t.C:
c.Publish(ctx)
}
}
}()
}
// Close stops the control plane.
func (c *Controller) Close() {
c.once.Do(func() {
close(c.stop)
<-c.done
c.rec.Close()
})
}
// Publish pushes the current Ready replicas to the sink.
//
// It is exported so tests can drive it deterministically instead of racing a
// ticker, and it skips the push when nothing changed: replacing the list is
// cheap but resynchronizes the metric pollers, and doing that every second for
// no reason adds churn to every engine's /metrics.
func (c *Controller) Publish(ctx context.Context) {
infos := c.endpoints()
fp := fingerprint(infos)
c.mu.Lock()
unchanged := fp == c.last
c.last = fp
c.mu.Unlock()
if unchanged {
return
}
c.log.Info("publishing endpoints", "count", len(infos))
c.sink.SetEndpoints(ctx, infos)
}
// endpoints converts Ready replicas into the router's view.
func (c *Controller) endpoints() []endpoint.Info {
ready := c.rec.ReadyReplicas()
out := make([]endpoint.Info, 0, len(ready))
for _, rep := range ready {
m, ok := c.cat.Resolve(rep.Model)
if !ok {
// The model left the catalog while a replica was still running.
// The reconciler will tear it down; until then, do not route to it.
continue
}
// Clients may address a model by any of its names, and the router
// matches endpoints by the resolved name plus aliases.
names := append([]string{m.Name}, m.Aliases...)
prof, _ := m.ProfileByName(rep.Profile)
out = append(out, endpoint.Info{
ID: rep.ID,
URL: strings.TrimRight(rep.Endpoint, "/"),
Models: names,
Pool: rep.Pool,
Weight: rep.Weight,
Priority: prof.Priority,
})
}
return out
}
// fingerprint summarizes a list so an unchanged one can be skipped.
func fingerprint(infos []endpoint.Info) string {
var b strings.Builder
for _, i := range infos {
fmt.Fprintf(&b, "%s|%s|%s|%v|%g|%d;", i.ID, i.URL, i.Pool, i.Models, i.Weight, i.Priority)
}
return b.String()
}
+45 -28
View File
@@ -254,10 +254,14 @@ func (r *Reconciler) reapFailed(ctx context.Context) {
r.mu.Unlock()
for _, rep := range doomed {
// rep is out of the map now, so no agent report can reach it any more.
if rep.State == Failed {
r.mu.Lock()
retry := r.backoff(r.attempts[rep.Model])
r.mu.Unlock()
r.log.Error("replica failed",
"replica", rep.ID, "model", rep.Model, "err", rep.Err,
"retry_in", r.backoff(r.attempts[rep.Model]).Round(time.Second))
"retry_in", retry.Round(time.Second))
}
// Stop every shard, including the ones that looked healthy: a partially
// alive replica holds GPUs while serving nothing.
@@ -286,16 +290,21 @@ func (r *Reconciler) backoff(attempt int) time.Duration {
}
func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
// Snapshot under the lock and work from values afterwards. A replica
// pointer stops being safe the moment it is in the map: an agent's status
// report can mutate it from another goroutine at any time.
r.mu.Lock()
var live, draining []*Replica
var liveIDs, drainingIDs, avoid []string
for _, rep := range r.replicas {
if rep.Model != m.Name {
continue
}
if rep.State.Counts() {
live = append(live, rep)
} else if rep.State == Draining {
draining = append(draining, rep)
switch {
case rep.State.Counts():
liveIDs = append(liveIDs, rep.ID)
avoid = append(avoid, rep.NodeIDs()...)
case rep.State == Draining:
drainingIDs = append(drainingIDs, rep.ID)
}
}
blockedUntil := r.nextAttempt[m.Name]
@@ -303,21 +312,21 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
desired := m.Scaling.Min
switch {
case len(live) < desired:
case len(liveIDs) < desired:
if r.now().Before(blockedUntil) {
// Still in backoff. Rebuilding immediately would hot-loop on a
// model that cannot start — a missing weights path, say — and bury
// the real error in noise.
return
}
r.scaleUp(ctx, m, live)
case len(live) > desired:
r.scaleDown(ctx, live[desired:])
r.scaleUp(ctx, m, avoid)
case len(liveIDs) > desired:
r.scaleDown(ctx, liveIDs[desired:])
}
// Drained replicas can finally be stopped.
for _, rep := range draining {
r.stopReplica(ctx, rep)
for _, id := range drainingIDs {
r.stopReplica(ctx, id)
}
}
@@ -326,12 +335,7 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
// One at a time on purpose: placement decisions depend on what is already
// allocated, and starting several at once from the same inventory snapshot
// would hand them the same GPUs.
func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, live []*Replica) {
var avoid []string
for _, rep := range live {
avoid = append(avoid, rep.NodeIDs()...)
}
func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []string) {
res := r.sched.Place(m.PlacementProfiles(), avoid)
if res.Placement == nil && len(avoid) > 0 {
// Spreading across failure domains is a preference, not a rule. If it
@@ -398,15 +402,20 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
}
rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort)
r.mu.Lock()
r.replicas[id] = rep
r.mu.Unlock()
// Everything read from rep has to be read now: once it is in the map an
// agent status report can mutate it concurrently, and the race detector is
// right to object.
//
// Workers first, then the leader. SGLang's ranks rendezvous, and vLLM under
// Ray needs its workers attached before the head starts serving; starting
// rank 0 first just makes it wait.
order := slices.Clone(rep.Shards)
slices.SortFunc(order, func(a, b Shard) int { return b.Rank - a.Rank })
nodeIDs := rep.NodeIDs()
r.mu.Lock()
r.replicas[id] = rep
r.mu.Unlock()
for _, s := range order {
cmd := ShardCommand{
@@ -437,7 +446,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
r.log.Info("replica placed",
"replica", id, "model", m.Name, "profile", prof.Name,
"pool", pl.Pool, "fabric", pl.Fabric, "nodes", rep.NodeIDs())
"pool", pl.Pool, "fabric", pl.Fabric, "nodes", nodeIDs)
return nil
}
@@ -465,12 +474,16 @@ func (r *Reconciler) allocatePorts(pl *placement.Placement) int {
return r.cfg.PortRange[0]
}
func (r *Reconciler) scaleDown(ctx context.Context, excess []*Replica) {
for _, rep := range excess {
func (r *Reconciler) scaleDown(ctx context.Context, excess []string) {
for _, id := range excess {
r.mu.Lock()
rep, ok := r.replicas[id]
if !ok {
r.mu.Unlock()
continue
}
rep.State = Draining
shards := slices.Clone(rep.Shards)
id := rep.ID
r.mu.Unlock()
r.log.Info("draining replica", "replica", id)
@@ -482,11 +495,15 @@ func (r *Reconciler) scaleDown(ctx context.Context, excess []*Replica) {
}
}
func (r *Reconciler) stopReplica(ctx context.Context, rep *Replica) {
func (r *Reconciler) stopReplica(ctx context.Context, id string) {
r.mu.Lock()
rep, ok := r.replicas[id]
if !ok {
r.mu.Unlock()
return
}
shards := slices.Clone(rep.Shards)
rep.State = Stopped
id := rep.ID
r.mu.Unlock()
for _, s := range shards {