Survive a control plane restart without orphaning live engines
CI / test (push) Successful in 1m44s

Not persisting replica state left one hole: restarting the control plane made it
forget every replica while the engines kept running. They held GPUs the
inventory believed were free, so the next placement failed to allocate, with an
error naming the new process rather than the old one.

The fix belongs on the agent, not in a database. A row can say "running" about a
process that died; the fleet cannot.

- the supervisor writes the full shard spec, not just a pid and command line. A
  pid identifies a process; the spec is what lets it be described back to the
  control plane and matched to a replica.
- Adopt takes over an engine a previous agent left running. It is not this
  process's child, so it cannot be waited on: it is watched by polling and
  stopped by signalling its group directly. Its earlier output is gone and the
  log says so rather than presenting emptiness as silence.
- every Register reports what the agent is running, not only the first, because
  a reconnect may be a control plane restart and has to recover the same way.
- the reconciler rebuilds replicas from those reports and reclaims their GPUs.
  Anything whose model or profile left the catalog is stopped; nothing will
  route to it and it is holding cards.
- an engine the agent cannot identify is killed. A process holding GPUs that
  nothing can describe is worse than no process, because placement hands those
  GPUs out again.

Stop was also wrong for adopted shards: it checked for an exec.Cmd, which they
do not have, and reported them stopped without killing anything.

TestAgentReconnects had been passing for no reason. httptest's
CloseClientConnections does nothing to an h2c stream — prior-knowledge h2c
hijacks the connection and httptest stops tracking hijacked ones — so the agent
never reconnected, and the assertion on the connection map passed on a stale
entry. Tests now sever connections through their own listener and count opened
streams.

Plan §11 also records what multi-node still needs, since placement landing early
makes it look further along than it is: workers never become Ready because the
agent health-checks every shard while only rank 0 serves HTTP; RayCommand is
written and tested but never called; and startup is fire-and-forget.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
ilkerandClaude Opus 5 committed 2026-09-16 06:15:04 +02:00
1 parent 40427b2705
commit 75a8e76706
13 files changed
+1312 -145

No files matched your search

+32
View File
@@ -2,6 +2,7 @@
package agentsvc
import (
"fmt"
"time"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
@@ -87,6 +88,33 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
return r
}
// AdoptedFromRegister converts an agent's running-shard report into the
// reconciler's view.
func AdoptedFromRegister(nodeID string, shards []*llmgwv1.RunningShard) []reconciler.AdoptedShard {
out := make([]reconciler.AdoptedShard, 0, len(shards))
for _, s := range shards {
a := reconciler.AdoptedShard{
NodeID: nodeID,
ReplicaID: s.GetReplicaId(),
ShardID: s.GetShardId(),
Rank: int(s.GetRank()),
GPUIndices: intSlice(s.GetGpuIndices()),
HCAs: s.GetHcas(),
HTTPPort: int(s.GetHttpPort()),
State: StateFromWire(s.GetState()),
Model: s.GetModel(),
Profile: s.GetProfile(),
Pool: s.GetPool(),
Fabric: s.GetFabric(),
}
if a.Rank == 0 && s.GetLeaderAddr() != "" {
a.Endpoint = fmt.Sprintf("http://%s:%d", s.GetLeaderAddr(), s.GetHttpPort())
}
out = append(out, a)
}
return out
}
// StartShardFromCommand converts a reconciler command into a wire message.
func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard {
p := cmd.Spec.Profile
@@ -109,6 +137,8 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard {
ModelPath: p.ModelPath, ServedModelName: p.ServedModelName,
},
StartupTimeoutMs: cmd.StartupTimeout.Milliseconds(),
Pool: cmd.Pool,
Fabric: cmd.Fabric,
}
}
@@ -126,6 +156,8 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration
HCAs: m.GetHcas(),
MgmtInterface: m.GetMgmtInterface(),
HTTPPort: int(m.GetHttpPort()),
Pool: m.GetPool(),
Fabric: m.GetFabric(),
Profile: engines.Profile{
Name: p.GetName(), Engine: p.GetEngine(), GPUClass: p.GetGpuClass(),
TP: int(p.GetTp()), PP: int(p.GetPp()),
+31 -1
View File
@@ -19,9 +19,11 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
)
// StatusSink receives shard transitions. The reconciler implements it.
// StatusSink receives shard transitions and recovers replica state from what
// agents report they are already running. The reconciler implements it.
type StatusSink interface {
OnShardStatus(replicaID, shardID string, state supervisor.State, errMsg string)
Adopt(shards []reconciler.AdoptedShard) (unclaimed []reconciler.AdoptedShard)
}
// Service is the control plane's AgentService implementation, and at the same
@@ -87,6 +89,7 @@ func (s *Service) Connect(
s.inv.Upsert(node)
s.log.Info("agent registered",
"node", nodeID, "gpus", len(node.GPUs), "fabrics", node.FabricNames(),
"running_shards", len(reg.GetRunningShards()),
"agent_version", reg.GetAgentVersion())
c := s.attach(ctx, nodeID)
@@ -98,6 +101,11 @@ func (s *Service) Connect(
return err
}
// Recover replica state from what this node is already running, before any
// reconciliation pass can conclude the cluster is empty and place
// duplicates (plan §6.5).
s.adopt(ctx, nodeID, reg.GetRunningShards(), c)
// One goroutine writes, this one reads. Splitting them is what lets a
// command be dispatched while the agent is mid-heartbeat.
var wg sync.WaitGroup
@@ -123,6 +131,28 @@ func (s *Service) Connect(
}
}
// adopt hands the agent's running shards to the reconciler and stops whatever
// nothing in the catalog claims.
func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.RunningShard, c *conn) {
if len(shards) == 0 {
return
}
unclaimed := s.status.Adopt(AdoptedFromRegister(nodeID, shards))
for _, u := range unclaimed {
// The model or profile left the catalog while this was running. Nobody
// will ever route to it, and it is holding GPUs.
s.log.Warn("stopping an unclaimed shard",
"node", nodeID, "shard", u.ShardID, "model", u.Model, "profile", u.Profile)
select {
case c.send <- &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: u.ShardID}},
}:
case <-ctx.Done():
return
}
}
}
func (s *Service) writeLoop(ctx context.Context,
stream *connect.BidiStream[llmgwv1.AgentMessage, llmgwv1.ControlMessage], c *conn,
) {
+96
View File
@@ -24,6 +24,11 @@ type ShardCommand struct {
Spec engines.ShardSpec
// StartupTimeout comes from the profile; model loading dominates it.
StartupTimeout time.Duration
// Pool and Fabric are control plane concepts the agent cannot derive. They
// travel with the command so the agent can report them back after a
// control plane restart (plan §6.5).
Pool string
Fabric string
}
// Dispatcher sends commands to agents.
@@ -436,6 +441,8 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
Profile: espec,
},
StartupTimeout: prof.StartupTimeout,
Pool: pl.Pool,
Fabric: pl.Fabric,
}
if err := r.disp.StartShard(ctx, cmd); err != nil {
r.markShard(id, s.ID, supervisor.Failed, err.Error())
@@ -518,6 +525,95 @@ func (r *Reconciler) stopReplica(ctx context.Context, id string) {
r.mu.Unlock()
}
// Adopt rebuilds replica state from shards an agent reports it is already
// running (plan §6.5).
//
// This is what makes a control plane restart survivable. Without it the
// reconciler sees an empty cluster, places duplicates, and the engines already
// running keep holding GPUs the inventory believes are free — so the duplicates
// fail to allocate, with an error naming the new process rather than the old.
//
// Shards whose model or profile is no longer in the catalog are returned for
// the caller to stop: nothing will ever claim them.
func (r *Reconciler) Adopt(shards []AdoptedShard) (unclaimed []AdoptedShard) {
byReplica := map[string][]AdoptedShard{}
for _, s := range shards {
byReplica[s.ReplicaID] = append(byReplica[s.ReplicaID], s)
}
r.mu.Lock()
defer r.mu.Unlock()
for replicaID, group := range byReplica {
if _, known := r.replicas[replicaID]; known {
// Already tracked — an agent reconnecting rather than a control
// plane restart. Its status reports keep the state current.
continue
}
first := group[0]
m, ok := r.cat.Resolve(first.Model)
if !ok {
unclaimed = append(unclaimed, group...)
continue
}
prof, ok := m.ProfileByName(first.Profile)
if !ok {
unclaimed = append(unclaimed, group...)
continue
}
rep := &Replica{
ID: replicaID, Model: m.Name, Profile: prof.Name,
Fabric: first.Fabric, Pool: first.Pool,
Weight: prof.Weight, CreatedAt: r.now(),
Endpoint: first.Endpoint,
}
for _, s := range group {
rep.Shards = append(rep.Shards, Shard{
ID: s.ShardID, NodeID: s.NodeID, Rank: s.Rank,
GPUIndices: s.GPUIndices, HCAs: s.HCAs,
HTTPPort: s.HTTPPort, State: s.State,
})
// Reclaim the GPUs, or the next placement will hand them out twice.
r.inv.Allocate(s.NodeID, inventory.Allocation{
ShardID: s.ShardID, ReplicaID: replicaID, GPUIndices: s.GPUIndices,
})
}
// The endpoint comes from the leader; a group missing rank 0 is a
// partial replica and will be failed by deriveState below.
for _, s := range group {
if s.Rank == 0 && s.Endpoint != "" {
rep.Endpoint = s.Endpoint
}
}
rep.deriveState()
r.replicas[replicaID] = rep
r.log.Info("adopted replica from the fleet",
"replica", replicaID, "model", m.Name, "shards", len(group), "state", rep.State)
}
return unclaimed
}
// AdoptedShard is one running shard an agent reported at registration.
type AdoptedShard struct {
NodeID string
ReplicaID string
ShardID string
Rank int
GPUIndices []int
HCAs []string
HTTPPort int
State supervisor.State
Model string
Profile string
Pool string
Fabric string
// Endpoint is the leader's base URL, set on the rank 0 shard.
Endpoint string
}
// OnShardStatus records a status report from an agent.
func (r *Reconciler) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) {
r.markShard(replicaID, shardID, st, errMsg)
@@ -596,3 +596,171 @@ func TestReplicaStateStrings(t *testing.T) {
t.Error("Counts is wrong")
}
}
// --- adoption after a control plane restart (plan §6.5) ---
func adopted(replicaID, shardID, nodeID string, rank int, gpus []int) AdoptedShard {
a := AdoptedShard{
ReplicaID: replicaID, ShardID: shardID, NodeID: nodeID, Rank: rank,
GPUIndices: gpus, HTTPPort: 20000, State: supervisor.Ready,
Model: "llama", Profile: "tp4", Pool: "none/h100-80g",
}
if rank == 0 {
a.Endpoint = "http://10.0.0." + nodeID + ":20000"
}
return a
}
func TestAdoptRebuildsReplica(t *testing.T) {
// The control plane restarted. Nothing is known, but a node reports a live
// replica. Without this the reconciler would place a duplicate while the
// original keeps holding GPUs.
h := newHarness(t, testModel(1), gpuNode("1", 8))
unclaimed := h.r.Adopt([]AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
if len(unclaimed) != 0 {
t.Fatalf("a valid shard was reported unclaimed: %+v", unclaimed)
}
reps := h.r.Replicas()
if len(reps) != 1 {
t.Fatalf("expected 1 adopted replica, got %d", len(reps))
}
rep := reps[0]
if rep.ID != "llama-99" || rep.Model != "llama" || rep.Profile != "tp4" {
t.Errorf("identity wrong: %+v", rep)
}
if rep.State != Ready {
t.Errorf("state=%s; every shard reported Ready", rep.State)
}
if rep.Endpoint == "" {
t.Error("no endpoint; the router would have nowhere to send")
}
if rep.Weight <= 0 {
t.Error("weight not taken from the catalog")
}
// The GPUs must be reclaimed, or the next placement hands them out twice.
if free := len(h.inv.FreeGPUs("1")); free != 4 {
t.Errorf("node has %d free GPUs, expected 4 after adoption", free)
}
}
func TestAdoptPreventsDuplicatePlacement(t *testing.T) {
// The whole point: after adopting, a reconciliation pass must see the
// desired count already met.
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.r.Adopt([]AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
h.disp.reset()
h.r.Reconcile(t.Context())
if got := len(h.disp.startedShards()); got != 0 {
t.Errorf("a duplicate replica was placed despite adoption (%d shards started)", got)
}
if got := len(h.r.Replicas()); got != 1 {
t.Errorf("expected 1 replica, got %d", got)
}
}
func TestAdoptMultiNodeReplica(t *testing.T) {
h := newHarness(t, testModel(1, catalog.Profile{
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
shards := []AdoptedShard{
adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7}),
adopted("llama-7", "llama-7-r1", "2", 1, []int{0, 1, 2, 3, 4, 5, 6, 7}),
}
for i := range shards {
shards[i].Profile = "2node"
shards[i].Fabric = "ib-a"
shards[i].Pool = "ib-a/h100-80g"
}
if unclaimed := h.r.Adopt(shards); len(unclaimed) != 0 {
t.Fatalf("unclaimed: %+v", unclaimed)
}
reps := h.r.Replicas()
if len(reps) != 1 || len(reps[0].Shards) != 2 {
t.Fatalf("the two shards were not grouped into one replica: %+v", reps)
}
if reps[0].Fabric != "ib-a" {
t.Errorf("fabric lost: %q", reps[0].Fabric)
}
// Both nodes' GPUs must be reclaimed.
for _, id := range []string{"1", "2"} {
if free := len(h.inv.FreeGPUs(id)); free != 0 {
t.Errorf("node %s has %d free GPUs after adoption", id, free)
}
}
}
func TestAdoptPartialReplicaFails(t *testing.T) {
// A multi-node replica missing a shard is useless: the survivors wait on
// collectives that will never complete while holding GPUs.
h := newHarness(t, testModel(1, catalog.Profile{
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
s := adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7})
s.Profile = "2node"
s.State = supervisor.Failed
h.r.Adopt([]AdoptedShard{s})
reps := h.r.Replicas()
if len(reps) != 1 || reps[0].State != Failed {
t.Fatalf("a failed adopted shard did not fail its replica: %+v", reps)
}
// And the next pass releases its GPUs.
h.r.Reconcile(t.Context())
if free := len(h.inv.FreeGPUs("1")); free != 8 {
t.Errorf("node has %d free GPUs; the failed adopted replica still holds some", free)
}
}
func TestAdoptUnknownModelIsUnclaimed(t *testing.T) {
// The model left the catalog while the engine kept running. Nothing will
// ever route to it, and it is holding GPUs.
h := newHarness(t, testModel(1), gpuNode("1", 8))
s := adopted("ghost-1", "ghost-1-r0", "1", 0, []int{0})
s.Model = "a-model-that-was-removed"
unclaimed := h.r.Adopt([]AdoptedShard{s})
if len(unclaimed) != 1 || unclaimed[0].ShardID != "ghost-1-r0" {
t.Fatalf("an orphaned shard was not reported unclaimed: %+v", unclaimed)
}
if got := len(h.r.Replicas()); got != 0 {
t.Errorf("a replica was created for an unknown model: %d", got)
}
}
func TestAdoptUnknownProfileIsUnclaimed(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
s := adopted("llama-9", "llama-9-r0", "1", 0, []int{0})
s.Profile = "a-profile-that-was-removed"
if unclaimed := h.r.Adopt([]AdoptedShard{s}); len(unclaimed) != 1 {
t.Errorf("a shard with an unknown profile was claimed: %+v", unclaimed)
}
}
func TestAdoptIsIdempotent(t *testing.T) {
// An agent reconnecting re-reports the same shards; that must not double
// the allocation.
h := newHarness(t, testModel(1), gpuNode("1", 8))
s := []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}
h.r.Adopt(s)
h.r.Adopt(s)
if got := len(h.r.Replicas()); got != 1 {
t.Errorf("re-reporting created %d replicas", got)
}
if free := len(h.inv.FreeGPUs("1")); free != 4 {
t.Errorf("node has %d free GPUs; the allocation was applied twice", free)
}
}