Survive a control plane restart without orphaning live engines

Not persisting replica state left one hole: restarting the control plane made it
forget every replica while the engines kept running. They held GPUs the
inventory believed were free, so the next placement failed to allocate, with an
error naming the new process rather than the old one.

The fix belongs on the agent, not in a database. A row can say "running" about a
process that died; the fleet cannot.

- the supervisor writes the full shard spec, not just a pid and command line. A
  pid identifies a process; the spec is what lets it be described back to the
  control plane and matched to a replica.
- Adopt takes over an engine a previous agent left running. It is not this
  process's child, so it cannot be waited on: it is watched by polling and
  stopped by signalling its group directly. Its earlier output is gone and the
  log says so rather than presenting emptiness as silence.
- every Register reports what the agent is running, not only the first, because
  a reconnect may be a control plane restart and has to recover the same way.
- the reconciler rebuilds replicas from those reports and reclaims their GPUs.
  Anything whose model or profile left the catalog is stopped; nothing will
  route to it and it is holding cards.
- an engine the agent cannot identify is killed. A process holding GPUs that
  nothing can describe is worse than no process, because placement hands those
  GPUs out again.

Stop was also wrong for adopted shards: it checked for an exec.Cmd, which they
do not have, and reported them stopped without killing anything.

TestAgentReconnects had been passing for no reason. httptest's
CloseClientConnections does nothing to an h2c stream — prior-knowledge h2c
hijacks the connection and httptest stops tracking hijacked ones — so the agent
never reconnected, and the assertion on the connection map passed on a stale
entry. Tests now sever connections through their own listener and count opened
streams.

Plan §11 also records what multi-node still needs, since placement landing early
makes it look further along than it is: workers never become Ready because the
agent health-checks every shard while only rank 0 serves HTTP; RayCommand is
written and tested but never called; and startup is fire-and-forget.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
ilkerandClaude Opus 5 committed 2026-09-16 06:15:04 +02:00
1 parent 40427b2705
commit 75a8e76706
13 files changed
+1312 -145

No files matched your search

+154 -46
View File
@@ -3,6 +3,7 @@ package supervisor
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"io"
@@ -10,7 +11,6 @@ import (
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
@@ -82,6 +82,9 @@ type Shard struct {
cmd *exec.Cmd
cancel context.CancelFunc
done chan struct{}
// adopted marks a shard inherited from a previous agent: it has no
// exec.Cmd, so it is watched by polling and killed by signal alone.
adopted bool
}
// Status is a snapshot of a shard, safe to send to the control plane.
@@ -217,7 +220,7 @@ func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmd engines.Command)
sh.state = Starting
sh.mu.Unlock()
if err := s.writePIDFile(sh); err != nil {
if err := s.writeState(sh); err != nil {
s.log.Warn("could not write pid file; adoption after a restart will not work",
"shard", sh.ID, "err", err)
}
@@ -268,7 +271,7 @@ func (s *Supervisor) wait(sh *Shard) {
final, code := sh.state, sh.exitCode
sh.mu.Unlock()
s.removePIDFile(sh)
s.removeState(sh)
if final == Failed {
s.log.Error("shard failed", "shard", sh.ID, "exit_code", code, "err", err,
"last_output", strings.Join(sh.Logs(5), " | "))
@@ -330,13 +333,15 @@ func (s *Supervisor) Stop(ctx context.Context, id string) error {
}
sh.mu.Lock()
c, pid := sh.cmd, sh.pid
pid := sh.pid
if sh.state.Live() {
sh.state = Draining // marks the exit as expected for wait()
sh.state = Draining // marks the exit as expected for wait() and poll()
}
sh.mu.Unlock()
if c == nil || pid == 0 {
// A pid of zero means the process never started; an adopted shard has a pid
// but no exec.Cmd, and must still be killed by signal.
if pid == 0 {
sh.setState(Stopped, nil)
return nil
}
@@ -347,18 +352,26 @@ func (s *Supervisor) Stop(ctx context.Context, id string) error {
s.log.Warn("SIGTERM failed", "shard", id, "err", err)
}
// force kills the group outright. exec.Cmd's cancel does this for shards we
// started; an adopted shard has none, so signal its group directly.
force := func() {
if sh.cancel != nil {
sh.cancel()
return
}
if err := killGroup(pid, syscall.SIGKILL); err != nil && !errors.Is(err, os.ErrProcessDone) {
s.log.Warn("SIGKILL failed", "shard", id, "err", err)
}
}
select {
case <-sh.done:
return nil
case <-time.After(s.cfg.StopTimeout):
s.log.Warn("shard did not exit in time, killing", "shard", id, "timeout", s.cfg.StopTimeout)
if sh.cancel != nil {
sh.cancel()
}
force()
case <-ctx.Done():
if sh.cancel != nil {
sh.cancel()
}
force()
return ctx.Err()
}
@@ -424,39 +437,68 @@ func (s *Supervisor) Statuses() []Status {
return out
}
// --- pid files, for adoption after an agent restart ---
// --- state files, for adoption after an agent restart ---
func (s *Supervisor) pidPath(id string) string {
return filepath.Join(s.cfg.WorkDir, "shard-"+id+".pid")
func (s *Supervisor) statePath(id string) string {
return filepath.Join(s.cfg.WorkDir, "shard-"+id+".json")
}
// writePIDFile records enough to recognise the process later.
// shardState is what survives an agent restart.
//
// The pid alone is not enough: pids are reused, and adopting an unrelated
// process would be worse than leaking the original. The command line is stored
// alongside it as a fingerprint.
func (s *Supervisor) writePIDFile(sh *Shard) error {
content := fmt.Sprintf("%d\n%s\n", sh.pid, strings.Join(sh.cmd.Args, "\x00"))
return os.WriteFile(s.pidPath(sh.ID), []byte(content), 0o600)
// The pid alone is not enough for two separate reasons. Pids are reused, so the
// command line is kept as a fingerprint — adopting an unrelated process would
// be worse than leaking the original. And the full spec is kept because a
// recovered shard has to be described back to the control plane: without it the
// control plane cannot tell which replica the process belongs to, and would
// place a duplicate while the original keeps holding the GPUs (plan §6.5).
type shardState struct {
ShardID string `json:"shard_id"`
ReplicaID string `json:"replica_id"`
PID int `json:"pid"`
Cmdline []string `json:"cmdline"`
StartedAt time.Time `json:"started_at"`
Spec engines.ShardSpec `json:"spec"`
}
func (s *Supervisor) removePIDFile(sh *Shard) {
_ = os.Remove(s.pidPath(sh.ID))
// writeState records everything needed to recognise and describe the process later.
func (s *Supervisor) writeState(sh *Shard) error {
b, err := json.Marshal(shardState{
ShardID: sh.ID,
ReplicaID: sh.Spec.ReplicaID,
PID: sh.pid,
Cmdline: sh.cmd.Args,
StartedAt: sh.started,
Spec: sh.Spec,
})
if err != nil {
return err
}
// Written atomically: a half-written state file after a crash would be
// indistinguishable from a corrupt one, and the shard would be killed.
tmp := s.statePath(sh.ID) + ".tmp"
if err := os.WriteFile(tmp, b, 0o600); err != nil {
return err
}
return os.Rename(tmp, s.statePath(sh.ID))
}
func (s *Supervisor) removeState(sh *Shard) {
_ = os.Remove(s.statePath(sh.ID))
}
// Orphan is an engine process left behind by a previous agent.
type Orphan struct {
ShardID string
PID int
Cmdline []string
// Alive reports whether the process still exists and matches its fingerprint.
ShardID string
ReplicaID string
PID int
Cmdline []string
Spec engines.ShardSpec
StartedAt time.Time
// Alive reports whether the process still exists.
Alive bool
}
// FindOrphans scans the work directory for processes a previous agent started.
//
// The control plane decides what happens next (plan §5.4): a shard of a
// multi-node replica cannot be adopted in isolation, because its peers are gone.
func (s *Supervisor) FindOrphans() ([]Orphan, error) {
entries, err := os.ReadDir(s.cfg.WorkDir)
if err != nil {
@@ -466,35 +508,101 @@ func (s *Supervisor) FindOrphans() ([]Orphan, error) {
var out []Orphan
for _, e := range entries {
name := e.Name()
if !strings.HasPrefix(name, "shard-") || !strings.HasSuffix(name, ".pid") {
if !strings.HasPrefix(name, "shard-") || !strings.HasSuffix(name, ".json") {
continue
}
id := strings.TrimSuffix(strings.TrimPrefix(name, "shard-"), ".pid")
b, err := os.ReadFile(filepath.Join(s.cfg.WorkDir, name))
if err != nil {
continue
}
lines := strings.SplitN(strings.TrimSpace(string(b)), "\n", 2)
pid, err := strconv.Atoi(strings.TrimSpace(lines[0]))
if err != nil {
var st shardState
if err := json.Unmarshal(b, &st); err != nil || st.ShardID == "" || st.PID <= 0 {
// A corrupt state file describes a process that cannot be
// identified, so it cannot be adopted safely. Leave it for
// KillOrphan to clean up by shard id from the filename.
id := strings.TrimSuffix(strings.TrimPrefix(name, "shard-"), ".json")
out = append(out, Orphan{ShardID: id})
continue
}
var cmdline []string
if len(lines) > 1 {
cmdline = strings.Split(lines[1], "\x00")
}
out = append(out, Orphan{
ShardID: id,
PID: pid,
Cmdline: cmdline,
Alive: processAlive(pid),
ShardID: st.ShardID,
ReplicaID: st.ReplicaID,
PID: st.PID,
Cmdline: st.Cmdline,
Spec: st.Spec,
StartedAt: st.StartedAt,
Alive: processAlive(st.PID),
})
}
return out, nil
}
// Adopt takes over a still-running engine from a previous agent.
//
// The process cannot be waited on — it is not this process's child — so it is
// watched by polling instead. That is the whole difference between an adopted
// shard and a started one; everything downstream treats them the same.
func (s *Supervisor) Adopt(o Orphan) (*Shard, error) {
if !o.Alive {
return nil, fmt.Errorf("shard %s: process %d is gone", o.ShardID, o.PID)
}
s.mu.Lock()
if _, exists := s.shards[o.ShardID]; exists {
s.mu.Unlock()
return nil, ErrAlreadyRunning
}
sh := &Shard{
ID: o.ShardID,
Spec: o.Spec,
state: Starting,
pid: o.PID,
logs: newRingBuffer(s.cfg.LogLines),
started: o.StartedAt,
done: make(chan struct{}),
adopted: true,
}
s.shards[o.ShardID] = sh
s.mu.Unlock()
// Output is gone with the previous agent; say so rather than presenting an
// empty log as if the engine had been silent.
sh.logs.add("[adopted from a previous agent; earlier output is not available]")
go s.poll(sh)
return sh, nil
}
// poll watches an adopted process, which cannot be reaped with Wait.
func (s *Supervisor) poll(sh *Shard) {
defer close(sh.done)
// Polled rather than waited on, so the interval bounds how quickly a stop
// is observed. Short enough that Stop does not sit out its whole timeout.
t := time.NewTicker(200 * time.Millisecond)
defer t.Stop()
for range t.C {
if processAlive(sh.pid) {
continue
}
sh.mu.Lock()
expected := sh.state == Draining || sh.state == Stopped
if expected {
sh.state = Stopped
} else {
sh.state = Failed
sh.err = errors.New("adopted engine exited")
}
sh.mu.Unlock()
s.removeState(sh)
s.log.Info("adopted shard exited", "shard", sh.ID, "expected", expected)
return
}
}
// KillOrphan terminates a leftover process group and removes its pid file.
func (s *Supervisor) KillOrphan(o Orphan) error {
if o.Alive {
@@ -513,9 +621,9 @@ func (s *Supervisor) KillOrphan(o Orphan) error {
_ = killGroup(o.PID, syscall.SIGKILL)
}
}
// An already-missing pid file is success: the previous agent may have
// An already-missing state file is success: the previous agent may have
// reaped the process itself between the scan and this call.
if err := os.Remove(s.pidPath(o.ShardID)); err != nil && !errors.Is(err, os.ErrNotExist) {
if err := os.Remove(s.statePath(o.ShardID)); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
return nil
+131 -6
View File
@@ -9,6 +9,7 @@ import (
"path/filepath"
"strings"
"sync/atomic"
"syscall"
"testing"
"time"
@@ -307,7 +308,8 @@ func TestLogsAreCaptured(t *testing.T) {
func TestFindOrphans(t *testing.T) {
s, _ := newSupervisor(t)
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a", HTTPPort: 20100}
sh, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30"))
if err != nil {
t.Fatal(err)
}
@@ -336,6 +338,14 @@ func TestFindOrphans(t *testing.T) {
if len(o.Cmdline) == 0 || !strings.Contains(strings.Join(o.Cmdline, " "), "sleep 30") {
t.Errorf("cmdline fingerprint missing: %v", o.Cmdline)
}
// And the spec, without which the control plane could not tell which
// replica the process belongs to and would place a duplicate.
if o.ReplicaID != "rep-1" {
t.Errorf("replica id not recovered: %q", o.ReplicaID)
}
if o.Spec.HTTPPort != 20100 {
t.Errorf("spec not recovered: %+v", o.Spec)
}
if err := s2.KillOrphan(o); err != nil {
t.Fatal(err)
@@ -350,20 +360,20 @@ func TestFindOrphans(t *testing.T) {
if processAlive(pid) {
t.Error("the orphan survived KillOrphan")
}
if _, err := os.Stat(s2.pidPath("a")); !os.IsNotExist(err) {
t.Error("the pid file was left behind")
if _, err := os.Stat(s2.statePath("a")); !os.IsNotExist(err) {
t.Error("the state file was left behind")
}
}
func TestPIDFileRemovedOnExit(t *testing.T) {
func TestStateFileRemovedOnExit(t *testing.T) {
s, _ := newSupervisor(t)
sh, _ := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 0"))
waitState(t, sh, Failed, 3*time.Second)
time.Sleep(50 * time.Millisecond)
if _, err := os.Stat(s.pidPath("a")); !os.IsNotExist(err) {
t.Error("the pid file outlived the process; a later scan would find a dead orphan")
if _, err := os.Stat(s.statePath("a")); !os.IsNotExist(err) {
t.Error("the state file outlived the process; a later scan would find a dead orphan")
}
}
@@ -407,3 +417,118 @@ func TestRingBuffer(t *testing.T) {
t.Errorf("tail(2)=%q", got)
}
}
func TestAdoptTakesOverRunningShard(t *testing.T) {
// The control plane restart fix depends on this: a shard that survived the
// agent must come back under supervision, not be restarted or leaked.
s, _ := newSupervisor(t)
spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a", HTTPPort: 20200}
sh, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30"))
if err != nil {
t.Fatal(err)
}
pid := sh.Status().PID
// A new supervisor over the same work dir, as after an agent restart.
s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 3 * time.Second},
&scriptedHealth{}, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatal(err)
}
orphans, err := s2.FindOrphans()
if err != nil {
t.Fatal(err)
}
if len(orphans) != 1 || !orphans[0].Alive {
t.Fatalf("orphan not found: %+v", orphans)
}
adopted, err := s2.Adopt(orphans[0])
if err != nil {
t.Fatal(err)
}
if got := adopted.Status().PID; got != pid {
t.Errorf("adopted pid %d, expected %d", got, pid)
}
if adopted.Spec.ReplicaID != "rep-1" {
t.Errorf("spec lost in adoption: %+v", adopted.Spec)
}
// The earlier output is gone; saying so beats presenting an empty log as if
// the engine had been silent.
if logs := strings.Join(adopted.Logs(5), " "); !strings.Contains(logs, "adopted") {
t.Errorf("adoption was not noted in the log: %q", logs)
}
// Stopping an adopted shard must actually kill it: it has no exec.Cmd, so
// the signal path is the only one available.
if err := s2.Stop(t.Context(), "a"); err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
if !processAlive(pid) {
return
}
time.Sleep(20 * time.Millisecond)
}
t.Error("the adopted process survived Stop")
}
func TestAdoptRefusesDeadProcess(t *testing.T) {
s, _ := newSupervisor(t)
if _, err := s.Adopt(Orphan{ShardID: "gone", PID: 999999, Alive: false}); err == nil {
t.Error("a dead process was adopted")
}
}
func TestAdoptedExitIsReported(t *testing.T) {
// An adopted engine dying has to be noticed; nothing waits on it.
s, _ := newSupervisor(t)
spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a"}
sh, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30"))
if err != nil {
t.Fatal(err)
}
pid := sh.Status().PID
s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second},
&scriptedHealth{}, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatal(err)
}
orphans, _ := s2.FindOrphans()
adopted, err := s2.Adopt(orphans[0])
if err != nil {
t.Fatal(err)
}
// Kill it behind the supervisor's back.
_ = killGroup(pid, syscall.SIGKILL)
waitState(t, adopted, Failed, 5*time.Second)
}
func TestCorruptStateFileIsCleanedUp(t *testing.T) {
// A half-written state file describes a process that cannot be identified,
// so it cannot be adopted safely.
s, _ := newSupervisor(t)
if err := os.WriteFile(s.statePath("broken"), []byte("{not json"), 0o600); err != nil {
t.Fatal(err)
}
orphans, err := s.FindOrphans()
if err != nil {
t.Fatal(err)
}
if len(orphans) != 1 || orphans[0].ShardID != "broken" || orphans[0].Alive {
t.Fatalf("corrupt state file not reported for cleanup: %+v", orphans)
}
if err := s.KillOrphan(orphans[0]); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(s.statePath("broken")); !os.IsNotExist(err) {
t.Error("the corrupt state file was left behind")
}
}