Files
llmgw/internal/agent/supervisor/supervisor_test.go
T
ilkerandClaude Opus 5 75a8e76706
CI / test (push) Successful in 1m44s
Survive a control plane restart without orphaning live engines
Not persisting replica state left one hole: restarting the control plane made it
forget every replica while the engines kept running. They held GPUs the
inventory believed were free, so the next placement failed to allocate, with an
error naming the new process rather than the old one.

The fix belongs on the agent, not in a database. A row can say "running" about a
process that died; the fleet cannot.

- the supervisor writes the full shard spec, not just a pid and command line. A
  pid identifies a process; the spec is what lets it be described back to the
  control plane and matched to a replica.
- Adopt takes over an engine a previous agent left running. It is not this
  process's child, so it cannot be waited on: it is watched by polling and
  stopped by signalling its group directly. Its earlier output is gone and the
  log says so rather than presenting emptiness as silence.
- every Register reports what the agent is running, not only the first, because
  a reconnect may be a control plane restart and has to recover the same way.
- the reconciler rebuilds replicas from those reports and reclaims their GPUs.
  Anything whose model or profile left the catalog is stopped; nothing will
  route to it and it is holding cards.
- an engine the agent cannot identify is killed. A process holding GPUs that
  nothing can describe is worse than no process, because placement hands those
  GPUs out again.

Stop was also wrong for adopted shards: it checked for an exec.Cmd, which they
do not have, and reported them stopped without killing anything.

TestAgentReconnects had been passing for no reason. httptest's
CloseClientConnections does nothing to an h2c stream — prior-knowledge h2c
hijacks the connection and httptest stops tracking hijacked ones — so the agent
never reconnected, and the assertion on the connection map passed on a stale
entry. Tests now sever connections through their own listener and count opened
streams.

Plan §11 also records what multi-node still needs, since placement landing early
makes it look further along than it is: workers never become Ready because the
agent health-checks every shard while only rank 0 serves HTTP; RayCommand is
written and tested but never called; and startup is fire-and-forget.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 06:15:04 +02:00

535 lines
15 KiB
Go

package supervisor
import (
"context"
"errors"
"fmt"
"log/slog"
"os"
"path/filepath"
"strings"
"sync/atomic"
"syscall"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
)
// scriptedHealth reports ready once the flag is set.
type scriptedHealth struct{ ready atomic.Bool }
func (h *scriptedHealth) Ready(context.Context, string, string) error {
if h.ready.Load() {
return nil
}
return errors.New("not ready")
}
func newSupervisor(t *testing.T) (*Supervisor, *scriptedHealth) {
t.Helper()
h := &scriptedHealth{}
s, err := New(Config{
WorkDir: t.TempDir(),
LogLines: 64,
StopTimeout: 3 * time.Second,
HealthInterval: 20 * time.Millisecond,
}, h, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { s.StopAll(context.Background()) })
return s, h
}
// shellCommand builds a Command running a shell script.
func shellCommand(script string) engines.Command {
return engines.Command{Path: "/bin/sh", Args: []string{"-c", script}, Env: os.Environ()}
}
func waitState(t *testing.T, sh *Shard, want State, limit time.Duration) {
t.Helper()
deadline := time.Now().Add(limit)
for time.Now().Before(deadline) {
if sh.Status().State == want {
return
}
time.Sleep(5 * time.Millisecond)
}
t.Fatalf("shard stayed in %s, expected %s", sh.Status().State, want)
}
func TestStartAndStop(t *testing.T) {
s, _ := newSupervisor(t)
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
if err != nil {
t.Fatal(err)
}
if st := sh.Status(); st.State != Starting || st.PID == 0 {
t.Fatalf("status after start: %+v", st)
}
if err := s.Stop(t.Context(), "a"); err != nil {
t.Fatal(err)
}
if st := sh.Status().State; st != Stopped {
t.Errorf("state after stop: %s", st)
}
}
func TestUnexpectedExitIsFailure(t *testing.T) {
// An engine exiting on its own is a failure even with status 0: it is meant
// to serve until told otherwise.
s, _ := newSupervisor(t)
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 0"))
if err != nil {
t.Fatal(err)
}
waitState(t, sh, Failed, 3*time.Second)
if !strings.Contains(sh.Status().Err, "unexpectedly") {
t.Errorf("err=%q", sh.Status().Err)
}
}
func TestNonZeroExitRecordsCode(t *testing.T) {
s, _ := newSupervisor(t)
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 42"))
if err != nil {
t.Fatal(err)
}
waitState(t, sh, Failed, 3*time.Second)
if got := sh.Status().ExitCode; got != 42 {
t.Errorf("exit code=%d, expected 42", got)
}
}
func TestStopKillsChildProcesses(t *testing.T) {
// The reason for the process group: engines fork workers that hold GPU
// memory, and signalling only the leader leaves them behind.
s, _ := newSupervisor(t)
dir := t.TempDir()
marker := filepath.Join(dir, "child.pid")
// The shell forks a child that writes its own pid, then both sleep.
script := fmt.Sprintf(`sh -c 'echo $$ > %s; sleep 30' & sleep 30`, marker)
if _, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand(script)); err != nil {
t.Fatal(err)
}
// Wait for the child to announce itself.
var childPID int
deadline := time.Now().Add(3 * time.Second)
for time.Now().Before(deadline) {
if b, err := os.ReadFile(marker); err == nil {
if _, err := fmt.Sscanf(strings.TrimSpace(string(b)), "%d", &childPID); err == nil && childPID > 0 {
break
}
}
time.Sleep(10 * time.Millisecond)
}
if childPID == 0 {
t.Fatal("child never started")
}
if err := s.Stop(t.Context(), "a"); err != nil {
t.Fatal(err)
}
deadline = time.Now().Add(3 * time.Second)
for time.Now().Before(deadline) {
if !processAlive(childPID) {
return
}
time.Sleep(20 * time.Millisecond)
}
t.Errorf("child process %d survived the stop; it would still hold GPU memory", childPID)
}
func TestStopKillsAfterTimeout(t *testing.T) {
// A process ignoring SIGTERM must still die.
s, _ := newSupervisor(t)
s.cfg.StopTimeout = 300 * time.Millisecond
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{},
shellCommand("trap '' TERM; sleep 30"))
if err != nil {
t.Fatal(err)
}
pid := sh.Status().PID
start := time.Now()
if err := s.Stop(t.Context(), "a"); err != nil {
t.Fatal(err)
}
if d := time.Since(start); d > 5*time.Second {
t.Errorf("stop took %v", d)
}
if processAlive(pid) {
t.Error("the process survived SIGKILL")
}
}
func TestStartIsIdempotent(t *testing.T) {
// The control plane retries commands, so a repeated start must not produce
// a second process (plan §5.6).
s, _ := newSupervisor(t)
first, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
if err != nil {
t.Fatal(err)
}
second, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
if !errors.Is(err, ErrAlreadyRunning) {
t.Fatalf("expected ErrAlreadyRunning, got %v", err)
}
if first.Status().PID != second.Status().PID {
t.Error("a second process was started for the same shard")
}
}
func TestRestartAfterTerminalState(t *testing.T) {
s, _ := newSupervisor(t)
sh, _ := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 1"))
waitState(t, sh, Failed, 3*time.Second)
// Once it has failed, the same id may be started again.
again, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
if err != nil {
t.Fatalf("restarting a failed shard: %v", err)
}
if again.Status().State != Starting {
t.Errorf("state=%s", again.Status().State)
}
}
func TestWaitReady(t *testing.T) {
s, h := newSupervisor(t)
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
if err != nil {
t.Fatal(err)
}
go func() {
time.Sleep(100 * time.Millisecond)
h.ready.Store(true)
}()
if err := s.WaitReady(t.Context(), sh, "http://x", "m", 3*time.Second); err != nil {
t.Fatal(err)
}
st := sh.Status()
if st.State != Ready {
t.Errorf("state=%s", st.State)
}
if st.ReadyAt.IsZero() {
t.Error("ReadyAt was not recorded")
}
}
func TestWaitReadyTimeoutIncludesLogs(t *testing.T) {
// When a model fails to load, the engine output is the only diagnosis
// available, so the error has to carry it.
s, _ := newSupervisor(t)
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{},
shellCommand("echo 'CUDA out of memory'; sleep 30"))
if err != nil {
t.Fatal(err)
}
time.Sleep(100 * time.Millisecond) // let the line be captured
err = s.WaitReady(t.Context(), sh, "http://x", "m", 200*time.Millisecond)
if err == nil {
t.Fatal("expected a timeout")
}
if !strings.Contains(err.Error(), "CUDA out of memory") {
t.Errorf("the error does not carry engine output: %v", err)
}
}
func TestWaitReadyDetectsEarlyExit(t *testing.T) {
s, _ := newSupervisor(t)
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 1"))
if err != nil {
t.Fatal(err)
}
// Should return as soon as the process dies, not after the full timeout.
start := time.Now()
if err := s.WaitReady(t.Context(), sh, "http://x", "m", 10*time.Second); err == nil {
t.Fatal("expected an error")
}
if d := time.Since(start); d > 3*time.Second {
t.Errorf("waited %v for a process that had already exited", d)
}
}
func TestDrain(t *testing.T) {
s, _ := newSupervisor(t)
sh, _ := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
if err := s.Drain("a"); err != nil {
t.Fatal(err)
}
if got := sh.Status().State; got != Draining {
t.Errorf("state=%s", got)
}
// A drained shard's process is still alive; only routing stops.
if !processAlive(sh.Status().PID) {
t.Error("draining killed the process; it should keep serving in-flight work")
}
}
func TestLogsAreCaptured(t *testing.T) {
s, _ := newSupervisor(t)
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{},
shellCommand("echo line1; echo line2 >&2; echo line3; sleep 0.2"))
if err != nil {
t.Fatal(err)
}
time.Sleep(400 * time.Millisecond)
logs := strings.Join(sh.Logs(10), "\n")
for _, want := range []string{"line1", "line2", "line3"} {
if !strings.Contains(logs, want) {
t.Errorf("%q missing from logs:\n%s", want, logs)
}
}
}
// --- orphan adoption ---
func TestFindOrphans(t *testing.T) {
s, _ := newSupervisor(t)
spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a", HTTPPort: 20100}
sh, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30"))
if err != nil {
t.Fatal(err)
}
pid := sh.Status().PID
// Simulate an agent restart: a new supervisor over the same work dir.
s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second},
&scriptedHealth{}, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatal(err)
}
orphans, err := s2.FindOrphans()
if err != nil {
t.Fatal(err)
}
if len(orphans) != 1 {
t.Fatalf("expected 1 orphan, got %d", len(orphans))
}
o := orphans[0]
if o.ShardID != "a" || o.PID != pid || !o.Alive {
t.Errorf("orphan=%+v (want shard a, pid %d, alive)", o, pid)
}
// The command line is stored as a fingerprint, because pids get reused and
// adopting an unrelated process is worse than leaking the original.
if len(o.Cmdline) == 0 || !strings.Contains(strings.Join(o.Cmdline, " "), "sleep 30") {
t.Errorf("cmdline fingerprint missing: %v", o.Cmdline)
}
// And the spec, without which the control plane could not tell which
// replica the process belongs to and would place a duplicate.
if o.ReplicaID != "rep-1" {
t.Errorf("replica id not recovered: %q", o.ReplicaID)
}
if o.Spec.HTTPPort != 20100 {
t.Errorf("spec not recovered: %+v", o.Spec)
}
if err := s2.KillOrphan(o); err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(3 * time.Second)
for time.Now().Before(deadline) {
if !processAlive(pid) {
break
}
time.Sleep(20 * time.Millisecond)
}
if processAlive(pid) {
t.Error("the orphan survived KillOrphan")
}
if _, err := os.Stat(s2.statePath("a")); !os.IsNotExist(err) {
t.Error("the state file was left behind")
}
}
func TestStateFileRemovedOnExit(t *testing.T) {
s, _ := newSupervisor(t)
sh, _ := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 0"))
waitState(t, sh, Failed, 3*time.Second)
time.Sleep(50 * time.Millisecond)
if _, err := os.Stat(s.statePath("a")); !os.IsNotExist(err) {
t.Error("the state file outlived the process; a later scan would find a dead orphan")
}
}
// --- state machine and ring buffer ---
func TestStateStrings(t *testing.T) {
for st, want := range map[State]string{
Pending: "pending", Starting: "starting", Loading: "loading",
Ready: "ready", Draining: "draining", Stopped: "stopped", Failed: "failed",
} {
if got := st.String(); got != want {
t.Errorf("%d -> %q, want %q", int(st), got, want)
}
}
if !Stopped.Terminal() || !Failed.Terminal() || Ready.Terminal() {
t.Error("Terminal is wrong")
}
if !Ready.Live() || !Draining.Live() || Stopped.Live() {
t.Error("Live is wrong")
}
}
func TestRingBuffer(t *testing.T) {
r := newRingBuffer(3)
if got := r.tail(10); len(got) != 0 {
t.Errorf("empty buffer returned %v", got)
}
r.add("a")
r.add("b")
if got := strings.Join(r.tail(10), ","); got != "a,b" {
t.Errorf("tail=%q", got)
}
r.add("c")
r.add("d") // evicts "a"
if got := strings.Join(r.tail(10), ","); got != "b,c,d" {
t.Errorf("after wrap tail=%q, want b,c,d", got)
}
if got := strings.Join(r.tail(2), ","); got != "c,d" {
t.Errorf("tail(2)=%q", got)
}
}
func TestAdoptTakesOverRunningShard(t *testing.T) {
// The control plane restart fix depends on this: a shard that survived the
// agent must come back under supervision, not be restarted or leaked.
s, _ := newSupervisor(t)
spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a", HTTPPort: 20200}
sh, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30"))
if err != nil {
t.Fatal(err)
}
pid := sh.Status().PID
// A new supervisor over the same work dir, as after an agent restart.
s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 3 * time.Second},
&scriptedHealth{}, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatal(err)
}
orphans, err := s2.FindOrphans()
if err != nil {
t.Fatal(err)
}
if len(orphans) != 1 || !orphans[0].Alive {
t.Fatalf("orphan not found: %+v", orphans)
}
adopted, err := s2.Adopt(orphans[0])
if err != nil {
t.Fatal(err)
}
if got := adopted.Status().PID; got != pid {
t.Errorf("adopted pid %d, expected %d", got, pid)
}
if adopted.Spec.ReplicaID != "rep-1" {
t.Errorf("spec lost in adoption: %+v", adopted.Spec)
}
// The earlier output is gone; saying so beats presenting an empty log as if
// the engine had been silent.
if logs := strings.Join(adopted.Logs(5), " "); !strings.Contains(logs, "adopted") {
t.Errorf("adoption was not noted in the log: %q", logs)
}
// Stopping an adopted shard must actually kill it: it has no exec.Cmd, so
// the signal path is the only one available.
if err := s2.Stop(t.Context(), "a"); err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
if !processAlive(pid) {
return
}
time.Sleep(20 * time.Millisecond)
}
t.Error("the adopted process survived Stop")
}
func TestAdoptRefusesDeadProcess(t *testing.T) {
s, _ := newSupervisor(t)
if _, err := s.Adopt(Orphan{ShardID: "gone", PID: 999999, Alive: false}); err == nil {
t.Error("a dead process was adopted")
}
}
func TestAdoptedExitIsReported(t *testing.T) {
// An adopted engine dying has to be noticed; nothing waits on it.
s, _ := newSupervisor(t)
spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a"}
sh, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30"))
if err != nil {
t.Fatal(err)
}
pid := sh.Status().PID
s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second},
&scriptedHealth{}, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatal(err)
}
orphans, _ := s2.FindOrphans()
adopted, err := s2.Adopt(orphans[0])
if err != nil {
t.Fatal(err)
}
// Kill it behind the supervisor's back.
_ = killGroup(pid, syscall.SIGKILL)
waitState(t, adopted, Failed, 5*time.Second)
}
func TestCorruptStateFileIsCleanedUp(t *testing.T) {
// A half-written state file describes a process that cannot be identified,
// so it cannot be adopted safely.
s, _ := newSupervisor(t)
if err := os.WriteFile(s.statePath("broken"), []byte("{not json"), 0o600); err != nil {
t.Fatal(err)
}
orphans, err := s.FindOrphans()
if err != nil {
t.Fatal(err)
}
if len(orphans) != 1 || orphans[0].ShardID != "broken" || orphans[0].Alive {
t.Fatalf("corrupt state file not reported for cleanup: %+v", orphans)
}
if err := s.KillOrphan(orphans[0]); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(s.statePath("broken")); !os.IsNotExist(err) {
t.Error("the corrupt state file was left behind")
}
}