CI / test (push) Successful in 45s
Everything here works without GPUs and is covered by tests. The control plane connection is not built yet, so llmgw agent itself is still a stub and the phase exit criterion is not met. - discovery: IB from /sys/class/infiniband, parsing the formats sysfs actually uses and telling native InfiniBand from RoCE; GPUs via nvidia-smi, the cgo-free path, with health flags for uncorrected ECC and memory already held by another process. A node without IB is not an error, it is a single-node island. - engines: vLLM and SGLang command and environment construction, kept free of process handling so it stays testable. The NCCL environment is the part the plan marks critical: NCCL_IB_HCA lists only the chosen fabric's adapters, because leaving it open lets NCCL pick an adapter on a fabric where the peer is unreachable, and the job then hangs instead of failing. - validation before launch: no mixed GPU types in a tensor-parallel group, the node/GPU/TP arithmetic has to add up, and fp8 is refused below compute capability 8.9 rather than silently falling back to a slower path. - supervisor: process groups, because engines fork workers that hold GPU memory and outlive a signal sent only to the leader; a bounded log tail, since engine output is the only diagnosis when loading fails; Starting separate from Loading, because a 70B model legitimately takes minutes; and orphan adoption through pid files fingerprinted with the command line, since pids get reused and adopting an unrelated process is worse than leaking the original. - readiness is two questions: /health says the process is up, /v1/models says the weights loaded. Routing on /health alone sends traffic into a loading model. - llmgw agent inventory reports what the scheduler would see on a node. One plan correction: GPU class sizes come from the driver in GiB, which does not always match the marketing name. An L40S is sold as 48GB and reports 44.99 GiB, so its class is l40s-45g, not l40s-48g. Consistency matters more than matching the box, because that number decides whether a model fits. Plan §2 and §5.1 updated. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
410 lines
11 KiB
Go
410 lines
11 KiB
Go
package supervisor
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
|
)
|
|
|
|
// scriptedHealth reports ready once the flag is set.
|
|
type scriptedHealth struct{ ready atomic.Bool }
|
|
|
|
func (h *scriptedHealth) Ready(context.Context, string, string) error {
|
|
if h.ready.Load() {
|
|
return nil
|
|
}
|
|
return errors.New("not ready")
|
|
}
|
|
|
|
func newSupervisor(t *testing.T) (*Supervisor, *scriptedHealth) {
|
|
t.Helper()
|
|
h := &scriptedHealth{}
|
|
s, err := New(Config{
|
|
WorkDir: t.TempDir(),
|
|
LogLines: 64,
|
|
StopTimeout: 3 * time.Second,
|
|
HealthInterval: 20 * time.Millisecond,
|
|
}, h, slog.New(slog.DiscardHandler))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { s.StopAll(context.Background()) })
|
|
return s, h
|
|
}
|
|
|
|
// shellCommand builds a Command running a shell script.
|
|
func shellCommand(script string) engines.Command {
|
|
return engines.Command{Path: "/bin/sh", Args: []string{"-c", script}, Env: os.Environ()}
|
|
}
|
|
|
|
func waitState(t *testing.T, sh *Shard, want State, limit time.Duration) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(limit)
|
|
for time.Now().Before(deadline) {
|
|
if sh.Status().State == want {
|
|
return
|
|
}
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
t.Fatalf("shard stayed in %s, expected %s", sh.Status().State, want)
|
|
}
|
|
|
|
func TestStartAndStop(t *testing.T) {
|
|
s, _ := newSupervisor(t)
|
|
|
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if st := sh.Status(); st.State != Starting || st.PID == 0 {
|
|
t.Fatalf("status after start: %+v", st)
|
|
}
|
|
|
|
if err := s.Stop(t.Context(), "a"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if st := sh.Status().State; st != Stopped {
|
|
t.Errorf("state after stop: %s", st)
|
|
}
|
|
}
|
|
|
|
func TestUnexpectedExitIsFailure(t *testing.T) {
|
|
// An engine exiting on its own is a failure even with status 0: it is meant
|
|
// to serve until told otherwise.
|
|
s, _ := newSupervisor(t)
|
|
|
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 0"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitState(t, sh, Failed, 3*time.Second)
|
|
if !strings.Contains(sh.Status().Err, "unexpectedly") {
|
|
t.Errorf("err=%q", sh.Status().Err)
|
|
}
|
|
}
|
|
|
|
func TestNonZeroExitRecordsCode(t *testing.T) {
|
|
s, _ := newSupervisor(t)
|
|
|
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 42"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitState(t, sh, Failed, 3*time.Second)
|
|
if got := sh.Status().ExitCode; got != 42 {
|
|
t.Errorf("exit code=%d, expected 42", got)
|
|
}
|
|
}
|
|
|
|
func TestStopKillsChildProcesses(t *testing.T) {
|
|
// The reason for the process group: engines fork workers that hold GPU
|
|
// memory, and signalling only the leader leaves them behind.
|
|
s, _ := newSupervisor(t)
|
|
dir := t.TempDir()
|
|
marker := filepath.Join(dir, "child.pid")
|
|
|
|
// The shell forks a child that writes its own pid, then both sleep.
|
|
script := fmt.Sprintf(`sh -c 'echo $$ > %s; sleep 30' & sleep 30`, marker)
|
|
if _, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand(script)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Wait for the child to announce itself.
|
|
var childPID int
|
|
deadline := time.Now().Add(3 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if b, err := os.ReadFile(marker); err == nil {
|
|
if _, err := fmt.Sscanf(strings.TrimSpace(string(b)), "%d", &childPID); err == nil && childPID > 0 {
|
|
break
|
|
}
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
if childPID == 0 {
|
|
t.Fatal("child never started")
|
|
}
|
|
|
|
if err := s.Stop(t.Context(), "a"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
deadline = time.Now().Add(3 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if !processAlive(childPID) {
|
|
return
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
t.Errorf("child process %d survived the stop; it would still hold GPU memory", childPID)
|
|
}
|
|
|
|
func TestStopKillsAfterTimeout(t *testing.T) {
|
|
// A process ignoring SIGTERM must still die.
|
|
s, _ := newSupervisor(t)
|
|
s.cfg.StopTimeout = 300 * time.Millisecond
|
|
|
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{},
|
|
shellCommand("trap '' TERM; sleep 30"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
pid := sh.Status().PID
|
|
|
|
start := time.Now()
|
|
if err := s.Stop(t.Context(), "a"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if d := time.Since(start); d > 5*time.Second {
|
|
t.Errorf("stop took %v", d)
|
|
}
|
|
if processAlive(pid) {
|
|
t.Error("the process survived SIGKILL")
|
|
}
|
|
}
|
|
|
|
func TestStartIsIdempotent(t *testing.T) {
|
|
// The control plane retries commands, so a repeated start must not produce
|
|
// a second process (plan §5.6).
|
|
s, _ := newSupervisor(t)
|
|
|
|
first, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
second, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
|
|
if !errors.Is(err, ErrAlreadyRunning) {
|
|
t.Fatalf("expected ErrAlreadyRunning, got %v", err)
|
|
}
|
|
if first.Status().PID != second.Status().PID {
|
|
t.Error("a second process was started for the same shard")
|
|
}
|
|
}
|
|
|
|
func TestRestartAfterTerminalState(t *testing.T) {
|
|
s, _ := newSupervisor(t)
|
|
|
|
sh, _ := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 1"))
|
|
waitState(t, sh, Failed, 3*time.Second)
|
|
|
|
// Once it has failed, the same id may be started again.
|
|
again, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
|
|
if err != nil {
|
|
t.Fatalf("restarting a failed shard: %v", err)
|
|
}
|
|
if again.Status().State != Starting {
|
|
t.Errorf("state=%s", again.Status().State)
|
|
}
|
|
}
|
|
|
|
func TestWaitReady(t *testing.T) {
|
|
s, h := newSupervisor(t)
|
|
|
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
go func() {
|
|
time.Sleep(100 * time.Millisecond)
|
|
h.ready.Store(true)
|
|
}()
|
|
|
|
if err := s.WaitReady(t.Context(), sh, "http://x", "m", 3*time.Second); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
st := sh.Status()
|
|
if st.State != Ready {
|
|
t.Errorf("state=%s", st.State)
|
|
}
|
|
if st.ReadyAt.IsZero() {
|
|
t.Error("ReadyAt was not recorded")
|
|
}
|
|
}
|
|
|
|
func TestWaitReadyTimeoutIncludesLogs(t *testing.T) {
|
|
// When a model fails to load, the engine output is the only diagnosis
|
|
// available, so the error has to carry it.
|
|
s, _ := newSupervisor(t)
|
|
|
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{},
|
|
shellCommand("echo 'CUDA out of memory'; sleep 30"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
time.Sleep(100 * time.Millisecond) // let the line be captured
|
|
|
|
err = s.WaitReady(t.Context(), sh, "http://x", "m", 200*time.Millisecond)
|
|
if err == nil {
|
|
t.Fatal("expected a timeout")
|
|
}
|
|
if !strings.Contains(err.Error(), "CUDA out of memory") {
|
|
t.Errorf("the error does not carry engine output: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestWaitReadyDetectsEarlyExit(t *testing.T) {
|
|
s, _ := newSupervisor(t)
|
|
|
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 1"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Should return as soon as the process dies, not after the full timeout.
|
|
start := time.Now()
|
|
if err := s.WaitReady(t.Context(), sh, "http://x", "m", 10*time.Second); err == nil {
|
|
t.Fatal("expected an error")
|
|
}
|
|
if d := time.Since(start); d > 3*time.Second {
|
|
t.Errorf("waited %v for a process that had already exited", d)
|
|
}
|
|
}
|
|
|
|
func TestDrain(t *testing.T) {
|
|
s, _ := newSupervisor(t)
|
|
|
|
sh, _ := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
|
|
if err := s.Drain("a"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := sh.Status().State; got != Draining {
|
|
t.Errorf("state=%s", got)
|
|
}
|
|
// A drained shard's process is still alive; only routing stops.
|
|
if !processAlive(sh.Status().PID) {
|
|
t.Error("draining killed the process; it should keep serving in-flight work")
|
|
}
|
|
}
|
|
|
|
func TestLogsAreCaptured(t *testing.T) {
|
|
s, _ := newSupervisor(t)
|
|
|
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{},
|
|
shellCommand("echo line1; echo line2 >&2; echo line3; sleep 0.2"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
time.Sleep(400 * time.Millisecond)
|
|
|
|
logs := strings.Join(sh.Logs(10), "\n")
|
|
for _, want := range []string{"line1", "line2", "line3"} {
|
|
if !strings.Contains(logs, want) {
|
|
t.Errorf("%q missing from logs:\n%s", want, logs)
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- orphan adoption ---
|
|
|
|
func TestFindOrphans(t *testing.T) {
|
|
s, _ := newSupervisor(t)
|
|
|
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
pid := sh.Status().PID
|
|
|
|
// Simulate an agent restart: a new supervisor over the same work dir.
|
|
s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second},
|
|
&scriptedHealth{}, slog.New(slog.DiscardHandler))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
orphans, err := s2.FindOrphans()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(orphans) != 1 {
|
|
t.Fatalf("expected 1 orphan, got %d", len(orphans))
|
|
}
|
|
o := orphans[0]
|
|
if o.ShardID != "a" || o.PID != pid || !o.Alive {
|
|
t.Errorf("orphan=%+v (want shard a, pid %d, alive)", o, pid)
|
|
}
|
|
// The command line is stored as a fingerprint, because pids get reused and
|
|
// adopting an unrelated process is worse than leaking the original.
|
|
if len(o.Cmdline) == 0 || !strings.Contains(strings.Join(o.Cmdline, " "), "sleep 30") {
|
|
t.Errorf("cmdline fingerprint missing: %v", o.Cmdline)
|
|
}
|
|
|
|
if err := s2.KillOrphan(o); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
deadline := time.Now().Add(3 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if !processAlive(pid) {
|
|
break
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
if processAlive(pid) {
|
|
t.Error("the orphan survived KillOrphan")
|
|
}
|
|
if _, err := os.Stat(s2.pidPath("a")); !os.IsNotExist(err) {
|
|
t.Error("the pid file was left behind")
|
|
}
|
|
}
|
|
|
|
func TestPIDFileRemovedOnExit(t *testing.T) {
|
|
s, _ := newSupervisor(t)
|
|
|
|
sh, _ := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 0"))
|
|
waitState(t, sh, Failed, 3*time.Second)
|
|
time.Sleep(50 * time.Millisecond)
|
|
|
|
if _, err := os.Stat(s.pidPath("a")); !os.IsNotExist(err) {
|
|
t.Error("the pid file outlived the process; a later scan would find a dead orphan")
|
|
}
|
|
}
|
|
|
|
// --- state machine and ring buffer ---
|
|
|
|
func TestStateStrings(t *testing.T) {
|
|
for st, want := range map[State]string{
|
|
Pending: "pending", Starting: "starting", Loading: "loading",
|
|
Ready: "ready", Draining: "draining", Stopped: "stopped", Failed: "failed",
|
|
} {
|
|
if got := st.String(); got != want {
|
|
t.Errorf("%d -> %q, want %q", int(st), got, want)
|
|
}
|
|
}
|
|
if !Stopped.Terminal() || !Failed.Terminal() || Ready.Terminal() {
|
|
t.Error("Terminal is wrong")
|
|
}
|
|
if !Ready.Live() || !Draining.Live() || Stopped.Live() {
|
|
t.Error("Live is wrong")
|
|
}
|
|
}
|
|
|
|
func TestRingBuffer(t *testing.T) {
|
|
r := newRingBuffer(3)
|
|
if got := r.tail(10); len(got) != 0 {
|
|
t.Errorf("empty buffer returned %v", got)
|
|
}
|
|
|
|
r.add("a")
|
|
r.add("b")
|
|
if got := strings.Join(r.tail(10), ","); got != "a,b" {
|
|
t.Errorf("tail=%q", got)
|
|
}
|
|
|
|
r.add("c")
|
|
r.add("d") // evicts "a"
|
|
if got := strings.Join(r.tail(10), ","); got != "b,c,d" {
|
|
t.Errorf("after wrap tail=%q, want b,c,d", got)
|
|
}
|
|
if got := strings.Join(r.tail(2), ","); got != "c,d" {
|
|
t.Errorf("tail(2)=%q", got)
|
|
}
|
|
}
|