Placement and environment generation had landed early, which made multi-node look further along than it was. A replica spanning nodes could not come up at all, and all three reasons came down to assumptions that only hold on one node. Not every rank serves HTTP. The agent health-checked every shard at its own port, but only the leader answers; workers timed out, reported Failed and took the replica with them. Engine.ServesHTTP now says which ranks serve, and a worker is judged by staying alive. That is deliberately weaker evidence: the replica's real readiness is the leader answering, which only happens once every worker has joined. A shard is not always one process. Engine.Commands returns a sequence — every command but the last is a sidecar kept alive for the shard's lifetime and torn down with it, and the last one's exit is the shard's. vLLM under Ray is why: the leader runs a Ray head and then the server against it, while a worker runs only the join and no vLLM process at all. RayCommand had been written and tested but never called, so launch_mode: ray would have run vllm serve against a cluster that did not exist. Sidecars are killed with the shard, because a Ray head left behind holds the port the next replica needs. Start order was backwards. Workers went first, on the theory that the head should wait for them. Both engines put the rendezvous point on rank 0, so a worker started first has nothing to connect to. vLLM's server blocks until the Ray cluster has the GPUs it needs, so the leader can start immediately and waits on its own. A command can now also require that rendezvous point to be listening before the next starts, bounded by a timeout so a head that never appears fails the shard instead of hanging it. Stop was also killing only the engine, leaving sidecars behind. None of this is verified against real engines — it is tested with stub processes. The Ray sequencing, and which launch mode a given vLLM version wants, can only be confirmed on hardware. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
75a8e76706
commit
1d1a6aebb0
11 files changed
+647
-125
No files matched your search
@@ -8,9 +8,11 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
@@ -30,15 +32,24 @@ type Config struct {
|
||||
StopTimeout time.Duration `koanf:"stop_timeout"`
|
||||
// HealthInterval is how often a starting shard is polled.
|
||||
HealthInterval time.Duration `koanf:"health_interval"`
|
||||
// RendezvousTimeout bounds the wait for a multi-node rendezvous point to
|
||||
// start listening.
|
||||
RendezvousTimeout time.Duration `koanf:"rendezvous_timeout"`
|
||||
// AliveGrace is how long a non-serving rank must stay up before it counts
|
||||
// as ready. There is no external signal for a worker, so staying alive is
|
||||
// the only evidence available.
|
||||
AliveGrace time.Duration `koanf:"alive_grace"`
|
||||
}
|
||||
|
||||
// DefaultConfig returns sensible defaults.
|
||||
func DefaultConfig() Config {
|
||||
return Config{
|
||||
WorkDir: "/var/lib/llmgw-agent",
|
||||
LogLines: 512,
|
||||
StopTimeout: 60 * time.Second,
|
||||
HealthInterval: 2 * time.Second,
|
||||
WorkDir: "/var/lib/llmgw-agent",
|
||||
LogLines: 512,
|
||||
StopTimeout: 60 * time.Second,
|
||||
HealthInterval: 2 * time.Second,
|
||||
RendezvousTimeout: 5 * time.Minute,
|
||||
AliveGrace: 5 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,6 +67,12 @@ func (c *Config) normalize() {
|
||||
if c.HealthInterval <= 0 {
|
||||
c.HealthInterval = d.HealthInterval
|
||||
}
|
||||
if c.RendezvousTimeout <= 0 {
|
||||
c.RendezvousTimeout = d.RendezvousTimeout
|
||||
}
|
||||
if c.AliveGrace <= 0 {
|
||||
c.AliveGrace = d.AliveGrace
|
||||
}
|
||||
}
|
||||
|
||||
// HealthChecker reports whether a shard is serving. It is an interface so the
|
||||
@@ -78,10 +95,12 @@ type Shard struct {
|
||||
readyAt time.Time
|
||||
exitCode int
|
||||
|
||||
logs *ringBuffer
|
||||
cmd *exec.Cmd
|
||||
cancel context.CancelFunc
|
||||
done chan struct{}
|
||||
logs *ringBuffer
|
||||
cmd *exec.Cmd
|
||||
// sidecars are the shard's supporting processes, torn down with it.
|
||||
sidecars []*exec.Cmd
|
||||
cancel context.CancelFunc
|
||||
done chan struct{}
|
||||
// adopted marks a shard inherited from a previous agent: it has no
|
||||
// exec.Cmd, so it is watched by polling and killed by signal alone.
|
||||
adopted bool
|
||||
@@ -157,7 +176,10 @@ var ErrAlreadyRunning = errors.New("shard is already running")
|
||||
// It is idempotent by ID: a repeated StartShard for the same shard is a no-op
|
||||
// rather than a second process, because the control plane retries commands
|
||||
// (plan §5.6).
|
||||
func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpec, cmd engines.Command) (*Shard, error) {
|
||||
func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpec, cmds []engines.Command) (*Shard, error) {
|
||||
if len(cmds) == 0 {
|
||||
return nil, fmt.Errorf("shard %s: no commands to run", id)
|
||||
}
|
||||
s.mu.Lock()
|
||||
if existing, ok := s.shards[id]; ok {
|
||||
s.mu.Unlock()
|
||||
@@ -180,7 +202,7 @@ func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpe
|
||||
s.shards[id] = sh
|
||||
s.mu.Unlock()
|
||||
|
||||
if err := s.launch(ctx, sh, cmd); err != nil {
|
||||
if err := s.launch(ctx, sh, cmds); err != nil {
|
||||
sh.setState(Failed, err)
|
||||
close(sh.done)
|
||||
return sh, err
|
||||
@@ -188,10 +210,40 @@ func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpe
|
||||
return sh, nil
|
||||
}
|
||||
|
||||
func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmd engines.Command) error {
|
||||
// launch starts a shard's processes in order.
|
||||
//
|
||||
// Every command but the last is a sidecar: it keeps running for the shard's
|
||||
// lifetime and is torn down with it, but its exit is not what defines the
|
||||
// shard's. The last command is the engine, and its exit is the shard's exit.
|
||||
func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Command) error {
|
||||
runCtx, cancel := context.WithCancel(context.WithoutCancel(ctx))
|
||||
sh.cancel = cancel
|
||||
|
||||
for i, c := range cmds[:len(cmds)-1] {
|
||||
proc, err := s.spawn(runCtx, sh, c)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return fmt.Errorf("sidecar %d (%s): %w", i, c.Name, err)
|
||||
}
|
||||
sh.mu.Lock()
|
||||
sh.sidecars = append(sh.sidecars, proc)
|
||||
sh.mu.Unlock()
|
||||
|
||||
if c.WaitForLeaderPort {
|
||||
// The next process cannot start until the rendezvous point is
|
||||
// listening: a Ray worker cannot join a head that is not up, and
|
||||
// the leader's server cannot claim GPUs from a cluster that does
|
||||
// not exist yet.
|
||||
addr := fmt.Sprintf("%s:%d", sh.Spec.LeaderAddr, sh.Spec.LeaderPort)
|
||||
if err := waitForPort(runCtx, addr, s.cfg.RendezvousTimeout); err != nil {
|
||||
cancel()
|
||||
return fmt.Errorf("waiting for %s (%s): %w", addr, c.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cmd := cmds[len(cmds)-1]
|
||||
|
||||
c := exec.CommandContext(runCtx, cmd.Path, cmd.Args...)
|
||||
c.Env = cmd.Env
|
||||
c.Dir = s.cfg.WorkDir
|
||||
@@ -225,13 +277,56 @@ func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmd engines.Command)
|
||||
"shard", sh.ID, "err", err)
|
||||
}
|
||||
|
||||
go s.pump(sh, stdout)
|
||||
go s.pump(sh, stdout, cmd.Name)
|
||||
go s.wait(sh)
|
||||
return nil
|
||||
}
|
||||
|
||||
// pump copies engine output into the ring buffer and the log.
|
||||
func (s *Supervisor) pump(sh *Shard, r io.Reader) {
|
||||
// spawn starts one process in its own group and pipes its output to the log.
|
||||
func (s *Supervisor) spawn(ctx context.Context, sh *Shard, c engines.Command) (*exec.Cmd, error) {
|
||||
proc := exec.CommandContext(ctx, c.Path, c.Args...)
|
||||
proc.Env = c.Env
|
||||
proc.Dir = s.cfg.WorkDir
|
||||
proc.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
proc.Cancel = func() error { return killGroup(proc.Process.Pid, syscall.SIGKILL) }
|
||||
|
||||
stdout, err := proc.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
proc.Stderr = proc.Stdout.(io.Writer)
|
||||
|
||||
if err := proc.Start(); err != nil {
|
||||
return nil, fmt.Errorf("starting %s: %w", c.Path, err)
|
||||
}
|
||||
go s.pump(sh, stdout, c.Name)
|
||||
// Reaped so it does not become a zombie; its exit does not end the shard.
|
||||
go func() { _ = proc.Wait() }()
|
||||
return proc, nil
|
||||
}
|
||||
|
||||
// waitForPort blocks until addr accepts a connection.
|
||||
func waitForPort(ctx context.Context, addr string, timeout time.Duration) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
var d net.Dialer
|
||||
for {
|
||||
conn, err := d.DialContext(ctx, "tcp", addr)
|
||||
if err == nil {
|
||||
_ = conn.Close()
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("not listening after %v: %w", timeout, ctx.Err())
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pump copies a process's output into the ring buffer and the log.
|
||||
func (s *Supervisor) pump(sh *Shard, r io.Reader, name string) {
|
||||
sc := bufio.NewScanner(r)
|
||||
// Engine tracebacks and progress bars produce long lines; the default
|
||||
// 64 KiB limit truncates them exactly when they matter.
|
||||
@@ -239,6 +334,11 @@ func (s *Supervisor) pump(sh *Shard, r io.Reader) {
|
||||
|
||||
for sc.Scan() {
|
||||
line := sc.Text()
|
||||
if name != "" {
|
||||
// Prefixed so the two processes of a Ray shard stay distinguishable
|
||||
// in a single log tail.
|
||||
line = "[" + name + "] " + line
|
||||
}
|
||||
sh.logs.add(line)
|
||||
s.log.Debug("engine", "shard", sh.ID, "line", line)
|
||||
}
|
||||
@@ -310,6 +410,32 @@ func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model st
|
||||
}
|
||||
}
|
||||
|
||||
// WaitAlive is readiness for a rank that serves no HTTP.
|
||||
//
|
||||
// A worker in a multi-node replica has no endpoint to poll, so the only
|
||||
// evidence available is that it started and stayed started. That is weaker than
|
||||
// a health check, and deliberately so: the replica's real readiness is the
|
||||
// leader answering, which only happens once every worker has joined. Polling a
|
||||
// worker's port instead would time out and fail a replica that is coming up
|
||||
// perfectly well.
|
||||
func (s *Supervisor) WaitAlive(ctx context.Context, sh *Shard) error {
|
||||
sh.setState(Loading, nil)
|
||||
|
||||
select {
|
||||
case <-sh.done:
|
||||
return fmt.Errorf("shard %s exited during startup: %s", sh.ID, sh.Status().Err)
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(s.cfg.AliveGrace):
|
||||
}
|
||||
|
||||
if st := sh.Status().State; st.Terminal() {
|
||||
return fmt.Errorf("shard %s is %s: %s", sh.ID, st, sh.Status().Err)
|
||||
}
|
||||
sh.setState(Ready, nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Drain marks a shard as taking no new work. The router stops routing to it;
|
||||
// the process keeps running until Stop.
|
||||
func (s *Supervisor) Drain(id string) error {
|
||||
@@ -334,11 +460,26 @@ func (s *Supervisor) Stop(ctx context.Context, id string) error {
|
||||
|
||||
sh.mu.Lock()
|
||||
pid := sh.pid
|
||||
sidecars := slices.Clone(sh.sidecars)
|
||||
if sh.state.Live() {
|
||||
sh.state = Draining // marks the exit as expected for wait() and poll()
|
||||
}
|
||||
sh.mu.Unlock()
|
||||
|
||||
// Sidecars go too. A Ray head left behind holds the port the next replica
|
||||
// needs, and would let a later shard join a cluster that serves nothing.
|
||||
defer func() {
|
||||
for _, sc := range sidecars {
|
||||
if sc.Process == nil {
|
||||
continue
|
||||
}
|
||||
if err := killGroup(sc.Process.Pid, syscall.SIGTERM); err != nil &&
|
||||
!errors.Is(err, os.ErrProcessDone) {
|
||||
s.log.Warn("stopping a sidecar failed", "shard", id, "pid", sc.Process.Pid, "err", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// A pid of zero means the process never started; an adopted shard has a pid
|
||||
// but no exec.Cmd, and must still be killed by signal.
|
||||
if pid == 0 {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -42,9 +43,9 @@ func newSupervisor(t *testing.T) (*Supervisor, *scriptedHealth) {
|
||||
return s, h
|
||||
}
|
||||
|
||||
// shellCommand builds a Command running a shell script.
|
||||
func shellCommand(script string) engines.Command {
|
||||
return engines.Command{Path: "/bin/sh", Args: []string{"-c", script}, Env: os.Environ()}
|
||||
// shellCommand builds the single-command sequence for a shell script.
|
||||
func shellCommand(script string) []engines.Command {
|
||||
return []engines.Command{{Path: "/bin/sh", Args: []string{"-c", script}, Env: os.Environ()}}
|
||||
}
|
||||
|
||||
func waitState(t *testing.T, sh *Shard, want State, limit time.Duration) {
|
||||
@@ -532,3 +533,197 @@ func TestCorruptStateFileIsCleanedUp(t *testing.T) {
|
||||
t.Error("the corrupt state file was left behind")
|
||||
}
|
||||
}
|
||||
|
||||
// --- multi-process shards (plan §11, phase 4) ---
|
||||
|
||||
func TestSidecarRunsBeforeEngine(t *testing.T) {
|
||||
// The Ray shape: a supporting process starts first and keeps running, then
|
||||
// the engine starts against it.
|
||||
s, _ := newSupervisor(t)
|
||||
dir := t.TempDir()
|
||||
order := filepath.Join(dir, "order")
|
||||
|
||||
// Each writes to the order file and to stdout: the file records the
|
||||
// sequence, stdout exercises per-process log attribution.
|
||||
cmds := []engines.Command{
|
||||
{Name: "sidecar", Path: "/bin/sh", Args: []string{"-c",
|
||||
fmt.Sprintf("echo sidecar >> %s; echo hello-from-sidecar; sleep 30", order)}, Env: os.Environ()},
|
||||
{Name: "engine", Path: "/bin/sh", Args: []string{"-c",
|
||||
fmt.Sprintf("echo engine >> %s; echo hello-from-engine; sleep 30", order)}, Env: os.Environ()},
|
||||
}
|
||||
|
||||
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, cmds)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if b, err := os.ReadFile(order); err == nil && strings.Count(string(b), "\n") == 2 {
|
||||
if got := strings.Fields(string(b)); got[0] != "sidecar" || got[1] != "engine" {
|
||||
t.Fatalf("start order was %v", got)
|
||||
}
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
|
||||
// The shard's pid is the engine's, not the sidecar's: the engine's exit is
|
||||
// what ends the shard.
|
||||
if sh.Status().PID == 0 {
|
||||
t.Fatal("no engine pid recorded")
|
||||
}
|
||||
// Both processes appear in one log tail, distinguishable by prefix.
|
||||
deadline = time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
logs := strings.Join(sh.Logs(20), "\n")
|
||||
if strings.Contains(logs, "[sidecar] hello-from-sidecar") &&
|
||||
strings.Contains(logs, "[engine] hello-from-engine") {
|
||||
return
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Errorf("output is not attributed per process:\n%s", strings.Join(sh.Logs(20), "\n"))
|
||||
}
|
||||
|
||||
func TestStopKillsSidecar(t *testing.T) {
|
||||
// A Ray head left behind holds the port the next replica needs, and would
|
||||
// let a later shard join a cluster that serves nothing.
|
||||
s, _ := newSupervisor(t)
|
||||
dir := t.TempDir()
|
||||
marker := filepath.Join(dir, "sidecar.pid")
|
||||
|
||||
cmds := []engines.Command{
|
||||
{Name: "sidecar", Path: "/bin/sh", Args: []string{"-c",
|
||||
fmt.Sprintf("echo $$ > %s; sleep 30", marker)}, Env: os.Environ()},
|
||||
{Name: "engine", Path: "/bin/sh", Args: []string{"-c", "sleep 30"}, Env: os.Environ()},
|
||||
}
|
||||
if _, err := s.Start(t.Context(), "a", engines.ShardSpec{}, cmds); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var sidePID int
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if b, err := os.ReadFile(marker); err == nil {
|
||||
if _, err := fmt.Sscanf(strings.TrimSpace(string(b)), "%d", &sidePID); err == nil && sidePID > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if sidePID == 0 {
|
||||
t.Fatal("the sidecar never started")
|
||||
}
|
||||
|
||||
if err := s.Stop(t.Context(), "a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deadline = time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if !processAlive(sidePID) {
|
||||
return
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Errorf("the sidecar (%d) survived the stop and still holds its port", sidePID)
|
||||
}
|
||||
|
||||
func TestSidecarFailureFailsTheShard(t *testing.T) {
|
||||
s, _ := newSupervisor(t)
|
||||
cmds := []engines.Command{
|
||||
{Name: "sidecar", Path: "/nonexistent/binary", Env: os.Environ()},
|
||||
{Name: "engine", Path: "/bin/sh", Args: []string{"-c", "sleep 30"}, Env: os.Environ()},
|
||||
}
|
||||
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, cmds)
|
||||
if err == nil {
|
||||
t.Fatal("a shard whose sidecar could not start was reported healthy")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "sidecar") {
|
||||
t.Errorf("the error does not identify the sidecar: %v", err)
|
||||
}
|
||||
if sh.Status().State != Failed {
|
||||
t.Errorf("state=%s", sh.Status().State)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitForLeaderPortBlocksUntilListening(t *testing.T) {
|
||||
// A Ray worker cannot join a head that is not up, and the leader's server
|
||||
// cannot claim GPUs from a cluster that does not exist yet.
|
||||
s, _ := newSupervisor(t)
|
||||
s.cfg.RendezvousTimeout = 5 * time.Second
|
||||
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
port := ln.Addr().(*net.TCPAddr).Port
|
||||
_ = ln.Close() // nothing listening yet
|
||||
|
||||
spec := engines.ShardSpec{LeaderAddr: "127.0.0.1", LeaderPort: port}
|
||||
cmds := []engines.Command{
|
||||
{Name: "sidecar", Path: "/bin/sh", Args: []string{"-c",
|
||||
fmt.Sprintf("sleep 0.4; nc -l %d", port)}, Env: os.Environ(),
|
||||
WaitForLeaderPort: true},
|
||||
{Name: "engine", Path: "/bin/sh", Args: []string{"-c", "sleep 30"}, Env: os.Environ()},
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
if _, err := s.Start(t.Context(), "a", spec, cmds); err != nil {
|
||||
t.Skipf("nc is unavailable for the rendezvous stub: %v", err)
|
||||
}
|
||||
if d := time.Since(start); d < 300*time.Millisecond {
|
||||
t.Errorf("the engine started after %v, before the rendezvous point was listening", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRendezvousTimeout(t *testing.T) {
|
||||
// Nothing ever listens, so the shard must fail rather than hang forever.
|
||||
s, _ := newSupervisor(t)
|
||||
s.cfg.RendezvousTimeout = 300 * time.Millisecond
|
||||
|
||||
spec := engines.ShardSpec{LeaderAddr: "127.0.0.1", LeaderPort: 1}
|
||||
cmds := []engines.Command{
|
||||
{Name: "sidecar", Path: "/bin/sh", Args: []string{"-c", "sleep 30"},
|
||||
Env: os.Environ(), WaitForLeaderPort: true},
|
||||
{Name: "engine", Path: "/bin/sh", Args: []string{"-c", "sleep 30"}, Env: os.Environ()},
|
||||
}
|
||||
if _, err := s.Start(t.Context(), "a", spec, cmds); err == nil {
|
||||
t.Fatal("the shard did not fail when the rendezvous point never appeared")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitAliveForNonServingRank(t *testing.T) {
|
||||
// A worker has no endpoint to poll, so staying up is the only evidence
|
||||
// available. This is what stopped multi-node replicas coming up at all.
|
||||
s, _ := newSupervisor(t)
|
||||
s.cfg.AliveGrace = 150 * time.Millisecond
|
||||
|
||||
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{Rank: 1}, shellCommand("sleep 30"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.WaitAlive(t.Context(), sh); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := sh.Status().State; got != Ready {
|
||||
t.Errorf("state=%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitAliveFailsOnEarlyExit(t *testing.T) {
|
||||
s, _ := newSupervisor(t)
|
||||
s.cfg.AliveGrace = 2 * time.Second
|
||||
|
||||
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{Rank: 1}, shellCommand("exit 1"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
start := time.Now()
|
||||
if err := s.WaitAlive(t.Context(), sh); err == nil {
|
||||
t.Fatal("a worker that died immediately was reported ready")
|
||||
}
|
||||
if d := time.Since(start); d > 1500*time.Millisecond {
|
||||
t.Errorf("waited %v for a process that had already exited", d)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user