An engine's output goes to a file, which an agent restart cannot break

Engines wrote to a pipe of the agent's. An agent that restarted and
adopted them left that pipe with no reader, and an engine's next write
past its buffer failed with EPIPE — in a Python engine, inside whatever it
was reporting on. Each process now writes to its own file in the agent's
work directory (0600), followed into the shard's log; an adopted engine's
file is read from the start, so its output from before the restart is in
its log too. A file past 64 MB is emptied in place; the files go with the
shard.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-10 10:25:04 +02:00
1 parent 9e7618c073
commit bb40d22eb8
3 files changed
+246 -33

No files matched your search

+31 -33
View File
@@ -1,12 +1,10 @@
package supervisor
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net"
"os"
@@ -104,6 +102,12 @@ type Shard struct {
ID string
Spec engines.ShardSpec
// outStop ends the followers of the processes' output files, and
// outDone is their having read to the end.
outStop chan struct{}
outOnce sync.Once
outDone sync.WaitGroup
mu sync.Mutex
state State
pid int
@@ -248,6 +252,7 @@ func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpe
logs: newRingBuffer(s.cfg.LogLines),
started: time.Now(),
done: make(chan struct{}),
outStop: make(chan struct{}),
}
s.shards[id] = sh
s.mu.Unlock()
@@ -315,14 +320,15 @@ func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Comma
// CommandContext kills only the leader, so replace that with a group kill.
c.Cancel = func() error { return killGroup(c.Process.Pid, syscall.SIGKILL) }
stdout, err := c.StdoutPipe()
out, err := s.openOutput(sh.ID, "")
if err != nil {
cancel()
return err
}
c.Stderr = c.Stdout.(io.Writer) // interleave, as the engine writes them
if err := c.Start(); err != nil {
c.Stdout, c.Stderr = out, out // interleaved, as the engine writes them
err = c.Start()
out.Close() // the process holds its own
if err != nil {
cancel()
return fmt.Errorf("starting %s: %w", cmd.Path, err)
}
@@ -339,7 +345,8 @@ func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Comma
"shard", sh.ID, "err", err)
}
go s.pump(sh, stdout, cmd.Name)
sh.outDone.Add(1)
go s.follow(sh, s.outputPath(sh.ID, ""), cmd.Name)
go s.wait(sh)
return nil
}
@@ -353,16 +360,18 @@ func (s *Supervisor) spawn(ctx context.Context, sh *Shard, c engines.Command) (*
s.dropTo(proc)
proc.Cancel = func() error { return killGroup(proc.Process.Pid, syscall.SIGKILL) }
stdout, err := proc.StdoutPipe()
out, err := s.openOutput(sh.ID, c.Name)
if err != nil {
return nil, err
}
proc.Stderr = proc.Stdout.(io.Writer)
if err := proc.Start(); err != nil {
proc.Stdout, proc.Stderr = out, out
err = proc.Start()
out.Close()
if err != nil {
return nil, fmt.Errorf("starting %s: %w", c.Path, err)
}
go s.pump(sh, stdout, c.Name)
sh.outDone.Add(1)
go s.follow(sh, s.outputPath(sh.ID, c.Name), c.Name)
// Reaped so it does not become a zombie; its exit does not end the shard.
go func() { _ = proc.Wait() }()
return proc, nil
@@ -395,24 +404,6 @@ func waitForPort(ctx context.Context, addr string, timeout time.Duration) (err e
}
// pump copies a process's output into the ring buffer and the log.
func (s *Supervisor) pump(sh *Shard, r io.Reader, name string) {
sc := bufio.NewScanner(r)
// Engine tracebacks and progress bars produce long lines; the default
// 64 KiB limit truncates them exactly when they matter.
sc.Buffer(make([]byte, 0, 64<<10), 1<<20)
for sc.Scan() {
line := sc.Text()
if name != "" {
// Prefixed so the two processes of a Ray shard stay distinguishable
// in a single log tail.
line = "[" + name + "] " + line
}
sh.logs.add(line)
s.log.Debug("engine", "shard", sh.ID, "line", line)
}
}
// wait reaps the process and records why it ended.
func (s *Supervisor) wait(sh *Shard) {
defer close(sh.done)
@@ -440,6 +431,7 @@ func (s *Supervisor) wait(sh *Shard) {
final, code := sh.state, sh.exitCode
sh.mu.Unlock()
s.stopOutput(sh)
s.removeState(sh)
if final == Failed {
s.log.Error("shard failed", "shard", sh.ID, "exit_code", code, "err", err,
@@ -760,6 +752,7 @@ func (s *Supervisor) writeState(sh *Shard) error {
func (s *Supervisor) removeState(sh *Shard) {
_ = os.Remove(s.statePath(sh.ID))
s.removeOutput(sh.ID)
}
// Orphan is an engine process left behind by a previous agent.
@@ -838,15 +831,18 @@ func (s *Supervisor) Adopt(o Orphan) (*Shard, error) {
logs: newRingBuffer(s.cfg.LogLines),
started: o.StartedAt,
done: make(chan struct{}),
outStop: make(chan struct{}),
adopted: true,
cmdline: o.Cmdline,
}
s.shards[o.ShardID] = sh
s.mu.Unlock()
// Output is gone with the previous agent; say so rather than presenting an
// empty log as if the engine had been silent.
sh.logs.add("[adopted from a previous agent; earlier output is not available]")
// Its output file is read from the start: what it wrote under the
// previous agent is there, and what it writes now goes on arriving.
sh.logs.add("[adopted from a previous agent]")
sh.outDone.Add(1)
go s.follow(sh, s.outputPath(sh.ID, ""), "")
go s.poll(sh)
return sh, nil
@@ -874,6 +870,7 @@ func (s *Supervisor) poll(sh *Shard) {
}
sh.mu.Unlock()
s.stopOutput(sh)
s.removeState(sh)
s.log.Info("adopted shard exited", "shard", sh.ID, "expected", expected)
return
@@ -900,6 +897,7 @@ func (s *Supervisor) KillOrphan(o Orphan) error {
}
// An already-missing state file is success: the previous agent may have
// reaped the process itself between the scan and this call.
s.removeOutput(o.ShardID)
if err := os.Remove(s.statePath(o.ShardID)); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}