An engine's output goes to a file, which an agent restart cannot break
Engines wrote to a pipe of the agent's. An agent that restarted and adopted them left that pipe with no reader, and an engine's next write past its buffer failed with EPIPE — in a Python engine, inside whatever it was reporting on. Each process now writes to its own file in the agent's work directory (0600), followed into the shard's log; an adopted engine's file is read from the start, so its output from before the restart is in its log too. A file past 64 MB is emptied in place; the files go with the shard. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
9e7618c073
commit
bb40d22eb8
3 files changed
+246
-33
No files matched your search
@@ -1,12 +1,10 @@
|
||||
package supervisor
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
@@ -104,6 +102,12 @@ type Shard struct {
|
||||
ID string
|
||||
Spec engines.ShardSpec
|
||||
|
||||
// outStop ends the followers of the processes' output files, and
|
||||
// outDone is their having read to the end.
|
||||
outStop chan struct{}
|
||||
outOnce sync.Once
|
||||
outDone sync.WaitGroup
|
||||
|
||||
mu sync.Mutex
|
||||
state State
|
||||
pid int
|
||||
@@ -248,6 +252,7 @@ func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpe
|
||||
logs: newRingBuffer(s.cfg.LogLines),
|
||||
started: time.Now(),
|
||||
done: make(chan struct{}),
|
||||
outStop: make(chan struct{}),
|
||||
}
|
||||
s.shards[id] = sh
|
||||
s.mu.Unlock()
|
||||
@@ -315,14 +320,15 @@ func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Comma
|
||||
// CommandContext kills only the leader, so replace that with a group kill.
|
||||
c.Cancel = func() error { return killGroup(c.Process.Pid, syscall.SIGKILL) }
|
||||
|
||||
stdout, err := c.StdoutPipe()
|
||||
out, err := s.openOutput(sh.ID, "")
|
||||
if err != nil {
|
||||
cancel()
|
||||
return err
|
||||
}
|
||||
c.Stderr = c.Stdout.(io.Writer) // interleave, as the engine writes them
|
||||
|
||||
if err := c.Start(); err != nil {
|
||||
c.Stdout, c.Stderr = out, out // interleaved, as the engine writes them
|
||||
err = c.Start()
|
||||
out.Close() // the process holds its own
|
||||
if err != nil {
|
||||
cancel()
|
||||
return fmt.Errorf("starting %s: %w", cmd.Path, err)
|
||||
}
|
||||
@@ -339,7 +345,8 @@ func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Comma
|
||||
"shard", sh.ID, "err", err)
|
||||
}
|
||||
|
||||
go s.pump(sh, stdout, cmd.Name)
|
||||
sh.outDone.Add(1)
|
||||
go s.follow(sh, s.outputPath(sh.ID, ""), cmd.Name)
|
||||
go s.wait(sh)
|
||||
return nil
|
||||
}
|
||||
@@ -353,16 +360,18 @@ func (s *Supervisor) spawn(ctx context.Context, sh *Shard, c engines.Command) (*
|
||||
s.dropTo(proc)
|
||||
proc.Cancel = func() error { return killGroup(proc.Process.Pid, syscall.SIGKILL) }
|
||||
|
||||
stdout, err := proc.StdoutPipe()
|
||||
out, err := s.openOutput(sh.ID, c.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
proc.Stderr = proc.Stdout.(io.Writer)
|
||||
|
||||
if err := proc.Start(); err != nil {
|
||||
proc.Stdout, proc.Stderr = out, out
|
||||
err = proc.Start()
|
||||
out.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("starting %s: %w", c.Path, err)
|
||||
}
|
||||
go s.pump(sh, stdout, c.Name)
|
||||
sh.outDone.Add(1)
|
||||
go s.follow(sh, s.outputPath(sh.ID, c.Name), c.Name)
|
||||
// Reaped so it does not become a zombie; its exit does not end the shard.
|
||||
go func() { _ = proc.Wait() }()
|
||||
return proc, nil
|
||||
@@ -395,24 +404,6 @@ func waitForPort(ctx context.Context, addr string, timeout time.Duration) (err e
|
||||
}
|
||||
|
||||
// pump copies a process's output into the ring buffer and the log.
|
||||
func (s *Supervisor) pump(sh *Shard, r io.Reader, name string) {
|
||||
sc := bufio.NewScanner(r)
|
||||
// Engine tracebacks and progress bars produce long lines; the default
|
||||
// 64 KiB limit truncates them exactly when they matter.
|
||||
sc.Buffer(make([]byte, 0, 64<<10), 1<<20)
|
||||
|
||||
for sc.Scan() {
|
||||
line := sc.Text()
|
||||
if name != "" {
|
||||
// Prefixed so the two processes of a Ray shard stay distinguishable
|
||||
// in a single log tail.
|
||||
line = "[" + name + "] " + line
|
||||
}
|
||||
sh.logs.add(line)
|
||||
s.log.Debug("engine", "shard", sh.ID, "line", line)
|
||||
}
|
||||
}
|
||||
|
||||
// wait reaps the process and records why it ended.
|
||||
func (s *Supervisor) wait(sh *Shard) {
|
||||
defer close(sh.done)
|
||||
@@ -440,6 +431,7 @@ func (s *Supervisor) wait(sh *Shard) {
|
||||
final, code := sh.state, sh.exitCode
|
||||
sh.mu.Unlock()
|
||||
|
||||
s.stopOutput(sh)
|
||||
s.removeState(sh)
|
||||
if final == Failed {
|
||||
s.log.Error("shard failed", "shard", sh.ID, "exit_code", code, "err", err,
|
||||
@@ -760,6 +752,7 @@ func (s *Supervisor) writeState(sh *Shard) error {
|
||||
|
||||
func (s *Supervisor) removeState(sh *Shard) {
|
||||
_ = os.Remove(s.statePath(sh.ID))
|
||||
s.removeOutput(sh.ID)
|
||||
}
|
||||
|
||||
// Orphan is an engine process left behind by a previous agent.
|
||||
@@ -838,15 +831,18 @@ func (s *Supervisor) Adopt(o Orphan) (*Shard, error) {
|
||||
logs: newRingBuffer(s.cfg.LogLines),
|
||||
started: o.StartedAt,
|
||||
done: make(chan struct{}),
|
||||
outStop: make(chan struct{}),
|
||||
adopted: true,
|
||||
cmdline: o.Cmdline,
|
||||
}
|
||||
s.shards[o.ShardID] = sh
|
||||
s.mu.Unlock()
|
||||
|
||||
// Output is gone with the previous agent; say so rather than presenting an
|
||||
// empty log as if the engine had been silent.
|
||||
sh.logs.add("[adopted from a previous agent; earlier output is not available]")
|
||||
// Its output file is read from the start: what it wrote under the
|
||||
// previous agent is there, and what it writes now goes on arriving.
|
||||
sh.logs.add("[adopted from a previous agent]")
|
||||
sh.outDone.Add(1)
|
||||
go s.follow(sh, s.outputPath(sh.ID, ""), "")
|
||||
|
||||
go s.poll(sh)
|
||||
return sh, nil
|
||||
@@ -874,6 +870,7 @@ func (s *Supervisor) poll(sh *Shard) {
|
||||
}
|
||||
sh.mu.Unlock()
|
||||
|
||||
s.stopOutput(sh)
|
||||
s.removeState(sh)
|
||||
s.log.Info("adopted shard exited", "shard", sh.ID, "expected", expected)
|
||||
return
|
||||
@@ -900,6 +897,7 @@ func (s *Supervisor) KillOrphan(o Orphan) error {
|
||||
}
|
||||
// An already-missing state file is success: the previous agent may have
|
||||
// reaped the process itself between the scan and this call.
|
||||
s.removeOutput(o.ShardID)
|
||||
if err := os.Remove(s.statePath(o.ShardID)); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user