An engine's output goes to a file, which an agent restart cannot break

Engines wrote to a pipe of the agent's. An agent that restarted and
adopted them left that pipe with no reader, and an engine's next write
past its buffer failed with EPIPE — in a Python engine, inside whatever it
was reporting on. Each process now writes to its own file in the agent's
work directory (0600), followed into the shard's log; an adopted engine's
file is read from the start, so its output from before the restart is in
its log too. A file past 64 MB is emptied in place; the files go with the
shard.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-10 10:25:04 +02:00
1 parent 9e7618c073
commit bb40d22eb8
3 files changed
+246 -33

No files matched your search

@@ -1,6 +1,7 @@
package supervisor
import (
"bytes"
"context"
"errors"
"fmt"
@@ -905,3 +906,78 @@ func TestAnAdoptedShardsComputeIsKeptWithoutCrashing(t *testing.T) {
t.Fatalf("lost: %+v", again)
}
}
func TestAnAdoptedEngineGoesOnWritingAndItsEarlierOutputIsKept(t *testing.T) {
// An engine's output went to a pipe of the agent's. The agent that
// restarted and adopted the engine left that pipe with no reader, and
// the engine's next write past the pipe's buffer failed with EPIPE —
// which, in a Python engine, raised inside whatever was being reported
// on and failed the request. Output goes to a file now: a reader is not
// something a file can lose, and the lines from before the restart are
// there for the agent after it.
s, _ := newSupervisor(t)
// 256 KiB at once outgrows a pipe's buffer; a reader gone means EPIPE.
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{ReplicaID: "r"},
shellCommand(`echo before-restart; sleep 1; head -c 262144 /dev/zero | tr '\0' x; echo; echo after-restart-done; sleep 30`))
if err != nil {
t.Fatal(err)
}
eventually(t, 3*time.Second, "the first line was never captured", func() bool {
return strings.Contains(strings.Join(sh.Logs(10), "\n"), "before-restart")
})
// The agent goes away without stopping the engine: nothing reads the
// output between here and the adoption.
pid := sh.Status().PID
sh.outOnce.Do(func() { close(sh.outStop) })
sh.outDone.Wait()
s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 3 * time.Second, LogLines: 64},
&scriptedHealth{}, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatal(err)
}
orphans, _ := s2.FindOrphans()
adopted, err := s2.Adopt(orphans[0])
if err != nil {
t.Fatal(err)
}
eventually(t, 5*time.Second, "the engine's output after the restart never arrived: "+strings.Join(adopted.Logs(5), " | "), func() bool {
return strings.Contains(strings.Join(adopted.Logs(64), "\n"), "after-restart-done")
})
if !strings.Contains(strings.Join(adopted.Logs(64), "\n"), "before-restart") {
t.Error("the output from before the restart is not in the adopted log")
}
if !processAlive(pid) {
t.Fatal("the engine died writing output nobody was reading")
}
if err := s2.Stop(t.Context(), "a"); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(s2.outputPath("a", "")); !errors.Is(err, os.ErrNotExist) {
t.Errorf("the output file was not removed with the shard: %v", err)
}
}
func TestAnOutputFileIsEmptiedPastItsBound(t *testing.T) {
s, _ := newSupervisor(t)
path := s.outputPath("a", "")
// A file already past the bound, as a long-running engine leaves one.
big, _ := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0o644)
_, _ = big.Write(bytes.Repeat([]byte("x"), maxOutputBytes+1))
big.Close()
sh := &Shard{ID: "a", logs: newRingBuffer(8), done: make(chan struct{}), outStop: make(chan struct{})}
sh.outDone.Add(1)
go s.follow(sh, path, "")
time.Sleep(300 * time.Millisecond)
f, _ := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0o644)
_, _ = f.WriteString("still here\n")
f.Close()
s.stopOutput(sh)
st, _ := os.Stat(path)
if st.Size() > 1024 {
t.Errorf("the file was not emptied: %d bytes", st.Size())
}
if !strings.Contains(strings.Join(sh.Logs(8), "\n"), "still here") {
t.Errorf("a line written after the file was emptied was lost: %v", sh.Logs(8))
}
}