Strata named by hand is reported, its engine stopped with it, its memory required, Pascal allowed
CI / test (push) Successful in 9m28s

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-30 09:49:16 +01:00
1 parent 73091d7c97
commit 91f668fe7e
8 files changed
+244 -2

No files matched your search

+69
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"os"
"strconv"
"strings"
"syscall"
)
@@ -63,3 +64,71 @@ func isZombie(pid int) bool {
}
return b[i+2] == 'Z'
}
// strays are a process's descendants that left its group, as an engine that
// starts its worker in a session of its own does (Strata): signalling the
// group misses them, and one left behind holds the card and the RAM the next
// replica needs. Found before the group is signalled, while they are still
// its descendants: after, they belong to init. Linux only; elsewhere none.
func strays(pid int) []int {
entries, err := os.ReadDir("/proc")
if err != nil {
return nil
}
parent := map[int]int{}
group := map[int]int{}
for _, e := range entries {
p, err := strconv.Atoi(e.Name())
if err != nil {
continue
}
if pp, pg, ok := parentAndGroup(p); ok {
parent[p], group[p] = pp, pg
}
}
var out []int
for p := range parent {
if group[p] == pid || p == pid {
continue
}
// Bounded: no engine's tree is this deep, and a pid reused while
// /proc was read must not loop.
for q, n := parent[p], 0; q > 1 && n < 32; q, n = parent[q], n+1 {
if q == pid {
out = append(out, p)
break
}
}
}
return out
}
// parentAndGroup reads a process's parent and group from /proc.
func parentAndGroup(pid int) (int, int, bool) {
b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat")
if err != nil {
return 0, 0, false
}
i := bytes.LastIndexByte(b, ')')
if i < 0 {
return 0, 0, false
}
f := strings.Fields(string(b[i+1:]))
if len(f) < 3 {
return 0, 0, false
}
pp, err1 := strconv.Atoi(f[1])
pg, err2 := strconv.Atoi(f[2])
return pp, pg, err1 == nil && err2 == nil
}
// signalStrays signals each stray, and its own group when it leads one.
func signalStrays(pids []int, sig syscall.Signal) {
for _, p := range pids {
if _, pg, ok := parentAndGroup(p); ok && pg == p {
_ = syscall.Kill(-p, sig)
continue
}
_ = syscall.Kill(p, sig)
}
}
+16 -1
View File
@@ -582,10 +582,24 @@ func (s *Supervisor) Stop(ctx context.Context, id string) (err error) {
}
// Signal the group, not the process: engines fork workers that hold GPU
// memory, and those outlive a signal sent only to the leader.
// memory, and those outlive a signal sent only to the leader. And what
// left the group, found while it is still the leader's.
left := strays(pid)
if err := killGroup(pid, syscall.SIGTERM); err != nil && !errors.Is(err, os.ErrProcessDone) {
s.log.Warn("SIGTERM failed", "shard", id, "err", err)
}
signalStrays(left, syscall.SIGTERM)
defer func() {
// Given as long as the group to go; then killed.
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) && slices.ContainsFunc(left, processAlive) {
time.Sleep(100 * time.Millisecond)
}
if slices.ContainsFunc(left, processAlive) {
s.log.Warn("killing an engine's processes that left its group", "shard", id, "pids", left)
signalStrays(left, syscall.SIGKILL)
}
}()
// force kills the group outright. exec.Cmd's cancel does this for shards we
// started; an adopted shard has none, so signal its group directly.
@@ -598,6 +612,7 @@ func (s *Supervisor) Stop(ctx context.Context, id string) (err error) {
if err := killGroup(pid, syscall.SIGKILL); err != nil && !errors.Is(err, os.ErrProcessDone) {
s.log.Warn("SIGKILL failed", "shard", id, "err", err)
}
signalStrays(left, syscall.SIGKILL)
}
select {
@@ -8,6 +8,7 @@ import (
"log/slog"
"net"
"os"
"os/exec"
"path/filepath"
"slices"
"strings"
@@ -981,3 +982,38 @@ func TestAnOutputFileIsEmptiedPastItsBound(t *testing.T) {
t.Errorf("a line written after the file was emptied was lost: %v", sh.Logs(8))
}
}
// An engine's worker in a session of its own, as Strata's is, is not in the
// group a stop signals: it is found and stopped as well.
func TestStopKillsAWorkerThatLeftTheGroup(t *testing.T) {
if _, err := os.Stat("/proc/self/stat"); err != nil {
t.Skip("needs /proc")
}
if _, err := exec.LookPath("setsid"); err != nil {
t.Skip("needs setsid")
}
s, _ := newSupervisor(t)
marker := filepath.Join(t.TempDir(), "worker.pid")
script := fmt.Sprintf(`setsid sh -c 'echo $$ > %s; sleep 30' & sleep 30`, marker)
if _, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand(script)); err != nil {
t.Fatal(err)
}
var worker int
for deadline := time.Now().Add(3 * time.Second); time.Now().Before(deadline) && worker == 0; time.Sleep(10 * time.Millisecond) {
if b, err := os.ReadFile(marker); err == nil {
_, _ = fmt.Sscanf(strings.TrimSpace(string(b)), "%d", &worker)
}
}
if worker == 0 {
t.Fatal("the worker never started")
}
if err := s.Stop(t.Context(), "a"); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(8 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if !processAlive(worker) {
return
}
}
t.Errorf("the worker %d that left the group survived the stop", worker)
}