Strata named by hand is reported, its engine stopped with it, its memory required, Pascal allowed
CI / test (push) Successful in 9m28s
CI / test (push) Successful in 9m28s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
73091d7c97
commit
91f668fe7e
8 files changed
+244
-2
No files matched your search
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
@@ -63,3 +64,71 @@ func isZombie(pid int) bool {
|
||||
}
|
||||
return b[i+2] == 'Z'
|
||||
}
|
||||
|
||||
// strays are a process's descendants that left its group, as an engine that
|
||||
// starts its worker in a session of its own does (Strata): signalling the
|
||||
// group misses them, and one left behind holds the card and the RAM the next
|
||||
// replica needs. Found before the group is signalled, while they are still
|
||||
// its descendants: after, they belong to init. Linux only; elsewhere none.
|
||||
func strays(pid int) []int {
|
||||
entries, err := os.ReadDir("/proc")
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
parent := map[int]int{}
|
||||
group := map[int]int{}
|
||||
for _, e := range entries {
|
||||
p, err := strconv.Atoi(e.Name())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if pp, pg, ok := parentAndGroup(p); ok {
|
||||
parent[p], group[p] = pp, pg
|
||||
}
|
||||
}
|
||||
var out []int
|
||||
for p := range parent {
|
||||
if group[p] == pid || p == pid {
|
||||
continue
|
||||
}
|
||||
// Bounded: no engine's tree is this deep, and a pid reused while
|
||||
// /proc was read must not loop.
|
||||
for q, n := parent[p], 0; q > 1 && n < 32; q, n = parent[q], n+1 {
|
||||
if q == pid {
|
||||
out = append(out, p)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// parentAndGroup reads a process's parent and group from /proc.
|
||||
func parentAndGroup(pid int) (int, int, bool) {
|
||||
b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat")
|
||||
if err != nil {
|
||||
return 0, 0, false
|
||||
}
|
||||
i := bytes.LastIndexByte(b, ')')
|
||||
if i < 0 {
|
||||
return 0, 0, false
|
||||
}
|
||||
f := strings.Fields(string(b[i+1:]))
|
||||
if len(f) < 3 {
|
||||
return 0, 0, false
|
||||
}
|
||||
pp, err1 := strconv.Atoi(f[1])
|
||||
pg, err2 := strconv.Atoi(f[2])
|
||||
return pp, pg, err1 == nil && err2 == nil
|
||||
}
|
||||
|
||||
// signalStrays signals each stray, and its own group when it leads one.
|
||||
func signalStrays(pids []int, sig syscall.Signal) {
|
||||
for _, p := range pids {
|
||||
if _, pg, ok := parentAndGroup(p); ok && pg == p {
|
||||
_ = syscall.Kill(-p, sig)
|
||||
continue
|
||||
}
|
||||
_ = syscall.Kill(p, sig)
|
||||
}
|
||||
}
|
||||
@@ -582,10 +582,24 @@ func (s *Supervisor) Stop(ctx context.Context, id string) (err error) {
|
||||
}
|
||||
|
||||
// Signal the group, not the process: engines fork workers that hold GPU
|
||||
// memory, and those outlive a signal sent only to the leader.
|
||||
// memory, and those outlive a signal sent only to the leader. And what
|
||||
// left the group, found while it is still the leader's.
|
||||
left := strays(pid)
|
||||
if err := killGroup(pid, syscall.SIGTERM); err != nil && !errors.Is(err, os.ErrProcessDone) {
|
||||
s.log.Warn("SIGTERM failed", "shard", id, "err", err)
|
||||
}
|
||||
signalStrays(left, syscall.SIGTERM)
|
||||
defer func() {
|
||||
// Given as long as the group to go; then killed.
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) && slices.ContainsFunc(left, processAlive) {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
if slices.ContainsFunc(left, processAlive) {
|
||||
s.log.Warn("killing an engine's processes that left its group", "shard", id, "pids", left)
|
||||
signalStrays(left, syscall.SIGKILL)
|
||||
}
|
||||
}()
|
||||
|
||||
// force kills the group outright. exec.Cmd's cancel does this for shards we
|
||||
// started; an adopted shard has none, so signal its group directly.
|
||||
@@ -598,6 +612,7 @@ func (s *Supervisor) Stop(ctx context.Context, id string) (err error) {
|
||||
if err := killGroup(pid, syscall.SIGKILL); err != nil && !errors.Is(err, os.ErrProcessDone) {
|
||||
s.log.Warn("SIGKILL failed", "shard", id, "err", err)
|
||||
}
|
||||
signalStrays(left, syscall.SIGKILL)
|
||||
}
|
||||
|
||||
select {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -981,3 +982,38 @@ func TestAnOutputFileIsEmptiedPastItsBound(t *testing.T) {
|
||||
t.Errorf("a line written after the file was emptied was lost: %v", sh.Logs(8))
|
||||
}
|
||||
}
|
||||
|
||||
// An engine's worker in a session of its own, as Strata's is, is not in the
|
||||
// group a stop signals: it is found and stopped as well.
|
||||
func TestStopKillsAWorkerThatLeftTheGroup(t *testing.T) {
|
||||
if _, err := os.Stat("/proc/self/stat"); err != nil {
|
||||
t.Skip("needs /proc")
|
||||
}
|
||||
if _, err := exec.LookPath("setsid"); err != nil {
|
||||
t.Skip("needs setsid")
|
||||
}
|
||||
s, _ := newSupervisor(t)
|
||||
marker := filepath.Join(t.TempDir(), "worker.pid")
|
||||
script := fmt.Sprintf(`setsid sh -c 'echo $$ > %s; sleep 30' & sleep 30`, marker)
|
||||
if _, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand(script)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var worker int
|
||||
for deadline := time.Now().Add(3 * time.Second); time.Now().Before(deadline) && worker == 0; time.Sleep(10 * time.Millisecond) {
|
||||
if b, err := os.ReadFile(marker); err == nil {
|
||||
_, _ = fmt.Sscanf(strings.TrimSpace(string(b)), "%d", &worker)
|
||||
}
|
||||
}
|
||||
if worker == 0 {
|
||||
t.Fatal("the worker never started")
|
||||
}
|
||||
if err := s.Stop(t.Context(), "a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(8 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if !processAlive(worker) {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Errorf("the worker %d that left the group survived the stop", worker)
|
||||
}
|
||||
Reference in new issue
Block a user