llmgw/internal/agent/supervisor/process_unix.go

package supervisor

import (
	"bytes"
	"os"
	"strconv"
	"strings"
	"syscall"
)

// killGroup signals a whole process group.
//
// Engines fork worker processes that hold GPU memory. Signalling only the
// leader leaves those workers alive, and the next launch then fails to
// allocate — with an error that points at the new process rather than the old.
func killGroup(pid int, sig syscall.Signal) error {
	// The negative pid addresses the group. Setpgid at launch made the leader's
	// pid the group id.
	if err := syscall.Kill(-pid, sig); err != nil {
		if err == syscall.ESRCH {
			return os.ErrProcessDone
		}
		return err
	}
	return nil
}

// processAlive reports whether a pid belongs to a process that is still doing
// something.
//
// A zombie is not. It has exited and is only waiting to be reaped, but
// kill(pid, 0) still succeeds on it — so a naive check reports a dead engine as
// running forever. That happens for real whenever an engine's parent dies
// before it does and nothing reaps the orphan: in a container, PID 1 is usually
// the workload rather than an init, and orphans simply accumulate.
func processAlive(pid int) bool {
	if pid <= 0 {
		return false
	}
	// Signal 0 performs the permission and existence checks without delivering
	// anything.
	if err := syscall.Kill(pid, 0); err != nil {
		// EPERM means it exists but belongs to someone else, which for our
		// purposes is still "there".
		return err == syscall.EPERM
	}
	return !isZombie(pid)
}

// isZombie reports whether a pid has exited but not been reaped.
//
// Only Linux exposes this; elsewhere there is no procfs and the answer is
// false, which degrades to the old behaviour rather than to something worse.
func isZombie(pid int) bool {
	b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat")
	if err != nil {
		return false
	}
	// The comm field is parenthesised and may itself contain spaces or
	// parentheses, so the state character is found from the last ')'.
	i := bytes.LastIndexByte(b, ')')
	if i < 0 || i+2 >= len(b) {
		return false
	}
	return b[i+2] == 'Z'
}

// strays are a process's descendants that left its group, as an engine that
// starts its worker in a session of its own does (Strata): signalling the
// group misses them, and one left behind holds the card and the RAM the next
// replica needs. Found before the group is signalled, while they are still
// its descendants: after, they belong to init. Linux only; elsewhere none.
func strays(pid int) []int {
	entries, err := os.ReadDir("/proc")
	if err != nil {
		return nil
	}
	parent := map[int]int{}
	group := map[int]int{}
	for _, e := range entries {
		p, err := strconv.Atoi(e.Name())
		if err != nil {
			continue
		}
		if pp, pg, ok := parentAndGroup(p); ok {
			parent[p], group[p] = pp, pg
		}
	}
	var out []int
	for p := range parent {
		if group[p] == pid || p == pid {
			continue
		}
		// Bounded: no engine's tree is this deep, and a pid reused while
		// /proc was read must not loop.
		for q, n := parent[p], 0; q > 1 && n < 32; q, n = parent[q], n+1 {
			if q == pid {
				out = append(out, p)
				break
			}
		}
	}
	return out
}

// parentAndGroup reads a process's parent and group from /proc.
func parentAndGroup(pid int) (int, int, bool) {
	b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat")
	if err != nil {
		return 0, 0, false
	}
	i := bytes.LastIndexByte(b, ')')
	if i < 0 {
		return 0, 0, false
	}
	f := strings.Fields(string(b[i+1:]))
	if len(f) < 3 {
		return 0, 0, false
	}
	pp, err1 := strconv.Atoi(f[1])
	pg, err2 := strconv.Atoi(f[2])
	return pp, pg, err1 == nil && err2 == nil
}

// signalStrays signals each stray, and its own group when it leads one.
func signalStrays(pids []int, sig syscall.Signal) {
	for _, p := range pids {
		if _, pg, ok := parentAndGroup(p); ok && pg == p {
			_ = syscall.Kill(-p, sig)
			continue
		}
		_ = syscall.Kill(p, sig)
	}
}