package supervisor import ( "bytes" "os" "strconv" "strings" "syscall" ) // killGroup signals a whole process group. // // Engines fork worker processes that hold GPU memory. Signalling only the // leader leaves those workers alive, and the next launch then fails to // allocate — with an error that points at the new process rather than the old. func killGroup(pid int, sig syscall.Signal) error { // The negative pid addresses the group. Setpgid at launch made the leader's // pid the group id. if err := syscall.Kill(-pid, sig); err != nil { if err == syscall.ESRCH { return os.ErrProcessDone } return err } return nil } // processAlive reports whether a pid belongs to a process that is still doing // something. // // A zombie is not. It has exited and is only waiting to be reaped, but // kill(pid, 0) still succeeds on it — so a naive check reports a dead engine as // running forever. That happens for real whenever an engine's parent dies // before it does and nothing reaps the orphan: in a container, PID 1 is usually // the workload rather than an init, and orphans simply accumulate. func processAlive(pid int) bool { if pid <= 0 { return false } // Signal 0 performs the permission and existence checks without delivering // anything. if err := syscall.Kill(pid, 0); err != nil { // EPERM means it exists but belongs to someone else, which for our // purposes is still "there". return err == syscall.EPERM } return !isZombie(pid) } // isZombie reports whether a pid has exited but not been reaped. // // Only Linux exposes this; elsewhere there is no procfs and the answer is // false, which degrades to the old behaviour rather than to something worse. func isZombie(pid int) bool { b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat") if err != nil { return false } // The comm field is parenthesised and may itself contain spaces or // parentheses, so the state character is found from the last ')'. i := bytes.LastIndexByte(b, ')') if i < 0 || i+2 >= len(b) { return false } return b[i+2] == 'Z' } // strays are a process's descendants that left its group, as an engine that // starts its worker in a session of its own does (Strata): signalling the // group misses them, and one left behind holds the card and the RAM the next // replica needs. Found before the group is signalled, while they are still // its descendants: after, they belong to init. Linux only; elsewhere none. func strays(pid int) []int { entries, err := os.ReadDir("/proc") if err != nil { return nil } parent := map[int]int{} group := map[int]int{} for _, e := range entries { p, err := strconv.Atoi(e.Name()) if err != nil { continue } if pp, pg, ok := parentAndGroup(p); ok { parent[p], group[p] = pp, pg } } var out []int for p := range parent { if group[p] == pid || p == pid { continue } // Bounded: no engine's tree is this deep, and a pid reused while // /proc was read must not loop. for q, n := parent[p], 0; q > 1 && n < 32; q, n = parent[q], n+1 { if q == pid { out = append(out, p) break } } } return out } // parentAndGroup reads a process's parent and group from /proc. func parentAndGroup(pid int) (int, int, bool) { b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat") if err != nil { return 0, 0, false } i := bytes.LastIndexByte(b, ')') if i < 0 { return 0, 0, false } f := strings.Fields(string(b[i+1:])) if len(f) < 3 { return 0, 0, false } pp, err1 := strconv.Atoi(f[1]) pg, err2 := strconv.Atoi(f[2]) return pp, pg, err1 == nil && err2 == nil } // signalStrays signals each stray, and its own group when it leads one. func signalStrays(pids []int, sig syscall.Signal) { for _, p := range pids { if _, pg, ok := parentAndGroup(p); ok && pg == p { _ = syscall.Kill(-p, sig) continue } _ = syscall.Kill(p, sig) } }