CI / test (push) Successful in 1m44s
Not persisting replica state left one hole: restarting the control plane made it forget every replica while the engines kept running. They held GPUs the inventory believed were free, so the next placement failed to allocate, with an error naming the new process rather than the old one. The fix belongs on the agent, not in a database. A row can say "running" about a process that died; the fleet cannot. - the supervisor writes the full shard spec, not just a pid and command line. A pid identifies a process; the spec is what lets it be described back to the control plane and matched to a replica. - Adopt takes over an engine a previous agent left running. It is not this process's child, so it cannot be waited on: it is watched by polling and stopped by signalling its group directly. Its earlier output is gone and the log says so rather than presenting emptiness as silence. - every Register reports what the agent is running, not only the first, because a reconnect may be a control plane restart and has to recover the same way. - the reconciler rebuilds replicas from those reports and reclaims their GPUs. Anything whose model or profile left the catalog is stopped; nothing will route to it and it is holding cards. - an engine the agent cannot identify is killed. A process holding GPUs that nothing can describe is worse than no process, because placement hands those GPUs out again. Stop was also wrong for adopted shards: it checked for an exec.Cmd, which they do not have, and reported them stopped without killing anything. TestAgentReconnects had been passing for no reason. httptest's CloseClientConnections does nothing to an h2c stream — prior-knowledge h2c hijacks the connection and httptest stops tracking hijacked ones — so the agent never reconnected, and the assertion on the connection map passed on a stale entry. Tests now sever connections through their own listener and count opened streams. Plan §11 also records what multi-node still needs, since placement landing early makes it look further along than it is: workers never become Ready because the agent health-checks every shard while only rank 0 serves HTTP; RayCommand is written and tested but never called; and startup is fire-and-forget. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
631 lines
16 KiB
Go
631 lines
16 KiB
Go
package supervisor
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
|
)
|
|
|
|
// Config configures the supervisor.
|
|
type Config struct {
|
|
// WorkDir holds pid files and per-shard scratch space.
|
|
WorkDir string `koanf:"work_dir"`
|
|
// LogLines is how many lines of engine output to retain per shard.
|
|
LogLines int `koanf:"log_lines"`
|
|
// StopTimeout is how long a process gets to exit after SIGTERM before it is
|
|
// killed. Engines need time to free GPU memory cleanly.
|
|
StopTimeout time.Duration `koanf:"stop_timeout"`
|
|
// HealthInterval is how often a starting shard is polled.
|
|
HealthInterval time.Duration `koanf:"health_interval"`
|
|
}
|
|
|
|
// DefaultConfig returns sensible defaults.
|
|
func DefaultConfig() Config {
|
|
return Config{
|
|
WorkDir: "/var/lib/llmgw-agent",
|
|
LogLines: 512,
|
|
StopTimeout: 60 * time.Second,
|
|
HealthInterval: 2 * time.Second,
|
|
}
|
|
}
|
|
|
|
func (c *Config) normalize() {
|
|
d := DefaultConfig()
|
|
if c.WorkDir == "" {
|
|
c.WorkDir = d.WorkDir
|
|
}
|
|
if c.LogLines <= 0 {
|
|
c.LogLines = d.LogLines
|
|
}
|
|
if c.StopTimeout <= 0 {
|
|
c.StopTimeout = d.StopTimeout
|
|
}
|
|
if c.HealthInterval <= 0 {
|
|
c.HealthInterval = d.HealthInterval
|
|
}
|
|
}
|
|
|
|
// HealthChecker reports whether a shard is serving. It is an interface so the
|
|
// supervisor can be tested without an HTTP engine.
|
|
type HealthChecker interface {
|
|
// Ready reports whether the shard at baseURL serves model.
|
|
Ready(ctx context.Context, baseURL, model string) error
|
|
}
|
|
|
|
// Shard is one supervised engine process.
|
|
type Shard struct {
|
|
ID string
|
|
Spec engines.ShardSpec
|
|
|
|
mu sync.Mutex
|
|
state State
|
|
pid int
|
|
err error
|
|
started time.Time
|
|
readyAt time.Time
|
|
exitCode int
|
|
|
|
logs *ringBuffer
|
|
cmd *exec.Cmd
|
|
cancel context.CancelFunc
|
|
done chan struct{}
|
|
// adopted marks a shard inherited from a previous agent: it has no
|
|
// exec.Cmd, so it is watched by polling and killed by signal alone.
|
|
adopted bool
|
|
}
|
|
|
|
// Status is a snapshot of a shard, safe to send to the control plane.
|
|
type Status struct {
|
|
ID string
|
|
State State
|
|
PID int
|
|
Err string
|
|
Started time.Time
|
|
ReadyAt time.Time
|
|
ExitCode int
|
|
}
|
|
|
|
// Status returns the shard's current status.
|
|
func (s *Shard) Status() Status {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
st := Status{
|
|
ID: s.ID, State: s.state, PID: s.pid,
|
|
Started: s.started, ReadyAt: s.readyAt, ExitCode: s.exitCode,
|
|
}
|
|
if s.err != nil {
|
|
st.Err = s.err.Error()
|
|
}
|
|
return st
|
|
}
|
|
|
|
// Logs returns the last n lines of engine output.
|
|
func (s *Shard) Logs(n int) []string { return s.logs.tail(n) }
|
|
|
|
func (s *Shard) setState(st State, err error) {
|
|
s.mu.Lock()
|
|
s.state = st
|
|
if err != nil {
|
|
s.err = err
|
|
}
|
|
if st == Ready && s.readyAt.IsZero() {
|
|
s.readyAt = time.Now()
|
|
}
|
|
s.mu.Unlock()
|
|
}
|
|
|
|
// Supervisor starts, watches and stops engine processes on this node.
|
|
type Supervisor struct {
|
|
cfg Config
|
|
log *slog.Logger
|
|
health HealthChecker
|
|
|
|
mu sync.Mutex
|
|
shards map[string]*Shard
|
|
}
|
|
|
|
// New builds a supervisor.
|
|
func New(cfg Config, health HealthChecker, log *slog.Logger) (*Supervisor, error) {
|
|
cfg.normalize()
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
if err := os.MkdirAll(cfg.WorkDir, 0o750); err != nil {
|
|
return nil, fmt.Errorf("work dir: %w", err)
|
|
}
|
|
return &Supervisor{cfg: cfg, log: log, health: health, shards: map[string]*Shard{}}, nil
|
|
}
|
|
|
|
// ErrAlreadyRunning means a shard with that ID is already supervised.
|
|
var ErrAlreadyRunning = errors.New("shard is already running")
|
|
|
|
// Start launches a shard.
|
|
//
|
|
// It is idempotent by ID: a repeated StartShard for the same shard is a no-op
|
|
// rather than a second process, because the control plane retries commands
|
|
// (plan §5.6).
|
|
func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpec, cmd engines.Command) (*Shard, error) {
|
|
s.mu.Lock()
|
|
if existing, ok := s.shards[id]; ok {
|
|
s.mu.Unlock()
|
|
if existing.Status().State.Terminal() {
|
|
s.Remove(id)
|
|
} else {
|
|
return existing, ErrAlreadyRunning
|
|
}
|
|
s.mu.Lock()
|
|
}
|
|
|
|
sh := &Shard{
|
|
ID: id,
|
|
Spec: spec,
|
|
state: Pending,
|
|
logs: newRingBuffer(s.cfg.LogLines),
|
|
started: time.Now(),
|
|
done: make(chan struct{}),
|
|
}
|
|
s.shards[id] = sh
|
|
s.mu.Unlock()
|
|
|
|
if err := s.launch(ctx, sh, cmd); err != nil {
|
|
sh.setState(Failed, err)
|
|
close(sh.done)
|
|
return sh, err
|
|
}
|
|
return sh, nil
|
|
}
|
|
|
|
func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmd engines.Command) error {
|
|
runCtx, cancel := context.WithCancel(context.WithoutCancel(ctx))
|
|
sh.cancel = cancel
|
|
|
|
c := exec.CommandContext(runCtx, cmd.Path, cmd.Args...)
|
|
c.Env = cmd.Env
|
|
c.Dir = s.cfg.WorkDir
|
|
// Its own process group, so stopping kills the engine's children too.
|
|
// Engines spawn workers; signalling only the parent leaves those workers
|
|
// holding GPU memory.
|
|
c.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
|
// CommandContext kills only the leader, so replace that with a group kill.
|
|
c.Cancel = func() error { return killGroup(c.Process.Pid, syscall.SIGKILL) }
|
|
|
|
stdout, err := c.StdoutPipe()
|
|
if err != nil {
|
|
cancel()
|
|
return err
|
|
}
|
|
c.Stderr = c.Stdout.(io.Writer) // interleave, as the engine writes them
|
|
|
|
if err := c.Start(); err != nil {
|
|
cancel()
|
|
return fmt.Errorf("starting %s: %w", cmd.Path, err)
|
|
}
|
|
|
|
sh.mu.Lock()
|
|
sh.cmd = c
|
|
sh.pid = c.Process.Pid
|
|
sh.state = Starting
|
|
sh.mu.Unlock()
|
|
|
|
if err := s.writeState(sh); err != nil {
|
|
s.log.Warn("could not write pid file; adoption after a restart will not work",
|
|
"shard", sh.ID, "err", err)
|
|
}
|
|
|
|
go s.pump(sh, stdout)
|
|
go s.wait(sh)
|
|
return nil
|
|
}
|
|
|
|
// pump copies engine output into the ring buffer and the log.
|
|
func (s *Supervisor) pump(sh *Shard, r io.Reader) {
|
|
sc := bufio.NewScanner(r)
|
|
// Engine tracebacks and progress bars produce long lines; the default
|
|
// 64 KiB limit truncates them exactly when they matter.
|
|
sc.Buffer(make([]byte, 0, 64<<10), 1<<20)
|
|
|
|
for sc.Scan() {
|
|
line := sc.Text()
|
|
sh.logs.add(line)
|
|
s.log.Debug("engine", "shard", sh.ID, "line", line)
|
|
}
|
|
}
|
|
|
|
// wait reaps the process and records why it ended.
|
|
func (s *Supervisor) wait(sh *Shard) {
|
|
defer close(sh.done)
|
|
|
|
err := sh.cmd.Wait()
|
|
|
|
sh.mu.Lock()
|
|
prev := sh.state
|
|
if ee := new(exec.ExitError); errors.As(err, &ee) {
|
|
sh.exitCode = ee.ExitCode()
|
|
}
|
|
switch {
|
|
case prev == Draining || prev == Stopped:
|
|
// Expected: we asked it to stop.
|
|
sh.state = Stopped
|
|
case err == nil:
|
|
// An engine exiting on its own is a failure, not a success: it is
|
|
// supposed to serve until told otherwise.
|
|
sh.state = Failed
|
|
sh.err = errors.New("engine exited unexpectedly")
|
|
default:
|
|
sh.state = Failed
|
|
sh.err = err
|
|
}
|
|
final, code := sh.state, sh.exitCode
|
|
sh.mu.Unlock()
|
|
|
|
s.removeState(sh)
|
|
if final == Failed {
|
|
s.log.Error("shard failed", "shard", sh.ID, "exit_code", code, "err", err,
|
|
"last_output", strings.Join(sh.Logs(5), " | "))
|
|
} else {
|
|
s.log.Info("shard stopped", "shard", sh.ID, "exit_code", code)
|
|
}
|
|
}
|
|
|
|
// WaitReady polls until the shard serves, it fails, or the timeout expires.
|
|
//
|
|
// The timeout comes from the profile because it is model-dependent: loading a
|
|
// 70B model across two nodes legitimately takes tens of minutes, while a small
|
|
// model failing to load for ten minutes means something is wrong.
|
|
func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model string, timeout time.Duration) error {
|
|
ctx, cancel := context.WithTimeout(ctx, timeout)
|
|
defer cancel()
|
|
|
|
t := time.NewTicker(s.cfg.HealthInterval)
|
|
defer t.Stop()
|
|
|
|
sh.setState(Loading, nil)
|
|
for {
|
|
select {
|
|
case <-sh.done:
|
|
return fmt.Errorf("shard %s exited during startup: %s", sh.ID, sh.Status().Err)
|
|
case <-ctx.Done():
|
|
last := strings.Join(sh.Logs(10), "\n")
|
|
return fmt.Errorf("shard %s not ready within %v; last output:\n%s", sh.ID, timeout, last)
|
|
case <-t.C:
|
|
if err := s.health.Ready(ctx, baseURL, model); err != nil {
|
|
continue
|
|
}
|
|
sh.setState(Ready, nil)
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
|
|
// Drain marks a shard as taking no new work. The router stops routing to it;
|
|
// the process keeps running until Stop.
|
|
func (s *Supervisor) Drain(id string) error {
|
|
sh, err := s.Get(id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if st := sh.Status().State; !st.Live() {
|
|
return fmt.Errorf("shard %s is %s, cannot drain", id, st)
|
|
}
|
|
sh.setState(Draining, nil)
|
|
return nil
|
|
}
|
|
|
|
// Stop terminates a shard: SIGTERM to the whole group, then SIGKILL after
|
|
// StopTimeout.
|
|
func (s *Supervisor) Stop(ctx context.Context, id string) error {
|
|
sh, err := s.Get(id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
sh.mu.Lock()
|
|
pid := sh.pid
|
|
if sh.state.Live() {
|
|
sh.state = Draining // marks the exit as expected for wait() and poll()
|
|
}
|
|
sh.mu.Unlock()
|
|
|
|
// A pid of zero means the process never started; an adopted shard has a pid
|
|
// but no exec.Cmd, and must still be killed by signal.
|
|
if pid == 0 {
|
|
sh.setState(Stopped, nil)
|
|
return nil
|
|
}
|
|
|
|
// Signal the group, not the process: engines fork workers that hold GPU
|
|
// memory, and those outlive a signal sent only to the leader.
|
|
if err := killGroup(pid, syscall.SIGTERM); err != nil && !errors.Is(err, os.ErrProcessDone) {
|
|
s.log.Warn("SIGTERM failed", "shard", id, "err", err)
|
|
}
|
|
|
|
// force kills the group outright. exec.Cmd's cancel does this for shards we
|
|
// started; an adopted shard has none, so signal its group directly.
|
|
force := func() {
|
|
if sh.cancel != nil {
|
|
sh.cancel()
|
|
return
|
|
}
|
|
if err := killGroup(pid, syscall.SIGKILL); err != nil && !errors.Is(err, os.ErrProcessDone) {
|
|
s.log.Warn("SIGKILL failed", "shard", id, "err", err)
|
|
}
|
|
}
|
|
|
|
select {
|
|
case <-sh.done:
|
|
return nil
|
|
case <-time.After(s.cfg.StopTimeout):
|
|
s.log.Warn("shard did not exit in time, killing", "shard", id, "timeout", s.cfg.StopTimeout)
|
|
force()
|
|
case <-ctx.Done():
|
|
force()
|
|
return ctx.Err()
|
|
}
|
|
|
|
select {
|
|
case <-sh.done:
|
|
case <-time.After(5 * time.Second):
|
|
return fmt.Errorf("shard %s survived SIGKILL", id)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// StopAll stops every live shard.
|
|
func (s *Supervisor) StopAll(ctx context.Context) {
|
|
for _, id := range s.IDs() {
|
|
if err := s.Stop(ctx, id); err != nil {
|
|
s.log.Warn("stopping shard failed", "shard", id, "err", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Get returns a supervised shard.
|
|
func (s *Supervisor) Get(id string) (*Shard, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
sh, ok := s.shards[id]
|
|
if !ok {
|
|
return nil, fmt.Errorf("unknown shard %q", id)
|
|
}
|
|
return sh, nil
|
|
}
|
|
|
|
// Remove forgets a terminated shard.
|
|
func (s *Supervisor) Remove(id string) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
delete(s.shards, id)
|
|
}
|
|
|
|
// IDs lists the supervised shards.
|
|
func (s *Supervisor) IDs() []string {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
out := make([]string, 0, len(s.shards))
|
|
for id := range s.shards {
|
|
out = append(out, id)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Statuses returns every shard's status.
|
|
func (s *Supervisor) Statuses() []Status {
|
|
s.mu.Lock()
|
|
shards := make([]*Shard, 0, len(s.shards))
|
|
for _, sh := range s.shards {
|
|
shards = append(shards, sh)
|
|
}
|
|
s.mu.Unlock()
|
|
|
|
out := make([]Status, 0, len(shards))
|
|
for _, sh := range shards {
|
|
out = append(out, sh.Status())
|
|
}
|
|
return out
|
|
}
|
|
|
|
// --- state files, for adoption after an agent restart ---
|
|
|
|
func (s *Supervisor) statePath(id string) string {
|
|
return filepath.Join(s.cfg.WorkDir, "shard-"+id+".json")
|
|
}
|
|
|
|
// shardState is what survives an agent restart.
|
|
//
|
|
// The pid alone is not enough for two separate reasons. Pids are reused, so the
|
|
// command line is kept as a fingerprint — adopting an unrelated process would
|
|
// be worse than leaking the original. And the full spec is kept because a
|
|
// recovered shard has to be described back to the control plane: without it the
|
|
// control plane cannot tell which replica the process belongs to, and would
|
|
// place a duplicate while the original keeps holding the GPUs (plan §6.5).
|
|
type shardState struct {
|
|
ShardID string `json:"shard_id"`
|
|
ReplicaID string `json:"replica_id"`
|
|
PID int `json:"pid"`
|
|
Cmdline []string `json:"cmdline"`
|
|
StartedAt time.Time `json:"started_at"`
|
|
Spec engines.ShardSpec `json:"spec"`
|
|
}
|
|
|
|
// writeState records everything needed to recognise and describe the process later.
|
|
func (s *Supervisor) writeState(sh *Shard) error {
|
|
b, err := json.Marshal(shardState{
|
|
ShardID: sh.ID,
|
|
ReplicaID: sh.Spec.ReplicaID,
|
|
PID: sh.pid,
|
|
Cmdline: sh.cmd.Args,
|
|
StartedAt: sh.started,
|
|
Spec: sh.Spec,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Written atomically: a half-written state file after a crash would be
|
|
// indistinguishable from a corrupt one, and the shard would be killed.
|
|
tmp := s.statePath(sh.ID) + ".tmp"
|
|
if err := os.WriteFile(tmp, b, 0o600); err != nil {
|
|
return err
|
|
}
|
|
return os.Rename(tmp, s.statePath(sh.ID))
|
|
}
|
|
|
|
func (s *Supervisor) removeState(sh *Shard) {
|
|
_ = os.Remove(s.statePath(sh.ID))
|
|
}
|
|
|
|
// Orphan is an engine process left behind by a previous agent.
|
|
type Orphan struct {
|
|
ShardID string
|
|
ReplicaID string
|
|
PID int
|
|
Cmdline []string
|
|
Spec engines.ShardSpec
|
|
StartedAt time.Time
|
|
// Alive reports whether the process still exists.
|
|
Alive bool
|
|
}
|
|
|
|
// FindOrphans scans the work directory for processes a previous agent started.
|
|
func (s *Supervisor) FindOrphans() ([]Orphan, error) {
|
|
entries, err := os.ReadDir(s.cfg.WorkDir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var out []Orphan
|
|
for _, e := range entries {
|
|
name := e.Name()
|
|
if !strings.HasPrefix(name, "shard-") || !strings.HasSuffix(name, ".json") {
|
|
continue
|
|
}
|
|
|
|
b, err := os.ReadFile(filepath.Join(s.cfg.WorkDir, name))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
var st shardState
|
|
if err := json.Unmarshal(b, &st); err != nil || st.ShardID == "" || st.PID <= 0 {
|
|
// A corrupt state file describes a process that cannot be
|
|
// identified, so it cannot be adopted safely. Leave it for
|
|
// KillOrphan to clean up by shard id from the filename.
|
|
id := strings.TrimSuffix(strings.TrimPrefix(name, "shard-"), ".json")
|
|
out = append(out, Orphan{ShardID: id})
|
|
continue
|
|
}
|
|
|
|
out = append(out, Orphan{
|
|
ShardID: st.ShardID,
|
|
ReplicaID: st.ReplicaID,
|
|
PID: st.PID,
|
|
Cmdline: st.Cmdline,
|
|
Spec: st.Spec,
|
|
StartedAt: st.StartedAt,
|
|
Alive: processAlive(st.PID),
|
|
})
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Adopt takes over a still-running engine from a previous agent.
|
|
//
|
|
// The process cannot be waited on — it is not this process's child — so it is
|
|
// watched by polling instead. That is the whole difference between an adopted
|
|
// shard and a started one; everything downstream treats them the same.
|
|
func (s *Supervisor) Adopt(o Orphan) (*Shard, error) {
|
|
if !o.Alive {
|
|
return nil, fmt.Errorf("shard %s: process %d is gone", o.ShardID, o.PID)
|
|
}
|
|
|
|
s.mu.Lock()
|
|
if _, exists := s.shards[o.ShardID]; exists {
|
|
s.mu.Unlock()
|
|
return nil, ErrAlreadyRunning
|
|
}
|
|
sh := &Shard{
|
|
ID: o.ShardID,
|
|
Spec: o.Spec,
|
|
state: Starting,
|
|
pid: o.PID,
|
|
logs: newRingBuffer(s.cfg.LogLines),
|
|
started: o.StartedAt,
|
|
done: make(chan struct{}),
|
|
adopted: true,
|
|
}
|
|
s.shards[o.ShardID] = sh
|
|
s.mu.Unlock()
|
|
|
|
// Output is gone with the previous agent; say so rather than presenting an
|
|
// empty log as if the engine had been silent.
|
|
sh.logs.add("[adopted from a previous agent; earlier output is not available]")
|
|
|
|
go s.poll(sh)
|
|
return sh, nil
|
|
}
|
|
|
|
// poll watches an adopted process, which cannot be reaped with Wait.
|
|
func (s *Supervisor) poll(sh *Shard) {
|
|
defer close(sh.done)
|
|
|
|
// Polled rather than waited on, so the interval bounds how quickly a stop
|
|
// is observed. Short enough that Stop does not sit out its whole timeout.
|
|
t := time.NewTicker(200 * time.Millisecond)
|
|
defer t.Stop()
|
|
for range t.C {
|
|
if processAlive(sh.pid) {
|
|
continue
|
|
}
|
|
sh.mu.Lock()
|
|
expected := sh.state == Draining || sh.state == Stopped
|
|
if expected {
|
|
sh.state = Stopped
|
|
} else {
|
|
sh.state = Failed
|
|
sh.err = errors.New("adopted engine exited")
|
|
}
|
|
sh.mu.Unlock()
|
|
|
|
s.removeState(sh)
|
|
s.log.Info("adopted shard exited", "shard", sh.ID, "expected", expected)
|
|
return
|
|
}
|
|
}
|
|
|
|
// KillOrphan terminates a leftover process group and removes its pid file.
|
|
func (s *Supervisor) KillOrphan(o Orphan) error {
|
|
if o.Alive {
|
|
if err := killGroup(o.PID, syscall.SIGTERM); err != nil {
|
|
return err
|
|
}
|
|
// Give it a moment to free GPU memory before insisting.
|
|
deadline := time.Now().Add(s.cfg.StopTimeout)
|
|
for time.Now().Before(deadline) {
|
|
if !processAlive(o.PID) {
|
|
break
|
|
}
|
|
time.Sleep(200 * time.Millisecond)
|
|
}
|
|
if processAlive(o.PID) {
|
|
_ = killGroup(o.PID, syscall.SIGKILL)
|
|
}
|
|
}
|
|
// An already-missing state file is success: the previous agent may have
|
|
// reaped the process itself between the scan and this call.
|
|
if err := os.Remove(s.statePath(o.ShardID)); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|