package supervisor import ( "bufio" "context" "errors" "fmt" "io" "log/slog" "os" "os/exec" "path/filepath" "strconv" "strings" "sync" "syscall" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" ) // Config configures the supervisor. type Config struct { // WorkDir holds pid files and per-shard scratch space. WorkDir string `koanf:"work_dir"` // LogLines is how many lines of engine output to retain per shard. LogLines int `koanf:"log_lines"` // StopTimeout is how long a process gets to exit after SIGTERM before it is // killed. Engines need time to free GPU memory cleanly. StopTimeout time.Duration `koanf:"stop_timeout"` // HealthInterval is how often a starting shard is polled. HealthInterval time.Duration `koanf:"health_interval"` } // DefaultConfig returns sensible defaults. func DefaultConfig() Config { return Config{ WorkDir: "/var/lib/llmgw-agent", LogLines: 512, StopTimeout: 60 * time.Second, HealthInterval: 2 * time.Second, } } func (c *Config) normalize() { d := DefaultConfig() if c.WorkDir == "" { c.WorkDir = d.WorkDir } if c.LogLines <= 0 { c.LogLines = d.LogLines } if c.StopTimeout <= 0 { c.StopTimeout = d.StopTimeout } if c.HealthInterval <= 0 { c.HealthInterval = d.HealthInterval } } // HealthChecker reports whether a shard is serving. It is an interface so the // supervisor can be tested without an HTTP engine. type HealthChecker interface { // Ready reports whether the shard at baseURL serves model. Ready(ctx context.Context, baseURL, model string) error } // Shard is one supervised engine process. type Shard struct { ID string Spec engines.ShardSpec mu sync.Mutex state State pid int err error started time.Time readyAt time.Time exitCode int logs *ringBuffer cmd *exec.Cmd cancel context.CancelFunc done chan struct{} } // Status is a snapshot of a shard, safe to send to the control plane. type Status struct { ID string State State PID int Err string Started time.Time ReadyAt time.Time ExitCode int } // Status returns the shard's current status. func (s *Shard) Status() Status { s.mu.Lock() defer s.mu.Unlock() st := Status{ ID: s.ID, State: s.state, PID: s.pid, Started: s.started, ReadyAt: s.readyAt, ExitCode: s.exitCode, } if s.err != nil { st.Err = s.err.Error() } return st } // Logs returns the last n lines of engine output. func (s *Shard) Logs(n int) []string { return s.logs.tail(n) } func (s *Shard) setState(st State, err error) { s.mu.Lock() s.state = st if err != nil { s.err = err } if st == Ready && s.readyAt.IsZero() { s.readyAt = time.Now() } s.mu.Unlock() } // Supervisor starts, watches and stops engine processes on this node. type Supervisor struct { cfg Config log *slog.Logger health HealthChecker mu sync.Mutex shards map[string]*Shard } // New builds a supervisor. func New(cfg Config, health HealthChecker, log *slog.Logger) (*Supervisor, error) { cfg.normalize() if log == nil { log = slog.Default() } if err := os.MkdirAll(cfg.WorkDir, 0o750); err != nil { return nil, fmt.Errorf("work dir: %w", err) } return &Supervisor{cfg: cfg, log: log, health: health, shards: map[string]*Shard{}}, nil } // ErrAlreadyRunning means a shard with that ID is already supervised. var ErrAlreadyRunning = errors.New("shard is already running") // Start launches a shard. // // It is idempotent by ID: a repeated StartShard for the same shard is a no-op // rather than a second process, because the control plane retries commands // (plan §5.6). func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpec, cmd engines.Command) (*Shard, error) { s.mu.Lock() if existing, ok := s.shards[id]; ok { s.mu.Unlock() if existing.Status().State.Terminal() { s.Remove(id) } else { return existing, ErrAlreadyRunning } s.mu.Lock() } sh := &Shard{ ID: id, Spec: spec, state: Pending, logs: newRingBuffer(s.cfg.LogLines), started: time.Now(), done: make(chan struct{}), } s.shards[id] = sh s.mu.Unlock() if err := s.launch(ctx, sh, cmd); err != nil { sh.setState(Failed, err) close(sh.done) return sh, err } return sh, nil } func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmd engines.Command) error { runCtx, cancel := context.WithCancel(context.WithoutCancel(ctx)) sh.cancel = cancel c := exec.CommandContext(runCtx, cmd.Path, cmd.Args...) c.Env = cmd.Env c.Dir = s.cfg.WorkDir // Its own process group, so stopping kills the engine's children too. // Engines spawn workers; signalling only the parent leaves those workers // holding GPU memory. c.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} // CommandContext kills only the leader, so replace that with a group kill. c.Cancel = func() error { return killGroup(c.Process.Pid, syscall.SIGKILL) } stdout, err := c.StdoutPipe() if err != nil { cancel() return err } c.Stderr = c.Stdout.(io.Writer) // interleave, as the engine writes them if err := c.Start(); err != nil { cancel() return fmt.Errorf("starting %s: %w", cmd.Path, err) } sh.mu.Lock() sh.cmd = c sh.pid = c.Process.Pid sh.state = Starting sh.mu.Unlock() if err := s.writePIDFile(sh); err != nil { s.log.Warn("could not write pid file; adoption after a restart will not work", "shard", sh.ID, "err", err) } go s.pump(sh, stdout) go s.wait(sh) return nil } // pump copies engine output into the ring buffer and the log. func (s *Supervisor) pump(sh *Shard, r io.Reader) { sc := bufio.NewScanner(r) // Engine tracebacks and progress bars produce long lines; the default // 64 KiB limit truncates them exactly when they matter. sc.Buffer(make([]byte, 0, 64<<10), 1<<20) for sc.Scan() { line := sc.Text() sh.logs.add(line) s.log.Debug("engine", "shard", sh.ID, "line", line) } } // wait reaps the process and records why it ended. func (s *Supervisor) wait(sh *Shard) { defer close(sh.done) err := sh.cmd.Wait() sh.mu.Lock() prev := sh.state if ee := new(exec.ExitError); errors.As(err, &ee) { sh.exitCode = ee.ExitCode() } switch { case prev == Draining || prev == Stopped: // Expected: we asked it to stop. sh.state = Stopped case err == nil: // An engine exiting on its own is a failure, not a success: it is // supposed to serve until told otherwise. sh.state = Failed sh.err = errors.New("engine exited unexpectedly") default: sh.state = Failed sh.err = err } final, code := sh.state, sh.exitCode sh.mu.Unlock() s.removePIDFile(sh) if final == Failed { s.log.Error("shard failed", "shard", sh.ID, "exit_code", code, "err", err, "last_output", strings.Join(sh.Logs(5), " | ")) } else { s.log.Info("shard stopped", "shard", sh.ID, "exit_code", code) } } // WaitReady polls until the shard serves, it fails, or the timeout expires. // // The timeout comes from the profile because it is model-dependent: loading a // 70B model across two nodes legitimately takes tens of minutes, while a small // model failing to load for ten minutes means something is wrong. func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model string, timeout time.Duration) error { ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() t := time.NewTicker(s.cfg.HealthInterval) defer t.Stop() sh.setState(Loading, nil) for { select { case <-sh.done: return fmt.Errorf("shard %s exited during startup: %s", sh.ID, sh.Status().Err) case <-ctx.Done(): last := strings.Join(sh.Logs(10), "\n") return fmt.Errorf("shard %s not ready within %v; last output:\n%s", sh.ID, timeout, last) case <-t.C: if err := s.health.Ready(ctx, baseURL, model); err != nil { continue } sh.setState(Ready, nil) return nil } } } // Drain marks a shard as taking no new work. The router stops routing to it; // the process keeps running until Stop. func (s *Supervisor) Drain(id string) error { sh, err := s.Get(id) if err != nil { return err } if st := sh.Status().State; !st.Live() { return fmt.Errorf("shard %s is %s, cannot drain", id, st) } sh.setState(Draining, nil) return nil } // Stop terminates a shard: SIGTERM to the whole group, then SIGKILL after // StopTimeout. func (s *Supervisor) Stop(ctx context.Context, id string) error { sh, err := s.Get(id) if err != nil { return err } sh.mu.Lock() c, pid := sh.cmd, sh.pid if sh.state.Live() { sh.state = Draining // marks the exit as expected for wait() } sh.mu.Unlock() if c == nil || pid == 0 { sh.setState(Stopped, nil) return nil } // Signal the group, not the process: engines fork workers that hold GPU // memory, and those outlive a signal sent only to the leader. if err := killGroup(pid, syscall.SIGTERM); err != nil && !errors.Is(err, os.ErrProcessDone) { s.log.Warn("SIGTERM failed", "shard", id, "err", err) } select { case <-sh.done: return nil case <-time.After(s.cfg.StopTimeout): s.log.Warn("shard did not exit in time, killing", "shard", id, "timeout", s.cfg.StopTimeout) if sh.cancel != nil { sh.cancel() } case <-ctx.Done(): if sh.cancel != nil { sh.cancel() } return ctx.Err() } select { case <-sh.done: case <-time.After(5 * time.Second): return fmt.Errorf("shard %s survived SIGKILL", id) } return nil } // StopAll stops every live shard. func (s *Supervisor) StopAll(ctx context.Context) { for _, id := range s.IDs() { if err := s.Stop(ctx, id); err != nil { s.log.Warn("stopping shard failed", "shard", id, "err", err) } } } // Get returns a supervised shard. func (s *Supervisor) Get(id string) (*Shard, error) { s.mu.Lock() defer s.mu.Unlock() sh, ok := s.shards[id] if !ok { return nil, fmt.Errorf("unknown shard %q", id) } return sh, nil } // Remove forgets a terminated shard. func (s *Supervisor) Remove(id string) { s.mu.Lock() defer s.mu.Unlock() delete(s.shards, id) } // IDs lists the supervised shards. func (s *Supervisor) IDs() []string { s.mu.Lock() defer s.mu.Unlock() out := make([]string, 0, len(s.shards)) for id := range s.shards { out = append(out, id) } return out } // Statuses returns every shard's status. func (s *Supervisor) Statuses() []Status { s.mu.Lock() shards := make([]*Shard, 0, len(s.shards)) for _, sh := range s.shards { shards = append(shards, sh) } s.mu.Unlock() out := make([]Status, 0, len(shards)) for _, sh := range shards { out = append(out, sh.Status()) } return out } // --- pid files, for adoption after an agent restart --- func (s *Supervisor) pidPath(id string) string { return filepath.Join(s.cfg.WorkDir, "shard-"+id+".pid") } // writePIDFile records enough to recognise the process later. // // The pid alone is not enough: pids are reused, and adopting an unrelated // process would be worse than leaking the original. The command line is stored // alongside it as a fingerprint. func (s *Supervisor) writePIDFile(sh *Shard) error { content := fmt.Sprintf("%d\n%s\n", sh.pid, strings.Join(sh.cmd.Args, "\x00")) return os.WriteFile(s.pidPath(sh.ID), []byte(content), 0o600) } func (s *Supervisor) removePIDFile(sh *Shard) { _ = os.Remove(s.pidPath(sh.ID)) } // Orphan is an engine process left behind by a previous agent. type Orphan struct { ShardID string PID int Cmdline []string // Alive reports whether the process still exists and matches its fingerprint. Alive bool } // FindOrphans scans the work directory for processes a previous agent started. // // The control plane decides what happens next (plan §5.4): a shard of a // multi-node replica cannot be adopted in isolation, because its peers are gone. func (s *Supervisor) FindOrphans() ([]Orphan, error) { entries, err := os.ReadDir(s.cfg.WorkDir) if err != nil { return nil, err } var out []Orphan for _, e := range entries { name := e.Name() if !strings.HasPrefix(name, "shard-") || !strings.HasSuffix(name, ".pid") { continue } id := strings.TrimSuffix(strings.TrimPrefix(name, "shard-"), ".pid") b, err := os.ReadFile(filepath.Join(s.cfg.WorkDir, name)) if err != nil { continue } lines := strings.SplitN(strings.TrimSpace(string(b)), "\n", 2) pid, err := strconv.Atoi(strings.TrimSpace(lines[0])) if err != nil { continue } var cmdline []string if len(lines) > 1 { cmdline = strings.Split(lines[1], "\x00") } out = append(out, Orphan{ ShardID: id, PID: pid, Cmdline: cmdline, Alive: processAlive(pid), }) } return out, nil } // KillOrphan terminates a leftover process group and removes its pid file. func (s *Supervisor) KillOrphan(o Orphan) error { if o.Alive { if err := killGroup(o.PID, syscall.SIGTERM); err != nil { return err } // Give it a moment to free GPU memory before insisting. deadline := time.Now().Add(s.cfg.StopTimeout) for time.Now().Before(deadline) { if !processAlive(o.PID) { break } time.Sleep(200 * time.Millisecond) } if processAlive(o.PID) { _ = killGroup(o.PID, syscall.SIGKILL) } } // An already-missing pid file is success: the previous agent may have // reaped the process itself between the scan and this call. if err := os.Remove(s.pidPath(o.ShardID)); err != nil && !errors.Is(err, os.ErrNotExist) { return err } return nil }