CI / test (push) Successful in 45s
Everything here works without GPUs and is covered by tests. The control plane connection is not built yet, so llmgw agent itself is still a stub and the phase exit criterion is not met. - discovery: IB from /sys/class/infiniband, parsing the formats sysfs actually uses and telling native InfiniBand from RoCE; GPUs via nvidia-smi, the cgo-free path, with health flags for uncorrected ECC and memory already held by another process. A node without IB is not an error, it is a single-node island. - engines: vLLM and SGLang command and environment construction, kept free of process handling so it stays testable. The NCCL environment is the part the plan marks critical: NCCL_IB_HCA lists only the chosen fabric's adapters, because leaving it open lets NCCL pick an adapter on a fabric where the peer is unreachable, and the job then hangs instead of failing. - validation before launch: no mixed GPU types in a tensor-parallel group, the node/GPU/TP arithmetic has to add up, and fp8 is refused below compute capability 8.9 rather than silently falling back to a slower path. - supervisor: process groups, because engines fork workers that hold GPU memory and outlive a signal sent only to the leader; a bounded log tail, since engine output is the only diagnosis when loading fails; Starting separate from Loading, because a 70B model legitimately takes minutes; and orphan adoption through pid files fingerprinted with the command line, since pids get reused and adopting an unrelated process is worse than leaking the original. - readiness is two questions: /health says the process is up, /v1/models says the weights loaded. Routing on /health alone sends traffic into a loading model. - llmgw agent inventory reports what the scheduler would see on a node. One plan correction: GPU class sizes come from the driver in GiB, which does not always match the marketing name. An L40S is sold as 48GB and reports 44.99 GiB, so its class is l40s-45g, not l40s-48g. Consistency matters more than matching the box, because that number decides whether a model fits. Plan §2 and §5.1 updated. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
523 lines
13 KiB
Go
523 lines
13 KiB
Go
package supervisor
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
|
)
|
|
|
|
// Config configures the supervisor.
|
|
type Config struct {
|
|
// WorkDir holds pid files and per-shard scratch space.
|
|
WorkDir string `koanf:"work_dir"`
|
|
// LogLines is how many lines of engine output to retain per shard.
|
|
LogLines int `koanf:"log_lines"`
|
|
// StopTimeout is how long a process gets to exit after SIGTERM before it is
|
|
// killed. Engines need time to free GPU memory cleanly.
|
|
StopTimeout time.Duration `koanf:"stop_timeout"`
|
|
// HealthInterval is how often a starting shard is polled.
|
|
HealthInterval time.Duration `koanf:"health_interval"`
|
|
}
|
|
|
|
// DefaultConfig returns sensible defaults.
|
|
func DefaultConfig() Config {
|
|
return Config{
|
|
WorkDir: "/var/lib/llmgw-agent",
|
|
LogLines: 512,
|
|
StopTimeout: 60 * time.Second,
|
|
HealthInterval: 2 * time.Second,
|
|
}
|
|
}
|
|
|
|
func (c *Config) normalize() {
|
|
d := DefaultConfig()
|
|
if c.WorkDir == "" {
|
|
c.WorkDir = d.WorkDir
|
|
}
|
|
if c.LogLines <= 0 {
|
|
c.LogLines = d.LogLines
|
|
}
|
|
if c.StopTimeout <= 0 {
|
|
c.StopTimeout = d.StopTimeout
|
|
}
|
|
if c.HealthInterval <= 0 {
|
|
c.HealthInterval = d.HealthInterval
|
|
}
|
|
}
|
|
|
|
// HealthChecker reports whether a shard is serving. It is an interface so the
|
|
// supervisor can be tested without an HTTP engine.
|
|
type HealthChecker interface {
|
|
// Ready reports whether the shard at baseURL serves model.
|
|
Ready(ctx context.Context, baseURL, model string) error
|
|
}
|
|
|
|
// Shard is one supervised engine process.
|
|
type Shard struct {
|
|
ID string
|
|
Spec engines.ShardSpec
|
|
|
|
mu sync.Mutex
|
|
state State
|
|
pid int
|
|
err error
|
|
started time.Time
|
|
readyAt time.Time
|
|
exitCode int
|
|
|
|
logs *ringBuffer
|
|
cmd *exec.Cmd
|
|
cancel context.CancelFunc
|
|
done chan struct{}
|
|
}
|
|
|
|
// Status is a snapshot of a shard, safe to send to the control plane.
|
|
type Status struct {
|
|
ID string
|
|
State State
|
|
PID int
|
|
Err string
|
|
Started time.Time
|
|
ReadyAt time.Time
|
|
ExitCode int
|
|
}
|
|
|
|
// Status returns the shard's current status.
|
|
func (s *Shard) Status() Status {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
st := Status{
|
|
ID: s.ID, State: s.state, PID: s.pid,
|
|
Started: s.started, ReadyAt: s.readyAt, ExitCode: s.exitCode,
|
|
}
|
|
if s.err != nil {
|
|
st.Err = s.err.Error()
|
|
}
|
|
return st
|
|
}
|
|
|
|
// Logs returns the last n lines of engine output.
|
|
func (s *Shard) Logs(n int) []string { return s.logs.tail(n) }
|
|
|
|
func (s *Shard) setState(st State, err error) {
|
|
s.mu.Lock()
|
|
s.state = st
|
|
if err != nil {
|
|
s.err = err
|
|
}
|
|
if st == Ready && s.readyAt.IsZero() {
|
|
s.readyAt = time.Now()
|
|
}
|
|
s.mu.Unlock()
|
|
}
|
|
|
|
// Supervisor starts, watches and stops engine processes on this node.
|
|
type Supervisor struct {
|
|
cfg Config
|
|
log *slog.Logger
|
|
health HealthChecker
|
|
|
|
mu sync.Mutex
|
|
shards map[string]*Shard
|
|
}
|
|
|
|
// New builds a supervisor.
|
|
func New(cfg Config, health HealthChecker, log *slog.Logger) (*Supervisor, error) {
|
|
cfg.normalize()
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
if err := os.MkdirAll(cfg.WorkDir, 0o750); err != nil {
|
|
return nil, fmt.Errorf("work dir: %w", err)
|
|
}
|
|
return &Supervisor{cfg: cfg, log: log, health: health, shards: map[string]*Shard{}}, nil
|
|
}
|
|
|
|
// ErrAlreadyRunning means a shard with that ID is already supervised.
|
|
var ErrAlreadyRunning = errors.New("shard is already running")
|
|
|
|
// Start launches a shard.
|
|
//
|
|
// It is idempotent by ID: a repeated StartShard for the same shard is a no-op
|
|
// rather than a second process, because the control plane retries commands
|
|
// (plan §5.6).
|
|
func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpec, cmd engines.Command) (*Shard, error) {
|
|
s.mu.Lock()
|
|
if existing, ok := s.shards[id]; ok {
|
|
s.mu.Unlock()
|
|
if existing.Status().State.Terminal() {
|
|
s.Remove(id)
|
|
} else {
|
|
return existing, ErrAlreadyRunning
|
|
}
|
|
s.mu.Lock()
|
|
}
|
|
|
|
sh := &Shard{
|
|
ID: id,
|
|
Spec: spec,
|
|
state: Pending,
|
|
logs: newRingBuffer(s.cfg.LogLines),
|
|
started: time.Now(),
|
|
done: make(chan struct{}),
|
|
}
|
|
s.shards[id] = sh
|
|
s.mu.Unlock()
|
|
|
|
if err := s.launch(ctx, sh, cmd); err != nil {
|
|
sh.setState(Failed, err)
|
|
close(sh.done)
|
|
return sh, err
|
|
}
|
|
return sh, nil
|
|
}
|
|
|
|
func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmd engines.Command) error {
|
|
runCtx, cancel := context.WithCancel(context.WithoutCancel(ctx))
|
|
sh.cancel = cancel
|
|
|
|
c := exec.CommandContext(runCtx, cmd.Path, cmd.Args...)
|
|
c.Env = cmd.Env
|
|
c.Dir = s.cfg.WorkDir
|
|
// Its own process group, so stopping kills the engine's children too.
|
|
// Engines spawn workers; signalling only the parent leaves those workers
|
|
// holding GPU memory.
|
|
c.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
|
// CommandContext kills only the leader, so replace that with a group kill.
|
|
c.Cancel = func() error { return killGroup(c.Process.Pid, syscall.SIGKILL) }
|
|
|
|
stdout, err := c.StdoutPipe()
|
|
if err != nil {
|
|
cancel()
|
|
return err
|
|
}
|
|
c.Stderr = c.Stdout.(io.Writer) // interleave, as the engine writes them
|
|
|
|
if err := c.Start(); err != nil {
|
|
cancel()
|
|
return fmt.Errorf("starting %s: %w", cmd.Path, err)
|
|
}
|
|
|
|
sh.mu.Lock()
|
|
sh.cmd = c
|
|
sh.pid = c.Process.Pid
|
|
sh.state = Starting
|
|
sh.mu.Unlock()
|
|
|
|
if err := s.writePIDFile(sh); err != nil {
|
|
s.log.Warn("could not write pid file; adoption after a restart will not work",
|
|
"shard", sh.ID, "err", err)
|
|
}
|
|
|
|
go s.pump(sh, stdout)
|
|
go s.wait(sh)
|
|
return nil
|
|
}
|
|
|
|
// pump copies engine output into the ring buffer and the log.
|
|
func (s *Supervisor) pump(sh *Shard, r io.Reader) {
|
|
sc := bufio.NewScanner(r)
|
|
// Engine tracebacks and progress bars produce long lines; the default
|
|
// 64 KiB limit truncates them exactly when they matter.
|
|
sc.Buffer(make([]byte, 0, 64<<10), 1<<20)
|
|
|
|
for sc.Scan() {
|
|
line := sc.Text()
|
|
sh.logs.add(line)
|
|
s.log.Debug("engine", "shard", sh.ID, "line", line)
|
|
}
|
|
}
|
|
|
|
// wait reaps the process and records why it ended.
|
|
func (s *Supervisor) wait(sh *Shard) {
|
|
defer close(sh.done)
|
|
|
|
err := sh.cmd.Wait()
|
|
|
|
sh.mu.Lock()
|
|
prev := sh.state
|
|
if ee := new(exec.ExitError); errors.As(err, &ee) {
|
|
sh.exitCode = ee.ExitCode()
|
|
}
|
|
switch {
|
|
case prev == Draining || prev == Stopped:
|
|
// Expected: we asked it to stop.
|
|
sh.state = Stopped
|
|
case err == nil:
|
|
// An engine exiting on its own is a failure, not a success: it is
|
|
// supposed to serve until told otherwise.
|
|
sh.state = Failed
|
|
sh.err = errors.New("engine exited unexpectedly")
|
|
default:
|
|
sh.state = Failed
|
|
sh.err = err
|
|
}
|
|
final, code := sh.state, sh.exitCode
|
|
sh.mu.Unlock()
|
|
|
|
s.removePIDFile(sh)
|
|
if final == Failed {
|
|
s.log.Error("shard failed", "shard", sh.ID, "exit_code", code, "err", err,
|
|
"last_output", strings.Join(sh.Logs(5), " | "))
|
|
} else {
|
|
s.log.Info("shard stopped", "shard", sh.ID, "exit_code", code)
|
|
}
|
|
}
|
|
|
|
// WaitReady polls until the shard serves, it fails, or the timeout expires.
|
|
//
|
|
// The timeout comes from the profile because it is model-dependent: loading a
|
|
// 70B model across two nodes legitimately takes tens of minutes, while a small
|
|
// model failing to load for ten minutes means something is wrong.
|
|
func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model string, timeout time.Duration) error {
|
|
ctx, cancel := context.WithTimeout(ctx, timeout)
|
|
defer cancel()
|
|
|
|
t := time.NewTicker(s.cfg.HealthInterval)
|
|
defer t.Stop()
|
|
|
|
sh.setState(Loading, nil)
|
|
for {
|
|
select {
|
|
case <-sh.done:
|
|
return fmt.Errorf("shard %s exited during startup: %s", sh.ID, sh.Status().Err)
|
|
case <-ctx.Done():
|
|
last := strings.Join(sh.Logs(10), "\n")
|
|
return fmt.Errorf("shard %s not ready within %v; last output:\n%s", sh.ID, timeout, last)
|
|
case <-t.C:
|
|
if err := s.health.Ready(ctx, baseURL, model); err != nil {
|
|
continue
|
|
}
|
|
sh.setState(Ready, nil)
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
|
|
// Drain marks a shard as taking no new work. The router stops routing to it;
|
|
// the process keeps running until Stop.
|
|
func (s *Supervisor) Drain(id string) error {
|
|
sh, err := s.Get(id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if st := sh.Status().State; !st.Live() {
|
|
return fmt.Errorf("shard %s is %s, cannot drain", id, st)
|
|
}
|
|
sh.setState(Draining, nil)
|
|
return nil
|
|
}
|
|
|
|
// Stop terminates a shard: SIGTERM to the whole group, then SIGKILL after
|
|
// StopTimeout.
|
|
func (s *Supervisor) Stop(ctx context.Context, id string) error {
|
|
sh, err := s.Get(id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
sh.mu.Lock()
|
|
c, pid := sh.cmd, sh.pid
|
|
if sh.state.Live() {
|
|
sh.state = Draining // marks the exit as expected for wait()
|
|
}
|
|
sh.mu.Unlock()
|
|
|
|
if c == nil || pid == 0 {
|
|
sh.setState(Stopped, nil)
|
|
return nil
|
|
}
|
|
|
|
// Signal the group, not the process: engines fork workers that hold GPU
|
|
// memory, and those outlive a signal sent only to the leader.
|
|
if err := killGroup(pid, syscall.SIGTERM); err != nil && !errors.Is(err, os.ErrProcessDone) {
|
|
s.log.Warn("SIGTERM failed", "shard", id, "err", err)
|
|
}
|
|
|
|
select {
|
|
case <-sh.done:
|
|
return nil
|
|
case <-time.After(s.cfg.StopTimeout):
|
|
s.log.Warn("shard did not exit in time, killing", "shard", id, "timeout", s.cfg.StopTimeout)
|
|
if sh.cancel != nil {
|
|
sh.cancel()
|
|
}
|
|
case <-ctx.Done():
|
|
if sh.cancel != nil {
|
|
sh.cancel()
|
|
}
|
|
return ctx.Err()
|
|
}
|
|
|
|
select {
|
|
case <-sh.done:
|
|
case <-time.After(5 * time.Second):
|
|
return fmt.Errorf("shard %s survived SIGKILL", id)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// StopAll stops every live shard.
|
|
func (s *Supervisor) StopAll(ctx context.Context) {
|
|
for _, id := range s.IDs() {
|
|
if err := s.Stop(ctx, id); err != nil {
|
|
s.log.Warn("stopping shard failed", "shard", id, "err", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Get returns a supervised shard.
|
|
func (s *Supervisor) Get(id string) (*Shard, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
sh, ok := s.shards[id]
|
|
if !ok {
|
|
return nil, fmt.Errorf("unknown shard %q", id)
|
|
}
|
|
return sh, nil
|
|
}
|
|
|
|
// Remove forgets a terminated shard.
|
|
func (s *Supervisor) Remove(id string) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
delete(s.shards, id)
|
|
}
|
|
|
|
// IDs lists the supervised shards.
|
|
func (s *Supervisor) IDs() []string {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
out := make([]string, 0, len(s.shards))
|
|
for id := range s.shards {
|
|
out = append(out, id)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Statuses returns every shard's status.
|
|
func (s *Supervisor) Statuses() []Status {
|
|
s.mu.Lock()
|
|
shards := make([]*Shard, 0, len(s.shards))
|
|
for _, sh := range s.shards {
|
|
shards = append(shards, sh)
|
|
}
|
|
s.mu.Unlock()
|
|
|
|
out := make([]Status, 0, len(shards))
|
|
for _, sh := range shards {
|
|
out = append(out, sh.Status())
|
|
}
|
|
return out
|
|
}
|
|
|
|
// --- pid files, for adoption after an agent restart ---
|
|
|
|
func (s *Supervisor) pidPath(id string) string {
|
|
return filepath.Join(s.cfg.WorkDir, "shard-"+id+".pid")
|
|
}
|
|
|
|
// writePIDFile records enough to recognise the process later.
|
|
//
|
|
// The pid alone is not enough: pids are reused, and adopting an unrelated
|
|
// process would be worse than leaking the original. The command line is stored
|
|
// alongside it as a fingerprint.
|
|
func (s *Supervisor) writePIDFile(sh *Shard) error {
|
|
content := fmt.Sprintf("%d\n%s\n", sh.pid, strings.Join(sh.cmd.Args, "\x00"))
|
|
return os.WriteFile(s.pidPath(sh.ID), []byte(content), 0o600)
|
|
}
|
|
|
|
func (s *Supervisor) removePIDFile(sh *Shard) {
|
|
_ = os.Remove(s.pidPath(sh.ID))
|
|
}
|
|
|
|
// Orphan is an engine process left behind by a previous agent.
|
|
type Orphan struct {
|
|
ShardID string
|
|
PID int
|
|
Cmdline []string
|
|
// Alive reports whether the process still exists and matches its fingerprint.
|
|
Alive bool
|
|
}
|
|
|
|
// FindOrphans scans the work directory for processes a previous agent started.
|
|
//
|
|
// The control plane decides what happens next (plan §5.4): a shard of a
|
|
// multi-node replica cannot be adopted in isolation, because its peers are gone.
|
|
func (s *Supervisor) FindOrphans() ([]Orphan, error) {
|
|
entries, err := os.ReadDir(s.cfg.WorkDir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var out []Orphan
|
|
for _, e := range entries {
|
|
name := e.Name()
|
|
if !strings.HasPrefix(name, "shard-") || !strings.HasSuffix(name, ".pid") {
|
|
continue
|
|
}
|
|
id := strings.TrimSuffix(strings.TrimPrefix(name, "shard-"), ".pid")
|
|
|
|
b, err := os.ReadFile(filepath.Join(s.cfg.WorkDir, name))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
lines := strings.SplitN(strings.TrimSpace(string(b)), "\n", 2)
|
|
pid, err := strconv.Atoi(strings.TrimSpace(lines[0]))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
var cmdline []string
|
|
if len(lines) > 1 {
|
|
cmdline = strings.Split(lines[1], "\x00")
|
|
}
|
|
|
|
out = append(out, Orphan{
|
|
ShardID: id,
|
|
PID: pid,
|
|
Cmdline: cmdline,
|
|
Alive: processAlive(pid),
|
|
})
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// KillOrphan terminates a leftover process group and removes its pid file.
|
|
func (s *Supervisor) KillOrphan(o Orphan) error {
|
|
if o.Alive {
|
|
if err := killGroup(o.PID, syscall.SIGTERM); err != nil {
|
|
return err
|
|
}
|
|
// Give it a moment to free GPU memory before insisting.
|
|
deadline := time.Now().Add(s.cfg.StopTimeout)
|
|
for time.Now().Before(deadline) {
|
|
if !processAlive(o.PID) {
|
|
break
|
|
}
|
|
time.Sleep(200 * time.Millisecond)
|
|
}
|
|
if processAlive(o.PID) {
|
|
_ = killGroup(o.PID, syscall.SIGKILL)
|
|
}
|
|
}
|
|
// An already-missing pid file is success: the previous agent may have
|
|
// reaped the process itself between the scan and this call.
|
|
if err := os.Remove(s.pidPath(o.ShardID)); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|