Not persisting replica state left one hole: restarting the control plane made it forget every replica while the engines kept running. They held GPUs the inventory believed were free, so the next placement failed to allocate, with an error naming the new process rather than the old one. The fix belongs on the agent, not in a database. A row can say "running" about a process that died; the fleet cannot. - the supervisor writes the full shard spec, not just a pid and command line. A pid identifies a process; the spec is what lets it be described back to the control plane and matched to a replica. - Adopt takes over an engine a previous agent left running. It is not this process's child, so it cannot be waited on: it is watched by polling and stopped by signalling its group directly. Its earlier output is gone and the log says so rather than presenting emptiness as silence. - every Register reports what the agent is running, not only the first, because a reconnect may be a control plane restart and has to recover the same way. - the reconciler rebuilds replicas from those reports and reclaims their GPUs. Anything whose model or profile left the catalog is stopped; nothing will route to it and it is holding cards. - an engine the agent cannot identify is killed. A process holding GPUs that nothing can describe is worse than no process, because placement hands those GPUs out again. Stop was also wrong for adopted shards: it checked for an exec.Cmd, which they do not have, and reported them stopped without killing anything. TestAgentReconnects had been passing for no reason. httptest's CloseClientConnections does nothing to an h2c stream — prior-knowledge h2c hijacks the connection and httptest stops tracking hijacked ones — so the agent never reconnected, and the assertion on the connection map passed on a stale entry. Tests now sever connections through their own listener and count opened streams. Plan §11 also records what multi-node still needs, since placement landing early makes it look further along than it is: workers never become Ready because the agent health-checks every shard while only rank 0 serves HTTP; RayCommand is written and tested but never called; and startup is fire-and-forget. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
296 lines
8.6 KiB
Go
296 lines
8.6 KiB
Go
// Package engines builds the command lines and environments that start
|
|
// inference engines (plan §5.3).
|
|
//
|
|
// Nothing here starts a process; that is the supervisor's job. Keeping command
|
|
// construction pure makes it testable without GPUs, which matters because the
|
|
// environment generation is where multi-node runs go wrong: a single missing
|
|
// NCCL variable turns a working tensor-parallel group into one that hangs on
|
|
// the first all-reduce.
|
|
package engines
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"slices"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// Profile is how a model runs on a GPU class. It mirrors the catalog profile
|
|
// (plan §6.1), reduced to what launching needs.
|
|
type Profile struct {
|
|
Name string
|
|
Engine string
|
|
GPUClass string
|
|
|
|
// TP and PP are tensor and pipeline parallel degrees.
|
|
TP int
|
|
PP int
|
|
// GPUsPerNode is how many GPUs each shard occupies.
|
|
GPUsPerNode int
|
|
// Nodes is how many nodes the replica spans. Above 1, a fabric is required.
|
|
Nodes int
|
|
|
|
// Args are passed through to the engine untouched.
|
|
Args []string
|
|
// Env overrides anything the engine adapter generates.
|
|
Env map[string]string
|
|
|
|
ModelPath string
|
|
// ServedModelName is the name clients use, which need not match the path.
|
|
ServedModelName string
|
|
}
|
|
|
|
// ShardSpec is one shard of one replica on this node.
|
|
type ShardSpec struct {
|
|
ReplicaID string
|
|
ShardID string
|
|
// Rank is the node rank; 0 is the leader and serves HTTP.
|
|
Rank int
|
|
NNodes int
|
|
|
|
// LeaderAddr and LeaderPort are the distributed init endpoint, always on the
|
|
// management network rather than the fabric: NCCL bootstraps over TCP and
|
|
// only then moves to RDMA.
|
|
LeaderAddr string
|
|
LeaderPort int
|
|
|
|
// GPUIndices are the local GPUs this shard uses.
|
|
GPUIndices []int
|
|
// HCAs are the adapters of the chosen fabric, and only that fabric. A shard
|
|
// must never be given HCAs from two fabrics.
|
|
HCAs []string
|
|
// MgmtInterface is the ethernet interface used for bootstrap traffic.
|
|
MgmtInterface string
|
|
|
|
HTTPPort int
|
|
Profile Profile
|
|
|
|
// Pool and Fabric are control plane concepts the agent cannot derive. They
|
|
// are carried so the agent can report them back after a restart, and are
|
|
// not used when building a command.
|
|
Pool string
|
|
Fabric string
|
|
}
|
|
|
|
// Leader reports whether this shard serves HTTP for the replica.
|
|
func (s ShardSpec) Leader() bool { return s.Rank == 0 }
|
|
|
|
// Command is a fully resolved launch: what to exec, with which environment.
|
|
type Command struct {
|
|
Path string
|
|
Args []string
|
|
// Env is the complete environment, not a delta.
|
|
Env []string
|
|
}
|
|
|
|
// Engine builds launch commands for one inference engine.
|
|
type Engine interface {
|
|
// Name is the engine's identifier in the catalog.
|
|
Name() string
|
|
// Validate rejects a profile that cannot work on these GPUs, before
|
|
// anything is started.
|
|
Validate(p Profile, gpus []GPUInfo) error
|
|
// BuildCommand produces the launch command for a shard.
|
|
BuildCommand(s ShardSpec) (Command, error)
|
|
// HealthPath and MetricsPath are appended to the shard's base URL.
|
|
HealthPath() string
|
|
MetricsPath() string
|
|
}
|
|
|
|
// GPUInfo is the subset of GPU facts validation needs.
|
|
type GPUInfo struct {
|
|
Index int
|
|
Name string
|
|
ComputeCapability string
|
|
MemoryTotalMB int
|
|
}
|
|
|
|
// Registry holds the known engines.
|
|
type Registry struct {
|
|
engines map[string]Engine
|
|
}
|
|
|
|
// NewRegistry returns a registry with the built-in engines.
|
|
func NewRegistry(cfg Config) *Registry {
|
|
r := &Registry{engines: map[string]Engine{}}
|
|
r.Add(&VLLM{cfg: cfg.VLLM})
|
|
r.Add(&SGLang{cfg: cfg.SGLang})
|
|
return r
|
|
}
|
|
|
|
// Add registers an engine, replacing any engine of the same name.
|
|
func (r *Registry) Add(e Engine) { r.engines[e.Name()] = e }
|
|
|
|
// Get returns an engine by name.
|
|
func (r *Registry) Get(name string) (Engine, error) {
|
|
e, ok := r.engines[name]
|
|
if !ok {
|
|
return nil, fmt.Errorf("unknown engine %q (have %s)", name, strings.Join(r.Names(), ", "))
|
|
}
|
|
return e, nil
|
|
}
|
|
|
|
// Names lists the registered engines.
|
|
func (r *Registry) Names() []string {
|
|
out := make([]string, 0, len(r.engines))
|
|
for n := range r.engines {
|
|
out = append(out, n)
|
|
}
|
|
slices.Sort(out)
|
|
return out
|
|
}
|
|
|
|
// Config configures the engine adapters.
|
|
type Config struct {
|
|
VLLM VLLMConfig `koanf:"vllm"`
|
|
SGLang SGLangConfig `koanf:"sglang"`
|
|
}
|
|
|
|
// --- shared helpers ---
|
|
|
|
// baseEnv builds the environment every engine shares.
|
|
//
|
|
// This is the part of §5.3 the plan marks critical. The rules encoded here:
|
|
//
|
|
// - CUDA_VISIBLE_DEVICES pins the shard to its GPUs, so two shards on one
|
|
// node cannot fight over the same card.
|
|
// - NCCL_IB_HCA lists only the chosen fabric's adapters. Leaving it unset
|
|
// lets NCCL pick adapters from another fabric, where the peer is
|
|
// unreachable, and the job hangs instead of failing.
|
|
// - NCCL_SOCKET_IFNAME pins bootstrap to the management interface. NCCL
|
|
// otherwise probes every interface, including IB-over-IP addresses that
|
|
// cannot route between fabrics.
|
|
// - NCCL_IB_DISABLE is set when there is no fabric, so NCCL stops looking for
|
|
// one instead of timing out.
|
|
func baseEnv(s ShardSpec) map[string]string {
|
|
env := map[string]string{
|
|
"CUDA_VISIBLE_DEVICES": joinInts(s.GPUIndices),
|
|
}
|
|
|
|
if len(s.HCAs) > 0 {
|
|
env["NCCL_IB_HCA"] = strings.Join(s.HCAs, ",")
|
|
env["NCCL_IB_DISABLE"] = "0"
|
|
} else {
|
|
// Single node, or a node with no fabric. Either way IB is not the path.
|
|
env["NCCL_IB_DISABLE"] = "1"
|
|
}
|
|
|
|
if s.MgmtInterface != "" {
|
|
env["NCCL_SOCKET_IFNAME"] = s.MgmtInterface
|
|
env["GLOO_SOCKET_IFNAME"] = s.MgmtInterface
|
|
}
|
|
|
|
// Profile overrides win: they are the escape hatch for the per-cluster
|
|
// tuning no adapter can guess.
|
|
for k, v := range s.Profile.Env {
|
|
env[k] = v
|
|
}
|
|
return env
|
|
}
|
|
|
|
// materializeEnv merges generated variables over the current process
|
|
// environment and returns it in exec form.
|
|
//
|
|
// Inheriting matters: engines need PATH, HOME, HF_HOME, LD_LIBRARY_PATH and
|
|
// whatever else the operator set up in the unit file.
|
|
func materializeEnv(generated map[string]string) []string {
|
|
merged := map[string]string{}
|
|
for _, kv := range os.Environ() {
|
|
if k, v, ok := strings.Cut(kv, "="); ok {
|
|
merged[k] = v
|
|
}
|
|
}
|
|
for k, v := range generated {
|
|
merged[k] = v
|
|
}
|
|
|
|
out := make([]string, 0, len(merged))
|
|
for k, v := range merged {
|
|
out = append(out, k+"="+v)
|
|
}
|
|
// Sorted so a command is reproducible and diffable in logs.
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
func joinInts(v []int) string {
|
|
parts := make([]string, len(v))
|
|
for i, n := range v {
|
|
parts[i] = strconv.Itoa(n)
|
|
}
|
|
return strings.Join(parts, ",")
|
|
}
|
|
|
|
// validateCommon holds the checks that apply to every engine.
|
|
func validateCommon(p Profile, gpus []GPUInfo) error {
|
|
if p.ModelPath == "" {
|
|
return fmt.Errorf("profile %q: model path is empty", p.Name)
|
|
}
|
|
if p.TP <= 0 {
|
|
return fmt.Errorf("profile %q: tensor parallel degree must be positive", p.Name)
|
|
}
|
|
if p.Nodes > 1 && p.GPUsPerNode <= 0 {
|
|
return fmt.Errorf("profile %q: gpus_per_node is required for a %d-node profile", p.Name, p.Nodes)
|
|
}
|
|
|
|
perNode := p.GPUsPerNode
|
|
if perNode == 0 {
|
|
perNode = p.TP
|
|
}
|
|
nodes := max(p.Nodes, 1)
|
|
if want := perNode * nodes; want != p.TP*max(p.PP, 1) {
|
|
return fmt.Errorf("profile %q: %d nodes x %d GPUs does not match TP %d x PP %d",
|
|
p.Name, nodes, perNode, p.TP, max(p.PP, 1))
|
|
}
|
|
if len(gpus) > 0 && len(gpus) < perNode {
|
|
return fmt.Errorf("profile %q: needs %d GPUs per node, node has %d", p.Name, perNode, len(gpus))
|
|
}
|
|
|
|
// A tensor-parallel group must be uniform: mixing classes makes the slowest
|
|
// card set the pace for every collective, and mismatched memory makes the
|
|
// smallest card decide what fits.
|
|
if len(gpus) > 1 {
|
|
first := gpus[0].Name
|
|
for _, g := range gpus[1:] {
|
|
if g.Name != first {
|
|
return fmt.Errorf("profile %q: GPUs are not uniform (%q and %q); "+
|
|
"a tensor-parallel group may not mix GPU types", p.Name, first, g.Name)
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// requireComputeCapability rejects profiles whose quantization the hardware
|
|
// cannot execute.
|
|
func requireComputeCapability(p Profile, gpus []GPUInfo, minCap float64, why string) error {
|
|
for _, g := range gpus {
|
|
cap, err := strconv.ParseFloat(g.ComputeCapability, 64)
|
|
if err != nil {
|
|
// Unknown capability: let it through rather than block a launch on
|
|
// a parsing problem. The engine will refuse it clearly enough.
|
|
continue
|
|
}
|
|
if cap < minCap {
|
|
return fmt.Errorf("profile %q: %s needs compute capability %.1f, GPU %d (%s) has %.1f",
|
|
p.Name, why, minCap, g.Index, g.Name, cap)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// quantizationOf extracts the --quantization argument, if present.
|
|
func quantizationOf(args []string) string {
|
|
for i, a := range args {
|
|
if a == "--quantization" && i+1 < len(args) {
|
|
return args[i+1]
|
|
}
|
|
if v, ok := strings.CutPrefix(a, "--quantization="); ok {
|
|
return v
|
|
}
|
|
}
|
|
return ""
|
|
}
|