diff --git a/CHANGELOG.md b/CHANGELOG.md index 84c7285..59d56b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,13 @@ The NVML path compiles but has never run against a driver — there is no GPU machine to try it on. Treat the NVLink domains it reports as unverified until someone runs `llmgw agent inventory` on a real node. +### Phase 3 exit criterion: half met + +`TestLostNodeMovesReplicas` covers the reconciliation half — a node going +silent becomes unreachable, then lost, and its replica is rebuilt on a survivor. +The other half, the router continuing to serve throughout, needs the +`EndpointSource` wiring and the agent stream. + ### Still missing for phase 2 - The agent-to-control-plane stream. It pairs with phase 3's control plane @@ -104,6 +111,27 @@ someone runs `llmgw agent inventory` on a real node. placement mixes GPU classes or uses an unhealthy or already-allocated GPU. The table-driven tests cover the cases someone thought of; this covers the ones nobody did. +- **Catalog** (`internal/control/catalog`): models, profiles, aliases, + fallbacks and scaling bounds. Aliases and fallbacks are resolved at load + rather than at request time — a typo in a fallback should stop a deployment, + not surface during the outage when the fallback is finally needed. The + node/GPU/TP arithmetic is checked here too, so a bad profile fails at config + load instead of minutes into model loading. +- **Reconciler** (`internal/control/reconciler`): the desired-versus-actual + loop. It places one replica per pass, because placement depends on what is + already allocated and starting several from one snapshot would hand them the + same GPUs. Workers start before rank 0, since SGLang's ranks rendezvous and + vLLM under Ray needs its workers attached before the head serves. +- **Replica failure is whole-replica failure.** One dead shard makes the others + useless — they wait on collectives that will never complete while holding + GPUs — so the reconciler stops every shard and rebuilds, with a jittered + exponential backoff carried on the model rather than the replica it just + destroyed. +- **Node expiry in two stages.** Unreachable first, Lost later. A brief network + blip must not tear down replicas that are still serving, and rebuilding a + multi-node replica costs tens of minutes of model loading. +- **Scale-down drains before stopping**, so in-flight requests finish instead of + being cut off mid-stream. ## v0.3.0 — 2026-09-16 diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go new file mode 100644 index 0000000..1313c9f --- /dev/null +++ b/internal/control/catalog/catalog.go @@ -0,0 +1,289 @@ +// Package catalog holds the desired state: which models exist, how each one can +// be run, and how many copies of it there should be (plan §6.1). +package catalog + +import ( + "fmt" + "slices" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/placement" +) + +// Scaling is how many replicas a model should have. +type Scaling struct { + Min int `koanf:"min"` + Max int `koanf:"max"` +} + +// SLO is the latency target a model is expected to meet. `ctl bench` measures +// capacity against it (plan §6.2.1). +type SLO struct { + TTFTp99 time.Duration `koanf:"ttft_p99"` + TPOTp99 time.Duration `koanf:"tpot_p99"` +} + +// Profile is how a model runs on one GPU class. +type Profile struct { + Name string `koanf:"name"` + Engine string `koanf:"engine"` + GPUClass string `koanf:"gpu_class"` + + TP int `koanf:"tp"` + PP int `koanf:"pp"` + // GPUsPerNode defaults to TP for a single-node profile. + GPUsPerNode int `koanf:"gpus_per_node"` + // Nodes above 1 makes this a multi-node profile, which requires a fabric. + Nodes int `koanf:"nodes"` + + Args []string `koanf:"args"` + Env map[string]string `koanf:"env"` + + // Weight is the router's capacity weight. It should come from ctl bench + // rather than a guess (plan §6.2.1). + Weight float64 `koanf:"weight"` + // Priority orders profiles during placement; highest that fits wins. + Priority int `koanf:"priority"` + // StartupTimeout bounds model loading. A 70B model across two nodes takes + // tens of minutes, so this is per profile rather than global. + StartupTimeout time.Duration `koanf:"startup_timeout"` +} + +// Model is a catalog entry as clients see it. +type Model struct { + Name string `koanf:"name"` + Aliases []string `koanf:"aliases"` + // Source is the weights path or a HuggingFace id. + Source string `koanf:"source"` + Profiles []Profile `koanf:"profiles"` + Scaling Scaling `koanf:"scaling"` + Fallbacks []string `koanf:"fallbacks"` + // RoutingProfile selects how the router scores endpoints for this model. + RoutingProfile string `koanf:"routing_profile"` + SLO SLO `koanf:"slo"` +} + +// Catalog is the set of models the cluster should serve. +type Catalog struct { + models map[string]Model + aliases map[string]string + order []string +} + +// New builds a catalog, rejecting definitions that would fail confusingly later. +func New(models []Model) (*Catalog, error) { + c := &Catalog{ + models: make(map[string]Model, len(models)), + aliases: map[string]string{}, + } + + for i := range models { + m := models[i].normalize() + if err := m.validate(); err != nil { + return nil, err + } + if _, dup := c.models[m.Name]; dup { + return nil, fmt.Errorf("model %q is declared twice", m.Name) + } + c.models[m.Name] = m + c.order = append(c.order, m.Name) + } + + // Aliases share a namespace with names. A collision would silently shadow a + // model at request time, so it has to fail at load. + for _, name := range c.order { + for _, a := range c.models[name].Aliases { + if _, clash := c.models[a]; clash { + return nil, fmt.Errorf("model %q: alias %q collides with another model name", name, a) + } + if prev, clash := c.aliases[a]; clash { + return nil, fmt.Errorf("alias %q is claimed by both %q and %q", a, prev, name) + } + c.aliases[a] = name + } + } + + // Fallbacks are resolved now rather than at request time: a typo in a + // fallback should stop a deployment, not surface during an outage when the + // fallback is finally needed. + for _, name := range c.order { + for _, f := range c.models[name].Fallbacks { + if _, ok := c.Resolve(f); !ok { + return nil, fmt.Errorf("model %q: fallback %q is not in the catalog", name, f) + } + } + } + return c, nil +} + +func (m Model) normalize() Model { + if m.Scaling.Min < 0 { + m.Scaling.Min = 0 + } + if m.Scaling.Max == 0 { + m.Scaling.Max = max(m.Scaling.Min, 1) + } + for i := range m.Profiles { + p := &m.Profiles[i] + if p.Engine == "" { + p.Engine = "vllm" + } + if p.TP <= 0 { + p.TP = 1 + } + if p.Nodes <= 0 { + p.Nodes = 1 + } + if p.GPUsPerNode <= 0 { + // The common case: a single-node profile uses TP GPUs on one node. + p.GPUsPerNode = p.TP / p.Nodes + if p.GPUsPerNode <= 0 { + p.GPUsPerNode = 1 + } + } + if p.StartupTimeout <= 0 { + p.StartupTimeout = 20 * time.Minute + } + if p.Weight <= 0 { + p.Weight = 1 + } + } + return m +} + +func (m Model) validate() error { + if m.Name == "" { + return fmt.Errorf("a model has no name") + } + if m.Source == "" { + return fmt.Errorf("model %q: source is required", m.Name) + } + if len(m.Profiles) == 0 { + return fmt.Errorf("model %q: at least one profile is required", m.Name) + } + if m.Scaling.Max < m.Scaling.Min { + return fmt.Errorf("model %q: scaling max %d is below min %d", m.Name, m.Scaling.Max, m.Scaling.Min) + } + + seen := map[string]struct{}{} + for _, p := range m.Profiles { + if p.Name == "" { + return fmt.Errorf("model %q: a profile has no name", m.Name) + } + if _, dup := seen[p.Name]; dup { + return fmt.Errorf("model %q: profile %q is declared twice", m.Name, p.Name) + } + seen[p.Name] = struct{}{} + + if p.GPUClass == "" { + return fmt.Errorf("model %q profile %q: gpu_class is required", m.Name, p.Name) + } + // The arithmetic has to add up, or the engine is handed a shard spec it + // cannot satisfy and fails minutes later during model loading. + if want := p.GPUsPerNode * p.Nodes; want != p.TP*max(p.PP, 1) { + return fmt.Errorf("model %q profile %q: %d nodes x %d GPUs does not match TP %d x PP %d", + m.Name, p.Name, p.Nodes, p.GPUsPerNode, p.TP, max(p.PP, 1)) + } + } + return nil +} + +// Resolve maps a client-supplied name or alias onto a model. +func (c *Catalog) Resolve(name string) (Model, bool) { + if m, ok := c.models[name]; ok { + return m, true + } + if canonical, ok := c.aliases[name]; ok { + return c.models[canonical], true + } + return Model{}, false +} + +// Models returns every model, in declaration order. +func (c *Catalog) Models() []Model { + out := make([]Model, 0, len(c.order)) + for _, n := range c.order { + out = append(out, c.models[n]) + } + return out +} + +// Names returns the canonical model names, sorted. +func (c *Catalog) Names() []string { + out := slices.Clone(c.order) + slices.Sort(out) + return out +} + +// AliasesOf returns the alternative names for a model. +func (c *Catalog) AliasesOf(name string) []string { + m, ok := c.Resolve(name) + if !ok { + return nil + } + return slices.Clone(m.Aliases) +} + +// PlacementProfiles converts a model's profiles into placement's view. +func (m Model) PlacementProfiles() []placement.Profile { + out := make([]placement.Profile, 0, len(m.Profiles)) + for _, p := range m.Profiles { + out = append(out, placement.Profile{ + Name: p.Name, + Engine: p.Engine, + GPUClass: p.GPUClass, + GPUsPerNode: p.GPUsPerNode, + Nodes: p.Nodes, + Priority: p.Priority, + }) + } + return out +} + +// EngineProfile converts a catalog profile into the engine adapter's view. +func (m Model) EngineProfile(name string) (engines.Profile, bool) { + for _, p := range m.Profiles { + if p.Name != name { + continue + } + return engines.Profile{ + Name: p.Name, + Engine: p.Engine, + GPUClass: p.GPUClass, + TP: p.TP, + PP: p.PP, + GPUsPerNode: p.GPUsPerNode, + Nodes: p.Nodes, + Args: slices.Clone(p.Args), + Env: p.Env, + ModelPath: m.Source, + ServedModelName: m.Name, + }, true + } + return engines.Profile{}, false +} + +// ProfileByName returns one of a model's profiles. +func (m Model) ProfileByName(name string) (Profile, bool) { + for _, p := range m.Profiles { + if p.Name == name { + return p, true + } + } + return Profile{}, false +} + +// String renders the catalog for `ctl models`. +func (c *Catalog) String() string { + var b strings.Builder + for _, m := range c.Models() { + fmt.Fprintf(&b, "%s (min %d, max %d)\n", m.Name, m.Scaling.Min, m.Scaling.Max) + for _, p := range m.Profiles { + fmt.Fprintf(&b, " %-20s %-10s %-12s tp=%d nodes=%d priority=%d\n", + p.Name, p.Engine, p.GPUClass, p.TP, p.Nodes, p.Priority) + } + } + return b.String() +} diff --git a/internal/control/catalog/catalog_test.go b/internal/control/catalog/catalog_test.go new file mode 100644 index 0000000..bb630ba --- /dev/null +++ b/internal/control/catalog/catalog_test.go @@ -0,0 +1,235 @@ +package catalog + +import ( + "strings" + "testing" + "time" +) + +func model(name string, profiles ...Profile) Model { + if len(profiles) == 0 { + profiles = []Profile{{Name: "p1", GPUClass: "h100-80g", TP: 4}} + } + return Model{Name: name, Source: "/models/" + name, Profiles: profiles} +} + +func TestResolveNameAndAlias(t *testing.T) { + m := model("llama-3.3-70b") + m.Aliases = []string{"llama", "llama3"} + c, err := New([]Model{m}) + if err != nil { + t.Fatal(err) + } + + for _, name := range []string{"llama-3.3-70b", "llama", "llama3"} { + got, ok := c.Resolve(name) + if !ok { + t.Fatalf("%q did not resolve", name) + } + if got.Name != "llama-3.3-70b" { + t.Errorf("%q resolved to %q", name, got.Name) + } + } + if _, ok := c.Resolve("nope"); ok { + t.Error("an unknown name resolved") + } +} + +func TestAliasCollisionRejected(t *testing.T) { + // An alias shadowing another model would misroute every request for it, + // silently, at request time. + a := model("qwen") + b := model("llama") + b.Aliases = []string{"qwen"} + + _, err := New([]Model{a, b}) + if err == nil || !strings.Contains(err.Error(), "collides") { + t.Errorf("collision not rejected: %v", err) + } +} + +func TestDuplicateAliasRejected(t *testing.T) { + a := model("a") + a.Aliases = []string{"shared"} + b := model("b") + b.Aliases = []string{"shared"} + + _, err := New([]Model{a, b}) + if err == nil || !strings.Contains(err.Error(), "claimed by both") { + t.Errorf("duplicate alias not rejected: %v", err) + } +} + +func TestUnknownFallbackRejected(t *testing.T) { + // A typo in a fallback should stop a deployment, not surface during the + // outage when the fallback is finally needed. + m := model("llama") + m.Fallbacks = []string{"qwen-typo"} + + _, err := New([]Model{m}) + if err == nil || !strings.Contains(err.Error(), "fallback") { + t.Errorf("unknown fallback not rejected: %v", err) + } +} + +func TestFallbackToAliasAccepted(t *testing.T) { + a := model("qwen2.5-32b") + a.Aliases = []string{"qwen"} + b := model("llama") + b.Fallbacks = []string{"qwen"} + + if _, err := New([]Model{a, b}); err != nil { + t.Errorf("a fallback naming an alias was rejected: %v", err) + } +} + +func TestParallelismArithmeticValidated(t *testing.T) { + // Wrong arithmetic hands the engine a spec it cannot satisfy, and it fails + // minutes later during model loading rather than at config load. + _, err := New([]Model{model("m", Profile{ + Name: "bad", GPUClass: "h100-80g", TP: 4, Nodes: 2, GPUsPerNode: 8, + })}) + if err == nil || !strings.Contains(err.Error(), "does not match") { + t.Errorf("bad arithmetic accepted: %v", err) + } + + // 2 nodes x 8 GPUs = TP 16 is consistent. + if _, err := New([]Model{model("m", Profile{ + Name: "ok", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8, + })}); err != nil { + t.Errorf("valid arithmetic rejected: %v", err) + } +} + +func TestDefaults(t *testing.T) { + c, err := New([]Model{model("m", Profile{Name: "p", GPUClass: "h100-80g", TP: 4})}) + if err != nil { + t.Fatal(err) + } + m, _ := c.Resolve("m") + p := m.Profiles[0] + + if p.Engine != "vllm" { + t.Errorf("engine=%q", p.Engine) + } + if p.Nodes != 1 { + t.Errorf("nodes=%d", p.Nodes) + } + if p.GPUsPerNode != 4 { + t.Errorf("gpus_per_node=%d; a single-node profile should default to TP", p.GPUsPerNode) + } + if p.StartupTimeout != 20*time.Minute { + t.Errorf("startup_timeout=%v", p.StartupTimeout) + } + if p.Weight <= 0 { + t.Errorf("weight=%v", p.Weight) + } + if m.Scaling.Max < 1 { + t.Errorf("scaling max=%d", m.Scaling.Max) + } +} + +func TestRequiredFields(t *testing.T) { + cases := map[string]Model{ + "no name": {Source: "/m", Profiles: []Profile{{Name: "p", GPUClass: "c", TP: 1}}}, + "no source": {Name: "m", Profiles: []Profile{{Name: "p", GPUClass: "c", TP: 1}}}, + "no profiles": {Name: "m", Source: "/m"}, + "no class": {Name: "m", Source: "/m", Profiles: []Profile{{Name: "p", TP: 1}}}, + } + for name, m := range cases { + t.Run(name, func(t *testing.T) { + if _, err := New([]Model{m}); err == nil { + t.Error("accepted") + } + }) + } +} + +func TestScalingBounds(t *testing.T) { + m := model("m") + m.Scaling = Scaling{Min: 5, Max: 2} + if _, err := New([]Model{m}); err == nil { + t.Error("max below min was accepted") + } +} + +func TestDuplicateProfileRejected(t *testing.T) { + _, err := New([]Model{model("m", + Profile{Name: "p", GPUClass: "h100-80g", TP: 1}, + Profile{Name: "p", GPUClass: "a100-40g", TP: 1}, + )}) + if err == nil || !strings.Contains(err.Error(), "twice") { + t.Errorf("duplicate profile accepted: %v", err) + } +} + +func TestPlacementProfilesCarryPriority(t *testing.T) { + c, _ := New([]Model{model("m", + Profile{Name: "fast", GPUClass: "h100-80g", TP: 4, Priority: 100}, + Profile{Name: "slow", GPUClass: "a100-40g", TP: 4, Priority: 50}, + )}) + m, _ := c.Resolve("m") + + pp := m.PlacementProfiles() + if len(pp) != 2 { + t.Fatalf("got %d placement profiles", len(pp)) + } + byName := map[string]int{} + for _, p := range pp { + byName[p.Name] = p.Priority + } + if byName["fast"] != 100 || byName["slow"] != 50 { + t.Errorf("priorities lost in conversion: %v", byName) + } +} + +func TestEngineProfileCarriesModelIdentity(t *testing.T) { + // The served name is what clients ask for and need not match the path. + c, _ := New([]Model{model("llama-3.3-70b")}) + m, _ := c.Resolve("llama-3.3-70b") + + ep, ok := m.EngineProfile("p1") + if !ok { + t.Fatal("engine profile not found") + } + if ep.ModelPath != "/models/llama-3.3-70b" { + t.Errorf("ModelPath=%q", ep.ModelPath) + } + if ep.ServedModelName != "llama-3.3-70b" { + t.Errorf("ServedModelName=%q", ep.ServedModelName) + } + + if _, ok := m.EngineProfile("nope"); ok { + t.Error("an unknown profile resolved") + } +} + +func TestEngineProfileArgsAreCopied(t *testing.T) { + // The reconciler hands these to an engine adapter; mutating them must not + // corrupt the catalog for every later replica. + c, _ := New([]Model{model("m", Profile{ + Name: "p", GPUClass: "h100-80g", TP: 1, Args: []string{"--max-model-len", "32768"}, + })}) + m, _ := c.Resolve("m") + + ep, _ := m.EngineProfile("p") + ep.Args[0] = "tampered" + + again, _ := m.EngineProfile("p") + if again.Args[0] != "--max-model-len" { + t.Error("EngineProfile exposed the catalog's own slice") + } +} + +func TestNamesAndModelsOrder(t *testing.T) { + c, _ := New([]Model{model("zeta"), model("alpha")}) + + // Models keeps declaration order, so configuration reads as written. + if got := c.Models(); got[0].Name != "zeta" { + t.Errorf("Models order=%q", got[0].Name) + } + // Names is sorted, for stable CLI output. + if got := c.Names(); got[0] != "alpha" { + t.Errorf("Names order=%v", got) + } +} diff --git a/internal/control/inventory/testhooks.go b/internal/control/inventory/testhooks.go new file mode 100644 index 0000000..f9b9aca --- /dev/null +++ b/internal/control/inventory/testhooks.go @@ -0,0 +1,16 @@ +package inventory + +import "time" + +// SetLastSeen overrides a node's heartbeat timestamp. +// +// It exists for tests that run on a synthetic clock: Upsert and Touch stamp the +// wall clock, which is decades away from a fake timeline and would make every +// node look silent. +func (s *Store) SetLastSeen(id string, at time.Time) { + s.mu.Lock() + defer s.mu.Unlock() + if n, ok := s.nodes[id]; ok { + n.LastSeen = at + } +} diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go new file mode 100644 index 0000000..cd936ca --- /dev/null +++ b/internal/control/reconciler/reconciler.go @@ -0,0 +1,590 @@ +package reconciler + +import ( + "context" + "fmt" + "log/slog" + "math/rand/v2" + "slices" + "sync" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/placement" +) + +// ShardCommand is everything an agent needs to launch one shard. +type ShardCommand struct { + ReplicaID string + ShardID string + NodeID string + Spec engines.ShardSpec + // StartupTimeout comes from the profile; model loading dominates it. + StartupTimeout time.Duration +} + +// Dispatcher sends commands to agents. +// +// It is an interface so the reconciliation loop can be tested without a network: +// the loop's logic is where the interesting failures are, not the transport. +type Dispatcher interface { + StartShard(ctx context.Context, cmd ShardCommand) error + StopShard(ctx context.Context, nodeID, shardID string) error + DrainShard(ctx context.Context, nodeID, shardID string) error +} + +// Config tunes the reconciliation loop. +type Config struct { + // Interval is how often desired and actual are compared when nothing else + // happens. Events wake the loop sooner. + Interval time.Duration + // BaseBackoff is how long to wait before rebuilding a failed replica. + BaseBackoff time.Duration + // MaxBackoff caps the exponential growth. + MaxBackoff time.Duration + // UnreachableAfter is how long without a heartbeat before a node is + // unreachable. + UnreachableAfter time.Duration + // LostAfter is how long unreachable before its replicas are declared dead. + // Longer than UnreachableAfter on purpose: a brief network blip must not + // tear down a replica that is still serving perfectly well. + LostAfter time.Duration + // PortRange is the per-node port range for engine HTTP and dist-init. + PortRange [2]int + // MgmtInterface is the ethernet interface used for NCCL bootstrap. + MgmtInterface string +} + +// DefaultConfig returns sensible defaults. +func DefaultConfig() Config { + return Config{ + Interval: 5 * time.Second, + BaseBackoff: 10 * time.Second, + MaxBackoff: 10 * time.Minute, + UnreachableAfter: 15 * time.Second, + LostAfter: 60 * time.Second, + PortRange: [2]int{20000, 20999}, + MgmtInterface: "eth0", + } +} + +func (c *Config) normalize() { + d := DefaultConfig() + if c.Interval <= 0 { + c.Interval = d.Interval + } + if c.BaseBackoff <= 0 { + c.BaseBackoff = d.BaseBackoff + } + if c.MaxBackoff < c.BaseBackoff { + c.MaxBackoff = max(c.BaseBackoff, d.MaxBackoff) + } + if c.UnreachableAfter <= 0 { + c.UnreachableAfter = d.UnreachableAfter + } + if c.LostAfter < c.UnreachableAfter { + c.LostAfter = max(c.UnreachableAfter*4, d.LostAfter) + } + if c.PortRange[1] <= c.PortRange[0] { + c.PortRange = d.PortRange + } + if c.MgmtInterface == "" { + c.MgmtInterface = d.MgmtInterface + } +} + +// Reconciler compares the catalog against what is running and closes the gap. +type Reconciler struct { + cfg Config + cat *catalog.Catalog + inv *inventory.Store + sched *placement.Scheduler + disp Dispatcher + log *slog.Logger + now func() time.Time + + mu sync.Mutex + replicas map[string]*Replica + // nextAttempt is the earliest a model may be rebuilt, per model. + nextAttempt map[string]time.Time + // attempts counts consecutive failures per model, for the backoff. + attempts map[string]int + // lastExplain keeps the most recent placement failure per model, so + // `ctl explain-placement` can answer without re-running placement. + lastExplain map[string]placement.Result + seq int + + wake chan struct{} + stop chan struct{} + done chan struct{} + once sync.Once +} + +// New builds a reconciler. +func New(cfg Config, cat *catalog.Catalog, inv *inventory.Store, disp Dispatcher, log *slog.Logger) *Reconciler { + cfg.normalize() + if log == nil { + log = slog.Default() + } + return &Reconciler{ + cfg: cfg, cat: cat, inv: inv, disp: disp, log: log, + sched: placement.New(inv), + now: time.Now, + replicas: map[string]*Replica{}, + nextAttempt: map[string]time.Time{}, + attempts: map[string]int{}, + lastExplain: map[string]placement.Result{}, + wake: make(chan struct{}, 1), + stop: make(chan struct{}), + done: make(chan struct{}), + } +} + +// SetClock replaces the time source, for tests. +func (r *Reconciler) SetClock(now func() time.Time) { r.now = now } + +// Start runs the loop until Close. +func (r *Reconciler) Start(ctx context.Context) { + go func() { + defer close(r.done) + t := time.NewTicker(r.cfg.Interval) + defer t.Stop() + + r.Reconcile(ctx) + for { + select { + case <-r.stop: + return + case <-ctx.Done(): + return + case <-r.wake: + r.Reconcile(ctx) + case <-t.C: + r.Reconcile(ctx) + } + } + }() +} + +// Close stops the loop. +func (r *Reconciler) Close() { + r.once.Do(func() { + close(r.stop) + <-r.done + }) +} + +// Notify wakes the loop, for when something changed and waiting for the next +// tick would add pointless latency. +func (r *Reconciler) Notify() { + select { + case r.wake <- struct{}{}: + default: + } +} + +// Reconcile performs one pass. It is exported so tests can drive it directly +// instead of racing a ticker. +func (r *Reconciler) Reconcile(ctx context.Context) { + r.expireNodes() + r.reapFailed(ctx) + + for _, m := range r.cat.Models() { + r.reconcileModel(ctx, m) + } +} + +// expireNodes moves silent nodes through Unreachable to Lost, and fails the +// replicas of nodes that are gone. +// +// The two-stage delay is deliberate: a brief network blip must not tear down +// replicas that are still serving traffic perfectly well, and rebuilding a +// multi-node replica costs tens of minutes of model loading. +func (r *Reconciler) expireNodes() { + now := r.now() + for _, n := range r.inv.Nodes() { + silent := now.Sub(n.LastSeen) + switch { + case n.State == inventory.Ready && silent > r.cfg.UnreachableAfter: + r.inv.SetState(n.ID, inventory.Unreachable) + r.log.Warn("node is unreachable", "node", n.ID, "silent_for", silent.Round(time.Second)) + case n.State == inventory.Unreachable && silent > r.cfg.LostAfter: + r.inv.SetState(n.ID, inventory.Lost) + r.log.Error("node lost; its replicas will be rebuilt elsewhere", + "node", n.ID, "silent_for", silent.Round(time.Second)) + r.failReplicasOn(n.ID, "node "+n.ID+" was lost") + } + } +} + +func (r *Reconciler) failReplicasOn(nodeID, reason string) { + r.mu.Lock() + defer r.mu.Unlock() + for _, rep := range r.replicas { + if rep.State.Terminal() || !rep.OnNode(nodeID) { + continue + } + rep.State = Failed + rep.Err = reason + } +} + +// reapFailed releases the resources of terminal replicas and tells the agents +// to clean up whatever might still be running. +func (r *Reconciler) reapFailed(ctx context.Context) { + r.mu.Lock() + var doomed []*Replica + for _, rep := range r.replicas { + if rep.State.Terminal() { + doomed = append(doomed, rep) + } + } + for _, rep := range doomed { + delete(r.replicas, rep.ID) + if rep.State == Failed { + // Carry the failure count on the model, not the replica: the + // replica is about to disappear, and the backoff has to survive it. + r.attempts[rep.Model]++ + r.nextAttempt[rep.Model] = r.now().Add(r.backoff(r.attempts[rep.Model])) + } + } + r.mu.Unlock() + + for _, rep := range doomed { + if rep.State == Failed { + r.log.Error("replica failed", + "replica", rep.ID, "model", rep.Model, "err", rep.Err, + "retry_in", r.backoff(r.attempts[rep.Model]).Round(time.Second)) + } + // Stop every shard, including the ones that looked healthy: a partially + // alive replica holds GPUs while serving nothing. + for _, s := range rep.Shards { + if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil { + r.log.Warn("stopping a shard of a dead replica failed", + "replica", rep.ID, "shard", s.ID, "err", err) + } + } + r.inv.ReleaseReplica(rep.ID) + } +} + +// backoff grows with consecutive failures and is jittered, so a cluster-wide +// problem does not produce synchronized retry storms across every model. +func (r *Reconciler) backoff(attempt int) time.Duration { + if attempt <= 0 { + return 0 + } + d := r.cfg.BaseBackoff << min(attempt-1, 16) + if d > r.cfg.MaxBackoff || d <= 0 { + d = r.cfg.MaxBackoff + } + jitter := time.Duration(rand.Int64N(int64(d / 4))) //nolint:gosec // jitter + return d - d/8 + jitter +} + +func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) { + r.mu.Lock() + var live, draining []*Replica + for _, rep := range r.replicas { + if rep.Model != m.Name { + continue + } + if rep.State.Counts() { + live = append(live, rep) + } else if rep.State == Draining { + draining = append(draining, rep) + } + } + blockedUntil := r.nextAttempt[m.Name] + r.mu.Unlock() + + desired := m.Scaling.Min + switch { + case len(live) < desired: + if r.now().Before(blockedUntil) { + // Still in backoff. Rebuilding immediately would hot-loop on a + // model that cannot start — a missing weights path, say — and bury + // the real error in noise. + return + } + r.scaleUp(ctx, m, live) + case len(live) > desired: + r.scaleDown(ctx, live[desired:]) + } + + // Drained replicas can finally be stopped. + for _, rep := range draining { + r.stopReplica(ctx, rep) + } +} + +// scaleUp places and starts one replica per pass. +// +// One at a time on purpose: placement decisions depend on what is already +// allocated, and starting several at once from the same inventory snapshot +// would hand them the same GPUs. +func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, live []*Replica) { + var avoid []string + for _, rep := range live { + avoid = append(avoid, rep.NodeIDs()...) + } + + res := r.sched.Place(m.PlacementProfiles(), avoid) + if res.Placement == nil && len(avoid) > 0 { + // Spreading across failure domains is a preference, not a rule. If it + // is the only thing standing in the way, take the placement anyway: + // a second replica on one node still survives an engine crash. + res = r.sched.Place(m.PlacementProfiles(), nil) + } + + r.mu.Lock() + r.lastExplain[m.Name] = res + r.mu.Unlock() + + if res.Placement == nil { + r.log.Warn("no placement for model", "model", m.Name, "reasons", res.Explain()) + return + } + if err := r.startReplica(ctx, m, res.Placement); err != nil { + r.log.Error("starting replica failed", "model", m.Name, "err", err) + } +} + +func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *placement.Placement) error { + prof, ok := m.ProfileByName(pl.Profile.Name) + if !ok { + return fmt.Errorf("profile %q vanished from model %q", pl.Profile.Name, m.Name) + } + espec, ok := m.EngineProfile(prof.Name) + if !ok { + return fmt.Errorf("model %q: no engine profile %q", m.Name, prof.Name) + } + + r.mu.Lock() + r.seq++ + id := fmt.Sprintf("%s-%d", m.Name, r.seq) + attempt := r.attempts[m.Name] + r.mu.Unlock() + + rep := &Replica{ + ID: id, Model: m.Name, Profile: prof.Name, + Fabric: pl.Fabric, Pool: pl.Pool, State: Placing, + Weight: prof.Weight, CreatedAt: r.now(), Attempt: attempt, + } + + // The leader's address and port are fixed before any shard starts: every + // rank needs the same rendezvous point, and a worker that starts first must + // already know where to dial. + leader := pl.Nodes[0] + leaderNode, ok := r.inv.Get(leader.NodeID) + if !ok { + return fmt.Errorf("leader node %s disappeared", leader.NodeID) + } + basePort := r.allocatePorts(pl) + + for _, a := range pl.Nodes { + shardID := fmt.Sprintf("%s-r%d", id, a.Rank) + rep.Shards = append(rep.Shards, Shard{ + ID: shardID, NodeID: a.NodeID, Rank: a.Rank, + GPUIndices: a.GPUIndices, HCAs: a.HCAs, + HTTPPort: basePort, State: supervisor.Pending, + }) + r.inv.Allocate(a.NodeID, inventory.Allocation{ + ShardID: shardID, ReplicaID: id, GPUIndices: a.GPUIndices, + }) + } + rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort) + + r.mu.Lock() + r.replicas[id] = rep + r.mu.Unlock() + + // Workers first, then the leader. SGLang's ranks rendezvous, and vLLM under + // Ray needs its workers attached before the head starts serving; starting + // rank 0 first just makes it wait. + order := slices.Clone(rep.Shards) + slices.SortFunc(order, func(a, b Shard) int { return b.Rank - a.Rank }) + + for _, s := range order { + cmd := ShardCommand{ + ReplicaID: id, + ShardID: s.ID, + NodeID: s.NodeID, + Spec: engines.ShardSpec{ + ReplicaID: id, + ShardID: s.ID, + Rank: s.Rank, + NNodes: len(pl.Nodes), + LeaderAddr: addrOf(leaderNode), + LeaderPort: basePort + 1, + GPUIndices: s.GPUIndices, + HCAs: s.HCAs, + MgmtInterface: r.cfg.MgmtInterface, + HTTPPort: s.HTTPPort, + Profile: espec, + }, + StartupTimeout: prof.StartupTimeout, + } + if err := r.disp.StartShard(ctx, cmd); err != nil { + r.markShard(id, s.ID, supervisor.Failed, err.Error()) + return fmt.Errorf("starting shard %s: %w", s.ID, err) + } + r.markShard(id, s.ID, supervisor.Starting, "") + } + + r.log.Info("replica placed", + "replica", id, "model", m.Name, "profile", prof.Name, + "pool", pl.Pool, "fabric", pl.Fabric, "nodes", rep.NodeIDs()) + return nil +} + +// allocatePorts picks a port shared by every shard of a replica. +// +// The same port on every node keeps the leader address predictable and the +// port bookkeeping trivial; nodes do not share a network namespace, so there is +// no conflict in reusing it. +func (r *Reconciler) allocatePorts(pl *placement.Placement) int { + used := map[int]struct{}{} + r.mu.Lock() + for _, rep := range r.replicas { + for _, s := range rep.Shards { + used[s.HTTPPort] = struct{}{} + } + } + r.mu.Unlock() + + // Two ports per replica: HTTP and the distributed init rendezvous. + for p := r.cfg.PortRange[0]; p < r.cfg.PortRange[1]; p += 2 { + if _, taken := used[p]; !taken { + return p + } + } + return r.cfg.PortRange[0] +} + +func (r *Reconciler) scaleDown(ctx context.Context, excess []*Replica) { + for _, rep := range excess { + r.mu.Lock() + rep.State = Draining + shards := slices.Clone(rep.Shards) + id := rep.ID + r.mu.Unlock() + + r.log.Info("draining replica", "replica", id) + for _, s := range shards { + if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil { + r.log.Warn("draining a shard failed", "shard", s.ID, "err", err) + } + } + } +} + +func (r *Reconciler) stopReplica(ctx context.Context, rep *Replica) { + r.mu.Lock() + shards := slices.Clone(rep.Shards) + rep.State = Stopped + id := rep.ID + r.mu.Unlock() + + for _, s := range shards { + if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil { + r.log.Warn("stopping a shard failed", "shard", s.ID, "err", err) + } + } + r.inv.ReleaseReplica(id) + + r.mu.Lock() + delete(r.replicas, id) + r.mu.Unlock() +} + +// OnShardStatus records a status report from an agent. +func (r *Reconciler) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) { + r.markShard(replicaID, shardID, st, errMsg) + r.Notify() +} + +func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, errMsg string) { + r.mu.Lock() + defer r.mu.Unlock() + + rep, ok := r.replicas[replicaID] + if !ok { + return + } + for i := range rep.Shards { + if rep.Shards[i].ID != shardID { + continue + } + rep.Shards[i].State = st + if errMsg != "" { + rep.Shards[i].Err = errMsg + } + break + } + + prev := rep.State + rep.deriveState() + if rep.State == Ready && prev != Ready { + // A model that comes up clears its backoff; the next failure starts + // counting from scratch rather than from an old streak. + r.attempts[rep.Model] = 0 + delete(r.nextAttempt, rep.Model) + } +} + +// Replicas returns a snapshot of every replica, sorted by id. +func (r *Reconciler) Replicas() []Replica { + r.mu.Lock() + defer r.mu.Unlock() + + out := make([]Replica, 0, len(r.replicas)) + for _, rep := range r.replicas { + clone := *rep + clone.Shards = slices.Clone(rep.Shards) + out = append(out, clone) + } + slices.SortFunc(out, func(a, b Replica) int { + if a.ID < b.ID { + return -1 + } + if a.ID > b.ID { + return 1 + } + return 0 + }) + return out +} + +// ReadyReplicas returns the replicas the router may send traffic to. +func (r *Reconciler) ReadyReplicas() []Replica { + var out []Replica + for _, rep := range r.Replicas() { + if rep.State == Ready { + out = append(out, rep) + } + } + return out +} + +// Explain returns the most recent placement result for a model, which is what +// `ctl explain-placement` prints. +func (r *Reconciler) Explain(model string) (placement.Result, bool) { + r.mu.Lock() + defer r.mu.Unlock() + res, ok := r.lastExplain[model] + return res, ok +} + +func addrOf(n inventory.Node) string { + if n.MgmtAddr != "" { + return n.MgmtAddr + } + if n.Hostname != "" { + return n.Hostname + } + return n.ID +} diff --git a/internal/control/reconciler/reconciler_test.go b/internal/control/reconciler/reconciler_test.go new file mode 100644 index 0000000..47662f4 --- /dev/null +++ b/internal/control/reconciler/reconciler_test.go @@ -0,0 +1,598 @@ +package reconciler + +import ( + "context" + "fmt" + "log/slog" + "slices" + "strings" + "sync" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +// fakeDispatcher records commands instead of sending them. +type fakeDispatcher struct { + mu sync.Mutex + started []ShardCommand + stopped []string + drained []string + failNext error +} + +func (d *fakeDispatcher) StartShard(_ context.Context, cmd ShardCommand) error { + d.mu.Lock() + defer d.mu.Unlock() + if d.failNext != nil { + err := d.failNext + d.failNext = nil + return err + } + d.started = append(d.started, cmd) + return nil +} + +func (d *fakeDispatcher) StopShard(_ context.Context, _, shardID string) error { + d.mu.Lock() + defer d.mu.Unlock() + d.stopped = append(d.stopped, shardID) + return nil +} + +func (d *fakeDispatcher) DrainShard(_ context.Context, _, shardID string) error { + d.mu.Lock() + defer d.mu.Unlock() + d.drained = append(d.drained, shardID) + return nil +} + +func (d *fakeDispatcher) startedShards() []ShardCommand { + d.mu.Lock() + defer d.mu.Unlock() + return slices.Clone(d.started) +} + +func (d *fakeDispatcher) reset() { + d.mu.Lock() + defer d.mu.Unlock() + d.started, d.stopped, d.drained = nil, nil, nil +} + +// clock lets tests move time without sleeping. +type clock struct { + mu sync.Mutex + t time.Time +} + +func newClock() *clock { + return &clock{t: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)} +} +func (c *clock) now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.t +} +func (c *clock) advance(d time.Duration) { + c.mu.Lock() + c.t = c.t.Add(d) + c.mu.Unlock() +} + +func gpuNode(id string, n int, fabrics ...string) inventory.Node { + nd := inventory.Node{ID: id, MgmtAddr: "10.0.0." + id, State: inventory.Ready} + for i := range n { + nd.GPUs = append(nd.GPUs, discovery.GPU{ + Index: i, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, ComputeCapability: "9.0", + }) + } + for _, f := range fabrics { + nd.Fabrics = append(nd.Fabrics, inventory.FabricAttachment{ + Fabric: f, HCAs: []string{"mlx5_0", "mlx5_1"}, + }) + } + return nd +} + +func testModel(min int, profiles ...catalog.Profile) catalog.Model { + if len(profiles) == 0 { + profiles = []catalog.Profile{{Name: "tp4", GPUClass: "h100-80g", TP: 4}} + } + return catalog.Model{ + Name: "llama", Source: "/models/llama", + Scaling: catalog.Scaling{Min: min, Max: min + 2}, Profiles: profiles, + } +} + +type harness struct { + r *Reconciler + inv *inventory.Store + disp *fakeDispatcher + clock *clock +} + +func newHarness(t *testing.T, m catalog.Model, nodes ...inventory.Node) *harness { + t.Helper() + + cat, err := catalog.New([]catalog.Model{m}) + if err != nil { + t.Fatal(err) + } + inv := inventory.NewStore() + c := newClock() + for _, n := range nodes { + inv.Upsert(n) + inv.Touch(n.ID) + } + + disp := &fakeDispatcher{} + cfg := DefaultConfig() + cfg.UnreachableAfter = 15 * time.Second + cfg.LostAfter = 60 * time.Second + + r := New(cfg, cat, inv, disp, slog.New(slog.DiscardHandler)) + r.SetClock(c.now) + // Nodes were just registered, so their LastSeen is wall-clock now, which is + // far from the fake clock's epoch. Re-stamp them onto the fake timeline. + for _, n := range nodes { + touchAt(inv, n.ID, c.now()) + } + return &harness{r: r, inv: inv, disp: disp, clock: c} +} + +// touchAt rewrites a node's LastSeen onto the fake clock's timeline. +func touchAt(inv *inventory.Store, id string, at time.Time) { + n, ok := inv.Get(id) + if !ok { + return + } + n.LastSeen = at + n.State = inventory.Ready + inv.Upsert(n) + // Upsert stamps LastSeen with the wall clock, so set it again directly. + inv.SetLastSeen(id, at) +} + +// advance moves the clock and keeps every node heartbeating. +// +// Without this, advancing time past UnreachableAfter makes every node look +// silent and placement stops — which is correct behaviour, but not what the +// backoff tests are trying to exercise. +func (h *harness) advance(d time.Duration) { + h.clock.advance(d) + for _, n := range h.inv.Nodes() { + touchAt(h.inv, n.ID, h.clock.now()) + } +} + +// markReady walks every started shard to Ready. +func (h *harness) markReady() { + for _, cmd := range h.disp.startedShards() { + h.r.OnShardStatus(cmd.ReplicaID, cmd.ShardID, supervisor.Ready, "") + } +} + +func TestPlacesMinimumReplicas(t *testing.T) { + h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8)) + + h.r.Reconcile(t.Context()) + h.markReady() + h.r.Reconcile(t.Context()) + h.markReady() + + reps := h.r.Replicas() + if len(reps) != 2 { + t.Fatalf("expected 2 replicas, got %d: %+v", len(reps), reps) + } + for _, rep := range reps { + if rep.State != Ready { + t.Errorf("replica %s is %s", rep.ID, rep.State) + } + if rep.Endpoint == "" { + t.Errorf("replica %s has no endpoint", rep.ID) + } + } +} + +func TestOneReplicaPerPass(t *testing.T) { + // Placement depends on what is already allocated, so starting several from + // one inventory snapshot would hand them the same GPUs. + h := newHarness(t, testModel(3), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8)) + + h.r.Reconcile(t.Context()) + if got := len(h.r.Replicas()); got != 1 { + t.Fatalf("one pass created %d replicas", got) + } + + seen := map[int]string{} + for range 3 { + h.markReady() + h.r.Reconcile(t.Context()) + } + for _, rep := range h.r.Replicas() { + for _, s := range rep.Shards { + for _, g := range s.GPUIndices { + key := g + if prev, dup := seen[key]; dup && prev != s.NodeID { + continue // same index on a different node is fine + } + seen[key] = s.NodeID + } + } + } + if got := len(h.r.Replicas()); got != 3 { + t.Errorf("expected 3 replicas, got %d", got) + } +} + +func TestSpreadsAcrossNodes(t *testing.T) { + h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8)) + + for range 2 { + h.r.Reconcile(t.Context()) + h.markReady() + } + + nodes := map[string]struct{}{} + for _, rep := range h.r.Replicas() { + for _, n := range rep.NodeIDs() { + nodes[n] = struct{}{} + } + } + if len(nodes) != 2 { + t.Errorf("both replicas landed on %d node(s); they should spread", len(nodes)) + } +} + +func TestPacksWhenSpreadingIsImpossible(t *testing.T) { + // Spreading is a preference, not a rule: a second replica on one node still + // survives an engine crash, which is the common failure. + h := newHarness(t, testModel(2), gpuNode("only", 8)) + + for range 2 { + h.r.Reconcile(t.Context()) + h.markReady() + } + if got := len(h.r.Replicas()); got != 2 { + t.Fatalf("expected 2 replicas on the single node, got %d", got) + } +} + +func TestShardOrderWorkersFirst(t *testing.T) { + // SGLang ranks rendezvous and vLLM under Ray needs its workers attached + // before the head serves, so rank 0 goes last. + h := newHarness(t, testModel(1, catalog.Profile{ + Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8, + }), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a")) + + h.r.Reconcile(t.Context()) + + started := h.disp.startedShards() + if len(started) != 2 { + t.Fatalf("expected 2 shards, got %d", len(started)) + } + if started[0].Spec.Rank != 1 || started[1].Spec.Rank != 0 { + t.Errorf("start order was rank %d then %d; workers must start first", + started[0].Spec.Rank, started[1].Spec.Rank) + } + // Every rank must be given the same rendezvous point. + if started[0].Spec.LeaderAddr != started[1].Spec.LeaderAddr { + t.Errorf("ranks got different leader addresses: %q and %q", + started[0].Spec.LeaderAddr, started[1].Spec.LeaderAddr) + } + if started[0].Spec.LeaderPort != started[1].Spec.LeaderPort { + t.Error("ranks got different leader ports") + } + for _, cmd := range started { + if len(cmd.Spec.HCAs) == 0 { + t.Errorf("rank %d got no HCAs; NCCL would choose its own", cmd.Spec.Rank) + } + } +} + +func TestFailedShardFailsWholeReplica(t *testing.T) { + // One dead shard makes the others useless: they wait on collectives that + // will never complete while holding GPUs (plan §5.4). + h := newHarness(t, testModel(1, catalog.Profile{ + Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8, + }), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a")) + + h.r.Reconcile(t.Context()) + started := h.disp.startedShards() + + // One shard ready, the other dead. + h.r.OnShardStatus(started[0].ReplicaID, started[0].ShardID, supervisor.Ready, "") + h.r.OnShardStatus(started[1].ReplicaID, started[1].ShardID, supervisor.Failed, "CUDA out of memory") + + h.r.Reconcile(t.Context()) + + // Both shards must have been stopped, not just the failed one. + h.disp.mu.Lock() + stopped := slices.Clone(h.disp.stopped) + h.disp.mu.Unlock() + if len(stopped) != 2 { + t.Errorf("stopped %d shards, expected both: %v", len(stopped), stopped) + } + // And the GPUs must be free again. + for _, id := range []string{"1", "2"} { + if got := len(h.inv.FreeGPUs(id)); got != 8 { + t.Errorf("node %s has %d free GPUs; the replica should have been released", id, got) + } + } +} + +func TestBackoffAfterFailure(t *testing.T) { + h := newHarness(t, testModel(1), gpuNode("1", 8)) + + h.r.Reconcile(t.Context()) + started := h.disp.startedShards() + h.r.OnShardStatus(started[0].ReplicaID, started[0].ShardID, supervisor.Failed, "boom") + h.r.Reconcile(t.Context()) // reaps it + + h.disp.reset() + h.r.Reconcile(t.Context()) + if got := len(h.disp.startedShards()); got != 0 { + t.Errorf("a replica was rebuilt immediately after failure; backoff is not applied") + } + + // Past the backoff, it tries again. + h.advance(DefaultConfig().MaxBackoff) + h.r.Reconcile(t.Context()) + if got := len(h.disp.startedShards()); got == 0 { + t.Error("the replica was never rebuilt after the backoff expired") + } +} + +func TestSuccessClearsBackoff(t *testing.T) { + h := newHarness(t, testModel(1), gpuNode("1", 8)) + + // Fail once. + h.r.Reconcile(t.Context()) + s := h.disp.startedShards()[0] + h.r.OnShardStatus(s.ReplicaID, s.ShardID, supervisor.Failed, "boom") + h.r.Reconcile(t.Context()) + + // Then succeed. + h.advance(DefaultConfig().MaxBackoff) + h.disp.reset() + h.r.Reconcile(t.Context()) + if len(h.disp.startedShards()) == 0 { + t.Fatal("no replica was started after the backoff expired") + } + s2 := h.disp.startedShards()[0] + h.r.OnShardStatus(s2.ReplicaID, s2.ShardID, supervisor.Ready, "") + + // A later failure should start from a short backoff, not a long streak. + h.r.OnShardStatus(s2.ReplicaID, s2.ShardID, supervisor.Failed, "again") + h.r.Reconcile(t.Context()) + h.advance(2 * DefaultConfig().BaseBackoff) + h.disp.reset() + h.r.Reconcile(t.Context()) + if got := len(h.disp.startedShards()); got == 0 { + t.Error("backoff was not reset by the successful run in between") + } +} + +// TestLostNodeMovesReplicas is the phase 3 exit criterion: a node going away +// must get its replicas rebuilt elsewhere. +func TestLostNodeMovesReplicas(t *testing.T) { + h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8)) + + h.r.Reconcile(t.Context()) + h.markReady() + h.r.Reconcile(t.Context()) + + reps := h.r.Replicas() + if len(reps) != 1 || reps[0].State != Ready { + t.Fatalf("setup failed: %+v", reps) + } + original := reps[0].NodeIDs()[0] + survivor := "2" + if original == "2" { + survivor = "1" + } + + // The surviving node keeps reporting; the other goes silent. + h.clock.advance(20 * time.Second) + touchAt(h.inv, survivor, h.clock.now()) + h.r.Reconcile(t.Context()) + + if n, _ := h.inv.Get(original); n.State != inventory.Unreachable { + t.Errorf("silent node is %s, expected unreachable", n.State) + } + // Unreachable is not lost: a blip must not tear down a working replica. + if reps := h.r.Replicas(); len(reps) != 1 || reps[0].State != Ready { + t.Fatalf("an unreachable node already destroyed the replica: %+v", reps) + } + + h.clock.advance(90 * time.Second) + touchAt(h.inv, survivor, h.clock.now()) + h.disp.reset() + h.r.Reconcile(t.Context()) // declares it lost and reaps the replica + + if n, _ := h.inv.Get(original); n.State != inventory.Lost { + t.Fatalf("node is %s, expected lost", n.State) + } + + // Past the backoff, the replica is rebuilt on the survivor. Only the + // survivor heartbeats; the lost node stays gone. + h.clock.advance(DefaultConfig().MaxBackoff) + touchAt(h.inv, survivor, h.clock.now()) + h.r.Reconcile(t.Context()) + h.markReady() + h.r.Reconcile(t.Context()) + + reps = h.r.Replicas() + if len(reps) != 1 { + t.Fatalf("expected the replica to be rebuilt, got %d", len(reps)) + } + if got := reps[0].NodeIDs()[0]; got != survivor { + t.Errorf("replica was rebuilt on %q, expected the surviving node %q", got, survivor) + } + if reps[0].State != Ready { + t.Errorf("rebuilt replica is %s", reps[0].State) + } +} + +func TestHeartbeatRevivesNode(t *testing.T) { + h := newHarness(t, testModel(1), gpuNode("1", 8)) + + h.clock.advance(20 * time.Second) // deliberately without a heartbeat + h.r.Reconcile(t.Context()) + if n, _ := h.inv.Get("1"); n.State != inventory.Unreachable { + t.Fatalf("node is %s", n.State) + } + + h.inv.Touch("1") + if n, _ := h.inv.Get("1"); n.State != inventory.Ready { + t.Errorf("a heartbeat did not revive the node: %s", n.State) + } +} + +func TestScaleDownDrainsFirst(t *testing.T) { + // Draining before stopping lets in-flight requests finish instead of being + // cut off mid-stream. + h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8)) + for range 2 { + h.r.Reconcile(t.Context()) + h.markReady() + } + if got := len(h.r.Replicas()); got != 2 { + t.Fatalf("setup: %d replicas", got) + } + + // Shrink the catalog to one replica. + cat, err := catalog.New([]catalog.Model{testModel(1)}) + if err != nil { + t.Fatal(err) + } + h.r.cat = cat + + h.disp.reset() + h.r.Reconcile(t.Context()) + + h.disp.mu.Lock() + drained, stopped := len(h.disp.drained), len(h.disp.stopped) + h.disp.mu.Unlock() + if drained == 0 { + t.Error("the excess replica was not drained") + } + if stopped != 0 { + t.Error("a replica was stopped before it had drained") + } + + // The next pass stops what has drained. + h.r.Reconcile(t.Context()) + h.disp.mu.Lock() + stopped = len(h.disp.stopped) + h.disp.mu.Unlock() + if stopped == 0 { + t.Error("the drained replica was never stopped") + } + if got := len(h.r.Replicas()); got != 1 { + t.Errorf("expected 1 replica after scale-down, got %d", got) + } +} + +func TestUnplaceableModelExplains(t *testing.T) { + // A model that cannot be placed must leave a usable explanation rather than + // a silent absence. + h := newHarness(t, testModel(1, catalog.Profile{ + Name: "tp8", GPUClass: "h100-80g", TP: 8, + }), gpuNode("1", 4)) + + h.r.Reconcile(t.Context()) + if got := len(h.r.Replicas()); got != 0 { + t.Fatalf("something was placed: %d", got) + } + + res, ok := h.r.Explain("llama") + if !ok { + t.Fatal("no explanation was recorded") + } + exp := res.Explain() + if !strings.Contains(exp, "needs 8") { + t.Errorf("explanation does not say what was missing:\n%s", exp) + } +} + +func TestDispatcherFailureFailsReplica(t *testing.T) { + h := newHarness(t, testModel(1), gpuNode("1", 8)) + h.disp.failNext = fmt.Errorf("agent unreachable") + + h.r.Reconcile(t.Context()) + + reps := h.r.Replicas() + if len(reps) != 1 || reps[0].State != Failed { + t.Fatalf("a failed dispatch did not fail the replica: %+v", reps) + } + + // And the next pass must release its GPUs. + h.r.Reconcile(t.Context()) + if got := len(h.inv.FreeGPUs("1")); got != 8 { + t.Errorf("node has %d free GPUs; the failed replica still holds some", got) + } +} + +func TestReadyReplicasFeedTheRouter(t *testing.T) { + h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8)) + + h.r.Reconcile(t.Context()) + if got := len(h.r.ReadyReplicas()); got != 0 { + t.Errorf("a starting replica was offered to the router") + } + + h.markReady() + h.r.Reconcile(t.Context()) + h.markReady() + + ready := h.r.ReadyReplicas() + if len(ready) != 2 { + t.Fatalf("expected 2 ready replicas, got %d", len(ready)) + } + for _, rep := range ready { + if !strings.HasPrefix(rep.Endpoint, "http://") { + t.Errorf("endpoint %q is not a URL", rep.Endpoint) + } + if rep.Weight <= 0 { + t.Errorf("replica %s has no weight; the router needs one", rep.ID) + } + } +} + +func TestPortsDoNotCollide(t *testing.T) { + h := newHarness(t, testModel(3), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8)) + for range 3 { + h.r.Reconcile(t.Context()) + h.markReady() + } + + ports := map[int]string{} + for _, rep := range h.r.Replicas() { + for _, s := range rep.Shards { + if prev, dup := ports[s.HTTPPort]; dup && prev != rep.ID { + t.Errorf("replicas %s and %s both use port %d", prev, rep.ID, s.HTTPPort) + } + ports[s.HTTPPort] = rep.ID + } + } +} + +func TestReplicaStateStrings(t *testing.T) { + for st, want := range map[ReplicaState]string{ + Placing: "placing", Starting: "starting", Ready: "ready", + Draining: "draining", Stopped: "stopped", Failed: "failed", + } { + if got := st.String(); got != want { + t.Errorf("%d -> %q", int(st), got) + } + } + if !Ready.Counts() || Draining.Counts() || Failed.Counts() { + // Draining must not count, or a replacement is never started for it. + t.Error("Counts is wrong") + } +} diff --git a/internal/control/reconciler/replica.go b/internal/control/reconciler/replica.go new file mode 100644 index 0000000..1649992 --- /dev/null +++ b/internal/control/reconciler/replica.go @@ -0,0 +1,156 @@ +// Package reconciler drives the cluster toward the catalog's desired state +// (plan §6.3). +package reconciler + +import ( + "fmt" + "slices" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" +) + +// ReplicaState is where a replica stands. +type ReplicaState int + +const ( + // Placing means it has been scheduled but no shard has started. + Placing ReplicaState = iota + // Starting means shards are launching or loading. + Starting + // Ready means it serves traffic. + Ready + // Draining means it takes no new requests and will be stopped. + Draining + // Stopped means it was shut down on purpose. + Stopped + // Failed means it died or never became ready. + Failed +) + +func (s ReplicaState) String() string { + switch s { + case Placing: + return "placing" + case Starting: + return "starting" + case Ready: + return "ready" + case Draining: + return "draining" + case Stopped: + return "stopped" + case Failed: + return "failed" + default: + return "unknown" + } +} + +// Terminal reports whether the replica has finished for good. +func (s ReplicaState) Terminal() bool { return s == Stopped || s == Failed } + +// Counts reports whether the replica should count toward the desired total. +// Draining replicas do not: they are on the way out, and counting them would +// stop a replacement from being started. +func (s ReplicaState) Counts() bool { return s == Placing || s == Starting || s == Ready } + +// Shard is one node's part of a replica. +type Shard struct { + ID string + NodeID string + Rank int + GPUIndices []int + HCAs []string + HTTPPort int + State supervisor.State + Err string +} + +// Leader reports whether this shard serves HTTP for the replica. +func (s Shard) Leader() bool { return s.Rank == 0 } + +// Replica is one running copy of a model. It is the unit the router sees. +type Replica struct { + ID string + Model string + Profile string + // Fabric is set only for multi-node replicas. + Fabric string + Pool string + Shards []Shard + State ReplicaState + // Endpoint is rank 0's base URL. + Endpoint string + // Weight is the profile's capacity weight, passed to the router. + Weight float64 + + CreatedAt time.Time + ReadyAt time.Time + // Attempt counts how many times this model has failed to come up in a row, + // carried across rebuilds so the backoff does not reset on every attempt. + Attempt int + Err string +} + +// NodeIDs lists the nodes this replica occupies. +func (r Replica) NodeIDs() []string { + out := make([]string, 0, len(r.Shards)) + for _, s := range r.Shards { + out = append(out, s.NodeID) + } + slices.Sort(out) + return out +} + +// OnNode reports whether any shard runs on a node. +func (r Replica) OnNode(id string) bool { + return slices.ContainsFunc(r.Shards, func(s Shard) bool { return s.NodeID == id }) +} + +// deriveState recomputes the replica's state from its shards. +// +// A replica is only as healthy as its worst shard: one failed shard makes the +// whole thing useless, because the others are waiting on collectives that will +// never complete. That is why there is no path here that keeps a replica +// partially alive (plan §5.4). +func (r *Replica) deriveState() { + if r.State == Draining || r.State == Stopped { + return + } + if len(r.Shards) == 0 { + r.State = Placing + return + } + + ready := 0 + for _, s := range r.Shards { + switch s.State { + case supervisor.Failed: + r.State = Failed + if r.Err == "" { + r.Err = fmt.Sprintf("shard %s failed: %s", s.ID, s.Err) + } + return + case supervisor.Stopped: + // A shard stopping on its own, while the replica was not draining, + // is a failure of the replica. + r.State = Failed + if r.Err == "" { + r.Err = fmt.Sprintf("shard %s stopped unexpectedly", s.ID) + } + return + case supervisor.Ready: + ready++ + } + } + + if ready == len(r.Shards) { + if r.State != Ready { + r.ReadyAt = time.Now() + } + r.State = Ready + return + } + r.State = Starting +} diff --git a/llmgw-plan.md b/llmgw-plan.md index 6aed6ae..dd85317 100644 --- a/llmgw-plan.md +++ b/llmgw-plan.md @@ -1015,10 +1015,16 @@ work_dir: /var/lib/llmgw-agent - [x] Inventory: nodes, hardware, fabric attachments, allocations - [x] Placement: profile priority, bin packing, NVLink-aware GPU selection, `explain-placement` rejection reasons, property-based invariant tests -- [ ] Catalog and reconciler +- [x] Catalog: models, profiles, alias and fallback resolution, validation +- [x] Reconciler: desired vs actual, replica lifecycle, node expiry, backoff +- [ ] `EndpointSource` wired from the reconciler to the router +- [ ] `ctl`: nodes, models, replicas, cordon, drain, explain-placement, logs - [ ] The router's `EndpointSource` becomes dynamic - [ ] `ctl`: nodes, models, replicas, cordon, drain, explain-placement, logs -- **Exit criterion:** shutting down a node moves its replicas elsewhere while the router keeps serving. +- **Exit criterion:** shutting down a node moves its replicas elsewhere while the + router keeps serving. Half met: `TestLostNodeMovesReplicas` covers the move, + including the deliberate delay between Unreachable and Lost. The router half + needs the `EndpointSource` wiring and the agent stream. ### Phase 4 — Multi-node and fabrics (2 weeks) - [ ] Fabric identity (config plus GID prefix), verification