Files
llmgw/internal/control/reconciler/replica.go
T
Ilker ManapandClaude Opus 5 9b32531ce3
CI / test (push) Successful in 1m29s
Stop an idle model to make room for one somebody asked for
On-demand loading worked right up to the moment the cluster was full and
then stopped meaning anything: the model somebody had just asked for could
not be placed, and the cards were held by a model nobody had asked for in an
hour. That is the only situation in which any of it matters.

Deliberately narrow. Only idle replicas of other on-demand models — a model
at min 1 is a promise that it stays up, and breaking it to serve a different
request would make every guarantee in the catalog conditional on load. Only
single-node replicas, since evicting a multi-node one frees several machines
and costs tens of minutes to rebuild. One at a time, because freeing
everything that qualifies would empty a cluster for a request that needed
one node. And never a replica with no reported usage: after a router restart
that is all of them, and unknown is not idle.

Writing the test found a gap the feature had rather than a bug in the test.
Two on-demand models on a cluster with room for one would take turns
evicting each other, at a model load per cycle — worse for both than one of
them waiting. A replica is now safe from eviction for a few minutes after it
starts serving, which bounds that.

Which in turn found a latent bug: deriveState stamped ReadyAt with
time.Now() inside a package that has an injectable clock, so the new
lifetime rule could not be tested at all — the fake clock said three minutes
and the wall clock said eight months. Invisible while nothing compared
ReadyAt to anything.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 17:51:07 +02:00

231 lines
7.3 KiB
Go

// Package reconciler drives the cluster toward the catalog's desired state
// (plan §6.3).
package reconciler
import (
"fmt"
"slices"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
)
// ReplicaState is where a replica stands.
type ReplicaState int
const (
// Placing means it has been scheduled but no shard has started.
Placing ReplicaState = iota
// Starting means shards are launching or loading.
Starting
// Ready means it serves traffic.
Ready
// Draining means it takes no new requests and will be stopped.
Draining
// Stopped means it was shut down on purpose.
Stopped
// Failed means it died or never became ready.
Failed
)
func (s ReplicaState) String() string {
switch s {
case Placing:
return "placing"
case Starting:
return "starting"
case Ready:
return "ready"
case Draining:
return "draining"
case Stopped:
return "stopped"
case Failed:
return "failed"
default:
return "unknown"
}
}
// Terminal reports whether the replica has finished for good.
func (s ReplicaState) Terminal() bool { return s == Stopped || s == Failed }
// Counts reports whether the replica should count toward the desired total.
// Draining replicas do not: they are on the way out, and counting them would
// stop a replacement from being started.
func (s ReplicaState) Counts() bool { return s == Placing || s == Starting || s == Ready }
// Shard is one node's part of a replica.
type Shard struct {
ID string
NodeID string
Rank int
GPUIndices []int
HCAs []string
HTTPPort int
State supervisor.State
Err string
}
// Leader reports whether this shard serves HTTP for the replica.
func (s Shard) Leader() bool { return s.Rank == 0 }
// Replica is one running copy of a model. It is the unit the router sees.
type Replica struct {
ID string
Model string
Profile string
// Fabric is set only for multi-node replicas.
Fabric string
Pool string
Shards []Shard
State ReplicaState
// Endpoint is rank 0's base URL.
Endpoint string
// Weight is the profile's capacity weight, passed to the router.
Weight float64
// WantShards is how many shards this replica should have, from its profile.
//
// Without it a replica cannot tell "every shard I have is ready" from
// "every shard I have left is ready". A control plane that adopted half of
// a two-node replica reported it Ready with one shard and no endpoint, and
// the router was handed a replica nothing could route to.
WantShards int
// AdoptedAt is set while a replica is being reassembled from what agents
// report, and cleared once every shard has arrived.
//
// A multi-node replica does not arrive at once: each node has its own
// agent, each agent registers separately, and a replica spanning two nodes
// is therefore reported in two halves a second or so apart. Between them it
// is legitimately short of shards, and failing it there condemned it before
// its other half had a chance to register — after which that half was
// stopped as unclaimed and the model rebuilt from nothing on every restart.
AdoptedAt time.Time
CreatedAt time.Time
ReadyAt time.Time
// LastUsedAt is when a request was last routed to this replica, zero if
// none has been since the router started.
//
// The difference between a replica that is holding eight cards for work and
// one that is holding them out of habit. Nothing recorded it, so on a fleet
// with more models than GPUs there was no way to answer the first question
// anyone asks: what can I stop?
LastUsedAt time.Time
// DrainingAt is when the replica stopped taking new work, so the wait for a
// replacement can be bounded.
DrainingAt time.Time
// Attempt counts how many times this model has failed to come up in a row,
// carried across rebuilds so the backoff does not reset on every attempt.
Attempt int
Err string
}
// NodeIDs lists the nodes this replica occupies.
func (r Replica) NodeIDs() []string {
out := make([]string, 0, len(r.Shards))
for _, s := range r.Shards {
out = append(out, s.NodeID)
}
slices.Sort(out)
return out
}
// OnNode reports whether any shard runs on a node.
func (r Replica) OnNode(id string) bool {
return slices.ContainsFunc(r.Shards, func(s Shard) bool { return s.NodeID == id })
}
// deriveState recomputes the replica's state from its shards.
//
// A replica is only as healthy as its worst shard: one failed shard makes the
// whole thing useless, because the others are waiting on collectives that will
// never complete. That is why there is no path here that keeps a replica
// partially alive (plan §5.4).
// hasLiveShard reports whether any shard has actually got somewhere.
//
// Starting and Pending are not evidence of anything: they are what a replica
// looks like both while an engine loads a model and when nothing was ever
// started at all. Loading means the port is open, which only a real process
// does.
func (r *Replica) hasLiveShard() bool {
for _, s := range r.Shards {
if s.State == supervisor.Ready || s.State == supervisor.Loading {
return true
}
}
return false
}
// deriveState recomputes the replica's state from its shards.
//
// Takes the time rather than reading the clock, because ReadyAt is now
// something decisions are made against — a replica is safe from eviction for a
// while after it starts serving — and a field stamped from the wall clock
// inside a package with an injectable one cannot be reasoned about in a test.
// It was invisible while nothing compared ReadyAt to anything.
func (r *Replica) deriveState(now time.Time) {
if r.State == Draining || r.State == Stopped {
return
}
if len(r.Shards) == 0 {
r.State = Placing
return
}
ready := 0
for _, s := range r.Shards {
switch s.State {
case supervisor.Failed:
r.State = Failed
if r.Err == "" {
r.Err = fmt.Sprintf("shard %s failed: %s", s.ID, s.Err)
}
return
case supervisor.Stopped:
// A shard stopping on its own, while the replica was not draining,
// is a failure of the replica.
r.State = Failed
if r.Err == "" {
r.Err = fmt.Sprintf("shard %s stopped unexpectedly", s.ID)
}
return
case supervisor.Ready:
ready++
}
}
// Both conditions. A replica missing a shard is not ready however healthy
// the shards it still has look — and rank 0's absence in particular means
// there is no endpoint to route to at all.
if r.WantShards > 0 && len(r.Shards) >= r.WantShards {
// Fully assembled. From here it is an ordinary replica: losing a shard
// now is a failure, not a half that has yet to report.
r.AdoptedAt = time.Time{}
}
if ready == len(r.Shards) && (r.WantShards == 0 || len(r.Shards) == r.WantShards) {
if r.State != Ready {
r.ReadyAt = now
}
r.State = Ready
return
}
if r.WantShards > 0 && len(r.Shards) < r.WantShards {
// Still assembling from the fleet: the rest of the shards are on nodes
// whose agents have not registered yet. Starting rather than Failed,
// which puts it under the ordinary startup timeout — so a half that
// never arrives is still failed, just by the deadline that already
// exists rather than a second before its sibling reports in.
if !r.AdoptedAt.IsZero() {
r.State = Starting
return
}
r.State = Failed
if r.Err == "" {
r.Err = fmt.Sprintf("replica has %d of %d shards", len(r.Shards), r.WantShards)
}
return
}
r.State = Starting
}