CI / test (push) Successful in 1m28s
Two things a cluster with many nodes and many GPUs needs that were missing, and they are the same thing seen from two distances. Idle tracking. The control plane knew what was placed and whether it was healthy, never whether anyone wanted it — so on a fleet with more models than cards there was no way to answer the first question anyone asks, which is what can I stop. The router is the only thing that knows, so it now reports it: a timestamp written in Acquire, where a request already meets an endpoint under a lock already held, and collected once a tick as a map. That shape is the point. Per-request notification would be a firehose carrying one bit — the newest timestamp wins — and this costs the same on four replicas as on four hundred. It travels through an interface in the direction opposite EndpointSink, for the same reason that one exists: the control plane decides what should run, the router knows whether anyone wants it, and neither should need to be in the other's process to say so. Unknown is kept distinct from idle throughout. A router that has just restarted knows nothing, and a zero timestamp would be the oldest idle replica in every comparison — so one restart would nominate an entire fleet for eviction. Absent from the map, "-" in the table, negative in the JSON, and IdleFor returns a second value rather than a duration anything could act on by mistake. ctl status. A per-node listing is right for four nodes and wrong for two hundred: how full am I, what is broken, what is holding cards nobody is using are aggregates, and reading them off a long table is a chore, which means it gets skipped. Capacity per GPU class with usage bars, free whole nodes per fabric — the number that decides whether a multi-node replica can land, since free cards on the wrong fabric are not capacity for a model that spans nodes — replica and model counts, models that cannot be placed, and what has gone unasked-for. Derived from the same state the tables show, so it cannot disagree with them. Unhealthy GPUs count as neither used nor free, because counting them free is how a cluster looks like it has room it does not. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
224 lines
6.8 KiB
Go
224 lines
6.8 KiB
Go
// Package reconciler drives the cluster toward the catalog's desired state
|
|
// (plan §6.3).
|
|
package reconciler
|
|
|
|
import (
|
|
"fmt"
|
|
"slices"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
|
)
|
|
|
|
// ReplicaState is where a replica stands.
|
|
type ReplicaState int
|
|
|
|
const (
|
|
// Placing means it has been scheduled but no shard has started.
|
|
Placing ReplicaState = iota
|
|
// Starting means shards are launching or loading.
|
|
Starting
|
|
// Ready means it serves traffic.
|
|
Ready
|
|
// Draining means it takes no new requests and will be stopped.
|
|
Draining
|
|
// Stopped means it was shut down on purpose.
|
|
Stopped
|
|
// Failed means it died or never became ready.
|
|
Failed
|
|
)
|
|
|
|
func (s ReplicaState) String() string {
|
|
switch s {
|
|
case Placing:
|
|
return "placing"
|
|
case Starting:
|
|
return "starting"
|
|
case Ready:
|
|
return "ready"
|
|
case Draining:
|
|
return "draining"
|
|
case Stopped:
|
|
return "stopped"
|
|
case Failed:
|
|
return "failed"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
// Terminal reports whether the replica has finished for good.
|
|
func (s ReplicaState) Terminal() bool { return s == Stopped || s == Failed }
|
|
|
|
// Counts reports whether the replica should count toward the desired total.
|
|
// Draining replicas do not: they are on the way out, and counting them would
|
|
// stop a replacement from being started.
|
|
func (s ReplicaState) Counts() bool { return s == Placing || s == Starting || s == Ready }
|
|
|
|
// Shard is one node's part of a replica.
|
|
type Shard struct {
|
|
ID string
|
|
NodeID string
|
|
Rank int
|
|
GPUIndices []int
|
|
HCAs []string
|
|
HTTPPort int
|
|
State supervisor.State
|
|
Err string
|
|
}
|
|
|
|
// Leader reports whether this shard serves HTTP for the replica.
|
|
func (s Shard) Leader() bool { return s.Rank == 0 }
|
|
|
|
// Replica is one running copy of a model. It is the unit the router sees.
|
|
type Replica struct {
|
|
ID string
|
|
Model string
|
|
Profile string
|
|
// Fabric is set only for multi-node replicas.
|
|
Fabric string
|
|
Pool string
|
|
Shards []Shard
|
|
State ReplicaState
|
|
// Endpoint is rank 0's base URL.
|
|
Endpoint string
|
|
// Weight is the profile's capacity weight, passed to the router.
|
|
Weight float64
|
|
// WantShards is how many shards this replica should have, from its profile.
|
|
//
|
|
// Without it a replica cannot tell "every shard I have is ready" from
|
|
// "every shard I have left is ready". A control plane that adopted half of
|
|
// a two-node replica reported it Ready with one shard and no endpoint, and
|
|
// the router was handed a replica nothing could route to.
|
|
WantShards int
|
|
// AdoptedAt is set while a replica is being reassembled from what agents
|
|
// report, and cleared once every shard has arrived.
|
|
//
|
|
// A multi-node replica does not arrive at once: each node has its own
|
|
// agent, each agent registers separately, and a replica spanning two nodes
|
|
// is therefore reported in two halves a second or so apart. Between them it
|
|
// is legitimately short of shards, and failing it there condemned it before
|
|
// its other half had a chance to register — after which that half was
|
|
// stopped as unclaimed and the model rebuilt from nothing on every restart.
|
|
AdoptedAt time.Time
|
|
|
|
CreatedAt time.Time
|
|
ReadyAt time.Time
|
|
// LastUsedAt is when a request was last routed to this replica, zero if
|
|
// none has been since the router started.
|
|
//
|
|
// The difference between a replica that is holding eight cards for work and
|
|
// one that is holding them out of habit. Nothing recorded it, so on a fleet
|
|
// with more models than GPUs there was no way to answer the first question
|
|
// anyone asks: what can I stop?
|
|
LastUsedAt time.Time
|
|
// DrainingAt is when the replica stopped taking new work, so the wait for a
|
|
// replacement can be bounded.
|
|
DrainingAt time.Time
|
|
// Attempt counts how many times this model has failed to come up in a row,
|
|
// carried across rebuilds so the backoff does not reset on every attempt.
|
|
Attempt int
|
|
Err string
|
|
}
|
|
|
|
// NodeIDs lists the nodes this replica occupies.
|
|
func (r Replica) NodeIDs() []string {
|
|
out := make([]string, 0, len(r.Shards))
|
|
for _, s := range r.Shards {
|
|
out = append(out, s.NodeID)
|
|
}
|
|
slices.Sort(out)
|
|
return out
|
|
}
|
|
|
|
// OnNode reports whether any shard runs on a node.
|
|
func (r Replica) OnNode(id string) bool {
|
|
return slices.ContainsFunc(r.Shards, func(s Shard) bool { return s.NodeID == id })
|
|
}
|
|
|
|
// deriveState recomputes the replica's state from its shards.
|
|
//
|
|
// A replica is only as healthy as its worst shard: one failed shard makes the
|
|
// whole thing useless, because the others are waiting on collectives that will
|
|
// never complete. That is why there is no path here that keeps a replica
|
|
// partially alive (plan §5.4).
|
|
// hasLiveShard reports whether any shard has actually got somewhere.
|
|
//
|
|
// Starting and Pending are not evidence of anything: they are what a replica
|
|
// looks like both while an engine loads a model and when nothing was ever
|
|
// started at all. Loading means the port is open, which only a real process
|
|
// does.
|
|
func (r *Replica) hasLiveShard() bool {
|
|
for _, s := range r.Shards {
|
|
if s.State == supervisor.Ready || s.State == supervisor.Loading {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (r *Replica) deriveState() {
|
|
if r.State == Draining || r.State == Stopped {
|
|
return
|
|
}
|
|
if len(r.Shards) == 0 {
|
|
r.State = Placing
|
|
return
|
|
}
|
|
|
|
ready := 0
|
|
for _, s := range r.Shards {
|
|
switch s.State {
|
|
case supervisor.Failed:
|
|
r.State = Failed
|
|
if r.Err == "" {
|
|
r.Err = fmt.Sprintf("shard %s failed: %s", s.ID, s.Err)
|
|
}
|
|
return
|
|
case supervisor.Stopped:
|
|
// A shard stopping on its own, while the replica was not draining,
|
|
// is a failure of the replica.
|
|
r.State = Failed
|
|
if r.Err == "" {
|
|
r.Err = fmt.Sprintf("shard %s stopped unexpectedly", s.ID)
|
|
}
|
|
return
|
|
case supervisor.Ready:
|
|
ready++
|
|
}
|
|
}
|
|
|
|
// Both conditions. A replica missing a shard is not ready however healthy
|
|
// the shards it still has look — and rank 0's absence in particular means
|
|
// there is no endpoint to route to at all.
|
|
if r.WantShards > 0 && len(r.Shards) >= r.WantShards {
|
|
// Fully assembled. From here it is an ordinary replica: losing a shard
|
|
// now is a failure, not a half that has yet to report.
|
|
r.AdoptedAt = time.Time{}
|
|
}
|
|
if ready == len(r.Shards) && (r.WantShards == 0 || len(r.Shards) == r.WantShards) {
|
|
if r.State != Ready {
|
|
r.ReadyAt = time.Now()
|
|
}
|
|
r.State = Ready
|
|
return
|
|
}
|
|
if r.WantShards > 0 && len(r.Shards) < r.WantShards {
|
|
// Still assembling from the fleet: the rest of the shards are on nodes
|
|
// whose agents have not registered yet. Starting rather than Failed,
|
|
// which puts it under the ordinary startup timeout — so a half that
|
|
// never arrives is still failed, just by the deadline that already
|
|
// exists rather than a second before its sibling reports in.
|
|
if !r.AdoptedAt.IsZero() {
|
|
r.State = Starting
|
|
return
|
|
}
|
|
r.State = Failed
|
|
if r.Err == "" {
|
|
r.Err = fmt.Sprintf("replica has %d of %d shards", len(r.Shards), r.WantShards)
|
|
}
|
|
return
|
|
}
|
|
r.State = Starting
|
|
}
|