The desired-versus-actual loop, with a Dispatcher interface where the agent stream will go. The loop's logic is where the interesting failures live, not the transport, so it is tested against a fake dispatcher and a synthetic clock. - catalog: models, profiles, aliases, fallbacks, scaling bounds. Aliases and fallbacks resolve at load rather than at request time, because a typo in a fallback should stop a deployment instead of surfacing during the outage when the fallback is finally needed. The node/GPU/TP arithmetic is checked here too, so a bad profile fails at config load rather than minutes into loading. - reconciler: one replica placed per pass, since placement depends on what is already allocated and several from one snapshot would be handed the same GPUs. Workers start before rank 0: SGLang's ranks rendezvous and vLLM under Ray needs its workers attached before the head serves. - a failed shard fails the whole replica. The others are waiting on collectives that will never complete while holding GPUs, so every shard is stopped and the replica rebuilt, with a jittered exponential backoff carried on the model rather than on the replica that was just destroyed. - node expiry has two stages, Unreachable then Lost, deliberately far apart: a network blip must not tear down replicas that are still serving, and rebuilding a multi-node replica costs tens of minutes of model loading. - scale-down drains before stopping, so in-flight requests finish. The phase 3 exit criterion is half met. TestLostNodeMovesReplicas covers the reconciliation half end to end. The router continuing to serve throughout needs the EndpointSource wiring and the agent stream. Writing the tests caught a real property of the design worth recording: advancing the clock makes every node look silent, so placement correctly stops. The test harness now heartbeats nodes as it advances time, except where a node going silent is the point. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
157 lines
3.8 KiB
Go
157 lines
3.8 KiB
Go
// Package reconciler drives the cluster toward the catalog's desired state
|
|
// (plan §6.3).
|
|
package reconciler
|
|
|
|
import (
|
|
"fmt"
|
|
"slices"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
|
)
|
|
|
|
// ReplicaState is where a replica stands.
|
|
type ReplicaState int
|
|
|
|
const (
|
|
// Placing means it has been scheduled but no shard has started.
|
|
Placing ReplicaState = iota
|
|
// Starting means shards are launching or loading.
|
|
Starting
|
|
// Ready means it serves traffic.
|
|
Ready
|
|
// Draining means it takes no new requests and will be stopped.
|
|
Draining
|
|
// Stopped means it was shut down on purpose.
|
|
Stopped
|
|
// Failed means it died or never became ready.
|
|
Failed
|
|
)
|
|
|
|
func (s ReplicaState) String() string {
|
|
switch s {
|
|
case Placing:
|
|
return "placing"
|
|
case Starting:
|
|
return "starting"
|
|
case Ready:
|
|
return "ready"
|
|
case Draining:
|
|
return "draining"
|
|
case Stopped:
|
|
return "stopped"
|
|
case Failed:
|
|
return "failed"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
// Terminal reports whether the replica has finished for good.
|
|
func (s ReplicaState) Terminal() bool { return s == Stopped || s == Failed }
|
|
|
|
// Counts reports whether the replica should count toward the desired total.
|
|
// Draining replicas do not: they are on the way out, and counting them would
|
|
// stop a replacement from being started.
|
|
func (s ReplicaState) Counts() bool { return s == Placing || s == Starting || s == Ready }
|
|
|
|
// Shard is one node's part of a replica.
|
|
type Shard struct {
|
|
ID string
|
|
NodeID string
|
|
Rank int
|
|
GPUIndices []int
|
|
HCAs []string
|
|
HTTPPort int
|
|
State supervisor.State
|
|
Err string
|
|
}
|
|
|
|
// Leader reports whether this shard serves HTTP for the replica.
|
|
func (s Shard) Leader() bool { return s.Rank == 0 }
|
|
|
|
// Replica is one running copy of a model. It is the unit the router sees.
|
|
type Replica struct {
|
|
ID string
|
|
Model string
|
|
Profile string
|
|
// Fabric is set only for multi-node replicas.
|
|
Fabric string
|
|
Pool string
|
|
Shards []Shard
|
|
State ReplicaState
|
|
// Endpoint is rank 0's base URL.
|
|
Endpoint string
|
|
// Weight is the profile's capacity weight, passed to the router.
|
|
Weight float64
|
|
|
|
CreatedAt time.Time
|
|
ReadyAt time.Time
|
|
// Attempt counts how many times this model has failed to come up in a row,
|
|
// carried across rebuilds so the backoff does not reset on every attempt.
|
|
Attempt int
|
|
Err string
|
|
}
|
|
|
|
// NodeIDs lists the nodes this replica occupies.
|
|
func (r Replica) NodeIDs() []string {
|
|
out := make([]string, 0, len(r.Shards))
|
|
for _, s := range r.Shards {
|
|
out = append(out, s.NodeID)
|
|
}
|
|
slices.Sort(out)
|
|
return out
|
|
}
|
|
|
|
// OnNode reports whether any shard runs on a node.
|
|
func (r Replica) OnNode(id string) bool {
|
|
return slices.ContainsFunc(r.Shards, func(s Shard) bool { return s.NodeID == id })
|
|
}
|
|
|
|
// deriveState recomputes the replica's state from its shards.
|
|
//
|
|
// A replica is only as healthy as its worst shard: one failed shard makes the
|
|
// whole thing useless, because the others are waiting on collectives that will
|
|
// never complete. That is why there is no path here that keeps a replica
|
|
// partially alive (plan §5.4).
|
|
func (r *Replica) deriveState() {
|
|
if r.State == Draining || r.State == Stopped {
|
|
return
|
|
}
|
|
if len(r.Shards) == 0 {
|
|
r.State = Placing
|
|
return
|
|
}
|
|
|
|
ready := 0
|
|
for _, s := range r.Shards {
|
|
switch s.State {
|
|
case supervisor.Failed:
|
|
r.State = Failed
|
|
if r.Err == "" {
|
|
r.Err = fmt.Sprintf("shard %s failed: %s", s.ID, s.Err)
|
|
}
|
|
return
|
|
case supervisor.Stopped:
|
|
// A shard stopping on its own, while the replica was not draining,
|
|
// is a failure of the replica.
|
|
r.State = Failed
|
|
if r.Err == "" {
|
|
r.Err = fmt.Sprintf("shard %s stopped unexpectedly", s.ID)
|
|
}
|
|
return
|
|
case supervisor.Ready:
|
|
ready++
|
|
}
|
|
}
|
|
|
|
if ready == len(r.Shards) {
|
|
if r.State != Ready {
|
|
r.ReadyAt = time.Now()
|
|
}
|
|
r.State = Ready
|
|
return
|
|
}
|
|
r.State = Starting
|
|
}
|