ctl talks to a management API on the admin listener, not the client-facing one. The two have different audiences: one serves inference to whoever holds an API key, the other lets a caller cordon nodes and read the cluster's shape. - nodes surfaces unhealthy GPUs and why they are excluded. That is usually the answer when a node looks idle but nothing will schedule onto it, and it was invisible before. - explain-placement prints every candidate considered and why it was rejected. - replicas shows the shards a replica is made of, with per-shard errors. - cordon, uncordon and drain are one endpoint with different targets, because the difference between cordon and drain is exactly what an operator needs to be sure of before running either. Draining had to be fixed first to make ctl drain worth having. A drained replica was stopped on the next pass, which cuts in-flight requests — the thing draining exists to avoid. Now a node marked Draining has its replicas drained, the model falls below its desired count, a replacement is placed elsewhere, and the original is stopped only once the replacement is serving. A grace period bounds it, because draining the only node that can host a model would otherwise never finish and the node could never be taken out of service. Verified against a two-node cluster of real binaries: draining moved a replica to the surviving node and traffic kept returning 200 throughout. ctl logs is deliberately absent. It needs a request/response pair the agent protocol does not have, and the failure path already ships an engine log tail with every status report — so the case that matters most is covered, and reading a healthy shard's output on demand is not worth a protocol change yet. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
160 lines
3.9 KiB
Go
160 lines
3.9 KiB
Go
// Package reconciler drives the cluster toward the catalog's desired state
|
|
// (plan §6.3).
|
|
package reconciler
|
|
|
|
import (
|
|
"fmt"
|
|
"slices"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
|
)
|
|
|
|
// ReplicaState is where a replica stands.
|
|
type ReplicaState int
|
|
|
|
const (
|
|
// Placing means it has been scheduled but no shard has started.
|
|
Placing ReplicaState = iota
|
|
// Starting means shards are launching or loading.
|
|
Starting
|
|
// Ready means it serves traffic.
|
|
Ready
|
|
// Draining means it takes no new requests and will be stopped.
|
|
Draining
|
|
// Stopped means it was shut down on purpose.
|
|
Stopped
|
|
// Failed means it died or never became ready.
|
|
Failed
|
|
)
|
|
|
|
func (s ReplicaState) String() string {
|
|
switch s {
|
|
case Placing:
|
|
return "placing"
|
|
case Starting:
|
|
return "starting"
|
|
case Ready:
|
|
return "ready"
|
|
case Draining:
|
|
return "draining"
|
|
case Stopped:
|
|
return "stopped"
|
|
case Failed:
|
|
return "failed"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
// Terminal reports whether the replica has finished for good.
|
|
func (s ReplicaState) Terminal() bool { return s == Stopped || s == Failed }
|
|
|
|
// Counts reports whether the replica should count toward the desired total.
|
|
// Draining replicas do not: they are on the way out, and counting them would
|
|
// stop a replacement from being started.
|
|
func (s ReplicaState) Counts() bool { return s == Placing || s == Starting || s == Ready }
|
|
|
|
// Shard is one node's part of a replica.
|
|
type Shard struct {
|
|
ID string
|
|
NodeID string
|
|
Rank int
|
|
GPUIndices []int
|
|
HCAs []string
|
|
HTTPPort int
|
|
State supervisor.State
|
|
Err string
|
|
}
|
|
|
|
// Leader reports whether this shard serves HTTP for the replica.
|
|
func (s Shard) Leader() bool { return s.Rank == 0 }
|
|
|
|
// Replica is one running copy of a model. It is the unit the router sees.
|
|
type Replica struct {
|
|
ID string
|
|
Model string
|
|
Profile string
|
|
// Fabric is set only for multi-node replicas.
|
|
Fabric string
|
|
Pool string
|
|
Shards []Shard
|
|
State ReplicaState
|
|
// Endpoint is rank 0's base URL.
|
|
Endpoint string
|
|
// Weight is the profile's capacity weight, passed to the router.
|
|
Weight float64
|
|
|
|
CreatedAt time.Time
|
|
ReadyAt time.Time
|
|
// DrainingAt is when the replica stopped taking new work, so the wait for a
|
|
// replacement can be bounded.
|
|
DrainingAt time.Time
|
|
// Attempt counts how many times this model has failed to come up in a row,
|
|
// carried across rebuilds so the backoff does not reset on every attempt.
|
|
Attempt int
|
|
Err string
|
|
}
|
|
|
|
// NodeIDs lists the nodes this replica occupies.
|
|
func (r Replica) NodeIDs() []string {
|
|
out := make([]string, 0, len(r.Shards))
|
|
for _, s := range r.Shards {
|
|
out = append(out, s.NodeID)
|
|
}
|
|
slices.Sort(out)
|
|
return out
|
|
}
|
|
|
|
// OnNode reports whether any shard runs on a node.
|
|
func (r Replica) OnNode(id string) bool {
|
|
return slices.ContainsFunc(r.Shards, func(s Shard) bool { return s.NodeID == id })
|
|
}
|
|
|
|
// deriveState recomputes the replica's state from its shards.
|
|
//
|
|
// A replica is only as healthy as its worst shard: one failed shard makes the
|
|
// whole thing useless, because the others are waiting on collectives that will
|
|
// never complete. That is why there is no path here that keeps a replica
|
|
// partially alive (plan §5.4).
|
|
func (r *Replica) deriveState() {
|
|
if r.State == Draining || r.State == Stopped {
|
|
return
|
|
}
|
|
if len(r.Shards) == 0 {
|
|
r.State = Placing
|
|
return
|
|
}
|
|
|
|
ready := 0
|
|
for _, s := range r.Shards {
|
|
switch s.State {
|
|
case supervisor.Failed:
|
|
r.State = Failed
|
|
if r.Err == "" {
|
|
r.Err = fmt.Sprintf("shard %s failed: %s", s.ID, s.Err)
|
|
}
|
|
return
|
|
case supervisor.Stopped:
|
|
// A shard stopping on its own, while the replica was not draining,
|
|
// is a failure of the replica.
|
|
r.State = Failed
|
|
if r.Err == "" {
|
|
r.Err = fmt.Sprintf("shard %s stopped unexpectedly", s.ID)
|
|
}
|
|
return
|
|
case supervisor.Ready:
|
|
ready++
|
|
}
|
|
}
|
|
|
|
if ready == len(r.Shards) {
|
|
if r.State != Ready {
|
|
r.ReadyAt = time.Now()
|
|
}
|
|
r.State = Ready
|
|
return
|
|
}
|
|
r.State = Starting
|
|
}
|