Files
llmgw/internal/control/reconciler/replica.go
T
Ilker ManapandClaude Opus 5 05bdc7d6d4 Add ctl bench, and fix two holes it walked into
`ctl bench` measures what a replica can actually serve, so weight,
target_waiting and kv_ceiling stop being numbers somebody typed. Concurrency
climbs until the SLO breaks and the last level that met it is the capacity:
peak throughput arrives long after latency has become unusable, so throughput
alone overstates what the hardware can serve. Queue depth and KV usage are
read from the engine while it is still meeting the target, which is exactly
what admission should treat as healthy.

Prompts are unique per request by default. With one repeated prompt the
engine's cache answers most of the prefill and the measurement describes the
cache rather than the hardware; --shared-prefix asks that question
deliberately, and it now produces byte-identical prompts, which the first
version did not — it dropped the random header and still picked words at
random, so the cache missed and the flag quietly measured the thing it exists
to avoid.

Running it against the cluster turned up two real defects:

A replica stuck in Starting never timed out. Every other way out of that state
needs the agent to report something, and when it never does the replica sits
there holding GPUs and counting towards the model's replica count while
serving nothing. Two of them had been like that for eight minutes past a
two-minute startup timeout, and nothing would ever have moved them.

They were stuck because adoption lost to a replica we had only placed. Replica
ids come from the model name and a counter, so a control plane that restarts
and places before its agents reconnect builds a different replica under the
same id on a freshly allocated port; the agent's report of the engine that was
actually serving was then discarded as already-known. The cluster pointed at
port 20004 while the engine answered on 20000. The fleet is authoritative
about what is running, so what we merely placed now loses to what an agent
reports — unless our copy has a shard that has actually come up, in which case
two things really are running and the stray is the one we did not ask for.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 09:51:37 +02:00

175 lines
4.4 KiB
Go

// Package reconciler drives the cluster toward the catalog's desired state
// (plan §6.3).
package reconciler
import (
"fmt"
"slices"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
)
// ReplicaState is where a replica stands.
type ReplicaState int
const (
// Placing means it has been scheduled but no shard has started.
Placing ReplicaState = iota
// Starting means shards are launching or loading.
Starting
// Ready means it serves traffic.
Ready
// Draining means it takes no new requests and will be stopped.
Draining
// Stopped means it was shut down on purpose.
Stopped
// Failed means it died or never became ready.
Failed
)
func (s ReplicaState) String() string {
switch s {
case Placing:
return "placing"
case Starting:
return "starting"
case Ready:
return "ready"
case Draining:
return "draining"
case Stopped:
return "stopped"
case Failed:
return "failed"
default:
return "unknown"
}
}
// Terminal reports whether the replica has finished for good.
func (s ReplicaState) Terminal() bool { return s == Stopped || s == Failed }
// Counts reports whether the replica should count toward the desired total.
// Draining replicas do not: they are on the way out, and counting them would
// stop a replacement from being started.
func (s ReplicaState) Counts() bool { return s == Placing || s == Starting || s == Ready }
// Shard is one node's part of a replica.
type Shard struct {
ID string
NodeID string
Rank int
GPUIndices []int
HCAs []string
HTTPPort int
State supervisor.State
Err string
}
// Leader reports whether this shard serves HTTP for the replica.
func (s Shard) Leader() bool { return s.Rank == 0 }
// Replica is one running copy of a model. It is the unit the router sees.
type Replica struct {
ID string
Model string
Profile string
// Fabric is set only for multi-node replicas.
Fabric string
Pool string
Shards []Shard
State ReplicaState
// Endpoint is rank 0's base URL.
Endpoint string
// Weight is the profile's capacity weight, passed to the router.
Weight float64
CreatedAt time.Time
ReadyAt time.Time
// DrainingAt is when the replica stopped taking new work, so the wait for a
// replacement can be bounded.
DrainingAt time.Time
// Attempt counts how many times this model has failed to come up in a row,
// carried across rebuilds so the backoff does not reset on every attempt.
Attempt int
Err string
}
// NodeIDs lists the nodes this replica occupies.
func (r Replica) NodeIDs() []string {
out := make([]string, 0, len(r.Shards))
for _, s := range r.Shards {
out = append(out, s.NodeID)
}
slices.Sort(out)
return out
}
// OnNode reports whether any shard runs on a node.
func (r Replica) OnNode(id string) bool {
return slices.ContainsFunc(r.Shards, func(s Shard) bool { return s.NodeID == id })
}
// deriveState recomputes the replica's state from its shards.
//
// A replica is only as healthy as its worst shard: one failed shard makes the
// whole thing useless, because the others are waiting on collectives that will
// never complete. That is why there is no path here that keeps a replica
// partially alive (plan §5.4).
// hasLiveShard reports whether any shard has actually got somewhere.
//
// Starting and Pending are not evidence of anything: they are what a replica
// looks like both while an engine loads a model and when nothing was ever
// started at all. Loading means the port is open, which only a real process
// does.
func (r *Replica) hasLiveShard() bool {
for _, s := range r.Shards {
if s.State == supervisor.Ready || s.State == supervisor.Loading {
return true
}
}
return false
}
func (r *Replica) deriveState() {
if r.State == Draining || r.State == Stopped {
return
}
if len(r.Shards) == 0 {
r.State = Placing
return
}
ready := 0
for _, s := range r.Shards {
switch s.State {
case supervisor.Failed:
r.State = Failed
if r.Err == "" {
r.Err = fmt.Sprintf("shard %s failed: %s", s.ID, s.Err)
}
return
case supervisor.Stopped:
// A shard stopping on its own, while the replica was not draining,
// is a failure of the replica.
r.State = Failed
if r.Err == "" {
r.Err = fmt.Sprintf("shard %s stopped unexpectedly", s.ID)
}
return
case supervisor.Ready:
ready++
}
}
if ready == len(r.Shards) {
if r.State != Ready {
r.ReadyAt = time.Now()
}
r.State = Ready
return
}
r.State = Starting
}