// Package reconciler drives the cluster toward the catalog's desired state // (plan §6.3). package reconciler import ( "fmt" "slices" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" ) // ReplicaState is where a replica stands. type ReplicaState int const ( // Placing means it has been scheduled but no shard has started. Placing ReplicaState = iota // Starting means shards are launching or loading. Starting // Ready means it serves traffic. Ready // Draining means it takes no new requests and will be stopped. Draining // Stopped means it was shut down on purpose. Stopped // Failed means it died or never became ready. Failed ) func (s ReplicaState) String() string { switch s { case Placing: return "placing" case Starting: return "starting" case Ready: return "ready" case Draining: return "draining" case Stopped: return "stopped" case Failed: return "failed" default: return "unknown" } } // Terminal reports whether the replica has finished for good. func (s ReplicaState) Terminal() bool { return s == Stopped || s == Failed } // Counts reports whether the replica should count toward the desired total. // Draining replicas do not: they are on the way out, and counting them would // stop a replacement from being started. func (s ReplicaState) Counts() bool { return s == Placing || s == Starting || s == Ready } // Shard is one node's part of a replica. type Shard struct { ID string NodeID string Rank int GPUIndices []int HCAs []string HTTPPort int State supervisor.State Err string } // Leader reports whether this shard serves HTTP for the replica. func (s Shard) Leader() bool { return s.Rank == 0 } // Replica is one running copy of a model. It is the unit the router sees. type Replica struct { ID string Model string Profile string // Fabric is set only for multi-node replicas. Fabric string Pool string Shards []Shard State ReplicaState // Endpoint is rank 0's base URL. Endpoint string // Weight is the profile's capacity weight, passed to the router. Weight float64 // WantShards is how many shards this replica should have, from its profile. // // Without it a replica cannot tell "every shard I have is ready" from // "every shard I have left is ready". A control plane that adopted half of // a two-node replica reported it Ready with one shard and no endpoint, and // the router was handed a replica nothing could route to. WantShards int // AdoptedAt is set while a replica is being reassembled from what agents // report, and cleared once every shard has arrived. // // A multi-node replica does not arrive at once: each node has its own // agent, each agent registers separately, and a replica spanning two nodes // is therefore reported in two halves a second or so apart. Between them it // is legitimately short of shards, and failing it there condemned it before // its other half had a chance to register — after which that half was // stopped as unclaimed and the model rebuilt from nothing on every restart. AdoptedAt time.Time CreatedAt time.Time ReadyAt time.Time // LastUsedAt is when a request was last routed to this replica, zero if // none has been since the router started. // // The difference between a replica that is holding eight cards for work and // one that is holding them out of habit. Nothing recorded it, so on a fleet // with more models than GPUs there was no way to answer the first question // anyone asks: what can I stop? LastUsedAt time.Time // DrainingAt is when the replica stopped taking new work, so the wait for a // replacement can be bounded. DrainingAt time.Time // Attempt counts how many times this model has failed to come up in a row, // carried across rebuilds so the backoff does not reset on every attempt. Attempt int Err string } // NodeIDs lists the nodes this replica occupies. func (r Replica) NodeIDs() []string { out := make([]string, 0, len(r.Shards)) for _, s := range r.Shards { out = append(out, s.NodeID) } slices.Sort(out) return out } // OnNode reports whether any shard runs on a node. func (r Replica) OnNode(id string) bool { return slices.ContainsFunc(r.Shards, func(s Shard) bool { return s.NodeID == id }) } // deriveState recomputes the replica's state from its shards. // // A replica is only as healthy as its worst shard: one failed shard makes the // whole thing useless, because the others are waiting on collectives that will // never complete. That is why there is no path here that keeps a replica // partially alive (plan §5.4). // hasLiveShard reports whether any shard has actually got somewhere. // // Starting and Pending are not evidence of anything: they are what a replica // looks like both while an engine loads a model and when nothing was ever // started at all. Loading means the port is open, which only a real process // does. func (r *Replica) hasLiveShard() bool { for _, s := range r.Shards { if s.State == supervisor.Ready || s.State == supervisor.Loading { return true } } return false } func (r *Replica) deriveState() { if r.State == Draining || r.State == Stopped { return } if len(r.Shards) == 0 { r.State = Placing return } ready := 0 for _, s := range r.Shards { switch s.State { case supervisor.Failed: r.State = Failed if r.Err == "" { r.Err = fmt.Sprintf("shard %s failed: %s", s.ID, s.Err) } return case supervisor.Stopped: // A shard stopping on its own, while the replica was not draining, // is a failure of the replica. r.State = Failed if r.Err == "" { r.Err = fmt.Sprintf("shard %s stopped unexpectedly", s.ID) } return case supervisor.Ready: ready++ } } // Both conditions. A replica missing a shard is not ready however healthy // the shards it still has look — and rank 0's absence in particular means // there is no endpoint to route to at all. if r.WantShards > 0 && len(r.Shards) >= r.WantShards { // Fully assembled. From here it is an ordinary replica: losing a shard // now is a failure, not a half that has yet to report. r.AdoptedAt = time.Time{} } if ready == len(r.Shards) && (r.WantShards == 0 || len(r.Shards) == r.WantShards) { if r.State != Ready { r.ReadyAt = time.Now() } r.State = Ready return } if r.WantShards > 0 && len(r.Shards) < r.WantShards { // Still assembling from the fleet: the rest of the shards are on nodes // whose agents have not registered yet. Starting rather than Failed, // which puts it under the ordinary startup timeout — so a half that // never arrives is still failed, just by the deadline that already // exists rather than a second before its sibling reports in. if !r.AdoptedAt.IsZero() { r.State = Starting return } r.State = Failed if r.Err == "" { r.Err = fmt.Sprintf("replica has %d of %d shards", len(r.Shards), r.WantShards) } return } r.State = Starting }