// Package reconciler drives the cluster toward the catalog's desired state // (plan §6.3). package reconciler import ( "fmt" "slices" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" ) // ReplicaState is where a replica stands. type ReplicaState int const ( // Placing means it has been scheduled but no shard has started. Placing ReplicaState = iota // Starting means shards are launching or loading. Starting // Ready means it serves traffic. Ready // Draining means it takes no new requests and will be stopped. Draining // Stopped means it was shut down on purpose. Stopped // Failed means it died or never became ready. Failed ) func (s ReplicaState) String() string { switch s { case Placing: return "placing" case Starting: return "starting" case Ready: return "ready" case Draining: return "draining" case Stopped: return "stopped" case Failed: return "failed" default: return "unknown" } } // Terminal reports whether the replica has finished for good. func (s ReplicaState) Terminal() bool { return s == Stopped || s == Failed } // Counts reports whether the replica should count toward the desired total. // Draining replicas do not: they are on the way out, and counting them would // stop a replacement from being started. func (s ReplicaState) Counts() bool { return s == Placing || s == Starting || s == Ready } // Shard is one node's part of a replica. type Shard struct { ID string NodeID string Rank int GPUIndices []int HCAs []string HTTPPort int State supervisor.State Err string } // Leader reports whether this shard serves HTTP for the replica. func (s Shard) Leader() bool { return s.Rank == 0 } // Replica is one running copy of a model. It is the unit the router sees. type Replica struct { ID string Model string Profile string // Fabric is set only for multi-node replicas. Fabric string Pool string Shards []Shard State ReplicaState // Endpoint is rank 0's base URL. Endpoint string // Weight is the profile's capacity weight, passed to the router. Weight float64 CreatedAt time.Time ReadyAt time.Time // DrainingAt is when the replica stopped taking new work, so the wait for a // replacement can be bounded. DrainingAt time.Time // Attempt counts how many times this model has failed to come up in a row, // carried across rebuilds so the backoff does not reset on every attempt. Attempt int Err string } // NodeIDs lists the nodes this replica occupies. func (r Replica) NodeIDs() []string { out := make([]string, 0, len(r.Shards)) for _, s := range r.Shards { out = append(out, s.NodeID) } slices.Sort(out) return out } // OnNode reports whether any shard runs on a node. func (r Replica) OnNode(id string) bool { return slices.ContainsFunc(r.Shards, func(s Shard) bool { return s.NodeID == id }) } // deriveState recomputes the replica's state from its shards. // // A replica is only as healthy as its worst shard: one failed shard makes the // whole thing useless, because the others are waiting on collectives that will // never complete. That is why there is no path here that keeps a replica // partially alive (plan §5.4). func (r *Replica) deriveState() { if r.State == Draining || r.State == Stopped { return } if len(r.Shards) == 0 { r.State = Placing return } ready := 0 for _, s := range r.Shards { switch s.State { case supervisor.Failed: r.State = Failed if r.Err == "" { r.Err = fmt.Sprintf("shard %s failed: %s", s.ID, s.Err) } return case supervisor.Stopped: // A shard stopping on its own, while the replica was not draining, // is a failure of the replica. r.State = Failed if r.Err == "" { r.Err = fmt.Sprintf("shard %s stopped unexpectedly", s.ID) } return case supervisor.Ready: ready++ } } if ready == len(r.Shards) { if r.State != Ready { r.ReadyAt = time.Now() } r.State = Ready return } r.State = Starting }