Not persisting replica state left one hole: restarting the control plane made it forget every replica while the engines kept running. They held GPUs the inventory believed were free, so the next placement failed to allocate, with an error naming the new process rather than the old one. The fix belongs on the agent, not in a database. A row can say "running" about a process that died; the fleet cannot. - the supervisor writes the full shard spec, not just a pid and command line. A pid identifies a process; the spec is what lets it be described back to the control plane and matched to a replica. - Adopt takes over an engine a previous agent left running. It is not this process's child, so it cannot be waited on: it is watched by polling and stopped by signalling its group directly. Its earlier output is gone and the log says so rather than presenting emptiness as silence. - every Register reports what the agent is running, not only the first, because a reconnect may be a control plane restart and has to recover the same way. - the reconciler rebuilds replicas from those reports and reclaims their GPUs. Anything whose model or profile left the catalog is stopped; nothing will route to it and it is holding cards. - an engine the agent cannot identify is killed. A process holding GPUs that nothing can describe is worse than no process, because placement hands those GPUs out again. Stop was also wrong for adopted shards: it checked for an exec.Cmd, which they do not have, and reported them stopped without killing anything. TestAgentReconnects had been passing for no reason. httptest's CloseClientConnections does nothing to an h2c stream — prior-knowledge h2c hijacks the connection and httptest stops tracking hijacked ones — so the agent never reconnected, and the assertion on the connection map passed on a stale entry. Tests now sever connections through their own listener and count opened streams. Plan §11 also records what multi-node still needs, since placement landing early makes it look further along than it is: workers never become Ready because the agent health-checks every shard while only rank 0 serves HTTP; RayCommand is written and tested but never called; and startup is fire-and-forget. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
40427b2705
commit
75a8e76706
13 files changed
+1312
-145
No files matched your search
+97
-5
@@ -149,6 +149,7 @@ func (a *Agent) Run(ctx context.Context) error {
|
||||
// an operator can see it and why it is useless.
|
||||
a.log.Error("hardware discovery failed", "err", err)
|
||||
}
|
||||
a.recoverOrphans()
|
||||
|
||||
attempt := 0
|
||||
for ctx.Err() == nil {
|
||||
@@ -185,6 +186,93 @@ func (a *Agent) reconnectDelay(attempt int) time.Duration {
|
||||
return d/2 + time.Duration(rand.Int64N(int64(d/2))) //nolint:gosec // jitter
|
||||
}
|
||||
|
||||
// recoverOrphans takes back engines a previous agent left running.
|
||||
//
|
||||
// Engines deliberately outlive their agent — the router talks to them directly,
|
||||
// so an agent restart must not interrupt served traffic. The cost is that they
|
||||
// have to be found again, and anything that cannot be identified has to be
|
||||
// killed: a process holding GPUs that nothing can describe is worse than no
|
||||
// process at all, because placement will hand those GPUs out again.
|
||||
func (a *Agent) recoverOrphans() {
|
||||
orphans, err := a.sup.FindOrphans()
|
||||
if err != nil {
|
||||
a.log.Warn("scanning for leftover engines failed", "err", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, o := range orphans {
|
||||
switch {
|
||||
case !o.Alive:
|
||||
// The process is gone; only its state file remains.
|
||||
if err := a.sup.KillOrphan(o); err != nil {
|
||||
a.log.Warn("cleaning up a dead shard's state failed", "shard", o.ShardID, "err", err)
|
||||
}
|
||||
case o.ReplicaID == "":
|
||||
// Alive, but its state file could not be read, so it cannot be
|
||||
// described to the control plane or matched to a replica.
|
||||
a.log.Warn("killing an unidentifiable engine", "shard", o.ShardID, "pid", o.PID)
|
||||
if err := a.sup.KillOrphan(o); err != nil {
|
||||
a.log.Warn("killing it failed", "shard", o.ShardID, "err", err)
|
||||
}
|
||||
default:
|
||||
sh, err := a.sup.Adopt(o)
|
||||
if err != nil {
|
||||
a.log.Warn("adopting a shard failed", "shard", o.ShardID, "err", err)
|
||||
continue
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.shardReplica[o.ShardID] = o.ReplicaID
|
||||
a.mu.Unlock()
|
||||
a.log.Info("adopted a running engine",
|
||||
"shard", o.ShardID, "replica", o.ReplicaID, "pid", o.PID)
|
||||
_ = sh
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runningShards describes what this agent is supervising, for registration.
|
||||
func (a *Agent) runningShards() []*llmgwv1.RunningShard {
|
||||
var out []*llmgwv1.RunningShard
|
||||
for _, st := range a.sup.Statuses() {
|
||||
if st.State.Terminal() {
|
||||
continue
|
||||
}
|
||||
sh, err := a.sup.Get(st.ID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
spec := sh.Spec
|
||||
out = append(out, &llmgwv1.RunningShard{
|
||||
ReplicaId: spec.ReplicaID,
|
||||
ShardId: st.ID,
|
||||
Rank: int32(spec.Rank),
|
||||
Nnodes: int32(spec.NNodes),
|
||||
GpuIndices: int32Slice(spec.GPUIndices),
|
||||
Hcas: spec.HCAs,
|
||||
HttpPort: int32(spec.HTTPPort),
|
||||
State: agentsvc.StateToWire(st.State),
|
||||
Model: spec.Profile.ServedModelName,
|
||||
Profile: spec.Profile.Name,
|
||||
Pool: spec.Pool,
|
||||
Fabric: spec.Fabric,
|
||||
LeaderAddr: a.cfg.MgmtAddr,
|
||||
LeaderPort: int32(spec.LeaderPort),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func int32Slice(v []int) []int32 {
|
||||
if len(v) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]int32, len(v))
|
||||
for i, x := range v {
|
||||
out[i] = int32(x)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// scan refreshes the node's hardware inventory.
|
||||
func (a *Agent) scan(ctx context.Context) error {
|
||||
inv, err := discovery.Discover(ctx, a.cfg.Discovery, a.log)
|
||||
@@ -208,12 +296,16 @@ func (a *Agent) session(ctx context.Context) error {
|
||||
inv := a.inv
|
||||
a.mu.Unlock()
|
||||
|
||||
reg := agentsvc.RegisterFromInventory(
|
||||
a.cfg.NodeID, a.cfg.Hostname, a.cfg.MgmtAddr, a.cfg.Version,
|
||||
a.cfg.Labels, inv, a.cfg.Fabrics)
|
||||
// Every registration reports what is already running, not just the first
|
||||
// one: a reconnect may be a control plane restart, and it has to recover
|
||||
// the same way (plan §6.5).
|
||||
reg.RunningShards = a.runningShards()
|
||||
|
||||
if err := stream.Send(&llmgwv1.AgentMessage{
|
||||
Msg: &llmgwv1.AgentMessage_Register{
|
||||
Register: agentsvc.RegisterFromInventory(
|
||||
a.cfg.NodeID, a.cfg.Hostname, a.cfg.MgmtAddr, a.cfg.Version,
|
||||
a.cfg.Labels, inv, a.cfg.Fabrics),
|
||||
},
|
||||
Msg: &llmgwv1.AgentMessage_Register{Register: reg},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("registering: %w", err)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user