Not persisting replica state left one hole: restarting the control plane made it forget every replica while the engines kept running. They held GPUs the inventory believed were free, so the next placement failed to allocate, with an error naming the new process rather than the old one. The fix belongs on the agent, not in a database. A row can say "running" about a process that died; the fleet cannot. - the supervisor writes the full shard spec, not just a pid and command line. A pid identifies a process; the spec is what lets it be described back to the control plane and matched to a replica. - Adopt takes over an engine a previous agent left running. It is not this process's child, so it cannot be waited on: it is watched by polling and stopped by signalling its group directly. Its earlier output is gone and the log says so rather than presenting emptiness as silence. - every Register reports what the agent is running, not only the first, because a reconnect may be a control plane restart and has to recover the same way. - the reconciler rebuilds replicas from those reports and reclaims their GPUs. Anything whose model or profile left the catalog is stopped; nothing will route to it and it is holding cards. - an engine the agent cannot identify is killed. A process holding GPUs that nothing can describe is worse than no process, because placement hands those GPUs out again. Stop was also wrong for adopted shards: it checked for an exec.Cmd, which they do not have, and reported them stopped without killing anything. TestAgentReconnects had been passing for no reason. httptest's CloseClientConnections does nothing to an h2c stream — prior-knowledge h2c hijacks the connection and httptest stops tracking hijacked ones — so the agent never reconnected, and the assertion on the connection map passed on a stale entry. Tests now sever connections through their own listener and count opened streams. Plan §11 also records what multi-node still needs, since placement landing early makes it look further along than it is: workers never become Ready because the agent health-checks every shard while only rank 0 serves HTTP; RayCommand is written and tested but never called; and startup is fire-and-forget. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
40427b2705
commit
75a8e76706
13 files changed
+1312
-145
No files matched your search
@@ -2,6 +2,7 @@
|
||||
package agentsvc
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
@@ -87,6 +88,33 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
|
||||
return r
|
||||
}
|
||||
|
||||
// AdoptedFromRegister converts an agent's running-shard report into the
|
||||
// reconciler's view.
|
||||
func AdoptedFromRegister(nodeID string, shards []*llmgwv1.RunningShard) []reconciler.AdoptedShard {
|
||||
out := make([]reconciler.AdoptedShard, 0, len(shards))
|
||||
for _, s := range shards {
|
||||
a := reconciler.AdoptedShard{
|
||||
NodeID: nodeID,
|
||||
ReplicaID: s.GetReplicaId(),
|
||||
ShardID: s.GetShardId(),
|
||||
Rank: int(s.GetRank()),
|
||||
GPUIndices: intSlice(s.GetGpuIndices()),
|
||||
HCAs: s.GetHcas(),
|
||||
HTTPPort: int(s.GetHttpPort()),
|
||||
State: StateFromWire(s.GetState()),
|
||||
Model: s.GetModel(),
|
||||
Profile: s.GetProfile(),
|
||||
Pool: s.GetPool(),
|
||||
Fabric: s.GetFabric(),
|
||||
}
|
||||
if a.Rank == 0 && s.GetLeaderAddr() != "" {
|
||||
a.Endpoint = fmt.Sprintf("http://%s:%d", s.GetLeaderAddr(), s.GetHttpPort())
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// StartShardFromCommand converts a reconciler command into a wire message.
|
||||
func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard {
|
||||
p := cmd.Spec.Profile
|
||||
@@ -109,6 +137,8 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard {
|
||||
ModelPath: p.ModelPath, ServedModelName: p.ServedModelName,
|
||||
},
|
||||
StartupTimeoutMs: cmd.StartupTimeout.Milliseconds(),
|
||||
Pool: cmd.Pool,
|
||||
Fabric: cmd.Fabric,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,6 +156,8 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration
|
||||
HCAs: m.GetHcas(),
|
||||
MgmtInterface: m.GetMgmtInterface(),
|
||||
HTTPPort: int(m.GetHttpPort()),
|
||||
Pool: m.GetPool(),
|
||||
Fabric: m.GetFabric(),
|
||||
Profile: engines.Profile{
|
||||
Name: p.GetName(), Engine: p.GetEngine(), GPUClass: p.GetGpuClass(),
|
||||
TP: int(p.GetTp()), PP: int(p.GetPp()),
|
||||
|
||||
@@ -19,9 +19,11 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
)
|
||||
|
||||
// StatusSink receives shard transitions. The reconciler implements it.
|
||||
// StatusSink receives shard transitions and recovers replica state from what
|
||||
// agents report they are already running. The reconciler implements it.
|
||||
type StatusSink interface {
|
||||
OnShardStatus(replicaID, shardID string, state supervisor.State, errMsg string)
|
||||
Adopt(shards []reconciler.AdoptedShard) (unclaimed []reconciler.AdoptedShard)
|
||||
}
|
||||
|
||||
// Service is the control plane's AgentService implementation, and at the same
|
||||
@@ -87,6 +89,7 @@ func (s *Service) Connect(
|
||||
s.inv.Upsert(node)
|
||||
s.log.Info("agent registered",
|
||||
"node", nodeID, "gpus", len(node.GPUs), "fabrics", node.FabricNames(),
|
||||
"running_shards", len(reg.GetRunningShards()),
|
||||
"agent_version", reg.GetAgentVersion())
|
||||
|
||||
c := s.attach(ctx, nodeID)
|
||||
@@ -98,6 +101,11 @@ func (s *Service) Connect(
|
||||
return err
|
||||
}
|
||||
|
||||
// Recover replica state from what this node is already running, before any
|
||||
// reconciliation pass can conclude the cluster is empty and place
|
||||
// duplicates (plan §6.5).
|
||||
s.adopt(ctx, nodeID, reg.GetRunningShards(), c)
|
||||
|
||||
// One goroutine writes, this one reads. Splitting them is what lets a
|
||||
// command be dispatched while the agent is mid-heartbeat.
|
||||
var wg sync.WaitGroup
|
||||
@@ -123,6 +131,28 @@ func (s *Service) Connect(
|
||||
}
|
||||
}
|
||||
|
||||
// adopt hands the agent's running shards to the reconciler and stops whatever
|
||||
// nothing in the catalog claims.
|
||||
func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.RunningShard, c *conn) {
|
||||
if len(shards) == 0 {
|
||||
return
|
||||
}
|
||||
unclaimed := s.status.Adopt(AdoptedFromRegister(nodeID, shards))
|
||||
for _, u := range unclaimed {
|
||||
// The model or profile left the catalog while this was running. Nobody
|
||||
// will ever route to it, and it is holding GPUs.
|
||||
s.log.Warn("stopping an unclaimed shard",
|
||||
"node", nodeID, "shard", u.ShardID, "model", u.Model, "profile", u.Profile)
|
||||
select {
|
||||
case c.send <- &llmgwv1.ControlMessage{
|
||||
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: u.ShardID}},
|
||||
}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) writeLoop(ctx context.Context,
|
||||
stream *connect.BidiStream[llmgwv1.AgentMessage, llmgwv1.ControlMessage], c *conn,
|
||||
) {
|
||||
|
||||
Reference in new issue
Block a user