Files
llmgw/internal/control/agentsvc/service.go
T
Ilker ManapandClaude Opus 5 aef1c143c5
CI / test (push) Successful in 1m19s
Let go of shards an agent says it is not running
Adoption handled "the agent has something we do not know about". The mirror
image was missing: "we think the agent has something it does not".

Both follow from the same rule — the fleet is authoritative about what is
running (plan §6.5) — and without the second half an engine that dies while
its agent survives leaves the control plane believing a replica is Ready
forever. Measured on the cluster: the engine was killed, the agent restarted
and registered with nothing running, and the replica stayed Ready pointing at
a dead port for two and a half minutes until I stopped watching. Traffic
survived, because the breaker took the endpoint out and retries covered it —
24 requests, none failed — so the only symptom was a cluster quietly running a
third short.

A registration follows a fresh connection, so anything dispatched over a
previous one is not running by definition; there is no window where a shard is
legitimately started but unreported.

The previous commit's disown rule was too broad, which this turned up: when a
known replica had a live shard it disowned everything the agent reported for
it, including the shards we already knew. A reconnecting agent would have been
told to stop the replica that was serving. It now disowns only what it does
not recognise.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 10:30:49 +02:00

359 lines
11 KiB
Go

package agentsvc
import (
"context"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"slices"
"sync"
"connectrpc.com/connect"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1/llmgwv1connect"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
)
// StatusSink receives shard transitions and recovers replica state from what
// agents report they are already running. The reconciler implements it.
type StatusSink interface {
OnShardStatus(replicaID, shardID string, state supervisor.State, errMsg string)
Adopt(nodeID string, shards []reconciler.AdoptedShard) (unclaimed []reconciler.AdoptedShard)
}
// Service is the control plane's AgentService implementation, and at the same
// time the reconciler's Dispatcher: it holds the open streams, so it is the
// only thing that can deliver a command to a node.
type Service struct {
inv *inventory.Store
status StatusSink
log *slog.Logger
// requireIdentity refuses agents that present no certificate. It is off
// only for the insecure development path.
requireIdentity bool
mu sync.RWMutex
conns map[string]*conn
}
// conn is one agent's open stream.
type conn struct {
nodeID string
// send is serialized through a channel rather than a mutex around the
// stream: connect's streams are not safe for concurrent writes, and the
// reconciler dispatches to many nodes at once.
send chan *llmgwv1.ControlMessage
closed chan struct{}
once sync.Once
}
func (c *conn) close() { c.once.Do(func() { close(c.closed) }) }
// New builds the service.
//
// requireIdentity refuses any agent that has not proved who it is. It should be
// true whenever TLS is configured, and is false only for local runs without
// certificates.
func New(inv *inventory.Store, status StatusSink, requireIdentity bool, log *slog.Logger) *Service {
if log == nil {
log = slog.Default()
}
return &Service{
inv: inv, status: status, log: log,
requireIdentity: requireIdentity,
conns: map[string]*conn{},
}
}
// Handler returns the HTTP path and handler to mount.
//
// The handler is wrapped so the verified client identity reaches the stream:
// connect exposes the peer's address but not its certificate.
func (s *Service) Handler() (string, http.Handler) {
path, h := llmgwv1connect.NewAgentServiceHandler(s)
return path, AuthMiddleware(h)
}
// Connect serves one agent's stream for the lifetime of that agent.
func (s *Service) Connect(
ctx context.Context,
stream *connect.BidiStream[llmgwv1.AgentMessage, llmgwv1.ControlMessage],
) error {
first, err := stream.Receive()
if err != nil {
return connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("no opening message: %w", err))
}
reg := first.GetRegister()
if reg == nil {
// Registration first, always: the control plane has to know which node
// it is talking to before it can route anything to it.
return connect.NewError(connect.CodeInvalidArgument,
errors.New("the first message must be a Register"))
}
nodeID := reg.GetNodeId()
if nodeID == "" {
return connect.NewError(connect.CodeInvalidArgument, errors.New("node_id is required"))
}
// The certificate decides who this is, not the message.
//
// Without this check an agent could register as any node in the fleet and
// receive that node's shards — which is the same exposure the stream had
// before TLS, just with encryption on top. mTLS is only worth having if the
// identity it proves is the one actually used.
if peer := peerFrom(ctx); peer.Verified {
if peer.CommonName != nodeID {
s.log.Warn("rejecting an agent claiming a node id its certificate does not authorize",
"certificate_cn", peer.CommonName, "claimed", nodeID)
return connect.NewError(connect.CodePermissionDenied,
fmt.Errorf("certificate authorizes node %q, not %q", peer.CommonName, nodeID))
}
} else if s.requireIdentity {
return connect.NewError(connect.CodeUnauthenticated,
errors.New("no verified client certificate"))
}
node := NodeFromRegister(reg)
s.inv.Upsert(node)
s.log.Info("agent registered",
"node", nodeID, "gpus", len(node.GPUs), "fabrics", node.FabricNames(),
"running_shards", len(reg.GetRunningShards()),
"agent_version", reg.GetAgentVersion())
c := s.attach(ctx, nodeID)
defer s.detach(nodeID, c)
if err := stream.Send(&llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Ack{Ack: &llmgwv1.Ack{MessageId: "register", Ok: true}},
}); err != nil {
return err
}
// Recover replica state from what this node is already running, before any
// reconciliation pass can conclude the cluster is empty and place
// duplicates (plan §6.5).
s.adopt(ctx, nodeID, reg.GetRunningShards(), c)
// One goroutine writes, this one reads. Splitting them is what lets a
// command be dispatched while the agent is mid-heartbeat.
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
s.writeLoop(ctx, stream, c)
}()
defer wg.Wait()
for {
msg, err := stream.Receive()
if err != nil {
c.close()
if ctx.Err() != nil || errors.Is(err, io.EOF) {
s.log.Info("agent disconnected", "node", nodeID)
return nil
}
s.log.Warn("agent stream failed", "node", nodeID, "err", err)
return nil
}
s.handle(nodeID, msg)
}
}
// adopt hands the agent's running shards to the reconciler and stops whatever
// nothing in the catalog claims.
func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.RunningShard, c *conn) {
if len(shards) == 0 {
return
}
unclaimed := s.status.Adopt(nodeID, AdoptedFromRegister(nodeID, shards))
for _, u := range unclaimed {
// The model or profile left the catalog while this was running. Nobody
// will ever route to it, and it is holding GPUs.
s.log.Warn("stopping an unclaimed shard",
"node", nodeID, "shard", u.ShardID, "model", u.Model, "profile", u.Profile)
select {
case c.send <- &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: u.ShardID}},
}:
case <-ctx.Done():
return
}
}
}
func (s *Service) writeLoop(ctx context.Context,
stream *connect.BidiStream[llmgwv1.AgentMessage, llmgwv1.ControlMessage], c *conn,
) {
for {
select {
case <-ctx.Done():
return
case <-c.closed:
return
case msg := <-c.send:
if err := stream.Send(msg); err != nil {
s.log.Warn("sending to agent failed", "node", c.nodeID, "err", err)
c.close()
return
}
}
}
}
func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
switch m := msg.Msg.(type) {
case *llmgwv1.AgentMessage_Heartbeat:
s.inv.Touch(nodeID)
s.applyHeartbeat(nodeID, m.Heartbeat)
case *llmgwv1.AgentMessage_Shard:
st := StateFromWire(m.Shard.GetState())
s.status.OnShardStatus(m.Shard.GetReplicaId(), m.Shard.GetShardId(), st, m.Shard.GetError())
if m.Shard.GetState() == llmgwv1.ShardStatus_STATE_FAILED {
s.log.Error("shard failed",
"node", nodeID, "shard", m.Shard.GetShardId(),
"err", m.Shard.GetError(), "exit_code", m.Shard.GetExitCode(),
"last_output", m.Shard.GetLastOutput())
}
case *llmgwv1.AgentMessage_Register:
// A re-registration on a live stream means the agent's view of its own
// hardware changed — a GPU fell out, say. Take it.
s.inv.Upsert(NodeFromRegister(m.Register))
s.log.Info("agent re-registered", "node", nodeID)
}
}
// applyHeartbeat folds in the cheap facts a heartbeat carries, so a foreign
// process appearing on a card after registration is noticed without waiting for
// a reconnect.
func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
used := hb.GetGpuMemoryUsedMb()
problems := hb.GetNewProblems()
if len(used) == 0 && len(problems) == 0 {
return
}
n, ok := s.inv.Get(nodeID)
if !ok {
return
}
for i := range n.GPUs {
if i < len(used) {
n.GPUs[i].MemoryUsedMB = int(used[i])
}
}
if len(problems) > 0 {
// Problems are reported per node, not per GPU, so attach them to the
// node's first GPU rather than guessing which one is affected. Any
// problem is enough to keep that GPU out of placement, which is the
// conservative direction.
n.GPUs[0].Health.Problems = append(n.GPUs[0].Health.Problems, problems...)
}
s.inv.Upsert(n)
}
func (s *Service) attach(ctx context.Context, nodeID string) *conn {
c := &conn{
nodeID: nodeID,
// Buffered so a burst of commands does not block the reconciler on a
// slow agent; if it fills, the agent is too far behind to be useful.
send: make(chan *llmgwv1.ControlMessage, 64),
closed: make(chan struct{}),
}
s.mu.Lock()
if prev, ok := s.conns[nodeID]; ok {
// A node reconnecting before the old stream was reaped. The new one
// wins: the old is by definition the dead one.
s.log.Warn("agent reconnected while an old stream was open", "node", nodeID)
prev.close()
}
s.conns[nodeID] = c
s.mu.Unlock()
go func() {
<-ctx.Done()
c.close()
}()
return c
}
func (s *Service) detach(nodeID string, c *conn) {
c.close()
s.mu.Lock()
if s.conns[nodeID] == c {
delete(s.conns, nodeID)
}
s.mu.Unlock()
// The node is not marked lost here. Losing a stream is not losing a node,
// and the reconciler's two-stage expiry already handles silence with the
// right delays (plan §6.3).
s.log.Info("agent stream closed", "node", nodeID)
}
// ConnectedNodes lists the nodes with an open stream.
func (s *Service) ConnectedNodes() []string {
s.mu.RLock()
defer s.mu.RUnlock()
out := make([]string, 0, len(s.conns))
for id := range s.conns {
out = append(out, id)
}
slices.Sort(out)
return out
}
// --- Dispatcher ---
// ErrNotConnected means no agent stream is open for that node.
var ErrNotConnected = errors.New("no agent connected")
// StartShard implements reconciler.Dispatcher.
func (s *Service) StartShard(_ context.Context, cmd reconciler.ShardCommand) error {
return s.send(cmd.NodeID, &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Start{Start: StartShardFromCommand(cmd)},
})
}
// StopShard implements reconciler.Dispatcher.
func (s *Service) StopShard(_ context.Context, nodeID, shardID string) error {
return s.send(nodeID, &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: shardID}},
})
}
// DrainShard implements reconciler.Dispatcher.
func (s *Service) DrainShard(_ context.Context, nodeID, shardID string) error {
return s.send(nodeID, &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Drain{Drain: &llmgwv1.DrainShard{ShardId: shardID}},
})
}
func (s *Service) send(nodeID string, msg *llmgwv1.ControlMessage) error {
s.mu.RLock()
c, ok := s.conns[nodeID]
s.mu.RUnlock()
if !ok {
return fmt.Errorf("%w: node %s", ErrNotConnected, nodeID)
}
select {
case c.send <- msg:
return nil
case <-c.closed:
return fmt.Errorf("%w: node %s stream closed", ErrNotConnected, nodeID)
default:
// The queue is full, so this agent is far enough behind that queueing
// more would only delay discovering it is broken.
return fmt.Errorf("node %s is not keeping up with commands", nodeID)
}
}