CI / test (push) Successful in 1m28s
The agent has always logged a percentage as it fetches weights, with a comment saying a replica pulling a hundred gigabytes and one that is wedged look identical from the control plane. They still did: the log is on the node, and the node is not where anyone is looking when a replica has been starting for ten minutes. Progress now rides on the shard status and appears under the replica in `ctl replicas` — a line rather than a column, because most replicas are not downloading anything and a column would be empty for all of them. It is deliberately not a state transition. Treating each chunk as one would re-mark the shard as starting on every report and reset the timer that decides when a slow start has become a failed one, so an hour-long download would keep a doomed replica alive indefinitely. It arrives on its own method for that reason, and a real transition clears it — otherwise a finished download sits in the listing at ninety-nine percent under a shard that has been serving for an hour. Watchers are keyed by source rather than by shard, because a fetch is single-flight: eight shards starting together produce one download and all eight are waiting on it. No percentage is invented when the server sent no length. One that stops moving is worse than a byte count that keeps rising, so the field is absent and the listing says "so far". Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
445 lines
15 KiB
Go
445 lines
15 KiB
Go
package agentsvc
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"slices"
|
|
"sync"
|
|
|
|
"connectrpc.com/connect"
|
|
|
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
|
"gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1/llmgwv1connect"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
|
)
|
|
|
|
// StatusSink receives shard transitions and recovers replica state from what
|
|
// agents report they are already running. The reconciler implements it.
|
|
type StatusSink interface {
|
|
OnShardStatus(replicaID, shardID string, state supervisor.State, errMsg string)
|
|
// OnFetchProgress is separate because it is not a transition: the shard is
|
|
// still starting, and treating each report as a state change would reset
|
|
// the timers that decide when a slow start has become a failed one.
|
|
OnFetchProgress(replicaID, shardID, source string, done, total int64)
|
|
Adopt(nodeID string, shards []reconciler.AdoptedShard) (unclaimed []reconciler.AdoptedShard)
|
|
}
|
|
|
|
// Service is the control plane's AgentService implementation, and at the same
|
|
// time the reconciler's Dispatcher: it holds the open streams, so it is the
|
|
// only thing that can deliver a command to a node.
|
|
type Service struct {
|
|
// leader gates whether this process accepts agent streams at all.
|
|
leader Leader
|
|
|
|
inv *inventory.Store
|
|
status StatusSink
|
|
log *slog.Logger
|
|
|
|
// requireIdentity refuses agents that present no certificate. It is off
|
|
// only for the insecure development path.
|
|
requireIdentity bool
|
|
|
|
mu sync.RWMutex
|
|
conns map[string]*conn
|
|
}
|
|
|
|
// conn is one agent's open stream.
|
|
type conn struct {
|
|
nodeID string
|
|
// send is serialized through a channel rather than a mutex around the
|
|
// stream: connect's streams are not safe for concurrent writes, and the
|
|
// reconciler dispatches to many nodes at once.
|
|
send chan *llmgwv1.ControlMessage
|
|
closed chan struct{}
|
|
once sync.Once
|
|
}
|
|
|
|
func (c *conn) close() { c.once.Do(func() { close(c.closed) }) }
|
|
|
|
// New builds the service.
|
|
//
|
|
// requireIdentity refuses any agent that has not proved who it is. It should be
|
|
// true whenever TLS is configured, and is false only for local runs without
|
|
// certificates.
|
|
func New(inv *inventory.Store, status StatusSink, requireIdentity bool, log *slog.Logger) *Service {
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
return &Service{
|
|
inv: inv, status: status, log: log,
|
|
requireIdentity: requireIdentity,
|
|
conns: map[string]*conn{},
|
|
}
|
|
}
|
|
|
|
// Handler returns the HTTP path and handler to mount.
|
|
//
|
|
// The handler is wrapped so the verified client identity reaches the stream:
|
|
// connect exposes the peer's address but not its certificate.
|
|
func (s *Service) Handler() (string, http.Handler) {
|
|
path, h := llmgwv1connect.NewAgentServiceHandler(s)
|
|
return path, AuthMiddleware(h)
|
|
}
|
|
|
|
// Connect serves one agent's stream for the lifetime of that agent.
|
|
func (s *Service) Connect(
|
|
ctx context.Context,
|
|
stream *connect.BidiStream[llmgwv1.AgentMessage, llmgwv1.ControlMessage],
|
|
) error {
|
|
// A standby must not hold an agent's stream. Commands travel down the
|
|
// stream the agent opened, so whichever control plane holds it is the only
|
|
// one that can start or stop anything on that node — and if that is not
|
|
// the leader, the leader's placements fail with "no agent connected" for a
|
|
// node that is plainly connected.
|
|
//
|
|
// Refused before the opening message is read, so the agent spends nothing
|
|
// on a connection that cannot serve it.
|
|
if s.leader != nil && !s.leader.IsLeader() {
|
|
return connect.NewError(connect.CodeUnavailable,
|
|
errors.New("this control plane is a standby; connect to the leader"))
|
|
}
|
|
|
|
first, err := stream.Receive()
|
|
if err != nil {
|
|
return connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("no opening message: %w", err))
|
|
}
|
|
reg := first.GetRegister()
|
|
if reg == nil {
|
|
// Registration first, always: the control plane has to know which node
|
|
// it is talking to before it can route anything to it.
|
|
return connect.NewError(connect.CodeInvalidArgument,
|
|
errors.New("the first message must be a Register"))
|
|
}
|
|
nodeID := reg.GetNodeId()
|
|
if nodeID == "" {
|
|
return connect.NewError(connect.CodeInvalidArgument, errors.New("node_id is required"))
|
|
}
|
|
|
|
// The certificate decides who this is, not the message.
|
|
//
|
|
// Without this check an agent could register as any node in the fleet and
|
|
// receive that node's shards — which is the same exposure the stream had
|
|
// before TLS, just with encryption on top. mTLS is only worth having if the
|
|
// identity it proves is the one actually used.
|
|
if peer := peerFrom(ctx); peer.Verified {
|
|
if peer.CommonName != nodeID {
|
|
s.log.Warn("rejecting an agent claiming a node id its certificate does not authorize",
|
|
"certificate_cn", peer.CommonName, "claimed", nodeID)
|
|
return connect.NewError(connect.CodePermissionDenied,
|
|
fmt.Errorf("certificate authorizes node %q, not %q", peer.CommonName, nodeID))
|
|
}
|
|
} else if s.requireIdentity {
|
|
return connect.NewError(connect.CodeUnauthenticated,
|
|
errors.New("no verified client certificate"))
|
|
}
|
|
|
|
node := NodeFromRegister(reg)
|
|
s.inv.Upsert(node)
|
|
s.log.Info("agent registered",
|
|
"node", nodeID, "gpus", len(node.GPUs), "fabrics", node.FabricNames(),
|
|
"running_shards", len(reg.GetRunningShards()),
|
|
"agent_version", reg.GetAgentVersion())
|
|
|
|
c := s.attach(ctx, nodeID)
|
|
defer s.detach(nodeID, c)
|
|
|
|
if err := stream.Send(&llmgwv1.ControlMessage{
|
|
Msg: &llmgwv1.ControlMessage_Ack{Ack: &llmgwv1.Ack{MessageId: "register", Ok: true}},
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Recover replica state from what this node is already running, before any
|
|
// reconciliation pass can conclude the cluster is empty and place
|
|
// duplicates (plan §6.5).
|
|
s.adopt(ctx, nodeID, reg.GetRunningShards(), c)
|
|
|
|
// One goroutine writes, this one reads. Splitting them is what lets a
|
|
// command be dispatched while the agent is mid-heartbeat.
|
|
var wg sync.WaitGroup
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
s.writeLoop(ctx, stream, c)
|
|
}()
|
|
defer wg.Wait()
|
|
|
|
for {
|
|
msg, err := stream.Receive()
|
|
if err != nil {
|
|
c.close()
|
|
if ctx.Err() != nil || errors.Is(err, io.EOF) {
|
|
s.log.Info("agent disconnected", "node", nodeID)
|
|
return nil
|
|
}
|
|
s.log.Warn("agent stream failed", "node", nodeID, "err", err)
|
|
return nil
|
|
}
|
|
s.handle(nodeID, msg)
|
|
}
|
|
}
|
|
|
|
// adopt hands the agent's running shards to the reconciler and stops whatever
|
|
// nothing in the catalog claims.
|
|
//
|
|
// An empty list is not skipped. "I am running nothing" is information — it is
|
|
// how the control plane learns that an engine died while its agent survived —
|
|
// and returning early on it made the reconciler's half of that exchange
|
|
// unreachable in production while its unit tests, which call the reconciler
|
|
// directly, kept passing.
|
|
func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.RunningShard, c *conn) {
|
|
unclaimed := s.status.Adopt(nodeID, AdoptedFromRegister(nodeID, shards))
|
|
for _, u := range unclaimed {
|
|
// The model or profile left the catalog while this was running. Nobody
|
|
// will ever route to it, and it is holding GPUs.
|
|
s.log.Warn("stopping an unclaimed shard",
|
|
"node", nodeID, "shard", u.ShardID, "model", u.Model, "profile", u.Profile)
|
|
select {
|
|
case c.send <- &llmgwv1.ControlMessage{
|
|
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: u.ShardID}},
|
|
}:
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Service) writeLoop(ctx context.Context,
|
|
stream *connect.BidiStream[llmgwv1.AgentMessage, llmgwv1.ControlMessage], c *conn,
|
|
) {
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-c.closed:
|
|
return
|
|
case msg := <-c.send:
|
|
if err := stream.Send(msg); err != nil {
|
|
s.log.Warn("sending to agent failed", "node", c.nodeID, "err", err)
|
|
c.close()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
|
|
switch m := msg.Msg.(type) {
|
|
case *llmgwv1.AgentMessage_Heartbeat:
|
|
s.inv.Touch(nodeID)
|
|
s.applyHeartbeat(nodeID, m.Heartbeat)
|
|
|
|
case *llmgwv1.AgentMessage_Shard:
|
|
// Progress rather than a transition. Falling through to OnShardStatus
|
|
// would re-mark the shard as starting on every chunk, which is how a
|
|
// download that takes an hour would keep a start timer at zero.
|
|
if f := m.Shard.GetFetch(); f != nil {
|
|
s.status.OnFetchProgress(m.Shard.GetReplicaId(), m.Shard.GetShardId(),
|
|
f.GetSource(), f.GetBytesDone(), f.GetBytesTotal())
|
|
return
|
|
}
|
|
st := StateFromWire(m.Shard.GetState())
|
|
s.status.OnShardStatus(m.Shard.GetReplicaId(), m.Shard.GetShardId(), st, m.Shard.GetError())
|
|
if m.Shard.GetState() == llmgwv1.ShardStatus_STATE_FAILED {
|
|
s.log.Error("shard failed",
|
|
"node", nodeID, "shard", m.Shard.GetShardId(),
|
|
"err", m.Shard.GetError(), "exit_code", m.Shard.GetExitCode(),
|
|
"last_output", m.Shard.GetLastOutput())
|
|
}
|
|
|
|
case *llmgwv1.AgentMessage_Register:
|
|
// A re-registration on a live stream means the agent's view of its own
|
|
// hardware changed — a GPU fell out, say. Take it.
|
|
s.inv.Upsert(NodeFromRegister(m.Register))
|
|
s.log.Info("agent re-registered", "node", nodeID)
|
|
}
|
|
}
|
|
|
|
// applyHeartbeat folds in the cheap facts a heartbeat carries, so a foreign
|
|
// process appearing on a card after registration is noticed without waiting for
|
|
// a reconnect.
|
|
func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
|
used := hb.GetGpuMemoryUsedMb()
|
|
problems := hb.GetNewProblems()
|
|
storage := hb.GetStorage()
|
|
if len(used) == 0 && len(problems) == 0 && len(storage) == 0 && !hb.GetWeightsChanged() {
|
|
return
|
|
}
|
|
|
|
n, ok := s.inv.Get(nodeID)
|
|
if !ok {
|
|
return
|
|
}
|
|
for i := range n.GPUs {
|
|
if i < len(used) {
|
|
n.GPUs[i].MemoryUsedMB = int(used[i])
|
|
}
|
|
}
|
|
if len(problems) > 0 {
|
|
// Problems are reported per node, not per GPU, so attach them to the
|
|
// node's first GPU rather than guessing which one is affected. Any
|
|
// problem is enough to keep that GPU out of placement, which is the
|
|
// conservative direction.
|
|
n.GPUs[0].Health.Problems = append(n.GPUs[0].Health.Problems, problems...)
|
|
}
|
|
|
|
// Disk is a fact about a moment, not about a machine. Sampled only at
|
|
// registration it is wrong on exactly the node worth watching — the one
|
|
// busy filling up — and it stays wrong until that agent restarts.
|
|
if len(storage) > 0 {
|
|
n.Storage = n.Storage[:0]
|
|
for _, st := range storage {
|
|
n.Storage = append(n.Storage, discovery.Storage{
|
|
Path: st.GetPath(),
|
|
FreeBytes: st.GetFreeBytes(),
|
|
TotalBytes: st.GetTotalBytes(),
|
|
Shared: st.GetShared(),
|
|
})
|
|
}
|
|
}
|
|
|
|
// Only when the agent says the list is meant. An empty list without the
|
|
// flag is silence, and treating it as "this node holds nothing" would
|
|
// empty a node's inventory on every heartbeat.
|
|
if hb.GetWeightsChanged() {
|
|
n.Weights = n.Weights[:0]
|
|
for _, w := range hb.GetWeights() {
|
|
n.Weights = append(n.Weights, discovery.LocalWeights{
|
|
Path: w.GetPath(), SizeBytes: w.GetSizeBytes(), Shared: w.GetShared(),
|
|
})
|
|
}
|
|
}
|
|
s.inv.Upsert(n)
|
|
}
|
|
|
|
func (s *Service) attach(ctx context.Context, nodeID string) *conn {
|
|
c := &conn{
|
|
nodeID: nodeID,
|
|
// Buffered so a burst of commands does not block the reconciler on a
|
|
// slow agent; if it fills, the agent is too far behind to be useful.
|
|
send: make(chan *llmgwv1.ControlMessage, 64),
|
|
closed: make(chan struct{}),
|
|
}
|
|
|
|
s.mu.Lock()
|
|
if prev, ok := s.conns[nodeID]; ok {
|
|
// A node reconnecting before the old stream was reaped. The new one
|
|
// wins: the old is by definition the dead one.
|
|
s.log.Warn("agent reconnected while an old stream was open", "node", nodeID)
|
|
prev.close()
|
|
}
|
|
s.conns[nodeID] = c
|
|
s.mu.Unlock()
|
|
|
|
go func() {
|
|
<-ctx.Done()
|
|
c.close()
|
|
}()
|
|
return c
|
|
}
|
|
|
|
func (s *Service) detach(nodeID string, c *conn) {
|
|
c.close()
|
|
s.mu.Lock()
|
|
if s.conns[nodeID] == c {
|
|
delete(s.conns, nodeID)
|
|
}
|
|
s.mu.Unlock()
|
|
|
|
// The node is not marked lost here. Losing a stream is not losing a node,
|
|
// and the reconciler's two-stage expiry already handles silence with the
|
|
// right delays (plan §6.3).
|
|
s.log.Info("agent stream closed", "node", nodeID)
|
|
}
|
|
|
|
// ConnectedNodes lists the nodes with an open stream.
|
|
func (s *Service) ConnectedNodes() []string {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
out := make([]string, 0, len(s.conns))
|
|
for id := range s.conns {
|
|
out = append(out, id)
|
|
}
|
|
slices.Sort(out)
|
|
return out
|
|
}
|
|
|
|
// --- Dispatcher ---
|
|
|
|
// Leader reports whether this process is the one that should be holding agent
|
|
// streams.
|
|
type Leader interface{ IsLeader() bool }
|
|
|
|
// SetLeadership makes this service refuse agents while it is a standby.
|
|
//
|
|
// Nil accepts them unconditionally, which is what a single control plane wants
|
|
// and what every deployment before leadership had.
|
|
func (s *Service) SetLeadership(l Leader) { s.leader = l }
|
|
|
|
// ErrNotConnected means no agent stream is open for that node.
|
|
var ErrNotConnected = errors.New("no agent connected")
|
|
|
|
// StartShard implements reconciler.Dispatcher.
|
|
func (s *Service) StartShard(_ context.Context, cmd reconciler.ShardCommand) error {
|
|
return s.send(cmd.NodeID, &llmgwv1.ControlMessage{
|
|
Msg: &llmgwv1.ControlMessage_Start{Start: StartShardFromCommand(cmd)},
|
|
})
|
|
}
|
|
|
|
// StopShard implements reconciler.Dispatcher.
|
|
func (s *Service) StopShard(_ context.Context, nodeID, shardID string) error {
|
|
return s.send(nodeID, &llmgwv1.ControlMessage{
|
|
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: shardID}},
|
|
})
|
|
}
|
|
|
|
// DrainShard implements reconciler.Dispatcher.
|
|
func (s *Service) DrainShard(_ context.Context, nodeID, shardID string) error {
|
|
return s.send(nodeID, &llmgwv1.ControlMessage{
|
|
Msg: &llmgwv1.ControlMessage_Drain{Drain: &llmgwv1.DrainShard{ShardId: shardID}},
|
|
})
|
|
}
|
|
|
|
// RemoveWeights asks a node to delete cached files no model needs.
|
|
//
|
|
// The node checks every path again before unlinking it, so this is a proposal
|
|
// rather than an instruction — which is the right shape for a message that
|
|
// deletes data the control plane cannot see.
|
|
func (s *Service) RemoveWeights(_ context.Context, nodeID string, paths []string) error {
|
|
if len(paths) == 0 {
|
|
return nil
|
|
}
|
|
return s.send(nodeID, &llmgwv1.ControlMessage{
|
|
Msg: &llmgwv1.ControlMessage_RemoveWeights{
|
|
RemoveWeights: &llmgwv1.RemoveWeights{Paths: paths},
|
|
},
|
|
})
|
|
}
|
|
|
|
func (s *Service) send(nodeID string, msg *llmgwv1.ControlMessage) error {
|
|
s.mu.RLock()
|
|
c, ok := s.conns[nodeID]
|
|
s.mu.RUnlock()
|
|
if !ok {
|
|
return fmt.Errorf("%w: node %s", ErrNotConnected, nodeID)
|
|
}
|
|
|
|
select {
|
|
case c.send <- msg:
|
|
return nil
|
|
case <-c.closed:
|
|
return fmt.Errorf("%w: node %s stream closed", ErrNotConnected, nodeID)
|
|
default:
|
|
// The queue is full, so this agent is far enough behind that queueing
|
|
// more would only delay discovering it is broken.
|
|
return fmt.Errorf("node %s is not keeping up with commands", nodeID)
|
|
}
|
|
}
|