Files
llmgw/internal/agent/agent.go
T
ilkerandClaude Opus 5 1d1a6aebb0 Phase 4: make a multi-node replica able to start
Placement and environment generation had landed early, which made multi-node
look further along than it was. A replica spanning nodes could not come up at
all, and all three reasons came down to assumptions that only hold on one node.

Not every rank serves HTTP. The agent health-checked every shard at its own
port, but only the leader answers; workers timed out, reported Failed and took
the replica with them. Engine.ServesHTTP now says which ranks serve, and a
worker is judged by staying alive. That is deliberately weaker evidence: the
replica's real readiness is the leader answering, which only happens once every
worker has joined.

A shard is not always one process. Engine.Commands returns a sequence — every
command but the last is a sidecar kept alive for the shard's lifetime and torn
down with it, and the last one's exit is the shard's. vLLM under Ray is why: the
leader runs a Ray head and then the server against it, while a worker runs only
the join and no vLLM process at all. RayCommand had been written and tested but
never called, so launch_mode: ray would have run vllm serve against a cluster
that did not exist. Sidecars are killed with the shard, because a Ray head left
behind holds the port the next replica needs.

Start order was backwards. Workers went first, on the theory that the head
should wait for them. Both engines put the rendezvous point on rank 0, so a
worker started first has nothing to connect to. vLLM's server blocks until the
Ray cluster has the GPUs it needs, so the leader can start immediately and waits
on its own. A command can now also require that rendezvous point to be listening
before the next starts, bounded by a timeout so a head that never appears fails
the shard instead of hanging it.

Stop was also killing only the engine, leaving sidecars behind.

None of this is verified against real engines — it is tested with stub
processes. The Ray sequencing, and which launch mode a given vLLM version wants,
can only be confirmed on hardware.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 06:20:03 +02:00

559 lines
16 KiB
Go

// Package agent is the node side of the control plane stream (plan §5.6).
package agent
import (
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"log/slog"
"math/rand/v2"
"net"
"net/http"
"sync"
"time"
"golang.org/x/net/http2"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1/llmgwv1connect"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// Config configures the agent.
type Config struct {
NodeID string `koanf:"node_id"`
Server string `koanf:"server"`
Hostname string `koanf:"hostname"`
MgmtAddr string `koanf:"mgmt_addr"`
Labels map[string]string `koanf:"labels"`
// MgmtInterface is the ethernet interface NCCL bootstraps over.
MgmtInterface string `koanf:"mgmt_interface"`
// Fabrics labels this node's HCAs. It is explicit on purpose: OpenSM's
// default subnet prefix is the same on every fabric, so nothing can be
// inferred from the wire (plan §5.2).
Fabrics []inventory.FabricAttachment `koanf:"fabrics"`
HeartbeatInterval time.Duration `koanf:"heartbeat_interval"`
// ReconnectBase is the first reconnect delay; it backs off from there.
ReconnectBase time.Duration `koanf:"reconnect_base"`
ReconnectMax time.Duration `koanf:"reconnect_max"`
Discovery discovery.Options `koanf:"discovery"`
Engines engines.Config `koanf:"engines"`
Superv supervisor.Config `koanf:"supervisor"`
Version string
}
// DefaultConfig returns sensible defaults.
func DefaultConfig() Config {
return Config{
MgmtInterface: "eth0",
HeartbeatInterval: 5 * time.Second,
ReconnectBase: time.Second,
ReconnectMax: 30 * time.Second,
Superv: supervisor.DefaultConfig(),
// NVML gives NVLink topology, so prefer it when this build has it.
Discovery: discovery.Options{PreferNVML: true},
}
}
func (c *Config) normalize() error {
d := DefaultConfig()
if c.NodeID == "" {
return errors.New("node_id is required")
}
if c.Server == "" {
return errors.New("server is required")
}
if c.MgmtInterface == "" {
c.MgmtInterface = d.MgmtInterface
}
if c.HeartbeatInterval <= 0 {
c.HeartbeatInterval = d.HeartbeatInterval
}
if c.ReconnectBase <= 0 {
c.ReconnectBase = d.ReconnectBase
}
if c.ReconnectMax < c.ReconnectBase {
c.ReconnectMax = max(c.ReconnectBase, d.ReconnectMax)
}
return nil
}
// Agent runs on a GPU node.
type Agent struct {
cfg Config
log *slog.Logger
sup *supervisor.Supervisor
reg *engines.Registry
http connectHTTPClient
mu sync.Mutex
inv discovery.Inventory
// shardReplica remembers which replica a shard belongs to, so a status
// report can name both. The control plane keys on the replica.
shardReplica map[string]string
}
// connectHTTPClient is the HTTP client connect uses; an interface so tests can
// hand in an httptest server's client.
type connectHTTPClient interface {
Do(*http.Request) (*http.Response, error)
}
// New builds an agent.
func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, error) {
if err := cfg.normalize(); err != nil {
return nil, err
}
if log == nil {
log = slog.Default()
}
if httpClient == nil {
httpClient = http.DefaultClient
}
health := supervisor.NewHTTPHealth(3 * time.Second)
sup, err := supervisor.New(cfg.Superv, health, log)
if err != nil {
return nil, err
}
return &Agent{
cfg: cfg, log: log, sup: sup,
reg: engines.NewRegistry(cfg.Engines),
http: httpClient,
shardReplica: map[string]string{},
}, nil
}
// Supervisor exposes the process supervisor, for tests and `ctl logs`.
func (a *Agent) Supervisor() *supervisor.Supervisor { return a.sup }
// Run connects to the control plane and stays connected until ctx ends.
//
// Reconnecting forever is the right behaviour for a node agent: a control plane
// restart or a network partition must not require touching every GPU node, and
// the engines keep serving traffic throughout because the router talks to them
// directly.
func (a *Agent) Run(ctx context.Context) error {
if err := a.scan(ctx); err != nil {
// Not fatal: a node whose GPUs cannot be read should still register, so
// an operator can see it and why it is useless.
a.log.Error("hardware discovery failed", "err", err)
}
a.recoverOrphans()
attempt := 0
for ctx.Err() == nil {
err := a.session(ctx)
if ctx.Err() != nil {
break
}
attempt++
delay := a.reconnectDelay(attempt)
a.log.Warn("disconnected from the control plane, retrying",
"err", err, "in", delay.Round(time.Millisecond), "attempt", attempt)
select {
case <-ctx.Done():
case <-time.After(delay):
}
if err == nil {
attempt = 0
}
}
// Engines keep running when the agent stops on purpose: an agent restart
// must not interrupt served traffic. Orphan adoption picks them up again.
return ctx.Err()
}
func (a *Agent) reconnectDelay(attempt int) time.Duration {
d := a.cfg.ReconnectBase << min(attempt-1, 16)
if d > a.cfg.ReconnectMax || d <= 0 {
d = a.cfg.ReconnectMax
}
// Jitter, so a fleet does not reconnect in lockstep after a control plane
// restart and knock it over again.
return d/2 + time.Duration(rand.Int64N(int64(d/2))) //nolint:gosec // jitter
}
// recoverOrphans takes back engines a previous agent left running.
//
// Engines deliberately outlive their agent — the router talks to them directly,
// so an agent restart must not interrupt served traffic. The cost is that they
// have to be found again, and anything that cannot be identified has to be
// killed: a process holding GPUs that nothing can describe is worse than no
// process at all, because placement will hand those GPUs out again.
func (a *Agent) recoverOrphans() {
orphans, err := a.sup.FindOrphans()
if err != nil {
a.log.Warn("scanning for leftover engines failed", "err", err)
return
}
for _, o := range orphans {
switch {
case !o.Alive:
// The process is gone; only its state file remains.
if err := a.sup.KillOrphan(o); err != nil {
a.log.Warn("cleaning up a dead shard's state failed", "shard", o.ShardID, "err", err)
}
case o.ReplicaID == "":
// Alive, but its state file could not be read, so it cannot be
// described to the control plane or matched to a replica.
a.log.Warn("killing an unidentifiable engine", "shard", o.ShardID, "pid", o.PID)
if err := a.sup.KillOrphan(o); err != nil {
a.log.Warn("killing it failed", "shard", o.ShardID, "err", err)
}
default:
sh, err := a.sup.Adopt(o)
if err != nil {
a.log.Warn("adopting a shard failed", "shard", o.ShardID, "err", err)
continue
}
a.mu.Lock()
a.shardReplica[o.ShardID] = o.ReplicaID
a.mu.Unlock()
a.log.Info("adopted a running engine",
"shard", o.ShardID, "replica", o.ReplicaID, "pid", o.PID)
_ = sh
}
}
}
// runningShards describes what this agent is supervising, for registration.
func (a *Agent) runningShards() []*llmgwv1.RunningShard {
var out []*llmgwv1.RunningShard
for _, st := range a.sup.Statuses() {
if st.State.Terminal() {
continue
}
sh, err := a.sup.Get(st.ID)
if err != nil {
continue
}
spec := sh.Spec
out = append(out, &llmgwv1.RunningShard{
ReplicaId: spec.ReplicaID,
ShardId: st.ID,
Rank: int32(spec.Rank),
Nnodes: int32(spec.NNodes),
GpuIndices: int32Slice(spec.GPUIndices),
Hcas: spec.HCAs,
HttpPort: int32(spec.HTTPPort),
State: agentsvc.StateToWire(st.State),
Model: spec.Profile.ServedModelName,
Profile: spec.Profile.Name,
Pool: spec.Pool,
Fabric: spec.Fabric,
LeaderAddr: a.cfg.MgmtAddr,
LeaderPort: int32(spec.LeaderPort),
})
}
return out
}
func int32Slice(v []int) []int32 {
if len(v) == 0 {
return nil
}
out := make([]int32, len(v))
for i, x := range v {
out[i] = int32(x)
}
return out
}
// scan refreshes the node's hardware inventory.
func (a *Agent) scan(ctx context.Context) error {
inv, err := discovery.Discover(ctx, a.cfg.Discovery, a.log)
a.mu.Lock()
a.inv = inv
a.mu.Unlock()
return err
}
// session runs one connection until it breaks.
func (a *Agent) session(ctx context.Context) error {
client := llmgwv1connect.NewAgentServiceClient(a.http, a.cfg.Server)
ctx, cancel := context.WithCancel(ctx)
defer cancel()
stream := client.Connect(ctx)
defer func() { _ = stream.CloseRequest() }()
a.mu.Lock()
inv := a.inv
a.mu.Unlock()
reg := agentsvc.RegisterFromInventory(
a.cfg.NodeID, a.cfg.Hostname, a.cfg.MgmtAddr, a.cfg.Version,
a.cfg.Labels, inv, a.cfg.Fabrics)
// Every registration reports what is already running, not just the first
// one: a reconnect may be a control plane restart, and it has to recover
// the same way (plan §6.5).
reg.RunningShards = a.runningShards()
if err := stream.Send(&llmgwv1.AgentMessage{
Msg: &llmgwv1.AgentMessage_Register{Register: reg},
}); err != nil {
return fmt.Errorf("registering: %w", err)
}
// Sends are serialized through this channel: connect streams are not safe
// for concurrent writes, and heartbeats race with status reports.
out := make(chan *llmgwv1.AgentMessage, 32)
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
a.sendLoop(ctx, stream, out)
}()
defer wg.Wait()
for {
msg, err := stream.Receive()
if err != nil {
cancel()
if errors.Is(err, io.EOF) || ctx.Err() != nil {
return nil
}
return err
}
a.handle(ctx, msg, out)
}
}
type sender interface {
Send(*llmgwv1.AgentMessage) error
}
func (a *Agent) sendLoop(ctx context.Context, s sender, out <-chan *llmgwv1.AgentMessage) {
t := time.NewTicker(a.cfg.HeartbeatInterval)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case msg := <-out:
if err := s.Send(msg); err != nil {
return
}
case <-t.C:
if err := s.Send(a.heartbeat()); err != nil {
return
}
}
}
}
func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
a.mu.Lock()
inv := a.inv
a.mu.Unlock()
hb := &llmgwv1.Heartbeat{NodeId: a.cfg.NodeID}
for _, g := range inv.GPUs {
hb.GpuMemoryUsedMb = append(hb.GpuMemoryUsedMb, int32(g.MemoryUsedMB))
}
return &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Heartbeat{Heartbeat: hb}}
}
func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out chan<- *llmgwv1.AgentMessage) {
switch m := msg.Msg.(type) {
case *llmgwv1.ControlMessage_Start:
go a.startShard(ctx, m.Start, out)
case *llmgwv1.ControlMessage_Stop:
go a.stopShard(ctx, m.Stop.GetShardId(), out)
case *llmgwv1.ControlMessage_Drain:
if err := a.sup.Drain(m.Drain.GetShardId()); err != nil {
a.log.Warn("draining failed", "shard", m.Drain.GetShardId(), "err", err)
}
case *llmgwv1.ControlMessage_Ack:
if !m.Ack.GetOk() {
a.log.Warn("control plane rejected a message",
"id", m.Ack.GetMessageId(), "err", m.Ack.GetError())
}
}
}
// startShard launches a shard and reports its progress.
//
// It runs in its own goroutine because model loading takes minutes, and the
// stream has to keep carrying heartbeats and other commands meanwhile.
func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<- *llmgwv1.AgentMessage) {
shardID, replicaID := m.GetShardId(), m.GetReplicaId()
a.mu.Lock()
a.shardReplica[shardID] = replicaID
a.mu.Unlock()
report := func(st supervisor.State, errMsg string, tail []string) {
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
NodeId: a.cfg.NodeID, ReplicaId: replicaID, ShardId: shardID,
State: agentsvc.StateToWire(st), Error: errMsg, LastOutput: tail,
}}}
select {
case out <- msg:
case <-ctx.Done():
}
}
spec, timeout := agentsvc.ShardSpecFromStart(m)
eng, err := a.reg.Get(spec.Profile.Engine)
if err != nil {
report(supervisor.Failed, err.Error(), nil)
return
}
a.mu.Lock()
inv := a.inv
a.mu.Unlock()
// Validate before launching. An engine that refuses a profile does so after
// minutes of loading; catching it here costs nothing.
if err := eng.Validate(spec.Profile, gpuInfo(inv, spec.GPUIndices)); err != nil {
report(supervisor.Failed, err.Error(), nil)
return
}
cmds, err := eng.Commands(spec)
if err != nil {
report(supervisor.Failed, err.Error(), nil)
return
}
sh, err := a.sup.Start(ctx, shardID, spec, cmds)
if err != nil {
if errors.Is(err, supervisor.ErrAlreadyRunning) {
// Commands are idempotent: the control plane retries, and a repeat
// must not produce a second process (plan §5.6).
report(sh.Status().State, "", nil)
return
}
report(supervisor.Failed, err.Error(), sh.Logs(10))
return
}
report(supervisor.Starting, "", nil)
// Only a rank that serves HTTP can be health-checked. A worker is judged by
// staying alive, and the replica's real readiness is the leader answering,
// which only happens once every worker has joined (plan §11, phase 4).
if eng.ServesHTTP(spec) {
base := fmt.Sprintf("http://127.0.0.1:%d", spec.HTTPPort)
err = a.sup.WaitReady(ctx, sh, base, spec.Profile.ServedModelName, timeout)
} else {
err = a.sup.WaitAlive(ctx, sh)
}
if err != nil {
report(supervisor.Failed, err.Error(), sh.Logs(20))
return
}
report(supervisor.Ready, "", nil)
// Watch for the engine dying later. Nothing else would notice: the router
// would simply see failing requests.
go a.watch(ctx, sh, replicaID, shardID, out)
}
// watch reports a shard that exits after it was ready.
func (a *Agent) watch(ctx context.Context, sh *supervisor.Shard, replicaID, shardID string, out chan<- *llmgwv1.AgentMessage) {
t := time.NewTicker(time.Second)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
st := sh.Status()
if !st.State.Terminal() {
continue
}
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
NodeId: a.cfg.NodeID, ReplicaId: replicaID, ShardId: shardID,
State: agentsvc.StateToWire(st.State), Error: st.Err,
ExitCode: int32(st.ExitCode), LastOutput: sh.Logs(20),
}}}
select {
case out <- msg:
case <-ctx.Done():
}
return
}
}
}
func (a *Agent) stopShard(ctx context.Context, shardID string, out chan<- *llmgwv1.AgentMessage) {
a.mu.Lock()
replicaID := a.shardReplica[shardID]
delete(a.shardReplica, shardID)
a.mu.Unlock()
if err := a.sup.Stop(ctx, shardID); err != nil {
a.log.Warn("stopping a shard failed", "shard", shardID, "err", err)
}
a.sup.Remove(shardID)
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
NodeId: a.cfg.NodeID, ReplicaId: replicaID, ShardId: shardID,
State: agentsvc.StateToWire(supervisor.Stopped),
}}}
select {
case out <- msg:
case <-ctx.Done():
}
}
// gpuInfo narrows the inventory to the GPUs a shard will use.
func gpuInfo(inv discovery.Inventory, indices []int) []engines.GPUInfo {
want := map[int]struct{}{}
for _, i := range indices {
want[i] = struct{}{}
}
var out []engines.GPUInfo
for _, g := range inv.GPUs {
if _, ok := want[g.Index]; !ok {
continue
}
out = append(out, engines.GPUInfo{
Index: g.Index, Name: g.Name,
ComputeCapability: g.ComputeCapability, MemoryTotalMB: g.MemoryTotalMB,
})
}
return out
}
// HTTP2Client returns the client the agent should use to reach the control
// plane over plaintext HTTP/2.
//
// connect's bidirectional streams require HTTP/2, and an agent talking to a
// control plane over a trusted network has no TLS to negotiate it with. Go's
// default transport will not speak HTTP/2 without TLS, so it has to be asked
// explicitly.
//
// mTLS is plan §5.6's target and replaces this; until then the stream must not
// cross an untrusted network.
func HTTP2Client(timeout time.Duration) *http.Client {
return &http.Client{
Transport: &http2.Transport{
AllowHTTP: true,
DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
return (&net.Dialer{Timeout: timeout}).DialContext(ctx, network, addr)
},
},
}
}