Agent stream: connect the fleet to the control plane
CI / test (push) Failing after 1m29s

One bidirectional connect-go stream per node, carrying registration, heartbeats
and shard status up, and start, stop and drain down. The agent dials out: GPU
nodes sit behind firewalls far more often than a control plane does, and an
outbound connection needs no inbound rule on every node in the fleet.

- registration carries the whole hardware inventory, so there is no separate
  discovery call and a reconnecting agent refreshes what the scheduler knows.
  The fabric labels travel with it; they cannot be inferred from the wire, so
  losing them in transit would silently disable multi-node placement.
- agentsvc is both the stream handler and the reconciler's dispatcher, because
  it holds the open streams and is therefore the only thing that can deliver a
  command to a node.
- the agent validates a profile before launching anything, since an engine that
  refuses a profile does so after minutes of loading, then supervises the
  process and reports every transition with the engine's output tail on failure.
- reconnects forever with jittered backoff. A control plane restart must not
  require touching every GPU node, and a fleet must not reconnect in lockstep
  and knock it over again. Engines keep serving throughout, because the router
  talks to them directly.

llmgw agent is real now, and llmgw server runs a control plane whenever any
model declares a profile. The static endpoint list stays for router-only runs.

Verified with real binaries as separate processes: the agent registers, a
replica is placed, an engine launches through the vLLM adapter's actual argument
construction, streaming traffic is served, and killing the agent moves the node
through unreachable to lost with placement explaining why nothing can run.

config.Model and catalog.Model were the same thing described twice, so the
router now reads catalog entries directly and the validation rules live in one
place.

Still missing: mTLS. The stream is plaintext HTTP/2 and must stay on a trusted
network until that lands.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
ilkerandClaude Opus 5 committed 2026-09-16 02:48:21 +02:00
1 parent c1b9ea1bc2
commit 2373726efd
24 files changed
+3645 -87

No files matched your search

+458
View File
@@ -0,0 +1,458 @@
// Package agent is the node side of the control plane stream (plan §5.6).
package agent
import (
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"log/slog"
"math/rand/v2"
"net"
"net/http"
"sync"
"time"
"golang.org/x/net/http2"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1/llmgwv1connect"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// Config configures the agent.
type Config struct {
NodeID string `koanf:"node_id"`
Server string `koanf:"server"`
Hostname string `koanf:"hostname"`
MgmtAddr string `koanf:"mgmt_addr"`
Labels map[string]string `koanf:"labels"`
// MgmtInterface is the ethernet interface NCCL bootstraps over.
MgmtInterface string `koanf:"mgmt_interface"`
// Fabrics labels this node's HCAs. It is explicit on purpose: OpenSM's
// default subnet prefix is the same on every fabric, so nothing can be
// inferred from the wire (plan §5.2).
Fabrics []inventory.FabricAttachment `koanf:"fabrics"`
HeartbeatInterval time.Duration `koanf:"heartbeat_interval"`
// ReconnectBase is the first reconnect delay; it backs off from there.
ReconnectBase time.Duration `koanf:"reconnect_base"`
ReconnectMax time.Duration `koanf:"reconnect_max"`
Discovery discovery.Options `koanf:"discovery"`
Engines engines.Config `koanf:"engines"`
Superv supervisor.Config `koanf:"supervisor"`
Version string
}
// DefaultConfig returns sensible defaults.
func DefaultConfig() Config {
return Config{
MgmtInterface: "eth0",
HeartbeatInterval: 5 * time.Second,
ReconnectBase: time.Second,
ReconnectMax: 30 * time.Second,
Superv: supervisor.DefaultConfig(),
// NVML gives NVLink topology, so prefer it when this build has it.
Discovery: discovery.Options{PreferNVML: true},
}
}
func (c *Config) normalize() error {
d := DefaultConfig()
if c.NodeID == "" {
return errors.New("node_id is required")
}
if c.Server == "" {
return errors.New("server is required")
}
if c.MgmtInterface == "" {
c.MgmtInterface = d.MgmtInterface
}
if c.HeartbeatInterval <= 0 {
c.HeartbeatInterval = d.HeartbeatInterval
}
if c.ReconnectBase <= 0 {
c.ReconnectBase = d.ReconnectBase
}
if c.ReconnectMax < c.ReconnectBase {
c.ReconnectMax = max(c.ReconnectBase, d.ReconnectMax)
}
return nil
}
// Agent runs on a GPU node.
type Agent struct {
cfg Config
log *slog.Logger
sup *supervisor.Supervisor
reg *engines.Registry
http connectHTTPClient
mu sync.Mutex
inv discovery.Inventory
// shardReplica remembers which replica a shard belongs to, so a status
// report can name both. The control plane keys on the replica.
shardReplica map[string]string
}
// connectHTTPClient is the HTTP client connect uses; an interface so tests can
// hand in an httptest server's client.
type connectHTTPClient interface {
Do(*http.Request) (*http.Response, error)
}
// New builds an agent.
func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, error) {
if err := cfg.normalize(); err != nil {
return nil, err
}
if log == nil {
log = slog.Default()
}
if httpClient == nil {
httpClient = http.DefaultClient
}
health := supervisor.NewHTTPHealth(3 * time.Second)
sup, err := supervisor.New(cfg.Superv, health, log)
if err != nil {
return nil, err
}
return &Agent{
cfg: cfg, log: log, sup: sup,
reg: engines.NewRegistry(cfg.Engines),
http: httpClient,
shardReplica: map[string]string{},
}, nil
}
// Supervisor exposes the process supervisor, for tests and `ctl logs`.
func (a *Agent) Supervisor() *supervisor.Supervisor { return a.sup }
// Run connects to the control plane and stays connected until ctx ends.
//
// Reconnecting forever is the right behaviour for a node agent: a control plane
// restart or a network partition must not require touching every GPU node, and
// the engines keep serving traffic throughout because the router talks to them
// directly.
func (a *Agent) Run(ctx context.Context) error {
if err := a.scan(ctx); err != nil {
// Not fatal: a node whose GPUs cannot be read should still register, so
// an operator can see it and why it is useless.
a.log.Error("hardware discovery failed", "err", err)
}
attempt := 0
for ctx.Err() == nil {
err := a.session(ctx)
if ctx.Err() != nil {
break
}
attempt++
delay := a.reconnectDelay(attempt)
a.log.Warn("disconnected from the control plane, retrying",
"err", err, "in", delay.Round(time.Millisecond), "attempt", attempt)
select {
case <-ctx.Done():
case <-time.After(delay):
}
if err == nil {
attempt = 0
}
}
// Engines keep running when the agent stops on purpose: an agent restart
// must not interrupt served traffic. Orphan adoption picks them up again.
return ctx.Err()
}
func (a *Agent) reconnectDelay(attempt int) time.Duration {
d := a.cfg.ReconnectBase << min(attempt-1, 16)
if d > a.cfg.ReconnectMax || d <= 0 {
d = a.cfg.ReconnectMax
}
// Jitter, so a fleet does not reconnect in lockstep after a control plane
// restart and knock it over again.
return d/2 + time.Duration(rand.Int64N(int64(d/2))) //nolint:gosec // jitter
}
// scan refreshes the node's hardware inventory.
func (a *Agent) scan(ctx context.Context) error {
inv, err := discovery.Discover(ctx, a.cfg.Discovery, a.log)
a.mu.Lock()
a.inv = inv
a.mu.Unlock()
return err
}
// session runs one connection until it breaks.
func (a *Agent) session(ctx context.Context) error {
client := llmgwv1connect.NewAgentServiceClient(a.http, a.cfg.Server)
ctx, cancel := context.WithCancel(ctx)
defer cancel()
stream := client.Connect(ctx)
defer func() { _ = stream.CloseRequest() }()
a.mu.Lock()
inv := a.inv
a.mu.Unlock()
if err := stream.Send(&llmgwv1.AgentMessage{
Msg: &llmgwv1.AgentMessage_Register{
Register: agentsvc.RegisterFromInventory(
a.cfg.NodeID, a.cfg.Hostname, a.cfg.MgmtAddr, a.cfg.Version,
a.cfg.Labels, inv, a.cfg.Fabrics),
},
}); err != nil {
return fmt.Errorf("registering: %w", err)
}
// Sends are serialized through this channel: connect streams are not safe
// for concurrent writes, and heartbeats race with status reports.
out := make(chan *llmgwv1.AgentMessage, 32)
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
a.sendLoop(ctx, stream, out)
}()
defer wg.Wait()
for {
msg, err := stream.Receive()
if err != nil {
cancel()
if errors.Is(err, io.EOF) || ctx.Err() != nil {
return nil
}
return err
}
a.handle(ctx, msg, out)
}
}
type sender interface {
Send(*llmgwv1.AgentMessage) error
}
func (a *Agent) sendLoop(ctx context.Context, s sender, out <-chan *llmgwv1.AgentMessage) {
t := time.NewTicker(a.cfg.HeartbeatInterval)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case msg := <-out:
if err := s.Send(msg); err != nil {
return
}
case <-t.C:
if err := s.Send(a.heartbeat()); err != nil {
return
}
}
}
}
func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
a.mu.Lock()
inv := a.inv
a.mu.Unlock()
hb := &llmgwv1.Heartbeat{NodeId: a.cfg.NodeID}
for _, g := range inv.GPUs {
hb.GpuMemoryUsedMb = append(hb.GpuMemoryUsedMb, int32(g.MemoryUsedMB))
}
return &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Heartbeat{Heartbeat: hb}}
}
func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out chan<- *llmgwv1.AgentMessage) {
switch m := msg.Msg.(type) {
case *llmgwv1.ControlMessage_Start:
go a.startShard(ctx, m.Start, out)
case *llmgwv1.ControlMessage_Stop:
go a.stopShard(ctx, m.Stop.GetShardId(), out)
case *llmgwv1.ControlMessage_Drain:
if err := a.sup.Drain(m.Drain.GetShardId()); err != nil {
a.log.Warn("draining failed", "shard", m.Drain.GetShardId(), "err", err)
}
case *llmgwv1.ControlMessage_Ack:
if !m.Ack.GetOk() {
a.log.Warn("control plane rejected a message",
"id", m.Ack.GetMessageId(), "err", m.Ack.GetError())
}
}
}
// startShard launches a shard and reports its progress.
//
// It runs in its own goroutine because model loading takes minutes, and the
// stream has to keep carrying heartbeats and other commands meanwhile.
func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<- *llmgwv1.AgentMessage) {
shardID, replicaID := m.GetShardId(), m.GetReplicaId()
a.mu.Lock()
a.shardReplica[shardID] = replicaID
a.mu.Unlock()
report := func(st supervisor.State, errMsg string, tail []string) {
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
NodeId: a.cfg.NodeID, ReplicaId: replicaID, ShardId: shardID,
State: agentsvc.StateToWire(st), Error: errMsg, LastOutput: tail,
}}}
select {
case out <- msg:
case <-ctx.Done():
}
}
spec, timeout := agentsvc.ShardSpecFromStart(m)
eng, err := a.reg.Get(spec.Profile.Engine)
if err != nil {
report(supervisor.Failed, err.Error(), nil)
return
}
a.mu.Lock()
inv := a.inv
a.mu.Unlock()
// Validate before launching. An engine that refuses a profile does so after
// minutes of loading; catching it here costs nothing.
if err := eng.Validate(spec.Profile, gpuInfo(inv, spec.GPUIndices)); err != nil {
report(supervisor.Failed, err.Error(), nil)
return
}
cmd, err := eng.BuildCommand(spec)
if err != nil {
report(supervisor.Failed, err.Error(), nil)
return
}
sh, err := a.sup.Start(ctx, shardID, spec, cmd)
if err != nil {
if errors.Is(err, supervisor.ErrAlreadyRunning) {
// Commands are idempotent: the control plane retries, and a repeat
// must not produce a second process (plan §5.6).
report(sh.Status().State, "", nil)
return
}
report(supervisor.Failed, err.Error(), sh.Logs(10))
return
}
report(supervisor.Starting, "", nil)
base := fmt.Sprintf("http://127.0.0.1:%d", spec.HTTPPort)
if err := a.sup.WaitReady(ctx, sh, base, spec.Profile.ServedModelName, timeout); err != nil {
report(supervisor.Failed, err.Error(), sh.Logs(20))
return
}
report(supervisor.Ready, "", nil)
// Watch for the engine dying later. Nothing else would notice: the router
// would simply see failing requests.
go a.watch(ctx, sh, replicaID, shardID, out)
}
// watch reports a shard that exits after it was ready.
func (a *Agent) watch(ctx context.Context, sh *supervisor.Shard, replicaID, shardID string, out chan<- *llmgwv1.AgentMessage) {
t := time.NewTicker(time.Second)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
st := sh.Status()
if !st.State.Terminal() {
continue
}
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
NodeId: a.cfg.NodeID, ReplicaId: replicaID, ShardId: shardID,
State: agentsvc.StateToWire(st.State), Error: st.Err,
ExitCode: int32(st.ExitCode), LastOutput: sh.Logs(20),
}}}
select {
case out <- msg:
case <-ctx.Done():
}
return
}
}
}
func (a *Agent) stopShard(ctx context.Context, shardID string, out chan<- *llmgwv1.AgentMessage) {
a.mu.Lock()
replicaID := a.shardReplica[shardID]
delete(a.shardReplica, shardID)
a.mu.Unlock()
if err := a.sup.Stop(ctx, shardID); err != nil {
a.log.Warn("stopping a shard failed", "shard", shardID, "err", err)
}
a.sup.Remove(shardID)
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
NodeId: a.cfg.NodeID, ReplicaId: replicaID, ShardId: shardID,
State: agentsvc.StateToWire(supervisor.Stopped),
}}}
select {
case out <- msg:
case <-ctx.Done():
}
}
// gpuInfo narrows the inventory to the GPUs a shard will use.
func gpuInfo(inv discovery.Inventory, indices []int) []engines.GPUInfo {
want := map[int]struct{}{}
for _, i := range indices {
want[i] = struct{}{}
}
var out []engines.GPUInfo
for _, g := range inv.GPUs {
if _, ok := want[g.Index]; !ok {
continue
}
out = append(out, engines.GPUInfo{
Index: g.Index, Name: g.Name,
ComputeCapability: g.ComputeCapability, MemoryTotalMB: g.MemoryTotalMB,
})
}
return out
}
// HTTP2Client returns the client the agent should use to reach the control
// plane over plaintext HTTP/2.
//
// connect's bidirectional streams require HTTP/2, and an agent talking to a
// control plane over a trusted network has no TLS to negotiate it with. Go's
// default transport will not speak HTTP/2 without TLS, so it has to be asked
// explicitly.
//
// mTLS is plan §5.6's target and replaces this; until then the stream must not
// cross an untrusted network.
func HTTP2Client(timeout time.Duration) *http.Client {
return &http.Client{
Transport: &http2.Transport{
AllowHTTP: true,
DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
return (&net.Dialer{Timeout: timeout}).DialContext(ctx, network, addr)
},
},
}
}
+403
View File
@@ -0,0 +1,403 @@
package agent
import (
"context"
"crypto/tls"
"errors"
"log/slog"
"net"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"slices"
"testing"
"time"
"golang.org/x/net/http2"
"golang.org/x/net/http2/h2c"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
)
// statusRecorder captures what the control plane hears about shards.
type statusRecorder struct{ ch chan status }
type status struct {
replicaID, shardID string
state supervisor.State
err string
}
func newRecorder() *statusRecorder { return &statusRecorder{ch: make(chan status, 128)} }
func (r *statusRecorder) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) {
select {
case r.ch <- status{replicaID, shardID, st, errMsg}:
default:
}
}
func (r *statusRecorder) await(t *testing.T, limit time.Duration, pred func(status) bool) status {
t.Helper()
deadline := time.After(limit)
var seen []status
for {
select {
case s := <-r.ch:
seen = append(seen, s)
if pred(s) {
return s
}
case <-deadline:
t.Fatalf("no matching status within %v; saw %+v", limit, seen)
return status{}
}
}
}
// fakeEngineScript writes a script that stands in for vLLM.
//
// It parses the real --port flag the vLLM adapter generates rather than being
// told out of band, so the adapter's argument construction is exercised too.
func fakeEngineScript(t *testing.T, body string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "fake-vllm")
script := `#!/bin/sh
port=""
while [ $# -gt 0 ]; do
case "$1" in
--port) port="$2"; shift 2 ;;
*) shift ;;
esac
done
` + body + "\n"
if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
t.Fatal(err)
}
return path
}
// servingEngine answers /health and /v1/models, which is what readiness needs.
func servingEngine(t *testing.T) string {
t.Helper()
py, err := exec.LookPath("python3")
if err != nil {
t.Skip("python3 is required for the fake engine")
}
return fakeEngineScript(t, `exec `+py+` -c '
import http.server, json, sys
port = int(sys.argv[1])
class H(http.server.BaseHTTPRequestHandler):
def do_GET(self):
if self.path == "/health":
self.send_response(200); self.end_headers(); return
if self.path == "/v1/models":
b = json.dumps({"object":"list","data":[{"id":"test-model"}]}).encode()
self.send_response(200)
self.send_header("content-type","application/json")
self.send_header("content-length", str(len(b)))
self.end_headers(); self.wfile.write(b); return
self.send_response(404); self.end_headers()
def log_message(self, *a): pass
http.server.HTTPServer(("127.0.0.1", port), H).serve_forever()
' "$port"`)
}
type harness struct {
svc *agentsvc.Service
inv *inventory.Store
rec *statusRecorder
srv *httptest.Server
ag *Agent
}
// start wires a real agent to a real service over h2c, so this exercises the
// protocol rather than two structs calling each other.
func start(t *testing.T, cfgFns ...func(*Config)) *harness {
t.Helper()
inv := inventory.NewStore()
rec := newRecorder()
svc := agentsvc.New(inv, rec, slog.New(slog.DiscardHandler))
path, handler := svc.Handler()
mux := http.NewServeMux()
mux.Handle(path, handler)
// connect's bidirectional streams need HTTP/2; h2c provides it without TLS.
srv := httptest.NewServer(h2c.NewHandler(mux, &http2.Server{}))
cfg := DefaultConfig()
cfg.NodeID = "n1"
cfg.Server = srv.URL
cfg.MgmtAddr = "127.0.0.1"
cfg.HeartbeatInterval = 50 * time.Millisecond
cfg.ReconnectBase = 20 * time.Millisecond
cfg.ReconnectMax = 100 * time.Millisecond
cfg.Discovery.SkipGPUs = true
cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent")
cfg.Superv = supervisor.Config{
WorkDir: t.TempDir(), LogLines: 64,
StopTimeout: 2 * time.Second, HealthInterval: 20 * time.Millisecond,
}
for _, f := range cfgFns {
f(&cfg)
}
client := &http.Client{Transport: &http2.Transport{
AllowHTTP: true,
DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, network, addr)
},
}}
ag, err := New(cfg, client, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
go func() { _ = ag.Run(ctx) }()
h := &harness{svc: svc, inv: inv, rec: rec, srv: srv, ag: ag}
t.Cleanup(func() {
cancel()
ag.Supervisor().StopAll(context.Background())
srv.Close()
})
h.waitRegistered(t)
return h
}
func (h *harness) waitRegistered(t *testing.T) {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
for time.Now().Before(deadline) {
if _, ok := h.inv.Get("n1"); ok && len(h.svc.ConnectedNodes()) == 1 {
return
}
time.Sleep(10 * time.Millisecond)
}
t.Fatal("the agent never registered")
}
// shardCommand builds a single-node vLLM shard on a free port.
func shardCommand(t *testing.T, shardID string) reconciler.ShardCommand {
t.Helper()
return reconciler.ShardCommand{
ReplicaID: "rep-1", ShardID: shardID, NodeID: "n1",
Spec: engines.ShardSpec{
ReplicaID: "rep-1", ShardID: shardID, Rank: 0, NNodes: 1,
GPUIndices: []int{0}, HTTPPort: freePort(t),
Profile: engines.Profile{
Name: "p", Engine: "vllm", TP: 1,
ModelPath: "/models/x", ServedModelName: "test-model",
},
},
StartupTimeout: 15 * time.Second,
}
}
func freePort(t *testing.T) int {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer ln.Close()
return ln.Addr().(*net.TCPAddr).Port
}
func TestRegistrationCarriesInventory(t *testing.T) {
h := start(t, func(c *Config) {
c.Hostname = "gpu-a-01"
c.Labels = map[string]string{"rack": "r12"}
c.Fabrics = []inventory.FabricAttachment{
{Fabric: "ib-a", HCAs: []string{"mlx5_0", "mlx5_1"}, RateGbps: 200},
}
})
n, ok := h.inv.Get("n1")
if !ok {
t.Fatal("node missing from inventory")
}
if n.Hostname != "gpu-a-01" || n.MgmtAddr != "127.0.0.1" {
t.Errorf("identity wrong: %+v", n)
}
if n.Labels["rack"] != "r12" {
t.Errorf("labels lost: %v", n.Labels)
}
// The fabric label cannot be inferred from the wire, so losing it in
// transit would silently disable multi-node placement.
if !n.HasFabric("ib-a") {
t.Fatalf("fabric attachment lost: %+v", n.Fabrics)
}
if got := n.HCAsFor("ib-a"); !slices.Equal(got, []string{"mlx5_0", "mlx5_1"}) {
t.Errorf("HCAs=%v", got)
}
}
func TestHeartbeatKeepsNodeAlive(t *testing.T) {
h := start(t)
n, _ := h.inv.Get("n1")
first := n.LastSeen
time.Sleep(200 * time.Millisecond)
n, _ = h.inv.Get("n1")
if !n.LastSeen.After(first) {
t.Error("heartbeats are not refreshing LastSeen; the node would be declared lost")
}
}
func TestStartAndStopShardEndToEnd(t *testing.T) {
h := start(t, func(c *Config) {
c.Engines.VLLM.Bin = servingEngine(t)
})
cmd := shardCommand(t, "shard-a")
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
t.Fatal(err)
}
got := h.rec.await(t, 30*time.Second, func(s status) bool {
return s.shardID == "shard-a" && s.state == supervisor.Ready
})
if got.replicaID != "rep-1" {
t.Errorf("replica id lost in transit: %q", got.replicaID)
}
if err := h.svc.StopShard(t.Context(), "n1", "shard-a"); err != nil {
t.Fatal(err)
}
h.rec.await(t, 15*time.Second, func(s status) bool {
return s.shardID == "shard-a" && s.state == supervisor.Stopped
})
}
func TestFailedShardCarriesDiagnosis(t *testing.T) {
// When a model fails to load the engine's output is the only diagnosis, so
// it has to survive the trip to the control plane.
h := start(t, func(c *Config) {
c.Engines.VLLM.Bin = fakeEngineScript(t, `echo 'CUDA out of memory' >&2; exit 1`)
})
if err := h.svc.StartShard(t.Context(), shardCommand(t, "shard-bad")); err != nil {
t.Fatal(err)
}
got := h.rec.await(t, 30*time.Second, func(s status) bool {
return s.shardID == "shard-bad" && s.state == supervisor.Failed
})
if got.err == "" {
t.Error("the failure carried no error message")
}
}
func TestEngineThatNeverBecomesReady(t *testing.T) {
// A process that runs but never serves must time out rather than hang the
// replica forever.
h := start(t, func(c *Config) {
c.Engines.VLLM.Bin = fakeEngineScript(t, `sleep 60`)
})
cmd := shardCommand(t, "shard-stuck")
cmd.StartupTimeout = 500 * time.Millisecond
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
t.Fatal(err)
}
h.rec.await(t, 20*time.Second, func(s status) bool {
return s.shardID == "shard-stuck" && s.state == supervisor.Failed
})
}
func TestValidationRejectsBeforeLaunch(t *testing.T) {
// An engine that refuses a profile does so after minutes of loading;
// catching it at dispatch costs nothing.
h := start(t, func(c *Config) {
c.Engines.VLLM.Bin = servingEngine(t)
})
cmd := shardCommand(t, "shard-invalid")
cmd.Spec.Profile.ModelPath = "" // validation requires one
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
t.Fatal(err)
}
got := h.rec.await(t, 15*time.Second, func(s status) bool {
return s.shardID == "shard-invalid" && s.state == supervisor.Failed
})
if got.err == "" {
t.Error("validation failure carried no reason")
}
}
func TestDispatchToUnconnectedNode(t *testing.T) {
h := start(t)
err := h.svc.StartShard(t.Context(), reconciler.ShardCommand{NodeID: "nowhere", ShardID: "x"})
if err == nil {
t.Fatal("dispatching to an unconnected node succeeded")
}
if !errors.Is(err, agentsvc.ErrNotConnected) {
t.Errorf("err=%v, expected ErrNotConnected", err)
}
}
func TestAgentReconnects(t *testing.T) {
// A control plane restart must not require touching every GPU node.
h := start(t)
h.srv.CloseClientConnections()
deadline := time.Now().Add(15 * time.Second)
for time.Now().Before(deadline) {
if len(h.svc.ConnectedNodes()) == 1 {
if err := h.svc.DrainShard(t.Context(), "n1", "nothing"); err != nil {
t.Fatalf("the reconnected stream does not accept commands: %v", err)
}
return
}
time.Sleep(20 * time.Millisecond)
}
t.Fatal("the agent never reconnected")
}
func TestIdempotentStart(t *testing.T) {
// The control plane retries commands; a repeat must not start a second
// process (plan §5.6).
h := start(t, func(c *Config) {
c.Engines.VLLM.Bin = fakeEngineScript(t, `sleep 30`)
})
cmd := shardCommand(t, "shard-idem")
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
t.Fatal(err)
}
h.rec.await(t, 15*time.Second, func(s status) bool {
return s.shardID == "shard-idem" && s.state == supervisor.Starting
})
sup := h.ag.Supervisor()
sh, err := sup.Get("shard-idem")
if err != nil {
t.Fatal(err)
}
pid := sh.Status().PID
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
t.Fatal(err)
}
time.Sleep(400 * time.Millisecond)
sh2, err := sup.Get("shard-idem")
if err != nil {
t.Fatal(err)
}
if sh2.Status().PID != pid {
t.Errorf("a second process was started: pid %d then %d", pid, sh2.Status().PID)
}
}
+6 -4
View File
@@ -16,14 +16,16 @@ var ErrNoNVML = errors.New("built without the nvml tag; rebuild with -tags nvml
// Options configure a full inventory scan.
type Options struct {
// NvidiaSMI is the path to nvidia-smi; empty means look it up on PATH.
NvidiaSMI string
// Worth configuring on nodes where the driver lives outside the default
// paths, which containerized driver installs often do.
NvidiaSMI string `koanf:"nvidia_smi"`
// IBRoot is the sysfs InfiniBand directory.
IBRoot string
IBRoot string `koanf:"ib_root"`
// PreferNVML uses the NVML backend when this binary has one. It is the only
// source of NVLink topology.
PreferNVML bool
PreferNVML bool `koanf:"prefer_nvml"`
// SkipGPUs omits GPU discovery, for nodes that have none.
SkipGPUs bool
SkipGPUs bool `koanf:"skip_gpus"`
}
// Discover builds the node's inventory.