One bidirectional connect-go stream per node, carrying registration, heartbeats and shard status up, and start, stop and drain down. The agent dials out: GPU nodes sit behind firewalls far more often than a control plane does, and an outbound connection needs no inbound rule on every node in the fleet. - registration carries the whole hardware inventory, so there is no separate discovery call and a reconnecting agent refreshes what the scheduler knows. The fabric labels travel with it; they cannot be inferred from the wire, so losing them in transit would silently disable multi-node placement. - agentsvc is both the stream handler and the reconciler's dispatcher, because it holds the open streams and is therefore the only thing that can deliver a command to a node. - the agent validates a profile before launching anything, since an engine that refuses a profile does so after minutes of loading, then supervises the process and reports every transition with the engine's output tail on failure. - reconnects forever with jittered backoff. A control plane restart must not require touching every GPU node, and a fleet must not reconnect in lockstep and knock it over again. Engines keep serving throughout, because the router talks to them directly. llmgw agent is real now, and llmgw server runs a control plane whenever any model declares a profile. The static endpoint list stays for router-only runs. Verified with real binaries as separate processes: the agent registers, a replica is placed, an engine launches through the vLLM adapter's actual argument construction, streaming traffic is served, and killing the agent moves the node through unreachable to lost with placement explaining why nothing can run. config.Model and catalog.Model were the same thing described twice, so the router now reads catalog entries directly and the validation rules live in one place. Still missing: mTLS. The stream is plaintext HTTP/2 and must stay on a trusted network until that lands. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
c1b9ea1bc2
commit
2373726efd
24 files changed
+3645
-87
No files matched your search
@@ -0,0 +1,458 @@
|
||||
// Package agent is the node side of the control plane stream (plan §5.6).
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math/rand/v2"
|
||||
"net"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/http2"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1/llmgwv1connect"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
// Config configures the agent.
|
||||
type Config struct {
|
||||
NodeID string `koanf:"node_id"`
|
||||
Server string `koanf:"server"`
|
||||
Hostname string `koanf:"hostname"`
|
||||
MgmtAddr string `koanf:"mgmt_addr"`
|
||||
Labels map[string]string `koanf:"labels"`
|
||||
|
||||
// MgmtInterface is the ethernet interface NCCL bootstraps over.
|
||||
MgmtInterface string `koanf:"mgmt_interface"`
|
||||
// Fabrics labels this node's HCAs. It is explicit on purpose: OpenSM's
|
||||
// default subnet prefix is the same on every fabric, so nothing can be
|
||||
// inferred from the wire (plan §5.2).
|
||||
Fabrics []inventory.FabricAttachment `koanf:"fabrics"`
|
||||
|
||||
HeartbeatInterval time.Duration `koanf:"heartbeat_interval"`
|
||||
// ReconnectBase is the first reconnect delay; it backs off from there.
|
||||
ReconnectBase time.Duration `koanf:"reconnect_base"`
|
||||
ReconnectMax time.Duration `koanf:"reconnect_max"`
|
||||
|
||||
Discovery discovery.Options `koanf:"discovery"`
|
||||
Engines engines.Config `koanf:"engines"`
|
||||
Superv supervisor.Config `koanf:"supervisor"`
|
||||
|
||||
Version string
|
||||
}
|
||||
|
||||
// DefaultConfig returns sensible defaults.
|
||||
func DefaultConfig() Config {
|
||||
return Config{
|
||||
MgmtInterface: "eth0",
|
||||
HeartbeatInterval: 5 * time.Second,
|
||||
ReconnectBase: time.Second,
|
||||
ReconnectMax: 30 * time.Second,
|
||||
Superv: supervisor.DefaultConfig(),
|
||||
// NVML gives NVLink topology, so prefer it when this build has it.
|
||||
Discovery: discovery.Options{PreferNVML: true},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Config) normalize() error {
|
||||
d := DefaultConfig()
|
||||
if c.NodeID == "" {
|
||||
return errors.New("node_id is required")
|
||||
}
|
||||
if c.Server == "" {
|
||||
return errors.New("server is required")
|
||||
}
|
||||
if c.MgmtInterface == "" {
|
||||
c.MgmtInterface = d.MgmtInterface
|
||||
}
|
||||
if c.HeartbeatInterval <= 0 {
|
||||
c.HeartbeatInterval = d.HeartbeatInterval
|
||||
}
|
||||
if c.ReconnectBase <= 0 {
|
||||
c.ReconnectBase = d.ReconnectBase
|
||||
}
|
||||
if c.ReconnectMax < c.ReconnectBase {
|
||||
c.ReconnectMax = max(c.ReconnectBase, d.ReconnectMax)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Agent runs on a GPU node.
|
||||
type Agent struct {
|
||||
cfg Config
|
||||
log *slog.Logger
|
||||
sup *supervisor.Supervisor
|
||||
reg *engines.Registry
|
||||
http connectHTTPClient
|
||||
|
||||
mu sync.Mutex
|
||||
inv discovery.Inventory
|
||||
// shardReplica remembers which replica a shard belongs to, so a status
|
||||
// report can name both. The control plane keys on the replica.
|
||||
shardReplica map[string]string
|
||||
}
|
||||
|
||||
// connectHTTPClient is the HTTP client connect uses; an interface so tests can
|
||||
// hand in an httptest server's client.
|
||||
type connectHTTPClient interface {
|
||||
Do(*http.Request) (*http.Response, error)
|
||||
}
|
||||
|
||||
// New builds an agent.
|
||||
func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, error) {
|
||||
if err := cfg.normalize(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if log == nil {
|
||||
log = slog.Default()
|
||||
}
|
||||
if httpClient == nil {
|
||||
httpClient = http.DefaultClient
|
||||
}
|
||||
|
||||
health := supervisor.NewHTTPHealth(3 * time.Second)
|
||||
sup, err := supervisor.New(cfg.Superv, health, log)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Agent{
|
||||
cfg: cfg, log: log, sup: sup,
|
||||
reg: engines.NewRegistry(cfg.Engines),
|
||||
http: httpClient,
|
||||
shardReplica: map[string]string{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Supervisor exposes the process supervisor, for tests and `ctl logs`.
|
||||
func (a *Agent) Supervisor() *supervisor.Supervisor { return a.sup }
|
||||
|
||||
// Run connects to the control plane and stays connected until ctx ends.
|
||||
//
|
||||
// Reconnecting forever is the right behaviour for a node agent: a control plane
|
||||
// restart or a network partition must not require touching every GPU node, and
|
||||
// the engines keep serving traffic throughout because the router talks to them
|
||||
// directly.
|
||||
func (a *Agent) Run(ctx context.Context) error {
|
||||
if err := a.scan(ctx); err != nil {
|
||||
// Not fatal: a node whose GPUs cannot be read should still register, so
|
||||
// an operator can see it and why it is useless.
|
||||
a.log.Error("hardware discovery failed", "err", err)
|
||||
}
|
||||
|
||||
attempt := 0
|
||||
for ctx.Err() == nil {
|
||||
err := a.session(ctx)
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
attempt++
|
||||
delay := a.reconnectDelay(attempt)
|
||||
a.log.Warn("disconnected from the control plane, retrying",
|
||||
"err", err, "in", delay.Round(time.Millisecond), "attempt", attempt)
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(delay):
|
||||
}
|
||||
if err == nil {
|
||||
attempt = 0
|
||||
}
|
||||
}
|
||||
|
||||
// Engines keep running when the agent stops on purpose: an agent restart
|
||||
// must not interrupt served traffic. Orphan adoption picks them up again.
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func (a *Agent) reconnectDelay(attempt int) time.Duration {
|
||||
d := a.cfg.ReconnectBase << min(attempt-1, 16)
|
||||
if d > a.cfg.ReconnectMax || d <= 0 {
|
||||
d = a.cfg.ReconnectMax
|
||||
}
|
||||
// Jitter, so a fleet does not reconnect in lockstep after a control plane
|
||||
// restart and knock it over again.
|
||||
return d/2 + time.Duration(rand.Int64N(int64(d/2))) //nolint:gosec // jitter
|
||||
}
|
||||
|
||||
// scan refreshes the node's hardware inventory.
|
||||
func (a *Agent) scan(ctx context.Context) error {
|
||||
inv, err := discovery.Discover(ctx, a.cfg.Discovery, a.log)
|
||||
a.mu.Lock()
|
||||
a.inv = inv
|
||||
a.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
|
||||
// session runs one connection until it breaks.
|
||||
func (a *Agent) session(ctx context.Context) error {
|
||||
client := llmgwv1connect.NewAgentServiceClient(a.http, a.cfg.Server)
|
||||
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
stream := client.Connect(ctx)
|
||||
defer func() { _ = stream.CloseRequest() }()
|
||||
|
||||
a.mu.Lock()
|
||||
inv := a.inv
|
||||
a.mu.Unlock()
|
||||
|
||||
if err := stream.Send(&llmgwv1.AgentMessage{
|
||||
Msg: &llmgwv1.AgentMessage_Register{
|
||||
Register: agentsvc.RegisterFromInventory(
|
||||
a.cfg.NodeID, a.cfg.Hostname, a.cfg.MgmtAddr, a.cfg.Version,
|
||||
a.cfg.Labels, inv, a.cfg.Fabrics),
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("registering: %w", err)
|
||||
}
|
||||
|
||||
// Sends are serialized through this channel: connect streams are not safe
|
||||
// for concurrent writes, and heartbeats race with status reports.
|
||||
out := make(chan *llmgwv1.AgentMessage, 32)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
a.sendLoop(ctx, stream, out)
|
||||
}()
|
||||
defer wg.Wait()
|
||||
|
||||
for {
|
||||
msg, err := stream.Receive()
|
||||
if err != nil {
|
||||
cancel()
|
||||
if errors.Is(err, io.EOF) || ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
a.handle(ctx, msg, out)
|
||||
}
|
||||
}
|
||||
|
||||
type sender interface {
|
||||
Send(*llmgwv1.AgentMessage) error
|
||||
}
|
||||
|
||||
func (a *Agent) sendLoop(ctx context.Context, s sender, out <-chan *llmgwv1.AgentMessage) {
|
||||
t := time.NewTicker(a.cfg.HeartbeatInterval)
|
||||
defer t.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case msg := <-out:
|
||||
if err := s.Send(msg); err != nil {
|
||||
return
|
||||
}
|
||||
case <-t.C:
|
||||
if err := s.Send(a.heartbeat()); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
|
||||
a.mu.Lock()
|
||||
inv := a.inv
|
||||
a.mu.Unlock()
|
||||
|
||||
hb := &llmgwv1.Heartbeat{NodeId: a.cfg.NodeID}
|
||||
for _, g := range inv.GPUs {
|
||||
hb.GpuMemoryUsedMb = append(hb.GpuMemoryUsedMb, int32(g.MemoryUsedMB))
|
||||
}
|
||||
return &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Heartbeat{Heartbeat: hb}}
|
||||
}
|
||||
|
||||
func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out chan<- *llmgwv1.AgentMessage) {
|
||||
switch m := msg.Msg.(type) {
|
||||
case *llmgwv1.ControlMessage_Start:
|
||||
go a.startShard(ctx, m.Start, out)
|
||||
case *llmgwv1.ControlMessage_Stop:
|
||||
go a.stopShard(ctx, m.Stop.GetShardId(), out)
|
||||
case *llmgwv1.ControlMessage_Drain:
|
||||
if err := a.sup.Drain(m.Drain.GetShardId()); err != nil {
|
||||
a.log.Warn("draining failed", "shard", m.Drain.GetShardId(), "err", err)
|
||||
}
|
||||
case *llmgwv1.ControlMessage_Ack:
|
||||
if !m.Ack.GetOk() {
|
||||
a.log.Warn("control plane rejected a message",
|
||||
"id", m.Ack.GetMessageId(), "err", m.Ack.GetError())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// startShard launches a shard and reports its progress.
|
||||
//
|
||||
// It runs in its own goroutine because model loading takes minutes, and the
|
||||
// stream has to keep carrying heartbeats and other commands meanwhile.
|
||||
func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<- *llmgwv1.AgentMessage) {
|
||||
shardID, replicaID := m.GetShardId(), m.GetReplicaId()
|
||||
|
||||
a.mu.Lock()
|
||||
a.shardReplica[shardID] = replicaID
|
||||
a.mu.Unlock()
|
||||
|
||||
report := func(st supervisor.State, errMsg string, tail []string) {
|
||||
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
|
||||
NodeId: a.cfg.NodeID, ReplicaId: replicaID, ShardId: shardID,
|
||||
State: agentsvc.StateToWire(st), Error: errMsg, LastOutput: tail,
|
||||
}}}
|
||||
select {
|
||||
case out <- msg:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}
|
||||
|
||||
spec, timeout := agentsvc.ShardSpecFromStart(m)
|
||||
|
||||
eng, err := a.reg.Get(spec.Profile.Engine)
|
||||
if err != nil {
|
||||
report(supervisor.Failed, err.Error(), nil)
|
||||
return
|
||||
}
|
||||
|
||||
a.mu.Lock()
|
||||
inv := a.inv
|
||||
a.mu.Unlock()
|
||||
|
||||
// Validate before launching. An engine that refuses a profile does so after
|
||||
// minutes of loading; catching it here costs nothing.
|
||||
if err := eng.Validate(spec.Profile, gpuInfo(inv, spec.GPUIndices)); err != nil {
|
||||
report(supervisor.Failed, err.Error(), nil)
|
||||
return
|
||||
}
|
||||
|
||||
cmd, err := eng.BuildCommand(spec)
|
||||
if err != nil {
|
||||
report(supervisor.Failed, err.Error(), nil)
|
||||
return
|
||||
}
|
||||
|
||||
sh, err := a.sup.Start(ctx, shardID, spec, cmd)
|
||||
if err != nil {
|
||||
if errors.Is(err, supervisor.ErrAlreadyRunning) {
|
||||
// Commands are idempotent: the control plane retries, and a repeat
|
||||
// must not produce a second process (plan §5.6).
|
||||
report(sh.Status().State, "", nil)
|
||||
return
|
||||
}
|
||||
report(supervisor.Failed, err.Error(), sh.Logs(10))
|
||||
return
|
||||
}
|
||||
report(supervisor.Starting, "", nil)
|
||||
|
||||
base := fmt.Sprintf("http://127.0.0.1:%d", spec.HTTPPort)
|
||||
if err := a.sup.WaitReady(ctx, sh, base, spec.Profile.ServedModelName, timeout); err != nil {
|
||||
report(supervisor.Failed, err.Error(), sh.Logs(20))
|
||||
return
|
||||
}
|
||||
report(supervisor.Ready, "", nil)
|
||||
|
||||
// Watch for the engine dying later. Nothing else would notice: the router
|
||||
// would simply see failing requests.
|
||||
go a.watch(ctx, sh, replicaID, shardID, out)
|
||||
}
|
||||
|
||||
// watch reports a shard that exits after it was ready.
|
||||
func (a *Agent) watch(ctx context.Context, sh *supervisor.Shard, replicaID, shardID string, out chan<- *llmgwv1.AgentMessage) {
|
||||
t := time.NewTicker(time.Second)
|
||||
defer t.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
st := sh.Status()
|
||||
if !st.State.Terminal() {
|
||||
continue
|
||||
}
|
||||
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
|
||||
NodeId: a.cfg.NodeID, ReplicaId: replicaID, ShardId: shardID,
|
||||
State: agentsvc.StateToWire(st.State), Error: st.Err,
|
||||
ExitCode: int32(st.ExitCode), LastOutput: sh.Logs(20),
|
||||
}}}
|
||||
select {
|
||||
case out <- msg:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Agent) stopShard(ctx context.Context, shardID string, out chan<- *llmgwv1.AgentMessage) {
|
||||
a.mu.Lock()
|
||||
replicaID := a.shardReplica[shardID]
|
||||
delete(a.shardReplica, shardID)
|
||||
a.mu.Unlock()
|
||||
|
||||
if err := a.sup.Stop(ctx, shardID); err != nil {
|
||||
a.log.Warn("stopping a shard failed", "shard", shardID, "err", err)
|
||||
}
|
||||
a.sup.Remove(shardID)
|
||||
|
||||
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
|
||||
NodeId: a.cfg.NodeID, ReplicaId: replicaID, ShardId: shardID,
|
||||
State: agentsvc.StateToWire(supervisor.Stopped),
|
||||
}}}
|
||||
select {
|
||||
case out <- msg:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}
|
||||
|
||||
// gpuInfo narrows the inventory to the GPUs a shard will use.
|
||||
func gpuInfo(inv discovery.Inventory, indices []int) []engines.GPUInfo {
|
||||
want := map[int]struct{}{}
|
||||
for _, i := range indices {
|
||||
want[i] = struct{}{}
|
||||
}
|
||||
var out []engines.GPUInfo
|
||||
for _, g := range inv.GPUs {
|
||||
if _, ok := want[g.Index]; !ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, engines.GPUInfo{
|
||||
Index: g.Index, Name: g.Name,
|
||||
ComputeCapability: g.ComputeCapability, MemoryTotalMB: g.MemoryTotalMB,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// HTTP2Client returns the client the agent should use to reach the control
|
||||
// plane over plaintext HTTP/2.
|
||||
//
|
||||
// connect's bidirectional streams require HTTP/2, and an agent talking to a
|
||||
// control plane over a trusted network has no TLS to negotiate it with. Go's
|
||||
// default transport will not speak HTTP/2 without TLS, so it has to be asked
|
||||
// explicitly.
|
||||
//
|
||||
// mTLS is plan §5.6's target and replaces this; until then the stream must not
|
||||
// cross an untrusted network.
|
||||
func HTTP2Client(timeout time.Duration) *http.Client {
|
||||
return &http.Client{
|
||||
Transport: &http2.Transport{
|
||||
AllowHTTP: true,
|
||||
DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
|
||||
return (&net.Dialer{Timeout: timeout}).DialContext(ctx, network, addr)
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,403 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/http2"
|
||||
"golang.org/x/net/http2/h2c"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
)
|
||||
|
||||
// statusRecorder captures what the control plane hears about shards.
|
||||
type statusRecorder struct{ ch chan status }
|
||||
|
||||
type status struct {
|
||||
replicaID, shardID string
|
||||
state supervisor.State
|
||||
err string
|
||||
}
|
||||
|
||||
func newRecorder() *statusRecorder { return &statusRecorder{ch: make(chan status, 128)} }
|
||||
|
||||
func (r *statusRecorder) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) {
|
||||
select {
|
||||
case r.ch <- status{replicaID, shardID, st, errMsg}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (r *statusRecorder) await(t *testing.T, limit time.Duration, pred func(status) bool) status {
|
||||
t.Helper()
|
||||
deadline := time.After(limit)
|
||||
var seen []status
|
||||
for {
|
||||
select {
|
||||
case s := <-r.ch:
|
||||
seen = append(seen, s)
|
||||
if pred(s) {
|
||||
return s
|
||||
}
|
||||
case <-deadline:
|
||||
t.Fatalf("no matching status within %v; saw %+v", limit, seen)
|
||||
return status{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// fakeEngineScript writes a script that stands in for vLLM.
|
||||
//
|
||||
// It parses the real --port flag the vLLM adapter generates rather than being
|
||||
// told out of band, so the adapter's argument construction is exercised too.
|
||||
func fakeEngineScript(t *testing.T, body string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "fake-vllm")
|
||||
script := `#!/bin/sh
|
||||
port=""
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--port) port="$2"; shift 2 ;;
|
||||
*) shift ;;
|
||||
esac
|
||||
done
|
||||
` + body + "\n"
|
||||
if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
// servingEngine answers /health and /v1/models, which is what readiness needs.
|
||||
func servingEngine(t *testing.T) string {
|
||||
t.Helper()
|
||||
py, err := exec.LookPath("python3")
|
||||
if err != nil {
|
||||
t.Skip("python3 is required for the fake engine")
|
||||
}
|
||||
return fakeEngineScript(t, `exec `+py+` -c '
|
||||
import http.server, json, sys
|
||||
port = int(sys.argv[1])
|
||||
class H(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
if self.path == "/health":
|
||||
self.send_response(200); self.end_headers(); return
|
||||
if self.path == "/v1/models":
|
||||
b = json.dumps({"object":"list","data":[{"id":"test-model"}]}).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("content-type","application/json")
|
||||
self.send_header("content-length", str(len(b)))
|
||||
self.end_headers(); self.wfile.write(b); return
|
||||
self.send_response(404); self.end_headers()
|
||||
def log_message(self, *a): pass
|
||||
http.server.HTTPServer(("127.0.0.1", port), H).serve_forever()
|
||||
' "$port"`)
|
||||
}
|
||||
|
||||
type harness struct {
|
||||
svc *agentsvc.Service
|
||||
inv *inventory.Store
|
||||
rec *statusRecorder
|
||||
srv *httptest.Server
|
||||
ag *Agent
|
||||
}
|
||||
|
||||
// start wires a real agent to a real service over h2c, so this exercises the
|
||||
// protocol rather than two structs calling each other.
|
||||
func start(t *testing.T, cfgFns ...func(*Config)) *harness {
|
||||
t.Helper()
|
||||
|
||||
inv := inventory.NewStore()
|
||||
rec := newRecorder()
|
||||
svc := agentsvc.New(inv, rec, slog.New(slog.DiscardHandler))
|
||||
|
||||
path, handler := svc.Handler()
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle(path, handler)
|
||||
// connect's bidirectional streams need HTTP/2; h2c provides it without TLS.
|
||||
srv := httptest.NewServer(h2c.NewHandler(mux, &http2.Server{}))
|
||||
|
||||
cfg := DefaultConfig()
|
||||
cfg.NodeID = "n1"
|
||||
cfg.Server = srv.URL
|
||||
cfg.MgmtAddr = "127.0.0.1"
|
||||
cfg.HeartbeatInterval = 50 * time.Millisecond
|
||||
cfg.ReconnectBase = 20 * time.Millisecond
|
||||
cfg.ReconnectMax = 100 * time.Millisecond
|
||||
cfg.Discovery.SkipGPUs = true
|
||||
cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent")
|
||||
cfg.Superv = supervisor.Config{
|
||||
WorkDir: t.TempDir(), LogLines: 64,
|
||||
StopTimeout: 2 * time.Second, HealthInterval: 20 * time.Millisecond,
|
||||
}
|
||||
for _, f := range cfgFns {
|
||||
f(&cfg)
|
||||
}
|
||||
|
||||
client := &http.Client{Transport: &http2.Transport{
|
||||
AllowHTTP: true,
|
||||
DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, network, addr)
|
||||
},
|
||||
}}
|
||||
|
||||
ag, err := New(cfg, client, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go func() { _ = ag.Run(ctx) }()
|
||||
|
||||
h := &harness{svc: svc, inv: inv, rec: rec, srv: srv, ag: ag}
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
ag.Supervisor().StopAll(context.Background())
|
||||
srv.Close()
|
||||
})
|
||||
h.waitRegistered(t)
|
||||
return h
|
||||
}
|
||||
|
||||
func (h *harness) waitRegistered(t *testing.T) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, ok := h.inv.Get("n1"); ok && len(h.svc.ConnectedNodes()) == 1 {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("the agent never registered")
|
||||
}
|
||||
|
||||
// shardCommand builds a single-node vLLM shard on a free port.
|
||||
func shardCommand(t *testing.T, shardID string) reconciler.ShardCommand {
|
||||
t.Helper()
|
||||
return reconciler.ShardCommand{
|
||||
ReplicaID: "rep-1", ShardID: shardID, NodeID: "n1",
|
||||
Spec: engines.ShardSpec{
|
||||
ReplicaID: "rep-1", ShardID: shardID, Rank: 0, NNodes: 1,
|
||||
GPUIndices: []int{0}, HTTPPort: freePort(t),
|
||||
Profile: engines.Profile{
|
||||
Name: "p", Engine: "vllm", TP: 1,
|
||||
ModelPath: "/models/x", ServedModelName: "test-model",
|
||||
},
|
||||
},
|
||||
StartupTimeout: 15 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
func freePort(t *testing.T) int {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ln.Close()
|
||||
return ln.Addr().(*net.TCPAddr).Port
|
||||
}
|
||||
|
||||
func TestRegistrationCarriesInventory(t *testing.T) {
|
||||
h := start(t, func(c *Config) {
|
||||
c.Hostname = "gpu-a-01"
|
||||
c.Labels = map[string]string{"rack": "r12"}
|
||||
c.Fabrics = []inventory.FabricAttachment{
|
||||
{Fabric: "ib-a", HCAs: []string{"mlx5_0", "mlx5_1"}, RateGbps: 200},
|
||||
}
|
||||
})
|
||||
|
||||
n, ok := h.inv.Get("n1")
|
||||
if !ok {
|
||||
t.Fatal("node missing from inventory")
|
||||
}
|
||||
if n.Hostname != "gpu-a-01" || n.MgmtAddr != "127.0.0.1" {
|
||||
t.Errorf("identity wrong: %+v", n)
|
||||
}
|
||||
if n.Labels["rack"] != "r12" {
|
||||
t.Errorf("labels lost: %v", n.Labels)
|
||||
}
|
||||
// The fabric label cannot be inferred from the wire, so losing it in
|
||||
// transit would silently disable multi-node placement.
|
||||
if !n.HasFabric("ib-a") {
|
||||
t.Fatalf("fabric attachment lost: %+v", n.Fabrics)
|
||||
}
|
||||
if got := n.HCAsFor("ib-a"); !slices.Equal(got, []string{"mlx5_0", "mlx5_1"}) {
|
||||
t.Errorf("HCAs=%v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeartbeatKeepsNodeAlive(t *testing.T) {
|
||||
h := start(t)
|
||||
|
||||
n, _ := h.inv.Get("n1")
|
||||
first := n.LastSeen
|
||||
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
n, _ = h.inv.Get("n1")
|
||||
if !n.LastSeen.After(first) {
|
||||
t.Error("heartbeats are not refreshing LastSeen; the node would be declared lost")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartAndStopShardEndToEnd(t *testing.T) {
|
||||
h := start(t, func(c *Config) {
|
||||
c.Engines.VLLM.Bin = servingEngine(t)
|
||||
})
|
||||
|
||||
cmd := shardCommand(t, "shard-a")
|
||||
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got := h.rec.await(t, 30*time.Second, func(s status) bool {
|
||||
return s.shardID == "shard-a" && s.state == supervisor.Ready
|
||||
})
|
||||
if got.replicaID != "rep-1" {
|
||||
t.Errorf("replica id lost in transit: %q", got.replicaID)
|
||||
}
|
||||
|
||||
if err := h.svc.StopShard(t.Context(), "n1", "shard-a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.rec.await(t, 15*time.Second, func(s status) bool {
|
||||
return s.shardID == "shard-a" && s.state == supervisor.Stopped
|
||||
})
|
||||
}
|
||||
|
||||
func TestFailedShardCarriesDiagnosis(t *testing.T) {
|
||||
// When a model fails to load the engine's output is the only diagnosis, so
|
||||
// it has to survive the trip to the control plane.
|
||||
h := start(t, func(c *Config) {
|
||||
c.Engines.VLLM.Bin = fakeEngineScript(t, `echo 'CUDA out of memory' >&2; exit 1`)
|
||||
})
|
||||
|
||||
if err := h.svc.StartShard(t.Context(), shardCommand(t, "shard-bad")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := h.rec.await(t, 30*time.Second, func(s status) bool {
|
||||
return s.shardID == "shard-bad" && s.state == supervisor.Failed
|
||||
})
|
||||
if got.err == "" {
|
||||
t.Error("the failure carried no error message")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineThatNeverBecomesReady(t *testing.T) {
|
||||
// A process that runs but never serves must time out rather than hang the
|
||||
// replica forever.
|
||||
h := start(t, func(c *Config) {
|
||||
c.Engines.VLLM.Bin = fakeEngineScript(t, `sleep 60`)
|
||||
})
|
||||
|
||||
cmd := shardCommand(t, "shard-stuck")
|
||||
cmd.StartupTimeout = 500 * time.Millisecond
|
||||
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.rec.await(t, 20*time.Second, func(s status) bool {
|
||||
return s.shardID == "shard-stuck" && s.state == supervisor.Failed
|
||||
})
|
||||
}
|
||||
|
||||
func TestValidationRejectsBeforeLaunch(t *testing.T) {
|
||||
// An engine that refuses a profile does so after minutes of loading;
|
||||
// catching it at dispatch costs nothing.
|
||||
h := start(t, func(c *Config) {
|
||||
c.Engines.VLLM.Bin = servingEngine(t)
|
||||
})
|
||||
|
||||
cmd := shardCommand(t, "shard-invalid")
|
||||
cmd.Spec.Profile.ModelPath = "" // validation requires one
|
||||
|
||||
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := h.rec.await(t, 15*time.Second, func(s status) bool {
|
||||
return s.shardID == "shard-invalid" && s.state == supervisor.Failed
|
||||
})
|
||||
if got.err == "" {
|
||||
t.Error("validation failure carried no reason")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDispatchToUnconnectedNode(t *testing.T) {
|
||||
h := start(t)
|
||||
|
||||
err := h.svc.StartShard(t.Context(), reconciler.ShardCommand{NodeID: "nowhere", ShardID: "x"})
|
||||
if err == nil {
|
||||
t.Fatal("dispatching to an unconnected node succeeded")
|
||||
}
|
||||
if !errors.Is(err, agentsvc.ErrNotConnected) {
|
||||
t.Errorf("err=%v, expected ErrNotConnected", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentReconnects(t *testing.T) {
|
||||
// A control plane restart must not require touching every GPU node.
|
||||
h := start(t)
|
||||
h.srv.CloseClientConnections()
|
||||
|
||||
deadline := time.Now().Add(15 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if len(h.svc.ConnectedNodes()) == 1 {
|
||||
if err := h.svc.DrainShard(t.Context(), "n1", "nothing"); err != nil {
|
||||
t.Fatalf("the reconnected stream does not accept commands: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("the agent never reconnected")
|
||||
}
|
||||
|
||||
func TestIdempotentStart(t *testing.T) {
|
||||
// The control plane retries commands; a repeat must not start a second
|
||||
// process (plan §5.6).
|
||||
h := start(t, func(c *Config) {
|
||||
c.Engines.VLLM.Bin = fakeEngineScript(t, `sleep 30`)
|
||||
})
|
||||
|
||||
cmd := shardCommand(t, "shard-idem")
|
||||
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.rec.await(t, 15*time.Second, func(s status) bool {
|
||||
return s.shardID == "shard-idem" && s.state == supervisor.Starting
|
||||
})
|
||||
|
||||
sup := h.ag.Supervisor()
|
||||
sh, err := sup.Get("shard-idem")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pid := sh.Status().PID
|
||||
|
||||
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(400 * time.Millisecond)
|
||||
|
||||
sh2, err := sup.Get("shard-idem")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sh2.Status().PID != pid {
|
||||
t.Errorf("a second process was started: pid %d then %d", pid, sh2.Status().PID)
|
||||
}
|
||||
}
|
||||
@@ -16,14 +16,16 @@ var ErrNoNVML = errors.New("built without the nvml tag; rebuild with -tags nvml
|
||||
// Options configure a full inventory scan.
|
||||
type Options struct {
|
||||
// NvidiaSMI is the path to nvidia-smi; empty means look it up on PATH.
|
||||
NvidiaSMI string
|
||||
// Worth configuring on nodes where the driver lives outside the default
|
||||
// paths, which containerized driver installs often do.
|
||||
NvidiaSMI string `koanf:"nvidia_smi"`
|
||||
// IBRoot is the sysfs InfiniBand directory.
|
||||
IBRoot string
|
||||
IBRoot string `koanf:"ib_root"`
|
||||
// PreferNVML uses the NVML backend when this binary has one. It is the only
|
||||
// source of NVLink topology.
|
||||
PreferNVML bool
|
||||
PreferNVML bool `koanf:"prefer_nvml"`
|
||||
// SkipGPUs omits GPU discovery, for nodes that have none.
|
||||
SkipGPUs bool
|
||||
SkipGPUs bool `koanf:"skip_gpus"`
|
||||
}
|
||||
|
||||
// Discover builds the node's inventory.
|
||||
|
||||
Reference in new issue
Block a user