CI / test (push) Successful in 1m26s
The default arrangement is the simple one: every node keeps its own copy. So a source naming a remote file is now fetched before the engine is launched, rather than left to fail as a missing path minutes into a load. Eight shards of a tensor-parallel replica start at the same instant and want the same file; they produce one download. An interrupted fetch resumes instead of starting over, which matters at forty gigabytes, and the download is written beside its target and renamed only when complete — a partial file under the model's name would be found by the next scan, reported as present, and handed to an engine that fails on it. The exotic arrangements need no code, which is the point. Weights on a Lustre mount or an image exported over the fabric are simply present, and present means nothing is fetched. The escape hatch from a distribution mechanism is a distribution mechanism that checks first. Wiring it up caught a regression in my own design. A bare model id — "Qwen/Qwen2.5-7B-Instruct" — is a documented source that vLLM and SGLang resolve themselves, and treating every non-remote source as a local path made every existing vLLM profile fail. Only an absolute or explicitly relative path is now checked as a file; anything else is passed through untouched. Three agent tests failed on the same change for a better reason: they used "/models/x", and the new check is exactly the one that catches that. Whole repositories of shards are deliberately not fetched here. vLLM and SGLang do that well already, and a second snapshot downloader would be work spent arriving back where we started. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
651 lines
19 KiB
Go
651 lines
19 KiB
Go
package agent
|
|
|
|
import (
|
|
"context"
|
|
"crypto/tls"
|
|
"errors"
|
|
"log/slog"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
|
|
"golang.org/x/net/http2"
|
|
"golang.org/x/net/http2/h2c"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
|
)
|
|
|
|
// statusRecorder captures what the control plane hears about shards, and what
|
|
// it is asked to adopt.
|
|
type statusRecorder struct {
|
|
ch chan status
|
|
|
|
mu sync.Mutex
|
|
calls []adoptCall
|
|
adopted []reconciler.AdoptedShard
|
|
unclaim []string // shard ids to report back as unclaimed
|
|
}
|
|
|
|
type status struct {
|
|
replicaID, shardID string
|
|
state supervisor.State
|
|
err string
|
|
}
|
|
|
|
func newRecorder() *statusRecorder { return &statusRecorder{ch: make(chan status, 128)} }
|
|
|
|
func (r *statusRecorder) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) {
|
|
select {
|
|
case r.ch <- status{replicaID, shardID, st, errMsg}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
func (r *statusRecorder) Adopt(nodeID string, shards []reconciler.AdoptedShard) []reconciler.AdoptedShard {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
r.calls = append(r.calls, adoptCall{nodeID: nodeID, shards: shards})
|
|
r.adopted = append(r.adopted, shards...)
|
|
|
|
var unclaimed []reconciler.AdoptedShard
|
|
for _, s := range shards {
|
|
if slices.Contains(r.unclaim, s.ShardID) {
|
|
unclaimed = append(unclaimed, s)
|
|
}
|
|
}
|
|
return unclaimed
|
|
}
|
|
|
|
// adoptCall records that the service reached the reconciler at all, which an
|
|
// empty shard list used to prevent.
|
|
type adoptCall struct {
|
|
nodeID string
|
|
shards []reconciler.AdoptedShard
|
|
}
|
|
|
|
func (r *statusRecorder) adoptCalls() []adoptCall {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
return slices.Clone(r.calls)
|
|
}
|
|
|
|
func (r *statusRecorder) adoptedShards() []reconciler.AdoptedShard {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
return slices.Clone(r.adopted)
|
|
}
|
|
|
|
func (r *statusRecorder) markUnclaimed(ids ...string) {
|
|
r.mu.Lock()
|
|
r.unclaim = append(r.unclaim, ids...)
|
|
r.mu.Unlock()
|
|
}
|
|
|
|
func (r *statusRecorder) await(t *testing.T, limit time.Duration, pred func(status) bool) status {
|
|
t.Helper()
|
|
deadline := time.After(limit)
|
|
var seen []status
|
|
for {
|
|
select {
|
|
case s := <-r.ch:
|
|
seen = append(seen, s)
|
|
if pred(s) {
|
|
return s
|
|
}
|
|
case <-deadline:
|
|
t.Fatalf("no matching status within %v; saw %+v", limit, seen)
|
|
return status{}
|
|
}
|
|
}
|
|
}
|
|
|
|
// fakeEngineScript writes a script that stands in for vLLM.
|
|
//
|
|
// It parses the real --port flag the vLLM adapter generates rather than being
|
|
// told out of band, so the adapter's argument construction is exercised too.
|
|
func fakeEngineScript(t *testing.T, body string) string {
|
|
t.Helper()
|
|
path := filepath.Join(t.TempDir(), "fake-vllm")
|
|
script := `#!/bin/sh
|
|
port=""
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--port) port="$2"; shift 2 ;;
|
|
*) shift ;;
|
|
esac
|
|
done
|
|
` + body + "\n"
|
|
if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return path
|
|
}
|
|
|
|
// servingEngine answers /health and /v1/models, which is what readiness needs.
|
|
func servingEngine(t *testing.T) string {
|
|
t.Helper()
|
|
py, err := exec.LookPath("python3")
|
|
if err != nil {
|
|
t.Skip("python3 is required for the fake engine")
|
|
}
|
|
return fakeEngineScript(t, `exec `+py+` -c '
|
|
import http.server, json, sys
|
|
port = int(sys.argv[1])
|
|
class H(http.server.BaseHTTPRequestHandler):
|
|
def do_GET(self):
|
|
if self.path == "/health":
|
|
self.send_response(200); self.end_headers(); return
|
|
if self.path == "/v1/models":
|
|
b = json.dumps({"object":"list","data":[{"id":"test-model"}]}).encode()
|
|
self.send_response(200)
|
|
self.send_header("content-type","application/json")
|
|
self.send_header("content-length", str(len(b)))
|
|
self.end_headers(); self.wfile.write(b); return
|
|
self.send_response(404); self.end_headers()
|
|
def log_message(self, *a): pass
|
|
http.server.HTTPServer(("127.0.0.1", port), H).serve_forever()
|
|
' "$port"`)
|
|
}
|
|
|
|
type harness struct {
|
|
svc *agentsvc.Service
|
|
inv *inventory.Store
|
|
rec *statusRecorder
|
|
srv *httptest.Server
|
|
ag *Agent
|
|
ln *dropListener
|
|
// streams counts how many times an agent has opened the Connect stream,
|
|
// which is the only honest way to tell a reconnect from a stale entry.
|
|
streams *atomic.Int64
|
|
}
|
|
|
|
// dropListener tracks accepted connections so a test can sever them.
|
|
//
|
|
// httptest's CloseClientConnections does not work here: h2c with prior
|
|
// knowledge hijacks the connection, and httptest stops tracking hijacked
|
|
// connections. A test relying on it would pass without ever reconnecting.
|
|
type dropListener struct {
|
|
net.Listener
|
|
mu sync.Mutex
|
|
conns []net.Conn
|
|
}
|
|
|
|
func (l *dropListener) Accept() (net.Conn, error) {
|
|
c, err := l.Listener.Accept()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
l.mu.Lock()
|
|
l.conns = append(l.conns, c)
|
|
l.mu.Unlock()
|
|
return c, nil
|
|
}
|
|
|
|
// drop severs every connection, as a control plane restart would.
|
|
func (l *dropListener) drop() {
|
|
l.mu.Lock()
|
|
conns := l.conns
|
|
l.conns = nil
|
|
l.mu.Unlock()
|
|
for _, c := range conns {
|
|
_ = c.Close()
|
|
}
|
|
}
|
|
|
|
// start wires a real agent to a real service over h2c, so this exercises the
|
|
// protocol rather than two structs calling each other.
|
|
func start(t *testing.T, cfgFns ...func(*Config)) *harness {
|
|
t.Helper()
|
|
return startWith(t, slog.New(slog.DiscardHandler), cfgFns...)
|
|
}
|
|
|
|
// startLogged is start with logging on, for diagnosing protocol behaviour.
|
|
func startLogged(t *testing.T, cfgFns ...func(*Config)) *harness {
|
|
t.Helper()
|
|
return startWith(t, slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})), cfgFns...)
|
|
}
|
|
|
|
func startWith(t *testing.T, log *slog.Logger, cfgFns ...func(*Config)) *harness {
|
|
t.Helper()
|
|
|
|
inv := inventory.NewStore()
|
|
rec := newRecorder()
|
|
svc := agentsvc.New(inv, rec, false, log)
|
|
|
|
path, handler := svc.Handler()
|
|
mux := http.NewServeMux()
|
|
mux.Handle(path, handler)
|
|
|
|
var streams atomic.Int64
|
|
counting := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
streams.Add(1)
|
|
mux.ServeHTTP(w, r)
|
|
})
|
|
|
|
raw, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ln := &dropListener{Listener: raw}
|
|
// connect's bidirectional streams need HTTP/2; h2c provides it without TLS.
|
|
srv := httptest.NewUnstartedServer(h2c.NewHandler(counting, &http2.Server{}))
|
|
srv.Listener = ln
|
|
srv.Start()
|
|
|
|
cfg := DefaultConfig()
|
|
cfg.NodeID = "n1"
|
|
cfg.Server = srv.URL
|
|
cfg.MgmtAddr = "127.0.0.1"
|
|
cfg.HeartbeatInterval = 50 * time.Millisecond
|
|
cfg.ReconnectBase = 20 * time.Millisecond
|
|
cfg.ReconnectMax = 100 * time.Millisecond
|
|
cfg.Discovery.SkipGPUs = true
|
|
cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent")
|
|
cfg.Superv = supervisor.Config{
|
|
WorkDir: t.TempDir(), LogLines: 64,
|
|
StopTimeout: 2 * time.Second, HealthInterval: 20 * time.Millisecond,
|
|
}
|
|
for _, f := range cfgFns {
|
|
f(&cfg)
|
|
}
|
|
|
|
client := &http.Client{Transport: &http2.Transport{
|
|
AllowHTTP: true,
|
|
DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
|
|
return (&net.Dialer{}).DialContext(ctx, network, addr)
|
|
},
|
|
}}
|
|
|
|
ag, err := New(cfg, client, log)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
go func() { _ = ag.Run(ctx) }()
|
|
|
|
h := &harness{svc: svc, inv: inv, rec: rec, srv: srv, ag: ag, ln: ln, streams: &streams}
|
|
t.Cleanup(func() {
|
|
cancel()
|
|
ag.Supervisor().StopAll(context.Background())
|
|
srv.Close()
|
|
})
|
|
h.waitRegistered(t)
|
|
return h
|
|
}
|
|
|
|
func (h *harness) waitRegistered(t *testing.T) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(10 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if _, ok := h.inv.Get("n1"); ok && len(h.svc.ConnectedNodes()) == 1 {
|
|
return
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
t.Fatal("the agent never registered")
|
|
}
|
|
|
|
// shardCommand builds a single-node vLLM shard on a free port.
|
|
//
|
|
// The model path is a real file. It used to be "/models/x", which stopped
|
|
// working once the agent began checking that weights exist before launching an
|
|
// engine — the check doing exactly what it is for, on a test that was relying
|
|
// on nobody looking.
|
|
func shardCommand(t *testing.T, shardID string) reconciler.ShardCommand {
|
|
t.Helper()
|
|
modelPath := filepath.Join(t.TempDir(), "model.gguf")
|
|
if err := os.WriteFile(modelPath, []byte("weights"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return reconciler.ShardCommand{
|
|
ReplicaID: "rep-1", ShardID: shardID, NodeID: "n1",
|
|
Spec: engines.ShardSpec{
|
|
ReplicaID: "rep-1", ShardID: shardID, Rank: 0, NNodes: 1,
|
|
GPUIndices: []int{0}, HTTPPort: freePort(t),
|
|
Profile: engines.Profile{
|
|
Name: "p", Engine: "vllm", TP: 1,
|
|
ModelPath: modelPath, ServedModelName: "test-model",
|
|
},
|
|
},
|
|
StartupTimeout: 15 * time.Second,
|
|
}
|
|
}
|
|
|
|
func freePort(t *testing.T) int {
|
|
t.Helper()
|
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer ln.Close()
|
|
return ln.Addr().(*net.TCPAddr).Port
|
|
}
|
|
|
|
func TestRegistrationCarriesInventory(t *testing.T) {
|
|
h := start(t, func(c *Config) {
|
|
c.Hostname = "gpu-a-01"
|
|
c.Labels = map[string]string{"rack": "r12"}
|
|
c.Fabrics = []inventory.FabricAttachment{
|
|
{Fabric: "ib-a", HCAs: []string{"mlx5_0", "mlx5_1"}, RateGbps: 200},
|
|
}
|
|
})
|
|
|
|
n, ok := h.inv.Get("n1")
|
|
if !ok {
|
|
t.Fatal("node missing from inventory")
|
|
}
|
|
if n.Hostname != "gpu-a-01" || n.MgmtAddr != "127.0.0.1" {
|
|
t.Errorf("identity wrong: %+v", n)
|
|
}
|
|
if n.Labels["rack"] != "r12" {
|
|
t.Errorf("labels lost: %v", n.Labels)
|
|
}
|
|
// The fabric label cannot be inferred from the wire, so losing it in
|
|
// transit would silently disable multi-node placement.
|
|
if !n.HasFabric("ib-a") {
|
|
t.Fatalf("fabric attachment lost: %+v", n.Fabrics)
|
|
}
|
|
if got := n.HCAsFor("ib-a"); !slices.Equal(got, []string{"mlx5_0", "mlx5_1"}) {
|
|
t.Errorf("HCAs=%v", got)
|
|
}
|
|
}
|
|
|
|
func TestHeartbeatKeepsNodeAlive(t *testing.T) {
|
|
h := start(t)
|
|
|
|
n, _ := h.inv.Get("n1")
|
|
first := n.LastSeen
|
|
|
|
time.Sleep(200 * time.Millisecond)
|
|
n, _ = h.inv.Get("n1")
|
|
if !n.LastSeen.After(first) {
|
|
t.Error("heartbeats are not refreshing LastSeen; the node would be declared lost")
|
|
}
|
|
}
|
|
|
|
func TestStartAndStopShardEndToEnd(t *testing.T) {
|
|
h := start(t, func(c *Config) {
|
|
c.Engines.VLLM.Bin = servingEngine(t)
|
|
})
|
|
|
|
cmd := shardCommand(t, "shard-a")
|
|
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
got := h.rec.await(t, 30*time.Second, func(s status) bool {
|
|
return s.shardID == "shard-a" && s.state == supervisor.Ready
|
|
})
|
|
if got.replicaID != "rep-1" {
|
|
t.Errorf("replica id lost in transit: %q", got.replicaID)
|
|
}
|
|
|
|
if err := h.svc.StopShard(t.Context(), "n1", "shard-a"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.rec.await(t, 15*time.Second, func(s status) bool {
|
|
return s.shardID == "shard-a" && s.state == supervisor.Stopped
|
|
})
|
|
}
|
|
|
|
func TestFailedShardCarriesDiagnosis(t *testing.T) {
|
|
// When a model fails to load the engine's output is the only diagnosis, so
|
|
// it has to survive the trip to the control plane.
|
|
h := start(t, func(c *Config) {
|
|
c.Engines.VLLM.Bin = fakeEngineScript(t, `echo 'CUDA out of memory' >&2; exit 1`)
|
|
})
|
|
|
|
if err := h.svc.StartShard(t.Context(), shardCommand(t, "shard-bad")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := h.rec.await(t, 30*time.Second, func(s status) bool {
|
|
return s.shardID == "shard-bad" && s.state == supervisor.Failed
|
|
})
|
|
if got.err == "" {
|
|
t.Error("the failure carried no error message")
|
|
}
|
|
}
|
|
|
|
func TestEngineThatNeverBecomesReady(t *testing.T) {
|
|
// A process that runs but never serves must time out rather than hang the
|
|
// replica forever.
|
|
h := start(t, func(c *Config) {
|
|
c.Engines.VLLM.Bin = fakeEngineScript(t, `sleep 60`)
|
|
})
|
|
|
|
cmd := shardCommand(t, "shard-stuck")
|
|
cmd.StartupTimeout = 500 * time.Millisecond
|
|
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.rec.await(t, 20*time.Second, func(s status) bool {
|
|
return s.shardID == "shard-stuck" && s.state == supervisor.Failed
|
|
})
|
|
}
|
|
|
|
func TestValidationRejectsBeforeLaunch(t *testing.T) {
|
|
// An engine that refuses a profile does so after minutes of loading;
|
|
// catching it at dispatch costs nothing.
|
|
h := start(t, func(c *Config) {
|
|
c.Engines.VLLM.Bin = servingEngine(t)
|
|
})
|
|
|
|
cmd := shardCommand(t, "shard-invalid")
|
|
cmd.Spec.Profile.ModelPath = "" // validation requires one
|
|
|
|
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := h.rec.await(t, 15*time.Second, func(s status) bool {
|
|
return s.shardID == "shard-invalid" && s.state == supervisor.Failed
|
|
})
|
|
if got.err == "" {
|
|
t.Error("validation failure carried no reason")
|
|
}
|
|
}
|
|
|
|
func TestDispatchToUnconnectedNode(t *testing.T) {
|
|
h := start(t)
|
|
|
|
err := h.svc.StartShard(t.Context(), reconciler.ShardCommand{NodeID: "nowhere", ShardID: "x"})
|
|
if err == nil {
|
|
t.Fatal("dispatching to an unconnected node succeeded")
|
|
}
|
|
if !errors.Is(err, agentsvc.ErrNotConnected) {
|
|
t.Errorf("err=%v, expected ErrNotConnected", err)
|
|
}
|
|
}
|
|
|
|
func TestAgentReconnects(t *testing.T) {
|
|
// A control plane restart must not require touching every GPU node.
|
|
h := start(t)
|
|
|
|
before := h.streams.Load()
|
|
h.ln.drop()
|
|
|
|
// Counting streams, not connected nodes: a stale entry in the connection
|
|
// map would make a node look connected without any reconnect having
|
|
// happened, and an assertion on it would pass vacuously.
|
|
deadline := time.Now().Add(15 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if h.streams.Load() > before && len(h.svc.ConnectedNodes()) == 1 {
|
|
if err := h.svc.DrainShard(t.Context(), "n1", "nothing"); err != nil {
|
|
t.Fatalf("the reconnected stream does not accept commands: %v", err)
|
|
}
|
|
return
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
t.Fatalf("the agent never reconnected (streams %d -> %d)", before, h.streams.Load())
|
|
}
|
|
|
|
func TestIdempotentStart(t *testing.T) {
|
|
// The control plane retries commands; a repeat must not start a second
|
|
// process (plan §5.6).
|
|
h := start(t, func(c *Config) {
|
|
c.Engines.VLLM.Bin = fakeEngineScript(t, `sleep 30`)
|
|
})
|
|
|
|
cmd := shardCommand(t, "shard-idem")
|
|
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.rec.await(t, 15*time.Second, func(s status) bool {
|
|
return s.shardID == "shard-idem" && s.state == supervisor.Starting
|
|
})
|
|
|
|
sup := h.ag.Supervisor()
|
|
sh, err := sup.Get("shard-idem")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
pid := sh.Status().PID
|
|
|
|
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
time.Sleep(400 * time.Millisecond)
|
|
|
|
sh2, err := sup.Get("shard-idem")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if sh2.Status().PID != pid {
|
|
t.Errorf("a second process was started: pid %d then %d", pid, sh2.Status().PID)
|
|
}
|
|
}
|
|
|
|
// TestRunningShardsReportedOnReconnect covers the control plane restart hole
|
|
// (plan §6.5): a reconnecting agent has to describe what it is already running,
|
|
// or the control plane concludes the cluster is empty and places duplicates
|
|
// while the originals keep holding GPUs.
|
|
func TestRunningShardsReportedOnReconnect(t *testing.T) {
|
|
h := start(t, func(c *Config) {
|
|
c.Engines.VLLM.Bin = fakeEngineScript(t, `sleep 60`)
|
|
})
|
|
|
|
cmd := shardCommand(t, "shard-live")
|
|
cmd.Pool = "ib-a/h100-80g"
|
|
cmd.Fabric = "ib-a"
|
|
cmd.Spec.HCAs = []string{"mlx5_0"}
|
|
|
|
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.rec.await(t, 15*time.Second, func(s status) bool {
|
|
return s.shardID == "shard-live" && s.state == supervisor.Starting
|
|
})
|
|
|
|
// Drop the stream, as a control plane restart would.
|
|
h.ln.drop()
|
|
|
|
deadline := time.Now().Add(15 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if got := h.rec.adoptedShards(); len(got) > 0 {
|
|
a := got[0]
|
|
if a.ShardID != "shard-live" || a.ReplicaID != "rep-1" {
|
|
t.Fatalf("wrong shard reported: %+v", a)
|
|
}
|
|
// Everything the control plane needs to rebuild the replica.
|
|
if a.Model != "test-model" || a.Profile != "p" {
|
|
t.Errorf("catalog identity lost: model=%q profile=%q", a.Model, a.Profile)
|
|
}
|
|
if a.Pool != "ib-a/h100-80g" || a.Fabric != "ib-a" {
|
|
t.Errorf("pool/fabric lost: %q %q", a.Pool, a.Fabric)
|
|
}
|
|
if !slices.Equal(a.HCAs, []string{"mlx5_0"}) {
|
|
t.Errorf("HCAs lost: %v", a.HCAs)
|
|
}
|
|
if a.Endpoint == "" {
|
|
t.Error("rank 0 reported no endpoint; the router would have nowhere to send")
|
|
}
|
|
if len(a.GPUIndices) == 0 {
|
|
t.Error("GPU indices lost; the inventory could not reclaim them")
|
|
}
|
|
return
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
t.Fatal("the reconnecting agent never reported its running shard")
|
|
}
|
|
|
|
// TestUnclaimedShardIsStopped covers the other half: a shard whose model left
|
|
// the catalog is holding GPUs nothing will ever route to.
|
|
func TestUnclaimedShardIsStopped(t *testing.T) {
|
|
h := start(t, func(c *Config) {
|
|
c.Engines.VLLM.Bin = fakeEngineScript(t, `sleep 60`)
|
|
})
|
|
|
|
cmd := shardCommand(t, "shard-orphaned")
|
|
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.rec.await(t, 15*time.Second, func(s status) bool {
|
|
return s.shardID == "shard-orphaned" && s.state == supervisor.Starting
|
|
})
|
|
|
|
sh, err := h.ag.Supervisor().Get("shard-orphaned")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
pid := sh.Status().PID
|
|
|
|
// The control plane will refuse to claim it on reconnect.
|
|
h.rec.markUnclaimed("shard-orphaned")
|
|
h.ln.drop()
|
|
|
|
deadline := time.Now().Add(20 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if !processAliveForTest(pid) {
|
|
return
|
|
}
|
|
time.Sleep(50 * time.Millisecond)
|
|
}
|
|
t.Error("an unclaimed shard kept running and holding its GPUs")
|
|
}
|
|
|
|
// processAliveForTest mirrors the supervisor's liveness check.
|
|
func processAliveForTest(pid int) bool {
|
|
return syscall.Kill(pid, 0) == nil
|
|
}
|
|
|
|
func TestAShardWithMissingWeightsFailsBeforeLaunching(t *testing.T) {
|
|
// The commonest setup mistake on a fresh node. Left to the engine it
|
|
// surfaces minutes into a load, in the engine's own words, on a machine
|
|
// nobody is watching — so it is caught here, where the message can say
|
|
// what is actually wrong.
|
|
a := &Agent{cfg: Config{WeightsCache: t.TempDir()}, log: slog.New(slog.DiscardHandler)}
|
|
_, err := a.weights().Ensure(t.Context(), filepath.Join(t.TempDir(), "not-there.gguf"))
|
|
if err == nil {
|
|
t.Fatal("a missing weights file was accepted")
|
|
}
|
|
if !strings.Contains(err.Error(), "not-there.gguf") {
|
|
t.Errorf("the error does not name the file: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestTheCacheDefaultsToTheFirstModelDirectory(t *testing.T) {
|
|
// A node that has said where its models live has already answered this;
|
|
// asking twice is a second thing to get out of step.
|
|
dir := t.TempDir()
|
|
a := &Agent{cfg: Config{ModelDirs: []string{dir, "/elsewhere"}}, log: slog.New(slog.DiscardHandler)}
|
|
if got := a.weights().CacheDir; got != dir {
|
|
t.Errorf("cache = %q, want the first model directory %q", got, dir)
|
|
}
|
|
}
|