From 2acd794022586f1c66b9ec1ad48a93ba041305b4 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 27 Sep 2026 16:11:55 +0200 Subject: [PATCH] Stop placing engines on nodes that do not have them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A node's registry holds an adapter for every engine whether or not its binary exists, the agent never said which were installed, and placement carried an Engine field it had never once read. So a vLLM replica was as happily placed on a node with no vLLM: the node registered healthy, the shard failed minutes later with "executable file not found" in the engine's own words, and the reconciler tried again. That is how fifteen gigabytes of safetensors reached this cluster's mirror before anybody noticed it had no card and no vLLM to run them with. Agents now probe what is installed and report it — the protocol has carried engine_versions since it was written and nothing ever filled it in. Placement filters on the key set, `ctl nodes` has a column, and the mirror page's serve warning is a fact instead of a guess about cards, which would have waved through a fleet of eight-GPU nodes that never had vLLM. Probing resolves binaries and runs none of them. The first version asked each engine for --version, which is fine for an engine and not fine for the wrapper scripts people put in front of one: a script that shifts past flags it does not recognise and then execs the server will start an inference server, at agent startup, on every node. The agent's own test suite installs exactly such a script and caught it. So presence is a path lookup, and the version is left to `ctl engine-check`, which is run by somebody who meant to run the engine. SGLang is the exception and has to execute, because a module has no path to look up — but the command is llmgw's own one-line import rather than the engine's launcher. Its exit code is the answer: python3 is on almost every machine, and treating a failed import as "installed" reported SGLang on every node in a fleet that had none, which is the failure being fixed, inverted. Probed once, not per reconnect, and bounded by the context that registration waits on: an agent that cannot reach the control plane is worse than one reporting an incomplete list. A node that reports no engines at all is not excluded anywhere — by placement, or by the page. That is what every agent older than this reports, and emptying a fleet on a control plane upgrade is a worse failure than the one being fixed. Co-Authored-By: Claude Opus 5 (1M context) --- internal/agent/agent.go | 28 +++ internal/agent/engines/probe.go | 190 +++++++++++++++++++ internal/agent/engines/probe_test.go | 107 +++++++++++ internal/cli/ctl.go | 8 +- internal/control/adminapi/api.go | 10 + internal/control/placement/placement.go | 15 ++ internal/control/placement/placement_test.go | 59 ++++++ internal/web/admin_serve.go | 45 ++++- internal/web/admin_serve_test.go | 59 ++++-- 9 files changed, 500 insertions(+), 21 deletions(-) create mode 100644 internal/agent/engines/probe.go create mode 100644 internal/agent/engines/probe_test.go diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 4d2e8c8..6460d34 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -209,6 +209,14 @@ type Agent struct { reg *engines.Registry http connectHTTPClient + // engineOnce probes the installed engines exactly once. + // + // Once, because probing runs each engine's --version and vLLM's imports + // torch — seconds on a cold cache. A reconnect is not a reason to pay that + // again, and what is installed does not change while the process runs. + engineOnce sync.Once + engineVers map[string]string + mu sync.Mutex inv discovery.Inventory // shardReplica remembers which replica a shard belongs to, so a status @@ -567,6 +575,11 @@ func (a *Agent) session(ctx context.Context, server string) error { // Every registration reports what is already running, not just the first // one: a reconnect may be a control plane restart, and it has to recover // the same way (plan §6.5). + // What this node can actually run, so the control plane stops placing + // engines that are not here. The registry holds an adapter for every + // engine whether or not its binary exists, so without this a node with no + // vLLM accepted a vLLM shard and failed it minutes later. + reg.EngineVersions = a.engines(parent) reg.RunningShards = a.runningShards() if err := stream.Send(&llmgwv1.AgentMessage{ @@ -1018,3 +1031,18 @@ func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) { } return &http.Client{Transport: &http2.Transport{TLSClientConfig: tc}}, nil } + +// engines is what this node can run, probed once. +// +// Failures are silence rather than an error: an engine that is not installed is +// the ordinary case on most nodes, and an agent that refused to register over +// one would take a working node out of the fleet for a missing engine it was +// never going to be asked to run. +func (a *Agent) engines(ctx context.Context) map[string]string { + a.engineOnce.Do(func() { + a.engineVers = a.reg.Probe(ctx) + a.log.Info("engines available on this node", + "engines", engines.ProbedNames(a.engineVers)) + }) + return a.engineVers +} diff --git a/internal/agent/engines/probe.go b/internal/agent/engines/probe.go new file mode 100644 index 0000000..7e9819a --- /dev/null +++ b/internal/agent/engines/probe.go @@ -0,0 +1,190 @@ +package engines + +import ( + "context" + "errors" + "os/exec" + "regexp" + "sort" + "strings" + "time" +) + +// Unknown is the version reported for an engine that is installed and would +// not say which version it is. +// +// A value, not an absent key, because the two mean different things and +// placement reads the keys: absent is "this node cannot run it", Unknown is +// "it is here and the probe could not get a number out of it". Collapsing them +// would make a slow engine look like a missing one and take a node out of +// scheduling for being slow to answer --version. +const Unknown = "unknown" + +// Prober is an engine that can say whether it is installed on this machine. +// +// Optional, and asserted for rather than part of Engine: an adapter that has no +// way to check should report nothing rather than be forced to invent an answer. +type Prober interface { + // Probe returns the engine's version, or an error when it is not installed. + Probe(ctx context.Context) (string, error) +} + +// Probe asks every registered engine whether it is here. +// +// The result is what the node reports at registration, and its key set is what +// placement filters on. Until this existed nobody knew: the registry registers +// every adapter whether or not its binary exists, so a node with no vLLM +// accepted a vLLM shard, registered healthy, and failed the shard minutes later +// with "executable file not found" — in the engine's own words, on a machine +// nobody was watching. +// +// Bounded, because it runs on the path to registering and an agent that cannot +// reach the control plane is worse than one that reports an incomplete list. +func (r *Registry) Probe(ctx context.Context) map[string]string { + out := map[string]string{} + for name, e := range r.engines { + p, ok := e.(Prober) + if !ok { + continue + } + v, err := p.Probe(ctx) + if err != nil { + continue + } + if v == "" { + v = Unknown + } + out[name] = v + } + return out +} + +// Names of the engines in a probe result, sorted. +func ProbedNames(m map[string]string) []string { + out := make([]string, 0, len(m)) + for n := range m { + out = append(out, n) + } + sort.Strings(out) + return out +} + +// probeTimeout bounds one engine's version check. +// +// Generous, because vLLM's --version imports torch and that is seconds on a +// cold page cache, and mean, because this is startup: an engine that has not +// answered in this long is reported as present-with-unknown-version rather +// than waited for. +const probeTimeout = 8 * time.Second + +// run executes a version command, saying whether the program was even there. +// +// The two failures are different answers and the caller has to tell them +// apart. A binary that is absent is an engine this node cannot run. A binary +// that is present and exits non-zero is an engine it probably can — unless the +// command was a test of whether a module imports, where a non-zero exit is the +// whole answer. +func run(ctx context.Context, bin string, args ...string) (out string, found bool, ok bool) { + path, err := exec.LookPath(bin) + if err != nil { + return "", false, false + } + ctx, cancel := context.WithTimeout(ctx, probeTimeout) + defer cancel() + + // Combined, not just stdout: engines print their version to either, and one + // that chose stderr would otherwise read as having no version at all. + b, err := exec.CommandContext(ctx, path, args...).CombinedOutput() + return string(b), true, err == nil +} + +// installedBinary reports an engine that is its own executable. +// +// It resolves the path and runs nothing. The first version of this ran the +// engine with --version, which is fine for an engine and not fine for the +// wrapper scripts people put in front of one: a script that shifts past flags +// it does not recognise and then execs the server will start an inference +// server, at agent startup, on every node. The agent's own test suite installs +// exactly such a script, and caught it. +// +// So the version is not read here. Presence is what placement needs, and what +// the version is for — telling whether an engine still accepts the flags llmgw +// builds — is `llmgw ctl engine-check`, which is run deliberately by somebody +// who meant to run the engine. +func installedBinary(bin string) (string, error) { + if _, err := exec.LookPath(bin); err != nil { + return "", errNotInstalled + } + return Unknown, nil +} + +// versionOfModule probes an engine that is a module inside an interpreter. +// +// This one does execute, and safely: the command is llmgw's own one-line +// import, not the engine's launcher, so there is no wrapper to shift past +// flags and start a server. There is also no alternative — a module has no +// path to look up. +// +// A non-zero exit is the answer: python3 exists on almost every machine, so +// presence of the interpreter says nothing. Treating a failed import as +// "installed" reported SGLang on every node in a fleet that had none, which +// would take nodes into scheduling for an engine they cannot run — the exact +// failure this whole change exists to stop. +func versionOfModule(ctx context.Context, python string, args ...string) (string, error) { + out, found, ok := run(ctx, python, args...) + if !found || !ok { + return "", errNotInstalled + } + return firstVersion(out), nil +} + +var errNotInstalled = errors.New("not installed on this node") + +var versionPattern = regexp.MustCompile(`\b\d+\.\d+(\.\d+)?\S*`) + +// firstVersion is the first thing in the output that looks like a version. +// +// Pattern rather than parsing each engine's format: they print "vllm 0.6.3", +// "version: b4123 (…)", and a paragraph with the number in the middle, and all +// of those change between releases. What this is for is telling one build from +// another, which the first number does. +func firstVersion(s string) string { + for _, line := range strings.Split(s, "\n") { + if m := versionPattern.FindString(line); m != "" { + return strings.TrimRight(m, ",;)") + } + } + return Unknown +} + +// Probe reports whether the llama.cpp server is installed. +func (l *LlamaCpp) Probe(context.Context) (string, error) { + bin := l.cfg.Bin + if bin == "" { + bin = "llama-server" + } + return installedBinary(bin) +} + +// Probe reports whether vLLM is installed. +func (v *VLLM) Probe(context.Context) (string, error) { + bin := v.cfg.Bin + if bin == "" { + bin = "vllm" + } + return installedBinary(bin) +} + +// Probe reports the SGLang version. +// +// Through the interpreter, because SGLang is a module rather than a binary: +// asking the shell for "sglang" finds nothing on a machine that can run it +// perfectly well. +func (g *SGLang) Probe(ctx context.Context) (string, error) { + python := g.cfg.Python + if python == "" { + python = "python3" + } + return versionOfModule(ctx, python, "-c", + "import sglang, sys; sys.stdout.write(getattr(sglang, '__version__', ''))") +} diff --git a/internal/agent/engines/probe_test.go b/internal/agent/engines/probe_test.go new file mode 100644 index 0000000..3d1175b --- /dev/null +++ b/internal/agent/engines/probe_test.go @@ -0,0 +1,107 @@ +package engines + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +// script writes an executable that stands in for an engine. +func script(t *testing.T, name, body string) string { + t.Helper() + dir := t.TempDir() + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte("#!/bin/sh\n"+body+"\n"), 0o755); err != nil { + t.Fatal(err) + } + return p +} + +func TestAnEngineThatIsNotInstalledIsAbsentFromTheReport(t *testing.T) { + r := NewRegistry(Config{}) + r.engines = map[string]Engine{ + "vllm": &VLLM{cfg: VLLMConfig{Bin: "/nonexistent/vllm"}}, + } + // Absent, not present-with-unknown-version. Placement reads the keys, and + // a node listed for an engine it cannot run is the failure being fixed. + if got := r.Probe(t.Context()); len(got) != 0 { + t.Errorf("expected nothing, got %v", got) + } +} + +func TestProbingNeverRunsTheEngine(t *testing.T) { + // A wrapper that shifts past flags it does not recognise and then starts + // the server. People put these in front of engines, and the agent's own + // tests install one. Probing with --version would launch an inference + // server at agent startup, on every node. + marker := filepath.Join(t.TempDir(), "ran") + bin := script(t, "fake-vllm", "touch "+marker+"; sleep 60") + + r := NewRegistry(Config{}) + r.engines = map[string]Engine{"vllm": &VLLM{cfg: VLLMConfig{Bin: bin}}} + + got := r.Probe(t.Context()) + if _, ok := got["vllm"]; !ok { + t.Error("a binary that is there should be reported present") + } + if _, err := os.Stat(marker); err == nil { + t.Fatal("the engine was executed by a probe") + } +} + +func TestAModuleThatWillNotImportIsNotInstalled(t *testing.T) { + // python3 is on almost every machine, so the interpreter being there says + // nothing about SGLang. Reporting it present put every node in a fleet + // forward for an engine none of them had. + bin := script(t, "python3", "exit 1") + r := NewRegistry(Config{}) + r.engines = map[string]Engine{"sglang": &SGLang{cfg: SGLangConfig{Python: bin}}} + + if got := r.Probe(t.Context()); len(got) != 0 { + t.Errorf("a failed import means not installed, got %v", got) + } +} + +func TestAModuleThatImportsIsReportedWithItsVersion(t *testing.T) { + bin := script(t, "python3", `echo "0.4.1"`) + r := NewRegistry(Config{}) + r.engines = map[string]Engine{"sglang": &SGLang{cfg: SGLangConfig{Python: bin}}} + + got := r.Probe(t.Context()) + if got["sglang"] != "0.4.1" { + t.Errorf("expected the version, got %v", got) + } +} + +func TestAVersionIsPickedOutOfWhateverIsPrinted(t *testing.T) { + // Only the module probe reaches this now, and a Python package version is + // dotted. A bare build number is deliberately not matched: it would turn a + // stray number in a line of prose into a version. + for in, want := range map[string]string{ + "0.4.1": "0.4.1", + "sglang 0.4.post1": "0.4.post1", + "warning: noise\n0.4.1": "0.4.1", + "": Unknown, + "no numbers here": Unknown, + "just 4123": Unknown, + } { + if got := firstVersion(in); got != want { + t.Errorf("firstVersion(%q) = %q, want %q", in, got, want) + } + } +} + +func TestProbingIsBoundedByTheContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + bin := script(t, "python3", "sleep 60") + r := NewRegistry(Config{}) + r.engines = map[string]Engine{"sglang": &SGLang{cfg: SGLangConfig{Python: bin}}} + + // Registration waits on this. An agent that cannot reach the control plane + // is worse than one reporting an incomplete list. + if got := r.Probe(ctx); len(got) != 0 { + t.Errorf("a cancelled probe should report nothing, got %v", got) + } +} diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index a8dd6fa..b6961bf 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -175,11 +175,15 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { } w := tab() - fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tDISK\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN") + fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tDISK\tENGINES\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN") for _, n := range nodes { - fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%s\t%d\t%s\n", + fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n", n.ID, n.State, n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB), diskColumn(n.DiskFreeGB, n.DiskTotalGB), + // What the node can run, which is the first thing to look + // at when a model will not place and the hardware looks + // fine. + join(n.Engines), join(n.Classes), join(n.Fabrics), n.Replicas, n.LastSeenAgo) } _ = w.Flush() diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index ad6c3c7..9303f57 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -73,6 +73,12 @@ type NodeView struct { // what decides whether a CPU profile can be placed here, so it belongs // where someone looks when a model will not schedule. MemoryGB int `json:"memory_gb,omitempty"` + // Engines are the inference engines installed here, as the agent found + // them. Absent means the agent is older than the probe and said nothing, + // which is not the same as having none — placement treats it that way too, + // or a control plane upgrade would empty a fleet. + Engines []string `json:"engines,omitempty"` + // Models are the weights this node can see. Absent means it was not asked // to look, which is not the same as holding none. Models []string `json:"models_on_disk,omitempty"` @@ -211,6 +217,10 @@ func (a *API) Nodes() []NodeView { v.DiskFreeGB = int(free / (1 << 30)) v.DiskTotalGB = int(total / (1 << 30)) } + for name := range n.EngineVersions { + v.Engines = append(v.Engines, name) + } + slices.Sort(v.Engines) for _, w := range n.Weights { v.Models = append(v.Models, w.Path) } diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index 951381e..34b1424 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -394,6 +394,21 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje reject("does not carry " + miss) continue } + // The engine has to be installed here. The field has been on this + // struct since placement was written and was never read, so a node + // with no vLLM was as good a candidate as one with it: the replica was + // placed, the node registered healthy, and the shard failed minutes + // later with "executable file not found" in the engine's own words. + // + // A node that reports no engines at all is not excluded. That is what + // every agent older than this reports, and emptying a fleet on a + // control plane upgrade is a worse failure than the one being fixed. + if p.Engine != "" && len(n.EngineVersions) > 0 { + if _, ok := n.EngineVersions[p.Engine]; !ok { + reject("does not have " + p.Engine + " installed") + continue + } + } // A node that does not report its memory is not excluded: an agent // older than that field would otherwise have its node quietly emptied // by a control plane upgrade, which is a worse failure than the one diff --git a/internal/control/placement/placement_test.go b/internal/control/placement/placement_test.go index c66c432..91dfd2f 100644 --- a/internal/control/placement/placement_test.go +++ b/internal/control/placement/placement_test.go @@ -679,3 +679,62 @@ func TestAProfileWithNoSourceIsUnaffected(t *testing.T) { t.Fatalf("nothing placed:\n%s", res.Explain()) } } + +// withEngines is a node that reports which engines it has. +func withEngines(id string, gpus int, names ...string) inventory.Node { + n := node(id, "h100-80g", gpus) + n.EngineVersions = map[string]string{} + for _, name := range names { + n.EngineVersions[name] = "unknown" + } + return n +} + +func TestAReplicaIsNotPlacedOnANodeWithoutItsEngine(t *testing.T) { + // The engine has been on the Profile struct since placement was written + // and was never read, so a node with no vLLM was as good a candidate as + // one with it: the replica was placed, the node registered healthy, and + // the shard failed minutes later with "executable file not found". + s := New(store(withEngines("no-vllm", 8, "llamacpp"), withEngines("has-vllm", 8, "vllm"))) + + res := s.Place([]Profile{{ + Name: "tp1", Engine: "vllm", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, + }}, nil) + if res.Placement == nil { + t.Fatalf("nothing was placed:\n%s", res.Explain()) + } + if got := res.Placement.Nodes[0].NodeID; got != "has-vllm" { + t.Errorf("placed on %q, which does not have vllm", got) + } +} + +func TestAModelWithNoNodeRunningItsEngineSaysSo(t *testing.T) { + s := New(store(withEngines("cpu-only", 8, "llamacpp"))) + + res := s.Place([]Profile{{ + Name: "tp1", Engine: "vllm", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, + }}, nil) + if res.Placement != nil { + t.Fatal("a model whose engine is installed nowhere should not be placed") + } + // The reason belongs in explain-placement, or the answer to "why is this + // not running" is a retry loop in the log. + if !strings.Contains(res.Explain(), "does not have vllm installed") { + t.Errorf("the rejection should name the engine:\n%s", res.Explain()) + } +} + +func TestANodeThatReportsNoEnginesIsStillEligible(t *testing.T) { + // Every agent older than the probe reports nothing. Excluding those would + // empty a fleet on a control plane upgrade, which is a worse failure than + // the one being fixed. + old := node("older-agent", "h100-80g", 8) + + s := New(store(old)) + res := s.Place([]Profile{{ + Name: "tp1", Engine: "vllm", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, + }}, nil) + if res.Placement == nil { + t.Errorf("an agent that says nothing must not be excluded:\n%s", res.Explain()) + } +} diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go index 4b871c0..61e83b4 100644 --- a/internal/web/admin_serve.go +++ b/internal/web/admin_serve.go @@ -75,14 +75,19 @@ func planServe(st mirror.Status, files []mirror.Status, base string, f.Profile = "cpu" default: f.Profile = "gpu" - // The control plane is not told which engines a node has installed, so - // this is the nearest thing it can check honestly. It is also the thing - // that was wrong here: fifteen gigabytes of safetensors arrived before - // anybody noticed the fleet had no card to run them on. - if gpus(fleet) == 0 { - f.Warning = "no node in this fleet reports a GPU, and " + - extOf(st.File) + " weights need vLLM or SGLang, which do. " + - "A GGUF build of the same model runs on the CPU nodes" + } + // Whether any node can actually run it. Agents report their installed + // engines now, so this is a fact rather than the guess it started as — + // which was cards, and would have passed a GPU node with no vLLM on it. + // + // A warning, not a refusal: an engine can be installed this afternoon, and + // a node that reports nothing is an older agent rather than an empty one. + if !runsAnywhere(fleet, f.Engine) { + f.Warning = "no node reports having " + f.Engine + " installed, and " + + extOf(st.File) + " weights need it" + if f.Engine != "llamacpp" && gpus(fleet) == 0 { + f.Warning += ". This fleet has no GPU either; a GGUF build of the " + + "same model runs on the CPU nodes" } } if cat != nil && cat.Declared(st.Model) { @@ -111,6 +116,30 @@ func extOf(file string) string { return file } +// runsAnywhere reports whether some node has this engine. +// +// True when nothing is known, so a fleet of agents older than the engine probe +// is not told every model is unservable. That matches what placement does with +// the same absence, and the two disagreeing would be worse than either. +func runsAnywhere(fleet Source, engine string) bool { + if fleet == nil || engine == "" { + return true + } + anyReported := false + for _, n := range fleet.Nodes() { + if len(n.Engines) == 0 { + continue + } + anyReported = true + for _, e := range n.Engines { + if e == engine { + return true + } + } + } + return !anyReported +} + func gpus(fleet Source) int { if fleet == nil { return 0 diff --git a/internal/web/admin_serve_test.go b/internal/web/admin_serve_test.go index 6f74e96..147ba00 100644 --- a/internal/web/admin_serve_test.go +++ b/internal/web/admin_serve_test.go @@ -28,11 +28,14 @@ func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error { } func (f *fakeCatalog) Declared(name string) bool { return name == f.declared } -// fleet reports a cluster with a given number of cards. -type fakeFleet struct{ gpus int } +// fleet reports a cluster with a given number of cards and engines. +type fakeFleet struct { + gpus int + engines []string +} func (f fakeFleet) Nodes() []adminapi.NodeView { - return []adminapi.NodeView{{ID: "n1", GPUTotal: f.gpus}} + return []adminapi.NodeView{{ID: "n1", GPUTotal: f.gpus, Engines: f.engines}} } func (f fakeFleet) Models() []adminapi.ModelView { return nil } func (f fakeFleet) Replicas(string) []adminapi.ReplicaView { return nil } @@ -44,10 +47,13 @@ func held(model, version, file string) mirror.Status { State: mirror.Present, BytesDone: 10, BytesTotal: 10} } -func servePage(t *testing.T, models []mirror.Status, cat CatalogAdmin, gpus int) *http.ServeMux { +func servePage(t *testing.T, models []mirror.Status, cat CatalogAdmin, gpus int, + engines ...string, +) *http.ServeMux { t.Helper() return mirrorPageDeps(t, MirrorDeps{ - Mirror: &fakeMirror{models: models}, Catalog: cat, Fleet: fakeFleet{gpus: gpus}, + Mirror: &fakeMirror{models: models}, Catalog: cat, + Fleet: fakeFleet{gpus: gpus, engines: engines}, }, true) } @@ -88,14 +94,19 @@ func TestAShardedVersionIsRefusedWithTheReason(t *testing.T) { func TestSafetensorsOnACPUFleetWarnsBeforeAnybodyTries(t *testing.T) { body := mirrorGet(t, servePage(t, []mirror.Status{ - held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 0), + held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 0, "llamacpp"), "/admin/mirror?serve=qwen:v1").Body.String() // Fifteen gigabytes arrived before anybody noticed the fleet had no card - // to run them on. The page can say so. - if !strings.Contains(body, "no node in this fleet reports a GPU") { + // and no vLLM to run them with. The page can say so. + if !strings.Contains(body, "no node reports having vllm installed") { t.Errorf("expected the warning; got %s", body) } + // And the way out, which on a fleet with no cards is a different build of + // the same model rather than installing vLLM. + if !strings.Contains(body, "GGUF build of the same model") { + t.Error("a fleet with no GPU should be pointed at a GGUF") + } // A warning, not a refusal: a card could be added tomorrow. if !strings.Contains(body, `action="/admin/mirror/serve"`) { t.Error("it should still be possible to add it") @@ -104,11 +115,11 @@ func TestSafetensorsOnACPUFleetWarnsBeforeAnybodyTries(t *testing.T) { func TestSafetensorsOnAGPUFleetIsNotWarnedAbout(t *testing.T) { body := mirrorGet(t, servePage(t, []mirror.Status{ - held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 8), + held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 8, "vllm"), "/admin/mirror?serve=qwen:v1").Body.String() - if strings.Contains(body, "reports a GPU") { - t.Error("a fleet with cards should not be warned about") + if strings.Contains(body, "no node reports having") { + t.Error("a fleet that runs vllm should not be warned about") } if !strings.Contains(body, `value="vllm"`) { t.Error("safetensors should propose vllm") @@ -196,3 +207,29 @@ func TestWithNoCatalogThereIsNoServeAction(t *testing.T) { t.Error("serving should not be offered without somewhere to persist it") } } + +func TestAGPUNodeWithoutVLLMIsStillWarnedAbout(t *testing.T) { + // The first version of this checked cards, which would have waved through + // a fleet of eight-GPU nodes that had never had vLLM installed. Agents + // report their engines now, so it is a fact. + body := mirrorGet(t, servePage(t, []mirror.Status{ + held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 8, "llamacpp"), + "/admin/mirror?serve=qwen:v1").Body.String() + + if !strings.Contains(body, "no node reports having vllm installed") { + t.Errorf("cards are not the question; the engine is. Got %s", body) + } +} + +func TestAFleetOfOlderAgentsIsNotToldEverythingIsUnservable(t *testing.T) { + // An agent older than the engine probe reports nothing, which is not the + // same as having no engines. Placement makes the same allowance, and the + // two disagreeing would be worse than either. + body := mirrorGet(t, servePage(t, []mirror.Status{ + held("qwen", "v1", "m.gguf")}, &fakeCatalog{}, 0), + "/admin/mirror?serve=qwen:v1").Body.String() + + if strings.Contains(body, "no node reports having") { + t.Error("silence from older agents must not read as a refusal") + } +}