diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 4d2e8c8..6460d34 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -209,6 +209,14 @@ type Agent struct { reg *engines.Registry http connectHTTPClient + // engineOnce probes the installed engines exactly once. + // + // Once, because probing runs each engine's --version and vLLM's imports + // torch — seconds on a cold cache. A reconnect is not a reason to pay that + // again, and what is installed does not change while the process runs. + engineOnce sync.Once + engineVers map[string]string + mu sync.Mutex inv discovery.Inventory // shardReplica remembers which replica a shard belongs to, so a status @@ -567,6 +575,11 @@ func (a *Agent) session(ctx context.Context, server string) error { // Every registration reports what is already running, not just the first // one: a reconnect may be a control plane restart, and it has to recover // the same way (plan §6.5). + // What this node can actually run, so the control plane stops placing + // engines that are not here. The registry holds an adapter for every + // engine whether or not its binary exists, so without this a node with no + // vLLM accepted a vLLM shard and failed it minutes later. + reg.EngineVersions = a.engines(parent) reg.RunningShards = a.runningShards() if err := stream.Send(&llmgwv1.AgentMessage{ @@ -1018,3 +1031,18 @@ func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) { } return &http.Client{Transport: &http2.Transport{TLSClientConfig: tc}}, nil } + +// engines is what this node can run, probed once. +// +// Failures are silence rather than an error: an engine that is not installed is +// the ordinary case on most nodes, and an agent that refused to register over +// one would take a working node out of the fleet for a missing engine it was +// never going to be asked to run. +func (a *Agent) engines(ctx context.Context) map[string]string { + a.engineOnce.Do(func() { + a.engineVers = a.reg.Probe(ctx) + a.log.Info("engines available on this node", + "engines", engines.ProbedNames(a.engineVers)) + }) + return a.engineVers +} diff --git a/internal/agent/engines/probe.go b/internal/agent/engines/probe.go new file mode 100644 index 0000000..7e9819a --- /dev/null +++ b/internal/agent/engines/probe.go @@ -0,0 +1,190 @@ +package engines + +import ( + "context" + "errors" + "os/exec" + "regexp" + "sort" + "strings" + "time" +) + +// Unknown is the version reported for an engine that is installed and would +// not say which version it is. +// +// A value, not an absent key, because the two mean different things and +// placement reads the keys: absent is "this node cannot run it", Unknown is +// "it is here and the probe could not get a number out of it". Collapsing them +// would make a slow engine look like a missing one and take a node out of +// scheduling for being slow to answer --version. +const Unknown = "unknown" + +// Prober is an engine that can say whether it is installed on this machine. +// +// Optional, and asserted for rather than part of Engine: an adapter that has no +// way to check should report nothing rather than be forced to invent an answer. +type Prober interface { + // Probe returns the engine's version, or an error when it is not installed. + Probe(ctx context.Context) (string, error) +} + +// Probe asks every registered engine whether it is here. +// +// The result is what the node reports at registration, and its key set is what +// placement filters on. Until this existed nobody knew: the registry registers +// every adapter whether or not its binary exists, so a node with no vLLM +// accepted a vLLM shard, registered healthy, and failed the shard minutes later +// with "executable file not found" — in the engine's own words, on a machine +// nobody was watching. +// +// Bounded, because it runs on the path to registering and an agent that cannot +// reach the control plane is worse than one that reports an incomplete list. +func (r *Registry) Probe(ctx context.Context) map[string]string { + out := map[string]string{} + for name, e := range r.engines { + p, ok := e.(Prober) + if !ok { + continue + } + v, err := p.Probe(ctx) + if err != nil { + continue + } + if v == "" { + v = Unknown + } + out[name] = v + } + return out +} + +// Names of the engines in a probe result, sorted. +func ProbedNames(m map[string]string) []string { + out := make([]string, 0, len(m)) + for n := range m { + out = append(out, n) + } + sort.Strings(out) + return out +} + +// probeTimeout bounds one engine's version check. +// +// Generous, because vLLM's --version imports torch and that is seconds on a +// cold page cache, and mean, because this is startup: an engine that has not +// answered in this long is reported as present-with-unknown-version rather +// than waited for. +const probeTimeout = 8 * time.Second + +// run executes a version command, saying whether the program was even there. +// +// The two failures are different answers and the caller has to tell them +// apart. A binary that is absent is an engine this node cannot run. A binary +// that is present and exits non-zero is an engine it probably can — unless the +// command was a test of whether a module imports, where a non-zero exit is the +// whole answer. +func run(ctx context.Context, bin string, args ...string) (out string, found bool, ok bool) { + path, err := exec.LookPath(bin) + if err != nil { + return "", false, false + } + ctx, cancel := context.WithTimeout(ctx, probeTimeout) + defer cancel() + + // Combined, not just stdout: engines print their version to either, and one + // that chose stderr would otherwise read as having no version at all. + b, err := exec.CommandContext(ctx, path, args...).CombinedOutput() + return string(b), true, err == nil +} + +// installedBinary reports an engine that is its own executable. +// +// It resolves the path and runs nothing. The first version of this ran the +// engine with --version, which is fine for an engine and not fine for the +// wrapper scripts people put in front of one: a script that shifts past flags +// it does not recognise and then execs the server will start an inference +// server, at agent startup, on every node. The agent's own test suite installs +// exactly such a script, and caught it. +// +// So the version is not read here. Presence is what placement needs, and what +// the version is for — telling whether an engine still accepts the flags llmgw +// builds — is `llmgw ctl engine-check`, which is run deliberately by somebody +// who meant to run the engine. +func installedBinary(bin string) (string, error) { + if _, err := exec.LookPath(bin); err != nil { + return "", errNotInstalled + } + return Unknown, nil +} + +// versionOfModule probes an engine that is a module inside an interpreter. +// +// This one does execute, and safely: the command is llmgw's own one-line +// import, not the engine's launcher, so there is no wrapper to shift past +// flags and start a server. There is also no alternative — a module has no +// path to look up. +// +// A non-zero exit is the answer: python3 exists on almost every machine, so +// presence of the interpreter says nothing. Treating a failed import as +// "installed" reported SGLang on every node in a fleet that had none, which +// would take nodes into scheduling for an engine they cannot run — the exact +// failure this whole change exists to stop. +func versionOfModule(ctx context.Context, python string, args ...string) (string, error) { + out, found, ok := run(ctx, python, args...) + if !found || !ok { + return "", errNotInstalled + } + return firstVersion(out), nil +} + +var errNotInstalled = errors.New("not installed on this node") + +var versionPattern = regexp.MustCompile(`\b\d+\.\d+(\.\d+)?\S*`) + +// firstVersion is the first thing in the output that looks like a version. +// +// Pattern rather than parsing each engine's format: they print "vllm 0.6.3", +// "version: b4123 (…)", and a paragraph with the number in the middle, and all +// of those change between releases. What this is for is telling one build from +// another, which the first number does. +func firstVersion(s string) string { + for _, line := range strings.Split(s, "\n") { + if m := versionPattern.FindString(line); m != "" { + return strings.TrimRight(m, ",;)") + } + } + return Unknown +} + +// Probe reports whether the llama.cpp server is installed. +func (l *LlamaCpp) Probe(context.Context) (string, error) { + bin := l.cfg.Bin + if bin == "" { + bin = "llama-server" + } + return installedBinary(bin) +} + +// Probe reports whether vLLM is installed. +func (v *VLLM) Probe(context.Context) (string, error) { + bin := v.cfg.Bin + if bin == "" { + bin = "vllm" + } + return installedBinary(bin) +} + +// Probe reports the SGLang version. +// +// Through the interpreter, because SGLang is a module rather than a binary: +// asking the shell for "sglang" finds nothing on a machine that can run it +// perfectly well. +func (g *SGLang) Probe(ctx context.Context) (string, error) { + python := g.cfg.Python + if python == "" { + python = "python3" + } + return versionOfModule(ctx, python, "-c", + "import sglang, sys; sys.stdout.write(getattr(sglang, '__version__', ''))") +} diff --git a/internal/agent/engines/probe_test.go b/internal/agent/engines/probe_test.go new file mode 100644 index 0000000..3d1175b --- /dev/null +++ b/internal/agent/engines/probe_test.go @@ -0,0 +1,107 @@ +package engines + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +// script writes an executable that stands in for an engine. +func script(t *testing.T, name, body string) string { + t.Helper() + dir := t.TempDir() + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte("#!/bin/sh\n"+body+"\n"), 0o755); err != nil { + t.Fatal(err) + } + return p +} + +func TestAnEngineThatIsNotInstalledIsAbsentFromTheReport(t *testing.T) { + r := NewRegistry(Config{}) + r.engines = map[string]Engine{ + "vllm": &VLLM{cfg: VLLMConfig{Bin: "/nonexistent/vllm"}}, + } + // Absent, not present-with-unknown-version. Placement reads the keys, and + // a node listed for an engine it cannot run is the failure being fixed. + if got := r.Probe(t.Context()); len(got) != 0 { + t.Errorf("expected nothing, got %v", got) + } +} + +func TestProbingNeverRunsTheEngine(t *testing.T) { + // A wrapper that shifts past flags it does not recognise and then starts + // the server. People put these in front of engines, and the agent's own + // tests install one. Probing with --version would launch an inference + // server at agent startup, on every node. + marker := filepath.Join(t.TempDir(), "ran") + bin := script(t, "fake-vllm", "touch "+marker+"; sleep 60") + + r := NewRegistry(Config{}) + r.engines = map[string]Engine{"vllm": &VLLM{cfg: VLLMConfig{Bin: bin}}} + + got := r.Probe(t.Context()) + if _, ok := got["vllm"]; !ok { + t.Error("a binary that is there should be reported present") + } + if _, err := os.Stat(marker); err == nil { + t.Fatal("the engine was executed by a probe") + } +} + +func TestAModuleThatWillNotImportIsNotInstalled(t *testing.T) { + // python3 is on almost every machine, so the interpreter being there says + // nothing about SGLang. Reporting it present put every node in a fleet + // forward for an engine none of them had. + bin := script(t, "python3", "exit 1") + r := NewRegistry(Config{}) + r.engines = map[string]Engine{"sglang": &SGLang{cfg: SGLangConfig{Python: bin}}} + + if got := r.Probe(t.Context()); len(got) != 0 { + t.Errorf("a failed import means not installed, got %v", got) + } +} + +func TestAModuleThatImportsIsReportedWithItsVersion(t *testing.T) { + bin := script(t, "python3", `echo "0.4.1"`) + r := NewRegistry(Config{}) + r.engines = map[string]Engine{"sglang": &SGLang{cfg: SGLangConfig{Python: bin}}} + + got := r.Probe(t.Context()) + if got["sglang"] != "0.4.1" { + t.Errorf("expected the version, got %v", got) + } +} + +func TestAVersionIsPickedOutOfWhateverIsPrinted(t *testing.T) { + // Only the module probe reaches this now, and a Python package version is + // dotted. A bare build number is deliberately not matched: it would turn a + // stray number in a line of prose into a version. + for in, want := range map[string]string{ + "0.4.1": "0.4.1", + "sglang 0.4.post1": "0.4.post1", + "warning: noise\n0.4.1": "0.4.1", + "": Unknown, + "no numbers here": Unknown, + "just 4123": Unknown, + } { + if got := firstVersion(in); got != want { + t.Errorf("firstVersion(%q) = %q, want %q", in, got, want) + } + } +} + +func TestProbingIsBoundedByTheContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + bin := script(t, "python3", "sleep 60") + r := NewRegistry(Config{}) + r.engines = map[string]Engine{"sglang": &SGLang{cfg: SGLangConfig{Python: bin}}} + + // Registration waits on this. An agent that cannot reach the control plane + // is worse than one reporting an incomplete list. + if got := r.Probe(ctx); len(got) != 0 { + t.Errorf("a cancelled probe should report nothing, got %v", got) + } +} diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index a8dd6fa..b6961bf 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -175,11 +175,15 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { } w := tab() - fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tDISK\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN") + fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tDISK\tENGINES\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN") for _, n := range nodes { - fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%s\t%d\t%s\n", + fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n", n.ID, n.State, n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB), diskColumn(n.DiskFreeGB, n.DiskTotalGB), + // What the node can run, which is the first thing to look + // at when a model will not place and the hardware looks + // fine. + join(n.Engines), join(n.Classes), join(n.Fabrics), n.Replicas, n.LastSeenAgo) } _ = w.Flush() diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index ad6c3c7..9303f57 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -73,6 +73,12 @@ type NodeView struct { // what decides whether a CPU profile can be placed here, so it belongs // where someone looks when a model will not schedule. MemoryGB int `json:"memory_gb,omitempty"` + // Engines are the inference engines installed here, as the agent found + // them. Absent means the agent is older than the probe and said nothing, + // which is not the same as having none — placement treats it that way too, + // or a control plane upgrade would empty a fleet. + Engines []string `json:"engines,omitempty"` + // Models are the weights this node can see. Absent means it was not asked // to look, which is not the same as holding none. Models []string `json:"models_on_disk,omitempty"` @@ -211,6 +217,10 @@ func (a *API) Nodes() []NodeView { v.DiskFreeGB = int(free / (1 << 30)) v.DiskTotalGB = int(total / (1 << 30)) } + for name := range n.EngineVersions { + v.Engines = append(v.Engines, name) + } + slices.Sort(v.Engines) for _, w := range n.Weights { v.Models = append(v.Models, w.Path) } diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index 951381e..34b1424 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -394,6 +394,21 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje reject("does not carry " + miss) continue } + // The engine has to be installed here. The field has been on this + // struct since placement was written and was never read, so a node + // with no vLLM was as good a candidate as one with it: the replica was + // placed, the node registered healthy, and the shard failed minutes + // later with "executable file not found" in the engine's own words. + // + // A node that reports no engines at all is not excluded. That is what + // every agent older than this reports, and emptying a fleet on a + // control plane upgrade is a worse failure than the one being fixed. + if p.Engine != "" && len(n.EngineVersions) > 0 { + if _, ok := n.EngineVersions[p.Engine]; !ok { + reject("does not have " + p.Engine + " installed") + continue + } + } // A node that does not report its memory is not excluded: an agent // older than that field would otherwise have its node quietly emptied // by a control plane upgrade, which is a worse failure than the one diff --git a/internal/control/placement/placement_test.go b/internal/control/placement/placement_test.go index c66c432..91dfd2f 100644 --- a/internal/control/placement/placement_test.go +++ b/internal/control/placement/placement_test.go @@ -679,3 +679,62 @@ func TestAProfileWithNoSourceIsUnaffected(t *testing.T) { t.Fatalf("nothing placed:\n%s", res.Explain()) } } + +// withEngines is a node that reports which engines it has. +func withEngines(id string, gpus int, names ...string) inventory.Node { + n := node(id, "h100-80g", gpus) + n.EngineVersions = map[string]string{} + for _, name := range names { + n.EngineVersions[name] = "unknown" + } + return n +} + +func TestAReplicaIsNotPlacedOnANodeWithoutItsEngine(t *testing.T) { + // The engine has been on the Profile struct since placement was written + // and was never read, so a node with no vLLM was as good a candidate as + // one with it: the replica was placed, the node registered healthy, and + // the shard failed minutes later with "executable file not found". + s := New(store(withEngines("no-vllm", 8, "llamacpp"), withEngines("has-vllm", 8, "vllm"))) + + res := s.Place([]Profile{{ + Name: "tp1", Engine: "vllm", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, + }}, nil) + if res.Placement == nil { + t.Fatalf("nothing was placed:\n%s", res.Explain()) + } + if got := res.Placement.Nodes[0].NodeID; got != "has-vllm" { + t.Errorf("placed on %q, which does not have vllm", got) + } +} + +func TestAModelWithNoNodeRunningItsEngineSaysSo(t *testing.T) { + s := New(store(withEngines("cpu-only", 8, "llamacpp"))) + + res := s.Place([]Profile{{ + Name: "tp1", Engine: "vllm", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, + }}, nil) + if res.Placement != nil { + t.Fatal("a model whose engine is installed nowhere should not be placed") + } + // The reason belongs in explain-placement, or the answer to "why is this + // not running" is a retry loop in the log. + if !strings.Contains(res.Explain(), "does not have vllm installed") { + t.Errorf("the rejection should name the engine:\n%s", res.Explain()) + } +} + +func TestANodeThatReportsNoEnginesIsStillEligible(t *testing.T) { + // Every agent older than the probe reports nothing. Excluding those would + // empty a fleet on a control plane upgrade, which is a worse failure than + // the one being fixed. + old := node("older-agent", "h100-80g", 8) + + s := New(store(old)) + res := s.Place([]Profile{{ + Name: "tp1", Engine: "vllm", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, + }}, nil) + if res.Placement == nil { + t.Errorf("an agent that says nothing must not be excluded:\n%s", res.Explain()) + } +} diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go index 4b871c0..61e83b4 100644 --- a/internal/web/admin_serve.go +++ b/internal/web/admin_serve.go @@ -75,14 +75,19 @@ func planServe(st mirror.Status, files []mirror.Status, base string, f.Profile = "cpu" default: f.Profile = "gpu" - // The control plane is not told which engines a node has installed, so - // this is the nearest thing it can check honestly. It is also the thing - // that was wrong here: fifteen gigabytes of safetensors arrived before - // anybody noticed the fleet had no card to run them on. - if gpus(fleet) == 0 { - f.Warning = "no node in this fleet reports a GPU, and " + - extOf(st.File) + " weights need vLLM or SGLang, which do. " + - "A GGUF build of the same model runs on the CPU nodes" + } + // Whether any node can actually run it. Agents report their installed + // engines now, so this is a fact rather than the guess it started as — + // which was cards, and would have passed a GPU node with no vLLM on it. + // + // A warning, not a refusal: an engine can be installed this afternoon, and + // a node that reports nothing is an older agent rather than an empty one. + if !runsAnywhere(fleet, f.Engine) { + f.Warning = "no node reports having " + f.Engine + " installed, and " + + extOf(st.File) + " weights need it" + if f.Engine != "llamacpp" && gpus(fleet) == 0 { + f.Warning += ". This fleet has no GPU either; a GGUF build of the " + + "same model runs on the CPU nodes" } } if cat != nil && cat.Declared(st.Model) { @@ -111,6 +116,30 @@ func extOf(file string) string { return file } +// runsAnywhere reports whether some node has this engine. +// +// True when nothing is known, so a fleet of agents older than the engine probe +// is not told every model is unservable. That matches what placement does with +// the same absence, and the two disagreeing would be worse than either. +func runsAnywhere(fleet Source, engine string) bool { + if fleet == nil || engine == "" { + return true + } + anyReported := false + for _, n := range fleet.Nodes() { + if len(n.Engines) == 0 { + continue + } + anyReported = true + for _, e := range n.Engines { + if e == engine { + return true + } + } + } + return !anyReported +} + func gpus(fleet Source) int { if fleet == nil { return 0 diff --git a/internal/web/admin_serve_test.go b/internal/web/admin_serve_test.go index 6f74e96..147ba00 100644 --- a/internal/web/admin_serve_test.go +++ b/internal/web/admin_serve_test.go @@ -28,11 +28,14 @@ func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error { } func (f *fakeCatalog) Declared(name string) bool { return name == f.declared } -// fleet reports a cluster with a given number of cards. -type fakeFleet struct{ gpus int } +// fleet reports a cluster with a given number of cards and engines. +type fakeFleet struct { + gpus int + engines []string +} func (f fakeFleet) Nodes() []adminapi.NodeView { - return []adminapi.NodeView{{ID: "n1", GPUTotal: f.gpus}} + return []adminapi.NodeView{{ID: "n1", GPUTotal: f.gpus, Engines: f.engines}} } func (f fakeFleet) Models() []adminapi.ModelView { return nil } func (f fakeFleet) Replicas(string) []adminapi.ReplicaView { return nil } @@ -44,10 +47,13 @@ func held(model, version, file string) mirror.Status { State: mirror.Present, BytesDone: 10, BytesTotal: 10} } -func servePage(t *testing.T, models []mirror.Status, cat CatalogAdmin, gpus int) *http.ServeMux { +func servePage(t *testing.T, models []mirror.Status, cat CatalogAdmin, gpus int, + engines ...string, +) *http.ServeMux { t.Helper() return mirrorPageDeps(t, MirrorDeps{ - Mirror: &fakeMirror{models: models}, Catalog: cat, Fleet: fakeFleet{gpus: gpus}, + Mirror: &fakeMirror{models: models}, Catalog: cat, + Fleet: fakeFleet{gpus: gpus, engines: engines}, }, true) } @@ -88,14 +94,19 @@ func TestAShardedVersionIsRefusedWithTheReason(t *testing.T) { func TestSafetensorsOnACPUFleetWarnsBeforeAnybodyTries(t *testing.T) { body := mirrorGet(t, servePage(t, []mirror.Status{ - held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 0), + held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 0, "llamacpp"), "/admin/mirror?serve=qwen:v1").Body.String() // Fifteen gigabytes arrived before anybody noticed the fleet had no card - // to run them on. The page can say so. - if !strings.Contains(body, "no node in this fleet reports a GPU") { + // and no vLLM to run them with. The page can say so. + if !strings.Contains(body, "no node reports having vllm installed") { t.Errorf("expected the warning; got %s", body) } + // And the way out, which on a fleet with no cards is a different build of + // the same model rather than installing vLLM. + if !strings.Contains(body, "GGUF build of the same model") { + t.Error("a fleet with no GPU should be pointed at a GGUF") + } // A warning, not a refusal: a card could be added tomorrow. if !strings.Contains(body, `action="/admin/mirror/serve"`) { t.Error("it should still be possible to add it") @@ -104,11 +115,11 @@ func TestSafetensorsOnACPUFleetWarnsBeforeAnybodyTries(t *testing.T) { func TestSafetensorsOnAGPUFleetIsNotWarnedAbout(t *testing.T) { body := mirrorGet(t, servePage(t, []mirror.Status{ - held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 8), + held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 8, "vllm"), "/admin/mirror?serve=qwen:v1").Body.String() - if strings.Contains(body, "reports a GPU") { - t.Error("a fleet with cards should not be warned about") + if strings.Contains(body, "no node reports having") { + t.Error("a fleet that runs vllm should not be warned about") } if !strings.Contains(body, `value="vllm"`) { t.Error("safetensors should propose vllm") @@ -196,3 +207,29 @@ func TestWithNoCatalogThereIsNoServeAction(t *testing.T) { t.Error("serving should not be offered without somewhere to persist it") } } + +func TestAGPUNodeWithoutVLLMIsStillWarnedAbout(t *testing.T) { + // The first version of this checked cards, which would have waved through + // a fleet of eight-GPU nodes that had never had vLLM installed. Agents + // report their engines now, so it is a fact. + body := mirrorGet(t, servePage(t, []mirror.Status{ + held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 8, "llamacpp"), + "/admin/mirror?serve=qwen:v1").Body.String() + + if !strings.Contains(body, "no node reports having vllm installed") { + t.Errorf("cards are not the question; the engine is. Got %s", body) + } +} + +func TestAFleetOfOlderAgentsIsNotToldEverythingIsUnservable(t *testing.T) { + // An agent older than the engine probe reports nothing, which is not the + // same as having no engines. Placement makes the same allowance, and the + // two disagreeing would be worse than either. + body := mirrorGet(t, servePage(t, []mirror.Status{ + held("qwen", "v1", "m.gguf")}, &fakeCatalog{}, 0), + "/admin/mirror?serve=qwen:v1").Body.String() + + if strings.Contains(body, "no node reports having") { + t.Error("silence from older agents must not read as a refusal") + } +}