From 590e5b9f3e06062b838af70ff960caff2aaaed41 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Tue, 29 Sep 2026 21:03:03 +0200 Subject: [PATCH] Ask a node only for engines it can install; doctor says when AVX2 is hidden A control plane of this release asks every node for vLLM. An agent of the last one does not know how to install it, and answered "unsupported", so a cluster half way through its upgrade showed every node as failing. Agents now say which engines they can install (Register.installable_engines), and are asked for no others; an agent that says nothing is asked for llama.cpp alone, as it always was. doctor reports the CPU's vector extensions, and warns when AVX2 is missing, with the fix for a VM: the test node for node add was one, on QEMU's default CPU model, where llama.cpp runs several times slower and nothing else looks wrong. node add shows it, since it runs doctor. The changelog's unreleased section was two releases' worth of sections one after the other, with two lists of fixes and the upgrade notes in the middle. It is one entry now: largest changes first, the fixes together, and every upgrade note at the end, with vLLM's default install and how to keep your own. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 328 +++++++++--------- api/gen/llmgw/v1/agent.pb.go | 24 +- api/proto/llmgw/v1/agent.proto | 7 + internal/agent/agent.go | 1 + internal/agent/provision/recipe.go | 14 + internal/control/agentsvc/installable_test.go | 27 ++ internal/control/agentsvc/service.go | 15 + internal/doctor/doctor.go | 18 + internal/doctor/doctor_test.go | 27 ++ 9 files changed, 295 insertions(+), 166 deletions(-) create mode 100644 internal/control/agentsvc/installable_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e6a2a6ba..1d1e6176 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ Until v1.0.0 the minor version is the breaking one: configuration formats and Go APIs change between minors without a deprecation period. ## Unreleased +Independent InfiniBand fabrics and NVLink domains, told apart, drawn and +placed on; a node checked by `llmgw doctor` and added by `llmgw node add`; vLLM +installed by the nodes themselves; people in many places at once; and a chat +page, editor setup and a model explorer for the people using it. ### Independent InfiniBand fabrics, told apart and drawn @@ -100,55 +104,36 @@ interface NCCL will use, the unit's memlock limit, and the NCCL environment a shard on each fabric would be given. `--json` writes every fact it read, to send back; it exits non-zero when something fails. -### Each node's CPU and memory, and a page per node +It also says when the CPU offers no AVX2 — a VM left on QEMU's default CPU +model, where llama.cpp runs several times slower and nothing else looks wrong — +and whether vLLM can be installed on the node, and why not. -The agent reads `/proc` between heartbeats: CPU across all cores, load, memory -used and available, and each engine's cores and resident memory. The cluster -page's node table shows CPU and RAM as bars; each node has a page with its -current load, its engines, its fabric ports, its GPUs, and an hour of charts -from Prometheus. +### vLLM installs itself on GPU nodes -### Connect an editor, and the Responses API +The agent installs vLLM as it does llama.cpp: on every node with an NVIDIA GPU +that can run it, with nothing done by hand. A Python 3.12 environment is built +by uv — itself a GitHub release checked against its checksum — from PyPI, +resolved as PyPI stood a day after that vLLM was published and installed with +`--require-hashes`, so every node gets the same files whenever it is built. The +resolved list stays in the environment as `llmgw-requirements.txt`. -- The account page makes a key for **Continue**, **Codex** or **Cline** and - shows what the tool needs, filled in: Continue's assistant file, Codex's - provider and profile, Cline's four settings. Continue and Codex get a one-line - command (`curl -fsSL /setup/?model=… | sh`) whose script holds - no secret — the key comes from the environment — keeps a copy of whatever it - replaces, and leaves the rest of the file alone. -- **`POST /v1/responses`** translates the Responses API, which Codex speaks, to - chat completions and back — instructions, messages, function and custom - tools with their calls and outputs, usage, streamed or not — down the same - path as any request. +- The default is vLLM 0.30.0, which takes every flag llmgw passes. Set + `engines.vllm` in `server.yaml` to change it, or to `""` to keep your own. +- It is built for CUDA 13: it needs driver 580 and a Turing GPU or newer, and + 25 GB free while it installs. A node that lacks one says which on the + cluster page; a node with no GPU skips vLLM without a word. +- gcc is installed with it, since Triton compiles its kernel launcher on the + first model load. +- SGLang installs the same way, on GPU nodes, when `engines.sglang` names a + version. -### The chat page +Checked by installing vLLM 0.30.0 this way on a node of the test cluster: 7.7 GB, +torch 2.13 for CUDA 13.0, importing cleanly. That node has no GPU, so vLLM +itself was not started there. -- A model that is not running shows a progress bar while it loads — waiting - for a node, downloading, starting — and the message is sent once it is up, - rather than failing after 45 seconds with "the model is loading". -- A finished reply is rendered as markdown, by the server under the model - cards' rules, with its maths typeset by KaTeX: "$5 and $10" stays prose. -- Stop, while a reply streams; its tokens, speed and time to first token under - it; a system prompt and a temperature; and the conversation kept in the - browser across a reload — the gateway still keeps none. - -### The mirror page - -- **Explore models** opens on what is trending on Hugging Face, with views for - new and most downloaded, filters for task and size, and each repository's - parameters, architecture (✓ where the nodes' llama.cpp loads it), context, - base model and license. "Only what these nodes can run" is on by default. -- A repository opens beneath its row with its model card, and a download - started there starts in place, its progress followed in the table above. -- What the mirror holds is grouped by model family. -- An importance matrix published beside a model's quantisations is no longer - offered as a model. - -### Your own use - -The account page (was "Your keys") shows the last thirty days of a person's own -use and, for each place they spend, a bar for every budget that applies — their -own, or a project's shared with others, said as such. +A node is asked only for engines its agent says it can install. An agent older +than this release says nothing, and is asked for llama.cpp alone, so a cluster +half way through its upgrade does not show every node as failing vLLM. ### `llmgw node add`: a node in one command @@ -181,114 +166,6 @@ longer lists vLLM as skipped on every CPU node. Tried on an empty VM, from the terminal screen and plain: joined with llama.cpp ready in under a minute, and 17 seconds to add it again. -### vLLM installs itself on GPU nodes - -The agent installs vLLM as it does llama.cpp: on every node with an NVIDIA GPU -that can run it, with nothing done by hand. A Python 3.12 environment is built -by uv — itself a GitHub release checked against its checksum — from PyPI, -resolved as PyPI stood a day after that vLLM was published and installed with -`--require-hashes`, so every node gets the same files whenever it is built. The -resolved list stays in the environment as `llmgw-requirements.txt`. - -- The default is vLLM 0.30.0, which takes every flag llmgw passes. Set - `engines.vllm` in `server.yaml` to change it, or to `""` to keep your own. -- It is built for CUDA 13: it needs driver 580 and a Turing GPU or newer, and - 25 GB free while it installs. A node that lacks one says which on the - cluster page; a node with no GPU skips vLLM without a word. -- gcc is installed with it, since Triton compiles its kernel launcher on the - first model load. -- SGLang installs the same way, on GPU nodes, when `engines.sglang` names a - version. -- `llmgw doctor` says whether vLLM can be installed on the node, and why not. - -Checked by installing vLLM 0.30.0 this way on a node of the test cluster: 7.7 GB, -torch 2.13 for CUDA 13.0, importing cleanly. That node has no GPU, so vLLM -itself was not started there. - -### The project's website, in every gateway - -`/about` is llmgw's own page: what it does, InfiniBand and NVLink, installing, -the reverse proxy configurations, and the release's files to download. It is -served without a key like `/`, whose footer links to it, and says nothing about -the installation. Its fonts are in the binary, so it makes no request elsewhere. -The published website is rendered from the same template. - -### The admin pages - -- The cluster page leads with capacity, nodes, models and replicas, then usage; - it refreshes in place instead of reloading, and its usage table names people - and projects instead of ids. -- The usage page's legend gives each line's total, not its last hour. -- The people page keeps an account on one line and its rarer actions in a menu. -- Every page fits a phone: a wide table scrolls in its own box. -- Static files are named by their content, so a deploy's new styles and - scripts reach browsers at once instead of a day later. - -### Metrics - -`llmgw_node_cpu_utilization_percent`, `llmgw_node_cpus`, `llmgw_node_load`, -`llmgw_node_memory_used_bytes`, `llmgw_node_memory_available_bytes`, -`llmgw_shard_cpu_cores`, `llmgw_shard_memory_bytes`, `llmgw_ib_port_active`, -`llmgw_ib_port_rate_gbps`, `llmgw_ib_port_counter_total`, -`llmgw_fabric_master_sm`, `llmgw_fabric_subnet_managers`, -`llmgw_fabric_problems`, `llmgw_gpu_nvlink_joined`, `llmgw_imex_up`. - -### Fixes - -- PostgreSQL queries with an apostrophe in a comment failed: the placeholder - rewriting read it as the start of a string. Comments are now skipped. -- Chart axes read "20.0BB" for sizes and 0% for a quiet CPU. - -### Upgrading - -Upgrade the control plane first, then the agents: an older agent is served as -before and simply reports none of the new measurements. Existing agent units -keep their old limits until `LimitMEMLOCK=infinity` is added to them. A node -that set `mgmt_interface` keeps it; one that did not now uses the interface -holding its management address rather than `eth0`. Multi-node vLLM replicas -start with `mp` unless `launch_mode: ray` is set. - -### An organization's own page - -A member of an organization can be made its **manager** (`users join --org … ---role manager`, or the star on the people page). A manager gets `/org`: the -organization's usage over time by team, project or person, its teams and -projects with limits and spend, and its members — read-only, and only what was -spent in its projects. Administrators can open every organization's page. - -### How a model runs, from the mirror page - -Each model a mirrored file is serving now shows how it runs — always on, on -demand, or stopped — and it can be changed there, without the CLI. The serve -form asks the same: on demand loads the model when somebody asks for it and -stops it after 15 minutes unused. A model declared in `server.yaml` shows its -mode and says to change it there, because the file is read on every start. - -### Weights the nodes cannot load are not fetched - -`kev-0.5b` was a GGUF whose architecture, `ggmlc`, llama.cpp does not know. It -was downloaded, catalogued and placed, and failed on every node it was tried on. -The architecture was in the first kilobytes of the file all along. Now: - -- The mirror page's registry listing shows each file's architecture and, for - one the nodes' llama.cpp cannot load, says so instead of offering to fetch - it. A vision projector (`mmproj`) is marked as not a model on its own. -- A fetch reads the file's header first — a range request, not a download — - and refuses one that cannot be served, unless "fetch anyway" is ticked. -- Serving from the mirror and `llmgw ctl model add` refuse the same. - -The list of architectures is llama.cpp's own (`LLM_ARCH_NAMES`) at the build -llmgw pins, b10991: 152 of them. A different build configured in `engines:` is -not refused, only not checked. - -### Fixes - -- Usage by team, project or organization, and the spend on the quotas page, - left out every request made on the chat page: they were joined through the - key, and the chat page has none. They now count where a request was spent. -- `/org` was answered with "no api key was provided" for a signed-in browser; - a test now drives every session page through the authentication middleware. - ### People belong to many places, or to none A person was in exactly one project, chosen at approval, and could not use the @@ -315,23 +192,104 @@ Leaving a project revokes the keys the person made there, unless another membership still reaches it. A personal key keeps no conversations or notes: those are kept per project. -**Upgrading.** Migration 0010 turns everyone's project into a membership and -attributes past usage to people where a key had an owner. On SQLite it rebuilds -the keys table; it was run against a copy of a production PostgreSQL store -before release. `llmgw users move` is gone: use `join` and `leave`. +### An organization's own page -### One menu on every page +A member of an organization can be made its **manager** (`users join --org … +--role manager`, or the star on the people page). A manager gets `/org`: the +organization's usage over time by team, project or person, its teams and +projects with limits and spend, and its members — read-only, and only what was +spent in its projects. Administrators can open every organization's page. -Each page drew its own menu, and no two agreed: different links, in a different +### The chat page + +- A model that is not running shows a progress bar while it loads — waiting + for a node, downloading, starting — and the message is sent once it is up, + rather than failing after 45 seconds with "the model is loading". +- A finished reply is rendered as markdown, by the server under the model + cards' rules, with its maths typeset by KaTeX: "$5 and $10" stays prose. +- Stop, while a reply streams; its tokens, speed and time to first token under + it; a system prompt and a temperature; and the conversation kept in the + browser across a reload — the gateway still keeps none. + +### Connect an editor, and the Responses API + +- The account page makes a key for **Continue**, **Codex** or **Cline** and + shows what the tool needs, filled in: Continue's assistant file, Codex's + provider and profile, Cline's four settings. Continue and Codex get a one-line + command (`curl -fsSL /setup/?model=… | sh`) whose script holds + no secret — the key comes from the environment — keeps a copy of whatever it + replaces, and leaves the rest of the file alone. +- **`POST /v1/responses`** translates the Responses API, which Codex speaks, to + chat completions and back — instructions, messages, function and custom + tools with their calls and outputs, usage, streamed or not — down the same + path as any request. + +### Your own use + +The account page (was "Your keys") shows the last thirty days of a person's own +use and, for each place they spend, a bar for every budget that applies — their +own, or a project's shared with others, said as such. + +### The mirror page + +- **Explore models** opens on what is trending on Hugging Face, with views for + new and most downloaded, filters for task and size, and each repository's + parameters, architecture (✓ where the nodes' llama.cpp loads it), context, + base model and license. "Only what these nodes can run" is on by default. +- A repository opens beneath its row with its model card, and a download + started there starts in place, its progress followed in the table above. +- What the mirror holds is grouped by model family. +- An importance matrix published beside a model's quantisations is no longer + offered as a model. + +**How a model runs.** Each model a mirrored file is serving now shows how it runs — always on, on +demand, or stopped — and it can be changed there, without the CLI. The serve +form asks the same: on demand loads the model when somebody asks for it and +stops it after 15 minutes unused. A model declared in `server.yaml` shows its +mode and says to change it there, because the file is read on every start. + +**Weights the nodes cannot load are not fetched.** `kev-0.5b` was a GGUF whose architecture, `ggmlc`, llama.cpp does not know. It +was downloaded, catalogued and placed, and failed on every node it was tried on. +The architecture was in the first kilobytes of the file all along. Now: + +- The mirror page's registry listing shows each file's architecture and, for + one the nodes' llama.cpp cannot load, says so instead of offering to fetch + it. A vision projector (`mmproj`) is marked as not a model on its own. +- A fetch reads the file's header first — a range request, not a download — + and refuses one that cannot be served, unless "fetch anyway" is ticked. +- Serving from the mirror and `llmgw ctl model add` refuse the same. + +The list of architectures is llama.cpp's own (`LLM_ARCH_NAMES`) at the build +llmgw pins, b10991: 152 of them. A different build configured in `engines:` is +not refused, only not checked. + +### Each node's CPU and memory, and a page per node + +The agent reads `/proc` between heartbeats: CPU across all cores, load, memory +used and available, and each engine's cores and resident memory. The cluster +page's node table shows CPU and RAM as bars; each node has a page with its +current load, its engines, its fabric ports, its GPUs, and an hour of charts +from Prometheus. + +### The admin pages + +- The cluster page leads with capacity, nodes, models and replicas, then usage; + it refreshes in place instead of reloading, and its usage table names people + and projects instead of ids. +- The usage page's legend gives each line's total, not its last hour. +- The people page keeps an account on one line and its rarer actions in a menu. +- Every page fits a phone: a wide table scrolls in its own box. +- Static files are named by their content, so a deploy's new styles and + scripts reach browsers at once instead of a day later. + +**One menu on every page.** Each page drew its own menu, and no two agreed: different links, in a different order, the page you were on sometimes listed and sometimes not. Every signed-in page now draws the same one — Cluster, People, Quotas, Usage, Mirror on the left for an administrator, Chat, Account and Sign out on the right for everyone — with the current page marked. The chat and key pages name the person's project by its path rather than its id. -### The quotas page says what actually applies - -A branch that sets no limit of its own used to read "—" in every column while +**The quotas page says what actually applies.** A branch that sets no limit of its own used to read "—" in every column while its organization held it to sixty requests a minute. Each limit now shows the value it is held to from above, muted and marked ↑, with where it comes from on hover; a larger value set below a stricter one says what caps it, for rate and @@ -342,6 +300,54 @@ form or saving one no longer reloads the page, and the explanations moved under the start of their path, which misplaced them when two organizations shared a name. +### The project's website, in every gateway + +`/about` is llmgw's own page: what it does, InfiniBand and NVLink, installing, +the reverse proxy configurations, and the release's files to download. It is +served without a key like `/`, whose footer links to it, and says nothing about +the installation. Its fonts are in the binary, so it makes no request elsewhere. +The published website is rendered from the same template. + +### Metrics + +`llmgw_node_cpu_utilization_percent`, `llmgw_node_cpus`, `llmgw_node_load`, +`llmgw_node_memory_used_bytes`, `llmgw_node_memory_available_bytes`, +`llmgw_shard_cpu_cores`, `llmgw_shard_memory_bytes`, `llmgw_ib_port_active`, +`llmgw_ib_port_rate_gbps`, `llmgw_ib_port_counter_total`, +`llmgw_fabric_master_sm`, `llmgw_fabric_subnet_managers`, +`llmgw_fabric_problems`, `llmgw_gpu_nvlink_joined`, `llmgw_imex_up`. + +### Fixes + +- PostgreSQL queries with an apostrophe in a comment failed: the placeholder + rewriting read it as the start of a string. Comments are now skipped. +- Chart axes read "20.0BB" for sizes and 0% for a quiet CPU. +- Usage by team, project or organization, and the spend on the quotas page, + left out every request made on the chat page: they were joined through the + key, and the chat page has none. They now count where a request was spent. +- `/org` was answered with "no api key was provided" for a signed-in browser; + a test now drives every session page through the authentication middleware. + +### Upgrading + +Upgrade the control plane first, then the agents: an older agent is served as +before and simply reports none of the new measurements. Existing agent units +keep their old limits until `LimitMEMLOCK=infinity` is added to them. A node +that set `mgmt_interface` keeps it; one that did not now uses the interface +holding its management address rather than `eth0`. Multi-node vLLM replicas +start with `mp` unless `launch_mode: ray` is set. + +The store migrates on its first start. Migration 0010 turns everyone's project into a membership and +attributes past usage to people where a key had an owner. On SQLite it rebuilds +the keys table; it was run against a copy of a production PostgreSQL store +before release. `llmgw users move` is gone: use `join` and `leave`. + +vLLM is installed on every GPU node that can run it, once its agent is +upgraded: about 8 GB each, from PyPI. Set `engines.vllm: ""` in `server.yaml` +before upgrading to keep vLLM your own. `deploy/bootstrap/onboard-node.sh` is +replaced by `llmgw node add`; it still works, for a machine holding the +authority but not this llmgw. + ## v0.9.2 — 2026-09-28 ### A new control plane serves a model by itself diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 65685a88..d8a29322 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -471,9 +471,15 @@ type Register struct { // built node with no engine installed yet was a candidate for every model. EnginesProbed bool `protobuf:"varint,14,opt,name=engines_probed,json=enginesProbed,proto3" json:"engines_probed,omitempty"` // imex is the service that lets a multi-node NVLink domain share memory. - Imex *IMEX `protobuf:"bytes,15,opt,name=imex,proto3" json:"imex,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Imex *IMEX `protobuf:"bytes,15,opt,name=imex,proto3" json:"imex,omitempty"` + // installable_engines are the engines this agent knows how to install + // itself. The control plane asks it for no others. Empty is an agent older + // than the field, which could install llama.cpp and nothing else: asked for + // vLLM, it answered "unsupported", and every node of a cluster half way + // through an upgrade showed as failing. + InstallableEngines []string `protobuf:"bytes,16,rep,name=installable_engines,json=installableEngines,proto3" json:"installable_engines,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Register) Reset() { @@ -611,6 +617,13 @@ func (x *Register) GetImex() *IMEX { return nil } +func (x *Register) GetInstallableEngines() []string { + if x != nil { + return x.InstallableEngines + } + return nil +} + // Host describes the operating system and processor of a node. type Host struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -3233,7 +3246,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x04logs\x18\a \x01(\v2\x15.llmgw.v1.LogsRequestH\x00R\x04logs\x12>\n" + "\vfabric_scan\x18\b \x01(\v2\x1b.llmgw.v1.FabricScanRequestH\x00R\n" + "fabricScanB\x05\n" + - "\x03msg\"\x96\x06\n" + + "\x03msg\"\xc7\x06\n" + "\bRegister\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" + "\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" + @@ -3250,7 +3263,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\astorage\x18\f \x03(\v2\x11.llmgw.v1.StorageR\astorage\x12\"\n" + "\x04host\x18\r \x01(\v2\x0e.llmgw.v1.HostR\x04host\x12%\n" + "\x0eengines_probed\x18\x0e \x01(\bR\renginesProbed\x12\"\n" + - "\x04imex\x18\x0f \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x1a9\n" + + "\x04imex\x18\x0f \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x12/\n" + + "\x13installable_engines\x18\x10 \x03(\tR\x12installableEngines\x1a9\n" + "\vLabelsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index f8876582..29bea3b2 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -118,6 +118,13 @@ message Register { bool engines_probed = 14; // imex is the service that lets a multi-node NVLink domain share memory. IMEX imex = 15; + + // installable_engines are the engines this agent knows how to install + // itself. The control plane asks it for no others. Empty is an agent older + // than the field, which could install llama.cpp and nothing else: asked for + // vLLM, it answered "unsupported", and every node of a cluster half way + // through an upgrade showed as failing. + repeated string installable_engines = 16; } // Host describes the operating system and processor of a node. diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 816582c7..5dc6342f 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -843,6 +843,7 @@ func (a *Agent) session(ctx context.Context, server string) error { // vLLM accepted a vLLM shard and failed it minutes later. reg.EngineVersions = a.engines(parent) reg.EnginesProbed = true + reg.InstallableEngines = provision.Installable() reg.RunningShards = a.runningShards() a.mu.Lock() h := a.prov.Host diff --git a/internal/agent/provision/recipe.go b/internal/agent/provision/recipe.go index 193711b0..e5d1fd87 100644 --- a/internal/agent/provision/recipe.go +++ b/internal/agent/provision/recipe.go @@ -11,6 +11,7 @@ package provision import ( "fmt" + "slices" "strings" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" @@ -71,6 +72,19 @@ var Recipes = map[string]Recipe{ "uv": {Repo: "astral-sh/uv", Choose: chooseUV}, } +// Installable are the engines this agent can install, as it tells the +// control plane: every recipe but uv, which is a tool for the others. +func Installable() []string { + var out []string + for name := range Recipes { + if name != "uv" { + out = append(out, name) + } + } + slices.Sort(out) + return out +} + // UVVersion is the uv that installs the Python engines. const UVVersion = "0.12.20" diff --git a/internal/control/agentsvc/installable_test.go b/internal/control/agentsvc/installable_test.go new file mode 100644 index 00000000..e560a969 --- /dev/null +++ b/internal/control/agentsvc/installable_test.go @@ -0,0 +1,27 @@ +package agentsvc + +import ( + "slices" + "testing" + + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision" +) + +func TestANodeIsAskedOnlyForEnginesItCanInstall(t *testing.T) { + // An agent older than the list says nothing, and could install llama.cpp + // alone. Asked for vLLM, it showed as failing until it was upgraded. + if got := installable(&llmgwv1.Register{}); !slices.Equal(got, []string{"llamacpp"}) { + t.Errorf("an agent that said nothing: %v", got) + } + // This agent says what it can, and that is vLLM and SGLang besides. + now := &llmgwv1.Register{InstallableEngines: provision.Installable()} + for _, e := range []string{"llamacpp", "vllm", "sglang"} { + if !slices.Contains(installable(now), e) { + t.Errorf("this agent does not offer %s: %v", e, installable(now)) + } + } + if slices.Contains(installable(now), "uv") { + t.Error("uv, a tool for the others, is offered as an engine") + } +} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index 89828f64..9a4207d6 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -177,7 +177,11 @@ func (s *Service) Connect( // month, and a changed version reaches every node on its next connect. if s.engines != nil { p := &llmgwv1.Provision{} + can := installable(reg) for _, e := range slices.Sorted(maps.Keys(s.engines)) { + if !slices.Contains(can, e) { + continue + } p.Engines = append(p.Engines, &llmgwv1.EngineWant{Engine: e, Version: s.engines[e]}) } if err := stream.Send(&llmgwv1.ControlMessage{ @@ -780,3 +784,14 @@ func rate(before, now uint64, secs float64) float64 { } return float64(now-before) / secs } + +// installable are the engines a node can be asked to install. An agent older +// than the list said nothing, and could install llama.cpp alone: asked for +// vLLM too, it reported it unsupported, and a cluster half way through an +// upgrade showed every such node as failing. +func installable(reg *llmgwv1.Register) []string { + if can := reg.GetInstallableEngines(); len(can) > 0 { + return can + } + return []string{"llamacpp"} +} diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index 84d876ce..191f9731 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -304,6 +304,24 @@ func Evaluate(f Facts) []Finding { out = append(out, Finding{Level: l, Area: area, Message: msg, Fix: fix}) } + // --- CPU --- + // Read from /proc/cpuinfo, so only on a node whose OS was read too; + // elsewhere an empty list means unknown, not missing. + if f.Host.Arch == "amd64" && f.Host.OSID != "" { + if f.Host.HasCPUFlag("avx2") { + add(OK, "cpu", "vector extensions: "+strings.Join(f.Host.CPUFlags, ", "), "") + } else { + has := "none of AVX, AVX2 or FMA" + if len(f.Host.CPUFlags) > 0 { + has = "only " + strings.Join(f.Host.CPUFlags, ", ") + } + // Nothing else looks wrong on such a node: it registers, answers + // correctly, and is several times slower than the hardware. + add(Warn, "cpu", "this CPU offers no AVX2 ("+has+"): llama.cpp runs several times slower on it", + "on a VM, give it the host's CPU type (Proxmox: qm set --cpu host, then restart the VM)") + } + } + // --- GPUs --- switch { case !f.SMIFound: diff --git a/internal/doctor/doctor_test.go b/internal/doctor/doctor_test.go index 2db7874e..87041dc2 100644 --- a/internal/doctor/doctor_test.go +++ b/internal/doctor/doctor_test.go @@ -173,3 +173,30 @@ func TestDoctorSaysWhetherVLLMCanBeInstalled(t *testing.T) { }) } } + +func TestDoctorSaysWhenTheCPUHidesAVX2(t *testing.T) { + area := func(fs []Finding, a string) *Finding { + for i := range fs { + if fs[i].Area == a { + return &fs[i] + } + } + return nil + } + // A VM with QEMU's default CPU model: nothing looks wrong but the speed. + f := area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "amd64"}}), "cpu") + if f == nil || f.Level != Warn || !strings.Contains(f.Fix, "--cpu host") { + t.Errorf("no AVX2: %+v", f) + } + f = area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "amd64", CPUFlags: []string{"avx", "avx2", "fma"}}}), "cpu") + if f == nil || f.Level != OK { + t.Errorf("with AVX2: %+v", f) + } + // Not read at all, or not x86: nothing to say. + if f := area(Evaluate(Facts{Host: discovery.Host{Arch: "amd64"}}), "cpu"); f != nil { + t.Errorf("unknown CPU: %+v", f) + } + if f := area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "arm64"}}), "cpu"); f != nil { + t.Errorf("ARM: %+v", f) + } +}