Ask a node only for engines it can install; doctor says when AVX2 is hidden
A control plane of this release asks every node for vLLM. An agent of the last one does not know how to install it, and answered "unsupported", so a cluster half way through its upgrade showed every node as failing. Agents now say which engines they can install (Register.installable_engines), and are asked for no others; an agent that says nothing is asked for llama.cpp alone, as it always was. doctor reports the CPU's vector extensions, and warns when AVX2 is missing, with the fix for a VM: the test node for node add was one, on QEMU's default CPU model, where llama.cpp runs several times slower and nothing else looks wrong. node add shows it, since it runs doctor. The changelog's unreleased section was two releases' worth of sections one after the other, with two lists of fixes and the upgrade notes in the middle. It is one entry now: largest changes first, the fixes together, and every upgrade note at the end, with vLLM's default install and how to keep your own. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
2a16c26575
commit
590e5b9f3e
9 files changed
+295
-166
No files matched your search
@@ -843,6 +843,7 @@ func (a *Agent) session(ctx context.Context, server string) error {
|
||||
// vLLM accepted a vLLM shard and failed it minutes later.
|
||||
reg.EngineVersions = a.engines(parent)
|
||||
reg.EnginesProbed = true
|
||||
reg.InstallableEngines = provision.Installable()
|
||||
reg.RunningShards = a.runningShards()
|
||||
a.mu.Lock()
|
||||
h := a.prov.Host
|
||||
|
||||
@@ -11,6 +11,7 @@ package provision
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
@@ -71,6 +72,19 @@ var Recipes = map[string]Recipe{
|
||||
"uv": {Repo: "astral-sh/uv", Choose: chooseUV},
|
||||
}
|
||||
|
||||
// Installable are the engines this agent can install, as it tells the
|
||||
// control plane: every recipe but uv, which is a tool for the others.
|
||||
func Installable() []string {
|
||||
var out []string
|
||||
for name := range Recipes {
|
||||
if name != "uv" {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// UVVersion is the uv that installs the Python engines.
|
||||
const UVVersion = "0.12.20"
|
||||
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package agentsvc
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
|
||||
)
|
||||
|
||||
func TestANodeIsAskedOnlyForEnginesItCanInstall(t *testing.T) {
|
||||
// An agent older than the list says nothing, and could install llama.cpp
|
||||
// alone. Asked for vLLM, it showed as failing until it was upgraded.
|
||||
if got := installable(&llmgwv1.Register{}); !slices.Equal(got, []string{"llamacpp"}) {
|
||||
t.Errorf("an agent that said nothing: %v", got)
|
||||
}
|
||||
// This agent says what it can, and that is vLLM and SGLang besides.
|
||||
now := &llmgwv1.Register{InstallableEngines: provision.Installable()}
|
||||
for _, e := range []string{"llamacpp", "vllm", "sglang"} {
|
||||
if !slices.Contains(installable(now), e) {
|
||||
t.Errorf("this agent does not offer %s: %v", e, installable(now))
|
||||
}
|
||||
}
|
||||
if slices.Contains(installable(now), "uv") {
|
||||
t.Error("uv, a tool for the others, is offered as an engine")
|
||||
}
|
||||
}
|
||||
@@ -177,7 +177,11 @@ func (s *Service) Connect(
|
||||
// month, and a changed version reaches every node on its next connect.
|
||||
if s.engines != nil {
|
||||
p := &llmgwv1.Provision{}
|
||||
can := installable(reg)
|
||||
for _, e := range slices.Sorted(maps.Keys(s.engines)) {
|
||||
if !slices.Contains(can, e) {
|
||||
continue
|
||||
}
|
||||
p.Engines = append(p.Engines, &llmgwv1.EngineWant{Engine: e, Version: s.engines[e]})
|
||||
}
|
||||
if err := stream.Send(&llmgwv1.ControlMessage{
|
||||
@@ -780,3 +784,14 @@ func rate(before, now uint64, secs float64) float64 {
|
||||
}
|
||||
return float64(now-before) / secs
|
||||
}
|
||||
|
||||
// installable are the engines a node can be asked to install. An agent older
|
||||
// than the list said nothing, and could install llama.cpp alone: asked for
|
||||
// vLLM too, it reported it unsupported, and a cluster half way through an
|
||||
// upgrade showed every such node as failing.
|
||||
func installable(reg *llmgwv1.Register) []string {
|
||||
if can := reg.GetInstallableEngines(); len(can) > 0 {
|
||||
return can
|
||||
}
|
||||
return []string{"llamacpp"}
|
||||
}
|
||||
@@ -304,6 +304,24 @@ func Evaluate(f Facts) []Finding {
|
||||
out = append(out, Finding{Level: l, Area: area, Message: msg, Fix: fix})
|
||||
}
|
||||
|
||||
// --- CPU ---
|
||||
// Read from /proc/cpuinfo, so only on a node whose OS was read too;
|
||||
// elsewhere an empty list means unknown, not missing.
|
||||
if f.Host.Arch == "amd64" && f.Host.OSID != "" {
|
||||
if f.Host.HasCPUFlag("avx2") {
|
||||
add(OK, "cpu", "vector extensions: "+strings.Join(f.Host.CPUFlags, ", "), "")
|
||||
} else {
|
||||
has := "none of AVX, AVX2 or FMA"
|
||||
if len(f.Host.CPUFlags) > 0 {
|
||||
has = "only " + strings.Join(f.Host.CPUFlags, ", ")
|
||||
}
|
||||
// Nothing else looks wrong on such a node: it registers, answers
|
||||
// correctly, and is several times slower than the hardware.
|
||||
add(Warn, "cpu", "this CPU offers no AVX2 ("+has+"): llama.cpp runs several times slower on it",
|
||||
"on a VM, give it the host's CPU type (Proxmox: qm set <vmid> --cpu host, then restart the VM)")
|
||||
}
|
||||
}
|
||||
|
||||
// --- GPUs ---
|
||||
switch {
|
||||
case !f.SMIFound:
|
||||
|
||||
@@ -173,3 +173,30 @@ func TestDoctorSaysWhetherVLLMCanBeInstalled(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoctorSaysWhenTheCPUHidesAVX2(t *testing.T) {
|
||||
area := func(fs []Finding, a string) *Finding {
|
||||
for i := range fs {
|
||||
if fs[i].Area == a {
|
||||
return &fs[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
// A VM with QEMU's default CPU model: nothing looks wrong but the speed.
|
||||
f := area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "amd64"}}), "cpu")
|
||||
if f == nil || f.Level != Warn || !strings.Contains(f.Fix, "--cpu host") {
|
||||
t.Errorf("no AVX2: %+v", f)
|
||||
}
|
||||
f = area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "amd64", CPUFlags: []string{"avx", "avx2", "fma"}}}), "cpu")
|
||||
if f == nil || f.Level != OK {
|
||||
t.Errorf("with AVX2: %+v", f)
|
||||
}
|
||||
// Not read at all, or not x86: nothing to say.
|
||||
if f := area(Evaluate(Facts{Host: discovery.Host{Arch: "amd64"}}), "cpu"); f != nil {
|
||||
t.Errorf("unknown CPU: %+v", f)
|
||||
}
|
||||
if f := area(Evaluate(Facts{Host: discovery.Host{OSID: "ubuntu", Arch: "arm64"}}), "cpu"); f != nil {
|
||||
t.Errorf("ARM: %+v", f)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user