A node ran whatever llama-server somebody had once unpacked on it. Onboarding checked an engine only when told its path, installed none, and a node with nothing on it registered as ready. The one time that went wrong it went wrong invisibly: the binary was there, executable, and could not start for want of libgomp1, while the node reported healthy. The control plane now says which engines every node has, at which version — `engines:` in server.yaml, defaulting to the llama.cpp build this release was checked against — and each agent converges on it. What only the node can know, the node works out: the build for its processor and driver (CUDA with its runtime when an NVIDIA driver is loaded, CPU otherwise), the OS packages that build needs from its own package sources, and whether the result runs. Every archive is checked against the sha256 GitHub publishes for it and unpacked refusing any path or link that leaves the install directory. ldd names a missing library outright; running `--version` proves the rest. Only then is a directory marked installed, and it is checked again every ten minutes, because an unrelated upgrade can remove a library. An engine is reported only once installed and seen to run, or when the agent's configuration names it on purpose — never because something happened to be on PATH. The agent now says when it has probed, so an empty list means "has none" rather than "too old to say": a node rebuilt from nothing is no longer a candidate for every model at the moment it can run none of them. Placement's refusal carries the node's own words, "installing: downloading llama-b10991-bin-ubuntu-x64.tar.gz (16 MB)", and so do `ctl nodes` and the cluster page, along with a warning for a CPU without AVX2 — the VM CPU type that makes everything five times slower and nothing look wrong. Agents also report their OS, architecture, vector extensions and NVIDIA driver. Onboarding installs llmgw itself from the CI release for the node's architecture, checked against the release's checksums, rather than copying whatever binary the operator's machine had. vLLM and SGLang are not installed yet: they are Python environments of several gigabytes, not an archive, and still have to be named in the agent's configuration. Also fixes a race on the weights fingerprint, which two heartbeat loops touch during a failover. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
255 lines
7.9 KiB
Go
255 lines
7.9 KiB
Go
package provision
|
|
|
|
import (
|
|
"archive/tar"
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
)
|
|
|
|
var ubuntu = discovery.Host{OSID: "ubuntu", OSVersion: "24.04", Arch: "amd64", CPUFlags: []string{"avx2"}}
|
|
|
|
// release is a fake GitHub: one release, its assets and their digests.
|
|
type release struct {
|
|
assets map[string][]byte
|
|
digests map[string]string
|
|
gets int
|
|
mu sync.Mutex
|
|
}
|
|
|
|
func tarGz(t *testing.T, files map[string]string, links map[string]string) []byte {
|
|
t.Helper()
|
|
var buf bytes.Buffer
|
|
gz := gzip.NewWriter(&buf)
|
|
tw := tar.NewWriter(gz)
|
|
for name, body := range files {
|
|
_ = tw.WriteHeader(&tar.Header{Name: name, Mode: 0o755, Size: int64(len(body)), Typeflag: tar.TypeReg})
|
|
_, _ = tw.Write([]byte(body))
|
|
}
|
|
for name, target := range links {
|
|
_ = tw.WriteHeader(&tar.Header{Name: name, Linkname: target, Typeflag: tar.TypeSymlink})
|
|
}
|
|
_ = tw.Close()
|
|
_ = gz.Close()
|
|
return buf.Bytes()
|
|
}
|
|
|
|
func (r *release) serve(t *testing.T) *httptest.Server {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
|
r.mu.Lock()
|
|
r.gets++
|
|
r.mu.Unlock()
|
|
if strings.HasPrefix(req.URL.Path, "/repos/") {
|
|
var out struct {
|
|
Assets []map[string]string `json:"assets"`
|
|
}
|
|
for name := range r.assets {
|
|
out.Assets = append(out.Assets, map[string]string{"name": name, "digest": r.digests[name]})
|
|
}
|
|
_ = json.NewEncoder(w).Encode(out)
|
|
return
|
|
}
|
|
body, ok := r.assets[filepath.Base(req.URL.Path)]
|
|
if !ok {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
_, _ = w.Write(body)
|
|
}))
|
|
t.Cleanup(srv.Close)
|
|
return srv
|
|
}
|
|
|
|
func newRelease(assets map[string][]byte) *release {
|
|
r := &release{assets: assets, digests: map[string]string{}}
|
|
for name, b := range assets {
|
|
sum := sha256.Sum256(b)
|
|
r.digests[name] = "sha256:" + hex.EncodeToString(sum[:])
|
|
}
|
|
return r
|
|
}
|
|
|
|
// fakeSystem answers dpkg-query, apt-get, ldd and the engine itself.
|
|
type fakeSystem struct {
|
|
mu sync.Mutex
|
|
installed map[string]bool
|
|
aptRuns int
|
|
lddOut string
|
|
calls []string
|
|
}
|
|
|
|
func (f *fakeSystem) run(_ context.Context, _ []string, name string, args ...string) ([]byte, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.calls = append(f.calls, name+" "+strings.Join(args, " "))
|
|
switch {
|
|
case name == "dpkg-query":
|
|
if f.installed[args[len(args)-1]] {
|
|
return []byte("ii "), nil
|
|
}
|
|
return []byte("un "), errors.New("exit 1")
|
|
case name == "apt-get":
|
|
f.aptRuns++
|
|
if args[1] == "install" {
|
|
for _, p := range args[4:] {
|
|
f.installed[p] = true
|
|
}
|
|
}
|
|
return nil, nil
|
|
case name == "ldd":
|
|
return []byte(f.lddOut), nil
|
|
case strings.HasSuffix(name, "llama-server"):
|
|
return []byte("version: 10991 (abc)\n"), nil
|
|
}
|
|
return nil, errors.New("unexpected command " + name)
|
|
}
|
|
|
|
func installer(t *testing.T, srv *httptest.Server, sys *fakeSystem, host discovery.Host) (*Installer, *[]Status) {
|
|
var seen []Status
|
|
var mu sync.Mutex
|
|
return &Installer{
|
|
Root: t.TempDir(), Host: host, Run: sys.run,
|
|
APIBase: srv.URL, DownloadBase: srv.URL, HTTP: srv.Client(),
|
|
OnStatus: func(s Status) { mu.Lock(); seen = append(seen, s); mu.Unlock() },
|
|
}, &seen
|
|
}
|
|
|
|
func llamaRelease(t *testing.T) *release {
|
|
return newRelease(map[string][]byte{
|
|
"llama-b1-bin-ubuntu-x64.tar.gz": tarGz(t,
|
|
map[string]string{"llama-b1/llama-server": "#!/bin/sh\n", "llama-b1/libllama.so.0": "x"},
|
|
map[string]string{"llama-b1/libllama.so": "libllama.so.0"}),
|
|
})
|
|
}
|
|
|
|
func TestAnEngineIsInstalledFromNothing(t *testing.T) {
|
|
// A node rebuilt from a bare image: no engine, no OpenMP runtime. What it
|
|
// ends with has to have come from the publisher, checked, and been run.
|
|
rel := llamaRelease(t)
|
|
sys := &fakeSystem{installed: map[string]bool{}}
|
|
in, seen := installer(t, rel.serve(t), sys, ubuntu)
|
|
|
|
got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
|
|
if len(got) != 1 || got[0].State != Ready {
|
|
t.Fatalf("not ready: %+v", got)
|
|
}
|
|
if !sys.installed["libgomp1"] {
|
|
t.Error("the OpenMP runtime the build links against was not installed")
|
|
}
|
|
if !strings.HasPrefix(got[0].Binary, in.Root) {
|
|
t.Errorf("binary %s is not the one installed under %s", got[0].Binary, in.Root)
|
|
}
|
|
if _, err := os.Lstat(filepath.Join(filepath.Dir(got[0].Binary), "libllama.so")); err != nil {
|
|
t.Errorf("the library symlink was not unpacked: %v", err)
|
|
}
|
|
var states []string
|
|
for _, s := range *seen {
|
|
states = append(states, s.State+": "+s.Detail)
|
|
}
|
|
if !strings.Contains(strings.Join(states, "\n"), "downloading") {
|
|
t.Errorf("a download was not reported while it ran:\n%s", strings.Join(states, "\n"))
|
|
}
|
|
}
|
|
|
|
func TestAnInstalledEngineIsCheckedNotFetchedAgain(t *testing.T) {
|
|
rel := llamaRelease(t)
|
|
sys := &fakeSystem{installed: map[string]bool{}}
|
|
in, _ := installer(t, rel.serve(t), sys, ubuntu)
|
|
w := []Want{{Engine: "llamacpp", Version: "b1"}}
|
|
in.Converge(t.Context(), w)
|
|
before := rel.gets
|
|
|
|
got := in.Converge(t.Context(), w)
|
|
if got[0].State != Ready {
|
|
t.Fatalf("second pass: %+v", got[0])
|
|
}
|
|
if rel.gets != before {
|
|
t.Errorf("a ready engine was downloaded again (%d requests)", rel.gets-before)
|
|
}
|
|
}
|
|
|
|
func TestAnEngineThatCannotLoadALibraryIsNotReady(t *testing.T) {
|
|
// Found on a node before: llama-server present, executable, and unable to
|
|
// start for want of a library — while the node reported ready.
|
|
rel := llamaRelease(t)
|
|
sys := &fakeSystem{installed: map[string]bool{"libgomp1": true},
|
|
lddOut: "\tlibcuda.so.1 => not found\n"}
|
|
in, _ := installer(t, rel.serve(t), sys, ubuntu)
|
|
|
|
got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
|
|
if got[0].State != Failed || !strings.Contains(got[0].Detail, "libcuda.so.1") {
|
|
t.Fatalf("expected a failure naming the library: %+v", got[0])
|
|
}
|
|
if _, err := os.Stat(filepath.Join(in.Root, "llamacpp", "b1-cpu", marker)); err == nil {
|
|
t.Error("a build that does not run was marked installed")
|
|
}
|
|
}
|
|
|
|
func TestADownloadThatDoesNotMatchItsChecksumIsRefused(t *testing.T) {
|
|
rel := llamaRelease(t)
|
|
for name := range rel.digests {
|
|
rel.digests[name] = "sha256:" + strings.Repeat("0", 64)
|
|
}
|
|
sys := &fakeSystem{installed: map[string]bool{"libgomp1": true}}
|
|
in, _ := installer(t, rel.serve(t), sys, ubuntu)
|
|
|
|
got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
|
|
if got[0].State != Failed || !strings.Contains(got[0].Detail, "checksum") {
|
|
t.Fatalf("a tampered download was not refused: %+v", got[0])
|
|
}
|
|
}
|
|
|
|
func TestAnArchiveCannotWriteOutsideItsDirectory(t *testing.T) {
|
|
rel := newRelease(map[string][]byte{
|
|
"llama-b1-bin-ubuntu-x64.tar.gz": tarGz(t,
|
|
map[string]string{"../../escaped": "x", "llama-server": ""}, nil),
|
|
})
|
|
sys := &fakeSystem{installed: map[string]bool{"libgomp1": true}}
|
|
in, _ := installer(t, rel.serve(t), sys, ubuntu)
|
|
|
|
got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
|
|
if got[0].State != Failed {
|
|
t.Fatalf("an archive escaping its directory was installed: %+v", got[0])
|
|
}
|
|
}
|
|
|
|
func TestTheBuildFollowsTheHardware(t *testing.T) {
|
|
gpu := ubuntu
|
|
gpu.NVIDIADriver = "570.86.15"
|
|
b, err := chooseLlamaCpp("b1", gpu)
|
|
if err != nil || !strings.HasPrefix(b.Variant, "cuda") || len(b.Assets) != 2 {
|
|
t.Errorf("an NVIDIA node got %+v, %v; expected the CUDA build and its runtime", b, err)
|
|
}
|
|
if _, err := chooseLlamaCpp("b1", discovery.Host{OSID: "rocky", Arch: "amd64"}); err == nil {
|
|
t.Error("a build was chosen for a system llama.cpp does not publish for")
|
|
}
|
|
|
|
slow := ubuntu
|
|
slow.CPUFlags = nil
|
|
st := (&Installer{Host: slow}).ready(Status{}, Build{CPUFlags: []string{"avx2"}}, "/x/llama-server", "")
|
|
if !strings.Contains(st.Detail, "avx2") {
|
|
t.Errorf("a CPU without AVX2 was not called out: %q", st.Detail)
|
|
}
|
|
}
|
|
|
|
func TestAnEngineTheAgentCannotInstallSaysSo(t *testing.T) {
|
|
in := &Installer{Root: t.TempDir(), Host: ubuntu}
|
|
got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.29.0"}})
|
|
if got[0].State != Unsupported {
|
|
t.Errorf("got %+v", got[0])
|
|
}
|
|
}
|