Files
llmgw/internal/agent/provision/install.go
T
Ilker ManapandClaude Opus 5.5 7d97c6b74f Install a node's engines on the node, instead of trusting what is there
A node ran whatever llama-server somebody had once unpacked on it. Onboarding
checked an engine only when told its path, installed none, and a node with
nothing on it registered as ready. The one time that went wrong it went wrong
invisibly: the binary was there, executable, and could not start for want of
libgomp1, while the node reported healthy.

The control plane now says which engines every node has, at which version —
`engines:` in server.yaml, defaulting to the llama.cpp build this release was
checked against — and each agent converges on it. What only the node can
know, the node works out: the build for its processor and driver (CUDA with
its runtime when an NVIDIA driver is loaded, CPU otherwise), the OS packages
that build needs from its own package sources, and whether the result runs.
Every archive is checked against the sha256 GitHub publishes for it and
unpacked refusing any path or link that leaves the install directory. ldd
names a missing library outright; running `--version` proves the rest. Only
then is a directory marked installed, and it is checked again every ten
minutes, because an unrelated upgrade can remove a library.

An engine is reported only once installed and seen to run, or when the
agent's configuration names it on purpose — never because something happened
to be on PATH. The agent now says when it has probed, so an empty list means
"has none" rather than "too old to say": a node rebuilt from nothing is no
longer a candidate for every model at the moment it can run none of them.
Placement's refusal carries the node's own words, "installing: downloading
llama-b10991-bin-ubuntu-x64.tar.gz (16 MB)", and so do `ctl nodes` and the
cluster page, along with a warning for a CPU without AVX2 — the VM CPU type
that makes everything five times slower and nothing look wrong.

Agents also report their OS, architecture, vector extensions and NVIDIA
driver. Onboarding installs llmgw itself from the CI release for the node's
architecture, checked against the release's checksums, rather than copying
whatever binary the operator's machine had.

vLLM and SGLang are not installed yet: they are Python environments of
several gigabytes, not an archive, and still have to be named in the agent's
configuration.

Also fixes a race on the weights fingerprint, which two heartbeat loops touch
during a failover.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-28 10:59:59 +02:00

535 lines
16 KiB
Go

package provision
import (
"archive/tar"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"log/slog"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
)
// Want is one engine at one version.
type Want struct {
Engine string
Version string
}
// The states an engine's installation can be in.
const (
Installing = "installing"
Ready = "ready"
Failed = "failed"
Unsupported = "unsupported"
)
// Status is where one engine's installation stands.
type Status struct {
Engine string
Version string
Variant string
State string
// Detail is for a person: what is happening, or what went wrong.
Detail string
// Binary is the executable, once ready.
Binary string
}
// Installer installs engines under one directory.
type Installer struct {
// Root is where engines are installed, one directory per build:
// <root>/<engine>/<version>-<variant>.
Root string
Host discovery.Host
Log *slog.Logger
HTTP *http.Client
// OnStatus hears every change, so a download that takes minutes is seen
// to be under way rather than silent.
OnStatus func(Status)
// Run executes a command; a seam for tests. Nil runs it for real.
Run func(ctx context.Context, env []string, name string, args ...string) ([]byte, error)
// APIBase and DownloadBase default to GitHub; tests point them at a fake.
APIBase string
DownloadBase string
}
// DefaultRoot is where engines are installed when nothing says otherwise.
const DefaultRoot = "/opt/llmgw/engines"
// marker is written into a build's directory once it has been installed and
// seen to run. Its absence means the directory is not to be trusted.
const marker = ".llmgw-installed"
type markerFile struct {
Engine string `json:"engine"`
Version string `json:"version"`
Variant string `json:"variant"`
Binary string `json:"binary"`
SHA256 map[string]string `json:"sha256"`
Installed time.Time `json:"installed"`
}
// Converge brings every wanted engine to ready, or says why it cannot.
//
// An engine already installed is checked again rather than believed: its
// directory can be damaged, or a library it needs removed by somebody's
// upgrade, and that is found here rather than by the next replica to fail.
func (in *Installer) Converge(ctx context.Context, wants []Want) []Status {
out := make([]Status, 0, len(wants))
for _, w := range wants {
st := in.one(ctx, w)
in.report(st)
out = append(out, st)
}
return out
}
func (in *Installer) one(ctx context.Context, w Want) Status {
st := Status{Engine: w.Engine, Version: w.Version}
recipe, ok := Recipes[w.Engine]
if !ok {
st.State = Unsupported
st.Detail = "this agent does not know how to install " + w.Engine +
"; install it by hand and name it in the agent's configuration"
return st
}
build, err := recipe.Choose(w.Version, in.Host)
if err != nil {
st.State, st.Detail = Unsupported, err.Error()
return st
}
st.Variant = build.Variant
dir := filepath.Join(in.root(), w.Engine, w.Version+"-"+build.Variant)
// Installed before: check it still runs, and trust nothing else about it.
if _, err := os.Stat(filepath.Join(dir, marker)); err == nil {
// The packages are checked too. Removing one is exactly the kind of
// change an unrelated upgrade makes, and ldd below would catch it only
// as a missing library with no hint of which package held it.
if err := in.packages(ctx, build.Packages, &st); err != nil {
st.State, st.Detail = Failed, err.Error()
return st
}
bin, version, err := in.verify(ctx, dir, build)
if err == nil {
return in.ready(st, build, bin, version)
}
in.log().Warn("an installed engine no longer runs; installing it again",
"engine", w.Engine, "version", w.Version, "err", err)
}
if err := in.install(ctx, recipe, w, build, dir, &st); err != nil {
st.State, st.Detail, st.Binary = Failed, err.Error(), ""
return st
}
bin, version, err := in.verify(ctx, dir, build)
if err != nil {
st.State, st.Detail = Failed, err.Error()
return st
}
return in.ready(st, build, bin, version)
}
func (in *Installer) ready(st Status, b Build, bin, version string) Status {
st.State, st.Binary = Ready, bin
st.Detail = strings.TrimSpace(filepath.Base(bin) + " " + version)
var missing []string
for _, f := range b.CPUFlags {
if !in.Host.HasCPUFlag(f) {
missing = append(missing, f)
}
}
if len(missing) > 0 {
// Said on every report, not once in a log: it is the difference between
// five tokens a second and forty, and a VM given QEMU's default CPU
// model looks entirely healthy otherwise.
st.Detail += "; this CPU lacks " + strings.Join(missing, ", ") +
", so it will run several times slower (a VM needs CPU type \"host\")"
}
return st
}
// install fetches, checks and unpacks a build into dir.
//
// Into a scratch directory renamed into place at the end, so a failure halfway
// leaves nothing that looks installed.
func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir string, st *Status) error {
if err := in.packages(ctx, b.Packages, st); err != nil {
return err
}
in.progress(st, "asking GitHub for the checksums of %s %s", w.Engine, w.Version)
sums, err := in.checksums(ctx, r.Repo, w.Version)
if err != nil {
return err
}
tmp := dir + ".partial"
_ = os.RemoveAll(tmp)
if err := os.MkdirAll(tmp, 0o755); err != nil {
return err
}
defer os.RemoveAll(tmp) //nolint:errcheck // gone either way once renamed
got := map[string]string{}
for _, asset := range b.Assets {
want, ok := sums[asset]
if !ok {
// Refused rather than installed unchecked: this is a binary that
// will run as root on every node.
return fmt.Errorf("%s %s publishes no checksum for %s, so it cannot be verified",
w.Engine, w.Version, asset)
}
archive := filepath.Join(tmp, asset+".download")
if err := in.download(ctx, r.Repo, w.Version, asset, archive, want, st); err != nil {
return err
}
in.progress(st, "unpacking %s", asset)
if err := untarGz(archive, tmp); err != nil {
return fmt.Errorf("unpacking %s: %w", asset, err)
}
_ = os.Remove(archive)
got[asset] = want
}
in.progress(st, "checking that %s runs", b.Binary)
bin, _, err := in.verify(ctx, tmp, b)
if err != nil {
return err
}
rel, _ := filepath.Rel(tmp, bin)
m, _ := json.MarshalIndent(markerFile{
Engine: w.Engine, Version: w.Version, Variant: b.Variant,
Binary: rel, SHA256: got, Installed: time.Now().UTC(),
}, "", " ")
if err := os.WriteFile(filepath.Join(tmp, marker), m, 0o644); err != nil { //nolint:gosec // not secret
return err
}
_ = os.RemoveAll(dir)
if err := os.Rename(tmp, dir); err != nil {
return err
}
in.log().Info("installed an engine", "engine", w.Engine, "version", w.Version,
"variant", b.Variant, "dir", dir)
return nil
}
// verify finds the binary and runs it.
//
// Two checks, because they fail differently. ldd names a missing library
// outright, where running the binary would only say it could not start. And
// running it is the only proof that the rest — the right instruction set, the
// right driver — is in place.
func (in *Installer) verify(ctx context.Context, dir string, b Build) (bin, version string, err error) {
bin, err = findFile(dir, b.Binary)
if err != nil {
return "", "", err
}
if out, err := in.run(ctx, nil, "ldd", bin); err == nil {
var missing []string
for _, line := range strings.Split(string(out), "\n") {
if strings.Contains(line, "not found") {
missing = append(missing, strings.TrimSpace(strings.SplitN(line, "=>", 2)[0]))
}
}
if len(missing) > 0 {
return "", "", fmt.Errorf("%s cannot load %s", b.Binary, strings.Join(missing, ", "))
}
}
tctx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
out, err := in.run(tctx, nil, bin, "--version")
if err != nil {
return "", "", fmt.Errorf("%s --version failed: %v: %s", b.Binary, err, lastLine(out))
}
return bin, firstVersionLine(out), nil
}
// packages installs the operating system packages a build needs.
//
// Only those missing, and from the node's own package sources: a package the
// base image already has is not touched, so an engine never upgrades
// something the rest of the machine depends on.
func (in *Installer) packages(ctx context.Context, pkgs []string, st *Status) error {
var missing []string
for _, p := range pkgs {
out, err := in.run(ctx, nil, "dpkg-query", "-W", "-f=${db:Status-Abbrev}", p)
if err != nil || !strings.HasPrefix(string(out), "ii") {
missing = append(missing, p)
}
}
if len(missing) == 0 {
return nil
}
in.progress(st, "installing %s", strings.Join(missing, ", "))
env := []string{"DEBIAN_FRONTEND=noninteractive"}
// A lock timeout rather than failing on the lock: a machine a minute old is
// running cloud-init's and unattended-upgrades' own apt, and the first
// attempt on every new node would otherwise fail.
lock := "-o=DPkg::Lock::Timeout=600"
if out, err := in.run(ctx, env, "apt-get", lock, "update"); err != nil {
return fmt.Errorf("apt-get update: %v: %s", err, lastLine(out))
}
args := append([]string{lock, "install", "-y", "--no-install-recommends"}, missing...)
if out, err := in.run(ctx, env, "apt-get", args...); err != nil {
return fmt.Errorf("installing %s: %v: %s", strings.Join(missing, ", "), err, lastLine(out))
}
return nil
}
// checksums reads the sha256 GitHub publishes for each asset of a release.
func (in *Installer) checksums(ctx context.Context, repo, tag string) (map[string]string, error) {
url := strings.TrimSuffix(in.apiBase(), "/") + "/repos/" + repo + "/releases/tags/" + tag
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/vnd.github+json")
resp, err := in.client().Do(req)
if err != nil {
return nil, fmt.Errorf("reaching GitHub for %s %s: %w", repo, tag, err)
}
defer resp.Body.Close()
switch {
case resp.StatusCode == http.StatusNotFound:
return nil, fmt.Errorf("%s has no release %s", repo, tag)
case resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests:
// Unauthenticated callers get sixty requests an hour per address, and
// a whole fleet behind one NAT shares them.
return nil, fmt.Errorf("GitHub is rate-limiting this address (%s); it retries later", resp.Status)
case resp.StatusCode != http.StatusOK:
return nil, fmt.Errorf("GitHub answered %s for %s %s", resp.Status, repo, tag)
}
var rel struct {
Assets []struct {
Name string `json:"name"`
Digest string `json:"digest"`
} `json:"assets"`
}
if err := json.NewDecoder(resp.Body).Decode(&rel); err != nil {
return nil, fmt.Errorf("reading %s %s from GitHub: %w", repo, tag, err)
}
out := map[string]string{}
for _, a := range rel.Assets {
if sum, ok := strings.CutPrefix(a.Digest, "sha256:"); ok && len(sum) == 64 {
out[a.Name] = sum
}
}
return out, nil
}
func (in *Installer) download(ctx context.Context, repo, tag, asset, dst, want string, st *Status) error {
url := strings.TrimSuffix(in.downloadBase(), "/") + "/" + repo + "/releases/download/" + tag + "/" + asset
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return err
}
resp, err := in.client().Do(req)
if err != nil {
return fmt.Errorf("downloading %s: %w", asset, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("downloading %s: %s", asset, resp.Status)
}
if resp.ContentLength > 0 {
in.progress(st, "downloading %s (%d MB)", asset, resp.ContentLength>>20)
} else {
in.progress(st, "downloading %s", asset)
}
f, err := os.Create(dst) //nolint:gosec // inside the install root
if err != nil {
return err
}
h := sha256.New()
_, err = io.Copy(io.MultiWriter(f, h), resp.Body)
if cerr := f.Close(); err == nil {
err = cerr
}
if err != nil {
return fmt.Errorf("downloading %s: %w", asset, err)
}
if got := hex.EncodeToString(h.Sum(nil)); got != want {
return fmt.Errorf("%s does not match the checksum its publisher gave (got %s, expected %s)",
asset, got[:12], want[:12])
}
return nil
}
// untarGz unpacks an archive under dir, refusing anything that would land
// outside it. A release archive is somebody else's file, and a path of
// "../../etc/cron.d/x" in it would otherwise be written as root.
func untarGz(archive, dir string) error {
f, err := os.Open(archive) //nolint:gosec // inside the install root
if err != nil {
return err
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
return err
}
tr := tar.NewReader(gz)
for {
h, err := tr.Next()
if errors.Is(err, io.EOF) {
return nil
}
if err != nil {
return err
}
target, err := within(dir, h.Name)
if err != nil {
return err
}
switch h.Typeflag {
case tar.TypeDir:
if err := os.MkdirAll(target, 0o755); err != nil {
return err
}
case tar.TypeReg:
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
return err
}
out, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fs.FileMode(h.Mode)&0o755) //nolint:gosec // checked above
if err != nil {
return err
}
if _, err := io.Copy(out, tr); err != nil { //nolint:gosec // a release archive, size bounded by its checksum
out.Close()
return err
}
if err := out.Close(); err != nil {
return err
}
case tar.TypeSymlink:
// Libraries are versioned through symlinks, and those are
// relative. One pointing anywhere else is refused.
if filepath.IsAbs(h.Linkname) {
return fmt.Errorf("%s links outside the archive", h.Name)
}
if _, err := within(dir, filepath.Join(filepath.Dir(h.Name), h.Linkname)); err != nil {
return fmt.Errorf("%s links outside the archive", h.Name)
}
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
return err
}
_ = os.Remove(target)
if err := os.Symlink(h.Linkname, target); err != nil {
return err
}
default:
// Devices, FIFOs and hard links have no business in a release.
}
}
}
func within(dir, name string) (string, error) {
target := filepath.Join(dir, name) //nolint:gosec // checked on the next line
rel, err := filepath.Rel(dir, target)
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("%s would unpack outside the install directory", name)
}
return target, nil
}
func findFile(dir, name string) (string, error) {
var found string
_ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
if err != nil || found != "" {
return nil //nolint:nilerr // an unreadable subtree is not the binary
}
if !d.IsDir() && d.Name() == name {
found = p
}
return nil
})
if found == "" {
return "", fmt.Errorf("the build has no %s in it", name)
}
return found, nil
}
func (in *Installer) progress(st *Status, format string, args ...any) {
st.State, st.Detail = Installing, fmt.Sprintf(format, args...)
in.report(*st)
}
func (in *Installer) report(st Status) {
if in.OnStatus != nil {
in.OnStatus(st)
}
}
func (in *Installer) run(ctx context.Context, env []string, name string, args ...string) ([]byte, error) {
if in.Run != nil {
return in.Run(ctx, env, name, args...)
}
cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // fixed commands and paths we installed
cmd.Env = append(os.Environ(), env...)
return cmd.CombinedOutput()
}
func (in *Installer) root() string {
if in.Root != "" {
return in.Root
}
return DefaultRoot
}
func (in *Installer) log() *slog.Logger {
if in.Log != nil {
return in.Log
}
return slog.Default()
}
func (in *Installer) client() *http.Client {
if in.HTTP != nil {
return in.HTTP
}
return &http.Client{Timeout: 30 * time.Minute}
}
func (in *Installer) apiBase() string {
if in.APIBase != "" {
return in.APIBase
}
return "https://api.github.com"
}
func (in *Installer) downloadBase() string {
if in.DownloadBase != "" {
return in.DownloadBase
}
return "https://github.com"
}
func lastLine(b []byte) string {
lines := strings.Split(strings.TrimSpace(string(b)), "\n")
return strings.TrimSpace(lines[len(lines)-1])
}
func firstVersionLine(b []byte) string {
for _, l := range strings.Split(string(b), "\n") {
if strings.Contains(strings.ToLower(l), "version") {
return strings.TrimSpace(l)
}
}
return ""
}