Files
llmgw/internal/agent/provision/install.go
T
Ilker ManapandClaude Opus 5.5 3a26277c32 Refuse names that leave their directory, on the mirror and the agent
Two places built a path from a name somebody else supplied and checked only
for slashes.

The mirror refused "a/b" as a model name but took "..", which is a single
path element too: a fetch for model ".." wrote beside the mirror root rather
than inside it. Model, version and file names now have to be one element
that does not start with a dot — the mirror's own checksum files are dotted
— on the fetch path and the API alike.

The agent took the engine version from the control plane as given and made
a directory of it under /opt/llmgw/engines, and a RemoveAll before replacing
one. A version of "../x" would have done both outside the install root, as
root. The agent now installs only versions that look like one — letters,
digits and ._+- — whatever the control plane sent.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 00:24:48 +02:00

729 lines
23 KiB
Go

package provision
import (
"archive/tar"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"log/slog"
"net/http"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
)
// Want is one engine at one version.
type Want struct {
Engine string
Version string
}
// The states an engine's installation can be in.
const (
Installing = "installing"
Ready = "ready"
Failed = "failed"
Unsupported = "unsupported"
// Skipped is an engine this node is not meant to run, such as vLLM on a
// machine with no GPU. Nothing is wrong, and nothing is shown as wrong.
Skipped = "skipped"
)
// Status is where one engine's installation stands.
type Status struct {
Engine string
Version string
Variant string
State string
// Detail is for a person: what is happening, or what went wrong.
Detail string
// Binary is the executable, once ready.
Binary string
}
// Installer installs engines under one directory.
type Installer struct {
// Root is where engines are installed, one directory per build:
// <root>/<engine>/<version>-<variant>.
Root string
Host discovery.Host
Log *slog.Logger
HTTP *http.Client
// OnStatus hears every change, so a download that takes minutes is seen
// to be under way rather than silent.
OnStatus func(Status)
// Run executes a command; a seam for tests. Nil runs it for real.
Run func(ctx context.Context, env []string, name string, args ...string) ([]byte, error)
// FreeSpace reports the gigabytes free under a directory; a seam for
// tests. Nil asks the filesystem.
FreeSpace func(dir string) (float64, error)
// RunStdin is Run with input; a seam for tests. Nil runs it for real.
RunStdin func(ctx context.Context, env []string, stdin, name string, args ...string) ([]byte, error)
// APIBase and DownloadBase default to GitHub, PyPIBase to PyPI; tests
// point them at a fake.
APIBase string
DownloadBase string
PyPIBase string
}
// validVersion is what a version may look like: "b10991", "0.29.0",
// "0.11.0+cu128". No separators, and not starting with a dot.
var validVersion = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._+-]{0,63}$`)
// DefaultRoot is where engines are installed when nothing says otherwise.
const DefaultRoot = "/opt/llmgw/engines"
// marker is written into a build's directory once it has been installed and
// seen to run. Its absence means the directory is not to be trusted.
const marker = ".llmgw-installed"
type markerFile struct {
Engine string `json:"engine"`
Version string `json:"version"`
Variant string `json:"variant"`
Binary string `json:"binary"`
SHA256 map[string]string `json:"sha256,omitempty"`
// For a Python engine: what was asked for, PyPI as of when, by which uv.
Requirements []string `json:"requirements,omitempty"`
ExcludeNewer string `json:"exclude_newer,omitempty"`
UV string `json:"uv,omitempty"`
Installed time.Time `json:"installed"`
}
// Converge brings every wanted engine to ready, or says why it cannot.
//
// An engine already installed is checked again rather than believed: its
// directory can be damaged, or a library it needs removed by somebody's
// upgrade, and that is found here rather than by the next replica to fail.
func (in *Installer) Converge(ctx context.Context, wants []Want) []Status {
out := make([]Status, 0, len(wants))
for _, w := range wants {
st := in.one(ctx, w)
in.report(st)
out = append(out, st)
}
return out
}
func (in *Installer) one(ctx context.Context, w Want) Status {
st := Status{Engine: w.Engine, Version: w.Version}
// Checked here, on the node, whatever the control plane checked: the
// version names a directory under the install root and is passed to a
// package manager, and "../x" would put an install — and the RemoveAll
// that replaces one — outside it, as root.
if !validVersion.MatchString(w.Version) {
st.State = Unsupported
st.Detail = fmt.Sprintf("%q is not a version this agent will install", w.Version)
return st
}
recipe, ok := Recipes[w.Engine]
if !ok {
st.State = Unsupported
st.Detail = "this agent does not know how to install " + w.Engine +
"; install it by hand and name it in the agent's configuration"
return st
}
build, err := recipe.Choose(w.Version, in.Host)
if err != nil {
st.State, st.Detail = Unsupported, err.Error()
var nh NotHere
if errors.As(err, &nh) {
st.State = Skipped
}
return st
}
st.Variant = build.Variant
dir := filepath.Join(in.root(), w.Engine, w.Version+"-"+build.Variant)
// Installed before: check it still runs, and trust nothing else about it.
if _, err := os.Stat(filepath.Join(dir, marker)); err == nil {
// The packages are checked too. Removing one is exactly the kind of
// change an unrelated upgrade makes, and ldd below would catch it only
// as a missing library with no hint of which package held it.
if err := in.packages(ctx, build.Packages, &st); err != nil {
st.State, st.Detail = Failed, err.Error()
return st
}
bin, version, err := in.verify(ctx, dir, build)
if err == nil {
return in.ready(st, build, bin, version)
}
in.log().Warn("an installed engine no longer runs; installing it again",
"engine", w.Engine, "version", w.Version, "err", err)
}
if err := in.install(ctx, recipe, w, build, dir, &st); err != nil {
st.State, st.Detail, st.Binary = Failed, err.Error(), ""
return st
}
bin, version, err := in.verify(ctx, dir, build)
if err != nil {
st.State, st.Detail = Failed, err.Error()
return st
}
return in.ready(st, build, bin, version)
}
func (in *Installer) ready(st Status, b Build, bin, version string) Status {
st.State, st.Binary = Ready, bin
st.Detail = strings.TrimSpace(filepath.Base(bin) + " " + version)
var missing []string
for _, f := range b.CPUFlags {
if !in.Host.HasCPUFlag(f) {
missing = append(missing, f)
}
}
if len(missing) > 0 {
// Said on every report, not once in a log: it is the difference between
// five tokens a second and forty, and a VM given QEMU's default CPU
// model looks entirely healthy otherwise.
st.Detail += "; this CPU lacks " + strings.Join(missing, ", ") +
", so it will run several times slower (a VM needs CPU type \"host\")"
}
return st
}
// install fetches, checks and unpacks a build into dir.
//
// Into a scratch directory renamed into place at the end, so a failure halfway
// leaves nothing that looks installed.
func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir string, st *Status) error {
if b.Python != "" {
return in.installPython(ctx, r, w, b, dir, st)
}
if err := in.packages(ctx, b.Packages, st); err != nil {
return err
}
in.progress(st, "asking GitHub for the checksums of %s %s", w.Engine, w.Version)
sums, err := in.checksums(ctx, r.Repo, w.Version)
if err != nil {
return err
}
tmp := dir + ".partial"
_ = os.RemoveAll(tmp)
if err := os.MkdirAll(tmp, 0o755); err != nil {
return err
}
defer os.RemoveAll(tmp) //nolint:errcheck // gone either way once renamed
got := map[string]string{}
for _, asset := range b.Assets {
want, ok := sums[asset]
if !ok {
// Refused rather than installed unchecked: this is a binary that
// will run as root on every node.
return fmt.Errorf("%s %s publishes no checksum for %s, so it cannot be verified",
w.Engine, w.Version, asset)
}
archive := filepath.Join(tmp, asset+".download")
if err := in.download(ctx, r.Repo, w.Version, asset, archive, want, st); err != nil {
return err
}
in.progress(st, "unpacking %s", asset)
if err := untarGz(archive, tmp); err != nil {
return fmt.Errorf("unpacking %s: %w", asset, err)
}
_ = os.Remove(archive)
got[asset] = want
}
in.progress(st, "checking that %s runs", b.Binary)
bin, _, err := in.verify(ctx, tmp, b)
if err != nil {
return err
}
rel, _ := filepath.Rel(tmp, bin)
m, _ := json.MarshalIndent(markerFile{
Engine: w.Engine, Version: w.Version, Variant: b.Variant,
Binary: rel, SHA256: got, Installed: time.Now().UTC(),
}, "", " ")
if err := os.WriteFile(filepath.Join(tmp, marker), m, 0o644); err != nil { //nolint:gosec // not secret
return err
}
_ = os.RemoveAll(dir)
if err := os.Rename(tmp, dir); err != nil {
return err
}
in.log().Info("installed an engine", "engine", w.Engine, "version", w.Version,
"variant", b.Variant, "dir", dir)
return nil
}
// verify finds the binary and runs it.
//
// Two checks, because they fail differently. ldd names a missing library
// outright, where running the binary would only say it could not start. And
// running it is the only proof that the rest — the right instruction set, the
// right driver — is in place.
func (in *Installer) verify(ctx context.Context, dir string, b Build) (bin, version string, err error) {
timeout := 60 * time.Second
if b.Python != "" {
// A script in the environment, not a binary to ldd; and importing
// torch and the engine behind it takes a while on a cold disk.
bin, timeout = filepath.Join(dir, "bin", b.Binary), 3*time.Minute
if _, err := os.Stat(bin); err != nil {
return "", "", fmt.Errorf("%s is not in the environment: %w", b.Binary, err)
}
} else {
bin, err = findFile(dir, b.Binary)
if err != nil {
return "", "", err
}
if out, err := in.run(ctx, nil, "ldd", bin); err == nil {
if missing := lddMissing(string(out)); len(missing) > 0 {
return "", "", fmt.Errorf("%s cannot run on this system: %s", b.Binary, strings.Join(missing, "; "))
}
}
}
args := b.VersionArgs
if len(args) == 0 {
args = []string{"--version"}
}
tctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
out, err := in.run(tctx, nil, bin, args...)
if err != nil {
return "", "", fmt.Errorf("%s --version failed: %v: %s", b.Binary, err, lastLine(out))
}
return bin, firstVersionLine(out), nil
}
// packages installs the operating system packages a build needs.
//
// Only those missing, and from the node's own package sources: a package the
// system already has is not touched, so an engine never upgrades something
// the rest of the machine depends on.
func (in *Installer) packages(ctx context.Context, byFamily map[string][]string, st *Status) error {
family := in.Host.Family()
pkgs := byFamily[family]
if len(pkgs) == 0 {
// A system of no family this knows is not refused: its packages may
// well be there, and if they are not, ldd names the missing library.
return nil
}
pm := packageManagers[family]
var missing []string
for _, p := range pkgs {
if _, err := in.run(ctx, nil, pm.query[0], append(pm.query[1:], p)...); err != nil {
missing = append(missing, p)
}
}
if len(missing) == 0 {
return nil
}
tool, cmds := pm.install(in, missing)
if tool == "" {
return fmt.Errorf("%s is missing and no package manager was found to install it",
strings.Join(missing, ", "))
}
in.progress(st, "installing %s with %s", strings.Join(missing, ", "), tool)
for _, c := range cmds {
if err := in.runWaitingForLock(ctx, pm.env, c, st); err != nil {
return err
}
}
return nil
}
// InstallPackages installs whichever of these packages the machine lacks,
// from its own package sources, waiting out another package manager's lock.
// byFamily is keyed by discovery.Host.Family: "deb" or "rpm".
func (in *Installer) InstallPackages(ctx context.Context, byFamily map[string][]string) error {
var st Status
return in.packages(ctx, byFamily, &st)
}
// lockWait bounds how long a package command waits for another to finish.
const lockWait = 10 * time.Minute
// lockRetry is how often a locked package manager is tried again.
var lockRetry = 10 * time.Second
// runWaitingForLock runs a package command, waiting while another holds the
// package manager's lock.
//
// A machine a minute old is running cloud-init's and unattended-upgrades' own
// apt. apt's DPkg::Lock::Timeout covers dpkg's lock and not the one `apt-get
// update` takes on its lists, so the first install on a new node failed with
// "Unable to lock directory /var/lib/apt/lists/" — found by building one.
func (in *Installer) runWaitingForLock(ctx context.Context, env, c []string, st *Status) error {
deadline := time.Now().Add(lockWait)
for {
out, err := in.run(ctx, env, c[0], c[1:]...)
if err == nil {
return nil
}
if !lockHeld(string(out)) || time.Now().After(deadline) {
return fmt.Errorf("%s: %v: %s", strings.Join(c, " "), err, lastLine(out))
}
in.progress(st, "waiting for another %s to finish (a new machine runs its own updates first)", c[0])
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(lockRetry):
}
}
}
// lockHeld reports whether a package command failed only because another
// held the lock.
func lockHeld(out string) bool {
for _, s := range []string{"Unable to lock", "Could not get lock", "is locked by another process",
"waiting for process with pid", "System management is locked"} {
if strings.Contains(out, s) {
return true
}
}
return false
}
// packageManager is how one packaging family asks and installs.
type packageManager struct {
// query exits zero when the package named after it is installed.
query []string
env []string
// install picks the tool present on this machine and the commands to run.
install func(in *Installer, pkgs []string) (tool string, cmds [][]string)
}
var packageManagers = map[string]packageManager{
"deb": {
// dpkg -s exits zero for a package merely known about, so the status
// is asked for and checked by the command itself: ${db:Status-Abbrev}
// is "ii " only when installed.
query: []string{"sh", "-c", `test "$(dpkg-query -W -f='${db:Status-Abbrev}' "$0" 2>/dev/null)" = "ii "`},
env: []string{"DEBIAN_FRONTEND=noninteractive"},
install: func(_ *Installer, pkgs []string) (string, [][]string) {
// A lock timeout rather than failing on the lock: a machine a
// minute old is running cloud-init's and unattended-upgrades' own
// apt, and the first attempt on every new node would fail.
lock := "-o=DPkg::Lock::Timeout=600"
return "apt-get", [][]string{
{"apt-get", lock, "update"},
append([]string{"apt-get", lock, "install", "-y", "--no-install-recommends"}, pkgs...),
}
},
},
"rpm": {
query: []string{"rpm", "-q"},
install: func(in *Installer, pkgs []string) (string, [][]string) {
for _, tool := range []string{"dnf", "yum"} {
if in.has(tool) {
return tool, [][]string{append([]string{tool, "install", "-y"}, pkgs...)}
}
}
if in.has("zypper") {
return "zypper", [][]string{append([]string{"zypper", "--non-interactive", "install"}, pkgs...)}
}
return "", nil
},
},
}
// has reports whether a command is on PATH. Asked through the same seam as
// every other command, so a test can say which tools a machine has.
func (in *Installer) has(tool string) bool {
_, err := in.run(context.Background(), nil, "sh", "-c", `command -v "$0"`, tool)
return err == nil
}
// checksums reads the sha256 GitHub publishes for each asset of a release.
func (in *Installer) checksums(ctx context.Context, repo, tag string) (map[string]string, error) {
url := strings.TrimSuffix(in.apiBase(), "/") + "/repos/" + repo + "/releases/tags/" + tag
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/vnd.github+json")
resp, err := in.client().Do(req)
if err != nil {
return nil, fmt.Errorf("reaching GitHub for %s %s: %w", repo, tag, err)
}
defer resp.Body.Close()
switch {
case resp.StatusCode == http.StatusNotFound:
return nil, fmt.Errorf("%s has no release %s", repo, tag)
case resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests:
// Unauthenticated callers get sixty requests an hour per address, and
// a whole fleet behind one NAT shares them.
return nil, fmt.Errorf("GitHub is rate-limiting this address (%s); it retries later", resp.Status)
case resp.StatusCode != http.StatusOK:
return nil, fmt.Errorf("GitHub answered %s for %s %s", resp.Status, repo, tag)
}
var rel struct {
Assets []struct {
Name string `json:"name"`
Digest string `json:"digest"`
} `json:"assets"`
}
if err := json.NewDecoder(resp.Body).Decode(&rel); err != nil {
return nil, fmt.Errorf("reading %s %s from GitHub: %w", repo, tag, err)
}
out := map[string]string{}
for _, a := range rel.Assets {
if sum, ok := strings.CutPrefix(a.Digest, "sha256:"); ok && len(sum) == 64 {
out[a.Name] = sum
}
}
return out, nil
}
func (in *Installer) download(ctx context.Context, repo, tag, asset, dst, want string, st *Status) error {
url := strings.TrimSuffix(in.downloadBase(), "/") + "/" + repo + "/releases/download/" + tag + "/" + asset
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return err
}
resp, err := in.client().Do(req)
if err != nil {
return fmt.Errorf("downloading %s: %w", asset, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("downloading %s: %s", asset, resp.Status)
}
if resp.ContentLength > 0 {
in.progress(st, "downloading %s (%d MB)", asset, resp.ContentLength>>20)
} else {
in.progress(st, "downloading %s", asset)
}
f, err := os.Create(dst) //nolint:gosec // inside the install root
if err != nil {
return err
}
h := sha256.New()
_, err = io.Copy(io.MultiWriter(f, h), resp.Body)
if cerr := f.Close(); err == nil {
err = cerr
}
if err != nil {
return fmt.Errorf("downloading %s: %w", asset, err)
}
if got := hex.EncodeToString(h.Sum(nil)); got != want {
return fmt.Errorf("%s does not match the checksum its publisher gave (got %s, expected %s)",
asset, got[:12], want[:12])
}
return nil
}
// untarGz unpacks an archive under dir, refusing anything that would land
// outside it. A release archive is somebody else's file, and a path of
// "../../etc/cron.d/x" in it would otherwise be written as root.
func untarGz(archive, dir string) error {
f, err := os.Open(archive) //nolint:gosec // inside the install root
if err != nil {
return err
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
return err
}
tr := tar.NewReader(gz)
for {
h, err := tr.Next()
if errors.Is(err, io.EOF) {
return nil
}
if err != nil {
return err
}
target, err := within(dir, h.Name)
if err != nil {
return err
}
switch h.Typeflag {
case tar.TypeDir:
if err := os.MkdirAll(target, 0o755); err != nil {
return err
}
case tar.TypeReg:
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
return err
}
out, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fs.FileMode(h.Mode)&0o755) //nolint:gosec // checked above
if err != nil {
return err
}
if _, err := io.Copy(out, tr); err != nil { //nolint:gosec // a release archive, size bounded by its checksum
out.Close()
return err
}
if err := out.Close(); err != nil {
return err
}
case tar.TypeSymlink:
// Libraries are versioned through symlinks, and those are
// relative. One pointing anywhere else is refused.
if filepath.IsAbs(h.Linkname) {
return fmt.Errorf("%s links outside the archive", h.Name)
}
if _, err := within(dir, filepath.Join(filepath.Dir(h.Name), h.Linkname)); err != nil {
return fmt.Errorf("%s links outside the archive", h.Name)
}
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
return err
}
_ = os.Remove(target)
if err := os.Symlink(h.Linkname, target); err != nil {
return err
}
default:
// Devices, FIFOs and hard links have no business in a release.
}
}
}
func within(dir, name string) (string, error) {
target := filepath.Join(dir, name) //nolint:gosec // checked on the next line
rel, err := filepath.Rel(dir, target)
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("%s would unpack outside the install directory", name)
}
return target, nil
}
// lddMissing reads what ldd could not satisfy, in two shapes:
//
// libgomp.so.1 => not found
// /lib64/libstdc++.so.6: version `GLIBCXX_3.4.30' not found (required by ...)
//
// The second is a system too old for the build — RHEL 9 against a build that
// wants GCC 12's libstdc++ — and is said as such, rather than as a library
// that is plainly there being "not found".
func lddMissing(out string) []string {
seen := map[string]bool{}
var missing []string
for _, line := range strings.Split(out, "\n") {
line = strings.TrimSpace(line)
if !strings.Contains(line, "not found") {
continue
}
var what string
if lib, rest, ok := strings.Cut(line, ": version `"); ok {
sym, _, _ := strings.Cut(rest, "'")
what = filepath.Base(lib) + " is too old (needs " + sym + ")"
} else {
what = strings.TrimSpace(strings.SplitN(line, "=>", 2)[0]) + " is not installed"
}
if !seen[what] {
seen[what] = true
missing = append(missing, what)
}
}
return missing
}
func findFile(dir, name string) (string, error) {
var found string
_ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
if err != nil || found != "" {
return nil //nolint:nilerr // an unreadable subtree is not the binary
}
if !d.IsDir() && d.Name() == name {
found = p
}
return nil
})
if found == "" {
return "", fmt.Errorf("the build has no %s in it", name)
}
return found, nil
}
func (in *Installer) progress(st *Status, format string, args ...any) {
st.State, st.Detail = Installing, fmt.Sprintf(format, args...)
in.report(*st)
}
func (in *Installer) report(st Status) {
if in.OnStatus != nil {
in.OnStatus(st)
}
}
func runWithStdin(ctx context.Context, env []string, stdin, name string, args ...string) ([]byte, error) {
cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // fixed commands and paths we installed
cmd.Env = append(os.Environ(), env...)
cmd.Stdin = strings.NewReader(stdin)
return cmd.CombinedOutput()
}
func (in *Installer) run(ctx context.Context, env []string, name string, args ...string) ([]byte, error) {
if in.Run != nil {
return in.Run(ctx, env, name, args...)
}
cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // fixed commands and paths we installed
cmd.Env = append(os.Environ(), env...)
return cmd.CombinedOutput()
}
func (in *Installer) root() string {
if in.Root != "" {
return in.Root
}
return DefaultRoot
}
func (in *Installer) log() *slog.Logger {
if in.Log != nil {
return in.Log
}
return slog.Default()
}
func (in *Installer) client() *http.Client {
if in.HTTP != nil {
return in.HTTP
}
return &http.Client{Timeout: 30 * time.Minute}
}
func (in *Installer) apiBase() string {
if in.APIBase != "" {
return in.APIBase
}
return "https://api.github.com"
}
func (in *Installer) downloadBase() string {
if in.DownloadBase != "" {
return in.DownloadBase
}
return "https://github.com"
}
func lastLine(b []byte) string {
lines := strings.Split(strings.TrimSpace(string(b)), "\n")
return strings.TrimSpace(lines[len(lines)-1])
}
func firstVersionLine(b []byte) string {
for _, l := range strings.Split(string(b), "\n") {
if strings.Contains(strings.ToLower(l), "version") {
return strings.TrimSpace(l)
}
}
// vllm --version says the number and nothing else.
for _, l := range strings.Split(string(b), "\n") {
if l = strings.TrimSpace(l); l != "" && l[0] >= '0' && l[0] <= '9' {
return l
}
}
return ""
}