Files
llmgw/internal/agent/provision/install.go
T
Ilker ManapandClaude Opus 5.5 4c5a2fd6da Install vLLM on every GPU node that can run it, as llama.cpp is
vLLM was installed by hand and named in each agent's configuration. The
agent now installs it: a Python 3.12 environment built by uv, which is
itself a GitHub release checked against its checksum, from PyPI resolved as
it stood a day after that vLLM was published and installed with
--require-hashes. The resolved list is kept in the environment. The
environment is built relocatable under a scratch name and renamed into place
once it runs, like an archive.

The default is 0.30.0, whose flags were checked against the ones llmgw
builds. It is built for CUDA 13, so it needs driver 580 and a Turing GPU or
newer, and 25 GB free while it installs; a node lacking one says which, and a
node with no GPU skips it without being shown as a fault. gcc comes with it
because Triton compiles its launcher at the first model load. SGLang
installs the same way when the control plane names a version. llmgw doctor
reports whether vLLM can be installed on the node.

Checked by installing 0.30.0 this way on a test node: 7.7 GB, torch 2.13 for
CUDA 13.0, importing cleanly; the node has no GPU, so vLLM was not started.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-29 13:15:54 +02:00

715 lines
22 KiB
Go

package provision
import (
"archive/tar"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"log/slog"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
)
// Want is one engine at one version.
type Want struct {
Engine string
Version string
}
// The states an engine's installation can be in.
const (
Installing = "installing"
Ready = "ready"
Failed = "failed"
Unsupported = "unsupported"
// Skipped is an engine this node is not meant to run, such as vLLM on a
// machine with no GPU. Nothing is wrong, and nothing is shown as wrong.
Skipped = "skipped"
)
// Status is where one engine's installation stands.
type Status struct {
Engine string
Version string
Variant string
State string
// Detail is for a person: what is happening, or what went wrong.
Detail string
// Binary is the executable, once ready.
Binary string
}
// Installer installs engines under one directory.
type Installer struct {
// Root is where engines are installed, one directory per build:
// <root>/<engine>/<version>-<variant>.
Root string
Host discovery.Host
Log *slog.Logger
HTTP *http.Client
// OnStatus hears every change, so a download that takes minutes is seen
// to be under way rather than silent.
OnStatus func(Status)
// Run executes a command; a seam for tests. Nil runs it for real.
Run func(ctx context.Context, env []string, name string, args ...string) ([]byte, error)
// FreeSpace reports the gigabytes free under a directory; a seam for
// tests. Nil asks the filesystem.
FreeSpace func(dir string) (float64, error)
// RunStdin is Run with input; a seam for tests. Nil runs it for real.
RunStdin func(ctx context.Context, env []string, stdin, name string, args ...string) ([]byte, error)
// APIBase and DownloadBase default to GitHub, PyPIBase to PyPI; tests
// point them at a fake.
APIBase string
DownloadBase string
PyPIBase string
}
// DefaultRoot is where engines are installed when nothing says otherwise.
const DefaultRoot = "/opt/llmgw/engines"
// marker is written into a build's directory once it has been installed and
// seen to run. Its absence means the directory is not to be trusted.
const marker = ".llmgw-installed"
type markerFile struct {
Engine string `json:"engine"`
Version string `json:"version"`
Variant string `json:"variant"`
Binary string `json:"binary"`
SHA256 map[string]string `json:"sha256,omitempty"`
// For a Python engine: what was asked for, PyPI as of when, by which uv.
Requirements []string `json:"requirements,omitempty"`
ExcludeNewer string `json:"exclude_newer,omitempty"`
UV string `json:"uv,omitempty"`
Installed time.Time `json:"installed"`
}
// Converge brings every wanted engine to ready, or says why it cannot.
//
// An engine already installed is checked again rather than believed: its
// directory can be damaged, or a library it needs removed by somebody's
// upgrade, and that is found here rather than by the next replica to fail.
func (in *Installer) Converge(ctx context.Context, wants []Want) []Status {
out := make([]Status, 0, len(wants))
for _, w := range wants {
st := in.one(ctx, w)
in.report(st)
out = append(out, st)
}
return out
}
func (in *Installer) one(ctx context.Context, w Want) Status {
st := Status{Engine: w.Engine, Version: w.Version}
recipe, ok := Recipes[w.Engine]
if !ok {
st.State = Unsupported
st.Detail = "this agent does not know how to install " + w.Engine +
"; install it by hand and name it in the agent's configuration"
return st
}
build, err := recipe.Choose(w.Version, in.Host)
if err != nil {
st.State, st.Detail = Unsupported, err.Error()
var nh NotHere
if errors.As(err, &nh) {
st.State = Skipped
}
return st
}
st.Variant = build.Variant
dir := filepath.Join(in.root(), w.Engine, w.Version+"-"+build.Variant)
// Installed before: check it still runs, and trust nothing else about it.
if _, err := os.Stat(filepath.Join(dir, marker)); err == nil {
// The packages are checked too. Removing one is exactly the kind of
// change an unrelated upgrade makes, and ldd below would catch it only
// as a missing library with no hint of which package held it.
if err := in.packages(ctx, build.Packages, &st); err != nil {
st.State, st.Detail = Failed, err.Error()
return st
}
bin, version, err := in.verify(ctx, dir, build)
if err == nil {
return in.ready(st, build, bin, version)
}
in.log().Warn("an installed engine no longer runs; installing it again",
"engine", w.Engine, "version", w.Version, "err", err)
}
if err := in.install(ctx, recipe, w, build, dir, &st); err != nil {
st.State, st.Detail, st.Binary = Failed, err.Error(), ""
return st
}
bin, version, err := in.verify(ctx, dir, build)
if err != nil {
st.State, st.Detail = Failed, err.Error()
return st
}
return in.ready(st, build, bin, version)
}
func (in *Installer) ready(st Status, b Build, bin, version string) Status {
st.State, st.Binary = Ready, bin
st.Detail = strings.TrimSpace(filepath.Base(bin) + " " + version)
var missing []string
for _, f := range b.CPUFlags {
if !in.Host.HasCPUFlag(f) {
missing = append(missing, f)
}
}
if len(missing) > 0 {
// Said on every report, not once in a log: it is the difference between
// five tokens a second and forty, and a VM given QEMU's default CPU
// model looks entirely healthy otherwise.
st.Detail += "; this CPU lacks " + strings.Join(missing, ", ") +
", so it will run several times slower (a VM needs CPU type \"host\")"
}
return st
}
// install fetches, checks and unpacks a build into dir.
//
// Into a scratch directory renamed into place at the end, so a failure halfway
// leaves nothing that looks installed.
func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir string, st *Status) error {
if b.Python != "" {
return in.installPython(ctx, r, w, b, dir, st)
}
if err := in.packages(ctx, b.Packages, st); err != nil {
return err
}
in.progress(st, "asking GitHub for the checksums of %s %s", w.Engine, w.Version)
sums, err := in.checksums(ctx, r.Repo, w.Version)
if err != nil {
return err
}
tmp := dir + ".partial"
_ = os.RemoveAll(tmp)
if err := os.MkdirAll(tmp, 0o755); err != nil {
return err
}
defer os.RemoveAll(tmp) //nolint:errcheck // gone either way once renamed
got := map[string]string{}
for _, asset := range b.Assets {
want, ok := sums[asset]
if !ok {
// Refused rather than installed unchecked: this is a binary that
// will run as root on every node.
return fmt.Errorf("%s %s publishes no checksum for %s, so it cannot be verified",
w.Engine, w.Version, asset)
}
archive := filepath.Join(tmp, asset+".download")
if err := in.download(ctx, r.Repo, w.Version, asset, archive, want, st); err != nil {
return err
}
in.progress(st, "unpacking %s", asset)
if err := untarGz(archive, tmp); err != nil {
return fmt.Errorf("unpacking %s: %w", asset, err)
}
_ = os.Remove(archive)
got[asset] = want
}
in.progress(st, "checking that %s runs", b.Binary)
bin, _, err := in.verify(ctx, tmp, b)
if err != nil {
return err
}
rel, _ := filepath.Rel(tmp, bin)
m, _ := json.MarshalIndent(markerFile{
Engine: w.Engine, Version: w.Version, Variant: b.Variant,
Binary: rel, SHA256: got, Installed: time.Now().UTC(),
}, "", " ")
if err := os.WriteFile(filepath.Join(tmp, marker), m, 0o644); err != nil { //nolint:gosec // not secret
return err
}
_ = os.RemoveAll(dir)
if err := os.Rename(tmp, dir); err != nil {
return err
}
in.log().Info("installed an engine", "engine", w.Engine, "version", w.Version,
"variant", b.Variant, "dir", dir)
return nil
}
// verify finds the binary and runs it.
//
// Two checks, because they fail differently. ldd names a missing library
// outright, where running the binary would only say it could not start. And
// running it is the only proof that the rest — the right instruction set, the
// right driver — is in place.
func (in *Installer) verify(ctx context.Context, dir string, b Build) (bin, version string, err error) {
timeout := 60 * time.Second
if b.Python != "" {
// A script in the environment, not a binary to ldd; and importing
// torch and the engine behind it takes a while on a cold disk.
bin, timeout = filepath.Join(dir, "bin", b.Binary), 3*time.Minute
if _, err := os.Stat(bin); err != nil {
return "", "", fmt.Errorf("%s is not in the environment: %w", b.Binary, err)
}
} else {
bin, err = findFile(dir, b.Binary)
if err != nil {
return "", "", err
}
if out, err := in.run(ctx, nil, "ldd", bin); err == nil {
if missing := lddMissing(string(out)); len(missing) > 0 {
return "", "", fmt.Errorf("%s cannot run on this system: %s", b.Binary, strings.Join(missing, "; "))
}
}
}
args := b.VersionArgs
if len(args) == 0 {
args = []string{"--version"}
}
tctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
out, err := in.run(tctx, nil, bin, args...)
if err != nil {
return "", "", fmt.Errorf("%s --version failed: %v: %s", b.Binary, err, lastLine(out))
}
return bin, firstVersionLine(out), nil
}
// packages installs the operating system packages a build needs.
//
// Only those missing, and from the node's own package sources: a package the
// system already has is not touched, so an engine never upgrades something
// the rest of the machine depends on.
func (in *Installer) packages(ctx context.Context, byFamily map[string][]string, st *Status) error {
family := in.Host.Family()
pkgs := byFamily[family]
if len(pkgs) == 0 {
// A system of no family this knows is not refused: its packages may
// well be there, and if they are not, ldd names the missing library.
return nil
}
pm := packageManagers[family]
var missing []string
for _, p := range pkgs {
if _, err := in.run(ctx, nil, pm.query[0], append(pm.query[1:], p)...); err != nil {
missing = append(missing, p)
}
}
if len(missing) == 0 {
return nil
}
tool, cmds := pm.install(in, missing)
if tool == "" {
return fmt.Errorf("%s is missing and no package manager was found to install it",
strings.Join(missing, ", "))
}
in.progress(st, "installing %s with %s", strings.Join(missing, ", "), tool)
for _, c := range cmds {
if err := in.runWaitingForLock(ctx, pm.env, c, st); err != nil {
return err
}
}
return nil
}
// InstallPackages installs whichever of these packages the machine lacks,
// from its own package sources, waiting out another package manager's lock.
// byFamily is keyed by discovery.Host.Family: "deb" or "rpm".
func (in *Installer) InstallPackages(ctx context.Context, byFamily map[string][]string) error {
var st Status
return in.packages(ctx, byFamily, &st)
}
// lockWait bounds how long a package command waits for another to finish.
const lockWait = 10 * time.Minute
// lockRetry is how often a locked package manager is tried again.
var lockRetry = 10 * time.Second
// runWaitingForLock runs a package command, waiting while another holds the
// package manager's lock.
//
// A machine a minute old is running cloud-init's and unattended-upgrades' own
// apt. apt's DPkg::Lock::Timeout covers dpkg's lock and not the one `apt-get
// update` takes on its lists, so the first install on a new node failed with
// "Unable to lock directory /var/lib/apt/lists/" — found by building one.
func (in *Installer) runWaitingForLock(ctx context.Context, env, c []string, st *Status) error {
deadline := time.Now().Add(lockWait)
for {
out, err := in.run(ctx, env, c[0], c[1:]...)
if err == nil {
return nil
}
if !lockHeld(string(out)) || time.Now().After(deadline) {
return fmt.Errorf("%s: %v: %s", strings.Join(c, " "), err, lastLine(out))
}
in.progress(st, "waiting for another %s to finish (a new machine runs its own updates first)", c[0])
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(lockRetry):
}
}
}
// lockHeld reports whether a package command failed only because another
// held the lock.
func lockHeld(out string) bool {
for _, s := range []string{"Unable to lock", "Could not get lock", "is locked by another process",
"waiting for process with pid", "System management is locked"} {
if strings.Contains(out, s) {
return true
}
}
return false
}
// packageManager is how one packaging family asks and installs.
type packageManager struct {
// query exits zero when the package named after it is installed.
query []string
env []string
// install picks the tool present on this machine and the commands to run.
install func(in *Installer, pkgs []string) (tool string, cmds [][]string)
}
var packageManagers = map[string]packageManager{
"deb": {
// dpkg -s exits zero for a package merely known about, so the status
// is asked for and checked by the command itself: ${db:Status-Abbrev}
// is "ii " only when installed.
query: []string{"sh", "-c", `test "$(dpkg-query -W -f='${db:Status-Abbrev}' "$0" 2>/dev/null)" = "ii "`},
env: []string{"DEBIAN_FRONTEND=noninteractive"},
install: func(_ *Installer, pkgs []string) (string, [][]string) {
// A lock timeout rather than failing on the lock: a machine a
// minute old is running cloud-init's and unattended-upgrades' own
// apt, and the first attempt on every new node would fail.
lock := "-o=DPkg::Lock::Timeout=600"
return "apt-get", [][]string{
{"apt-get", lock, "update"},
append([]string{"apt-get", lock, "install", "-y", "--no-install-recommends"}, pkgs...),
}
},
},
"rpm": {
query: []string{"rpm", "-q"},
install: func(in *Installer, pkgs []string) (string, [][]string) {
for _, tool := range []string{"dnf", "yum"} {
if in.has(tool) {
return tool, [][]string{append([]string{tool, "install", "-y"}, pkgs...)}
}
}
if in.has("zypper") {
return "zypper", [][]string{append([]string{"zypper", "--non-interactive", "install"}, pkgs...)}
}
return "", nil
},
},
}
// has reports whether a command is on PATH. Asked through the same seam as
// every other command, so a test can say which tools a machine has.
func (in *Installer) has(tool string) bool {
_, err := in.run(context.Background(), nil, "sh", "-c", `command -v "$0"`, tool)
return err == nil
}
// checksums reads the sha256 GitHub publishes for each asset of a release.
func (in *Installer) checksums(ctx context.Context, repo, tag string) (map[string]string, error) {
url := strings.TrimSuffix(in.apiBase(), "/") + "/repos/" + repo + "/releases/tags/" + tag
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/vnd.github+json")
resp, err := in.client().Do(req)
if err != nil {
return nil, fmt.Errorf("reaching GitHub for %s %s: %w", repo, tag, err)
}
defer resp.Body.Close()
switch {
case resp.StatusCode == http.StatusNotFound:
return nil, fmt.Errorf("%s has no release %s", repo, tag)
case resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests:
// Unauthenticated callers get sixty requests an hour per address, and
// a whole fleet behind one NAT shares them.
return nil, fmt.Errorf("GitHub is rate-limiting this address (%s); it retries later", resp.Status)
case resp.StatusCode != http.StatusOK:
return nil, fmt.Errorf("GitHub answered %s for %s %s", resp.Status, repo, tag)
}
var rel struct {
Assets []struct {
Name string `json:"name"`
Digest string `json:"digest"`
} `json:"assets"`
}
if err := json.NewDecoder(resp.Body).Decode(&rel); err != nil {
return nil, fmt.Errorf("reading %s %s from GitHub: %w", repo, tag, err)
}
out := map[string]string{}
for _, a := range rel.Assets {
if sum, ok := strings.CutPrefix(a.Digest, "sha256:"); ok && len(sum) == 64 {
out[a.Name] = sum
}
}
return out, nil
}
func (in *Installer) download(ctx context.Context, repo, tag, asset, dst, want string, st *Status) error {
url := strings.TrimSuffix(in.downloadBase(), "/") + "/" + repo + "/releases/download/" + tag + "/" + asset
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return err
}
resp, err := in.client().Do(req)
if err != nil {
return fmt.Errorf("downloading %s: %w", asset, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("downloading %s: %s", asset, resp.Status)
}
if resp.ContentLength > 0 {
in.progress(st, "downloading %s (%d MB)", asset, resp.ContentLength>>20)
} else {
in.progress(st, "downloading %s", asset)
}
f, err := os.Create(dst) //nolint:gosec // inside the install root
if err != nil {
return err
}
h := sha256.New()
_, err = io.Copy(io.MultiWriter(f, h), resp.Body)
if cerr := f.Close(); err == nil {
err = cerr
}
if err != nil {
return fmt.Errorf("downloading %s: %w", asset, err)
}
if got := hex.EncodeToString(h.Sum(nil)); got != want {
return fmt.Errorf("%s does not match the checksum its publisher gave (got %s, expected %s)",
asset, got[:12], want[:12])
}
return nil
}
// untarGz unpacks an archive under dir, refusing anything that would land
// outside it. A release archive is somebody else's file, and a path of
// "../../etc/cron.d/x" in it would otherwise be written as root.
func untarGz(archive, dir string) error {
f, err := os.Open(archive) //nolint:gosec // inside the install root
if err != nil {
return err
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
return err
}
tr := tar.NewReader(gz)
for {
h, err := tr.Next()
if errors.Is(err, io.EOF) {
return nil
}
if err != nil {
return err
}
target, err := within(dir, h.Name)
if err != nil {
return err
}
switch h.Typeflag {
case tar.TypeDir:
if err := os.MkdirAll(target, 0o755); err != nil {
return err
}
case tar.TypeReg:
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
return err
}
out, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fs.FileMode(h.Mode)&0o755) //nolint:gosec // checked above
if err != nil {
return err
}
if _, err := io.Copy(out, tr); err != nil { //nolint:gosec // a release archive, size bounded by its checksum
out.Close()
return err
}
if err := out.Close(); err != nil {
return err
}
case tar.TypeSymlink:
// Libraries are versioned through symlinks, and those are
// relative. One pointing anywhere else is refused.
if filepath.IsAbs(h.Linkname) {
return fmt.Errorf("%s links outside the archive", h.Name)
}
if _, err := within(dir, filepath.Join(filepath.Dir(h.Name), h.Linkname)); err != nil {
return fmt.Errorf("%s links outside the archive", h.Name)
}
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
return err
}
_ = os.Remove(target)
if err := os.Symlink(h.Linkname, target); err != nil {
return err
}
default:
// Devices, FIFOs and hard links have no business in a release.
}
}
}
func within(dir, name string) (string, error) {
target := filepath.Join(dir, name) //nolint:gosec // checked on the next line
rel, err := filepath.Rel(dir, target)
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("%s would unpack outside the install directory", name)
}
return target, nil
}
// lddMissing reads what ldd could not satisfy, in two shapes:
//
// libgomp.so.1 => not found
// /lib64/libstdc++.so.6: version `GLIBCXX_3.4.30' not found (required by ...)
//
// The second is a system too old for the build — RHEL 9 against a build that
// wants GCC 12's libstdc++ — and is said as such, rather than as a library
// that is plainly there being "not found".
func lddMissing(out string) []string {
seen := map[string]bool{}
var missing []string
for _, line := range strings.Split(out, "\n") {
line = strings.TrimSpace(line)
if !strings.Contains(line, "not found") {
continue
}
var what string
if lib, rest, ok := strings.Cut(line, ": version `"); ok {
sym, _, _ := strings.Cut(rest, "'")
what = filepath.Base(lib) + " is too old (needs " + sym + ")"
} else {
what = strings.TrimSpace(strings.SplitN(line, "=>", 2)[0]) + " is not installed"
}
if !seen[what] {
seen[what] = true
missing = append(missing, what)
}
}
return missing
}
func findFile(dir, name string) (string, error) {
var found string
_ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
if err != nil || found != "" {
return nil //nolint:nilerr // an unreadable subtree is not the binary
}
if !d.IsDir() && d.Name() == name {
found = p
}
return nil
})
if found == "" {
return "", fmt.Errorf("the build has no %s in it", name)
}
return found, nil
}
func (in *Installer) progress(st *Status, format string, args ...any) {
st.State, st.Detail = Installing, fmt.Sprintf(format, args...)
in.report(*st)
}
func (in *Installer) report(st Status) {
if in.OnStatus != nil {
in.OnStatus(st)
}
}
func runWithStdin(ctx context.Context, env []string, stdin, name string, args ...string) ([]byte, error) {
cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // fixed commands and paths we installed
cmd.Env = append(os.Environ(), env...)
cmd.Stdin = strings.NewReader(stdin)
return cmd.CombinedOutput()
}
func (in *Installer) run(ctx context.Context, env []string, name string, args ...string) ([]byte, error) {
if in.Run != nil {
return in.Run(ctx, env, name, args...)
}
cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // fixed commands and paths we installed
cmd.Env = append(os.Environ(), env...)
return cmd.CombinedOutput()
}
func (in *Installer) root() string {
if in.Root != "" {
return in.Root
}
return DefaultRoot
}
func (in *Installer) log() *slog.Logger {
if in.Log != nil {
return in.Log
}
return slog.Default()
}
func (in *Installer) client() *http.Client {
if in.HTTP != nil {
return in.HTTP
}
return &http.Client{Timeout: 30 * time.Minute}
}
func (in *Installer) apiBase() string {
if in.APIBase != "" {
return in.APIBase
}
return "https://api.github.com"
}
func (in *Installer) downloadBase() string {
if in.DownloadBase != "" {
return in.DownloadBase
}
return "https://github.com"
}
func lastLine(b []byte) string {
lines := strings.Split(strings.TrimSpace(string(b)), "\n")
return strings.TrimSpace(lines[len(lines)-1])
}
func firstVersionLine(b []byte) string {
for _, l := range strings.Split(string(b), "\n") {
if strings.Contains(strings.ToLower(l), "version") {
return strings.TrimSpace(l)
}
}
// vllm --version says the number and nothing else.
for _, l := range strings.Split(string(b), "\n") {
if l = strings.TrimSpace(l); l != "" && l[0] >= '0' && l[0] <= '9' {
return l
}
}
return ""
}