llmgw/internal/agent/provision/install.go

package provision

import (
	"archive/tar"
	"compress/gzip"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"log/slog"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/trace"

	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
	"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
)

// Want is one engine at one version.
type Want struct {
	Engine  string
	Version string
	// Variant is a build chosen for this node ("cu129"); empty leaves it to
	// the node's hardware.
	Variant string
	// Offline, when set, is where the engine comes from instead of its
	// publisher: a cluster with no way to the internet installs from its
	// mirror's copy of an offline bundle.
	Offline *Offline
}

// Offline is an engine version's files on a cluster's own mirror.
//
// The files come from the mirror and their checksums from the control plane,
// which read them from the bundle they were imported from. The mirror serves
// plain HTTP to every node; a checksum taken from beside the file there would
// prove only that the file and the checksum came from the same place.
type Offline struct {
	// Base is where the version's files are: <mirror>/_llmgw/engines/<engine>/<version>.
	Base string
	// SHA256 is each file's checksum, by name, as the publisher gave it
	// when the bundle was made.
	SHA256 map[string]string
}

// The states an engine's installation can be in.
const (
	Installing  = "installing"
	Ready       = "ready"
	Failed      = "failed"
	Unsupported = "unsupported"
	// Skipped is an engine this node is not meant to run, such as vLLM on a
	// machine with no GPU. Nothing is wrong, and nothing is shown as wrong.
	Skipped = "skipped"
)

// Status is where one engine's installation stands.
type Status struct {
	Engine  string
	Version string
	Variant string
	State   string
	// Detail is for a person: what is happening, or what went wrong.
	Detail string
	// Binary is the executable, once ready.
	Binary string
}

// Installer installs engines under one directory.
type Installer struct {
	// Root is where engines are installed, one directory per build:
	// <root>/<engine>/<version>-<variant>.
	Root string
	Host discovery.Host
	Log  *slog.Logger
	HTTP *http.Client
	// OnStatus hears every change, so a download that takes minutes is seen
	// to be under way rather than silent.
	OnStatus func(Status)

	// Run executes a command; a seam for tests. Nil runs it for real.
	Run func(ctx context.Context, env []string, name string, args ...string) ([]byte, error)
	// FreeSpace reports the gigabytes free under a directory; a seam for
	// tests. Nil asks the filesystem.
	FreeSpace func(dir string) (float64, error)
	// RunStdin is Run with input; a seam for tests. Nil runs it for real.
	RunStdin func(ctx context.Context, env []string, stdin, name string, args ...string) ([]byte, error)
	// APIBase and DownloadBase default to GitHub, PyPIBase to PyPI; tests
	// point them at a fake.
	APIBase      string
	DownloadBase string
	PyPIBase     string
}

// validVersion is what a version may look like: "b10991", "0.29.0",
// "0.11.0+cu128". No separators, and not starting with a dot.
var validVersion = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._+-]{0,63}$`)

// DefaultRoot is where engines are installed when nothing says otherwise.
const DefaultRoot = "/opt/llmgw/engines"

// marker is written into a build's directory once it has been installed and
// seen to run. Its absence means the directory is not to be trusted.
const marker = ".llmgw-installed"

type markerFile struct {
	Engine  string            `json:"engine"`
	Version string            `json:"version"`
	Variant string            `json:"variant"`
	Binary  string            `json:"binary"`
	SHA256  map[string]string `json:"sha256,omitempty"`
	// For a Python engine: what was asked for, PyPI as of when, by which uv.
	Requirements []string  `json:"requirements,omitempty"`
	ExcludeNewer string    `json:"exclude_newer,omitempty"`
	UV           string    `json:"uv,omitempty"`
	Installed    time.Time `json:"installed"`
}

// Converge brings every wanted engine to ready, or says why it cannot.
//
// An engine already installed is checked again rather than believed: its
// directory can be damaged, or a library it needs removed by somebody's
// upgrade, and that is found here rather than by the next replica to fail.
func (in *Installer) Converge(ctx context.Context, wants []Want) []Status {
	out := make([]Status, 0, len(wants))
	for _, w := range wants {
		st := in.one(ctx, w)
		in.report(st)
		out = append(out, st)
	}
	return out
}

func (in *Installer) one(ctx context.Context, w Want) Status {
	st := Status{Engine: w.Engine, Version: w.Version}
	// Checked here, on the node, whatever the control plane checked: the
	// version names a directory under the install root and is passed to a
	// package manager, and "../x" would put an install — and the RemoveAll
	// that replaces one — outside it, as root.
	if !validVersion.MatchString(w.Version) {
		st.State = Unsupported
		st.Detail = fmt.Sprintf("%q is not a version this agent will install", w.Version)
		return st
	}
	recipe, ok := Recipes[w.Engine]
	if !ok {
		st.State = Unsupported
		st.Detail = "this agent does not know how to install " + w.Engine +
			"; install it by hand and name it in the agent's configuration"
		return st
	}
	if w.Variant != "" && !validVersion.MatchString(w.Variant) {
		st.State = Unsupported
		st.Detail = fmt.Sprintf("%q is not a build this agent will install", w.Variant)
		return st
	}
	builds, err := recipe.Candidates(w.Version, w.Variant, in.Host)
	if err != nil {
		st.State, st.Detail = Unsupported, err.Error()
		var nh NotHere
		if errors.As(err, &nh) {
			st.State = Skipped
		}
		return st
	}
	if w.Offline != nil && recipe.Repo == "" {
		// Said, not attempted: uv would spend minutes failing to reach PyPI,
		// and the failure would read like a network fault rather than the
		// design of an offline site.
		st.State = Unsupported
		st.Detail = "this cluster installs offline, and " + w.Engine + " comes from PyPI, which an offline bundle " +
			"does not carry: install it on the node by hand and name it in agent.yaml, or set engines." +
			w.Engine + ` to "" in server.yaml`
		return st
	}

	// Installed before: whichever candidate is, best first, checked that it
	// still runs and trusted for nothing else.
	for _, build := range builds {
		dir := in.dirFor(w, build)
		if _, err := os.Stat(filepath.Join(dir, marker)); err != nil {
			continue
		}
		st := st.as(w, build)
		// The packages are checked too. Removing one is exactly the kind of
		// change an unrelated upgrade makes, and ldd below would catch it
		// only as a missing library with no hint of which package held it.
		if err := in.packages(ctx, build.Packages, &st); err != nil {
			st.State, st.Detail = Failed, err.Error()
			return st
		}
		bin, version, err := in.verify(ctx, dir, build)
		if err == nil {
			return in.ready(st, build, bin, version)
		}
		in.log().Warn("an installed engine no longer runs; installing it again",
			"engine", w.Engine, "version", st.Version, "variant", build.Variant, "err", err)
		break
	}

	// Each candidate in turn until one installs and is seen to work: a build
	// the table was wrong about — a driver too old for it after all, a GPU
	// it has no kernels for — costs its download, and the next is tried.
	var tried []string
	for _, build := range builds {
		st := st.as(w, build)
		if err := in.install(ctx, recipe, w, build, in.dirFor(w, build), &st); err != nil {
			tried = append(tried, build.Variant+": "+err.Error())
			in.log().Warn("an engine build did not work here; trying the next",
				"engine", w.Engine, "variant", build.Variant, "err", err)
			continue
		}
		bin, version, err := in.verify(ctx, in.dirFor(w, build), build)
		if err != nil {
			tried = append(tried, build.Variant+": "+err.Error())
			continue
		}
		return in.ready(st, build, bin, version)
	}
	st.State, st.Binary = Failed, ""
	st.Detail = strings.Join(tried, "; then ")
	if len(builds) == 1 {
		_, st.Detail, _ = strings.Cut(st.Detail, ": ")
	}
	return st
}

// dirFor is where a build of an engine is installed: by the version it is
// and its variant, so two builds of one version sit side by side.
func (in *Installer) dirFor(w Want, b Build) string {
	return filepath.Join(in.root(), w.Engine, b.versionOr(w.Version)+"-"+b.Variant)
}

// versionOr is the version a build is: its own, when it is not the one asked
// for.
func (b Build) versionOr(asked string) string {
	if b.Version != "" {
		return b.Version
	}
	return asked
}

// as is a status for one build of what was asked for.
func (st Status) as(w Want, b Build) Status {
	st.Variant, st.Version = b.Variant, b.versionOr(w.Version)
	return st
}

func (in *Installer) ready(st Status, b Build, bin, version string) Status {
	st.State, st.Binary = Ready, bin
	st.Detail = strings.TrimSpace(filepath.Base(bin) + " " + version)
	if b.Note != "" {
		st.Detail += "; " + b.Note
	}
	var missing []string
	for _, f := range b.CPUFlags {
		if !in.Host.HasCPUFlag(f) {
			missing = append(missing, f)
		}
	}
	if len(missing) > 0 {
		// Said on every report, not once in a log: it is the difference between
		// five tokens a second and forty, and a VM given QEMU's default CPU
		// model looks entirely healthy otherwise.
		st.Detail += "; this CPU lacks " + strings.Join(missing, ", ") +
			", so it will run several times slower (a VM needs CPU type \"host\")"
	}
	return st
}

// install fetches, checks and unpacks a build into dir.
//
// Into a scratch directory renamed into place at the end, so a failure halfway
// leaves nothing that looks installed.
//
// Traced as "install engine", a trace of its own: an install is not part of
// any one start, and runs once per version rather than once per replica.
func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir string, st *Status) (err error) {
	ctx, span := obs.Tracer("gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision").Start(ctx, "install engine",
		trace.WithAttributes(
			attribute.String("llmgw.engine", w.Engine),
			attribute.String("llmgw.engine_version", w.Version),
			attribute.String("llmgw.engine_build", w.Variant),
		))
	defer func() { obs.End(span, err) }()
	if b.Python != "" {
		return in.installPython(ctx, r, w, b, dir, st)
	}
	if err := in.packages(ctx, b.Packages, st); err != nil {
		return err
	}

	var sums map[string]string
	if w.Offline != nil {
		sums = w.Offline.SHA256
	} else {
		in.progress(st, "asking GitHub for the checksums of %s %s", w.Engine, w.Version)
		var err error
		if sums, err = in.checksums(ctx, r.Repo, w.Version); err != nil {
			return err
		}
	}

	tmp := dir + ".partial"
	_ = os.RemoveAll(tmp)
	if err := os.MkdirAll(tmp, 0o755); err != nil {
		return err
	}
	defer os.RemoveAll(tmp) //nolint:errcheck // gone either way once renamed

	got := map[string]string{}
	for _, asset := range b.Assets {
		want, ok := sums[asset]
		switch {
		case !ok && w.Offline != nil:
			return fmt.Errorf("the offline bundle has no %s for %s %s: make one that has "+
				"(llmgw bundle create --engine %s=%s) and import it", asset, w.Engine, w.Version, w.Engine, w.Version)
		case !ok:
			// Refused rather than installed unchecked: this is a binary that
			// will run as root on every node.
			return fmt.Errorf("%s %s publishes no checksum for %s, so it cannot be verified",
				w.Engine, w.Version, asset)
		}
		url := strings.TrimSuffix(in.downloadBase(), "/") + "/" + r.Repo + "/releases/download/" + w.Version + "/" + asset
		if w.Offline != nil {
			url = strings.TrimSuffix(w.Offline.Base, "/") + "/" + asset
		}
		archive := filepath.Join(tmp, asset+".download")
		if err := in.download(ctx, url, asset, archive, want, st); err != nil {
			return err
		}
		in.progress(st, "unpacking %s", asset)
		if err := untarGz(archive, tmp); err != nil {
			return fmt.Errorf("unpacking %s: %w", asset, err)
		}
		_ = os.Remove(archive)
		got[asset] = want
	}

	in.progress(st, "checking that %s runs", b.Binary)
	bin, _, err := in.verify(ctx, tmp, b)
	if err != nil {
		return err
	}
	rel, _ := filepath.Rel(tmp, bin)
	m, _ := json.MarshalIndent(markerFile{
		Engine: w.Engine, Version: w.Version, Variant: b.Variant,
		Binary: rel, SHA256: got, Installed: time.Now().UTC(),
	}, "", "  ")
	if err := os.WriteFile(filepath.Join(tmp, marker), m, 0o644); err != nil { //nolint:gosec // not secret
		return err
	}
	_ = os.RemoveAll(dir)
	if err := os.Rename(tmp, dir); err != nil {
		return err
	}
	in.log().Info("installed an engine", "engine", w.Engine, "version", w.Version,
		"variant", b.Variant, "dir", dir)
	return nil
}

// verify finds the binary and runs it.
//
// Two checks, because they fail differently. ldd names a missing library
// outright, where running the binary would only say it could not start. And
// running it is the only proof that the rest — the right instruction set, the
// right driver — is in place.
func (in *Installer) verify(ctx context.Context, dir string, b Build) (bin, version string, err error) {
	timeout := 60 * time.Second
	if b.Python != "" {
		// A script in the environment, not a binary to ldd; and importing
		// torch and the engine behind it takes a while on a cold disk.
		bin, timeout = filepath.Join(dir, "bin", b.Binary), 3*time.Minute
		if _, err := os.Stat(bin); err != nil {
			return "", "", fmt.Errorf("%s is not in the environment: %w", b.Binary, err)
		}
	} else {
		bin, err = findFile(dir, b.Binary)
		if err != nil {
			return "", "", err
		}
		if out, err := in.run(ctx, nil, "ldd", bin); err == nil {
			if missing := lddMissing(string(out)); len(missing) > 0 {
				return "", "", fmt.Errorf("%s cannot run on this system: %s", b.Binary, strings.Join(missing, "; "))
			}
		}
	}
	args := b.VersionArgs
	if len(args) == 0 {
		args = []string{"--version"}
	}
	tctx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()
	out, err := in.run(tctx, nil, bin, args...)
	if err != nil {
		return "", "", fmt.Errorf("%s --version failed: %v: %s", b.Binary, err, lastLine(out))
	}
	if b.GPUBackend != "" {
		if err := in.verifyGPUBackend(tctx, dir, bin, b); err != nil {
			return "", "", err
		}
	}
	return bin, firstVersionLine(out), nil
}

// verifyGPUBackend checks a GPU build can compute on this node's GPUs.
//
// The binary's own ldd says nothing about it: the GPU backend is a library
// the server opens at run time, and when it cannot — cuBLAS missing, a driver
// too old — the server carries on with its CPU backend. So the build is asked
// which devices it sees, and when it sees none, ldd on the backend says why.
func (in *Installer) verifyGPUBackend(ctx context.Context, dir, bin string, b Build) error {
	out, err := in.run(ctx, nil, bin, "--list-devices")
	if err != nil {
		return fmt.Errorf("%s --list-devices failed: %v: %s", b.Binary, err, lastLine(out))
	}
	if listsDevice(string(out)) {
		return nil
	}
	why := "its GPU backend did not load"
	if lib, err := findFile(dir, b.GPUBackend); err != nil {
		why = "the build has no " + b.GPUBackend
	} else if out, err := in.run(ctx, nil, "ldd", lib); err == nil {
		if missing := lddMissing(string(out)); len(missing) > 0 {
			why = b.GPUBackend + " cannot load: " + strings.Join(missing, "; ")
		}
	}
	return fmt.Errorf("the %s build of %s sees no GPU, and would run on the CPU: %s", b.Variant, b.Binary, why)
}

// listsDevice reads llama-server --list-devices: a device line after
// "Available devices:", which holds "(none)" when there is none.
func listsDevice(out string) bool {
	listing := false
	for _, line := range strings.Split(out, "\n") {
		line = strings.TrimSpace(line)
		switch {
		case strings.HasPrefix(line, "Available devices"):
			listing = true
		case listing && line != "" && line != "(none)":
			return true
		}
	}
	return false
}

// packages installs the operating system packages a build needs.
//
// Only those missing, and from the node's own package sources: a package the
// system already has is not touched, so an engine never upgrades something
// the rest of the machine depends on.
func (in *Installer) packages(ctx context.Context, byFamily map[string][]string, st *Status) error {
	family := in.Host.Family()
	pkgs := byFamily[family]
	if len(pkgs) == 0 {
		// A system of no family this knows is not refused: its packages may
		// well be there, and if they are not, ldd names the missing library.
		return nil
	}
	pm := packageManagers[family]
	var missing []string
	for _, p := range pkgs {
		if _, err := in.run(ctx, nil, pm.query[0], append(pm.query[1:], p)...); err != nil {
			missing = append(missing, p)
		}
	}
	if len(missing) == 0 {
		return nil
	}
	tool, cmds := pm.install(in, missing)
	if tool == "" {
		return fmt.Errorf("%s is missing and no package manager was found to install it",
			strings.Join(missing, ", "))
	}
	in.progress(st, "installing %s with %s", strings.Join(missing, ", "), tool)
	for _, c := range cmds {
		if err := in.runWaitingForLock(ctx, pm.env, c, st); err != nil {
			return err
		}
	}
	return nil
}

// InstallPackages installs whichever of these packages the machine lacks,
// from its own package sources, waiting out another package manager's lock.
// byFamily is keyed by discovery.Host.Family: "deb" or "rpm".
func (in *Installer) InstallPackages(ctx context.Context, byFamily map[string][]string) error {
	var st Status
	return in.packages(ctx, byFamily, &st)
}

// lockWait bounds how long a package command waits for another to finish.
const lockWait = 10 * time.Minute

// lockRetry is how often a locked package manager is tried again.
var lockRetry = 10 * time.Second

// runWaitingForLock runs a package command, waiting while another holds the
// package manager's lock.
//
// A machine a minute old is running cloud-init's and unattended-upgrades' own
// apt. apt's DPkg::Lock::Timeout covers dpkg's lock and not the one `apt-get
// update` takes on its lists, so the first install on a new node failed with
// "Unable to lock directory /var/lib/apt/lists/" — found by building one.
func (in *Installer) runWaitingForLock(ctx context.Context, env, c []string, st *Status) error {
	deadline := time.Now().Add(lockWait)
	for {
		out, err := in.run(ctx, env, c[0], c[1:]...)
		if err == nil {
			return nil
		}
		if !lockHeld(string(out)) || time.Now().After(deadline) {
			return fmt.Errorf("%s: %v: %s", strings.Join(c, " "), err, lastLine(out))
		}
		in.progress(st, "waiting for another %s to finish (a new machine runs its own updates first)", c[0])
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-time.After(lockRetry):
		}
	}
}

// lockHeld reports whether a package command failed only because another
// held the lock.
func lockHeld(out string) bool {
	for _, s := range []string{"Unable to lock", "Could not get lock", "is locked by another process",
		"waiting for process with pid", "System management is locked"} {
		if strings.Contains(out, s) {
			return true
		}
	}
	return false
}

// packageManager is how one packaging family asks and installs.
type packageManager struct {
	// query exits zero when the package named after it is installed.
	query []string
	env   []string
	// install picks the tool present on this machine and the commands to run.
	install func(in *Installer, pkgs []string) (tool string, cmds [][]string)
}

var packageManagers = map[string]packageManager{
	"deb": {
		// dpkg -s exits zero for a package merely known about, so the status
		// is asked for and checked by the command itself: ${db:Status-Abbrev}
		// is "ii " only when installed.
		query: []string{"sh", "-c", `test "$(dpkg-query -W -f='${db:Status-Abbrev}' "$0" 2>/dev/null)" = "ii "`},
		env:   []string{"DEBIAN_FRONTEND=noninteractive"},
		install: func(_ *Installer, pkgs []string) (string, [][]string) {
			// A lock timeout rather than failing on the lock: a machine a
			// minute old is running cloud-init's and unattended-upgrades' own
			// apt, and the first attempt on every new node would fail.
			lock := "-o=DPkg::Lock::Timeout=600"
			return "apt-get", [][]string{
				{"apt-get", lock, "update"},
				append([]string{"apt-get", lock, "install", "-y", "--no-install-recommends"}, pkgs...),
			}
		},
	},
	"rpm": {
		query: []string{"rpm", "-q"},
		install: func(in *Installer, pkgs []string) (string, [][]string) {
			for _, tool := range []string{"dnf", "yum"} {
				if in.has(tool) {
					return tool, [][]string{append([]string{tool, "install", "-y"}, pkgs...)}
				}
			}
			if in.has("zypper") {
				return "zypper", [][]string{append([]string{"zypper", "--non-interactive", "install"}, pkgs...)}
			}
			return "", nil
		},
	},
}

// has reports whether a command is on PATH. Asked through the same seam as
// every other command, so a test can say which tools a machine has.
func (in *Installer) has(tool string) bool {
	_, err := in.run(context.Background(), nil, "sh", "-c", `command -v "$0"`, tool)
	return err == nil
}

// Published is the sha256 an engine's publisher gives for each file of a
// version, and where each is downloaded from: what an offline bundle is made
// of, read the way a node online reads it.
func (in *Installer) Published(ctx context.Context, engine, version string) (sums map[string]string, url func(asset string) string, err error) {
	r, ok := Recipes[engine]
	if !ok || r.Repo == "" {
		return nil, nil, fmt.Errorf("%s is not published as release files", engine)
	}
	if !validVersion.MatchString(version) {
		return nil, nil, fmt.Errorf("%q is not a version of %s", version, engine)
	}
	sums, err = in.checksums(ctx, r.Repo, version)
	if err != nil {
		return nil, nil, err
	}
	return sums, func(asset string) string {
		return strings.TrimSuffix(in.downloadBase(), "/") + "/" + r.Repo + "/releases/download/" + version + "/" + asset
	}, nil
}

// checksums reads the sha256 GitHub publishes for each asset of a release.
func (in *Installer) checksums(ctx context.Context, repo, tag string) (map[string]string, error) {
	url := strings.TrimSuffix(in.apiBase(), "/") + "/repos/" + repo + "/releases/tags/" + tag
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Accept", "application/vnd.github+json")
	resp, err := in.client().Do(req)
	if err != nil {
		return nil, fmt.Errorf("reaching GitHub for %s %s: %w", repo, tag, err)
	}
	defer resp.Body.Close()
	switch {
	case resp.StatusCode == http.StatusNotFound:
		return nil, fmt.Errorf("%s has no release %s", repo, tag)
	case resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests:
		// Unauthenticated callers get sixty requests an hour per address, and
		// a whole fleet behind one NAT shares them.
		return nil, fmt.Errorf("GitHub is rate-limiting this address (%s); it retries later", resp.Status)
	case resp.StatusCode != http.StatusOK:
		return nil, fmt.Errorf("GitHub answered %s for %s %s", resp.Status, repo, tag)
	}
	var rel struct {
		Assets []struct {
			Name   string `json:"name"`
			Digest string `json:"digest"`
		} `json:"assets"`
	}
	if err := json.NewDecoder(resp.Body).Decode(&rel); err != nil {
		return nil, fmt.Errorf("reading %s %s from GitHub: %w", repo, tag, err)
	}
	out := map[string]string{}
	for _, a := range rel.Assets {
		if sum, ok := strings.CutPrefix(a.Digest, "sha256:"); ok && len(sum) == 64 {
			out[a.Name] = sum
		}
	}
	return out, nil
}

func (in *Installer) download(ctx context.Context, url, asset, dst, want string, st *Status) error {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return err
	}
	resp, err := in.client().Do(req)
	if err != nil {
		return fmt.Errorf("downloading %s: %w", asset, err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("downloading %s: %s", asset, resp.Status)
	}
	if resp.ContentLength > 0 {
		in.progress(st, "downloading %s (%d MB)", asset, resp.ContentLength>>20)
	} else {
		in.progress(st, "downloading %s", asset)
	}
	f, err := os.Create(dst) //nolint:gosec // inside the install root
	if err != nil {
		return err
	}
	h := sha256.New()
	_, err = io.Copy(io.MultiWriter(f, h), resp.Body)
	if cerr := f.Close(); err == nil {
		err = cerr
	}
	if err != nil {
		return fmt.Errorf("downloading %s: %w", asset, err)
	}
	if got := hex.EncodeToString(h.Sum(nil)); got != want {
		return fmt.Errorf("%s does not match the checksum its publisher gave (got %s, expected %s)",
			asset, got[:12], want[:12])
	}
	return nil
}

// untarGz unpacks an archive under dir, refusing anything that would land
// outside it. A release archive is somebody else's file, and a path of
// "../../etc/cron.d/x" in it would otherwise be written as root.
//
// Every write goes through an os.Root, which refuses to leave dir however the
// path gets there. Checking names alone was not enough: "a -> ." followed by
// "a/a/b -> ../.." reads as inside and resolves two levels above it, and a
// file written through that lands wherever the links point.
func untarGz(archive, dir string) error {
	f, err := os.Open(archive) //nolint:gosec // inside the install root
	if err != nil {
		return err
	}
	defer f.Close()
	gz, err := gzip.NewReader(f)
	if err != nil {
		return err
	}
	root, err := os.OpenRoot(dir)
	if err != nil {
		return err
	}
	defer root.Close()
	tr := tar.NewReader(gz)
	for {
		h, err := tr.Next()
		if errors.Is(err, io.EOF) {
			return nil
		}
		if err != nil {
			return err
		}
		name := filepath.Clean(h.Name)
		if _, err := within(dir, name); err != nil {
			return err
		}
		switch h.Typeflag {
		case tar.TypeDir:
			if err := root.MkdirAll(name, 0o755); err != nil {
				return err
			}
		case tar.TypeReg:
			if err := root.MkdirAll(filepath.Dir(name), 0o755); err != nil {
				return err
			}
			out, err := root.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fs.FileMode(h.Mode)&0o755)
			if err != nil {
				return fmt.Errorf("%s: %w", h.Name, err)
			}
			if _, err := io.Copy(out, tr); err != nil { //nolint:gosec // a release archive, size bounded by its checksum
				out.Close()
				return err
			}
			if err := out.Close(); err != nil {
				return err
			}
		case tar.TypeSymlink:
			// Libraries are versioned through symlinks to a file beside them —
			// libllama.so -> libllama.so.0 — and that is all a release needs.
			// A target with a separator in it could point anywhere once other
			// links are followed, and a binary found through one would be run
			// as root, so it is refused rather than reasoned about.
			if h.Linkname == "" || strings.ContainsAny(h.Linkname, `/\`) || h.Linkname == ".." || h.Linkname == "." {
				return fmt.Errorf("%s links to %q; only a file beside it is allowed", h.Name, h.Linkname)
			}
			if err := root.MkdirAll(filepath.Dir(name), 0o755); err != nil {
				return err
			}
			_ = root.Remove(name)
			if err := root.Symlink(h.Linkname, name); err != nil {
				return err
			}
		default:
			// Devices, FIFOs and hard links have no business in a release.
		}
	}
}

func within(dir, name string) (string, error) {
	target := filepath.Join(dir, name) //nolint:gosec // checked on the next line
	rel, err := filepath.Rel(dir, target)
	if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
		return "", fmt.Errorf("%s would unpack outside the install directory", name)
	}
	return target, nil
}

// lddMissing reads what ldd could not satisfy, in two shapes:
//
//	libgomp.so.1 => not found
//	/lib64/libstdc++.so.6: version `GLIBCXX_3.4.30' not found (required by ...)
//
// The second is a system too old for the build — RHEL 9 against a build that
// wants GCC 12's libstdc++ — and is said as such, rather than as a library
// that is plainly there being "not found".
func lddMissing(out string) []string {
	seen := map[string]bool{}
	var missing []string
	for _, line := range strings.Split(out, "\n") {
		line = strings.TrimSpace(line)
		if !strings.Contains(line, "not found") {
			continue
		}
		var what string
		if lib, rest, ok := strings.Cut(line, ": version `"); ok {
			sym, _, _ := strings.Cut(rest, "'")
			what = filepath.Base(lib) + " is too old (needs " + sym + ")"
		} else {
			what = strings.TrimSpace(strings.SplitN(line, "=>", 2)[0]) + " is not installed"
		}
		if !seen[what] {
			seen[what] = true
			missing = append(missing, what)
		}
	}
	return missing
}

func findFile(dir, name string) (string, error) {
	var found string
	_ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
		if err != nil || found != "" {
			return nil //nolint:nilerr // an unreadable subtree is not the binary
		}
		if !d.IsDir() && d.Name() == name {
			found = p
		}
		return nil
	})
	if found == "" {
		return "", fmt.Errorf("the build has no %s in it", name)
	}
	return found, nil
}

func (in *Installer) progress(st *Status, format string, args ...any) {
	st.State, st.Detail = Installing, fmt.Sprintf(format, args...)
	in.report(*st)
}

func (in *Installer) report(st Status) {
	if in.OnStatus != nil {
		in.OnStatus(st)
	}
}

func runWithStdin(ctx context.Context, env []string, stdin, name string, args ...string) ([]byte, error) {
	cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // fixed commands and paths we installed
	cmd.Env = append(os.Environ(), env...)
	cmd.Stdin = strings.NewReader(stdin)
	return cmd.CombinedOutput()
}

func (in *Installer) run(ctx context.Context, env []string, name string, args ...string) ([]byte, error) {
	if in.Run != nil {
		return in.Run(ctx, env, name, args...)
	}
	cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // fixed commands and paths we installed
	cmd.Env = append(os.Environ(), env...)
	return cmd.CombinedOutput()
}

func (in *Installer) root() string {
	if in.Root != "" {
		return in.Root
	}
	return DefaultRoot
}

func (in *Installer) log() *slog.Logger {
	if in.Log != nil {
		return in.Log
	}
	return slog.Default()
}

func (in *Installer) client() *http.Client {
	if in.HTTP != nil {
		return in.HTTP
	}
	return &http.Client{Timeout: 30 * time.Minute}
}

func (in *Installer) apiBase() string {
	if in.APIBase != "" {
		return in.APIBase
	}
	return "https://api.github.com"
}

func (in *Installer) downloadBase() string {
	if in.DownloadBase != "" {
		return in.DownloadBase
	}
	return "https://github.com"
}

func lastLine(b []byte) string {
	lines := strings.Split(strings.TrimSpace(string(b)), "\n")
	return strings.TrimSpace(lines[len(lines)-1])
}

func firstVersionLine(b []byte) string {
	for _, l := range strings.Split(string(b), "\n") {
		if strings.Contains(strings.ToLower(l), "version") {
			return strings.TrimSpace(l)
		}
	}
	// vllm --version says the number and nothing else.
	for _, l := range strings.Split(string(b), "\n") {
		if l = strings.TrimSpace(l); l != "" && l[0] >= '0' && l[0] <= '9' {
			return l
		}
	}
	return ""
}