llmgw/internal/agent/provision/install.go

package provision

import (
	"archive/tar"
	"compress/gzip"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"log/slog"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
)

// Want is one engine at one version.
type Want struct {
	Engine  string
	Version string
}

// The states an engine's installation can be in.
const (
	Installing  = "installing"
	Ready       = "ready"
	Failed      = "failed"
	Unsupported = "unsupported"
)

// Status is where one engine's installation stands.
type Status struct {
	Engine  string
	Version string
	Variant string
	State   string
	// Detail is for a person: what is happening, or what went wrong.
	Detail string
	// Binary is the executable, once ready.
	Binary string
}

// Installer installs engines under one directory.
type Installer struct {
	// Root is where engines are installed, one directory per build:
	// <root>/<engine>/<version>-<variant>.
	Root string
	Host discovery.Host
	Log  *slog.Logger
	HTTP *http.Client
	// OnStatus hears every change, so a download that takes minutes is seen
	// to be under way rather than silent.
	OnStatus func(Status)

	// Run executes a command; a seam for tests. Nil runs it for real.
	Run func(ctx context.Context, env []string, name string, args ...string) ([]byte, error)
	// APIBase and DownloadBase default to GitHub; tests point them at a fake.
	APIBase      string
	DownloadBase string
}

// DefaultRoot is where engines are installed when nothing says otherwise.
const DefaultRoot = "/opt/llmgw/engines"

// marker is written into a build's directory once it has been installed and
// seen to run. Its absence means the directory is not to be trusted.
const marker = ".llmgw-installed"

type markerFile struct {
	Engine    string            `json:"engine"`
	Version   string            `json:"version"`
	Variant   string            `json:"variant"`
	Binary    string            `json:"binary"`
	SHA256    map[string]string `json:"sha256"`
	Installed time.Time         `json:"installed"`
}

// Converge brings every wanted engine to ready, or says why it cannot.
//
// An engine already installed is checked again rather than believed: its
// directory can be damaged, or a library it needs removed by somebody's
// upgrade, and that is found here rather than by the next replica to fail.
func (in *Installer) Converge(ctx context.Context, wants []Want) []Status {
	out := make([]Status, 0, len(wants))
	for _, w := range wants {
		st := in.one(ctx, w)
		in.report(st)
		out = append(out, st)
	}
	return out
}

func (in *Installer) one(ctx context.Context, w Want) Status {
	st := Status{Engine: w.Engine, Version: w.Version}
	recipe, ok := Recipes[w.Engine]
	if !ok {
		st.State = Unsupported
		st.Detail = "this agent does not know how to install " + w.Engine +
			"; install it by hand and name it in the agent's configuration"
		return st
	}
	build, err := recipe.Choose(w.Version, in.Host)
	if err != nil {
		st.State, st.Detail = Unsupported, err.Error()
		return st
	}
	st.Variant = build.Variant
	dir := filepath.Join(in.root(), w.Engine, w.Version+"-"+build.Variant)

	// Installed before: check it still runs, and trust nothing else about it.
	if _, err := os.Stat(filepath.Join(dir, marker)); err == nil {
		// The packages are checked too. Removing one is exactly the kind of
		// change an unrelated upgrade makes, and ldd below would catch it only
		// as a missing library with no hint of which package held it.
		if err := in.packages(ctx, build.Packages, &st); err != nil {
			st.State, st.Detail = Failed, err.Error()
			return st
		}
		bin, version, err := in.verify(ctx, dir, build)
		if err == nil {
			return in.ready(st, build, bin, version)
		}
		in.log().Warn("an installed engine no longer runs; installing it again",
			"engine", w.Engine, "version", w.Version, "err", err)
	}

	if err := in.install(ctx, recipe, w, build, dir, &st); err != nil {
		st.State, st.Detail, st.Binary = Failed, err.Error(), ""
		return st
	}
	bin, version, err := in.verify(ctx, dir, build)
	if err != nil {
		st.State, st.Detail = Failed, err.Error()
		return st
	}
	return in.ready(st, build, bin, version)
}

func (in *Installer) ready(st Status, b Build, bin, version string) Status {
	st.State, st.Binary = Ready, bin
	st.Detail = strings.TrimSpace(filepath.Base(bin) + " " + version)
	var missing []string
	for _, f := range b.CPUFlags {
		if !in.Host.HasCPUFlag(f) {
			missing = append(missing, f)
		}
	}
	if len(missing) > 0 {
		// Said on every report, not once in a log: it is the difference between
		// five tokens a second and forty, and a VM given QEMU's default CPU
		// model looks entirely healthy otherwise.
		st.Detail += "; this CPU lacks " + strings.Join(missing, ", ") +
			", so it will run several times slower (a VM needs CPU type \"host\")"
	}
	return st
}

// install fetches, checks and unpacks a build into dir.
//
// Into a scratch directory renamed into place at the end, so a failure halfway
// leaves nothing that looks installed.
func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir string, st *Status) error {
	if err := in.packages(ctx, b.Packages, st); err != nil {
		return err
	}

	in.progress(st, "asking GitHub for the checksums of %s %s", w.Engine, w.Version)
	sums, err := in.checksums(ctx, r.Repo, w.Version)
	if err != nil {
		return err
	}

	tmp := dir + ".partial"
	_ = os.RemoveAll(tmp)
	if err := os.MkdirAll(tmp, 0o755); err != nil {
		return err
	}
	defer os.RemoveAll(tmp) //nolint:errcheck // gone either way once renamed

	got := map[string]string{}
	for _, asset := range b.Assets {
		want, ok := sums[asset]
		if !ok {
			// Refused rather than installed unchecked: this is a binary that
			// will run as root on every node.
			return fmt.Errorf("%s %s publishes no checksum for %s, so it cannot be verified",
				w.Engine, w.Version, asset)
		}
		archive := filepath.Join(tmp, asset+".download")
		if err := in.download(ctx, r.Repo, w.Version, asset, archive, want, st); err != nil {
			return err
		}
		in.progress(st, "unpacking %s", asset)
		if err := untarGz(archive, tmp); err != nil {
			return fmt.Errorf("unpacking %s: %w", asset, err)
		}
		_ = os.Remove(archive)
		got[asset] = want
	}

	in.progress(st, "checking that %s runs", b.Binary)
	bin, _, err := in.verify(ctx, tmp, b)
	if err != nil {
		return err
	}
	rel, _ := filepath.Rel(tmp, bin)
	m, _ := json.MarshalIndent(markerFile{
		Engine: w.Engine, Version: w.Version, Variant: b.Variant,
		Binary: rel, SHA256: got, Installed: time.Now().UTC(),
	}, "", "  ")
	if err := os.WriteFile(filepath.Join(tmp, marker), m, 0o644); err != nil { //nolint:gosec // not secret
		return err
	}
	_ = os.RemoveAll(dir)
	if err := os.Rename(tmp, dir); err != nil {
		return err
	}
	in.log().Info("installed an engine", "engine", w.Engine, "version", w.Version,
		"variant", b.Variant, "dir", dir)
	return nil
}

// verify finds the binary and runs it.
//
// Two checks, because they fail differently. ldd names a missing library
// outright, where running the binary would only say it could not start. And
// running it is the only proof that the rest — the right instruction set, the
// right driver — is in place.
func (in *Installer) verify(ctx context.Context, dir string, b Build) (bin, version string, err error) {
	bin, err = findFile(dir, b.Binary)
	if err != nil {
		return "", "", err
	}
	if out, err := in.run(ctx, nil, "ldd", bin); err == nil {
		if missing := lddMissing(string(out)); len(missing) > 0 {
			return "", "", fmt.Errorf("%s cannot run on this system: %s", b.Binary, strings.Join(missing, "; "))
		}
	}
	tctx, cancel := context.WithTimeout(ctx, 60*time.Second)
	defer cancel()
	out, err := in.run(tctx, nil, bin, "--version")
	if err != nil {
		return "", "", fmt.Errorf("%s --version failed: %v: %s", b.Binary, err, lastLine(out))
	}
	return bin, firstVersionLine(out), nil
}

// packages installs the operating system packages a build needs.
//
// Only those missing, and from the node's own package sources: a package the
// system already has is not touched, so an engine never upgrades something
// the rest of the machine depends on.
func (in *Installer) packages(ctx context.Context, byFamily map[string][]string, st *Status) error {
	family := in.Host.Family()
	pkgs := byFamily[family]
	if len(pkgs) == 0 {
		// A system of no family this knows is not refused: its packages may
		// well be there, and if they are not, ldd names the missing library.
		return nil
	}
	pm := packageManagers[family]
	var missing []string
	for _, p := range pkgs {
		if _, err := in.run(ctx, nil, pm.query[0], append(pm.query[1:], p)...); err != nil {
			missing = append(missing, p)
		}
	}
	if len(missing) == 0 {
		return nil
	}
	tool, cmds := pm.install(in, missing)
	if tool == "" {
		return fmt.Errorf("%s is missing and no package manager was found to install it",
			strings.Join(missing, ", "))
	}
	in.progress(st, "installing %s with %s", strings.Join(missing, ", "), tool)
	for _, c := range cmds {
		if err := in.runWaitingForLock(ctx, pm.env, c, st); err != nil {
			return err
		}
	}
	return nil
}

// InstallPackages installs whichever of these packages the machine lacks,
// from its own package sources, waiting out another package manager's lock.
// byFamily is keyed by discovery.Host.Family: "deb" or "rpm".
func (in *Installer) InstallPackages(ctx context.Context, byFamily map[string][]string) error {
	var st Status
	return in.packages(ctx, byFamily, &st)
}

// lockWait bounds how long a package command waits for another to finish.
const lockWait = 10 * time.Minute

// lockRetry is how often a locked package manager is tried again.
var lockRetry = 10 * time.Second

// runWaitingForLock runs a package command, waiting while another holds the
// package manager's lock.
//
// A machine a minute old is running cloud-init's and unattended-upgrades' own
// apt. apt's DPkg::Lock::Timeout covers dpkg's lock and not the one `apt-get
// update` takes on its lists, so the first install on a new node failed with
// "Unable to lock directory /var/lib/apt/lists/" — found by building one.
func (in *Installer) runWaitingForLock(ctx context.Context, env, c []string, st *Status) error {
	deadline := time.Now().Add(lockWait)
	for {
		out, err := in.run(ctx, env, c[0], c[1:]...)
		if err == nil {
			return nil
		}
		if !lockHeld(string(out)) || time.Now().After(deadline) {
			return fmt.Errorf("%s: %v: %s", strings.Join(c, " "), err, lastLine(out))
		}
		in.progress(st, "waiting for another %s to finish (a new machine runs its own updates first)", c[0])
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-time.After(lockRetry):
		}
	}
}

// lockHeld reports whether a package command failed only because another
// held the lock.
func lockHeld(out string) bool {
	for _, s := range []string{"Unable to lock", "Could not get lock", "is locked by another process",
		"waiting for process with pid", "System management is locked"} {
		if strings.Contains(out, s) {
			return true
		}
	}
	return false
}

// packageManager is how one packaging family asks and installs.
type packageManager struct {
	// query exits zero when the package named after it is installed.
	query []string
	env   []string
	// install picks the tool present on this machine and the commands to run.
	install func(in *Installer, pkgs []string) (tool string, cmds [][]string)
}

var packageManagers = map[string]packageManager{
	"deb": {
		// dpkg -s exits zero for a package merely known about, so the status
		// is asked for and checked by the command itself: ${db:Status-Abbrev}
		// is "ii " only when installed.
		query: []string{"sh", "-c", `test "$(dpkg-query -W -f='${db:Status-Abbrev}' "$0" 2>/dev/null)" = "ii "`},
		env:   []string{"DEBIAN_FRONTEND=noninteractive"},
		install: func(_ *Installer, pkgs []string) (string, [][]string) {
			// A lock timeout rather than failing on the lock: a machine a
			// minute old is running cloud-init's and unattended-upgrades' own
			// apt, and the first attempt on every new node would fail.
			lock := "-o=DPkg::Lock::Timeout=600"
			return "apt-get", [][]string{
				{"apt-get", lock, "update"},
				append([]string{"apt-get", lock, "install", "-y", "--no-install-recommends"}, pkgs...),
			}
		},
	},
	"rpm": {
		query: []string{"rpm", "-q"},
		install: func(in *Installer, pkgs []string) (string, [][]string) {
			for _, tool := range []string{"dnf", "yum"} {
				if in.has(tool) {
					return tool, [][]string{append([]string{tool, "install", "-y"}, pkgs...)}
				}
			}
			if in.has("zypper") {
				return "zypper", [][]string{append([]string{"zypper", "--non-interactive", "install"}, pkgs...)}
			}
			return "", nil
		},
	},
}

// has reports whether a command is on PATH. Asked through the same seam as
// every other command, so a test can say which tools a machine has.
func (in *Installer) has(tool string) bool {
	_, err := in.run(context.Background(), nil, "sh", "-c", `command -v "$0"`, tool)
	return err == nil
}

// checksums reads the sha256 GitHub publishes for each asset of a release.
func (in *Installer) checksums(ctx context.Context, repo, tag string) (map[string]string, error) {
	url := strings.TrimSuffix(in.apiBase(), "/") + "/repos/" + repo + "/releases/tags/" + tag
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Accept", "application/vnd.github+json")
	resp, err := in.client().Do(req)
	if err != nil {
		return nil, fmt.Errorf("reaching GitHub for %s %s: %w", repo, tag, err)
	}
	defer resp.Body.Close()
	switch {
	case resp.StatusCode == http.StatusNotFound:
		return nil, fmt.Errorf("%s has no release %s", repo, tag)
	case resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests:
		// Unauthenticated callers get sixty requests an hour per address, and
		// a whole fleet behind one NAT shares them.
		return nil, fmt.Errorf("GitHub is rate-limiting this address (%s); it retries later", resp.Status)
	case resp.StatusCode != http.StatusOK:
		return nil, fmt.Errorf("GitHub answered %s for %s %s", resp.Status, repo, tag)
	}
	var rel struct {
		Assets []struct {
			Name   string `json:"name"`
			Digest string `json:"digest"`
		} `json:"assets"`
	}
	if err := json.NewDecoder(resp.Body).Decode(&rel); err != nil {
		return nil, fmt.Errorf("reading %s %s from GitHub: %w", repo, tag, err)
	}
	out := map[string]string{}
	for _, a := range rel.Assets {
		if sum, ok := strings.CutPrefix(a.Digest, "sha256:"); ok && len(sum) == 64 {
			out[a.Name] = sum
		}
	}
	return out, nil
}

func (in *Installer) download(ctx context.Context, repo, tag, asset, dst, want string, st *Status) error {
	url := strings.TrimSuffix(in.downloadBase(), "/") + "/" + repo + "/releases/download/" + tag + "/" + asset
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return err
	}
	resp, err := in.client().Do(req)
	if err != nil {
		return fmt.Errorf("downloading %s: %w", asset, err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("downloading %s: %s", asset, resp.Status)
	}
	if resp.ContentLength > 0 {
		in.progress(st, "downloading %s (%d MB)", asset, resp.ContentLength>>20)
	} else {
		in.progress(st, "downloading %s", asset)
	}
	f, err := os.Create(dst) //nolint:gosec // inside the install root
	if err != nil {
		return err
	}
	h := sha256.New()
	_, err = io.Copy(io.MultiWriter(f, h), resp.Body)
	if cerr := f.Close(); err == nil {
		err = cerr
	}
	if err != nil {
		return fmt.Errorf("downloading %s: %w", asset, err)
	}
	if got := hex.EncodeToString(h.Sum(nil)); got != want {
		return fmt.Errorf("%s does not match the checksum its publisher gave (got %s, expected %s)",
			asset, got[:12], want[:12])
	}
	return nil
}

// untarGz unpacks an archive under dir, refusing anything that would land
// outside it. A release archive is somebody else's file, and a path of
// "../../etc/cron.d/x" in it would otherwise be written as root.
func untarGz(archive, dir string) error {
	f, err := os.Open(archive) //nolint:gosec // inside the install root
	if err != nil {
		return err
	}
	defer f.Close()
	gz, err := gzip.NewReader(f)
	if err != nil {
		return err
	}
	tr := tar.NewReader(gz)
	for {
		h, err := tr.Next()
		if errors.Is(err, io.EOF) {
			return nil
		}
		if err != nil {
			return err
		}
		target, err := within(dir, h.Name)
		if err != nil {
			return err
		}
		switch h.Typeflag {
		case tar.TypeDir:
			if err := os.MkdirAll(target, 0o755); err != nil {
				return err
			}
		case tar.TypeReg:
			if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
				return err
			}
			out, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fs.FileMode(h.Mode)&0o755) //nolint:gosec // checked above
			if err != nil {
				return err
			}
			if _, err := io.Copy(out, tr); err != nil { //nolint:gosec // a release archive, size bounded by its checksum
				out.Close()
				return err
			}
			if err := out.Close(); err != nil {
				return err
			}
		case tar.TypeSymlink:
			// Libraries are versioned through symlinks, and those are
			// relative. One pointing anywhere else is refused.
			if filepath.IsAbs(h.Linkname) {
				return fmt.Errorf("%s links outside the archive", h.Name)
			}
			if _, err := within(dir, filepath.Join(filepath.Dir(h.Name), h.Linkname)); err != nil {
				return fmt.Errorf("%s links outside the archive", h.Name)
			}
			if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
				return err
			}
			_ = os.Remove(target)
			if err := os.Symlink(h.Linkname, target); err != nil {
				return err
			}
		default:
			// Devices, FIFOs and hard links have no business in a release.
		}
	}
}

func within(dir, name string) (string, error) {
	target := filepath.Join(dir, name) //nolint:gosec // checked on the next line
	rel, err := filepath.Rel(dir, target)
	if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
		return "", fmt.Errorf("%s would unpack outside the install directory", name)
	}
	return target, nil
}

// lddMissing reads what ldd could not satisfy, in two shapes:
//
//	libgomp.so.1 => not found
//	/lib64/libstdc++.so.6: version `GLIBCXX_3.4.30' not found (required by ...)
//
// The second is a system too old for the build — RHEL 9 against a build that
// wants GCC 12's libstdc++ — and is said as such, rather than as a library
// that is plainly there being "not found".
func lddMissing(out string) []string {
	seen := map[string]bool{}
	var missing []string
	for _, line := range strings.Split(out, "\n") {
		line = strings.TrimSpace(line)
		if !strings.Contains(line, "not found") {
			continue
		}
		var what string
		if lib, rest, ok := strings.Cut(line, ": version `"); ok {
			sym, _, _ := strings.Cut(rest, "'")
			what = filepath.Base(lib) + " is too old (needs " + sym + ")"
		} else {
			what = strings.TrimSpace(strings.SplitN(line, "=>", 2)[0]) + " is not installed"
		}
		if !seen[what] {
			seen[what] = true
			missing = append(missing, what)
		}
	}
	return missing
}

func findFile(dir, name string) (string, error) {
	var found string
	_ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
		if err != nil || found != "" {
			return nil //nolint:nilerr // an unreadable subtree is not the binary
		}
		if !d.IsDir() && d.Name() == name {
			found = p
		}
		return nil
	})
	if found == "" {
		return "", fmt.Errorf("the build has no %s in it", name)
	}
	return found, nil
}

func (in *Installer) progress(st *Status, format string, args ...any) {
	st.State, st.Detail = Installing, fmt.Sprintf(format, args...)
	in.report(*st)
}

func (in *Installer) report(st Status) {
	if in.OnStatus != nil {
		in.OnStatus(st)
	}
}

func (in *Installer) run(ctx context.Context, env []string, name string, args ...string) ([]byte, error) {
	if in.Run != nil {
		return in.Run(ctx, env, name, args...)
	}
	cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // fixed commands and paths we installed
	cmd.Env = append(os.Environ(), env...)
	return cmd.CombinedOutput()
}

func (in *Installer) root() string {
	if in.Root != "" {
		return in.Root
	}
	return DefaultRoot
}

func (in *Installer) log() *slog.Logger {
	if in.Log != nil {
		return in.Log
	}
	return slog.Default()
}

func (in *Installer) client() *http.Client {
	if in.HTTP != nil {
		return in.HTTP
	}
	return &http.Client{Timeout: 30 * time.Minute}
}

func (in *Installer) apiBase() string {
	if in.APIBase != "" {
		return in.APIBase
	}
	return "https://api.github.com"
}

func (in *Installer) downloadBase() string {
	if in.DownloadBase != "" {
		return in.DownloadBase
	}
	return "https://github.com"
}

func lastLine(b []byte) string {
	lines := strings.Split(strings.TrimSpace(string(b)), "\n")
	return strings.TrimSpace(lines[len(lines)-1])
}

func firstVersionLine(b []byte) string {
	for _, l := range strings.Split(string(b), "\n") {
		if strings.Contains(strings.ToLower(l), "version") {
			return strings.TrimSpace(l)
		}
	}
	return ""
}