llmgw/internal/agent/slurmnode/slurmnode.go

// Package slurmnode keeps this machine's part in the Slurm cluster run beside
// llmgw (BACKLOG P), as the leader says: slurmctld on the controller, slurmd
// on a node, munge on both, installed from the machine's own package sources.
//
// What the leader sends is wanted state, kept by Keeper's loop: sent again,
// or checked again a minute later, it changes nothing that is already so.
package slurmnode

import (
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"log/slog"
	"net/http"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"slices"
	"strconv"
	"strings"
	"sync"
	"time"

	"google.golang.org/protobuf/proto"

	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
	"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)

// Host is what Keeper does to the machine; Local in the agent, a fake in
// tests.
type Host interface {
	Run(ctx context.Context, env []string, name string, args ...string) ([]byte, error)
	LookPath(name string) (string, error)
	ReadFile(path string) ([]byte, error)
	// WriteFile writes b at path with mode, owned by owner ("" for root),
	// making its directory.
	WriteFile(path string, b []byte, mode fs.FileMode, owner string) error
	// MkdirAll makes path, owned by owner.
	MkdirAll(path string, mode fs.FileMode, owner string) error
	Remove(path string) error
	// Download fetches url to path, kept only if its SHA-256 is sha; a file
	// already there with it is not fetched again.
	Download(ctx context.Context, url, path, sha string) error
}

// Installer installs packages by family, "deb" and "rpm".
type Installer interface {
	InstallPackages(ctx context.Context, byFamily map[string][]string) error
}

// Reasons llmgw drains a node with; one drained for another reason, by a
// Slurm administrator, is never resumed by llmgw.
const (
	ReasonServing = "llmgw: serving models"
	reasonPrefix  = "llmgw"
)

// DropIn is the unit drop-in that starts slurmd configless.
const DropIn = "/etc/systemd/system/slurmd.service.d/llmgw.conf"

// Keeper keeps the machine as the last Slurm message says.
type Keeper struct {
	host   Host
	inst   Installer
	log    *slog.Logger
	marker string

	mu   sync.Mutex
	want *llmgwv1.Slurm
	kick chan struct{}
	// resend has the next status sent even if unchanged: a new stream's
	// control plane may never have had it.
	resend bool
	// installed is true once the packages were found or put there, so a
	// minute's check does not look again.
	installed map[string]bool
	files     string
	// acctDone is the digest of the password the database's user was last
	// given; clusterAdded the cluster last found in the accounting.
	acctDone     string
	clusterAdded string
}

// New is a Keeper; marker is a file it keeps while it runs Slurm here, so
// an agent started again knows to stop what it started when told it has no
// part any more.
func New(host Host, inst Installer, marker string, log *slog.Logger) *Keeper {
	if log == nil {
		log = slog.Default()
	}
	return &Keeper{host: host, inst: inst, log: log, marker: marker,
		kick: make(chan struct{}, 1), installed: map[string]bool{}}
}

// Apply is a new message from the leader: kept from now on.
func (k *Keeper) Apply(m *llmgwv1.Slurm) {
	k.mu.Lock()
	k.want = proto.CloneOf(m)
	k.mu.Unlock()
	select {
	case k.kick <- struct{}{}:
	default:
	}
}

// HasPart is whether the last message gives this machine a part in Slurm.
func (k *Keeper) HasPart() bool {
	k.mu.Lock()
	defer k.mu.Unlock()
	return k.want.GetController() || k.want.GetNode()
}

// Resend has the status sent again on the next pass, which it starts now.
func (k *Keeper) Resend() {
	k.mu.Lock()
	k.resend = true
	k.mu.Unlock()
	select {
	case k.kick <- struct{}{}:
	default:
	}
}

// Every is how often the machine is checked again without a new message.
var Every = time.Minute

// resendAfter is how often an unchanged status is sent anyway.
const resendAfter = 5 * time.Minute

// Run keeps the machine until ctx ends, sending its status when it changes
// and every few minutes while it has a part.
func (k *Keeper) Run(ctx context.Context, send func(*llmgwv1.SlurmStatus)) {
	t := time.NewTicker(Every)
	defer t.Stop()
	var last *llmgwv1.SlurmStatus
	var sent time.Time
	for {
		select {
		case <-ctx.Done():
			return
		case <-k.kick:
		case <-t.C:
		}
		k.mu.Lock()
		want, again := k.want, k.resend
		k.resend = false
		k.mu.Unlock()
		if want == nil {
			continue
		}
		st := k.Reconcile(ctx, want)
		if st == nil {
			last = nil
			continue
		}
		if again || !proto.Equal(st, last) || time.Since(sent) > resendAfter {
			send(st)
			last, sent = st, time.Now()
		}
	}
}

// Reconcile makes the machine what want says once, and answers its status;
// nil when it has no part in Slurm.
func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.SlurmStatus {
	ctx, cancel := context.WithTimeout(ctx, 20*time.Minute)
	defer cancel()
	if !want.GetController() && !want.GetNode() {
		k.stop(ctx)
		return nil
	}
	st := &llmgwv1.SlurmStatus{}
	fail := func(what string, err error) *llmgwv1.SlurmStatus {
		st.Error = what + ": " + err.Error()
		k.log.Warn("keeping Slurm on this node: "+what, "err", err)
		return st
	}
	if err := k.ensurePackages(ctx, want); err != nil {
		return fail("installing Slurm", err)
	}
	// From the daemon, not scontrol: a client command looks for its files
	// even to say its version, and a configless node has none before slurmd
	// has fetched them.
	daemon := "slurmd"
	if !want.GetNode() {
		daemon = "slurmctld"
	}
	out, err := k.host.Run(ctx, nil, daemon, "-V")
	if err != nil {
		return fail("reading Slurm's version", fmt.Errorf("%v: %s", err, lastLine(out)))
	}
	v, err := slurm.ParseVersion(string(out))
	if err != nil {
		return fail("reading Slurm's version", err)
	}
	st.Version = strings.TrimSpace(string(out))
	if i := strings.LastIndexByte(st.Version, ' '); i >= 0 {
		st.Version = st.Version[i+1:]
	}
	if slurm.TooOld(v) {
		return fail("Slurm", fmt.Errorf("%d.%02d is older than %d.%02d, the first to run jobs in cgroup v2; "+
			"a newer release of the distribution, or Slurm's own packages, has one", v[0], v[1],
			slurm.MinVersion[0], slurm.MinVersion[1]))
	}
	if err := k.host.WriteFile(k.marker, []byte("llmgw runs Slurm here (BACKLOG P)\n"), 0o644, ""); err != nil {
		return fail("keeping its marker", err)
	}
	if err := k.munge(ctx, want.GetMungeKey()); err != nil {
		return fail("munge", err)
	}
	// The shared files, then the users whose homes are on them. Neither
	// keeps Slurm from running: what goes wrong is said, and Slurm kept.
	var shErr, usErr error
	st.Shared, shErr = k.shared(ctx, want)
	if shErr != nil {
		k.log.Warn("keeping the Slurm cluster's shared files on this node", "err", shErr)
	}
	if shErr == nil || !want.GetMakeHomes() || want.GetShared().GetPath() == "" {
		st.Users, usErr = k.users(ctx, want)
	} else {
		// Homes made now would be made beside the shared files, not on them.
		usErr = errors.New("waiting for the shared files, where the homes are")
	}
	if usErr != nil {
		k.log.Warn("keeping the Slurm users on this node", "err", usErr)
	}
	// Jobs in containers; Slurm runs without them, and is kept anyway.
	var ctErr error
	if st.Containers, ctErr = k.containers(ctx, want); ctErr != nil {
		k.log.Warn("setting up jobs in containers on this node", "err", ctErr)
	}
	defer func() {
		switch {
		case st.Error != "":
		case shErr != nil:
			st.Error = "shared files: " + shErr.Error()
		case usErr != nil:
			st.Error = "users: " + usErr.Error()
		case ctErr != nil:
			st.Error = "containers: " + ctErr.Error()
		}
	}()
	if want.GetController() {
		// Accounting first: slurmctld reports to it from its first start.
		// Without it slurmctld still runs jobs, a pass of backfill late
		// each; so a failure here is said, and slurmctld is kept anyway.
		var acctErr error
		if acctErr = k.accounting(ctx, want.GetMungeKey(), want.GetFiles()); acctErr != nil {
			k.log.Warn("keeping Slurm's accounting on this node", "err", acctErr)
		}
		st.SlurmdbdRunning = k.active(ctx, "slurmdbd")
		defer func() {
			if acctErr != nil && st.Error == "" {
				st.Error = "slurmdbd: " + acctErr.Error()
			}
		}()
		sum, err := k.controller(ctx, want.GetFiles())
		st.FilesSha256 = sum
		if err != nil {
			return fail("slurmctld", err)
		}
		st.SlurmctldRunning = k.active(ctx, "slurmctld")
	}
	if want.GetNode() {
		if err := k.node(ctx, want, st); err != nil {
			return fail("slurmd", err)
		}
	}
	return st
}

func (k *Keeper) ensurePackages(ctx context.Context, want *llmgwv1.Slurm) error {
	need := map[string][2]string{
		"munged":   {"munge", "munge"},
		"scontrol": {"slurm-client", "slurm"},
	}
	if want.GetController() {
		need["slurmctld"] = [2]string{"slurmctld", "slurm-slurmctld"}
		need["slurmdbd"] = [2]string{"slurmdbd", "slurm-slurmdbd"}
		need["mariadbd"] = [2]string{"mariadb-server", "mariadb-server"}
	}
	if want.GetNode() {
		need["slurmd"] = [2]string{"slurmd", "slurm-slurmd"}
	}
	sharedPackages(want, need)
	pkgs := map[string][]string{}
	for bin, p := range need {
		if k.installed[bin] {
			continue
		}
		if _, err := k.host.LookPath(bin); err == nil {
			k.installed[bin] = true
			continue
		}
		pkgs["deb"] = append(pkgs["deb"], p[0])
		pkgs["rpm"] = append(pkgs["rpm"], p[1])
	}
	if len(pkgs) == 0 {
		return nil
	}
	slices.Sort(pkgs["deb"])
	slices.Sort(pkgs["rpm"])
	if k.inst == nil {
		return fmt.Errorf("%s missing, and this agent installs nothing", strings.Join(pkgs["deb"], ", "))
	}
	k.log.Info("installing Slurm, as the cluster's settings say", "packages", pkgs["deb"])
	if err := k.inst.InstallPackages(ctx, pkgs); err != nil {
		return err
	}
	if slices.Contains(pkgs["deb"], "mariadb-server") {
		// Installed by llmgw: its settings are llmgw's to keep. One that was
		// here before is another's, and is left as it is.
		if err := k.host.WriteFile(k.mariaDBMarker(), []byte("llmgw installed MariaDB here, for Slurm's accounting\n"),
			0o644, ""); err != nil {
			return err
		}
	}
	for bin := range need {
		if _, err := k.host.LookPath(bin); err != nil {
			return fmt.Errorf("%s is still missing after installing %s", bin, strings.Join(pkgs["deb"], ", "))
		}
		k.installed[bin] = true
	}
	return nil
}

// munge writes the cluster's key and keeps munged running on it.
func (k *Keeper) munge(ctx context.Context, key []byte) error {
	if len(key) < 32 {
		return errors.New("the leader sent no key")
	}
	changed, err := k.write(slurm.MungeKeyPath, key, 0o400, "munge")
	if err != nil {
		return err
	}
	if changed {
		k.log.Info("munge key written; munge starts again on it")
		return k.systemctl(ctx, "enable", "--now", "munge", "&&restart")
	}
	if !k.active(ctx, "munge") {
		return k.systemctl(ctx, "enable", "--now", "munge")
	}
	return nil
}

// controller writes the files and keeps slurmctld running on them,
// answering their digest.
func (k *Keeper) controller(ctx context.Context, files map[string]string) (string, error) {
	if files["slurm.conf"] == "" {
		return "", errors.New("the leader sent no slurm.conf")
	}
	for _, d := range []string{slurm.StateDir, slurm.LogDir} {
		if err := k.host.MkdirAll(d, 0o755, "slurm"); err != nil {
			return "", err
		}
	}
	names := make([]string, 0, len(files))
	for n := range files {
		names = append(names, n)
	}
	slices.Sort(names)
	h := sha256.New()
	changed := false
	for _, n := range names {
		if strings.ContainsAny(n, "/\\") || n == "" || n == "." || n == ".." {
			return "", fmt.Errorf("the leader sent a file named %q", n)
		}
		fmt.Fprintf(h, "%s\x00%s\x00", n, files[n])
		c, err := k.write(filepath.Join(slurm.ConfDir, n), []byte(files[n]), 0o644, "")
		if err != nil {
			return "", err
		}
		changed = changed || c
	}
	sum := hex.EncodeToString(h.Sum(nil))
	if changed || sum != k.files {
		if err := k.systemctl(ctx, "enable", "slurmctld"); err != nil {
			return sum, err
		}
		// Nodes added or taken away need a restart before 23.11 and a
		// reconfigure after it; a restart does both, and running jobs live
		// through it.
		k.log.Info("slurmctld starts again on new files")
		if err := k.systemctl(ctx, "restart", "slurmctld"); err != nil {
			return sum, err
		}
		if err := k.waitPing(ctx); err != nil {
			return sum, err
		}
		// Hands the configless slurmd the new files.
		if out, err := k.host.Run(ctx, nil, "scontrol", "reconfigure"); err != nil {
			return sum, fmt.Errorf("scontrol reconfigure: %v: %s", err, lastLine(out))
		}
		k.files = sum
		return sum, nil
	}
	if !k.active(ctx, "slurmctld") {
		return sum, k.systemctl(ctx, "enable", "--now", "slurmctld")
	}
	return sum, nil
}

func (k *Keeper) waitPing(ctx context.Context) error {
	var last []byte
	for range 30 {
		out, err := k.host.Run(ctx, nil, "scontrol", "ping")
		if err == nil && bytes.Contains(out, []byte("UP")) {
			return nil
		}
		last = out
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-time.After(pingWait):
		}
	}
	return fmt.Errorf("slurmctld did not answer after starting: %s", lastLine(last))
}

var pingWait = time.Second

// node keeps slurmd running configless and puts the node in or out of the
// controller's hands as lent says.
func (k *Keeper) node(ctx context.Context, want *llmgwv1.Slurm, st *llmgwv1.SlurmStatus) error {
	name, server := want.GetName(), want.GetConfServer()
	if name == "" || server == "" {
		return errors.New("the leader sent no name or controller")
	}
	if strings.ContainsAny(name+server, " \n\t\"'\\$`") {
		return fmt.Errorf("the leader sent a name or controller with spaces or quotes in it")
	}
	for _, d := range []string{slurm.SpoolDir, slurm.LogDir} {
		if err := k.host.MkdirAll(d, 0o755, "slurm"); err != nil {
			return err
		}
	}
	bin, err := k.host.LookPath("slurmd")
	if err != nil {
		return err
	}
	unit := "# Written by llmgw (BACKLOG P): slurmd fetches its files from slurmctld.\n" +
		"[Unit]\nConditionPathExists=\nStartLimitIntervalSec=0\n" +
		// --systemd as the packaged unit has it (Type=notify, since 22.05):
		// without it systemd waits for a readiness that never comes.
		"[Service]\nExecStart=\nExecStart=" + bin + " --systemd --conf-server " + server + " -N " + name + "\n" +
		// A node new to the controller is refused until the leader has
		// written it into slurm.conf; tried again until it is.
		"Restart=on-failure\nRestartSec=15\n"
	changed, err := k.write(DropIn, []byte(unit), 0o644, "")
	if err != nil {
		return err
	}
	if out, err := k.host.Run(ctx, nil, "slurmd", "-C"); err == nil {
		st.Hardware = strings.TrimSpace(string(out))
	}
	if changed {
		if err := k.systemctl(ctx, "daemon-reload"); err != nil {
			return err
		}
		if err := k.systemctl(ctx, "enable", "slurmd"); err != nil {
			return err
		}
		if err := k.systemctl(ctx, "restart", "slurmd"); err != nil {
			return err
		}
	} else if !k.active(ctx, "slurmd") {
		if err := k.systemctl(ctx, "enable", "--now", "slurmd"); err != nil {
			return err
		}
	}
	st.SlurmdRunning = k.active(ctx, "slurmd")

	st.Jobs = k.localJobs(ctx)
	env := k.clientEnv()
	out, err := k.host.Run(ctx, env, "scontrol", "show", "node", "-o", name)
	if err != nil {
		// Not in slurm.conf yet: the leader writes it once this node's
		// hardware is in.
		return nil
	}
	ns, err := slurm.ParseNodeState(string(out))
	if err != nil {
		return nil
	}
	ours := ns.Reason == "" || strings.HasPrefix(ns.Reason, reasonPrefix)
	switch {
	case want.GetLent() && ns.Drained() && ours:
		k.log.Info("this node is lent to Slurm: resumed there")
		if out, err := k.host.Run(ctx, env, "scontrol", "update", "NodeName="+name, "State=RESUME"); err != nil {
			return fmt.Errorf("resuming the node: %v: %s", err, lastLine(out))
		}
	case !want.GetLent() && !strings.Contains(ns.State, "DRAIN"):
		k.log.Info("this node is llmgw's: drained in Slurm, its jobs left to finish")
		if out, err := k.host.Run(ctx, env, "scontrol", "update", "NodeName="+name, "State=DRAIN",
			"Reason="+ReasonServing); err != nil {
			return fmt.Errorf("draining the node: %v: %s", err, lastLine(out))
		}
	}
	if out, err := k.host.Run(ctx, env, "scontrol", "show", "node", "-o", name); err == nil {
		if ns2, err := slurm.ParseNodeState(string(out)); err == nil {
			ns = ns2
		}
	}
	st.State, st.Reason = ns.State, ns.Reason
	return nil
}

// localJobs counts the job steps running here, from their slurmstepd, not
// from the controller: a job outlives a controller that is down, and is
// what decides whether llmgw may have the cards back. The one slurmstepd
// that is no job's, "slurmstepd infinity" holding slurmd's cgroup v2 scope,
// is not counted. -1 when it cannot tell.
func (k *Keeper) localJobs(ctx context.Context) int32 {
	// pgrep exits 1 with no output when there is none.
	out, err := k.host.Run(ctx, nil, "pgrep", "-a", "-x", "slurmstepd")
	if err != nil && len(bytes.TrimSpace(out)) > 0 {
		return -1
	}
	n := int32(0)
	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
		f := strings.Fields(line)
		if len(f) < 2 || f[len(f)-1] == "infinity" {
			continue
		}
		n++
	}
	return n
}

// clientEnv points scontrol at slurmd's copy of the files on a node that is
// not the controller, which has none of its own.
func (k *Keeper) clientEnv() []string {
	if _, err := k.host.ReadFile(filepath.Join(slurm.ConfDir, "slurm.conf")); err == nil {
		return nil
	}
	return []string{"SLURM_CONF=" + filepath.Join(slurm.SpoolDir, "conf-cache", "slurm.conf")}
}

// stop stops what llmgw started here, if it did.
func (k *Keeper) stop(ctx context.Context) {
	if _, err := k.host.ReadFile(k.marker); err != nil {
		return
	}
	k.log.Info("this node has no part in Slurm any more: slurmd, slurmctld and slurmdbd stopped; the packages and MariaDB stay")
	// MariaDB stays: it may have been the machine's before, and holds the
	// jobs' records.
	for _, u := range []string{"slurmd", "slurmctld", "slurmdbd"} {
		_ = k.systemctl(ctx, "disable", "--now", u)
	}
	_ = k.host.Remove(DropIn)
	_ = k.systemctl(ctx, "daemon-reload")
	k.unshare(ctx, "", false, "")
	if err := k.removeUsers(ctx, k.madeUsers(), nil); err != nil {
		k.log.Warn("could not take away every Slurm user llmgw made", "err", err)
	}
	_ = k.host.Remove(k.marker)
	k.files, k.acctDone, k.clusterAdded = "", "", ""
}

// write writes a file when it differs, saying whether it did.
func (k *Keeper) write(path string, b []byte, mode fs.FileMode, owner string) (bool, error) {
	if old, err := k.host.ReadFile(path); err == nil && bytes.Equal(old, b) {
		return false, nil
	}
	return true, k.host.WriteFile(path, b, mode, owner)
}

// systemctl runs systemctl; a last argument "&&restart" restarts the unit
// after.
func (k *Keeper) systemctl(ctx context.Context, args ...string) error {
	restart := false
	if n := len(args); n > 0 && args[n-1] == "&&restart" {
		args, restart = args[:n-1], true
	}
	if out, err := k.host.Run(ctx, nil, "systemctl", args...); err != nil {
		return fmt.Errorf("systemctl %s: %v: %s", strings.Join(args, " "), err, lastLine(out))
	}
	if restart {
		u := args[len(args)-1]
		if out, err := k.host.Run(ctx, nil, "systemctl", "restart", u); err != nil {
			return fmt.Errorf("systemctl restart %s: %v: %s", u, err, lastLine(out))
		}
	}
	return nil
}

func (k *Keeper) active(ctx context.Context, unit string) bool {
	out, err := k.host.Run(ctx, nil, "systemctl", "is-active", unit)
	return err == nil && strings.TrimSpace(string(out)) == "active"
}

func lastLine(b []byte) string {
	s := strings.TrimSpace(string(b))
	if i := strings.LastIndexByte(s, '\n'); i >= 0 {
		s = s[i+1:]
	}
	return s
}

// Local is the machine the agent runs on.
type Local struct{}

// Run runs a command, with env added to the agent's.
func (Local) Run(ctx context.Context, env []string, name string, args ...string) ([]byte, error) {
	cmd := exec.CommandContext(ctx, name, args...)
	if len(env) > 0 {
		cmd.Env = append(os.Environ(), env...)
	}
	return cmd.CombinedOutput()
}

// LookPath finds a command, in the sbin directories too: an agent started
// by systemd has them, one started by hand may not.
func (Local) LookPath(name string) (string, error) {
	if p, err := exec.LookPath(name); err == nil {
		return p, nil
	}
	// /usr/libexec: Red Hat's mariadbd.
	for _, d := range []string{"/usr/sbin", "/sbin", "/usr/local/sbin", "/usr/libexec"} {
		p := filepath.Join(d, name)
		if fi, err := os.Stat(p); err == nil && !fi.IsDir() {
			return p, nil
		}
	}
	return "", fmt.Errorf("%s is not installed", name)
}

// ReadFile reads a file.
func (Local) ReadFile(path string) ([]byte, error) { return os.ReadFile(path) } //nolint:gosec // our paths

// WriteFile writes a file whole, through a temporary one beside it.
func (Local) WriteFile(path string, b []byte, mode fs.FileMode, owner string) error {
	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { //nolint:gosec // system directories
		return err
	}
	tmp := path + ".llmgw-new"
	if err := os.WriteFile(tmp, b, mode); err != nil {
		return err
	}
	if err := os.Chmod(tmp, mode); err != nil {
		return err
	}
	if err := chown(tmp, owner); err != nil {
		_ = os.Remove(tmp)
		return err
	}
	return os.Rename(tmp, path)
}

// Download fetches url to path through a temporary file beside it, kept
// only when its SHA-256 is sha.
func (Local) Download(ctx context.Context, url, path, sha string) error {
	if f, err := os.Open(path); err == nil { //nolint:gosec // our paths
		h := sha256.New()
		_, cerr := io.Copy(h, f)
		_ = f.Close()
		if cerr == nil && hex.EncodeToString(h.Sum(nil)) == sha {
			return nil
		}
	}
	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { //nolint:gosec // system directories
		return err
	}
	ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
	defer cancel()
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return err
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return fmt.Errorf("fetching %s: %w", url, err)
	}
	defer func() { _ = resp.Body.Close() }()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("fetching %s: %s", url, resp.Status)
	}
	tmp := path + ".llmgw-new"
	f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) //nolint:gosec // our paths
	if err != nil {
		return err
	}
	h := sha256.New()
	_, err = io.Copy(io.MultiWriter(f, h), io.LimitReader(resp.Body, 1<<30))
	if cerr := f.Close(); err == nil {
		err = cerr
	}
	if err != nil {
		_ = os.Remove(tmp)
		return fmt.Errorf("fetching %s: %w", url, err)
	}
	if got := hex.EncodeToString(h.Sum(nil)); got != sha {
		_ = os.Remove(tmp)
		return fmt.Errorf("%s has SHA-256 %s, not the %s llmgw was built with: not used", url, got, sha)
	}
	return os.Rename(tmp, path)
}

// MkdirAll makes a directory owned by owner.
func (Local) MkdirAll(path string, mode fs.FileMode, owner string) error {
	if err := os.MkdirAll(path, mode); err != nil {
		return err
	}
	return chown(path, owner)
}

// Remove removes a file, and is content when there is none.
func (Local) Remove(path string) error {
	if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) {
		return err
	}
	return nil
}

func chown(path, owner string) error {
	if owner == "" {
		return nil
	}
	u, err := user.Lookup(owner)
	if err != nil {
		return fmt.Errorf("user %s: %w (installed with its package)", owner, err)
	}
	uid, _ := strconv.Atoi(u.Uid)
	gid, _ := strconv.Atoi(u.Gid)
	return os.Chown(path, uid, gid)
}