diff --git a/internal/agent/discovery/host.go b/internal/agent/discovery/host.go index f0dbcccc..3891658e 100644 --- a/internal/agent/discovery/host.go +++ b/internal/agent/discovery/host.go @@ -14,6 +14,10 @@ type Host struct { OSID string `json:"os_id,omitempty"` OSVersion string `json:"os_version,omitempty"` OSCodename string `json:"os_codename,omitempty"` + // OSLike is ID_LIKE: the families this system says it follows, "debian" + // for Ubuntu, "rhel centos fedora" for Rocky. It is what decides the + // package manager, which the name alone does not for every derivative. + OSLike []string `json:"os_like,omitempty"` // Arch is Debian's spelling, "amd64" or "arm64", because that is what // package names and release archives are keyed by. Arch string `json:"arch,omitempty"` @@ -24,6 +28,19 @@ type Host struct { NVIDIADriver string `json:"nvidia_driver,omitempty"` } +// Family is the packaging family: "deb", "rpm", or "" when neither is known. +func (h Host) Family() string { + for _, id := range append([]string{h.OSID}, h.OSLike...) { + switch id { + case "debian", "ubuntu": + return "deb" + case "rhel", "fedora", "centos", "rocky", "almalinux", "suse", "opensuse": + return "rpm" + } + } + return "" +} + // HasCPUFlag reports whether the processor has an extension. func (h Host) HasCPUFlag(f string) bool { for _, have := range h.CPUFlags { @@ -69,6 +86,8 @@ func hostFromOSRelease(path string) Host { h.OSVersion = v case "VERSION_CODENAME": h.OSCodename = v + case "ID_LIKE": + h.OSLike = strings.Fields(v) } } return h diff --git a/internal/agent/provision/install.go b/internal/agent/provision/install.go index 2c11357e..d5f13d44 100644 --- a/internal/agent/provision/install.go +++ b/internal/agent/provision/install.go @@ -240,14 +240,8 @@ func (in *Installer) verify(ctx context.Context, dir string, b Build) (bin, vers return "", "", err } if out, err := in.run(ctx, nil, "ldd", bin); err == nil { - var missing []string - for _, line := range strings.Split(string(out), "\n") { - if strings.Contains(line, "not found") { - missing = append(missing, strings.TrimSpace(strings.SplitN(line, "=>", 2)[0])) - } - } - if len(missing) > 0 { - return "", "", fmt.Errorf("%s cannot load %s", b.Binary, strings.Join(missing, ", ")) + if missing := lddMissing(string(out)); len(missing) > 0 { + return "", "", fmt.Errorf("%s cannot run on this system: %s", b.Binary, strings.Join(missing, "; ")) } } tctx, cancel := context.WithTimeout(ctx, 60*time.Second) @@ -262,35 +256,90 @@ func (in *Installer) verify(ctx context.Context, dir string, b Build) (bin, vers // packages installs the operating system packages a build needs. // // Only those missing, and from the node's own package sources: a package the -// base image already has is not touched, so an engine never upgrades -// something the rest of the machine depends on. -func (in *Installer) packages(ctx context.Context, pkgs []string, st *Status) error { +// system already has is not touched, so an engine never upgrades something +// the rest of the machine depends on. +func (in *Installer) packages(ctx context.Context, byFamily map[string][]string, st *Status) error { + family := in.Host.Family() + pkgs := byFamily[family] + if len(pkgs) == 0 { + // A system of no family this knows is not refused: its packages may + // well be there, and if they are not, ldd names the missing library. + return nil + } + pm := packageManagers[family] var missing []string for _, p := range pkgs { - out, err := in.run(ctx, nil, "dpkg-query", "-W", "-f=${db:Status-Abbrev}", p) - if err != nil || !strings.HasPrefix(string(out), "ii") { + if _, err := in.run(ctx, nil, pm.query[0], append(pm.query[1:], p)...); err != nil { missing = append(missing, p) } } if len(missing) == 0 { return nil } - in.progress(st, "installing %s", strings.Join(missing, ", ")) - env := []string{"DEBIAN_FRONTEND=noninteractive"} - // A lock timeout rather than failing on the lock: a machine a minute old is - // running cloud-init's and unattended-upgrades' own apt, and the first - // attempt on every new node would otherwise fail. - lock := "-o=DPkg::Lock::Timeout=600" - if out, err := in.run(ctx, env, "apt-get", lock, "update"); err != nil { - return fmt.Errorf("apt-get update: %v: %s", err, lastLine(out)) + tool, cmds := pm.install(in, missing) + if tool == "" { + return fmt.Errorf("%s is missing and no package manager was found to install it", + strings.Join(missing, ", ")) } - args := append([]string{lock, "install", "-y", "--no-install-recommends"}, missing...) - if out, err := in.run(ctx, env, "apt-get", args...); err != nil { - return fmt.Errorf("installing %s: %v: %s", strings.Join(missing, ", "), err, lastLine(out)) + in.progress(st, "installing %s with %s", strings.Join(missing, ", "), tool) + for _, c := range cmds { + if out, err := in.run(ctx, pm.env, c[0], c[1:]...); err != nil { + return fmt.Errorf("%s: %v: %s", strings.Join(c, " "), err, lastLine(out)) + } } return nil } +// packageManager is how one packaging family asks and installs. +type packageManager struct { + // query exits zero when the package named after it is installed. + query []string + env []string + // install picks the tool present on this machine and the commands to run. + install func(in *Installer, pkgs []string) (tool string, cmds [][]string) +} + +var packageManagers = map[string]packageManager{ + "deb": { + // dpkg -s exits zero for a package merely known about, so the status + // is asked for and checked by the command itself: ${db:Status-Abbrev} + // is "ii " only when installed. + query: []string{"sh", "-c", `test "$(dpkg-query -W -f='${db:Status-Abbrev}' "$0" 2>/dev/null)" = "ii "`}, + env: []string{"DEBIAN_FRONTEND=noninteractive"}, + install: func(_ *Installer, pkgs []string) (string, [][]string) { + // A lock timeout rather than failing on the lock: a machine a + // minute old is running cloud-init's and unattended-upgrades' own + // apt, and the first attempt on every new node would fail. + lock := "-o=DPkg::Lock::Timeout=600" + return "apt-get", [][]string{ + {"apt-get", lock, "update"}, + append([]string{"apt-get", lock, "install", "-y", "--no-install-recommends"}, pkgs...), + } + }, + }, + "rpm": { + query: []string{"rpm", "-q"}, + install: func(in *Installer, pkgs []string) (string, [][]string) { + for _, tool := range []string{"dnf", "yum"} { + if in.has(tool) { + return tool, [][]string{append([]string{tool, "install", "-y"}, pkgs...)} + } + } + if in.has("zypper") { + return "zypper", [][]string{append([]string{"zypper", "--non-interactive", "install"}, pkgs...)} + } + return "", nil + }, + }, +} + +// has reports whether a command is on PATH. Asked through the same seam as +// every other command, so a test can say which tools a machine has. +func (in *Installer) has(tool string) bool { + _, err := in.run(context.Background(), nil, "sh", "-c", `command -v "$0"`, tool) + return err == nil +} + // checksums reads the sha256 GitHub publishes for each asset of a release. func (in *Installer) checksums(ctx context.Context, repo, tag string) (map[string]string, error) { url := strings.TrimSuffix(in.apiBase(), "/") + "/repos/" + repo + "/releases/tags/" + tag @@ -447,6 +496,37 @@ func within(dir, name string) (string, error) { return target, nil } +// lddMissing reads what ldd could not satisfy, in two shapes: +// +// libgomp.so.1 => not found +// /lib64/libstdc++.so.6: version `GLIBCXX_3.4.30' not found (required by ...) +// +// The second is a system too old for the build — RHEL 9 against a build that +// wants GCC 12's libstdc++ — and is said as such, rather than as a library +// that is plainly there being "not found". +func lddMissing(out string) []string { + seen := map[string]bool{} + var missing []string + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if !strings.Contains(line, "not found") { + continue + } + var what string + if lib, rest, ok := strings.Cut(line, ": version `"); ok { + sym, _, _ := strings.Cut(rest, "'") + what = filepath.Base(lib) + " is too old (needs " + sym + ")" + } else { + what = strings.TrimSpace(strings.SplitN(line, "=>", 2)[0]) + " is not installed" + } + if !seen[what] { + seen[what] = true + missing = append(missing, what) + } + } + return missing +} + func findFile(dir, name string) (string, error) { var found string _ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { diff --git a/internal/agent/provision/install_test.go b/internal/agent/provision/install_test.go index 70808f48..0e01c316 100644 --- a/internal/agent/provision/install_test.go +++ b/internal/agent/provision/install_test.go @@ -82,30 +82,41 @@ func newRelease(assets map[string][]byte) *release { return r } -// fakeSystem answers dpkg-query, apt-get, ldd and the engine itself. +// fakeSystem answers the package tools, ldd and the engine itself. type fakeSystem struct { mu sync.Mutex installed map[string]bool - aptRuns int + tools map[string]bool lddOut string calls []string } +var errExit = errors.New("exit 1") + func (f *fakeSystem) run(_ context.Context, _ []string, name string, args ...string) ([]byte, error) { f.mu.Lock() defer f.mu.Unlock() f.calls = append(f.calls, name+" "+strings.Join(args, " ")) switch { - case name == "dpkg-query": + case name == "sh" && strings.Contains(args[1], "dpkg-query"), + name == "rpm" && args[0] == "-q": if f.installed[args[len(args)-1]] { - return []byte("ii "), nil + return nil, nil } - return []byte("un "), errors.New("exit 1") - case name == "apt-get": - f.aptRuns++ - if args[1] == "install" { - for _, p := range args[4:] { - f.installed[p] = true + return nil, errExit + case name == "sh" && strings.Contains(args[1], "command -v"): + if f.tools[args[2]] { + return nil, nil + } + return nil, errExit + case name == "apt-get" || name == "dnf": + for i, a := range args { + if a == "install" { + for _, p := range args[i+1:] { + if !strings.HasPrefix(p, "-") { + f.installed[p] = true + } + } } } return nil, nil @@ -233,8 +244,8 @@ func TestTheBuildFollowsTheHardware(t *testing.T) { if err != nil || !strings.HasPrefix(b.Variant, "cuda") || len(b.Assets) != 2 { t.Errorf("an NVIDIA node got %+v, %v; expected the CUDA build and its runtime", b, err) } - if _, err := chooseLlamaCpp("b1", discovery.Host{OSID: "rocky", Arch: "amd64"}); err == nil { - t.Error("a build was chosen for a system llama.cpp does not publish for") + if _, err := chooseLlamaCpp("b1", discovery.Host{OSID: "ubuntu", Arch: "riscv64"}); err == nil { + t.Error("a build was chosen for an architecture llama.cpp does not publish for") } slow := ubuntu @@ -252,3 +263,40 @@ func TestAnEngineTheAgentCannotInstallSaysSo(t *testing.T) { t.Errorf("got %+v", got[0]) } } + +func TestARedHatFamilyNodeInstallsWithItsOwnPackageManager(t *testing.T) { + // The build is not Ubuntu's in any way that matters; the package that + // carries OpenMP's runtime is named differently, and installed by dnf. + rocky := discovery.Host{OSID: "rocky", OSVersion: "10.0", OSLike: []string{"rhel", "centos", "fedora"}, + Arch: "amd64", CPUFlags: []string{"avx2"}} + rel := llamaRelease(t) + sys := &fakeSystem{installed: map[string]bool{}, tools: map[string]bool{"dnf": true}} + in, _ := installer(t, rel.serve(t), sys, rocky) + + got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}) + if got[0].State != Ready { + t.Fatalf("not ready: %+v", got[0]) + } + if !sys.installed["libgomp"] || sys.installed["libgomp1"] { + t.Errorf("installed %v; expected the rpm name, libgomp", sys.installed) + } +} + +func TestASystemTooOldForTheBuildSaysWhatItLacks(t *testing.T) { + // RHEL 9: glibc is new enough, libstdc++ is one GCC release short. ldd + // finds the library and reports the symbol, and "libstdc++ not found" + // would send somebody looking for a package that is installed. + rel := llamaRelease(t) + sys := &fakeSystem{installed: map[string]bool{"libgomp1": true}, + lddOut: "\t/lib64/libstdc++.so.6: version `GLIBCXX_3.4.30' not found (required by /x/libllama.so)\n" + + "\t/lib64/libstdc++.so.6: version `GLIBCXX_3.4.30' not found (required by /x/llama-server)\n"} + in, _ := installer(t, rel.serve(t), sys, ubuntu) + + got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}) + if got[0].State != Failed || !strings.Contains(got[0].Detail, "libstdc++.so.6 is too old (needs GLIBCXX_3.4.30)") { + t.Fatalf("detail %q", got[0].Detail) + } + if strings.Count(got[0].Detail, "GLIBCXX") != 1 { + t.Errorf("the same shortfall was listed more than once: %q", got[0].Detail) + } +} diff --git a/internal/agent/provision/recipe.go b/internal/agent/provision/recipe.go index 0b9910ec..a6d0a0da 100644 --- a/internal/agent/provision/recipe.go +++ b/internal/agent/provision/recipe.go @@ -23,9 +23,10 @@ type Build struct { // Assets are the release files to fetch, unpacked in order into one // directory. Assets []string - // Packages are operating system packages the build links against and the - // base image does not carry. - Packages []string + // Packages are operating system packages the build links against and a + // minimal install may not carry, by packaging family ("deb", "rpm"): the + // same library is libgomp1 on one and libgomp on the other. + Packages map[string][]string // Binary is the executable's file name, found anywhere in the unpacked // tree. Binary string @@ -57,13 +58,11 @@ var Recipes = map[string]Recipe{ const cudaForLlamaCpp = "12.8" func chooseLlamaCpp(version string, h discovery.Host) (Build, error) { - // The published Linux builds are built on Ubuntu, and are what llama.cpp - // tests. Another distribution may well run them, but finding out on a - // production node is not the place to learn it. - if h.OSID != "ubuntu" { - return Build{}, fmt.Errorf("llama.cpp publishes builds for Ubuntu, and this node runs %s", - orUnknown(h.OSID)) - } + // The published Linux builds are built on Ubuntu, but nothing about them + // is Ubuntu's: they need glibc 2.34 and the libstdc++ of GCC 12, which + // Debian 12 and RHEL 10 have and RHEL 9 does not. So the distribution's + // name decides nothing here. The build is tried everywhere, and ldd says + // whether this system can load it — in the words of the missing symbol. var arch string switch h.Arch { case "amd64": @@ -71,16 +70,16 @@ func chooseLlamaCpp(version string, h discovery.Host) (Build, error) { case "arm64": arch = "arm64" default: - return Build{}, fmt.Errorf("llama.cpp publishes no Ubuntu build for %s", orUnknown(h.Arch)) + return Build{}, fmt.Errorf("llama.cpp publishes no Linux build for %s", orUnknown(h.Arch)) } b := Build{ Variant: "cpu", Assets: []string{fmt.Sprintf("llama-%s-bin-ubuntu-%s.tar.gz", version, arch)}, - // Everything else the build links is in the archive or in any Ubuntu - // install; OpenMP's runtime is the one a cloud image lacks, and the - // server does not start without it. - Packages: []string{"libgomp1"}, + // Everything else the build links is in the archive or in any current + // glibc system; OpenMP's runtime is the one a cloud image lacks, and + // the server does not start without it. + Packages: map[string][]string{"deb": {"libgomp1"}, "rpm": {"libgomp"}}, Binary: "llama-server", } if arch == "x64" {