From f5d6210b866af4c733f8c051775094e30bc1f7ac Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Mon, 28 Sep 2026 11:10:06 +0200 Subject: [PATCH] Install engines on any glibc system, with its own package manager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The llama.cpp build is published from Ubuntu but is not Ubuntu's in any way that matters: it needs glibc 2.34 and GCC 12's libstdc++, which Debian 12 and RHEL 10 have and RHEL 9 does not. Refusing every system not called "ubuntu" turned away the first two for their name. The build is now tried everywhere, and ldd decides. What ldd says is made readable. "/lib64/libstdc++.so.6: version `GLIBCXX_3.4.30' not found" is a system too old for the build, and is reported as "libstdc++.so.6 is too old (needs GLIBCXX_3.4.30)", once — rather than as a missing library that is plainly installed. Packages are named per family — libgomp1 for Debian's, libgomp for Red Hat's — and installed with whichever of apt-get, dnf, yum or zypper the machine has, decided from ID_LIKE rather than a list of distribution names. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/agent/discovery/host.go | 19 ++++ internal/agent/provision/install.go | 128 ++++++++++++++++++----- internal/agent/provision/install_test.go | 72 ++++++++++--- internal/agent/provision/recipe.go | 29 +++-- 4 files changed, 197 insertions(+), 51 deletions(-) diff --git a/internal/agent/discovery/host.go b/internal/agent/discovery/host.go index f0dbccc..3891658 100644 --- a/internal/agent/discovery/host.go +++ b/internal/agent/discovery/host.go @@ -14,6 +14,10 @@ type Host struct { OSID string `json:"os_id,omitempty"` OSVersion string `json:"os_version,omitempty"` OSCodename string `json:"os_codename,omitempty"` + // OSLike is ID_LIKE: the families this system says it follows, "debian" + // for Ubuntu, "rhel centos fedora" for Rocky. It is what decides the + // package manager, which the name alone does not for every derivative. + OSLike []string `json:"os_like,omitempty"` // Arch is Debian's spelling, "amd64" or "arm64", because that is what // package names and release archives are keyed by. Arch string `json:"arch,omitempty"` @@ -24,6 +28,19 @@ type Host struct { NVIDIADriver string `json:"nvidia_driver,omitempty"` } +// Family is the packaging family: "deb", "rpm", or "" when neither is known. +func (h Host) Family() string { + for _, id := range append([]string{h.OSID}, h.OSLike...) { + switch id { + case "debian", "ubuntu": + return "deb" + case "rhel", "fedora", "centos", "rocky", "almalinux", "suse", "opensuse": + return "rpm" + } + } + return "" +} + // HasCPUFlag reports whether the processor has an extension. func (h Host) HasCPUFlag(f string) bool { for _, have := range h.CPUFlags { @@ -69,6 +86,8 @@ func hostFromOSRelease(path string) Host { h.OSVersion = v case "VERSION_CODENAME": h.OSCodename = v + case "ID_LIKE": + h.OSLike = strings.Fields(v) } } return h diff --git a/internal/agent/provision/install.go b/internal/agent/provision/install.go index 2c11357..d5f13d4 100644 --- a/internal/agent/provision/install.go +++ b/internal/agent/provision/install.go @@ -240,14 +240,8 @@ func (in *Installer) verify(ctx context.Context, dir string, b Build) (bin, vers return "", "", err } if out, err := in.run(ctx, nil, "ldd", bin); err == nil { - var missing []string - for _, line := range strings.Split(string(out), "\n") { - if strings.Contains(line, "not found") { - missing = append(missing, strings.TrimSpace(strings.SplitN(line, "=>", 2)[0])) - } - } - if len(missing) > 0 { - return "", "", fmt.Errorf("%s cannot load %s", b.Binary, strings.Join(missing, ", ")) + if missing := lddMissing(string(out)); len(missing) > 0 { + return "", "", fmt.Errorf("%s cannot run on this system: %s", b.Binary, strings.Join(missing, "; ")) } } tctx, cancel := context.WithTimeout(ctx, 60*time.Second) @@ -262,35 +256,90 @@ func (in *Installer) verify(ctx context.Context, dir string, b Build) (bin, vers // packages installs the operating system packages a build needs. // // Only those missing, and from the node's own package sources: a package the -// base image already has is not touched, so an engine never upgrades -// something the rest of the machine depends on. -func (in *Installer) packages(ctx context.Context, pkgs []string, st *Status) error { +// system already has is not touched, so an engine never upgrades something +// the rest of the machine depends on. +func (in *Installer) packages(ctx context.Context, byFamily map[string][]string, st *Status) error { + family := in.Host.Family() + pkgs := byFamily[family] + if len(pkgs) == 0 { + // A system of no family this knows is not refused: its packages may + // well be there, and if they are not, ldd names the missing library. + return nil + } + pm := packageManagers[family] var missing []string for _, p := range pkgs { - out, err := in.run(ctx, nil, "dpkg-query", "-W", "-f=${db:Status-Abbrev}", p) - if err != nil || !strings.HasPrefix(string(out), "ii") { + if _, err := in.run(ctx, nil, pm.query[0], append(pm.query[1:], p)...); err != nil { missing = append(missing, p) } } if len(missing) == 0 { return nil } - in.progress(st, "installing %s", strings.Join(missing, ", ")) - env := []string{"DEBIAN_FRONTEND=noninteractive"} - // A lock timeout rather than failing on the lock: a machine a minute old is - // running cloud-init's and unattended-upgrades' own apt, and the first - // attempt on every new node would otherwise fail. - lock := "-o=DPkg::Lock::Timeout=600" - if out, err := in.run(ctx, env, "apt-get", lock, "update"); err != nil { - return fmt.Errorf("apt-get update: %v: %s", err, lastLine(out)) + tool, cmds := pm.install(in, missing) + if tool == "" { + return fmt.Errorf("%s is missing and no package manager was found to install it", + strings.Join(missing, ", ")) } - args := append([]string{lock, "install", "-y", "--no-install-recommends"}, missing...) - if out, err := in.run(ctx, env, "apt-get", args...); err != nil { - return fmt.Errorf("installing %s: %v: %s", strings.Join(missing, ", "), err, lastLine(out)) + in.progress(st, "installing %s with %s", strings.Join(missing, ", "), tool) + for _, c := range cmds { + if out, err := in.run(ctx, pm.env, c[0], c[1:]...); err != nil { + return fmt.Errorf("%s: %v: %s", strings.Join(c, " "), err, lastLine(out)) + } } return nil } +// packageManager is how one packaging family asks and installs. +type packageManager struct { + // query exits zero when the package named after it is installed. + query []string + env []string + // install picks the tool present on this machine and the commands to run. + install func(in *Installer, pkgs []string) (tool string, cmds [][]string) +} + +var packageManagers = map[string]packageManager{ + "deb": { + // dpkg -s exits zero for a package merely known about, so the status + // is asked for and checked by the command itself: ${db:Status-Abbrev} + // is "ii " only when installed. + query: []string{"sh", "-c", `test "$(dpkg-query -W -f='${db:Status-Abbrev}' "$0" 2>/dev/null)" = "ii "`}, + env: []string{"DEBIAN_FRONTEND=noninteractive"}, + install: func(_ *Installer, pkgs []string) (string, [][]string) { + // A lock timeout rather than failing on the lock: a machine a + // minute old is running cloud-init's and unattended-upgrades' own + // apt, and the first attempt on every new node would fail. + lock := "-o=DPkg::Lock::Timeout=600" + return "apt-get", [][]string{ + {"apt-get", lock, "update"}, + append([]string{"apt-get", lock, "install", "-y", "--no-install-recommends"}, pkgs...), + } + }, + }, + "rpm": { + query: []string{"rpm", "-q"}, + install: func(in *Installer, pkgs []string) (string, [][]string) { + for _, tool := range []string{"dnf", "yum"} { + if in.has(tool) { + return tool, [][]string{append([]string{tool, "install", "-y"}, pkgs...)} + } + } + if in.has("zypper") { + return "zypper", [][]string{append([]string{"zypper", "--non-interactive", "install"}, pkgs...)} + } + return "", nil + }, + }, +} + +// has reports whether a command is on PATH. Asked through the same seam as +// every other command, so a test can say which tools a machine has. +func (in *Installer) has(tool string) bool { + _, err := in.run(context.Background(), nil, "sh", "-c", `command -v "$0"`, tool) + return err == nil +} + // checksums reads the sha256 GitHub publishes for each asset of a release. func (in *Installer) checksums(ctx context.Context, repo, tag string) (map[string]string, error) { url := strings.TrimSuffix(in.apiBase(), "/") + "/repos/" + repo + "/releases/tags/" + tag @@ -447,6 +496,37 @@ func within(dir, name string) (string, error) { return target, nil } +// lddMissing reads what ldd could not satisfy, in two shapes: +// +// libgomp.so.1 => not found +// /lib64/libstdc++.so.6: version `GLIBCXX_3.4.30' not found (required by ...) +// +// The second is a system too old for the build — RHEL 9 against a build that +// wants GCC 12's libstdc++ — and is said as such, rather than as a library +// that is plainly there being "not found". +func lddMissing(out string) []string { + seen := map[string]bool{} + var missing []string + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if !strings.Contains(line, "not found") { + continue + } + var what string + if lib, rest, ok := strings.Cut(line, ": version `"); ok { + sym, _, _ := strings.Cut(rest, "'") + what = filepath.Base(lib) + " is too old (needs " + sym + ")" + } else { + what = strings.TrimSpace(strings.SplitN(line, "=>", 2)[0]) + " is not installed" + } + if !seen[what] { + seen[what] = true + missing = append(missing, what) + } + } + return missing +} + func findFile(dir, name string) (string, error) { var found string _ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { diff --git a/internal/agent/provision/install_test.go b/internal/agent/provision/install_test.go index 70808f4..0e01c31 100644 --- a/internal/agent/provision/install_test.go +++ b/internal/agent/provision/install_test.go @@ -82,30 +82,41 @@ func newRelease(assets map[string][]byte) *release { return r } -// fakeSystem answers dpkg-query, apt-get, ldd and the engine itself. +// fakeSystem answers the package tools, ldd and the engine itself. type fakeSystem struct { mu sync.Mutex installed map[string]bool - aptRuns int + tools map[string]bool lddOut string calls []string } +var errExit = errors.New("exit 1") + func (f *fakeSystem) run(_ context.Context, _ []string, name string, args ...string) ([]byte, error) { f.mu.Lock() defer f.mu.Unlock() f.calls = append(f.calls, name+" "+strings.Join(args, " ")) switch { - case name == "dpkg-query": + case name == "sh" && strings.Contains(args[1], "dpkg-query"), + name == "rpm" && args[0] == "-q": if f.installed[args[len(args)-1]] { - return []byte("ii "), nil + return nil, nil } - return []byte("un "), errors.New("exit 1") - case name == "apt-get": - f.aptRuns++ - if args[1] == "install" { - for _, p := range args[4:] { - f.installed[p] = true + return nil, errExit + case name == "sh" && strings.Contains(args[1], "command -v"): + if f.tools[args[2]] { + return nil, nil + } + return nil, errExit + case name == "apt-get" || name == "dnf": + for i, a := range args { + if a == "install" { + for _, p := range args[i+1:] { + if !strings.HasPrefix(p, "-") { + f.installed[p] = true + } + } } } return nil, nil @@ -233,8 +244,8 @@ func TestTheBuildFollowsTheHardware(t *testing.T) { if err != nil || !strings.HasPrefix(b.Variant, "cuda") || len(b.Assets) != 2 { t.Errorf("an NVIDIA node got %+v, %v; expected the CUDA build and its runtime", b, err) } - if _, err := chooseLlamaCpp("b1", discovery.Host{OSID: "rocky", Arch: "amd64"}); err == nil { - t.Error("a build was chosen for a system llama.cpp does not publish for") + if _, err := chooseLlamaCpp("b1", discovery.Host{OSID: "ubuntu", Arch: "riscv64"}); err == nil { + t.Error("a build was chosen for an architecture llama.cpp does not publish for") } slow := ubuntu @@ -252,3 +263,40 @@ func TestAnEngineTheAgentCannotInstallSaysSo(t *testing.T) { t.Errorf("got %+v", got[0]) } } + +func TestARedHatFamilyNodeInstallsWithItsOwnPackageManager(t *testing.T) { + // The build is not Ubuntu's in any way that matters; the package that + // carries OpenMP's runtime is named differently, and installed by dnf. + rocky := discovery.Host{OSID: "rocky", OSVersion: "10.0", OSLike: []string{"rhel", "centos", "fedora"}, + Arch: "amd64", CPUFlags: []string{"avx2"}} + rel := llamaRelease(t) + sys := &fakeSystem{installed: map[string]bool{}, tools: map[string]bool{"dnf": true}} + in, _ := installer(t, rel.serve(t), sys, rocky) + + got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}) + if got[0].State != Ready { + t.Fatalf("not ready: %+v", got[0]) + } + if !sys.installed["libgomp"] || sys.installed["libgomp1"] { + t.Errorf("installed %v; expected the rpm name, libgomp", sys.installed) + } +} + +func TestASystemTooOldForTheBuildSaysWhatItLacks(t *testing.T) { + // RHEL 9: glibc is new enough, libstdc++ is one GCC release short. ldd + // finds the library and reports the symbol, and "libstdc++ not found" + // would send somebody looking for a package that is installed. + rel := llamaRelease(t) + sys := &fakeSystem{installed: map[string]bool{"libgomp1": true}, + lddOut: "\t/lib64/libstdc++.so.6: version `GLIBCXX_3.4.30' not found (required by /x/libllama.so)\n" + + "\t/lib64/libstdc++.so.6: version `GLIBCXX_3.4.30' not found (required by /x/llama-server)\n"} + in, _ := installer(t, rel.serve(t), sys, ubuntu) + + got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}) + if got[0].State != Failed || !strings.Contains(got[0].Detail, "libstdc++.so.6 is too old (needs GLIBCXX_3.4.30)") { + t.Fatalf("detail %q", got[0].Detail) + } + if strings.Count(got[0].Detail, "GLIBCXX") != 1 { + t.Errorf("the same shortfall was listed more than once: %q", got[0].Detail) + } +} diff --git a/internal/agent/provision/recipe.go b/internal/agent/provision/recipe.go index 0b9910e..a6d0a0d 100644 --- a/internal/agent/provision/recipe.go +++ b/internal/agent/provision/recipe.go @@ -23,9 +23,10 @@ type Build struct { // Assets are the release files to fetch, unpacked in order into one // directory. Assets []string - // Packages are operating system packages the build links against and the - // base image does not carry. - Packages []string + // Packages are operating system packages the build links against and a + // minimal install may not carry, by packaging family ("deb", "rpm"): the + // same library is libgomp1 on one and libgomp on the other. + Packages map[string][]string // Binary is the executable's file name, found anywhere in the unpacked // tree. Binary string @@ -57,13 +58,11 @@ var Recipes = map[string]Recipe{ const cudaForLlamaCpp = "12.8" func chooseLlamaCpp(version string, h discovery.Host) (Build, error) { - // The published Linux builds are built on Ubuntu, and are what llama.cpp - // tests. Another distribution may well run them, but finding out on a - // production node is not the place to learn it. - if h.OSID != "ubuntu" { - return Build{}, fmt.Errorf("llama.cpp publishes builds for Ubuntu, and this node runs %s", - orUnknown(h.OSID)) - } + // The published Linux builds are built on Ubuntu, but nothing about them + // is Ubuntu's: they need glibc 2.34 and the libstdc++ of GCC 12, which + // Debian 12 and RHEL 10 have and RHEL 9 does not. So the distribution's + // name decides nothing here. The build is tried everywhere, and ldd says + // whether this system can load it — in the words of the missing symbol. var arch string switch h.Arch { case "amd64": @@ -71,16 +70,16 @@ func chooseLlamaCpp(version string, h discovery.Host) (Build, error) { case "arm64": arch = "arm64" default: - return Build{}, fmt.Errorf("llama.cpp publishes no Ubuntu build for %s", orUnknown(h.Arch)) + return Build{}, fmt.Errorf("llama.cpp publishes no Linux build for %s", orUnknown(h.Arch)) } b := Build{ Variant: "cpu", Assets: []string{fmt.Sprintf("llama-%s-bin-ubuntu-%s.tar.gz", version, arch)}, - // Everything else the build links is in the archive or in any Ubuntu - // install; OpenMP's runtime is the one a cloud image lacks, and the - // server does not start without it. - Packages: []string{"libgomp1"}, + // Everything else the build links is in the archive or in any current + // glibc system; OpenMP's runtime is the one a cloud image lacks, and + // the server does not start without it. + Packages: map[string][]string{"deb": {"libgomp1"}, "rpm": {"libgomp"}}, Binary: "llama-server", } if arch == "x64" {