diff --git a/CHANGELOG.md b/CHANGELOG.md index 388a44b..a4e6a0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -123,6 +123,24 @@ leaving the model with nothing. Now: stopping — is waited for, as before. - **`ctl restart --in-place`** stops each replica first, whatever room there is. +### Every GPU node gets a vLLM it can run, chosen on the cluster page + +The vLLM a node installs is the build its hardware runs: CUDA 13 from PyPI +where the driver (580+) and the GPUs (Turing+) allow, and otherwise the same +version's CUDA 12.9 build from vLLM's GitHub release, pinned to GitHub's +digest, with PyTorch's CUDA 12.9 torch (driver 525+, Volta+). One version +across the cluster, so the flags llmgw builds — checked against that version +— hold on every node. + +- **The install computes on the GPU before it counts**: a small tensor + operation in the new environment. A build without kernels for the card + imports and prints its version, and failed only on its first model; now it + fails to install, saying so. +- **The build is chosen per node** on the cluster page (*vLLM build*: auto, + CUDA 13, CUDA 12.9, none), with `llmgw ctl engine-build NODE vllm BUILD`, or + `POST /api/v1/nodes/{id}/engines`. The choice is kept in the store and sent + to the node at once. + ### Engines per node, and nodes that do not run them all A cluster with older GPUs could not add them cleanly: `node add` refused a diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 04ad5c0..5e9f4fd71 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -2370,9 +2370,12 @@ func (x *Provision) GetEngines() []*EngineWant { // EngineWant names one engine at one version. type EngineWant struct { - state protoimpl.MessageState `protogen:"open.v1"` - Engine string `protobuf:"bytes,1,opt,name=engine,proto3" json:"engine,omitempty"` // "llamacpp" - Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` // "b10991" + state protoimpl.MessageState `protogen:"open.v1"` + Engine string `protobuf:"bytes,1,opt,name=engine,proto3" json:"engine,omitempty"` // "llamacpp" + Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` // "b10991" + // variant is the build to install, when one is chosen for the node rather + // than left to its hardware: "cu13", "cu129". Empty is the agent's choice. + Variant string `protobuf:"bytes,3,opt,name=variant,proto3" json:"variant,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2421,6 +2424,13 @@ func (x *EngineWant) GetVersion() string { return "" } +func (x *EngineWant) GetVariant() string { + if x != nil { + return x.Variant + } + return "" +} + // ProvisionStatus is where one engine's installation stands on a node. type ProvisionStatus struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -4068,11 +4078,12 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\";\n" + "\tProvision\x12.\n" + - "\aengines\x18\x01 \x03(\v2\x14.llmgw.v1.EngineWantR\aengines\">\n" + + "\aengines\x18\x01 \x03(\v2\x14.llmgw.v1.EngineWantR\aengines\"X\n" + "\n" + "EngineWant\x12\x16\n" + "\x06engine\x18\x01 \x01(\tR\x06engine\x12\x18\n" + - "\aversion\x18\x02 \x01(\tR\aversion\"\x8b\x01\n" + + "\aversion\x18\x02 \x01(\tR\aversion\x12\x18\n" + + "\avariant\x18\x03 \x01(\tR\avariant\"\x8b\x01\n" + "\x0fProvisionStatus\x12\x16\n" + "\x06engine\x18\x01 \x01(\tR\x06engine\x12\x18\n" + "\aversion\x18\x02 \x01(\tR\aversion\x12\x18\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 45818fb..16d4613 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -465,6 +465,9 @@ message Provision { message EngineWant { string engine = 1; // "llamacpp" string version = 2; // "b10991" + // variant is the build to install, when one is chosen for the node rather + // than left to its hardware: "cu13", "cu129". Empty is the agent's choice. + string variant = 3; } // ProvisionStatus is where one engine's installation stands on a node. diff --git a/docs/install.md b/docs/install.md index 98699c1..e00c03c 100644 --- a/docs/install.md +++ b/docs/install.md @@ -377,9 +377,12 @@ node gets the same dependencies whenever it is built, and installed with the environment as `llmgw-requirements.txt`. It needs: - **driver 580 or newer and a Turing GPU or newer** (compute capability 7.5) - for vLLM 0.26 and later, which are built for CUDA 13. A V100 cannot run - them; vLLM before 0.26 is built for CUDA 12, and runs on driver 525 and - Volta (7.0); + for the CUDA 13 build of vLLM 0.26 and later, from PyPI. A node that cannot + run it — an older driver, a V100 — gets **the same version built for CUDA + 12.9** instead, from vLLM's GitHub release (checked against the digest + GitHub publishes) with PyTorch's CUDA 12.9 torch: driver 525 or newer, Volta + (7.0) or newer. Either way the install ends by computing something on the + node's GPU, and a build that cannot fails to install, saying why; - **about 25 GB free** under the engines directory while it installs; - **gcc**, which Triton builds its kernel launcher with. The agent installs it. @@ -400,6 +403,19 @@ you installed yourself on a node, name its path in that node's `agent.yaml` (`engines.llamacpp.bin`, `engines.vllm.bin`, `engines.sglang.python`): the agent then installs none of its own beside it. +#### Choosing a node's vLLM build + +Which build a GPU node installs is chosen on the cluster page, in the **vLLM +build** column — *auto* (what its hardware runs, the default), *CUDA 13*, +*CUDA 12.9*, or *none* — or from the command line: + +``` +llmgw ctl engine-build gpu07 vllm cu129 +``` + +The choice is kept by the control plane and the node is told at once; it +installs the build in the background, and the column shows how that stands. + #### Different GPUs, different engines A cluster of two GPU generations may need two builds of an engine, or one on diff --git a/internal/agent/provision.go b/internal/agent/provision.go index ea3f448..fbd3d28 100644 --- a/internal/agent/provision.go +++ b/internal/agent/provision.go @@ -88,7 +88,7 @@ func (a *Agent) setProvision(p *llmgwv1.Provision) { // the installed build is used first — and quietly replace theirs. continue } - wants = append(wants, provision.Want{Engine: e.GetEngine(), Version: e.GetVersion()}) + wants = append(wants, provision.Want{Engine: e.GetEngine(), Version: e.GetVersion(), Variant: e.GetVariant()}) } a.mu.Lock() a.provWants = wants diff --git a/internal/agent/provision/install.go b/internal/agent/provision/install.go index c7f088d..9bb83af 100644 --- a/internal/agent/provision/install.go +++ b/internal/agent/provision/install.go @@ -27,6 +27,9 @@ import ( type Want struct { Engine string Version string + // Variant is a build chosen for this node ("cu129"); empty leaves it to + // the node's hardware. + Variant string } // The states an engine's installation can be in. @@ -135,7 +138,12 @@ func (in *Installer) one(ctx context.Context, w Want) Status { "; install it by hand and name it in the agent's configuration" return st } - build, err := recipe.Choose(w.Version, in.Host) + if w.Variant != "" && !validVersion.MatchString(w.Variant) { + st.State = Unsupported + st.Detail = fmt.Sprintf("%q is not a build this agent will install", w.Variant) + return st + } + build, err := recipe.Choose(w.Version, w.Variant, in.Host) if err != nil { st.State, st.Detail = Unsupported, err.Error() var nh NotHere diff --git a/internal/agent/provision/install_test.go b/internal/agent/provision/install_test.go index 4afbd57..07ee82c 100644 --- a/internal/agent/provision/install_test.go +++ b/internal/agent/provision/install_test.go @@ -255,11 +255,11 @@ func TestAnArchiveCannotWriteOutsideItsDirectory(t *testing.T) { func TestTheBuildFollowsTheHardware(t *testing.T) { gpu := ubuntu gpu.NVIDIADriver = "570.86.15" - b, err := chooseLlamaCpp("b1", gpu) + b, err := chooseLlamaCpp("b1", "", gpu) if err != nil || !strings.HasPrefix(b.Variant, "cuda") || len(b.Assets) != 2 { t.Errorf("an NVIDIA node got %+v, %v; expected the CUDA build and its runtime", b, err) } - if _, err := chooseLlamaCpp("b1", discovery.Host{OSID: "ubuntu", Arch: "riscv64"}); err == nil { + if _, err := chooseLlamaCpp("b1", "", discovery.Host{OSID: "ubuntu", Arch: "riscv64"}); err == nil { t.Error("a build was chosen for an architecture llama.cpp does not publish for") } diff --git a/internal/agent/provision/python.go b/internal/agent/provision/python.go index dd84ec2..9ea78df 100644 --- a/internal/agent/provision/python.go +++ b/internal/agent/provision/python.go @@ -86,28 +86,62 @@ func (in *Installer) installPython(ctx context.Context, r Recipe, w Want, b Buil } python := filepath.Join(tmp, "bin", "python") - in.progress(st, "resolving %s against PyPI as of %s", strings.Join(b.Requirements, " "), asOf[:10]) + reqs := b.Requirements + if b.Wheel != nil { + // The engine from its GitHub release, pinned to the digest GitHub + // publishes for the file: the same trust as an archive, and uv + // refuses a download that does not match it. + in.progress(st, "asking GitHub for %s's digest", b.Wheel.Asset) + sums, err := in.checksums(ctx, b.Wheel.Repo, b.Wheel.Tag) + if err != nil { + return err + } + sum, ok := sums[b.Wheel.Asset] + if !ok { + return fmt.Errorf("%s %s publishes no %s", b.Wheel.Repo, b.Wheel.Tag, b.Wheel.Asset) + } + url := strings.TrimSuffix(in.downloadBase(), "/") + "/" + b.Wheel.Repo + "/releases/download/" + + b.Wheel.Tag + "/" + b.Wheel.Asset + reqs = append([]string{b.Wheel.Project + " @ " + url + "#sha256=" + sum}, reqs...) + } + // A second index, for the torch the engine was built on. Every package + // is taken from whichever of the two has the version resolved — both are + // the publishers' own — and each file is still pinned by its hash. + var index []string + if b.ExtraIndex != "" { + index = []string{"--extra-index-url", b.ExtraIndex, "--index-strategy", "unsafe-best-match"} + } + + in.progress(st, "resolving %s against PyPI as of %s", strings.Join(reqs, " "), asOf[:10]) lock := filepath.Join(tmp, lockName) - args := []string{"pip", "compile", "--python", python, "--exclude-newer", asOf, - "--generate-hashes", "--quiet", "--output-file", lock, "-"} - if out, err := in.runStdin(ctx, env, strings.Join(b.Requirements, "\n")+"\n", uv.Binary, args...); err != nil { - return fmt.Errorf("resolving %s: %v: %s", strings.Join(b.Requirements, " "), err, tail(out)) + args := append([]string{"pip", "compile", "--python", python, "--exclude-newer", asOf, + "--generate-hashes", "--quiet", "--output-file", lock}, index...) + args = append(args, "-") + if out, err := in.runStdin(ctx, env, strings.Join(reqs, "\n")+"\n", uv.Binary, args...); err != nil { + return fmt.Errorf("resolving %s: %v: %s", strings.Join(reqs, " "), err, tail(out)) } in.progress(st, "downloading and installing %s and its dependencies: several gigabytes, which takes minutes", - strings.Join(b.Requirements, " ")) - if out, err := in.run(ctx, env, uv.Binary, "pip", "install", "--python", python, - "--require-hashes", "--no-deps", "-r", lock); err != nil { - return fmt.Errorf("installing %s: %v: %s", strings.Join(b.Requirements, " "), err, tail(out)) + strings.Join(reqs, " ")) + install := append([]string{"pip", "install", "--python", python, + "--require-hashes", "--no-deps", "-r", lock}, index...) + if out, err := in.run(ctx, env, uv.Binary, install...); err != nil { + return fmt.Errorf("installing %s: %v: %s", strings.Join(reqs, " "), err, tail(out)) } in.progress(st, "checking that %s runs", w.Engine) if _, _, err := in.verify(ctx, tmp, b); err != nil { return err } + if b.GPUCheck && in.Host.NVIDIADriver != "" { + in.progress(st, "checking that %s (%s) computes on this node's GPU", w.Engine, b.Variant) + if err := in.gpuCheck(ctx, python, w, b); err != nil { + return err + } + } m, _ := json.MarshalIndent(markerFile{ Engine: w.Engine, Version: w.Version, Variant: b.Variant, - Binary: filepath.Join("bin", b.Binary), Requirements: b.Requirements, + Binary: filepath.Join("bin", b.Binary), Requirements: reqs, ExcludeNewer: asOf, UV: UVVersion, Installed: time.Now().UTC(), }, "", " ") if err := os.WriteFile(filepath.Join(tmp, marker), m, 0o644); err != nil { //nolint:gosec // not secret @@ -203,3 +237,26 @@ func tail(b []byte) string { } return strings.Join(lines, " / ") } + +// gpuProbe is what the environment is asked to run on the GPU: one small +// operation, which needs the torch to have kernels for the card. +const gpuProbe = `import torch +assert torch.cuda.is_available(), "torch sees no CUDA device" +x = torch.ones(64, device="cuda") +print(float((x * 2).sum()), torch.cuda.get_device_name(0), torch.cuda.get_device_capability(0))` + +// gpuCheck runs gpuProbe in a freshly installed environment. +func (in *Installer) gpuCheck(ctx context.Context, python string, w Want, b Build) error { + ctx, cancel := context.WithTimeout(ctx, 3*time.Minute) + defer cancel() + out, err := in.run(ctx, nil, python, "-c", gpuProbe) + if err != nil { + hint := "" + if strings.Contains(string(out), "no kernel image") { + hint = "; this build has no kernels for this GPU, so try the other build for this node" + } + return fmt.Errorf("%s %s (%s) installed but cannot compute on this node's GPU: %s%s", + w.Engine, w.Version, b.Variant, tail(out), hint) + } + return nil +} diff --git a/internal/agent/provision/python_test.go b/internal/agent/provision/python_test.go index bc01c5f..31d48a8 100644 --- a/internal/agent/provision/python_test.go +++ b/internal/agent/provision/python_test.go @@ -2,6 +2,7 @@ package provision import ( "context" + "errors" "net/http" "net/http/httptest" "os" @@ -23,6 +24,9 @@ type fakePython struct { pmu sync.Mutex commands []string failPip bool + // noKernel makes the GPU check fail as a torch without kernels for the + // card does. + noKernel bool } func (f *fakePython) run(ctx context.Context, env []string, name string, args ...string) ([]byte, error) { @@ -43,6 +47,11 @@ func (f *fakePython) run(ctx context.Context, env []string, name string, args .. return nil, os.WriteFile(filepath.Join(venv, "bin", "vllm"), []byte("#!/bin/sh\n"), 0o755) case strings.HasSuffix(name, "/bin/vllm") && args[0] == "--version": return []byte("0.30.0\n"), nil + case strings.HasSuffix(name, "/bin/python") && args[0] == "-c": + if f.noKernel { + return []byte("RuntimeError: CUDA error: no kernel image is available for execution on the device\n"), errExit + } + return []byte("128.0 Tesla V100-SXM2-32GB (7, 0)\n"), nil } return f.fakeSystem.run(ctx, env, name, args...) } @@ -59,6 +68,8 @@ func pythonInstaller(t *testing.T, host discovery.Host, sys *fakePython) (*Insta uv := newRelease(map[string][]byte{ "uv-x86_64-unknown-linux-gnu.tar.gz": tarGz(t, map[string]string{"uv-x86_64-unknown-linux-gnu/uv": "#!/bin/sh\n"}, nil), + // The fake GitHub serves every repository's release from this one. + "vllm-0.30.0+cu129-cp38-abi3-manylinux_2_28_x86_64.whl": []byte("the cu129 wheel"), }) gh := uv.serve(t) mux := http.NewServeMux() @@ -130,8 +141,9 @@ func TestVLLMIsNotForEveryNode(t *testing.T) { says string }{ {"no GPU", ubuntu, Skipped, "has none"}, - {"old driver", discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15"}, Unsupported, "driver 580"}, - {"V100", discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 7.0}, Unsupported, "7.5"}, + // Too old for CUDA 13 and for its CUDA 12.9 build both. + {"a driver too old for either", discovery.Host{Arch: "amd64", NVIDIADriver: "470.256.02"}, Unsupported, "driver 525"}, + {"Pascal", discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 6.1}, Unsupported, "7.0 or newer"}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -182,3 +194,77 @@ func TestAVersionPyPIDoesNotHaveIsSaidSo(t *testing.T) { t.Errorf("got %s %q", got[0].State, got[0].Detail) } } + +func TestAnOlderGPUGetsTheSameVLLMBuiltForCUDA12(t *testing.T) { + // A V100: CUDA 13 dropped it. The same vLLM's CUDA 12.9 build, from the + // project's GitHub release, with the torch PyTorch publishes for 12.9. + volta := discovery.Host{OSID: "ubuntu", Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.0} + sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}} + in, gh := pythonInstaller(t, volta, sys) + got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0"}}) + if got[0].State != Ready || got[0].Variant != "cu129" { + t.Fatalf("a V100 node: %+v", got[0]) + } + if _, err := os.Stat(filepath.Join(in.Root, "vllm", "0.30.0-cu129", marker)); err != nil { + t.Errorf("not installed under its own build's name: %v", err) + } + all := strings.Join(sys.commands, "\n") + wheel := "vllm-0.30.0+cu129-cp38-abi3-manylinux_2_28_x86_64.whl" + for _, want := range []string{ + // Pinned to the digest GitHub publishes for the wheel. + "vllm @ " + in.DownloadBase + "/vllm-project/vllm/releases/download/v0.30.0/" + wheel + "#sha256=" + + strings.TrimPrefix(gh.digests[wheel], "sha256:"), + "--extra-index-url https://download.pytorch.org/whl/cu129 --index-strategy unsafe-best-match -", + "--require-hashes --no-deps -r", + // Computing on the card, not only importing. + "/bin/python -c import torch", + } { + if !strings.Contains(all, want) { + t.Errorf("no %q in:\n%s", want, all) + } + } + for _, c := range sys.commands { + if strings.Contains(c, "pip install") && !strings.Contains(c, "download.pytorch.org/whl/cu129") { + t.Errorf("installed without the index its torch comes from: %s", c) + } + } +} + +func TestABuildChosenForANodeIsTheOneInstalled(t *testing.T) { + sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}} + in, _ := pythonInstaller(t, gpuNode, sys) + if got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0", Variant: "cu129"}}); got[0].Variant != "cu129" || got[0].State != Ready { + t.Errorf("cu129 chosen for a node that runs cu13 too: %+v", got[0]) + } + volta := discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 7.0} + if _, err := chooseVLLM("0.30.0", "cu13", volta); err == nil || !strings.Contains(err.Error(), "7.5") { + t.Errorf("cu13 chosen for a V100 should be refused, saying why: %v", err) + } + if _, err := chooseVLLM("0.30.0", "cu200", gpuNode); err == nil { + t.Error("a build that does not exist was accepted") + } +} + +func TestABuildThatCannotComputeOnTheCardIsNotInstalled(t *testing.T) { + sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}, noKernel: true} + in, _ := pythonInstaller(t, gpuNode, sys) + got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0"}}) + if got[0].State != Failed || !strings.Contains(got[0].Detail, "no kernels for this GPU") { + t.Fatalf("a build without kernels for the card: %+v", got[0]) + } + if _, err := os.Stat(filepath.Join(in.Root, "vllm", "0.30.0-cu13", marker)); err == nil { + t.Error("marked installed") + } +} + +func TestOnlyACUDARequirementFallsBackToTheCUDA12Build(t *testing.T) { + // A machine vLLM is not published for is refused, not given the other + // build to fail with. + if _, err := chooseVLLM("0.30.0", "", discovery.Host{NVIDIADriver: "580.65.06"}); err == nil || + !strings.Contains(err.Error(), "no Linux build") { + t.Errorf("an unknown architecture: %v", err) + } + if _, err := chooseVLLM("0.30.0", "cu129", ubuntu); !errors.As(err, new(NotHere)) { + t.Errorf("a node with no GPU chosen cu129: %v", err) + } +} diff --git a/internal/agent/provision/recipe.go b/internal/agent/provision/recipe.go index 49c3c66..3c9ffa8 100644 --- a/internal/agent/provision/recipe.go +++ b/internal/agent/provision/recipe.go @@ -10,6 +10,7 @@ package provision import ( + "errors" "fmt" "slices" "strings" @@ -48,12 +49,28 @@ type Build struct { // build must list a device with it, or it is not installed — a llama.cpp // whose CUDA backend cannot load runs on the CPU without complaint. GPUBackend string + // Wheel, when set, is the engine itself as a wheel from a GitHub + // release rather than from PyPI, checked against the digest GitHub + // publishes for it. ExtraIndex is a package index besides PyPI that the + // rest resolves from too: the CUDA 12.9 torch the wheel was built on. + Wheel *Wheel + ExtraIndex string + // GPUCheck, on a node with an NVIDIA GPU, has the installed environment + // compute something on one before the build counts as installed. A build + // with no kernels for the card — a GPU generation its torch dropped — + // imports and prints its version, and fails on its first model. + GPUCheck bool // FreeGB is the disk space the installation needs before it starts: a // Python engine is several gigabytes, and one that fills the disk halfway // leaves the node worse off than one that never began. FreeGB int } +// Wheel is a package published as a GitHub release asset. +type Wheel struct { + Project, Repo, Tag, Asset string +} + // Recipe knows how to install one engine. type Recipe struct { // Repo is the GitHub repository whose releases carry the builds; empty @@ -61,8 +78,10 @@ type Recipe struct { Repo string // PyPI is the project an engine installed from PyPI is published as. PyPI string - // Choose picks the build for a machine, or says why none fits. - Choose func(version string, h discovery.Host) (Build, error) + // Choose picks the build for a machine, or says why none fits. variant + // is a build chosen for the node ("cu129"); empty leaves it to the + // machine. + Choose func(version, variant string, h discovery.Host) (Build, error) } // Recipes are the engines this agent can install. @@ -104,7 +123,7 @@ type NotHere struct{ Reason string } func (e NotHere) Error() string { return e.Reason } -func chooseUV(version string, h discovery.Host) (Build, error) { +func chooseUV(version, _ string, h discovery.Host) (Build, error) { var arch string switch h.Arch { case "amd64": @@ -202,22 +221,121 @@ func versionBefore(a, b string) bool { // that load fails long after the installation looked finished. var compilerPackages = map[string][]string{"deb": {"gcc"}, "rpm": {"gcc"}} -func chooseVLLM(version string, h discovery.Host) (Build, error) { - variant, err := cudaGate("vLLM", version, h) - if err != nil { - return Build{}, err +// VLLMVariants are the builds of vLLM a node can be given by choice: CUDA +// 13 from PyPI, and CUDA 12.9 from the project's GitHub release — the same +// version, built for the drivers and GPUs CUDA 13 left behind. +var VLLMVariants = []string{"cu13", "cu129"} + +// pytorchCU129 is where the CUDA 12.9 build's torch is published. PyPI's +// torch is built for CUDA 13; the cu129 vLLM wheel needs this one. +const pytorchCU129 = "https://download.pytorch.org/whl/cu129" + +// chooseVLLM picks the vLLM build for a machine. +// +// Left to the machine (variant ""), it is CUDA 13 where that runs, and the +// same version's CUDA 12.9 build where only that does: one version across +// the cluster, which matters because the flags llmgw builds are checked +// against one version, and an older vLLM on the older cards would be a +// second set of flags nobody checked. A variant chosen for the node is that +// build, if the node can run it. +func chooseVLLM(version, variant string, h discovery.Host) (Build, error) { + // What no build of it gets past: a machine it is not published for, or + // one with no NVIDIA GPU. Only a CUDA requirement falls back to the + // CUDA 12.9 build. + switch h.Arch { + case "amd64", "arm64": + default: + return Build{}, fmt.Errorf("vLLM publishes no Linux build for %s", orUnknown(h.Arch)) } - return Build{ - Variant: variant, - Python: pythonForEngines, - Requirements: []string{"vllm==" + version}, - Binary: "vllm", - Packages: compilerPackages, - FreeGB: 25, - }, nil + if h.NVIDIADriver == "" { + return Build{}, NotHere{"vLLM runs on NVIDIA GPUs and this node has none; it serves llama.cpp models"} + } + pypi := func(v string) Build { + return Build{ + Variant: v, + Python: pythonForEngines, + Requirements: []string{"vllm==" + version}, + Binary: "vllm", + Packages: compilerPackages, + FreeGB: 25, + GPUCheck: true, + } + } + cu129 := func() (Build, error) { + if err := cudaFits("vLLM", version, "12.9", minDriverForCUDA12, minComputeForCUDA12, h); err != nil { + return Build{}, err + } + arch := map[string]string{"amd64": "x86_64", "arm64": "aarch64"}[h.Arch] + b := pypi("cu129") + b.Requirements = nil + b.Wheel = &Wheel{ + Project: "vllm", Repo: "vllm-project/vllm", Tag: "v" + version, + Asset: fmt.Sprintf("vllm-%s+cu129-cp38-abi3-manylinux_2_28_%s.whl", version, arch), + } + b.ExtraIndex = pytorchCU129 + return b, nil + } + modern := !versionBefore(version, cuda13Since["vLLM"]) + switch variant { + case "": + v, err := cudaGate("vLLM", version, h) + if err == nil { + return pypi(v), nil + } + var nh NotHere + if errors.As(err, &nh) || !modern { + return Build{}, err + } + b, err129 := cu129() + if err129 != nil { + return Build{}, fmt.Errorf("%v; nor can its CUDA 12.9 build: %v", err, err129) + } + return b, nil + case "cu13": + if !modern { + return Build{}, fmt.Errorf("vLLM %s is built for CUDA 12; there is no CUDA 13 build of it", version) + } + if _, err := cudaGate("vLLM", version, h); err != nil { + return Build{}, err + } + return pypi("cu13"), nil + case "cu129": + if !modern { + // Before 0.26 PyPI's own build is CUDA 12, and is the one given. + v, err := cudaGate("vLLM", version, h) + if err != nil { + return Build{}, err + } + return pypi(v), nil + } + if _, err := cudaGate("vLLM", version, h); err != nil { + var nh NotHere + if errors.As(err, &nh) { + return Build{}, err + } + } + return cu129() + } + return Build{}, fmt.Errorf("vLLM has no build %q; the builds are %s", variant, strings.Join(VLLMVariants, ", ")) } -func chooseSGLang(version string, h discovery.Host) (Build, error) { +// cudaFits checks a node against a CUDA's driver and GPU requirements. +func cudaFits(engine, version, cuda string, minDriver int, minCC float64, h discovery.Host) error { + if major := leadingInt(h.NVIDIADriver); major > 0 && major < minDriver { + return fmt.Errorf("%s %s built for CUDA %s needs NVIDIA driver %d or newer; this node has %s", + engine, version, cuda, minDriver, h.NVIDIADriver) + } + if cc := h.MinComputeCapability; cc > 0 && cc < minCC { + return fmt.Errorf("%s %s built for CUDA %s needs GPUs of compute capability %.1f or newer; this node has %.1f", + engine, version, cuda, minCC, cc) + } + return nil +} + +func chooseSGLang(version, chosen string, h discovery.Host) (Build, error) { + if chosen != "" { + return Build{}, fmt.Errorf("SGLang is installed in one build only, not %q", chosen) + } variant, err := cudaGate("SGLang", version, h) if err != nil { return Build{}, err @@ -252,7 +370,7 @@ func leadingInt(s string) int { // else — no toolkit. const cudaForLlamaCpp = "12.8" -func chooseLlamaCpp(version string, h discovery.Host) (Build, error) { +func chooseLlamaCpp(version, _ string, h discovery.Host) (Build, error) { // The published Linux builds are built on Ubuntu, but nothing about them // is Ubuntu's: they need glibc 2.34 and the libstdc++ of GCC 12, which // Debian 12 and RHEL 10 have and RHEL 9 does not. So the distribution's diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index 41c91ff..e1bdff9 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -143,6 +143,7 @@ func newCtlCmd() *cobra.Command { newRetryCmd(client), newUpgradeCmd(client), newEngineCheckCmd(), + newEngineBuildCmd(client), newToolCheckCmd(), newWeightsCmd(client, &asJSON), newCapacityCmd(client, &asJSON), diff --git a/internal/cli/enginebuild.go b/internal/cli/enginebuild.go new file mode 100644 index 0000000..7727af8 --- /dev/null +++ b/internal/cli/enginebuild.go @@ -0,0 +1,38 @@ +package cli + +import ( + "encoding/json" + "fmt" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" +) + +func newEngineBuildCmd(client func() *ctlClient) *cobra.Command { + return &cobra.Command{ + Use: "engine-build ", + Short: "Choose which build of an engine a GPU node installs", + Long: "A node installs the build of vLLM its hardware runs: CUDA 13 where the\n" + + "driver and GPUs allow, else the same version built for CUDA 12.9. This\n" + + "chooses instead, as the cluster page's vLLM build column does:\n\n" + + " auto what the hardware runs (the default)\n" + + " cu13 the CUDA 13 build, from PyPI\n" + + " cu129 the CUDA 12.9 build, from vLLM's GitHub release\n" + + " none no vLLM on this node\n\n" + + "The node is told at once and installs it in the background; a build it\n" + + "cannot compute with on its GPU fails to install, saying why, and the\n" + + "cluster page shows it.\n\n" + + " llmgw ctl engine-build gpu07 vllm cu129", + Args: cobra.ExactArgs(3), + RunE: func(cmd *cobra.Command, args []string) error { + body, _ := json.Marshal(adminapi.NodeEngineRequest{Engine: args[1], Build: args[2]}) + if err := client().post(cmd.Context(), "/api/v1/nodes/"+args[0]+"/engines", body, nil); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "%s: %s build %s; the node installs it in the background (llmgw ctl nodes shows it)\n", + args[0], args[1], args[2]) + return nil + }, + } +} diff --git a/internal/cli/server.go b/internal/cli/server.go index ceff217..70daa17 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -495,6 +495,8 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live // And what each model was measured to serve, for scaling and for // choosing between profiles. ctl.Reconciler().SetMeasurements(control.Measurements(st, log)) + // The engine builds chosen per node on the cluster page. + ctl.SetEngineChoices(control.NewEngineChoices(ctx, st, log)) log.Info("competing to be the control plane that places work", "owner", lead.Owner()) } @@ -504,6 +506,10 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live return cfg.EnginesFor(n.ID, slices.Sorted(maps.Keys(n.GPUsByClass())), n.Labels) }) log.Info("nodes install these engines", "engines", cfg.WantedEngines(), "overrides", len(cfg.EngineOverrides)) + if ch := ctl.EngineChoices(); ch != nil { + svc.SetEngineChoices(ch.For) + ch.OnChange(svc.Reprovision) + } // A standby must refuse agents as well as refusing to place. Commands // travel down the stream the agent opened, so a standby holding one is a // node the leader cannot command — and the leader's placements would fail diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index cfc3ebd..b1d5146 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -9,6 +9,7 @@ package adminapi import ( "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -111,6 +112,7 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("GET /api/v1/explain-placement", a.explainPlacement) mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState) mux.HandleFunc("POST /api/v1/nodes/{id}/upgrade", a.upgradeNode) + mux.HandleFunc("POST /api/v1/nodes/{id}/engines", a.setNodeEngine) mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode) mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest) mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs) @@ -156,6 +158,10 @@ type NodeView struct { // Provision is where installing each wanted engine stands. An engine is in // Engines only once its installation here is ready. Provision []inventory.EngineInstall `json:"provision,omitempty"` + // EngineBuilds are the builds chosen for this node, by engine: "auto" + // (its hardware decides), a build ("cu129"), or "none". Only on a node + // with GPUs, the only kind with a choice to make. + EngineBuilds map[string]string `json:"engine_builds,omitempty"` // OS is the operating system and architecture, as the agent read them. OS string `json:"os,omitempty"` // Warnings are facts about the machine that make it slow or unusable @@ -473,6 +479,13 @@ func (a *API) Nodes() []NodeView { } slices.Sort(v.Engines) v.Provision = n.Provision + if ch := a.ctl.EngineChoices(); ch != nil && len(n.GPUs) > 0 { + chosen := ch.For(n.ID) + v.EngineBuilds = map[string]string{"vllm": "auto"} + for e, b := range chosen { + v.EngineBuilds[e] = b + } + } if h := n.Host; h.OSID != "" { v.OS = strings.TrimSpace(h.OSID + " " + h.OSVersion + " " + h.Arch) if h.Arch == "amd64" && !h.HasCPUFlag("avx2") { @@ -1532,3 +1545,47 @@ func (a *API) setAdapters(w http.ResponseWriter, r *http.Request) { } writeJSON(w, http.StatusOK, AdaptersRequest{Adapters: out.Adapters}) } + +// NodeEngineRequest chooses one engine's build for a node. +type NodeEngineRequest struct { + Engine string `json:"engine"` + // Build is "auto", a build such as "cu129", or "none". + Build string `json:"build"` +} + +// setNodeEngine chooses an engine's build for a node, and has the node +// install it. +func (a *API) setNodeEngine(w http.ResponseWriter, r *http.Request) { + ch := a.ctl.EngineChoices() + if ch == nil { + writeError(w, http.StatusNotImplemented, "this control plane has no store to keep the choice in") + return + } + id := r.PathValue("id") + if _, ok := a.ctl.Inventory().Get(id); !ok { + writeError(w, http.StatusNotFound, "no node "+id) + return + } + var req NodeEngineRequest + if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error()) + return + } + if err := ch.Set(r.Context(), id, req.Engine, req.Build, actorOf(r)); err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusOK, req) +} + +// SetNodeEngine chooses an engine's build for a node, for the cluster page. +func (a *API) SetNodeEngine(ctx context.Context, node, engine, build, by string) error { + ch := a.ctl.EngineChoices() + if ch == nil { + return errors.New("this control plane has no store to keep the choice in") + } + if _, ok := a.ctl.Inventory().Get(node); !ok { + return errors.New("no node " + node) + } + return ch.Set(ctx, node, engine, build, by) +} diff --git a/internal/control/adminapi/nodeengine_test.go b/internal/control/adminapi/nodeengine_test.go new file mode 100644 index 0000000..fc94641 --- /dev/null +++ b/internal/control/adminapi/nodeengine_test.go @@ -0,0 +1,63 @@ +package adminapi + +import ( + "context" + "log/slog" + "net/http" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +// choiceStore keeps engine choices in memory. +type choiceStore struct{ m map[string]map[string]string } + +func (c *choiceStore) NodeEngines(context.Context) (map[string]map[string]string, error) { + return c.m, nil +} +func (c *choiceStore) SetNodeEngine(_ context.Context, node, engine, variant, _ string) error { + if c.m[node] == nil { + c.m[node] = map[string]string{} + } + c.m[node][engine] = variant + return nil +} + +func TestAnEngineBuildIsChosenForANodeAndTheNodeIsTold(t *testing.T) { + a, ctl := setup(t, testModels(), gpuNode("gpu07", 8), inventory.Node{ID: "cpu01", State: inventory.Ready}) + ch := control.NewEngineChoices(t.Context(), &choiceStore{m: map[string]map[string]string{}}, slog.New(slog.DiscardHandler)) + var told []string + ch.OnChange(func(node string) error { told = append(told, node); return nil }) + ctl.SetEngineChoices(ch) + + if code := call(t, a, http.MethodPost, "/api/v1/nodes/gpu07/engines", `{"engine":"vllm","build":"cu129"}`, nil); code != http.StatusOK { + t.Fatalf("status %d", code) + } + if len(told) != 1 || told[0] != "gpu07" { + t.Errorf("the node was not told: %v", told) + } + var nodes []NodeView + call(t, a, http.MethodGet, "/api/v1/nodes", "", &nodes) + for _, n := range nodes { + switch n.ID { + case "gpu07": + if n.EngineBuilds["vllm"] != "cu129" { + t.Errorf("gpu07 shows %v", n.EngineBuilds) + } + case "cpu01": + if n.EngineBuilds != nil { + t.Errorf("a node with no GPU has no build to choose: %v", n.EngineBuilds) + } + } + } + for _, bad := range []string{`{"engine":"vllm","build":"cu200"}`, `{"engine":"llamacpp","build":"cu129"}`} { + if code := call(t, a, http.MethodPost, "/api/v1/nodes/gpu07/engines", bad, nil); code != http.StatusBadRequest { + t.Errorf("%s: status %d", bad, code) + } + } + if code := call(t, a, http.MethodPost, "/api/v1/nodes/gpu07/engines", `{"engine":"vllm","build":"auto"}`, nil); code != http.StatusOK || + ch.For("gpu07")["vllm"] != "" { + t.Errorf("auto should leave it to the node again: %d %v", code, ch.For("gpu07")) + } +} diff --git a/internal/control/agentsvc/installable_test.go b/internal/control/agentsvc/installable_test.go index 5ee5a40..d8675e0 100644 --- a/internal/control/agentsvc/installable_test.go +++ b/internal/control/agentsvc/installable_test.go @@ -43,14 +43,40 @@ func TestEachNodeIsToldTheEnginesDecidedForIt(t *testing.T) { } return out } - if got := want(s.provisionFor(inventory.Node{ID: "volta"}, reg)); got["vllm"] != "0.25.1" { + if got := want(s.provisionFor(inventory.Node{ID: "volta"}, installable(reg))); got["vllm"] != "0.25.1" { t.Errorf("the Volta node: %v", got) } - if got := want(s.provisionFor(inventory.Node{ID: "hopper"}, reg)); got["vllm"] != "0.30.0" { + if got := want(s.provisionFor(inventory.Node{ID: "hopper"}, installable(reg))); got["vllm"] != "0.30.0" { t.Errorf("the Hopper node: %v", got) } // A node that disabled vLLM does not offer it, and is not asked for it. - if got := want(s.provisionFor(inventory.Node{ID: "hopper"}, &llmgwv1.Register{InstallableEngines: []string{"llamacpp"}})); got["vllm"] != "" { + if got := want(s.provisionFor(inventory.Node{ID: "hopper"}, []string{"llamacpp"})); got["vllm"] != "" { t.Errorf("asked a node for an engine it does not install: %v", got) } } + +func TestABuildChosenForANodeIsWhatItIsTold(t *testing.T) { + s := &Service{} + s.SetEngines(map[string]string{"llamacpp": "b1", "vllm": "0.30.0"}) + s.SetEngineChoices(func(node string) map[string]string { + return map[string]map[string]string{"gpu07": {"vllm": "cu129"}, "gpu08": {"vllm": "none"}}[node] + }) + can := []string{"llamacpp", "vllm"} + variant := func(node string) (string, bool) { + for _, e := range s.provisionFor(inventory.Node{ID: node}, can).GetEngines() { + if e.GetEngine() == "vllm" { + return e.GetVariant(), true + } + } + return "", false + } + if v, ok := variant("gpu07"); !ok || v != "cu129" { + t.Errorf("gpu07: %q, %v", v, ok) + } + if _, ok := variant("gpu08"); ok { + t.Error("gpu08 was asked for a vLLM it was chosen to run none of") + } + if v, ok := variant("gpu09"); !ok || v != "" { + t.Errorf("gpu09, left to its hardware: %q, %v", v, ok) + } +} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index f40f7e5..c2951ed 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -56,6 +56,10 @@ type Service struct { // enginesFor is what a node is told to install, set once at start: the // cluster's engines, with any override that picks the node laid over. enginesFor func(node inventory.Node) map[string]string + // choices are the engine builds chosen per node, and installableBy what + // each connected node said it can install, for telling it again. + choices func(nodeID string) map[string]string + installableBy map[string][]string // logWaits are the log requests awaiting a reply, by request id. logMu sync.Mutex @@ -78,6 +82,29 @@ func (s *Service) SetEngines(e map[string]string) { // of two GPU generations needs two vLLM builds, or vLLM on only one. func (s *Service) SetEnginesFor(f func(node inventory.Node) map[string]string) { s.enginesFor = f } +// SetEngineChoices sets where the builds chosen for each node come from: by +// engine, a build ("cu129") or "none". An engine not named is left to the +// node's hardware. +func (s *Service) SetEngineChoices(f func(nodeID string) map[string]string) { s.choices = f } + +// Reprovision tells a connected node again what to install, after a choice +// for it changed; one not connected is told when it next connects. +func (s *Service) Reprovision(nodeID string) error { + if s.enginesFor == nil { + return nil + } + s.mu.RLock() + can, ok := s.installableBy[nodeID] + s.mu.RUnlock() + node, known := s.inv.Get(nodeID) + if !ok || !known { + return nil + } + return s.send(nodeID, &llmgwv1.ControlMessage{ + Msg: &llmgwv1.ControlMessage_Provision{Provision: s.provisionFor(node, can)}, + }) +} + // conn is one agent's open stream. type conn struct { nodeID string @@ -188,7 +215,14 @@ func (s *Service) Connect( // node rebuilt from nothing is told the same as one that has been here a // month, and a changed version reaches every node on its next connect. if s.enginesFor != nil { - p := s.provisionFor(node, reg) + can := installable(reg) + s.mu.Lock() + if s.installableBy == nil { + s.installableBy = map[string][]string{} + } + s.installableBy[nodeID] = can + s.mu.Unlock() + p := s.provisionFor(node, can) if err := stream.Send(&llmgwv1.ControlMessage{ Msg: &llmgwv1.ControlMessage_Provision{Provision: p}, }); err != nil { @@ -856,15 +890,18 @@ func rate(before, now uint64, secs float64) float64 { // provisionFor is what a node is told to install: the engines decided for // it, those it can install. -func (s *Service) provisionFor(node inventory.Node, reg *llmgwv1.Register) *llmgwv1.Provision { +func (s *Service) provisionFor(node inventory.Node, can []string) *llmgwv1.Provision { p := &llmgwv1.Provision{} - can := installable(reg) wants := s.enginesFor(node) + var chosen map[string]string + if s.choices != nil { + chosen = s.choices(node.ID) + } for _, e := range slices.Sorted(maps.Keys(wants)) { - if !slices.Contains(can, e) { + if !slices.Contains(can, e) || chosen[e] == "none" { continue } - p.Engines = append(p.Engines, &llmgwv1.EngineWant{Engine: e, Version: wants[e]}) + p.Engines = append(p.Engines, &llmgwv1.EngineWant{Engine: e, Version: wants[e], Variant: chosen[e]}) } return p } diff --git a/internal/control/control.go b/internal/control/control.go index 54ce8cd..bf67ad8 100644 --- a/internal/control/control.go +++ b/internal/control/control.go @@ -75,15 +75,18 @@ func DefaultConfig() Config { // Controller owns the control plane's state. type Controller struct { - cfg Config - log *slog.Logger - inv *inventory.Store - cat *catalog.Live - rec *reconciler.Reconciler - lead *Leadership - disp reconciler.Dispatcher - sink EndpointSink - usage UsageSource + // engineChoices are the engine builds chosen per node; nil without a + // store to keep them in. + engineChoices *EngineChoices + cfg Config + log *slog.Logger + inv *inventory.Store + cat *catalog.Live + rec *reconciler.Reconciler + lead *Leadership + disp reconciler.Dispatcher + sink EndpointSink + usage UsageSource mu sync.Mutex last string // fingerprint of the last published list @@ -408,3 +411,9 @@ func EngineKeys(secret []byte) func(replicaID string) string { // QuotaUse is every capacity quota beside what its scope holds. func (c *Controller) QuotaUse() []reconciler.QuotaUse { return c.rec.QuotaUse() } + +// SetEngineChoices gives the controller the engine builds chosen per node. +func (c *Controller) SetEngineChoices(e *EngineChoices) { c.engineChoices = e } + +// EngineChoices are the engine builds chosen per node, or nil. +func (c *Controller) EngineChoices() *EngineChoices { return c.engineChoices } diff --git a/internal/control/enginechoice.go b/internal/control/enginechoice.go new file mode 100644 index 0000000..7e4bcfb --- /dev/null +++ b/internal/control/enginechoice.go @@ -0,0 +1,108 @@ +package control + +import ( + "context" + "fmt" + "log/slog" + "maps" + "slices" + "strings" + "sync" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision" +) + +// EngineChoiceStore is where the engine builds chosen per node are kept. +type EngineChoiceStore interface { + NodeEngines(ctx context.Context) (map[string]map[string]string, error) + SetNodeEngine(ctx context.Context, node, engine, variant, by string) error +} + +// EngineChoices are the engine builds chosen for nodes, rather than left to +// their hardware: vLLM's CUDA 12.9 build on a node its CUDA 13 build would +// also install on, or no vLLM at all. Kept in the store, held here for the +// agent service to ask on every registration, and pushed to a node the +// moment its choice changes. +type EngineChoices struct { + st EngineChoiceStore + log *slog.Logger + + mu sync.RWMutex + byNode map[string]map[string]string + notify func(node string) error +} + +// NewEngineChoices reads what was chosen before. +func NewEngineChoices(ctx context.Context, st EngineChoiceStore, log *slog.Logger) *EngineChoices { + c := &EngineChoices{st: st, log: log, byNode: map[string]map[string]string{}} + if got, err := st.NodeEngines(ctx); err != nil { + log.Error("could not read the engine builds chosen per node; every node installs what its hardware runs", "err", err) + } else { + c.byNode = got + } + return c +} + +// OnChange is called with a node whose choice changed, to tell it. +func (c *EngineChoices) OnChange(f func(node string) error) { + c.mu.Lock() + c.notify = f + c.mu.Unlock() +} + +// For is what was chosen for one node, by engine: a build, or "none". +func (c *EngineChoices) For(node string) map[string]string { + c.mu.RLock() + defer c.mu.RUnlock() + return maps.Clone(c.byNode[node]) +} + +// EngineBuilds are what can be chosen for each engine that has a choice: +// "auto" is the node's hardware deciding, "none" no engine at all. +func EngineBuilds(engine string) []string { + switch engine { + case "vllm": + return append(append([]string{"auto"}, provision.VLLMVariants...), "none") + case "sglang": + return []string{"auto", "none"} + } + return nil +} + +// Set chooses an engine's build for a node, keeps it, and tells the node. +func (c *EngineChoices) Set(ctx context.Context, node, engine, build, by string) error { + builds := EngineBuilds(engine) + if builds == nil { + return fmt.Errorf("%s has no builds to choose between", engine) + } + if !slices.Contains(builds, build) { + return fmt.Errorf("%s has no build %q; choose %s", engine, build, strings.Join(builds, ", ")) + } + variant := build + if build == "auto" { + variant = "" + } + if err := c.st.SetNodeEngine(ctx, node, engine, variant, by); err != nil { + return err + } + c.mu.Lock() + if variant == "" { + delete(c.byNode[node], engine) + } else { + if c.byNode[node] == nil { + c.byNode[node] = map[string]string{} + } + c.byNode[node][engine] = variant + } + notify := c.notify + c.mu.Unlock() + c.log.Info("engine build chosen for a node", "node", node, "engine", engine, "build", build, "by", by) + if notify != nil { + if err := notify(node); err != nil { + // Kept and applied on its next connection; said, since a node that + // is not connected is the usual reason. + c.log.Warn("the node was not told yet; it is when it next connects", "node", node, "err", err) + } + } + return nil +} diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index ef931d2..d39eddf 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -581,7 +581,7 @@ func Evaluate(f Facts) []Finding { // the same decision the agent makes, so the report and the node agree. func vllmFindings(f Facts, add func(Level, string, string, string)) { v := f.VLLMVersion - b, err := provision.Recipes["vllm"].Choose(v, f.Host) + b, err := provision.Recipes["vllm"].Choose(v, "", f.Host) var nh provision.NotHere switch { case errors.As(err, &nh): diff --git a/internal/doctor/doctor_test.go b/internal/doctor/doctor_test.go index 1609719..79ed89e 100644 --- a/internal/doctor/doctor_test.go +++ b/internal/doctor/doctor_test.go @@ -153,9 +153,10 @@ func TestDoctorSaysWhetherVLLMCanBeInstalled(t *testing.T) { {"CPU node", Facts{Host: discovery.Host{Arch: "amd64"}, VLLMVersion: "0.30.0"}, Info, "has none"}, // What vLLM cannot do keeps vLLM off the node, not the node out of // the cluster: it still serves llama.cpp. - {"old driver", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15"}, VLLMVersion: "0.30.0"}, Warn, "driver 580"}, - {"volta", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 7.0}, VLLMVersion: "0.30.0"}, - Warn, "before 0.26 is built for CUDA 12"}, + {"a driver too old for either build", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "470.256.02"}, VLLMVersion: "0.30.0"}, Warn, "driver 525"}, + // What CUDA 13 cannot run, its CUDA 12.9 build of the same version can. + {"volta", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.0}, + VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 200}, OK, "(cu129) can be installed"}, {"volta, a CUDA 12 vLLM", Facts{Host: discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.0}, VLLMVersion: "0.25.1", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 200}, OK, "(cu12) can be installed"}, {"small disk", Facts{Host: gpu, VLLMVersion: "0.30.0", EnginesDir: "/opt/llmgw/engines", EnginesFreeGB: 9}, Warn, "9.0 GB"}, diff --git a/internal/router/handlers.go b/internal/router/handlers.go index c47c60f..7a3df40 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -196,6 +196,11 @@ func (r *Router) Handler() http.Handler { if c, ok := r.clusterView.(web.CapacityAdmin); ok { pages.SetCapacity(c) } + // The engine build each GPU node installs, chosen on the cluster + // page. + if e, ok := r.clusterView.(web.NodeEngineAdmin); ok { + pages.RegisterNodeEngines(mux, e) + } pages.RegisterAdminQuotas(mux, r.store, r.acct) // And the report of what that tree actually used. pages.RegisterAdminUsage(mux, r.store, r.acct) diff --git a/internal/store/copy.go b/internal/store/copy.go index 11ef3e5..1c72157 100644 --- a/internal/store/copy.go +++ b/internal/store/copy.go @@ -45,6 +45,7 @@ var copyOrder = []string{ // derived from this, and a new secret would lock the router out of all // of them at the moment of the move. "cluster_secrets", + "node_engines", } // notCopied are tables deliberately left behind. diff --git a/internal/store/migrations/postgres/0014_node_engines.sql b/internal/store/migrations/postgres/0014_node_engines.sql new file mode 100644 index 0000000..3c66553 --- /dev/null +++ b/internal/store/migrations/postgres/0014_node_engines.sql @@ -0,0 +1,11 @@ +-- The build of an engine chosen for a node, where it is not left to the +-- node's hardware: vLLM's CUDA 12.9 build on a node that could run CUDA 13's, +-- or no vLLM at all. A node with no row here installs what its hardware runs. +CREATE TABLE node_engines ( + node_id TEXT NOT NULL, + engine TEXT NOT NULL, + variant TEXT NOT NULL, + updated_at BIGINT NOT NULL, + updated_by TEXT NOT NULL, + PRIMARY KEY (node_id, engine) +); diff --git a/internal/store/migrations/sqlite/0014_node_engines.sql b/internal/store/migrations/sqlite/0014_node_engines.sql new file mode 100644 index 0000000..3c66553 --- /dev/null +++ b/internal/store/migrations/sqlite/0014_node_engines.sql @@ -0,0 +1,11 @@ +-- The build of an engine chosen for a node, where it is not left to the +-- node's hardware: vLLM's CUDA 12.9 build on a node that could run CUDA 13's, +-- or no vLLM at all. A node with no row here installs what its hardware runs. +CREATE TABLE node_engines ( + node_id TEXT NOT NULL, + engine TEXT NOT NULL, + variant TEXT NOT NULL, + updated_at BIGINT NOT NULL, + updated_by TEXT NOT NULL, + PRIMARY KEY (node_id, engine) +); diff --git a/internal/store/nodeengines.go b/internal/store/nodeengines.go new file mode 100644 index 0000000..a862431 --- /dev/null +++ b/internal/store/nodeengines.go @@ -0,0 +1,44 @@ +package store + +import ( + "context" + "time" +) + +// NodeEngines are the engine builds chosen per node: node id, then engine, +// then the build ("cu129") or "none". A node or engine not here is left to +// the node's hardware. +func (s *DB) NodeEngines(ctx context.Context) (map[string]map[string]string, error) { + rows, err := s.db.QueryContext(ctx, `SELECT node_id, engine, variant FROM node_engines`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]map[string]string{} + for rows.Next() { + var node, engine, variant string + if err := rows.Scan(&node, &engine, &variant); err != nil { + return nil, err + } + if out[node] == nil { + out[node] = map[string]string{} + } + out[node][engine] = variant + } + return out, rows.Err() +} + +// SetNodeEngine chooses an engine's build for a node; an empty variant +// leaves it to the node's hardware again. +func (s *DB) SetNodeEngine(ctx context.Context, node, engine, variant, by string) error { + if variant == "" { + _, err := s.db.ExecContext(ctx, `DELETE FROM node_engines WHERE node_id = ? AND engine = ?`, node, engine) + return err + } + _, err := s.db.ExecContext(ctx, + `INSERT INTO node_engines (node_id, engine, variant, updated_at, updated_by) VALUES (?, ?, ?, ?, ?) + ON CONFLICT (node_id, engine) DO UPDATE SET variant = excluded.variant, + updated_at = excluded.updated_at, updated_by = excluded.updated_by`, + node, engine, variant, time.Now().UnixMilli(), by) + return err +} diff --git a/internal/store/nodeengines_test.go b/internal/store/nodeengines_test.go new file mode 100644 index 0000000..8a3c40f --- /dev/null +++ b/internal/store/nodeengines_test.go @@ -0,0 +1,24 @@ +package store + +import "testing" + +func TestAnEngineBuildIsChosenPerNodeAndCanBeLeftToItAgain(t *testing.T) { + db := capacityStore(t) + ctx := t.Context() + if err := db.SetNodeEngine(ctx, "gpu07", "vllm", "cu129", "ilker"); err != nil { + t.Fatal(err) + } + if err := db.SetNodeEngine(ctx, "gpu07", "vllm", "none", "ilker"); err != nil { + t.Fatal(err) + } + got, err := db.NodeEngines(ctx) + if err != nil || got["gpu07"]["vllm"] != "none" { + t.Fatalf("got %v, %v; the second choice replaces the first", got, err) + } + if err := db.SetNodeEngine(ctx, "gpu07", "vllm", "", "ilker"); err != nil { + t.Fatal(err) + } + if got, _ := db.NodeEngines(ctx); len(got["gpu07"]) != 0 { + t.Errorf("left to the node again, still %v", got) + } +} diff --git a/internal/web/admin_nodeengine.go b/internal/web/admin_nodeengine.go new file mode 100644 index 0000000..60a5100 --- /dev/null +++ b/internal/web/admin_nodeengine.go @@ -0,0 +1,49 @@ +package web + +import ( + "context" + "net/http" + "net/url" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// NodeEngineAdmin chooses an engine's build for a node: "auto", a build +// such as "cu129", or "none". +type NodeEngineAdmin interface { + SetNodeEngine(ctx context.Context, node, engine, build, by string) error +} + +// RegisterNodeEngines mounts the choice of engine builds on the cluster +// page: for an administrator, with the session's CSRF token, and answered by +// going back to the page with what happened. +func (p *AccountPages) RegisterNodeEngines(mux *http.ServeMux, a NodeEngineAdmin) { + mux.HandleFunc("POST /admin/nodes/{id}/engines", p.adminOnly(func(w http.ResponseWriter, r *http.Request, admin store.User) { + sess, _, _ := p.session(r) + if !p.accounts.CheckCSRF(sess, r.FormValue("csrf")) { + http.Error(w, "bad request", http.StatusBadRequest) + return + } + node, engine, build := r.PathValue("id"), r.FormValue("engine"), r.FormValue("build") + q := url.Values{} + if err := a.SetNodeEngine(r.Context(), node, engine, build, admin.Email); err != nil { + p.log.Warn("an engine build was not chosen", "admin", admin.Email, "node", node, "err", err) + q.Set("problem", node+": "+err.Error()) + } else { + p.log.Info("an engine build was chosen", "admin", admin.Email, "node", node, "engine", engine, "build", build) + q.Set("notice", node+" installs "+engine+" "+buildWords(build)+ + "; the node is told now, and installs it in the background") + } + http.Redirect(w, r, "/admin?"+q.Encode(), http.StatusSeeOther) + })) +} + +func buildWords(b string) string { + switch b { + case "auto": + return "in the build its hardware runs" + case "none": + return "no longer" + } + return "built for " + map[string]string{"cu13": "CUDA 13", "cu129": "CUDA 12.9"}[b] +} diff --git a/internal/web/admin_nodeengine_test.go b/internal/web/admin_nodeengine_test.go new file mode 100644 index 0000000..37b5008 --- /dev/null +++ b/internal/web/admin_nodeengine_test.go @@ -0,0 +1,75 @@ +package web + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// buildFleet is one GPU node whose vLLM build was chosen, and one CPU node. +type buildFleet struct{ fakeFleet } + +func (buildFleet) Nodes() []adminapi.NodeView { + return []adminapi.NodeView{ + {ID: "gpu07", State: "ready", GPUTotal: 8, GPUFree: 8, + EngineBuilds: map[string]string{"vllm": "cu129"}, + Provision: []inventory.EngineInstall{{Engine: "vllm", Version: "0.30.0", Variant: "cu129", State: "installing"}}}, + {ID: "cpu01", State: "ready"}, + } +} + +type recordedChoice struct{ node, engine, build, by string } + +type fakeEngineAdmin struct{ got []recordedChoice } + +func (f *fakeEngineAdmin) SetNodeEngine(_ context.Context, node, engine, build, by string) error { + f.got = append(f.got, recordedChoice{node, engine, build, by}) + return nil +} + +func TestTheVLLMBuildIsChosenOnTheClusterPage(t *testing.T) { + a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}} + p := NewAccountPages(a, nil, nil, "v0", slog.New(slog.NewTextHandler(io.Discard, nil))) + mux := http.NewServeMux() + mux.HandleFunc("GET /admin", p.AdminPage(Dashboard(buildFleet{}, nil, "v1", time.Second, false, NavExtras{}, nil))) + fe := &fakeEngineAdmin{} + p.RegisterNodeEngines(mux, fe) + + body := mirrorGet(t, mux, "/admin").Body.String() + if !strings.Contains(body, `action="/admin/nodes/gpu07/engines"`) || + !strings.Contains(body, ``) { + t.Fatalf("no choice of build for the GPU node, with its current one selected: %s", body) + } + if strings.Contains(body, `action="/admin/nodes/cpu01/engines"`) { + t.Error("a node with no GPU was offered a vLLM build") + } + if !strings.Contains(body, "CUDA 12.9 installing") { + t.Error("what is installed is not shown beside the choice") + } + + // The read-only page on the admin listener has no session, so no form. + _, ro := get(t, Dashboard(buildFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil) + if strings.Contains(ro, "/engines\"") { + t.Error("the admin listener's page offers a change it cannot sign") + } + + w := mirrorPost(t, mux, "/admin/nodes/gpu07/engines", url.Values{"csrf": {"tok"}, "engine": {"vllm"}, "build": {"none"}}) + if w.Code != http.StatusSeeOther || len(fe.got) != 1 || fe.got[0] != (recordedChoice{"gpu07", "vllm", "none", "a@example.com"}) { + t.Fatalf("status %d, chose %+v", w.Code, fe.got) + } + if loc := w.Header().Get("Location"); !strings.Contains(loc, "notice=") { + t.Errorf("redirected to %s, saying nothing", loc) + } + if w := mirrorPost(t, mux, "/admin/nodes/gpu07/engines", url.Values{"csrf": {"bad"}, "engine": {"vllm"}, "build": {"none"}}); w.Code != http.StatusBadRequest || len(fe.got) != 1 { + t.Errorf("a forged form was accepted: %d", w.Code) + } +} diff --git a/internal/web/dashboard.go b/internal/web/dashboard.go index 0e5ba67..536e184 100644 --- a/internal/web/dashboard.go +++ b/internal/web/dashboard.go @@ -19,13 +19,16 @@ type DashboardData struct { // of a GPU table that would only say there are none. CPU cpuCapacity // Menu is the shared top menu; Nav below is what the cluster page adds. - Menu Nav - Nodes []adminapi.NodeView - Models []adminapi.ModelView - Replicas []adminapi.ReplicaView - Unplaced []adminapi.ExplainView - Usage []store.KeyUsage - Summary adminapi.Summary + Menu Nav + // Notice and Problem are what the last change made from this page did, + // carried over the redirect back to it. + Notice, Problem string + Nodes []adminapi.NodeView + Models []adminapi.ModelView + Replicas []adminapi.ReplicaView + Unplaced []adminapi.ExplainView + Usage []store.KeyUsage + Summary adminapi.Summary // Fabrics is the physical fabrics and what disagrees about them, from a // source that can say. Fabrics *fabrics.Report @@ -170,6 +173,9 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat Now: time.Now().UTC().Format(time.RFC3339), AuthEnabled: usage != nil, } + if d.Menu.Admin { + d.Notice, d.Problem = r.URL.Query().Get("notice"), r.URL.Query().Get("problem") + } if src != nil { d.Nodes = src.Nodes() d.CPU = cpuCapacityOf(d.Nodes) diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index c34f9ac..0ca38e7 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -50,8 +50,9 @@ {{else}}

No nodes have registered.

{{end}}

Nodes

+{{with .Notice}}

{{.}}

{{end}}{{with .Problem}}

{{.}}

{{end}} {{if .Nodes}} -{{if .AnyGPUs}}{{end}}{{if .AnyClasses}}{{end}}{{if .AnyFabrics}}{{end}} +
NodeStateCPURAMGPUsDiskEnginesClassesFabricsModelsReplicasSeen
{{if .AnyGPUs}}{{end}}{{if .AnyGPUs}}{{end}}{{if .AnyClasses}}{{end}}{{if .AnyFabrics}}{{end}} {{range .Nodes}} @@ -66,6 +67,23 @@ was never asked about its disk must not read as a full one. */}} + {{/* Which vLLM build the node installs: left to its hardware, or chosen. + What is installed beside it, so a choice and its outcome are read + together. Only a GPU node has one; changing it is for an + administrator, on the page reached through a session. */}} + {{if $.AnyGPUs}}{{end}} {{if $.AnyClasses}}{{end}} {{if $.AnyFabrics}}{{end}} @@ -75,6 +93,13 @@ {{range .Provision}}{{if and (ne .State "ready") (ne .State "skipped")}}{{end}}{{end}} {{range .Warnings}}{{end}} {{end}}
NodeStateCPURAMGPUsDiskEnginesvLLM buildClassesFabricsModelsReplicasSeen
{{shortNode .ID $.NodePrefix}}{{if .MgmtAddr}} {{.MgmtAddr}}{{end}} logs{{if .DiskTotalGB}}{{.DiskFreeGB}}G/{{.DiskTotalGB}}G{{else}}—{{end}} {{if .Engines}}{{join .Engines}}{{else}}none yet{{end}}{{if .GPUTotal}}{{$node := .}} + {{range .Provision}}{{if eq .Engine "vllm"}}{{with .Variant}}{{buildLabel .}} {{end}}{{.State}}{{end}}{{end}} + {{if and $.Menu.CSRF .EngineBuilds}} +
+ + + + +
+ {{end}} + {{else}}—{{end}}
{{join .Classes}}{{join .Fabrics}}{{if .Models}}{{len .Models}}{{else}}—{{end}}
{{.Engine}} {{.Version}}{{if .Variant}} ({{.Variant}}){{end}} {{.State}}: {{.Detail}}
{{.}}
+ {{else}}

No agents have registered.

{{end}}

Models

diff --git a/internal/web/web.go b/internal/web/web.go index 60a389d..534aebd 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -26,6 +26,7 @@ import ( "sync" "time" + "gitea.mynodes.xyz/ilker/llmgw/internal/control" "gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) @@ -118,7 +119,25 @@ var funcs = template.FuncMap{ }, // shortNode is a node's name without what every node's name starts with. "shortNode": strings.TrimPrefix, - "age": age, + // engineBuilds are the builds an engine can be given on a node, and + // buildLabel says one in words. + "engineBuilds": control.EngineBuilds, + "buildLabel": func(b string) string { + switch b { + case "auto": + return "auto: what the hardware runs" + case "cu13": + return "CUDA 13" + case "cu129": + return "CUDA 12.9" + case "cu12": + return "CUDA 12" + case "none": + return "none" + } + return b + }, + "age": age, // pctbar is a percentage as the minibar template takes it: whole, and // held to 0..100 so a measurement a little over does not draw past its box. "pctbar": func(v float64) struct{ P int } {