Give each GPU node a vLLM build it can run, chosen on the cluster page
A node that cannot run the CUDA 13 build installs the same version's CUDA 12.9 build from vLLM's GitHub release, with PyTorch's cu129 torch; the install computes on the GPU before it counts. The build can be chosen per node — auto, cu13, cu129, none — on the cluster page, with ctl engine-build, or over the admin API; the control plane keeps the choice and tells the node at once. Refs #2. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
2735c5a318
commit
4f280a3546
32 files changed
+1004
-71
No files matched your search
@@ -86,28 +86,62 @@ func (in *Installer) installPython(ctx context.Context, r Recipe, w Want, b Buil
|
||||
}
|
||||
python := filepath.Join(tmp, "bin", "python")
|
||||
|
||||
in.progress(st, "resolving %s against PyPI as of %s", strings.Join(b.Requirements, " "), asOf[:10])
|
||||
reqs := b.Requirements
|
||||
if b.Wheel != nil {
|
||||
// The engine from its GitHub release, pinned to the digest GitHub
|
||||
// publishes for the file: the same trust as an archive, and uv
|
||||
// refuses a download that does not match it.
|
||||
in.progress(st, "asking GitHub for %s's digest", b.Wheel.Asset)
|
||||
sums, err := in.checksums(ctx, b.Wheel.Repo, b.Wheel.Tag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sum, ok := sums[b.Wheel.Asset]
|
||||
if !ok {
|
||||
return fmt.Errorf("%s %s publishes no %s", b.Wheel.Repo, b.Wheel.Tag, b.Wheel.Asset)
|
||||
}
|
||||
url := strings.TrimSuffix(in.downloadBase(), "/") + "/" + b.Wheel.Repo + "/releases/download/" +
|
||||
b.Wheel.Tag + "/" + b.Wheel.Asset
|
||||
reqs = append([]string{b.Wheel.Project + " @ " + url + "#sha256=" + sum}, reqs...)
|
||||
}
|
||||
// A second index, for the torch the engine was built on. Every package
|
||||
// is taken from whichever of the two has the version resolved — both are
|
||||
// the publishers' own — and each file is still pinned by its hash.
|
||||
var index []string
|
||||
if b.ExtraIndex != "" {
|
||||
index = []string{"--extra-index-url", b.ExtraIndex, "--index-strategy", "unsafe-best-match"}
|
||||
}
|
||||
|
||||
in.progress(st, "resolving %s against PyPI as of %s", strings.Join(reqs, " "), asOf[:10])
|
||||
lock := filepath.Join(tmp, lockName)
|
||||
args := []string{"pip", "compile", "--python", python, "--exclude-newer", asOf,
|
||||
"--generate-hashes", "--quiet", "--output-file", lock, "-"}
|
||||
if out, err := in.runStdin(ctx, env, strings.Join(b.Requirements, "\n")+"\n", uv.Binary, args...); err != nil {
|
||||
return fmt.Errorf("resolving %s: %v: %s", strings.Join(b.Requirements, " "), err, tail(out))
|
||||
args := append([]string{"pip", "compile", "--python", python, "--exclude-newer", asOf,
|
||||
"--generate-hashes", "--quiet", "--output-file", lock}, index...)
|
||||
args = append(args, "-")
|
||||
if out, err := in.runStdin(ctx, env, strings.Join(reqs, "\n")+"\n", uv.Binary, args...); err != nil {
|
||||
return fmt.Errorf("resolving %s: %v: %s", strings.Join(reqs, " "), err, tail(out))
|
||||
}
|
||||
|
||||
in.progress(st, "downloading and installing %s and its dependencies: several gigabytes, which takes minutes",
|
||||
strings.Join(b.Requirements, " "))
|
||||
if out, err := in.run(ctx, env, uv.Binary, "pip", "install", "--python", python,
|
||||
"--require-hashes", "--no-deps", "-r", lock); err != nil {
|
||||
return fmt.Errorf("installing %s: %v: %s", strings.Join(b.Requirements, " "), err, tail(out))
|
||||
strings.Join(reqs, " "))
|
||||
install := append([]string{"pip", "install", "--python", python,
|
||||
"--require-hashes", "--no-deps", "-r", lock}, index...)
|
||||
if out, err := in.run(ctx, env, uv.Binary, install...); err != nil {
|
||||
return fmt.Errorf("installing %s: %v: %s", strings.Join(reqs, " "), err, tail(out))
|
||||
}
|
||||
|
||||
in.progress(st, "checking that %s runs", w.Engine)
|
||||
if _, _, err := in.verify(ctx, tmp, b); err != nil {
|
||||
return err
|
||||
}
|
||||
if b.GPUCheck && in.Host.NVIDIADriver != "" {
|
||||
in.progress(st, "checking that %s (%s) computes on this node's GPU", w.Engine, b.Variant)
|
||||
if err := in.gpuCheck(ctx, python, w, b); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
m, _ := json.MarshalIndent(markerFile{
|
||||
Engine: w.Engine, Version: w.Version, Variant: b.Variant,
|
||||
Binary: filepath.Join("bin", b.Binary), Requirements: b.Requirements,
|
||||
Binary: filepath.Join("bin", b.Binary), Requirements: reqs,
|
||||
ExcludeNewer: asOf, UV: UVVersion, Installed: time.Now().UTC(),
|
||||
}, "", " ")
|
||||
if err := os.WriteFile(filepath.Join(tmp, marker), m, 0o644); err != nil { //nolint:gosec // not secret
|
||||
@@ -203,3 +237,26 @@ func tail(b []byte) string {
|
||||
}
|
||||
return strings.Join(lines, " / ")
|
||||
}
|
||||
|
||||
// gpuProbe is what the environment is asked to run on the GPU: one small
|
||||
// operation, which needs the torch to have kernels for the card.
|
||||
const gpuProbe = `import torch
|
||||
assert torch.cuda.is_available(), "torch sees no CUDA device"
|
||||
x = torch.ones(64, device="cuda")
|
||||
print(float((x * 2).sum()), torch.cuda.get_device_name(0), torch.cuda.get_device_capability(0))`
|
||||
|
||||
// gpuCheck runs gpuProbe in a freshly installed environment.
|
||||
func (in *Installer) gpuCheck(ctx context.Context, python string, w Want, b Build) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
|
||||
defer cancel()
|
||||
out, err := in.run(ctx, nil, python, "-c", gpuProbe)
|
||||
if err != nil {
|
||||
hint := ""
|
||||
if strings.Contains(string(out), "no kernel image") {
|
||||
hint = "; this build has no kernels for this GPU, so try the other build for this node"
|
||||
}
|
||||
return fmt.Errorf("%s %s (%s) installed but cannot compute on this node's GPU: %s%s",
|
||||
w.Engine, w.Version, b.Variant, tail(out), hint)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user