Install vLLM on every GPU node that can run it, as llama.cpp is
vLLM was installed by hand and named in each agent's configuration. The agent now installs it: a Python 3.12 environment built by uv, which is itself a GitHub release checked against its checksum, from PyPI resolved as it stood a day after that vLLM was published and installed with --require-hashes. The resolved list is kept in the environment. The environment is built relocatable under a scratch name and renamed into place once it runs, like an archive. The default is 0.30.0, whose flags were checked against the ones llmgw builds. It is built for CUDA 13, so it needs driver 580 and a Turing GPU or newer, and 25 GB free while it installs; a node lacking one says which, and a node with no GPU skips it without being shown as a fault. gcc comes with it because Triton compiles its launcher at the first model load. SGLang installs the same way when the control plane names a version. llmgw doctor reports whether vLLM can be installed on the node. Checked by installing 0.30.0 this way on a test node: 7.7 GB, torch 2.13 for CUDA 13.0, importing cleanly; the node has no GPU, so vLLM was not started. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
e751c89937
commit
4c5a2fd6da
21 files changed
+853
-100
No files matched your search
@@ -34,6 +34,9 @@ const (
|
||||
Ready = "ready"
|
||||
Failed = "failed"
|
||||
Unsupported = "unsupported"
|
||||
// Skipped is an engine this node is not meant to run, such as vLLM on a
|
||||
// machine with no GPU. Nothing is wrong, and nothing is shown as wrong.
|
||||
Skipped = "skipped"
|
||||
)
|
||||
|
||||
// Status is where one engine's installation stands.
|
||||
@@ -62,9 +65,16 @@ type Installer struct {
|
||||
|
||||
// Run executes a command; a seam for tests. Nil runs it for real.
|
||||
Run func(ctx context.Context, env []string, name string, args ...string) ([]byte, error)
|
||||
// APIBase and DownloadBase default to GitHub; tests point them at a fake.
|
||||
// FreeSpace reports the gigabytes free under a directory; a seam for
|
||||
// tests. Nil asks the filesystem.
|
||||
FreeSpace func(dir string) (float64, error)
|
||||
// RunStdin is Run with input; a seam for tests. Nil runs it for real.
|
||||
RunStdin func(ctx context.Context, env []string, stdin, name string, args ...string) ([]byte, error)
|
||||
// APIBase and DownloadBase default to GitHub, PyPIBase to PyPI; tests
|
||||
// point them at a fake.
|
||||
APIBase string
|
||||
DownloadBase string
|
||||
PyPIBase string
|
||||
}
|
||||
|
||||
// DefaultRoot is where engines are installed when nothing says otherwise.
|
||||
@@ -75,12 +85,16 @@ const DefaultRoot = "/opt/llmgw/engines"
|
||||
const marker = ".llmgw-installed"
|
||||
|
||||
type markerFile struct {
|
||||
Engine string `json:"engine"`
|
||||
Version string `json:"version"`
|
||||
Variant string `json:"variant"`
|
||||
Binary string `json:"binary"`
|
||||
SHA256 map[string]string `json:"sha256"`
|
||||
Installed time.Time `json:"installed"`
|
||||
Engine string `json:"engine"`
|
||||
Version string `json:"version"`
|
||||
Variant string `json:"variant"`
|
||||
Binary string `json:"binary"`
|
||||
SHA256 map[string]string `json:"sha256,omitempty"`
|
||||
// For a Python engine: what was asked for, PyPI as of when, by which uv.
|
||||
Requirements []string `json:"requirements,omitempty"`
|
||||
ExcludeNewer string `json:"exclude_newer,omitempty"`
|
||||
UV string `json:"uv,omitempty"`
|
||||
Installed time.Time `json:"installed"`
|
||||
}
|
||||
|
||||
// Converge brings every wanted engine to ready, or says why it cannot.
|
||||
@@ -110,6 +124,10 @@ func (in *Installer) one(ctx context.Context, w Want) Status {
|
||||
build, err := recipe.Choose(w.Version, in.Host)
|
||||
if err != nil {
|
||||
st.State, st.Detail = Unsupported, err.Error()
|
||||
var nh NotHere
|
||||
if errors.As(err, &nh) {
|
||||
st.State = Skipped
|
||||
}
|
||||
return st
|
||||
}
|
||||
st.Variant = build.Variant
|
||||
@@ -168,6 +186,9 @@ func (in *Installer) ready(st Status, b Build, bin, version string) Status {
|
||||
// Into a scratch directory renamed into place at the end, so a failure halfway
|
||||
// leaves nothing that looks installed.
|
||||
func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir string, st *Status) error {
|
||||
if b.Python != "" {
|
||||
return in.installPython(ctx, r, w, b, dir, st)
|
||||
}
|
||||
if err := in.packages(ctx, b.Packages, st); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -235,18 +256,32 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir
|
||||
// running it is the only proof that the rest — the right instruction set, the
|
||||
// right driver — is in place.
|
||||
func (in *Installer) verify(ctx context.Context, dir string, b Build) (bin, version string, err error) {
|
||||
bin, err = findFile(dir, b.Binary)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if out, err := in.run(ctx, nil, "ldd", bin); err == nil {
|
||||
if missing := lddMissing(string(out)); len(missing) > 0 {
|
||||
return "", "", fmt.Errorf("%s cannot run on this system: %s", b.Binary, strings.Join(missing, "; "))
|
||||
timeout := 60 * time.Second
|
||||
if b.Python != "" {
|
||||
// A script in the environment, not a binary to ldd; and importing
|
||||
// torch and the engine behind it takes a while on a cold disk.
|
||||
bin, timeout = filepath.Join(dir, "bin", b.Binary), 3*time.Minute
|
||||
if _, err := os.Stat(bin); err != nil {
|
||||
return "", "", fmt.Errorf("%s is not in the environment: %w", b.Binary, err)
|
||||
}
|
||||
} else {
|
||||
bin, err = findFile(dir, b.Binary)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if out, err := in.run(ctx, nil, "ldd", bin); err == nil {
|
||||
if missing := lddMissing(string(out)); len(missing) > 0 {
|
||||
return "", "", fmt.Errorf("%s cannot run on this system: %s", b.Binary, strings.Join(missing, "; "))
|
||||
}
|
||||
}
|
||||
}
|
||||
tctx, cancel := context.WithTimeout(ctx, 60*time.Second)
|
||||
args := b.VersionArgs
|
||||
if len(args) == 0 {
|
||||
args = []string{"--version"}
|
||||
}
|
||||
tctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
out, err := in.run(tctx, nil, bin, "--version")
|
||||
out, err := in.run(tctx, nil, bin, args...)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("%s --version failed: %v: %s", b.Binary, err, lastLine(out))
|
||||
}
|
||||
@@ -607,6 +642,13 @@ func (in *Installer) report(st Status) {
|
||||
}
|
||||
}
|
||||
|
||||
func runWithStdin(ctx context.Context, env []string, stdin, name string, args ...string) ([]byte, error) {
|
||||
cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // fixed commands and paths we installed
|
||||
cmd.Env = append(os.Environ(), env...)
|
||||
cmd.Stdin = strings.NewReader(stdin)
|
||||
return cmd.CombinedOutput()
|
||||
}
|
||||
|
||||
func (in *Installer) run(ctx context.Context, env []string, name string, args ...string) ([]byte, error) {
|
||||
if in.Run != nil {
|
||||
return in.Run(ctx, env, name, args...)
|
||||
@@ -662,5 +704,11 @@ func firstVersionLine(b []byte) string {
|
||||
return strings.TrimSpace(l)
|
||||
}
|
||||
}
|
||||
// vllm --version says the number and nothing else.
|
||||
for _, l := range strings.Split(string(b), "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" && l[0] >= '0' && l[0] <= '9' {
|
||||
return l
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
Reference in new issue
Block a user