package engines import ( "context" "errors" "os/exec" "regexp" "sort" "strings" "time" ) // Unknown is the version reported for an engine that is installed and would // not say which version it is. // // A value, not an absent key, because the two mean different things and // placement reads the keys: absent is "this node cannot run it", Unknown is // "it is here and the probe could not get a number out of it". Collapsing them // would make a slow engine look like a missing one and take a node out of // scheduling for being slow to answer --version. const Unknown = "unknown" // Prober is an engine that can say whether it is installed on this machine. // // Optional, and asserted for rather than part of Engine: an adapter that has no // way to check should report nothing rather than be forced to invent an answer. type Prober interface { // Probe returns the engine's version, or an error when it is not installed. Probe(ctx context.Context) (string, error) } // Probe asks every registered engine whether it is here. // // The result is what the node reports at registration, and its key set is what // placement filters on. Until this existed nobody knew: the registry registers // every adapter whether or not its binary exists, so a node with no vLLM // accepted a vLLM shard, registered healthy, and failed the shard minutes later // with "executable file not found" — in the engine's own words, on a machine // nobody was watching. // // Bounded, because it runs on the path to registering and an agent that cannot // reach the control plane is worse than one that reports an incomplete list. func (r *Registry) Probe(ctx context.Context) map[string]string { out := map[string]string{} for name, e := range r.engines { p, ok := e.(Prober) if !ok { continue } v, err := p.Probe(ctx) if err != nil { continue } if v == "" { v = Unknown } out[name] = v } return out } // Names of the engines in a probe result, sorted. func ProbedNames(m map[string]string) []string { out := make([]string, 0, len(m)) for n := range m { out = append(out, n) } sort.Strings(out) return out } // probeTimeout bounds one engine's version check. // // Generous, because vLLM's --version imports torch and that is seconds on a // cold page cache, and mean, because this is startup: an engine that has not // answered in this long is reported as present-with-unknown-version rather // than waited for. const probeTimeout = 8 * time.Second // run executes a version command, saying whether the program was even there. // // The two failures are different answers and the caller has to tell them // apart. A binary that is absent is an engine this node cannot run. A binary // that is present and exits non-zero is an engine it probably can — unless the // command was a test of whether a module imports, where a non-zero exit is the // whole answer. func run(ctx context.Context, bin string, args ...string) (out string, found bool, ok bool) { path, err := exec.LookPath(bin) if err != nil { return "", false, false } ctx, cancel := context.WithTimeout(ctx, probeTimeout) defer cancel() // Combined, not just stdout: engines print their version to either, and one // that chose stderr would otherwise read as having no version at all. b, err := exec.CommandContext(ctx, path, args...).CombinedOutput() return string(b), true, err == nil } // installedBinary reports an engine that is its own executable. // // It resolves the path and runs nothing. The first version of this ran the // engine with --version, which is fine for an engine and not fine for the // wrapper scripts people put in front of one: a script that shifts past flags // it does not recognise and then execs the server will start an inference // server, at agent startup, on every node. The agent's own test suite installs // exactly such a script, and caught it. // // So the version is not read here. Presence is what placement needs, and what // the version is for — telling whether an engine still accepts the flags llmgw // builds — is `llmgw ctl engine-check`, which is run deliberately by somebody // who meant to run the engine. func installedBinary(bin string) (string, error) { if _, err := exec.LookPath(bin); err != nil { return "", errNotInstalled } return Unknown, nil } // versionOfModule probes an engine that is a module inside an interpreter. // // This one does execute, and safely: the command is llmgw's own one-line // import, not the engine's launcher, so there is no wrapper to shift past // flags and start a server. There is also no alternative — a module has no // path to look up. // // A non-zero exit is the answer: python3 exists on almost every machine, so // presence of the interpreter says nothing. Treating a failed import as // "installed" reported SGLang on every node in a fleet that had none, which // would take nodes into scheduling for an engine they cannot run — the exact // failure this whole change exists to stop. func versionOfModule(ctx context.Context, python string, args ...string) (string, error) { out, found, ok := run(ctx, python, args...) if !found || !ok { return "", errNotInstalled } return firstVersion(out), nil } var errNotInstalled = errors.New("not installed on this node") var versionPattern = regexp.MustCompile(`\b\d+\.\d+(\.\d+)?\S*`) // firstVersion is the first thing in the output that looks like a version. // // Pattern rather than parsing each engine's format: they print "vllm 0.6.3", // "version: b4123 (…)", and a paragraph with the number in the middle, and all // of those change between releases. What this is for is telling one build from // another, which the first number does. func firstVersion(s string) string { for _, line := range strings.Split(s, "\n") { if m := versionPattern.FindString(line); m != "" { return strings.TrimRight(m, ",;)") } } return Unknown } // Configured is an engine whose location the agent's configuration names. type Configured interface { Configured() bool } // ProbeConfigured probes only the engines the configuration names. // // Probe looks everywhere — PATH, the default interpreter — and reports what // it finds, which is exactly the trust in whatever happens to be on a machine // that installing engines replaces. A node the agent provisions reports what // it installed, plus what the operator named on purpose; nothing else. func (r *Registry) ProbeConfigured(ctx context.Context) map[string]string { out := map[string]string{} for name, e := range r.engines { c, ok := e.(Configured) if !ok || !c.Configured() { continue } p, ok := e.(Prober) if !ok { continue } if v, err := p.Probe(ctx); err == nil { if v == "" { v = Unknown } out[name] = v } } return out } // Probe reports whether the llama.cpp server is installed. func (l *LlamaCpp) Probe(context.Context) (string, error) { return installedBinary(l.bin()) } // Configured reports whether the configuration names vLLM. func (v *VLLM) Configured() bool { return v.cfg.Bin != "" } // Configured reports whether the configuration names SGLang's interpreter. func (g *SGLang) Configured() bool { return g.cfg.Python != "" } // Probe reports whether vLLM is installed. func (v *VLLM) Probe(context.Context) (string, error) { return installedBinary(v.bin()) } // Probe reports the SGLang version. // // Through the interpreter, because SGLang is a module rather than a binary: // asking the shell for "sglang" finds nothing on a machine that can run it // perfectly well. func (g *SGLang) Probe(ctx context.Context) (string, error) { return versionOfModule(ctx, g.python(), "-c", "import sglang, sys; sys.stdout.write(getattr(sglang, '__version__', ''))") }