diff --git a/docs/node-onboarding.md b/docs/node-onboarding.md index 8c22e43..6748bd6 100644 --- a/docs/node-onboarding.md +++ b/docs/node-onboarding.md @@ -92,6 +92,15 @@ a flag that only appears there is exactly the one nobody notices has gone — an checks every flag against the engine's own `--help`. No GPU required: a CLI prints its help on any machine. +Some programs do not give up their whole help for `--help`. vLLM prints a +summary of section names and expects `--help=all`; checking against the summary +reports every flag as missing, confidently and wrongly. `engine-check` knows +that one. If you add an engine, check that what you get back is the full list. + +vLLM also refuses to do anything at all on a machine with no accelerator, +including print its help, unless `VLLM_TARGET_DEVICE` says what to assume. +`engine-check` sets it. + Give it the **whole** help. Truncating llama.cpp's 725 lines to 400 reports five flags as missing that the running cluster uses; the command warns when the text it was given does not document `--help` itself, which is the cheapest sign it diff --git a/internal/cli/enginecheck.go b/internal/cli/enginecheck.go index 390db06..bad7d12 100644 --- a/internal/cli/enginecheck.go +++ b/internal/cli/enginecheck.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os/exec" + "path/filepath" "sort" "strings" "time" @@ -16,6 +17,7 @@ import ( func newEngineCheckCmd() *cobra.Command { var ( help string + bin string nodes int tp int perNode int @@ -42,63 +44,112 @@ func newEngineCheckCmd() *cobra.Command { return err } + // The binary lives where the agent's configuration says, which is + // rarely on PATH: a check that cannot find the engine reports + // nothing rather than reporting a problem. + cfg := engines.Config{} + if bin != "" { + switch name { + case "llamacpp": + cfg.LlamaCpp.Bin = bin + case "vllm": + cfg.VLLM.Bin = bin + case "sglang": + cfg.SGLang.Python = bin + case "sim": + cfg.Sim.Bin = bin + } + reg = engines.NewRegistry(cfg) + if eng, err = reg.Get(name); err != nil { + return err + } + } + spec := representativeShard(name, nodes, tp, perNode) cmds, err := eng.Commands(spec) if err != nil { return fmt.Errorf("building a command for %s: %w", name, err) } - text := help - if text == "" { - if text, err = runHelp(cmd.Context(), cmds); err != nil { - return err - } - } - - // A truncated help produces confident nonsense: every flag below the - // cut looks missing. Real help text documents --help itself, which - // is the cheapest signal that we are looking at all of it. - if !mentions(text, "--help") { - cmd.PrintErrln("warning: this help text does not document --help, " + - "which usually means it was truncated. Flags below the cut will " + - "look missing.") - } - + out, errOut := cmd.OutOrStdout(), cmd.ErrOrStderr() + var checked, skipped int var unknown []string - seen := map[string]bool{} - for _, c := range cmds { + + // Per command, and that distinction is the whole point. A shard can + // be several processes — vLLM under Ray is `ray start` and then + // `vllm serve` — and they are different programs with different + // vocabularies. Checking the sidecar's flags against the engine's + // help reports --head and --num-gpus as missing, which is true of + // vllm and irrelevant, because ray is what receives them. + for i, c := range cmds { + fmt.Fprintf(errOut, " %s %s\n", c.Path, strings.Join(c.Args, " ")) + + text := "" + switch { + case help != "" && i == len(cmds)-1: + // A supplied help belongs to the engine, which is the last + // command; the sidecars have their own and we do not have it. + text = help + case help != "": + skipped++ + fmt.Fprintf(errOut, " skipped: --help-output describes the engine, not %s\n", c.Name) + continue + default: + var err error + if text, err = runHelp(cmd.Context(), c); err != nil { + skipped++ + fmt.Fprintf(errOut, " skipped: %v\n", err) + continue + } + } + + // A truncated help produces confident nonsense: every flag below + // the cut looks missing. Real help documents --help itself, + // which is the cheapest signal that we have all of it. + if !mentions(text, "--help") { + fmt.Fprintf(errOut, " warning: this help does not document --help, "+ + "which usually means it was truncated; flags below the cut will look missing\n") + } + + seen := map[string]bool{} for _, a := range c.Args { flag, _, _ := strings.Cut(a, "=") if !strings.HasPrefix(flag, "--") || seen[flag] { continue } seen[flag] = true + checked++ if !mentions(text, flag) { - unknown = append(unknown, flag) + unknown = append(unknown, c.Name+": "+flag) } } } sort.Strings(unknown) - out := cmd.OutOrStdout() - fmt.Fprintf(out, "%s: %d flags checked\n", name, len(seen)) - for _, c := range cmds { - fmt.Fprintf(cmd.ErrOrStderr(), " %s %s\n", c.Path, strings.Join(c.Args, " ")) + fmt.Fprintf(out, "%s: %d flags checked", name, checked) + if skipped > 0 { + fmt.Fprintf(out, ", %d commands skipped", skipped) } + fmt.Fprintln(out) + if len(unknown) == 0 { - fmt.Fprintln(out, "every flag llmgw builds is accepted by this engine") + if checked == 0 { + return fmt.Errorf("nothing was checked; install the engine here or pass --help-output") + } + fmt.Fprintln(out, "every flag llmgw builds is accepted") return nil } - fmt.Fprintf(out, "\n%d flags this engine does not mention:\n", len(unknown)) + fmt.Fprintf(out, "\n%d flags not mentioned by the program they are passed to:\n", len(unknown)) for _, f := range unknown { fmt.Fprintf(out, " %s\n", f) } - return fmt.Errorf("%s does not accept %d of the flags llmgw builds", name, len(unknown)) + return fmt.Errorf("%s: %d flags are not accepted", name, len(unknown)) }, } f := cmd.Flags() f.StringVar(&help, "help-output", "", "a saved --help to check against, instead of running the engine") + f.StringVar(&bin, "bin", "", "where the engine lives, when it is not on PATH") f.IntVar(&nodes, "nodes", 2, "shape the command for a replica of this many nodes") f.IntVar(&tp, "tp", 8, "tensor parallel degree") f.IntVar(&perNode, "gpus-per-node", 4, "GPUs each node contributes") @@ -132,16 +183,20 @@ func seq(n int) []int { return out } -// runHelp asks the engine to describe itself. +// helpArgs is how each program is asked for its *complete* help. // -// The last command is the one that serves; the sidecars before it are a -// different program with a different vocabulary, so its help would be the wrong -// help to check against. -func runHelp(ctx context.Context, cmds []engines.Command) (string, error) { - c := cmds[len(cmds)-1] +// Not every program gives it all up for --help. vLLM prints a summary listing +// section names and tells you to ask again per section; checking against that +// reports every flag as missing, confidently and wrongly. This is exactly what +// happened the first time this ran against a real vLLM. +var helpArgs = map[string][]string{ + "vllm": {"--help=all"}, +} - // Everything up to the first flag, plus --help. Passing the real arguments - // would start the engine. +// runHelp asks one program to describe itself. +func runHelp(ctx context.Context, c engines.Command) (string, error) { + // Everything up to the first flag, plus the help request. Passing the real + // arguments would start the engine. var args []string for _, a := range c.Args { if strings.HasPrefix(a, "-") { @@ -149,18 +204,27 @@ func runHelp(ctx context.Context, cmds []engines.Command) (string, error) { } args = append(args, a) } - args = append(args, "--help") + ask, ok := helpArgs[filepath.Base(c.Path)] + if !ok { + ask = []string{"--help"} + } + args = append(args, ask...) ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() - out, err := exec.CommandContext(ctx, c.Path, args...).CombinedOutput() - if err != nil && len(out) == 0 { - return "", fmt.Errorf("running %s %s: %w\n"+ - "(is the engine installed here? use --help-output to check against a saved one)", - c.Path, strings.Join(args, " "), err) + ec := exec.CommandContext(ctx, c.Path, args...) + // vLLM refuses to do anything at all on a machine with no accelerator, + // including print its help, unless it is told what to assume. Asking an + // engine about its flags should not require the hardware it runs on. + ec.Env = append(ec.Environ(), "VLLM_TARGET_DEVICE=cpu") + + raw, err := ec.CombinedOutput() + if err != nil && len(raw) == 0 { + return "", fmt.Errorf("could not run %s: %w (install it here, or pass --help-output)", + c.Path, err) } - return string(out), nil + return string(raw), nil } // mentions reports whether the help text documents a flag. diff --git a/llmgw-plan.md b/llmgw-plan.md index e59b102..3414a01 100644 --- a/llmgw-plan.md +++ b/llmgw-plan.md @@ -1408,11 +1408,18 @@ The launch path is now built too: real collective run. Those need hardware. The GPUs here are an inventory, the fabric is a sysfs-shaped directory, and the engine speaks no NCCL. - [x] `ctl engine-check`, which asks an engine whether it still accepts the - flags llmgw builds. No GPU needed — a CLI prints its help anywhere. - Verified against llama.cpp b10991: all seven flags accepted. -- [ ] **Run `engine-check` against vLLM and SGLang.** The launch *sequence* is - exercised on the simulated cluster and the *arguments* now have a way to - be checked; what is missing is an installed engine to check them against. + flags llmgw builds. No GPU needed — a CLI prints its help anywhere, and + vLLM prints its own once told to assume a device. +- [x] **Checked against real engines.** vLLM 0.29.0: all five `vllm serve` flags + accepted (`--host`, `--port`, `--tensor-parallel-size`, + `--served-model-name`, `--distributed-executor-backend`). llama.cpp + b10991: all seven accepted. +- [ ] **The Ray sidecar is still unchecked.** vLLM 0.29 does not install Ray, so + `ray start --head --block --port --num-gpus` has never been run against + Ray's own help. `launch_mode: mp` avoids it entirely and is worth + preferring until someone checks. +- [ ] **SGLang is unchecked**, for the same reason as Ray: not installed + anywhere yet. - **Exit criterion (met on the simulated cluster):** a 2-node replica on each fabric; killing a node rebuilds the replica within one fabric and never