Files
llmgw/internal/agent/engines/unsafeenv.go
T
Ilker ManapandClaude Opus 5.5 c0d5194ea0 Strata runs as an engine: a 125B MoE on one card
Strata (github.com/Niko1221/Strata) keeps a mixture-of-experts model's
experts in RAM and the ones used most in VRAM; on cuda01's RTX 6000 it
serves Qwen3.8-Flash-Next 125B at 70-87 tokens a second to one client.

- engines/strata.go: the adapter. The agent starts llmgw strata-serve,
  which runs Strata's setup the way its desktop app does
  (STRATA_CONTROL_INSTALL: installs nothing, writes only under the
  replica's and the node's directories) on the fetched GGUF files, points
  the configuration it writes at the installed engine, and execs Strata's
  server. The key goes in STRATA_API_KEY, never on a command line.
- A profile's args go to setup: --family, --size (setup's --model, which
  is the source everywhere else), --context, --parallel, ... Flags that
  move files or pick the engine or GPU are refused.
- provision: the source pinned by digest in a Python environment, and the
  release's prebuilt engine unpacked beside it, checked against the digest
  GitHub publishes: CUDA 13 (7.5+, driver 580) or CUDA 12.9 (6.0-8.9,
  driver 575). Python builds can now carry release assets.
- Metrics are vLLM's names under /metrics/prometheus.
- The catalog wants one card and a family, and a directory, not one .gguf.

Opt-in: not in DefaultEngines.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-29 15:09:13 +01:00

268 lines
12 KiB
Go

package engines
import (
"fmt"
"os"
"strings"
)
// A model profile's env and args are the operator's escape hatch for tuning
// an engine, and they reach a process the agent starts as root on every node
// the model is placed on. Some of what an engine accepts is not tuning but a
// way to run other code, or to read or write files, in that process: a
// loader preload, an NCCL network plugin, a CUDA injection library, vLLM's
// --trust-remote-code, llama.cpp's --log-file pointed at /etc/cron.d. Taken
// from a model definition, which the admin API or a config file supplies,
// they would turn "add a model" into "run this as root on every node".
//
// So env is an allowlist — the families of variables engines are tuned with
// — less any whose name says it names a path, a plugin or a library, and args
// are checked against what each engine takes that reads, writes or loads.
// Both are refused where a model is defined and again where an engine starts.
// envFamilies are the prefixes of variables engines are tuned with.
var envFamilies = []string{
"CUDA_", "NCCL_", "NVIDIA_", "NVTE_", "GLOO_", "UCX_", "TORCH_", "PYTORCH_",
"VLLM_", "SGLANG_", "GGML_", "LLAMA_", "OMP_", "MKL_", "OPENBLAS_", "KMP_",
"HF_", "TRANSFORMERS_", "TOKENIZERS_", "TRITON_", "FLASHINFER_",
"TRTLLM_", "TLLM_",
}
// envExact are single variables that are tuning too.
var envExact = map[string]bool{
"PYTHONUNBUFFERED": true, "PYTHONHASHSEED": true, "TZ": true,
}
// envNever are names in an allowed family that load code or move where the
// engine reads and writes.
var envNever = []string{
"PLUGIN", "INJECTION", "PRELOAD", "LIBRARY", "_LIB",
"_PATH", "_DIR", "_HOME", "_FILE", "_ROOT", "_CACHE", "_CONFIG",
"_ADDRESS", "_ENDPOINT", "_TOKEN", "_URL",
}
// UnsafeEnv reports whether a profile may not set an environment variable,
// and why.
func UnsafeEnv(key string) (bool, string) {
k := strings.ToUpper(strings.TrimSpace(key))
if k == "" {
return true, "it has no name"
}
allowed := envExact[k]
for _, f := range envFamilies {
if strings.HasPrefix(k, f) {
allowed = true
break
}
}
if !allowed {
return true, "only the variables engines are tuned with are allowed (" +
strings.Join(envFamilies, "*, ") + "*)"
}
for _, n := range envNever {
if strings.Contains(k, n) && !envExact[k] {
return true, "it names a path, a plugin, a library or an address, which is a way to make the engine " +
"read, write or load something other than its model"
}
}
// llama.cpp reads every flag from LLAMA_ARG_<FLAG> too.
if rest, ok := strings.CutPrefix(k, "LLAMA_ARG_"); ok {
flag := "--" + strings.ToLower(strings.ReplaceAll(rest, "_", "-"))
if why := unsafeFlag("llamacpp", flag, ""); why != "" {
return true, "it is " + flag + ", which " + why
}
}
return false, ""
}
// flagsNever are flags no profile may give, by engine; "" is every engine.
// Each reads or writes files the agent did not choose, loads code, or moves
// what the agent itself sets: where the engine listens and the key it
// demands.
var flagsNever = map[string]map[string]string{
"": {
"--host": "is set by the agent", "--port": "is set by the agent",
"--api-key": "is set by the agent", "--api-key-file": "is set by the agent",
"--model": "is the model's source, set by the agent", "-m": "is the model's source, set by the agent",
"--ssl-keyfile": "reads a file", "--ssl-certfile": "reads a file",
"--ssl-key-file": "reads a file", "--ssl-cert-file": "reads a file",
"--trust-remote-code": "runs Python the model repository ships",
},
"llamacpp": {
"--log-file": "writes a file", "--slot-save-path": "writes files",
"--path": "serves a directory's files over HTTP", "--rpc": "sends the model's work to other machines",
"--chat-template-file": "reads a file", "--system-prompt-file": "reads a file",
"--grammar-file": "reads a file", "--json-schema-file": "reads a file",
"--lora": "loads a file; adapters are set in the model's adapters",
"--lora-scaled": "loads a file; adapters are set in the model's adapters",
"--control-vector": "loads a file", "--control-vector-scaled": "loads a file",
"--mmproj": "loads a file; the projector is fetched with the model",
"--model-url": "downloads a model", "-mu": "downloads a model",
"--hf-repo": "downloads a model", "-hf": "downloads a model", "--hf-file": "downloads a model",
"--hf-token": "is a credential", "--alias": "is set by the agent",
"--model-draft": "loads a file", "-md": "loads a file",
"--prompt-cache": "writes a file", "--logdir": "writes files",
// The short spellings of the same.
"-mm": "loads a file; the projector is fetched with the model", "--mmproj-url": "downloads a file",
"-mmu": "downloads a file", "-hfr": "downloads a model", "-hff": "downloads a model",
"--hf-repo-draft": "downloads a model", "-hfd": "downloads a model", "-hfrd": "downloads a model",
"--hf-repo-v": "downloads a model", "-hfv": "downloads a model", "--hf-file-v": "downloads a model",
"-hffv": "downloads a model", "-hft": "is a credential", "-jf": "reads a file", "-a": "is set by the agent",
"--model-vocoder": "loads a file", "-mv": "loads a file",
},
"vllm": {
"--download-dir": "writes files", "--tokenizer": "reads another repository or path",
"--chat-template": "reads a file", "--allowed-local-media-path": "serves local files to requests",
"--lora-modules": "loads files; adapters are set in the model's adapters",
"--tool-parser-plugin": "runs a Python file", "--middleware": "imports Python",
"--worker-extension-cls": "imports Python", "--worker-cls": "imports Python",
"--load-format": "changes how weights are loaded, some ways running code",
"--otlp-traces-endpoint": "sends request traces elsewhere",
"--served-model-name": "is set by the agent",
"--enable-log-requests": "logs prompts",
},
"sglang": {
"--tokenizer-path": "reads another repository or path", "--chat-template": "reads a file",
"--file-storage-path": "writes files", "--log-requests": "logs prompts",
"--model-path": "is the model's source, set by the agent", "--served-model-name": "is set by the agent",
"--lora-paths": "loads files", "--download-dir": "writes files",
"--load-format": "changes how weights are loaded, some ways running code",
},
// ComfyUI's main.py, behind the gate that sets where it listens and
// where its files are.
"comfyui": {
"--listen": "is set by the agent", "--base-directory": "is set by the agent",
"--extra-model-paths-config": "is set by the agent",
"--output-directory": "writes files", "--temp-directory": "writes files",
"--input-directory": "reads files", "--user-directory": "writes files",
"--database-url": "writes a database elsewhere", "--front-end-root": "serves a directory",
"--enable-cors-header": "lets other sites' pages call it",
"--enable-manager": "installs nodes, which run Python",
"--tls-keyfile": "reads a file", "--tls-certfile": "reads a file",
"--comfy-api-base": "sends work to another service",
},
// Strata's setup.py, which a profile's arguments go to: llmgw
// strata-serve sets where the model is, where its files go and where
// the server listens, and the engine is the one the agent installed.
// Its own --model, the size, is spelt --size in a profile, since
// --model is the source everywhere else.
"strata": {
"--gguf-dir": "is the model's source, set by the agent", "--models-dir": "is set by the agent",
"--data-dir": "is set by the agent", "--move-data": "moves files", "--set-default-data-dir": "writes settings",
"--feed": "downloads an engine", "--prebuilt": "loads an engine", "--build": "compiles an engine",
"--update": "downloads an engine", "--rollback-engine": "changes the engine",
"--dsv41-package": "loads a file", "--inspect": "reads another repository or path",
"--source": "downloads from elsewhere", "--setup": "is not for a server",
"--browser": "opens a browser", "--gpu": "is set by the agent", "--gpus": "is set by the agent",
"--gpu-roles": "is set by the agent", "--backend": "is set by the agent",
"--experimental-speed-projection": "changes what the model refuses",
},
// trtllm-serve spells its flags with underscores, so the shared list's
// hyphenated names do not reach them.
"trtllm": {
"--trust_remote_code": "runs Python the model repository ships",
"--served_model_name": "is set by the agent",
"--config": "reads a file; the agent writes the options file itself",
"--extra_llm_api_options": "reads a file; the agent writes the options file itself",
"--tokenizer": "reads another repository or path",
"--custom_tokenizer": "imports Python", "--post_processor_hook": "imports Python",
"--custom_module_dirs": "imports Python", "--middleware": "imports Python",
"--chat_template": "reads a file",
"--metadata_server_config_file": "reads a file",
"--visual_gen_args": "reads a file",
"--report_addr": "writes a file",
"--otlp_traces_endpoint": "sends request traces elsewhere",
"--disagg_cluster_uri": "joins the server to another cluster",
"--server_role": "makes the server part of another cluster",
"--allow_request_chat_template": "lets a request supply the chat template",
"--grpc": "serves without the key the agent sets",
"--num_serve_frontends": "starts front ends that do not demand the key the agent sets",
"--telemetry": "sends usage to NVIDIA",
"--tensor_parallel_size": "is set by the agent from tp",
"--tp_size": "is set by the agent from tp",
"--pipeline_parallel_size": "is set by the agent from pp",
"--pp_size": "is set by the agent from pp",
},
}
// unsafeFlag is why an engine may not be given a flag, or "".
func unsafeFlag(engine, flag, _ string) string {
name, _, _ := strings.Cut(flag, "=")
if !strings.HasPrefix(name, "-") {
return ""
}
// Written as the engine would read it: vLLM takes --load_format for
// --load-format, TensorRT-LLM the other way round. Short flags (-m,
// -hf) are matched as written.
norm := func(f string) string {
if strings.HasPrefix(f, "--") {
return "--" + strings.ReplaceAll(f[2:], "_", "-")
}
return f
}
given := norm(name)
// argparse also takes any unambiguous beginning of a long flag for the
// flag — --trust-remote is --trust-remote-code, --li is ComfyUI's
// --listen — so for the engines that parse with it, a beginning of a
// refused flag is refused. llama.cpp and TensorRT-LLM take no such
// thing, and --chat-template there is a flag of its own.
abbreviates := argparseEngines[engine] && strings.HasPrefix(given, "--") && len(given) >= 4
for _, set := range []string{"", engine} {
for never, why := range flagsNever[set] {
if why == "" {
continue
}
n := norm(never)
if n == given || (abbreviates && strings.HasPrefix(n, "--") && strings.HasPrefix(n, given)) {
return why
}
}
}
return ""
}
// argparseEngines read their flags with Python's argparse, which takes a
// long flag's beginning for the flag.
var argparseEngines = map[string]bool{"vllm": true, "sglang": true, "comfyui": true, "strata": true}
// UnsafeArgs refuses the first of a profile's arguments an engine may not be
// given, saying why.
func UnsafeArgs(engine string, args []string) error {
for _, a := range args {
if !strings.HasPrefix(a, "-") {
continue // a value
}
if why := unsafeFlag(engine, a, ""); why != "" {
name, _, _ := strings.Cut(a, "=")
return fmt.Errorf("argument %s is not allowed: it %s", name, why)
}
}
return nil
}
// safetensorsOnly has vLLM or SGLang read only safetensors from a model it
// downloads itself, by its Hugging Face id: left to choose, either takes
// the repository's pickle files (pytorch_model.bin) when it has no
// safetensors, and loading those runs whatever they hold. A path was
// fetched by the agent, which checked it, or put there by the operator.
func safetensorsOnly(p Profile) []string {
if p.AllowPickle || !hubID(p.ModelPath) {
return nil
}
return []string{"--load-format", "safetensors"}
}
// hubID reports whether a model path is a Hugging Face id, org/name, rather
// than a file or directory.
func hubID(s string) bool {
if s == "" || strings.HasPrefix(s, "/") || strings.HasPrefix(s, ".") || strings.Contains(s, "://") {
return false
}
// org/name:Q4_K_M is a GGUF in a repository, which vLLM reads as one.
if strings.Count(s, "/") != 1 || strings.Contains(s, ":") {
return false
}
_, err := os.Stat(s)
return err != nil
}