TensorRT-LLM as an engine, and models from Ollama into the mirror

Engine trtllm runs trtllm-serve on one node, through a launcher of
llmgw's that demands the replica's key and turns on the Prometheus
endpoint its queue and cache gauges are read from; the router reads it
at /prometheus/metrics. Brought by the operator (engines.trtllm.python),
not installed; checked by ctl engine-check trtllm. Written against the
documented CLI, server and metrics and tested with stand-ins: not yet
run on a datacenter GPU.

mirror import-ollama copies a model's GGUF, and a vision model's
projector, from Ollama's directory into the mirror, checked against the
digest Ollama named each blob by; mirror import does the same for any
local file. The console's Mirror view imports from Ollama with I.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:02:01 +02:00
1 parent 30f7f3474a
commit 7940e8593c
28 files changed
+2376 -18

No files matched your search

+12 -1
View File
@@ -168,6 +168,13 @@ type CPUFallback interface {
Devices(ctx context.Context) ([]string, error)
}
// HelpCommand is an engine whose program is not asked for its help by
// running the command's path with --help, because the path is an interpreter
// starting it through a launcher.
type HelpCommand interface {
HelpCommand(c Command) (path string, args []string)
}
// GPUInfo is the subset of GPU facts validation needs.
type GPUInfo struct {
Index int
@@ -187,6 +194,7 @@ func NewRegistry(cfg Config) *Registry {
r.Add(&VLLM{cfg: cfg.VLLM})
r.Add(&SGLang{cfg: cfg.SGLang})
r.Add(&LlamaCpp{cfg: cfg.LlamaCpp})
r.Add(&TRTLLM{cfg: cfg.TRTLLM})
r.Add(&Sim{cfg: cfg.Sim})
return r
}
@@ -232,7 +240,10 @@ type Config struct {
VLLM VLLMConfig `koanf:"vllm"`
SGLang SGLangConfig `koanf:"sglang"`
LlamaCpp LlamaCppConfig `koanf:"llamacpp"`
Sim SimConfig `koanf:"sim"`
// TRTLLM is TensorRT-LLM, which the agent does not install: a node
// offers it only when its interpreter is named here.
TRTLLM TRTLLMConfig `koanf:"trtllm"`
Sim SimConfig `koanf:"sim"`
}
// --- shared helpers ---
+2 -2
View File
@@ -457,14 +457,14 @@ func TestValidateRequiresModelPath(t *testing.T) {
func TestRegistry(t *testing.T) {
r := NewRegistry(Config{})
if got := r.Names(); !slices.Equal(got, []string{"llamacpp", "sglang", "sim", "vllm"}) {
if got := r.Names(); !slices.Equal(got, []string{"llamacpp", "sglang", "sim", "trtllm", "vllm"}) {
t.Errorf("registry contents: %v", got)
}
if _, err := r.Get("vllm"); err != nil {
t.Errorf("vllm missing: %v", err)
}
err := func() error { _, err := r.Get("nope"); return err }()
if err == nil || !strings.Contains(err.Error(), "llamacpp, sglang, sim, vllm") {
if err == nil || !strings.Contains(err.Error(), "llamacpp, sglang, sim, trtllm, vllm") {
t.Errorf("an unknown engine should list what is available: %v", err)
}
}
+218
View File
@@ -0,0 +1,218 @@
package engines
import (
"context"
"fmt"
"strconv"
)
// TRTLLMConfig locates a TensorRT-LLM the operator installed.
//
// llmgw does not install TensorRT-LLM. It is a pip package pinned to one
// CUDA, one PyTorch and an MPI the machine must provide, and NVIDIA's own
// advice is its container; an installer that guessed at all of that would
// fail on most machines it met, a long way into a large download. So it is
// brought by the operator — a virtual environment, or NVIDIA's container —
// and named here.
type TRTLLMConfig struct {
// Python is the interpreter of the environment TensorRT-LLM is installed
// in: <venv>/bin/python for a virtual environment, python3 in NVIDIA's
// container. Naming it is what makes this node offer the engine; empty
// offers nothing.
//
// The interpreter rather than trtllm-serve, because llmgw starts the
// server through a few lines of its own that make it demand the
// gateway's key, which trtllm-serve has no option for.
Python string `koanf:"python"`
}
// TRTLLM builds TensorRT-LLM launch commands: its OpenAI-compatible server,
// trtllm-serve, on one node.
//
// Written against TensorRT-LLM's documented command line and server, and not
// yet run on hardware by anyone working on llmgw: no datacenter GPU was at
// hand. `llmgw ctl engine-check trtllm` asks an installed one whether it
// takes the flags built here.
type TRTLLM struct{ cfg TRTLLMConfig }
// Name identifies the engine in the catalog.
func (*TRTLLM) Name() string { return "trtllm" }
// HealthPath is trtllm-serve's liveness endpoint: 200 once the executor is
// up, 503 while it is not. The supervisor's /v1/models step is what says the
// model is being served.
func (*TRTLLM) HealthPath() string { return "/health" }
// MetricsPath is trtllm-serve's Prometheus endpoint. Its /metrics is a
// different thing — a JSON list of iteration statistics, drained by each
// read — and a scrape there would both fail to parse and steal the numbers
// from anyone else reading them.
func (*TRTLLM) MetricsPath() string { return "/prometheus/metrics" }
// ServesHTTP reports whether this rank answers the API; a TensorRT-LLM
// replica is one node, and so one rank.
func (*TRTLLM) ServesHTTP(s ShardSpec) bool { return s.Leader() }
// Validate checks the profile against the GPUs it would run on.
//
// One node: trtllm-serve spans machines only when started under MPI or
// Slurm from one of them (trtllm-llmapi-launch), and the agents each start
// their own node's part, which is not that. Ampere or newer: the oldest
// generation current TensorRT-LLM builds kernels for.
func (*TRTLLM) Validate(p Profile, gpus []GPUInfo) error {
if p.Nodes > 1 {
return fmt.Errorf("profile %q: trtllm runs a replica on one node; "+
"use vllm or sglang for a %d-node replica", p.Name, p.Nodes)
}
if err := validateCommon(p, gpus); err != nil {
return err
}
return requireComputeCapability(p, gpus, 8.0, "TensorRT-LLM")
}
// Commands returns the processes for one shard, which is one process. Tensor
// and pipeline parallel ranks on the node are trtllm-serve's own MPI workers,
// started by it.
func (t *TRTLLM) Commands(s ShardSpec) ([]Command, error) {
if s.NNodes > 1 {
return nil, fmt.Errorf("trtllm: a replica spanning %d nodes is not something llmgw starts; "+
"TensorRT-LLM runs one node per replica here", s.NNodes)
}
p := s.Profile
args := []string{"-c", trtllmLauncher,
// The subcommand by name: trtllm-serve takes a bare model as "serve",
// but saying it costs nothing and does not depend on that default.
"serve", p.ModelPath,
"--host", s.Host(),
"--port", strconv.Itoa(s.HTTPPort),
"--tp_size", strconv.Itoa(p.TP),
}
if p.PP > 1 {
args = append(args, "--pp_size", strconv.Itoa(p.PP))
}
if p.ServedModelName != "" {
// Without it the model is named after its directory, and the agent's
// readiness check, which looks for this name in /v1/models, never
// finds it.
args = append(args, "--served_model_name", p.ServedModelName)
}
// What makes the queue and the cache measurable: the options file the
// launcher writes turns on the Prometheus endpoint and the iteration
// statistics its gauges are set from. A placeholder here, so the flag is
// one engine-check sees and checks; the launcher replaces it.
args = append(args, "--extra_llm_api_options", trtllmOptionsPlaceholder)
args = append(args, p.Args...)
env := baseEnv(s)
if s.APIKey != "" {
env[trtllmKeyEnv] = s.APIKey
}
// TensorRT-LLM reports usage to NVIDIA unless told not to. A gateway on
// somebody's own network does not phone home on their behalf, and a
// model's profile cannot make it: these are set after the profile's env.
env["TRTLLM_NO_USAGE_STATS"] = "1"
env["DO_NOT_TRACK"] = "1"
return []Command{{Name: "trtllm-serve", Path: t.python(), Args: args, Env: materializeEnv(env)}}, nil
}
// python is the interpreter to start trtllm-serve with.
func (t *TRTLLM) python() string {
if t.cfg.Python != "" {
return t.cfg.Python
}
return "python3"
}
// Configured reports whether the configuration names TensorRT-LLM's
// interpreter, which is the only way a node offers it.
func (t *TRTLLM) Configured() bool { return t.cfg.Python != "" }
// Probe reports the installed TensorRT-LLM's version.
//
// From the package's metadata rather than by importing it: the import loads
// CUDA, MPI and PyTorch and takes longer than a probe may, while what
// placement needs is whether the package is there. Whether it runs is
// engine-check's question, and the replica's.
func (t *TRTLLM) Probe(ctx context.Context) (string, error) {
return versionOfModule(ctx, t.python(), "-c",
"import importlib.metadata as m, importlib.util as u, sys\n"+
"if u.find_spec('tensorrt_llm') is None: sys.exit(1)\n"+
"sys.stdout.write(m.version('tensorrt_llm'))")
}
// HelpCommand is how to ask trtllm-serve for its help: through its module,
// since the command itself is llmgw's launcher.
func (t *TRTLLM) HelpCommand(Command) (string, []string) {
return t.python(), []string{"-m", "tensorrt_llm.commands.serve", "serve", "--help"}
}
const (
trtllmKeyEnv = "LLMGW_ENGINE_KEY"
trtllmOptionsPlaceholder = "@llmgw-options"
)
// trtllmLauncher starts trtllm-serve inside the process, after two things
// trtllm-serve has no flag for.
//
// The key. trtllm-serve checks none, and an engine listens on the node's
// management address, where without one anybody who can reach the port talks
// to the model past the gateway's authentication, quotas and accounting. So
// every FastAPI application the process makes — trtllm-serve's server is one
// — is given a middleware demanding it, before trtllm-serve imports anything.
// /health is left open: it says only up or down, and a supervisor of
// TensorRT-LLM's own may ask it without a key. The key comes in the
// environment and is taken out of it, so the MPI workers and anything else
// started from here never hold it.
//
// The options. The Prometheus endpoint exists only with return_perf_metrics,
// and its queue gauges move only with enable_iter_perf_stats; both are
// settings of TensorRT-LLM's options file, not flags. The file is written to
// an unnamed temporary file and passed by descriptor: nothing is left on disk
// however the server ends.
//
// trtllm-serve can start further HTTP front ends as copies of its own command
// line (--num_serve_frontends); those would not run this, so the flag is
// refused in a profile, and a copy started anyway would be given a program
// path that cannot exist — /dev/null is no directory — and fail rather than
// serve without the key. Click names itself by the path's last element, so
// trtllm-serve's messages still say trtllm-serve.
const trtllmLauncher = `import hmac, os, sys, tempfile
key = os.environ.pop("` + trtllmKeyEnv + `", "")
if key:
import fastapi
want = ("Bearer " + key).encode()
denied = b'{"error":{"message":"a valid API key is required","type":"invalid_request_error","code":"invalid_api_key"}}'
class RequireKey:
def __init__(self, app):
self.app = app
async def __call__(self, scope, receive, send):
if scope["type"] in ("http", "websocket") and scope.get("path") != "/health":
got = b""
for name, value in scope.get("headers") or ():
if name == b"authorization":
got = value
if not hmac.compare_digest(got, want):
if scope["type"] == "websocket":
await send({"type": "websocket.close", "code": 1008})
return
await send({"type": "http.response.start", "status": 401,
"headers": [(b"content-type", b"application/json")]})
await send({"type": "http.response.body", "body": denied})
return
await self.app(scope, receive, send)
plain_init = fastapi.FastAPI.__init__
def init_with_key(self, *a, **kw):
plain_init(self, *a, **kw)
self.add_middleware(RequireKey)
fastapi.FastAPI.__init__ = init_with_key
args = sys.argv[1:]
if "` + trtllmOptionsPlaceholder + `" in args:
options = tempfile.TemporaryFile()
options.write(b"return_perf_metrics: true\nenable_iter_perf_stats: true\n")
options.flush()
options.seek(0)
args[args.index("` + trtllmOptionsPlaceholder + `")] = "/dev/fd/%d" % options.fileno()
sys.argv = ["/dev/null/trtllm-serve"] + args
from tensorrt_llm.commands.serve import main
main()
`
+316
View File
@@ -0,0 +1,316 @@
package engines
import (
"encoding/json"
"os"
"os/exec"
"path/filepath"
"slices"
"strings"
"testing"
)
func trtllmSpec() ShardSpec {
s := singleNodeSpec()
s.Profile.Engine = "trtllm"
s.Profile.ServedModelName = "llama-3.1-70b"
return s
}
func TestTRTLLMRunsTrtllmServeWithTheFlagsItDocuments(t *testing.T) {
s := trtllmSpec()
s.BindAddr = "10.0.0.147"
s.Profile.Args = []string{"--max_batch_size", "64"}
cmd := buildOne(t, &TRTLLM{cfg: TRTLLMConfig{Python: "/opt/trtllm/bin/python"}}, s)
if cmd.Path != "/opt/trtllm/bin/python" {
t.Errorf("Path=%q, expected the configured interpreter", cmd.Path)
}
if cmd.Args[0] != "-c" || !strings.Contains(cmd.Args[1], "from tensorrt_llm.commands.serve import main") {
t.Fatalf("not started through the launcher: %v", cmd.Args[:2])
}
rest := cmd.Args[2:]
if rest[0] != "serve" || rest[1] != "/models/llama" {
t.Errorf("expected serve and the model first, got %v", rest[:2])
}
for flag, want := range map[string]string{
"--host": "10.0.0.147", "--port": "20000", "--tp_size": "4",
"--served_model_name": "llama-3.1-70b",
"--extra_llm_api_options": trtllmOptionsPlaceholder,
} {
if got := argValue(rest, flag); got != want {
t.Errorf("%s=%q, want %q (args %v)", flag, got, want, rest)
}
}
// One pipeline stage is the default, and saying so would only be one
// more flag for a version to rename.
if slices.Contains(rest, "--pp_size") {
t.Errorf("--pp_size given for a single stage: %v", rest)
}
// The profile's own, last, so a tuning flag is the operator's to set.
if got := rest[len(rest)-2:]; !slices.Equal(got, []string{"--max_batch_size", "64"}) {
t.Errorf("profile args are not last: %v", rest)
}
s.Profile.TP, s.Profile.PP = 2, 2
if got := argValue(buildOne(t, &TRTLLM{}, s).Args, "--pp_size"); got != "2" {
t.Errorf("--pp_size=%q for two stages", got)
}
if got := buildOne(t, &TRTLLM{}, s).Path; got != "python3" {
t.Errorf("unconfigured, the interpreter is %q", got)
}
}
func TestTRTLLMsKeyIsNotOnItsCommandLine(t *testing.T) {
s := trtllmSpec()
s.APIKey = "k1"
cmd := buildOne(t, &TRTLLM{}, s)
for _, a := range cmd.Args {
if strings.Contains(a, "k1") && a != cmd.Args[1] {
t.Errorf("the key is in the arguments: %v", cmd.Args)
}
}
if envMap(cmd.Env)[trtllmKeyEnv] != "k1" {
t.Error("the key is not in the environment")
}
}
func TestTRTLLMDoesNotReportUsageToNVIDIA(t *testing.T) {
s := trtllmSpec()
s.Profile.Env = map[string]string{"TRTLLM_NO_USAGE_STATS": "0", "TRTLLM_ENABLE_PDL": "1"}
env := envMap(buildOne(t, &TRTLLM{}, s).Env)
if env["TRTLLM_NO_USAGE_STATS"] != "1" || env["DO_NOT_TRACK"] != "1" {
t.Errorf("usage reporting is not off: %q %q", env["TRTLLM_NO_USAGE_STATS"], env["DO_NOT_TRACK"])
}
// Tuning still reaches it.
if env["TRTLLM_ENABLE_PDL"] != "1" {
t.Error("a profile's TensorRT-LLM tuning was dropped")
}
}
func TestTRTLLMIsReadFromItsPrometheusEndpoint(t *testing.T) {
// /metrics on trtllm-serve is JSON, and each read drains what it returns.
if got := (&TRTLLM{}).MetricsPath(); got != "/prometheus/metrics" {
t.Errorf("MetricsPath=%q", got)
}
if !(&TRTLLM{}).ServesHTTP(trtllmSpec()) {
t.Error("the one rank of a replica serves its API")
}
}
func TestTRTLLMRunsAReplicaOnOneNode(t *testing.T) {
p := trtllmSpec().Profile
p.Nodes, p.GPUsPerNode, p.TP = 2, 4, 8
err := (&TRTLLM{}).Validate(p, nil)
if err == nil || !strings.Contains(err.Error(), "one node") {
t.Errorf("a two-node profile was accepted: %v", err)
}
s := multiNodeSpec(0)
s.Profile.Engine = "trtllm"
if _, err := (&TRTLLM{}).Commands(s); err == nil {
t.Error("a command was built for a shard of a two-node replica")
}
}
func TestTRTLLMNeedsAmpereOrNewer(t *testing.T) {
p := trtllmSpec().Profile
p.TP = 2
v100 := []GPUInfo{{Index: 0, Name: "Tesla V100", ComputeCapability: "7.0"}, {Index: 1, Name: "Tesla V100", ComputeCapability: "7.0"}}
if err := (&TRTLLM{}).Validate(p, v100); err == nil {
t.Error("a V100 was accepted")
}
h100 := []GPUInfo{{Index: 0, Name: "H100", ComputeCapability: "9.0"}, {Index: 1, Name: "H100", ComputeCapability: "9.0"}}
if err := (&TRTLLM{}).Validate(p, h100); err != nil {
t.Errorf("two H100s were refused: %v", err)
}
}
func TestTRTLLMProfilesMayNotOpenWhatTheAgentCloses(t *testing.T) {
for _, args := range [][]string{
{"--trust_remote_code"},
{"--num_serve_frontends", "4"},
{"--grpc"},
{"--config", "/etc/shadow"},
{"--extra_llm_api_options=/tmp/x.yaml"},
{"--served_model_name", "other"},
{"--middleware", "evil.mw"},
{"--tp_size", "8"},
} {
if err := UnsafeArgs("trtllm", args); err == nil {
t.Errorf("%v was allowed", args)
}
}
if err := UnsafeArgs("trtllm", []string{"--max_batch_size", "64", "--set", "kv_cache_config.enable_block_reuse=false",
"--free_gpu_memory_fraction", "0.85"}); err != nil {
t.Errorf("tuning was refused: %v", err)
}
}
func TestTRTLLMIsOfferedOnlyWhenItsPackageIsThere(t *testing.T) {
there := script(t, "python3", `echo "1.3.0rc29"`)
r := NewRegistry(Config{})
r.engines = map[string]Engine{"trtllm": &TRTLLM{cfg: TRTLLMConfig{Python: there}}}
if got := r.ProbeConfigured(t.Context()); got["trtllm"] != "1.3.0rc29" {
t.Errorf("expected the version, got %v", got)
}
// An interpreter without the package, which every machine has.
absent := script(t, "python3", "exit 1")
r.engines = map[string]Engine{"trtllm": &TRTLLM{cfg: TRTLLMConfig{Python: absent}}}
if got := r.ProbeConfigured(t.Context()); len(got) != 0 {
t.Errorf("reported without its package: %v", got)
}
// Not named, not offered: python3 on PATH is nobody's TensorRT-LLM.
r.engines = map[string]Engine{"trtllm": &TRTLLM{}}
if got := r.ProbeConfigured(t.Context()); len(got) != 0 {
t.Errorf("offered without being configured: %v", got)
}
}
// fakeTRTLLM is a Python path holding stand-ins for fastapi and
// tensorrt_llm: enough of each for the launcher to do what it does in front
// of the real server, and for the stand-in server to report what it saw.
func fakeTRTLLM(t *testing.T) string {
t.Helper()
dir := t.TempDir()
files := map[string]string{
"fastapi/__init__.py": `
class FastAPI:
def __init__(self, *a, **kw):
self.user_middleware = []
def add_middleware(self, cls, **kw):
self.user_middleware.append(cls)
`,
"tensorrt_llm/__init__.py": "",
"tensorrt_llm/commands/__init__.py": "",
"tensorrt_llm/commands/serve.py": `
import asyncio, json, os, sys
import fastapi
def main():
args = sys.argv[1:]
with open(args[args.index("--extra_llm_api_options") + 1]) as f:
options = f.read()
app = fastapi.FastAPI()
async def server(scope, receive, send):
await send({"type": "http.response.start", "status": 200, "headers": []})
await send({"type": "http.response.body", "body": b"ok"})
handler = server
for mw in reversed(app.user_middleware):
handler = mw(handler)
async def ask(path, auth=None):
headers = [(b"authorization", auth.encode())] if auth else []
sent = []
async def send(msg):
sent.append(msg)
async def receive():
return {"type": "http.request"}
await handler({"type": "http", "path": path, "headers": headers}, receive, send)
return sent[0]["status"]
async def run():
return {
"health": await ask("/health"),
"anonymous": await ask("/v1/chat/completions"),
"wrong": await ask("/v1/models", "Bearer nope"),
"keyed": await ask("/prometheus/metrics", "Bearer k1"),
}
out = asyncio.run(run())
out.update(options=options, argv0=sys.argv[0], args=args,
key_left=os.environ.get("LLMGW_ENGINE_KEY"))
print(json.dumps(out))
`,
}
for name, body := range files {
p := filepath.Join(dir, name)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
return dir
}
type launched struct {
Health, Anonymous, Wrong, Keyed int
Options, Argv0 string
Args []string
KeyLeft *string `json:"key_left"`
}
func launch(t *testing.T, s ShardSpec) launched {
t.Helper()
python, err := exec.LookPath("python3")
if err != nil {
t.Skip("no python3 here to run the launcher with")
}
cmd := buildOne(t, &TRTLLM{cfg: TRTLLMConfig{Python: python}}, s)
c := exec.Command(cmd.Path, cmd.Args...) //nolint:gosec // the command under test
c.Env = append(cmd.Env, "PYTHONPATH="+fakeTRTLLM(t))
out, err := c.Output()
if err != nil {
var stderr string
if ee, ok := err.(*exec.ExitError); ok {
stderr = string(ee.Stderr)
}
t.Fatalf("the launcher failed: %v\n%s", err, stderr)
}
var got launched
if err := json.Unmarshal(out, &got); err != nil {
t.Fatalf("%v: %s", err, out)
}
return got
}
func TestTheTRTLLMLauncherDemandsTheKey(t *testing.T) {
s := trtllmSpec()
s.APIKey = "k1"
got := launch(t, s)
if got.Anonymous != 401 || got.Wrong != 401 {
t.Errorf("served without the key: anonymous %d, wrong key %d", got.Anonymous, got.Wrong)
}
if got.Keyed != 200 {
t.Errorf("refused the key: %d", got.Keyed)
}
// Up or down is all /health says, and it is asked without one.
if got.Health != 200 {
t.Errorf("/health answered %d", got.Health)
}
// Taken out of the environment, so nothing the server starts holds it.
if got.KeyLeft != nil {
t.Errorf("the key was left in the environment: %q", *got.KeyLeft)
}
// A copy of this command line, as trtllm-serve's extra front ends are,
// names no program that exists and so cannot serve without the key.
if _, err := os.Stat(got.Argv0); err == nil || filepath.Base(got.Argv0) != "trtllm-serve" {
t.Errorf("argv[0] is %q", got.Argv0)
}
}
func TestTheTRTLLMLauncherTurnsOnTheMetricsTheRouterReads(t *testing.T) {
got := launch(t, trtllmSpec())
for _, want := range []string{"return_perf_metrics: true", "enable_iter_perf_stats: true"} {
if !strings.Contains(got.Options, want) {
t.Errorf("the options file lacks %q: %q", want, got.Options)
}
}
path := argValue(got.Args, "--extra_llm_api_options")
if !strings.HasPrefix(path, "/dev/fd/") {
t.Errorf("the options were not passed by descriptor: %q", path)
}
if slices.Contains(got.Args, trtllmOptionsPlaceholder) {
t.Error("the placeholder reached trtllm-serve")
}
// Without a key there is nothing to demand.
if got.Anonymous != 200 {
t.Errorf("with no key set, a request was refused: %d", got.Anonymous)
}
}
+27
View File
@@ -24,6 +24,7 @@ var envFamilies = []string{
"CUDA_", "NCCL_", "NVIDIA_", "NVTE_", "GLOO_", "UCX_", "TORCH_", "PYTORCH_",
"VLLM_", "SGLANG_", "GGML_", "LLAMA_", "OMP_", "MKL_", "OPENBLAS_", "KMP_",
"HF_", "TRANSFORMERS_", "TOKENIZERS_", "TRITON_", "FLASHINFER_",
"TRTLLM_", "TLLM_",
}
// envExact are single variables that are tuning too.
@@ -118,6 +119,32 @@ var flagsNever = map[string]map[string]string{
"--model-path": "is the model's source, set by the agent", "--served-model-name": "is set by the agent",
"--lora-paths": "loads files", "--download-dir": "writes files",
},
// trtllm-serve spells its flags with underscores, so the shared list's
// hyphenated names do not reach them.
"trtllm": {
"--trust_remote_code": "runs Python the model repository ships",
"--served_model_name": "is set by the agent",
"--config": "reads a file; the agent writes the options file itself",
"--extra_llm_api_options": "reads a file; the agent writes the options file itself",
"--tokenizer": "reads another repository or path",
"--custom_tokenizer": "imports Python", "--post_processor_hook": "imports Python",
"--custom_module_dirs": "imports Python", "--middleware": "imports Python",
"--chat_template": "reads a file",
"--metadata_server_config_file": "reads a file",
"--visual_gen_args": "reads a file",
"--report_addr": "writes a file",
"--otlp_traces_endpoint": "sends request traces elsewhere",
"--disagg_cluster_uri": "joins the server to another cluster",
"--server_role": "makes the server part of another cluster",
"--allow_request_chat_template": "lets a request supply the chat template",
"--grpc": "serves without the key the agent sets",
"--num_serve_frontends": "starts front ends that do not demand the key the agent sets",
"--telemetry": "sends usage to NVIDIA",
"--tensor_parallel_size": "is set by the agent from tp",
"--tp_size": "is set by the agent from tp",
"--pipeline_parallel_size": "is set by the agent from pp",
"--pp_size": "is set by the agent from pp",
},
}
// unsafeFlag is why an engine may not be given a flag, or "".