TensorRT-LLM as an engine, and models from Ollama into the mirror
Engine trtllm runs trtllm-serve on one node, through a launcher of llmgw's that demands the replica's key and turns on the Prometheus endpoint its queue and cache gauges are read from; the router reads it at /prometheus/metrics. Brought by the operator (engines.trtllm.python), not installed; checked by ctl engine-check trtllm. Written against the documented CLI, server and metrics and tested with stand-ins: not yet run on a datacenter GPU. mirror import-ollama copies a model's GGUF, and a vision model's projector, from Ollama's directory into the mirror, checked against the digest Ollama named each blob by; mirror import does the same for any local file. The console's Mirror view imports from Ollama with I. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
30f7f3474a
commit
7940e8593c
28 files changed
+2376
-18
No files matched your search
@@ -168,6 +168,13 @@ type CPUFallback interface {
|
||||
Devices(ctx context.Context) ([]string, error)
|
||||
}
|
||||
|
||||
// HelpCommand is an engine whose program is not asked for its help by
|
||||
// running the command's path with --help, because the path is an interpreter
|
||||
// starting it through a launcher.
|
||||
type HelpCommand interface {
|
||||
HelpCommand(c Command) (path string, args []string)
|
||||
}
|
||||
|
||||
// GPUInfo is the subset of GPU facts validation needs.
|
||||
type GPUInfo struct {
|
||||
Index int
|
||||
@@ -187,6 +194,7 @@ func NewRegistry(cfg Config) *Registry {
|
||||
r.Add(&VLLM{cfg: cfg.VLLM})
|
||||
r.Add(&SGLang{cfg: cfg.SGLang})
|
||||
r.Add(&LlamaCpp{cfg: cfg.LlamaCpp})
|
||||
r.Add(&TRTLLM{cfg: cfg.TRTLLM})
|
||||
r.Add(&Sim{cfg: cfg.Sim})
|
||||
return r
|
||||
}
|
||||
@@ -232,7 +240,10 @@ type Config struct {
|
||||
VLLM VLLMConfig `koanf:"vllm"`
|
||||
SGLang SGLangConfig `koanf:"sglang"`
|
||||
LlamaCpp LlamaCppConfig `koanf:"llamacpp"`
|
||||
Sim SimConfig `koanf:"sim"`
|
||||
// TRTLLM is TensorRT-LLM, which the agent does not install: a node
|
||||
// offers it only when its interpreter is named here.
|
||||
TRTLLM TRTLLMConfig `koanf:"trtllm"`
|
||||
Sim SimConfig `koanf:"sim"`
|
||||
}
|
||||
|
||||
// --- shared helpers ---
|
||||
|
||||
@@ -457,14 +457,14 @@ func TestValidateRequiresModelPath(t *testing.T) {
|
||||
|
||||
func TestRegistry(t *testing.T) {
|
||||
r := NewRegistry(Config{})
|
||||
if got := r.Names(); !slices.Equal(got, []string{"llamacpp", "sglang", "sim", "vllm"}) {
|
||||
if got := r.Names(); !slices.Equal(got, []string{"llamacpp", "sglang", "sim", "trtllm", "vllm"}) {
|
||||
t.Errorf("registry contents: %v", got)
|
||||
}
|
||||
if _, err := r.Get("vllm"); err != nil {
|
||||
t.Errorf("vllm missing: %v", err)
|
||||
}
|
||||
err := func() error { _, err := r.Get("nope"); return err }()
|
||||
if err == nil || !strings.Contains(err.Error(), "llamacpp, sglang, sim, vllm") {
|
||||
if err == nil || !strings.Contains(err.Error(), "llamacpp, sglang, sim, trtllm, vllm") {
|
||||
t.Errorf("an unknown engine should list what is available: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
package engines
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// TRTLLMConfig locates a TensorRT-LLM the operator installed.
|
||||
//
|
||||
// llmgw does not install TensorRT-LLM. It is a pip package pinned to one
|
||||
// CUDA, one PyTorch and an MPI the machine must provide, and NVIDIA's own
|
||||
// advice is its container; an installer that guessed at all of that would
|
||||
// fail on most machines it met, a long way into a large download. So it is
|
||||
// brought by the operator — a virtual environment, or NVIDIA's container —
|
||||
// and named here.
|
||||
type TRTLLMConfig struct {
|
||||
// Python is the interpreter of the environment TensorRT-LLM is installed
|
||||
// in: <venv>/bin/python for a virtual environment, python3 in NVIDIA's
|
||||
// container. Naming it is what makes this node offer the engine; empty
|
||||
// offers nothing.
|
||||
//
|
||||
// The interpreter rather than trtllm-serve, because llmgw starts the
|
||||
// server through a few lines of its own that make it demand the
|
||||
// gateway's key, which trtllm-serve has no option for.
|
||||
Python string `koanf:"python"`
|
||||
}
|
||||
|
||||
// TRTLLM builds TensorRT-LLM launch commands: its OpenAI-compatible server,
|
||||
// trtllm-serve, on one node.
|
||||
//
|
||||
// Written against TensorRT-LLM's documented command line and server, and not
|
||||
// yet run on hardware by anyone working on llmgw: no datacenter GPU was at
|
||||
// hand. `llmgw ctl engine-check trtllm` asks an installed one whether it
|
||||
// takes the flags built here.
|
||||
type TRTLLM struct{ cfg TRTLLMConfig }
|
||||
|
||||
// Name identifies the engine in the catalog.
|
||||
func (*TRTLLM) Name() string { return "trtllm" }
|
||||
|
||||
// HealthPath is trtllm-serve's liveness endpoint: 200 once the executor is
|
||||
// up, 503 while it is not. The supervisor's /v1/models step is what says the
|
||||
// model is being served.
|
||||
func (*TRTLLM) HealthPath() string { return "/health" }
|
||||
|
||||
// MetricsPath is trtllm-serve's Prometheus endpoint. Its /metrics is a
|
||||
// different thing — a JSON list of iteration statistics, drained by each
|
||||
// read — and a scrape there would both fail to parse and steal the numbers
|
||||
// from anyone else reading them.
|
||||
func (*TRTLLM) MetricsPath() string { return "/prometheus/metrics" }
|
||||
|
||||
// ServesHTTP reports whether this rank answers the API; a TensorRT-LLM
|
||||
// replica is one node, and so one rank.
|
||||
func (*TRTLLM) ServesHTTP(s ShardSpec) bool { return s.Leader() }
|
||||
|
||||
// Validate checks the profile against the GPUs it would run on.
|
||||
//
|
||||
// One node: trtllm-serve spans machines only when started under MPI or
|
||||
// Slurm from one of them (trtllm-llmapi-launch), and the agents each start
|
||||
// their own node's part, which is not that. Ampere or newer: the oldest
|
||||
// generation current TensorRT-LLM builds kernels for.
|
||||
func (*TRTLLM) Validate(p Profile, gpus []GPUInfo) error {
|
||||
if p.Nodes > 1 {
|
||||
return fmt.Errorf("profile %q: trtllm runs a replica on one node; "+
|
||||
"use vllm or sglang for a %d-node replica", p.Name, p.Nodes)
|
||||
}
|
||||
if err := validateCommon(p, gpus); err != nil {
|
||||
return err
|
||||
}
|
||||
return requireComputeCapability(p, gpus, 8.0, "TensorRT-LLM")
|
||||
}
|
||||
|
||||
// Commands returns the processes for one shard, which is one process. Tensor
|
||||
// and pipeline parallel ranks on the node are trtllm-serve's own MPI workers,
|
||||
// started by it.
|
||||
func (t *TRTLLM) Commands(s ShardSpec) ([]Command, error) {
|
||||
if s.NNodes > 1 {
|
||||
return nil, fmt.Errorf("trtllm: a replica spanning %d nodes is not something llmgw starts; "+
|
||||
"TensorRT-LLM runs one node per replica here", s.NNodes)
|
||||
}
|
||||
p := s.Profile
|
||||
args := []string{"-c", trtllmLauncher,
|
||||
// The subcommand by name: trtllm-serve takes a bare model as "serve",
|
||||
// but saying it costs nothing and does not depend on that default.
|
||||
"serve", p.ModelPath,
|
||||
"--host", s.Host(),
|
||||
"--port", strconv.Itoa(s.HTTPPort),
|
||||
"--tp_size", strconv.Itoa(p.TP),
|
||||
}
|
||||
if p.PP > 1 {
|
||||
args = append(args, "--pp_size", strconv.Itoa(p.PP))
|
||||
}
|
||||
if p.ServedModelName != "" {
|
||||
// Without it the model is named after its directory, and the agent's
|
||||
// readiness check, which looks for this name in /v1/models, never
|
||||
// finds it.
|
||||
args = append(args, "--served_model_name", p.ServedModelName)
|
||||
}
|
||||
// What makes the queue and the cache measurable: the options file the
|
||||
// launcher writes turns on the Prometheus endpoint and the iteration
|
||||
// statistics its gauges are set from. A placeholder here, so the flag is
|
||||
// one engine-check sees and checks; the launcher replaces it.
|
||||
args = append(args, "--extra_llm_api_options", trtllmOptionsPlaceholder)
|
||||
args = append(args, p.Args...)
|
||||
|
||||
env := baseEnv(s)
|
||||
if s.APIKey != "" {
|
||||
env[trtllmKeyEnv] = s.APIKey
|
||||
}
|
||||
// TensorRT-LLM reports usage to NVIDIA unless told not to. A gateway on
|
||||
// somebody's own network does not phone home on their behalf, and a
|
||||
// model's profile cannot make it: these are set after the profile's env.
|
||||
env["TRTLLM_NO_USAGE_STATS"] = "1"
|
||||
env["DO_NOT_TRACK"] = "1"
|
||||
return []Command{{Name: "trtllm-serve", Path: t.python(), Args: args, Env: materializeEnv(env)}}, nil
|
||||
}
|
||||
|
||||
// python is the interpreter to start trtllm-serve with.
|
||||
func (t *TRTLLM) python() string {
|
||||
if t.cfg.Python != "" {
|
||||
return t.cfg.Python
|
||||
}
|
||||
return "python3"
|
||||
}
|
||||
|
||||
// Configured reports whether the configuration names TensorRT-LLM's
|
||||
// interpreter, which is the only way a node offers it.
|
||||
func (t *TRTLLM) Configured() bool { return t.cfg.Python != "" }
|
||||
|
||||
// Probe reports the installed TensorRT-LLM's version.
|
||||
//
|
||||
// From the package's metadata rather than by importing it: the import loads
|
||||
// CUDA, MPI and PyTorch and takes longer than a probe may, while what
|
||||
// placement needs is whether the package is there. Whether it runs is
|
||||
// engine-check's question, and the replica's.
|
||||
func (t *TRTLLM) Probe(ctx context.Context) (string, error) {
|
||||
return versionOfModule(ctx, t.python(), "-c",
|
||||
"import importlib.metadata as m, importlib.util as u, sys\n"+
|
||||
"if u.find_spec('tensorrt_llm') is None: sys.exit(1)\n"+
|
||||
"sys.stdout.write(m.version('tensorrt_llm'))")
|
||||
}
|
||||
|
||||
// HelpCommand is how to ask trtllm-serve for its help: through its module,
|
||||
// since the command itself is llmgw's launcher.
|
||||
func (t *TRTLLM) HelpCommand(Command) (string, []string) {
|
||||
return t.python(), []string{"-m", "tensorrt_llm.commands.serve", "serve", "--help"}
|
||||
}
|
||||
|
||||
const (
|
||||
trtllmKeyEnv = "LLMGW_ENGINE_KEY"
|
||||
trtllmOptionsPlaceholder = "@llmgw-options"
|
||||
)
|
||||
|
||||
// trtllmLauncher starts trtllm-serve inside the process, after two things
|
||||
// trtllm-serve has no flag for.
|
||||
//
|
||||
// The key. trtllm-serve checks none, and an engine listens on the node's
|
||||
// management address, where without one anybody who can reach the port talks
|
||||
// to the model past the gateway's authentication, quotas and accounting. So
|
||||
// every FastAPI application the process makes — trtllm-serve's server is one
|
||||
// — is given a middleware demanding it, before trtllm-serve imports anything.
|
||||
// /health is left open: it says only up or down, and a supervisor of
|
||||
// TensorRT-LLM's own may ask it without a key. The key comes in the
|
||||
// environment and is taken out of it, so the MPI workers and anything else
|
||||
// started from here never hold it.
|
||||
//
|
||||
// The options. The Prometheus endpoint exists only with return_perf_metrics,
|
||||
// and its queue gauges move only with enable_iter_perf_stats; both are
|
||||
// settings of TensorRT-LLM's options file, not flags. The file is written to
|
||||
// an unnamed temporary file and passed by descriptor: nothing is left on disk
|
||||
// however the server ends.
|
||||
//
|
||||
// trtllm-serve can start further HTTP front ends as copies of its own command
|
||||
// line (--num_serve_frontends); those would not run this, so the flag is
|
||||
// refused in a profile, and a copy started anyway would be given a program
|
||||
// path that cannot exist — /dev/null is no directory — and fail rather than
|
||||
// serve without the key. Click names itself by the path's last element, so
|
||||
// trtllm-serve's messages still say trtllm-serve.
|
||||
const trtllmLauncher = `import hmac, os, sys, tempfile
|
||||
key = os.environ.pop("` + trtllmKeyEnv + `", "")
|
||||
if key:
|
||||
import fastapi
|
||||
want = ("Bearer " + key).encode()
|
||||
denied = b'{"error":{"message":"a valid API key is required","type":"invalid_request_error","code":"invalid_api_key"}}'
|
||||
class RequireKey:
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope["type"] in ("http", "websocket") and scope.get("path") != "/health":
|
||||
got = b""
|
||||
for name, value in scope.get("headers") or ():
|
||||
if name == b"authorization":
|
||||
got = value
|
||||
if not hmac.compare_digest(got, want):
|
||||
if scope["type"] == "websocket":
|
||||
await send({"type": "websocket.close", "code": 1008})
|
||||
return
|
||||
await send({"type": "http.response.start", "status": 401,
|
||||
"headers": [(b"content-type", b"application/json")]})
|
||||
await send({"type": "http.response.body", "body": denied})
|
||||
return
|
||||
await self.app(scope, receive, send)
|
||||
plain_init = fastapi.FastAPI.__init__
|
||||
def init_with_key(self, *a, **kw):
|
||||
plain_init(self, *a, **kw)
|
||||
self.add_middleware(RequireKey)
|
||||
fastapi.FastAPI.__init__ = init_with_key
|
||||
args = sys.argv[1:]
|
||||
if "` + trtllmOptionsPlaceholder + `" in args:
|
||||
options = tempfile.TemporaryFile()
|
||||
options.write(b"return_perf_metrics: true\nenable_iter_perf_stats: true\n")
|
||||
options.flush()
|
||||
options.seek(0)
|
||||
args[args.index("` + trtllmOptionsPlaceholder + `")] = "/dev/fd/%d" % options.fileno()
|
||||
sys.argv = ["/dev/null/trtllm-serve"] + args
|
||||
from tensorrt_llm.commands.serve import main
|
||||
main()
|
||||
`
|
||||
@@ -0,0 +1,316 @@
|
||||
package engines
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func trtllmSpec() ShardSpec {
|
||||
s := singleNodeSpec()
|
||||
s.Profile.Engine = "trtllm"
|
||||
s.Profile.ServedModelName = "llama-3.1-70b"
|
||||
return s
|
||||
}
|
||||
|
||||
func TestTRTLLMRunsTrtllmServeWithTheFlagsItDocuments(t *testing.T) {
|
||||
s := trtllmSpec()
|
||||
s.BindAddr = "10.0.0.147"
|
||||
s.Profile.Args = []string{"--max_batch_size", "64"}
|
||||
cmd := buildOne(t, &TRTLLM{cfg: TRTLLMConfig{Python: "/opt/trtllm/bin/python"}}, s)
|
||||
|
||||
if cmd.Path != "/opt/trtllm/bin/python" {
|
||||
t.Errorf("Path=%q, expected the configured interpreter", cmd.Path)
|
||||
}
|
||||
if cmd.Args[0] != "-c" || !strings.Contains(cmd.Args[1], "from tensorrt_llm.commands.serve import main") {
|
||||
t.Fatalf("not started through the launcher: %v", cmd.Args[:2])
|
||||
}
|
||||
rest := cmd.Args[2:]
|
||||
if rest[0] != "serve" || rest[1] != "/models/llama" {
|
||||
t.Errorf("expected serve and the model first, got %v", rest[:2])
|
||||
}
|
||||
for flag, want := range map[string]string{
|
||||
"--host": "10.0.0.147", "--port": "20000", "--tp_size": "4",
|
||||
"--served_model_name": "llama-3.1-70b",
|
||||
"--extra_llm_api_options": trtllmOptionsPlaceholder,
|
||||
} {
|
||||
if got := argValue(rest, flag); got != want {
|
||||
t.Errorf("%s=%q, want %q (args %v)", flag, got, want, rest)
|
||||
}
|
||||
}
|
||||
// One pipeline stage is the default, and saying so would only be one
|
||||
// more flag for a version to rename.
|
||||
if slices.Contains(rest, "--pp_size") {
|
||||
t.Errorf("--pp_size given for a single stage: %v", rest)
|
||||
}
|
||||
// The profile's own, last, so a tuning flag is the operator's to set.
|
||||
if got := rest[len(rest)-2:]; !slices.Equal(got, []string{"--max_batch_size", "64"}) {
|
||||
t.Errorf("profile args are not last: %v", rest)
|
||||
}
|
||||
|
||||
s.Profile.TP, s.Profile.PP = 2, 2
|
||||
if got := argValue(buildOne(t, &TRTLLM{}, s).Args, "--pp_size"); got != "2" {
|
||||
t.Errorf("--pp_size=%q for two stages", got)
|
||||
}
|
||||
if got := buildOne(t, &TRTLLM{}, s).Path; got != "python3" {
|
||||
t.Errorf("unconfigured, the interpreter is %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTRTLLMsKeyIsNotOnItsCommandLine(t *testing.T) {
|
||||
s := trtllmSpec()
|
||||
s.APIKey = "k1"
|
||||
cmd := buildOne(t, &TRTLLM{}, s)
|
||||
for _, a := range cmd.Args {
|
||||
if strings.Contains(a, "k1") && a != cmd.Args[1] {
|
||||
t.Errorf("the key is in the arguments: %v", cmd.Args)
|
||||
}
|
||||
}
|
||||
if envMap(cmd.Env)[trtllmKeyEnv] != "k1" {
|
||||
t.Error("the key is not in the environment")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTRTLLMDoesNotReportUsageToNVIDIA(t *testing.T) {
|
||||
s := trtllmSpec()
|
||||
s.Profile.Env = map[string]string{"TRTLLM_NO_USAGE_STATS": "0", "TRTLLM_ENABLE_PDL": "1"}
|
||||
env := envMap(buildOne(t, &TRTLLM{}, s).Env)
|
||||
if env["TRTLLM_NO_USAGE_STATS"] != "1" || env["DO_NOT_TRACK"] != "1" {
|
||||
t.Errorf("usage reporting is not off: %q %q", env["TRTLLM_NO_USAGE_STATS"], env["DO_NOT_TRACK"])
|
||||
}
|
||||
// Tuning still reaches it.
|
||||
if env["TRTLLM_ENABLE_PDL"] != "1" {
|
||||
t.Error("a profile's TensorRT-LLM tuning was dropped")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTRTLLMIsReadFromItsPrometheusEndpoint(t *testing.T) {
|
||||
// /metrics on trtllm-serve is JSON, and each read drains what it returns.
|
||||
if got := (&TRTLLM{}).MetricsPath(); got != "/prometheus/metrics" {
|
||||
t.Errorf("MetricsPath=%q", got)
|
||||
}
|
||||
if !(&TRTLLM{}).ServesHTTP(trtllmSpec()) {
|
||||
t.Error("the one rank of a replica serves its API")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTRTLLMRunsAReplicaOnOneNode(t *testing.T) {
|
||||
p := trtllmSpec().Profile
|
||||
p.Nodes, p.GPUsPerNode, p.TP = 2, 4, 8
|
||||
err := (&TRTLLM{}).Validate(p, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "one node") {
|
||||
t.Errorf("a two-node profile was accepted: %v", err)
|
||||
}
|
||||
s := multiNodeSpec(0)
|
||||
s.Profile.Engine = "trtllm"
|
||||
if _, err := (&TRTLLM{}).Commands(s); err == nil {
|
||||
t.Error("a command was built for a shard of a two-node replica")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTRTLLMNeedsAmpereOrNewer(t *testing.T) {
|
||||
p := trtllmSpec().Profile
|
||||
p.TP = 2
|
||||
v100 := []GPUInfo{{Index: 0, Name: "Tesla V100", ComputeCapability: "7.0"}, {Index: 1, Name: "Tesla V100", ComputeCapability: "7.0"}}
|
||||
if err := (&TRTLLM{}).Validate(p, v100); err == nil {
|
||||
t.Error("a V100 was accepted")
|
||||
}
|
||||
h100 := []GPUInfo{{Index: 0, Name: "H100", ComputeCapability: "9.0"}, {Index: 1, Name: "H100", ComputeCapability: "9.0"}}
|
||||
if err := (&TRTLLM{}).Validate(p, h100); err != nil {
|
||||
t.Errorf("two H100s were refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTRTLLMProfilesMayNotOpenWhatTheAgentCloses(t *testing.T) {
|
||||
for _, args := range [][]string{
|
||||
{"--trust_remote_code"},
|
||||
{"--num_serve_frontends", "4"},
|
||||
{"--grpc"},
|
||||
{"--config", "/etc/shadow"},
|
||||
{"--extra_llm_api_options=/tmp/x.yaml"},
|
||||
{"--served_model_name", "other"},
|
||||
{"--middleware", "evil.mw"},
|
||||
{"--tp_size", "8"},
|
||||
} {
|
||||
if err := UnsafeArgs("trtllm", args); err == nil {
|
||||
t.Errorf("%v was allowed", args)
|
||||
}
|
||||
}
|
||||
if err := UnsafeArgs("trtllm", []string{"--max_batch_size", "64", "--set", "kv_cache_config.enable_block_reuse=false",
|
||||
"--free_gpu_memory_fraction", "0.85"}); err != nil {
|
||||
t.Errorf("tuning was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTRTLLMIsOfferedOnlyWhenItsPackageIsThere(t *testing.T) {
|
||||
there := script(t, "python3", `echo "1.3.0rc29"`)
|
||||
r := NewRegistry(Config{})
|
||||
r.engines = map[string]Engine{"trtllm": &TRTLLM{cfg: TRTLLMConfig{Python: there}}}
|
||||
if got := r.ProbeConfigured(t.Context()); got["trtllm"] != "1.3.0rc29" {
|
||||
t.Errorf("expected the version, got %v", got)
|
||||
}
|
||||
|
||||
// An interpreter without the package, which every machine has.
|
||||
absent := script(t, "python3", "exit 1")
|
||||
r.engines = map[string]Engine{"trtllm": &TRTLLM{cfg: TRTLLMConfig{Python: absent}}}
|
||||
if got := r.ProbeConfigured(t.Context()); len(got) != 0 {
|
||||
t.Errorf("reported without its package: %v", got)
|
||||
}
|
||||
|
||||
// Not named, not offered: python3 on PATH is nobody's TensorRT-LLM.
|
||||
r.engines = map[string]Engine{"trtllm": &TRTLLM{}}
|
||||
if got := r.ProbeConfigured(t.Context()); len(got) != 0 {
|
||||
t.Errorf("offered without being configured: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeTRTLLM is a Python path holding stand-ins for fastapi and
|
||||
// tensorrt_llm: enough of each for the launcher to do what it does in front
|
||||
// of the real server, and for the stand-in server to report what it saw.
|
||||
func fakeTRTLLM(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
files := map[string]string{
|
||||
"fastapi/__init__.py": `
|
||||
class FastAPI:
|
||||
def __init__(self, *a, **kw):
|
||||
self.user_middleware = []
|
||||
def add_middleware(self, cls, **kw):
|
||||
self.user_middleware.append(cls)
|
||||
`,
|
||||
"tensorrt_llm/__init__.py": "",
|
||||
"tensorrt_llm/commands/__init__.py": "",
|
||||
"tensorrt_llm/commands/serve.py": `
|
||||
import asyncio, json, os, sys
|
||||
import fastapi
|
||||
|
||||
def main():
|
||||
args = sys.argv[1:]
|
||||
with open(args[args.index("--extra_llm_api_options") + 1]) as f:
|
||||
options = f.read()
|
||||
app = fastapi.FastAPI()
|
||||
|
||||
async def server(scope, receive, send):
|
||||
await send({"type": "http.response.start", "status": 200, "headers": []})
|
||||
await send({"type": "http.response.body", "body": b"ok"})
|
||||
|
||||
handler = server
|
||||
for mw in reversed(app.user_middleware):
|
||||
handler = mw(handler)
|
||||
|
||||
async def ask(path, auth=None):
|
||||
headers = [(b"authorization", auth.encode())] if auth else []
|
||||
sent = []
|
||||
async def send(msg):
|
||||
sent.append(msg)
|
||||
async def receive():
|
||||
return {"type": "http.request"}
|
||||
await handler({"type": "http", "path": path, "headers": headers}, receive, send)
|
||||
return sent[0]["status"]
|
||||
|
||||
async def run():
|
||||
return {
|
||||
"health": await ask("/health"),
|
||||
"anonymous": await ask("/v1/chat/completions"),
|
||||
"wrong": await ask("/v1/models", "Bearer nope"),
|
||||
"keyed": await ask("/prometheus/metrics", "Bearer k1"),
|
||||
}
|
||||
|
||||
out = asyncio.run(run())
|
||||
out.update(options=options, argv0=sys.argv[0], args=args,
|
||||
key_left=os.environ.get("LLMGW_ENGINE_KEY"))
|
||||
print(json.dumps(out))
|
||||
`,
|
||||
}
|
||||
for name, body := range files {
|
||||
p := filepath.Join(dir, name)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
type launched struct {
|
||||
Health, Anonymous, Wrong, Keyed int
|
||||
Options, Argv0 string
|
||||
Args []string
|
||||
KeyLeft *string `json:"key_left"`
|
||||
}
|
||||
|
||||
func launch(t *testing.T, s ShardSpec) launched {
|
||||
t.Helper()
|
||||
python, err := exec.LookPath("python3")
|
||||
if err != nil {
|
||||
t.Skip("no python3 here to run the launcher with")
|
||||
}
|
||||
cmd := buildOne(t, &TRTLLM{cfg: TRTLLMConfig{Python: python}}, s)
|
||||
c := exec.Command(cmd.Path, cmd.Args...) //nolint:gosec // the command under test
|
||||
c.Env = append(cmd.Env, "PYTHONPATH="+fakeTRTLLM(t))
|
||||
out, err := c.Output()
|
||||
if err != nil {
|
||||
var stderr string
|
||||
if ee, ok := err.(*exec.ExitError); ok {
|
||||
stderr = string(ee.Stderr)
|
||||
}
|
||||
t.Fatalf("the launcher failed: %v\n%s", err, stderr)
|
||||
}
|
||||
var got launched
|
||||
if err := json.Unmarshal(out, &got); err != nil {
|
||||
t.Fatalf("%v: %s", err, out)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func TestTheTRTLLMLauncherDemandsTheKey(t *testing.T) {
|
||||
s := trtllmSpec()
|
||||
s.APIKey = "k1"
|
||||
got := launch(t, s)
|
||||
|
||||
if got.Anonymous != 401 || got.Wrong != 401 {
|
||||
t.Errorf("served without the key: anonymous %d, wrong key %d", got.Anonymous, got.Wrong)
|
||||
}
|
||||
if got.Keyed != 200 {
|
||||
t.Errorf("refused the key: %d", got.Keyed)
|
||||
}
|
||||
// Up or down is all /health says, and it is asked without one.
|
||||
if got.Health != 200 {
|
||||
t.Errorf("/health answered %d", got.Health)
|
||||
}
|
||||
// Taken out of the environment, so nothing the server starts holds it.
|
||||
if got.KeyLeft != nil {
|
||||
t.Errorf("the key was left in the environment: %q", *got.KeyLeft)
|
||||
}
|
||||
// A copy of this command line, as trtllm-serve's extra front ends are,
|
||||
// names no program that exists and so cannot serve without the key.
|
||||
if _, err := os.Stat(got.Argv0); err == nil || filepath.Base(got.Argv0) != "trtllm-serve" {
|
||||
t.Errorf("argv[0] is %q", got.Argv0)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheTRTLLMLauncherTurnsOnTheMetricsTheRouterReads(t *testing.T) {
|
||||
got := launch(t, trtllmSpec())
|
||||
for _, want := range []string{"return_perf_metrics: true", "enable_iter_perf_stats: true"} {
|
||||
if !strings.Contains(got.Options, want) {
|
||||
t.Errorf("the options file lacks %q: %q", want, got.Options)
|
||||
}
|
||||
}
|
||||
path := argValue(got.Args, "--extra_llm_api_options")
|
||||
if !strings.HasPrefix(path, "/dev/fd/") {
|
||||
t.Errorf("the options were not passed by descriptor: %q", path)
|
||||
}
|
||||
if slices.Contains(got.Args, trtllmOptionsPlaceholder) {
|
||||
t.Error("the placeholder reached trtllm-serve")
|
||||
}
|
||||
// Without a key there is nothing to demand.
|
||||
if got.Anonymous != 200 {
|
||||
t.Errorf("with no key set, a request was refused: %d", got.Anonymous)
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,7 @@ var envFamilies = []string{
|
||||
"CUDA_", "NCCL_", "NVIDIA_", "NVTE_", "GLOO_", "UCX_", "TORCH_", "PYTORCH_",
|
||||
"VLLM_", "SGLANG_", "GGML_", "LLAMA_", "OMP_", "MKL_", "OPENBLAS_", "KMP_",
|
||||
"HF_", "TRANSFORMERS_", "TOKENIZERS_", "TRITON_", "FLASHINFER_",
|
||||
"TRTLLM_", "TLLM_",
|
||||
}
|
||||
|
||||
// envExact are single variables that are tuning too.
|
||||
@@ -118,6 +119,32 @@ var flagsNever = map[string]map[string]string{
|
||||
"--model-path": "is the model's source, set by the agent", "--served-model-name": "is set by the agent",
|
||||
"--lora-paths": "loads files", "--download-dir": "writes files",
|
||||
},
|
||||
// trtllm-serve spells its flags with underscores, so the shared list's
|
||||
// hyphenated names do not reach them.
|
||||
"trtllm": {
|
||||
"--trust_remote_code": "runs Python the model repository ships",
|
||||
"--served_model_name": "is set by the agent",
|
||||
"--config": "reads a file; the agent writes the options file itself",
|
||||
"--extra_llm_api_options": "reads a file; the agent writes the options file itself",
|
||||
"--tokenizer": "reads another repository or path",
|
||||
"--custom_tokenizer": "imports Python", "--post_processor_hook": "imports Python",
|
||||
"--custom_module_dirs": "imports Python", "--middleware": "imports Python",
|
||||
"--chat_template": "reads a file",
|
||||
"--metadata_server_config_file": "reads a file",
|
||||
"--visual_gen_args": "reads a file",
|
||||
"--report_addr": "writes a file",
|
||||
"--otlp_traces_endpoint": "sends request traces elsewhere",
|
||||
"--disagg_cluster_uri": "joins the server to another cluster",
|
||||
"--server_role": "makes the server part of another cluster",
|
||||
"--allow_request_chat_template": "lets a request supply the chat template",
|
||||
"--grpc": "serves without the key the agent sets",
|
||||
"--num_serve_frontends": "starts front ends that do not demand the key the agent sets",
|
||||
"--telemetry": "sends usage to NVIDIA",
|
||||
"--tensor_parallel_size": "is set by the agent from tp",
|
||||
"--tp_size": "is set by the agent from tp",
|
||||
"--pipeline_parallel_size": "is set by the agent from pp",
|
||||
"--pp_size": "is set by the agent from pp",
|
||||
},
|
||||
}
|
||||
|
||||
// unsafeFlag is why an engine may not be given a flag, or "".
|
||||
|
||||
Reference in new issue
Block a user