TensorRT-LLM as an engine, and models from Ollama into the mirror

Engine trtllm runs trtllm-serve on one node, through a launcher of
llmgw's that demands the replica's key and turns on the Prometheus
endpoint its queue and cache gauges are read from; the router reads it
at /prometheus/metrics. Brought by the operator (engines.trtllm.python),
not installed; checked by ctl engine-check trtllm. Written against the
documented CLI, server and metrics and tested with stand-ins: not yet
run on a datacenter GPU.

mirror import-ollama copies a model's GGUF, and a vision model's
projector, from Ollama's directory into the mirror, checked against the
digest Ollama named each blob by; mirror import does the same for any
local file. The console's Mirror view imports from Ollama with I.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:02:01 +02:00
1 parent 30f7f3474a
commit 7940e8593c
28 files changed
+2376 -18

No files matched your search

+77
View File
@@ -1,6 +1,11 @@
package cli
import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
)
@@ -69,3 +74,75 @@ func TestATruncatedHelpIsSuspected(t *testing.T) {
t.Error("a truncated help was taken for a complete one")
}
}
// fakeInterpreter stands in for TensorRT-LLM's Python: asked for
// trtllm-serve's help through its module, it prints the help given; asked
// anything else, it fails, as running the launcher here would.
func fakeInterpreter(t *testing.T, help string) string {
t.Helper()
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "help.txt"), []byte(help), 0o644); err != nil {
t.Fatal(err)
}
p := filepath.Join(dir, "python")
body := "#!/bin/sh\n" +
`if [ "$*" = "-m tensorrt_llm.commands.serve serve --help" ]; then cat "` + filepath.Join(dir, "help.txt") + `"; exit 0; fi` + "\n" +
"echo \"unexpected: $*\" >&2; exit 2\n"
if err := os.WriteFile(p, []byte(body), 0o755); err != nil { //nolint:gosec // an executable stand-in
t.Fatal(err)
}
return p
}
// trtllmHelp is the part of trtllm-serve serve --help (1.3) that holds the
// flags llmgw builds.
const trtllmHelp = `Usage: trtllm-serve serve [OPTIONS] MODEL
Options:
--host TEXT Hostname of the server.
--port INTEGER Port of the server.
--tensor_parallel_size, --tp_size INTEGER
Tensor parallelism size.
--pipeline_parallel_size, --pp_size INTEGER
Pipeline parallelism size.
--config, --extra_llm_api_options TEXT
Path to a YAML configuration file.
--served_model_name TEXT The model name used in the API.
--help Show this message and exit.
`
func runEngineCheck(t *testing.T, args ...string) (string, error) {
t.Helper()
c := newEngineCheckCmd()
var out bytes.Buffer
c.SetOut(&out)
c.SetErr(&out)
c.SetArgs(args)
err := c.ExecuteContext(context.Background())
return out.String(), err
}
func TestEngineCheckAsksTrtllmServeThroughItsModule(t *testing.T) {
out, err := runEngineCheck(t, "trtllm", "--bin", fakeInterpreter(t, trtllmHelp))
if err != nil {
t.Fatalf("%v\n%s", err, out)
}
if !strings.Contains(out, "every flag llmgw builds is accepted") {
t.Errorf("unexpected report:\n%s", out)
}
// The interpreter's own help would describe Python, not trtllm-serve; and
// one node is what trtllm runs, so no multi-node flag is in the command.
if strings.Contains(out, "unexpected:") || strings.Contains(out, "skipped") {
t.Errorf("the help was not asked of trtllm-serve:\n%s", out)
}
}
func TestEngineCheckCatchesATensorRTLLMTooOldForTheServedName(t *testing.T) {
// TensorRT-LLM 1.2 has no --served_model_name, and a replica of it would
// never be seen serving the name the agent waits for.
old := strings.ReplaceAll(trtllmHelp, " --served_model_name TEXT The model name used in the API.\n", "")
out, err := runEngineCheck(t, "trtllm", "--bin", fakeInterpreter(t, old))
if err == nil || !strings.Contains(out, "trtllm-serve: --served_model_name") {
t.Errorf("an old trtllm-serve passed: %v\n%s", err, out)
}
}