TensorRT-LLM as an engine, and models from Ollama into the mirror

Engine trtllm runs trtllm-serve on one node, through a launcher of
llmgw's that demands the replica's key and turns on the Prometheus
endpoint its queue and cache gauges are read from; the router reads it
at /prometheus/metrics. Brought by the operator (engines.trtllm.python),
not installed; checked by ctl engine-check trtllm. Written against the
documented CLI, server and metrics and tested with stand-ins: not yet
run on a datacenter GPU.

mirror import-ollama copies a model's GGUF, and a vision model's
projector, from Ollama's directory into the mirror, checked against the
digest Ollama named each blob by; mirror import does the same for any
local file. The console's Mirror view imports from Ollama with I.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:02:01 +02:00
1 parent 30f7f3474a
commit 7940e8593c
28 files changed
+2376 -18

No files matched your search

+15 -6
View File
@@ -57,6 +57,8 @@ func newEngineCheckCmd() *cobra.Command {
cfg.VLLM.Bin = bin
case "sglang":
cfg.SGLang.Python = bin
case "trtllm":
cfg.TRTLLM.Python = bin
case "sim":
cfg.Sim.Bin = bin
}
@@ -97,7 +99,7 @@ func newEngineCheckCmd() *cobra.Command {
continue
default:
var err error
if text, err = runHelp(cmd.Context(), c); err != nil {
if text, err = runHelp(cmd.Context(), eng, c); err != nil {
skipped++
fmt.Fprintf(errOut, " skipped: %v\n", err)
continue
@@ -167,9 +169,10 @@ func newEngineCheckCmd() *cobra.Command {
// representativeShard is the widest command llmgw would ever build, because a
// flag that only appears in the multi-node case is exactly the one nobody
// notices has been renamed.
// notices has been renamed. For an engine that runs a replica on one node,
// that is one node.
func representativeShard(engine string, nodes, tp, perNode int) engines.ShardSpec {
if nodes < 1 {
if nodes < 1 || engine == "trtllm" {
nodes = 1
}
return engines.ShardSpec{
@@ -203,9 +206,10 @@ var helpArgs = map[string][]string{
}
// runHelp asks one program to describe itself.
func runHelp(ctx context.Context, c engines.Command) (string, error) {
func runHelp(ctx context.Context, eng engines.Engine, c engines.Command) (string, error) {
// Everything up to the first flag, plus the help request. Passing the real
// arguments would start the engine.
path := c.Path
var args []string
for _, a := range c.Args {
if strings.HasPrefix(a, "-") {
@@ -218,11 +222,16 @@ func runHelp(ctx context.Context, c engines.Command) (string, error) {
ask = []string{"--help"}
}
args = append(args, ask...)
// An engine started through a launcher of llmgw's is asked through its
// own entry point: the interpreter's --help describes the interpreter.
if h, ok := eng.(engines.HelpCommand); ok {
path, args = h.HelpCommand(c)
}
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
ec := exec.CommandContext(ctx, c.Path, args...)
ec := exec.CommandContext(ctx, path, args...)
// vLLM refuses to do anything at all on a machine with no accelerator,
// including print its help, unless it is told what to assume. Asking an
// engine about its flags should not require the hardware it runs on.
@@ -231,7 +240,7 @@ func runHelp(ctx context.Context, c engines.Command) (string, error) {
raw, err := ec.CombinedOutput()
if err != nil && len(raw) == 0 {
return "", fmt.Errorf("could not run %s: %w (install it here, or pass --help-output)",
c.Path, err)
path, err)
}
return string(raw), nil
}