TensorRT-LLM as an engine, and models from Ollama into the mirror

Engine trtllm runs trtllm-serve on one node, through a launcher of
llmgw's that demands the replica's key and turns on the Prometheus
endpoint its queue and cache gauges are read from; the router reads it
at /prometheus/metrics. Brought by the operator (engines.trtllm.python),
not installed; checked by ctl engine-check trtllm. Written against the
documented CLI, server and metrics and tested with stand-ins: not yet
run on a datacenter GPU.

mirror import-ollama copies a model's GGUF, and a vision model's
projector, from Ollama's directory into the mirror, checked against the
digest Ollama named each blob by; mirror import does the same for any
local file. The console's Mirror view imports from Ollama with I.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:02:01 +02:00
1 parent 30f7f3474a
commit 7940e8593c
28 files changed
+2376 -18

No files matched your search

+13
View File
@@ -717,6 +717,19 @@ func (m Model) validate() error {
}
continue
}
// TensorRT-LLM spans machines only under MPI started from one of them,
// which the agents, each starting its own node, do not do; and it
// loads a Hugging Face checkpoint, not llama.cpp's format.
if p.Engine == "trtllm" {
if p.Nodes > 1 {
return fmt.Errorf("model %q profile %q: trtllm runs a replica on one node; "+
"use vllm or sglang for a %d-node replica", m.Name, p.Name, p.Nodes)
}
if strings.HasSuffix(strings.ToLower(strings.TrimSuffix(m.Source, "/")), ".gguf") {
return fmt.Errorf("model %q profile %q: trtllm loads a Hugging Face checkpoint, not a .gguf; "+
"serve a GGUF with llamacpp", m.Name, p.Name)
}
}
// The arithmetic has to add up, or the engine is handed a shard spec it
// cannot satisfy and fails minutes later during model loading.
if want := p.GPUsPerNode * p.Nodes; want != p.TP*max(p.PP, 1) {
+53
View File
@@ -0,0 +1,53 @@
package catalog
import (
"strings"
"testing"
)
func TestATensorRTLLMProfileIsAGPUProfileOnOneNode(t *testing.T) {
ok := Model{Name: "llama", Source: "meta-llama/Llama-3.1-70B-Instruct", Profiles: []Profile{{
Name: "h100", Engine: "trtllm", GPUClass: "h100-80g", TP: 4,
Args: []string{"--max_batch_size", "64"},
}}}
c, err := New([]Model{ok})
if err != nil {
t.Fatalf("a four-GPU TensorRT-LLM profile was refused: %v", err)
}
m, _ := c.Resolve("llama")
if p := m.PlacementProfiles()[0]; p.GPUsPerNode != 4 || p.Nodes != 1 {
t.Errorf("placed as %+v, expected four GPUs of one node", p)
}
for want, bad := range map[string]Model{
// Two nodes need MPI started from one of them.
"one node": {Name: "x", Source: "org/x", Profiles: []Profile{{
Name: "p", Engine: "trtllm", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8}}},
// No CPU build exists.
"gpu_class is required": {Name: "x", Source: "org/x", Profiles: []Profile{{Name: "p", Engine: "trtllm"}}},
// llama.cpp's format.
"not a .gguf": {Name: "x", Source: "http://mirror:8081/x/v1/x-q4.gguf", Profiles: []Profile{{
Name: "p", Engine: "trtllm", GPUClass: "h100-80g"}}},
// What its launcher closes, a profile may not open.
"--trust_remote_code is not allowed": {Name: "x", Source: "org/x", Profiles: []Profile{{
Name: "p", Engine: "trtllm", GPUClass: "h100-80g", Args: []string{"--trust_remote_code"}}}},
} {
if _, err := New([]Model{bad}); err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("expected %q, got %v", want, err)
}
}
}
func TestATensorRTLLMModelIsGivenTheRepositoryNotItsPage(t *testing.T) {
// trtllm-serve downloads a repository itself, as vLLM does.
m := Model{Name: "q", Source: "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct", Profiles: []Profile{{
Name: "p", Engine: "trtllm", GPUClass: "l40s-48g"}}}
c, err := New([]Model{m})
if err != nil {
t.Fatal(err)
}
got, _ := c.Resolve("q")
if !strings.HasSuffix(got.Source, "Qwen/Qwen2.5-7B-Instruct") || strings.Contains(got.Source, "huggingface.co") {
t.Errorf("source %q", got.Source)
}
}