TensorRT-LLM as an engine, and models from Ollama into the mirror
Engine trtllm runs trtllm-serve on one node, through a launcher of llmgw's that demands the replica's key and turns on the Prometheus endpoint its queue and cache gauges are read from; the router reads it at /prometheus/metrics. Brought by the operator (engines.trtllm.python), not installed; checked by ctl engine-check trtllm. Written against the documented CLI, server and metrics and tested with stand-ins: not yet run on a datacenter GPU. mirror import-ollama copies a model's GGUF, and a vision model's projector, from Ollama's directory into the mirror, checked against the digest Ollama named each blob by; mirror import does the same for any local file. The console's Mirror view imports from Ollama with I. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
30f7f3474a
commit
7940e8593c
28 files changed
+2376
-18
No files matched your search
@@ -717,6 +717,19 @@ func (m Model) validate() error {
|
||||
}
|
||||
continue
|
||||
}
|
||||
// TensorRT-LLM spans machines only under MPI started from one of them,
|
||||
// which the agents, each starting its own node, do not do; and it
|
||||
// loads a Hugging Face checkpoint, not llama.cpp's format.
|
||||
if p.Engine == "trtllm" {
|
||||
if p.Nodes > 1 {
|
||||
return fmt.Errorf("model %q profile %q: trtllm runs a replica on one node; "+
|
||||
"use vllm or sglang for a %d-node replica", m.Name, p.Name, p.Nodes)
|
||||
}
|
||||
if strings.HasSuffix(strings.ToLower(strings.TrimSuffix(m.Source, "/")), ".gguf") {
|
||||
return fmt.Errorf("model %q profile %q: trtllm loads a Hugging Face checkpoint, not a .gguf; "+
|
||||
"serve a GGUF with llamacpp", m.Name, p.Name)
|
||||
}
|
||||
}
|
||||
// The arithmetic has to add up, or the engine is handed a shard spec it
|
||||
// cannot satisfy and fails minutes later during model loading.
|
||||
if want := p.GPUsPerNode * p.Nodes; want != p.TP*max(p.PP, 1) {
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestATensorRTLLMProfileIsAGPUProfileOnOneNode(t *testing.T) {
|
||||
ok := Model{Name: "llama", Source: "meta-llama/Llama-3.1-70B-Instruct", Profiles: []Profile{{
|
||||
Name: "h100", Engine: "trtllm", GPUClass: "h100-80g", TP: 4,
|
||||
Args: []string{"--max_batch_size", "64"},
|
||||
}}}
|
||||
c, err := New([]Model{ok})
|
||||
if err != nil {
|
||||
t.Fatalf("a four-GPU TensorRT-LLM profile was refused: %v", err)
|
||||
}
|
||||
m, _ := c.Resolve("llama")
|
||||
if p := m.PlacementProfiles()[0]; p.GPUsPerNode != 4 || p.Nodes != 1 {
|
||||
t.Errorf("placed as %+v, expected four GPUs of one node", p)
|
||||
}
|
||||
|
||||
for want, bad := range map[string]Model{
|
||||
// Two nodes need MPI started from one of them.
|
||||
"one node": {Name: "x", Source: "org/x", Profiles: []Profile{{
|
||||
Name: "p", Engine: "trtllm", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8}}},
|
||||
// No CPU build exists.
|
||||
"gpu_class is required": {Name: "x", Source: "org/x", Profiles: []Profile{{Name: "p", Engine: "trtllm"}}},
|
||||
// llama.cpp's format.
|
||||
"not a .gguf": {Name: "x", Source: "http://mirror:8081/x/v1/x-q4.gguf", Profiles: []Profile{{
|
||||
Name: "p", Engine: "trtllm", GPUClass: "h100-80g"}}},
|
||||
// What its launcher closes, a profile may not open.
|
||||
"--trust_remote_code is not allowed": {Name: "x", Source: "org/x", Profiles: []Profile{{
|
||||
Name: "p", Engine: "trtllm", GPUClass: "h100-80g", Args: []string{"--trust_remote_code"}}}},
|
||||
} {
|
||||
if _, err := New([]Model{bad}); err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("expected %q, got %v", want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestATensorRTLLMModelIsGivenTheRepositoryNotItsPage(t *testing.T) {
|
||||
// trtllm-serve downloads a repository itself, as vLLM does.
|
||||
m := Model{Name: "q", Source: "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct", Profiles: []Profile{{
|
||||
Name: "p", Engine: "trtllm", GPUClass: "l40s-48g"}}}
|
||||
c, err := New([]Model{m})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := c.Resolve("q")
|
||||
if !strings.HasSuffix(got.Source, "Qwen/Qwen2.5-7B-Instruct") || strings.Contains(got.Source, "huggingface.co") {
|
||||
t.Errorf("source %q", got.Source)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user