Ilker ManapandClaude Opus 5.5
7940e8593c
Engine trtllm runs trtllm-serve on one node, through a launcher of llmgw's that demands the replica's key and turns on the Prometheus endpoint its queue and cache gauges are read from; the router reads it at /prometheus/metrics. Brought by the operator (engines.trtllm.python), not installed; checked by ctl engine-check trtllm. Written against the documented CLI, server and metrics and tested with stand-ins: not yet run on a datacenter GPU. mirror import-ollama copies a model's GGUF, and a vision model's projector, from Ollama's directory into the mirror, checked against the digest Ollama named each blob by; mirror import does the same for any local file. The console's Mirror view imports from Ollama with I. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
54 lines
2.0 KiB
Go
54 lines
2.0 KiB
Go
package catalog
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestATensorRTLLMProfileIsAGPUProfileOnOneNode(t *testing.T) {
|
|
ok := Model{Name: "llama", Source: "meta-llama/Llama-3.1-70B-Instruct", Profiles: []Profile{{
|
|
Name: "h100", Engine: "trtllm", GPUClass: "h100-80g", TP: 4,
|
|
Args: []string{"--max_batch_size", "64"},
|
|
}}}
|
|
c, err := New([]Model{ok})
|
|
if err != nil {
|
|
t.Fatalf("a four-GPU TensorRT-LLM profile was refused: %v", err)
|
|
}
|
|
m, _ := c.Resolve("llama")
|
|
if p := m.PlacementProfiles()[0]; p.GPUsPerNode != 4 || p.Nodes != 1 {
|
|
t.Errorf("placed as %+v, expected four GPUs of one node", p)
|
|
}
|
|
|
|
for want, bad := range map[string]Model{
|
|
// Two nodes need MPI started from one of them.
|
|
"one node": {Name: "x", Source: "org/x", Profiles: []Profile{{
|
|
Name: "p", Engine: "trtllm", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8}}},
|
|
// No CPU build exists.
|
|
"gpu_class is required": {Name: "x", Source: "org/x", Profiles: []Profile{{Name: "p", Engine: "trtllm"}}},
|
|
// llama.cpp's format.
|
|
"not a .gguf": {Name: "x", Source: "http://mirror:8081/x/v1/x-q4.gguf", Profiles: []Profile{{
|
|
Name: "p", Engine: "trtllm", GPUClass: "h100-80g"}}},
|
|
// What its launcher closes, a profile may not open.
|
|
"--trust_remote_code is not allowed": {Name: "x", Source: "org/x", Profiles: []Profile{{
|
|
Name: "p", Engine: "trtllm", GPUClass: "h100-80g", Args: []string{"--trust_remote_code"}}}},
|
|
} {
|
|
if _, err := New([]Model{bad}); err == nil || !strings.Contains(err.Error(), want) {
|
|
t.Errorf("expected %q, got %v", want, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestATensorRTLLMModelIsGivenTheRepositoryNotItsPage(t *testing.T) {
|
|
// trtllm-serve downloads a repository itself, as vLLM does.
|
|
m := Model{Name: "q", Source: "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct", Profiles: []Profile{{
|
|
Name: "p", Engine: "trtllm", GPUClass: "l40s-48g"}}}
|
|
c, err := New([]Model{m})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, _ := c.Resolve("q")
|
|
if !strings.HasSuffix(got.Source, "Qwen/Qwen2.5-7B-Instruct") || strings.Contains(got.Source, "huggingface.co") {
|
|
t.Errorf("source %q", got.Source)
|
|
}
|
|
}
|