TensorRT-LLM as an engine, and models from Ollama into the mirror
Engine trtllm runs trtllm-serve on one node, through a launcher of llmgw's that demands the replica's key and turns on the Prometheus endpoint its queue and cache gauges are read from; the router reads it at /prometheus/metrics. Brought by the operator (engines.trtllm.python), not installed; checked by ctl engine-check trtllm. Written against the documented CLI, server and metrics and tested with stand-ins: not yet run on a datacenter GPU. mirror import-ollama copies a model's GGUF, and a vision model's projector, from Ollama's directory into the mirror, checked against the digest Ollama named each blob by; mirror import does the same for any local file. The console's Mirror view imports from Ollama with I. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
30f7f3474a
commit
7940e8593c
28 files changed
+2376
-18
No files matched your search
@@ -1,6 +1,11 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -69,3 +74,75 @@ func TestATruncatedHelpIsSuspected(t *testing.T) {
|
||||
t.Error("a truncated help was taken for a complete one")
|
||||
}
|
||||
}
|
||||
|
||||
// fakeInterpreter stands in for TensorRT-LLM's Python: asked for
|
||||
// trtllm-serve's help through its module, it prints the help given; asked
|
||||
// anything else, it fails, as running the launcher here would.
|
||||
func fakeInterpreter(t *testing.T, help string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "help.txt"), []byte(help), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := filepath.Join(dir, "python")
|
||||
body := "#!/bin/sh\n" +
|
||||
`if [ "$*" = "-m tensorrt_llm.commands.serve serve --help" ]; then cat "` + filepath.Join(dir, "help.txt") + `"; exit 0; fi` + "\n" +
|
||||
"echo \"unexpected: $*\" >&2; exit 2\n"
|
||||
if err := os.WriteFile(p, []byte(body), 0o755); err != nil { //nolint:gosec // an executable stand-in
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// trtllmHelp is the part of trtllm-serve serve --help (1.3) that holds the
|
||||
// flags llmgw builds.
|
||||
const trtllmHelp = `Usage: trtllm-serve serve [OPTIONS] MODEL
|
||||
|
||||
Options:
|
||||
--host TEXT Hostname of the server.
|
||||
--port INTEGER Port of the server.
|
||||
--tensor_parallel_size, --tp_size INTEGER
|
||||
Tensor parallelism size.
|
||||
--pipeline_parallel_size, --pp_size INTEGER
|
||||
Pipeline parallelism size.
|
||||
--config, --extra_llm_api_options TEXT
|
||||
Path to a YAML configuration file.
|
||||
--served_model_name TEXT The model name used in the API.
|
||||
--help Show this message and exit.
|
||||
`
|
||||
|
||||
func runEngineCheck(t *testing.T, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
c := newEngineCheckCmd()
|
||||
var out bytes.Buffer
|
||||
c.SetOut(&out)
|
||||
c.SetErr(&out)
|
||||
c.SetArgs(args)
|
||||
err := c.ExecuteContext(context.Background())
|
||||
return out.String(), err
|
||||
}
|
||||
|
||||
func TestEngineCheckAsksTrtllmServeThroughItsModule(t *testing.T) {
|
||||
out, err := runEngineCheck(t, "trtllm", "--bin", fakeInterpreter(t, trtllmHelp))
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
if !strings.Contains(out, "every flag llmgw builds is accepted") {
|
||||
t.Errorf("unexpected report:\n%s", out)
|
||||
}
|
||||
// The interpreter's own help would describe Python, not trtllm-serve; and
|
||||
// one node is what trtllm runs, so no multi-node flag is in the command.
|
||||
if strings.Contains(out, "unexpected:") || strings.Contains(out, "skipped") {
|
||||
t.Errorf("the help was not asked of trtllm-serve:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineCheckCatchesATensorRTLLMTooOldForTheServedName(t *testing.T) {
|
||||
// TensorRT-LLM 1.2 has no --served_model_name, and a replica of it would
|
||||
// never be seen serving the name the agent waits for.
|
||||
old := strings.ReplaceAll(trtllmHelp, " --served_model_name TEXT The model name used in the API.\n", "")
|
||||
out, err := runEngineCheck(t, "trtllm", "--bin", fakeInterpreter(t, old))
|
||||
if err == nil || !strings.Contains(out, "trtllm-serve: --served_model_name") {
|
||||
t.Errorf("an old trtllm-serve passed: %v\n%s", err, out)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user