TensorRT-LLM as an engine, and models from Ollama into the mirror
Engine trtllm runs trtllm-serve on one node, through a launcher of llmgw's that demands the replica's key and turns on the Prometheus endpoint its queue and cache gauges are read from; the router reads it at /prometheus/metrics. Brought by the operator (engines.trtllm.python), not installed; checked by ctl engine-check trtllm. Written against the documented CLI, server and metrics and tested with stand-ins: not yet run on a datacenter GPU. mirror import-ollama copies a model's GGUF, and a vision model's projector, from Ollama's directory into the mirror, checked against the digest Ollama named each blob by; mirror import does the same for any local file. The console's Mirror view imports from Ollama with I. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
30f7f3474a
commit
7940e8593c
28 files changed
+2376
-18
No files matched your search
@@ -57,6 +57,8 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
cfg.VLLM.Bin = bin
|
||||
case "sglang":
|
||||
cfg.SGLang.Python = bin
|
||||
case "trtllm":
|
||||
cfg.TRTLLM.Python = bin
|
||||
case "sim":
|
||||
cfg.Sim.Bin = bin
|
||||
}
|
||||
@@ -97,7 +99,7 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
continue
|
||||
default:
|
||||
var err error
|
||||
if text, err = runHelp(cmd.Context(), c); err != nil {
|
||||
if text, err = runHelp(cmd.Context(), eng, c); err != nil {
|
||||
skipped++
|
||||
fmt.Fprintf(errOut, " skipped: %v\n", err)
|
||||
continue
|
||||
@@ -167,9 +169,10 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
|
||||
// representativeShard is the widest command llmgw would ever build, because a
|
||||
// flag that only appears in the multi-node case is exactly the one nobody
|
||||
// notices has been renamed.
|
||||
// notices has been renamed. For an engine that runs a replica on one node,
|
||||
// that is one node.
|
||||
func representativeShard(engine string, nodes, tp, perNode int) engines.ShardSpec {
|
||||
if nodes < 1 {
|
||||
if nodes < 1 || engine == "trtllm" {
|
||||
nodes = 1
|
||||
}
|
||||
return engines.ShardSpec{
|
||||
@@ -203,9 +206,10 @@ var helpArgs = map[string][]string{
|
||||
}
|
||||
|
||||
// runHelp asks one program to describe itself.
|
||||
func runHelp(ctx context.Context, c engines.Command) (string, error) {
|
||||
func runHelp(ctx context.Context, eng engines.Engine, c engines.Command) (string, error) {
|
||||
// Everything up to the first flag, plus the help request. Passing the real
|
||||
// arguments would start the engine.
|
||||
path := c.Path
|
||||
var args []string
|
||||
for _, a := range c.Args {
|
||||
if strings.HasPrefix(a, "-") {
|
||||
@@ -218,11 +222,16 @@ func runHelp(ctx context.Context, c engines.Command) (string, error) {
|
||||
ask = []string{"--help"}
|
||||
}
|
||||
args = append(args, ask...)
|
||||
// An engine started through a launcher of llmgw's is asked through its
|
||||
// own entry point: the interpreter's --help describes the interpreter.
|
||||
if h, ok := eng.(engines.HelpCommand); ok {
|
||||
path, args = h.HelpCommand(c)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
ec := exec.CommandContext(ctx, c.Path, args...)
|
||||
ec := exec.CommandContext(ctx, path, args...)
|
||||
// vLLM refuses to do anything at all on a machine with no accelerator,
|
||||
// including print its help, unless it is told what to assume. Asking an
|
||||
// engine about its flags should not require the hardware it runs on.
|
||||
@@ -231,7 +240,7 @@ func runHelp(ctx context.Context, c engines.Command) (string, error) {
|
||||
raw, err := ec.CombinedOutput()
|
||||
if err != nil && len(raw) == 0 {
|
||||
return "", fmt.Errorf("could not run %s: %w (install it here, or pass --help-output)",
|
||||
c.Path, err)
|
||||
path, err)
|
||||
}
|
||||
return string(raw), nil
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user