TensorRT-LLM as an engine, and models from Ollama into the mirror

Engine trtllm runs trtllm-serve on one node, through a launcher of
llmgw's that demands the replica's key and turns on the Prometheus
endpoint its queue and cache gauges are read from; the router reads it
at /prometheus/metrics. Brought by the operator (engines.trtllm.python),
not installed; checked by ctl engine-check trtllm. Written against the
documented CLI, server and metrics and tested with stand-ins: not yet
run on a datacenter GPU.

mirror import-ollama copies a model's GGUF, and a vision model's
projector, from Ollama's directory into the mirror, checked against the
digest Ollama named each blob by; mirror import does the same for any
local file. The console's Mirror view imports from Ollama with I.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:02:01 +02:00
1 parent 30f7f3474a
commit 7940e8593c
28 files changed
+2376 -18

No files matched your search

+34 -2
View File
@@ -1,6 +1,7 @@
package cli
import (
"bytes"
"context"
"errors"
"fmt"
@@ -178,10 +179,41 @@ func (m *consoleModel) mirrorFetchForm() {
}})
}
// mirrorImportOllamaForm puts a model from an Ollama on this machine into a
// mirror on this machine: the control plane's own, when the mirror runs
// beside it. A mirror elsewhere cannot read this machine's files, and its
// API is given none; there it is `llmgw mirror import-ollama`, on the mirror.
func (m *consoleModel) mirrorImportOllamaForm() {
m.openForm(&formDialog{title: "Import from Ollama",
note: "From an Ollama on this machine into the mirror's directory on this machine.\n" +
"A mirror elsewhere: run llmgw mirror import-ollama there.",
fields: []formField{
textF("model", "", "as Ollama names it, like llama3.2:3b"),
textF("ollama dir", "", "optional: found where Ollama keeps it"),
textF("mirror dir", "/srv/models", "the mirror's directory on this machine"),
},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
if v[0] == "" {
return nil, fmt.Errorf("a model, as Ollama names it")
}
if st, err := os.Stat(v[2]); err != nil || !st.IsDir() {
return nil, fmt.Errorf("%s is not a directory on this machine", v[2])
}
title := "Import " + v[0]
m.busy = title
o := ollamaImport{name: v[0], ollamaDir: v[1], mirrorDir: v[2], base: m.mirror.res.Base, quiet: true}
return func() tea.Msg {
var out bytes.Buffer
err := o.run(context.Background(), &out)
return actionDoneMsg{title: title, text: strings.TrimSpace(out.String()), err: err}
}, nil
}})
}
func (m consoleModel) mirrorPanel(pw int) (string, string, string) {
keys := "→ or Enter open · ↑↓ choose · Q leave"
if m.inPanel {
keys = "↑↓ row · Enter actions · A download one · ← back · Q leave"
keys = "↑↓ row · Enter actions · A download one · I import from Ollama · ← back · Q leave"
}
if m.mirror.err != nil {
return "Mirror", "\n " + sBad.Render(m.mirror.err.Error()), keys
@@ -216,7 +248,7 @@ func (m consoleModel) mirrorPanel(pw int) (string, string, string) {
rows = append(rows, []string{g.model, g.version, what, state, bytesShort(total), used})
}
if len(rows) == 0 {
return "Mirror", head + " " + sDim.Render("Empty. A downloads one."), keys
return "Mirror", head + " " + sDim.Render("Empty. A downloads one; I imports one from Ollama."), keys
}
return "Mirror", head + table([]string{"MODEL", "VERSION", "FILE", "STATE", "SIZE", "USED BY"}, rows, m.row, pw, m.inPanel), keys
}