TensorRT-LLM as an engine, and models from Ollama into the mirror
Engine trtllm runs trtllm-serve on one node, through a launcher of llmgw's that demands the replica's key and turns on the Prometheus endpoint its queue and cache gauges are read from; the router reads it at /prometheus/metrics. Brought by the operator (engines.trtllm.python), not installed; checked by ctl engine-check trtllm. Written against the documented CLI, server and metrics and tested with stand-ins: not yet run on a datacenter GPU. mirror import-ollama copies a model's GGUF, and a vision model's projector, from Ollama's directory into the mirror, checked against the digest Ollama named each blob by; mirror import does the same for any local file. The console's Mirror view imports from Ollama with I. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
30f7f3474a
commit
7940e8593c
28 files changed
+2376
-18
No files matched your search
@@ -1,6 +1,7 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -178,10 +179,41 @@ func (m *consoleModel) mirrorFetchForm() {
|
||||
}})
|
||||
}
|
||||
|
||||
// mirrorImportOllamaForm puts a model from an Ollama on this machine into a
|
||||
// mirror on this machine: the control plane's own, when the mirror runs
|
||||
// beside it. A mirror elsewhere cannot read this machine's files, and its
|
||||
// API is given none; there it is `llmgw mirror import-ollama`, on the mirror.
|
||||
func (m *consoleModel) mirrorImportOllamaForm() {
|
||||
m.openForm(&formDialog{title: "Import from Ollama",
|
||||
note: "From an Ollama on this machine into the mirror's directory on this machine.\n" +
|
||||
"A mirror elsewhere: run llmgw mirror import-ollama there.",
|
||||
fields: []formField{
|
||||
textF("model", "", "as Ollama names it, like llama3.2:3b"),
|
||||
textF("ollama dir", "", "optional: found where Ollama keeps it"),
|
||||
textF("mirror dir", "/srv/models", "the mirror's directory on this machine"),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
if v[0] == "" {
|
||||
return nil, fmt.Errorf("a model, as Ollama names it")
|
||||
}
|
||||
if st, err := os.Stat(v[2]); err != nil || !st.IsDir() {
|
||||
return nil, fmt.Errorf("%s is not a directory on this machine", v[2])
|
||||
}
|
||||
title := "Import " + v[0]
|
||||
m.busy = title
|
||||
o := ollamaImport{name: v[0], ollamaDir: v[1], mirrorDir: v[2], base: m.mirror.res.Base, quiet: true}
|
||||
return func() tea.Msg {
|
||||
var out bytes.Buffer
|
||||
err := o.run(context.Background(), &out)
|
||||
return actionDoneMsg{title: title, text: strings.TrimSpace(out.String()), err: err}
|
||||
}, nil
|
||||
}})
|
||||
}
|
||||
|
||||
func (m consoleModel) mirrorPanel(pw int) (string, string, string) {
|
||||
keys := "→ or Enter open · ↑↓ choose · Q leave"
|
||||
if m.inPanel {
|
||||
keys = "↑↓ row · Enter actions · A download one · ← back · Q leave"
|
||||
keys = "↑↓ row · Enter actions · A download one · I import from Ollama · ← back · Q leave"
|
||||
}
|
||||
if m.mirror.err != nil {
|
||||
return "Mirror", "\n " + sBad.Render(m.mirror.err.Error()), keys
|
||||
@@ -216,7 +248,7 @@ func (m consoleModel) mirrorPanel(pw int) (string, string, string) {
|
||||
rows = append(rows, []string{g.model, g.version, what, state, bytesShort(total), used})
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return "Mirror", head + " " + sDim.Render("Empty. A downloads one."), keys
|
||||
return "Mirror", head + " " + sDim.Render("Empty. A downloads one; I imports one from Ollama."), keys
|
||||
}
|
||||
return "Mirror", head + table([]string{"MODEL", "VERSION", "FILE", "STATE", "SIZE", "USED BY"}, rows, m.row, pw, m.inPanel), keys
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user