TensorRT-LLM as an engine, and models from Ollama into the mirror
Engine trtllm runs trtllm-serve on one node, through a launcher of llmgw's that demands the replica's key and turns on the Prometheus endpoint its queue and cache gauges are read from; the router reads it at /prometheus/metrics. Brought by the operator (engines.trtllm.python), not installed; checked by ctl engine-check trtllm. Written against the documented CLI, server and metrics and tested with stand-ins: not yet run on a datacenter GPU. mirror import-ollama copies a model's GGUF, and a vision model's projector, from Ollama's directory into the mirror, checked against the digest Ollama named each blob by; mirror import does the same for any local file. The console's Mirror view imports from Ollama with I. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
30f7f3474a
commit
7940e8593c
28 files changed
+2376
-18
No files matched your search
@@ -0,0 +1,144 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
)
|
||||
|
||||
// runMirror runs `llmgw mirror ...` as an operator would and returns what it
|
||||
// printed.
|
||||
func runMirror(t *testing.T, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
c := Root()
|
||||
var out bytes.Buffer
|
||||
c.SetOut(&out)
|
||||
c.SetErr(&out)
|
||||
c.SetArgs(append([]string{"mirror"}, args...))
|
||||
err := c.ExecuteContext(t.Context())
|
||||
return out.String(), err
|
||||
}
|
||||
|
||||
// ollamaWith lays out an Ollama models directory holding one model, as
|
||||
// Ollama writes it, and returns the directory and the weights' sha256.
|
||||
func ollamaWith(t *testing.T, name, tag string, weights []byte, params string) (string, string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(t.TempDir(), "models")
|
||||
blob := func(content []byte) string {
|
||||
s := sha256.Sum256(content)
|
||||
h := hex.EncodeToString(s[:])
|
||||
if err := os.MkdirAll(filepath.Join(dir, "blobs"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "blobs", "sha256-"+h), content, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return h
|
||||
}
|
||||
w := blob(weights)
|
||||
p := blob([]byte(params))
|
||||
manifest := `{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json",` +
|
||||
`"config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:` + blob([]byte("{}")) + `","size":2},` +
|
||||
`"layers":[{"mediaType":"application/vnd.ollama.image.model","digest":"sha256:` + w + `","size":` + strconv.Itoa(len(weights)) + `},` +
|
||||
`{"mediaType":"application/vnd.ollama.image.params","digest":"sha256:` + p + `","size":` + strconv.Itoa(len(params)) + `}]}`
|
||||
mdir := filepath.Join(dir, "manifests", "registry.ollama.ai", "library", name)
|
||||
if err := os.MkdirAll(mdir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(mdir, tag), []byte(manifest), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return dir, w
|
||||
}
|
||||
|
||||
func TestImportOllamaListsThenImportsAModelForLlamaCpp(t *testing.T) {
|
||||
weights := []byte("GGUF\x03\x00\x00\x00qwen weights")
|
||||
ollama, sum := ollamaWith(t, "qwen2.5", "7b", weights, `{"num_ctx":16384}`)
|
||||
root := t.TempDir()
|
||||
|
||||
out, err := runMirror(t, "--dir", root, "import-ollama", "--ollama-dir", ollama)
|
||||
if err != nil || !strings.Contains(out, " qwen2.5:7b\n") {
|
||||
t.Fatalf("listing: %v\n%s", err, out)
|
||||
}
|
||||
|
||||
out, err = runMirror(t, "--dir", root, "import-ollama", "qwen2.5:7b", "--ollama-dir", ollama,
|
||||
"--base", "http://models.internal:8081")
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
file := filepath.Join(root, "qwen2.5", "7b-"+sum[:12], "qwen2.5-7b.gguf")
|
||||
if b, err := os.ReadFile(file); err != nil || !bytes.Equal(b, weights) {
|
||||
t.Fatalf("the GGUF is not in the mirror at %s: %v", file, err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"sha256 " + sum,
|
||||
"source: http://models.internal:8081/qwen2.5/7b-" + sum[:12] + "/qwen2.5-7b.gguf",
|
||||
"engine: llamacpp",
|
||||
`args: ["--ctx-size", "16384"]`,
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("the output does not say %q:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheConsoleImportsFromOllamaIntoTheMirrorBesideIt(t *testing.T) {
|
||||
weights := []byte("GGUF\x03\x00\x00\x00llama weights")
|
||||
ollama, sum := ollamaWith(t, "llama3.2", "3b", weights, `{}`)
|
||||
root := t.TempDir()
|
||||
|
||||
m, _ := consoleWith(t, cMirror)
|
||||
var tm tea.Model = press(m, "i")
|
||||
f := tm.(consoleModel).formD
|
||||
if f == nil || f.title != "Import from Ollama" {
|
||||
t.Fatalf("I opened %+v", f)
|
||||
}
|
||||
f.fields[0].text.SetValue("llama3.2:3b")
|
||||
f.fields[1].text.SetValue(ollama)
|
||||
f.fields[2].text.SetValue(root)
|
||||
for range len(f.fields) {
|
||||
tm = press(tm, "tab")
|
||||
}
|
||||
tm = pressRun(tm, "enter")
|
||||
|
||||
if _, err := os.Stat(filepath.Join(root, "llama3.2", "3b-"+sum[:12], "llama3.2-3b.gguf")); err != nil {
|
||||
t.Fatalf("not imported: %v", err)
|
||||
}
|
||||
text := tm.(consoleModel).dialogText
|
||||
if !strings.Contains(text, "engine: llamacpp") || strings.Contains(text, "(100%)") {
|
||||
t.Errorf("the console said:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportCopiesALocalGGUFIntoTheMirror(t *testing.T) {
|
||||
src := filepath.Join(t.TempDir(), "q4_k_m.gguf")
|
||||
content := []byte("GGUF\x03\x00\x00\x00converted")
|
||||
if err := os.WriteFile(src, content, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root := t.TempDir()
|
||||
out, err := runMirror(t, "--dir", root, "import", src, "--model", "qwen", "--version", "2026-10")
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
if b, err := os.ReadFile(filepath.Join(root, "qwen", "2026-10", "q4_k_m.gguf")); err != nil || !bytes.Equal(b, content) {
|
||||
t.Fatalf("not imported: %v", err)
|
||||
}
|
||||
if !strings.Contains(out, "source: http://<this-host>:8081/qwen/2026-10/q4_k_m.gguf") {
|
||||
t.Errorf("no source given:\n%s", out)
|
||||
}
|
||||
// The layout is the point: a version is never replaced.
|
||||
if _, err := runMirror(t, "--dir", root, "import", src, "--model", "qwen", "--version", "2026-10"); err == nil {
|
||||
t.Error("an existing version was replaced")
|
||||
}
|
||||
if _, err := runMirror(t, "--dir", root, "import", src); err == nil {
|
||||
t.Error("imported without --model")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user