TensorRT-LLM as an engine, and models from Ollama into the mirror

Engine trtllm runs trtllm-serve on one node, through a launcher of
llmgw's that demands the replica's key and turns on the Prometheus
endpoint its queue and cache gauges are read from; the router reads it
at /prometheus/metrics. Brought by the operator (engines.trtllm.python),
not installed; checked by ctl engine-check trtllm. Written against the
documented CLI, server and metrics and tested with stand-ins: not yet
run on a datacenter GPU.

mirror import-ollama copies a model's GGUF, and a vision model's
projector, from Ollama's directory into the mirror, checked against the
digest Ollama named each blob by; mirror import does the same for any
local file. The console's Mirror view imports from Ollama with I.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:02:01 +02:00
1 parent 30f7f3474a
commit 7940e8593c
28 files changed
+2376 -18

No files matched your search

+131
View File
@@ -0,0 +1,131 @@
package mirror
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"os"
"path/filepath"
"strings"
)
// Import copies a file on this machine into the mirror at model/version/file:
// weights somebody already has — a GGUF they converted, Ollama's blob of
// one — put where every node can fetch them, with the checksum nodes check
// them against.
//
// The same discipline as Add: written under a temporary name and renamed
// when whole, the checksum and the source beside it first, and an existing
// version never replaced. wantSHA256, when given, is what the file must hash
// to — Ollama names each blob by its digest — and a file that does not is
// refused rather than copied to every node.
//
// source is what is recorded as where the file came from.
func Import(ctx context.Context, root, src string, e Entry, wantSHA256, source string,
progress func(done, total int64),
) (Added, error) {
if e.Model == "" || e.Version == "" || e.File == "" {
return Added{}, fmt.Errorf("a model, a version and a filename are all required")
}
for _, n := range []string{e.Model, e.Version, e.File} {
if err := ValidName(n); err != nil {
return Added{}, err
}
}
in, err := os.Open(src) //nolint:gosec // a file the operator named
if err != nil {
return Added{}, err
}
defer in.Close()
st, err := in.Stat()
if err != nil {
return Added{}, err
}
if !st.Mode().IsRegular() {
return Added{}, fmt.Errorf("%s is not a file", src)
}
if strings.HasSuffix(strings.ToLower(e.File), ".gguf") {
if err := isGGUF(in, src); err != nil {
return Added{}, err
}
}
final := e.Path(root)
if _, err := os.Stat(final); err == nil {
return Added{}, fmt.Errorf("%s/%s/%s: %w", e.Model, e.Version, e.File, ErrExists)
}
if err := os.MkdirAll(filepath.Dir(final), 0o755); err != nil { //nolint:gosec // served to every node
return Added{}, err
}
part := final + partSuffix
// Truncated rather than refused when there: a .part is what an import cut
// short left, and the next attempt is the one that should finish it.
out, err := os.OpenFile(part, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) //nolint:gosec // inside the mirror root
if err != nil {
return Added{}, err
}
// Hashed on the way through: a forty gigabyte file is read once, not
// copied and then read again.
h := sha256.New()
written, err := copyWithProgress(io.MultiWriter(out, h), ctxReader{ctx, in}, st.Size(), progress)
closeErr := out.Close()
if err == nil {
err = closeErr
}
if err == nil && written != st.Size() {
err = fmt.Errorf("%s: copied %d bytes of %d; it changed while being read", src, written, st.Size())
}
sum := hex.EncodeToString(h.Sum(nil))
if err == nil && wantSHA256 != "" && !strings.EqualFold(sum, wantSHA256) {
err = fmt.Errorf("%s is damaged: it should hash to sha256 %s and hashes to %s; "+
"nothing was added", src, wantSHA256, sum)
}
if err == nil {
err = writeChecksum(final, sum)
}
if err == nil {
err = writeSource(final, source)
}
if err == nil {
err = os.Rename(part, final)
}
if err != nil {
_ = os.Remove(part)
return Added{}, err
}
e.Bytes = written
return Added{Entry: e, SHA256: sum}, nil
}
// ggufMagic opens every GGUF file.
const ggufMagic = "GGUF"
// isGGUF checks a file named .gguf is one, and leaves it read from the start.
// A file that is not — an HTML error page saved under the name, Ollama's
// newer safetensors-based layers — would otherwise reach every node and fail
// there, in llama.cpp's words.
func isGGUF(f *os.File, name string) error {
head := make([]byte, len(ggufMagic))
if _, err := io.ReadFull(f, head); err != nil || string(head) != ggufMagic {
return fmt.Errorf("%s is not a GGUF file: it does not begin with %q", name, ggufMagic)
}
_, err := f.Seek(0, io.SeekStart)
return err
}
// ctxReader stops a copy when its context ends: an operator's ^C on a long
// import leaves no partial file behind.
type ctxReader struct {
ctx context.Context
r io.Reader
}
func (c ctxReader) Read(p []byte) (int, error) {
if err := c.ctx.Err(); err != nil {
return 0, err
}
return c.r.Read(p)
}
+352
View File
@@ -0,0 +1,352 @@
package mirror
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
)
// Ollama keeps a model as a manifest and the blobs it names. The manifest is
// at manifests/<host>/<namespace>/<model>/<tag> under its models directory —
// registry.ollama.ai/library/llama3.2/3b for "llama3.2:3b" — and each layer
// is a file blobs/sha256-<hex>, named by its own digest. The layer whose media
// type is application/vnd.ollama.image.model is the weights as a GGUF, which
// is what llama.cpp runs; a vision model has its projector in a second GGUF.
//
// What an operator already has in Ollama is weights they chose and have
// tried. Importing them puts that file in the mirror, checked against the
// digest Ollama named it by, rather than downloading it again from somewhere
// that may not be the same file.
// Ollama's defaults for the parts of a name left out.
const (
ollamaHost = "registry.ollama.ai"
ollamaNamespace = "library"
ollamaTag = "latest"
)
// The layer media types an import reads, or says it leaves.
const (
ollamaModelLayer = "application/vnd.ollama.image.model"
ollamaProjectorLayer = "application/vnd.ollama.image.projector"
ollamaParamsLayer = "application/vnd.ollama.image.params"
ollamaTensorLayer = "application/vnd.ollama.image.tensor"
)
// OllamaName is a model as Ollama names it.
type OllamaName struct {
Host, Namespace, Model, Tag string
}
// ParseOllamaName reads a name as Ollama does: model, namespace/model or
// host/namespace/model, with an optional :tag, and its defaults for whatever
// is left out — "llama3.2" is registry.ollama.ai/library/llama3.2:latest.
func ParseOllamaName(s string) (OllamaName, error) {
s = strings.TrimSpace(s)
var n OllamaName
// A digest pins a name in Ollama; the manifest on disk is by tag.
s, _, _ = strings.Cut(s, "@")
// "/" cannot be in a tag, so a colon after the last slash is the tag's.
if i := strings.LastIndex(s, ":"); i > strings.LastIndex(s, "/") {
s, n.Tag = s[:i], s[i+1:]
}
parts := strings.Split(s, "/")
switch len(parts) {
case 1:
n.Model = parts[0]
case 2:
n.Namespace, n.Model = parts[0], parts[1]
case 3:
n.Host, n.Namespace, n.Model = parts[0], parts[1], parts[2]
default:
return OllamaName{}, fmt.Errorf("%q is not an Ollama model name: model[:tag], namespace/model[:tag] "+
"or host/namespace/model[:tag]", s)
}
if n.Host == "" {
n.Host = ollamaHost
}
if n.Namespace == "" {
n.Namespace = ollamaNamespace
}
if n.Tag == "" {
n.Tag = ollamaTag
}
for _, p := range []string{n.Host, n.Namespace, n.Model, n.Tag} {
// Each part is a directory of Ollama's; one that is not a single path
// element would read a manifest from somewhere else.
if p == "" || p == "." || p == ".." || strings.ContainsAny(p, `\`+"\x00") {
return OllamaName{}, fmt.Errorf("%q is not an Ollama model name", s)
}
}
return n, nil
}
// String is the name the way Ollama shows it: without the parts that are its
// defaults.
func (n OllamaName) String() string {
name := n.Model + ":" + n.Tag
switch {
case !strings.EqualFold(n.Host, ollamaHost):
return n.Host + "/" + n.Namespace + "/" + name
case !strings.EqualFold(n.Namespace, ollamaNamespace):
return n.Namespace + "/" + name
}
return name
}
func (n OllamaName) manifestPath(dir string) string {
return filepath.Join(dir, "manifests", n.Host, n.Namespace, n.Model, n.Tag)
}
// OllamaDir finds Ollama's models directory: the one given (or its models
// subdirectory, for a ~/.ollama given whole), else OLLAMA_MODELS, else the
// places Ollama keeps it — this user's ~/.ollama/models, and on Linux the
// home of the ollama user its installer makes.
func OllamaDir(given string) (string, error) {
var candidates []string
if given != "" {
candidates = []string{given, filepath.Join(given, "models")}
} else {
if env := os.Getenv("OLLAMA_MODELS"); env != "" {
candidates = append(candidates, env)
}
if home, err := os.UserHomeDir(); err == nil {
candidates = append(candidates, filepath.Join(home, ".ollama", "models"))
}
candidates = append(candidates,
"/usr/share/ollama/.ollama/models", // Ollama's Linux installer
"/var/lib/ollama/.ollama/models", // some distributions' packages
)
}
for _, c := range candidates {
if st, err := os.Stat(filepath.Join(c, "manifests")); err == nil && st.IsDir() {
return c, nil
}
}
if given != "" {
return "", fmt.Errorf("%s holds no Ollama models: there is no manifests directory in it or in its models", given)
}
return "", fmt.Errorf("no Ollama models found in %s; name the directory with --ollama-dir",
strings.Join(candidates, ", "))
}
// ListOllama lists the models in an Ollama models directory.
func ListOllama(dir string) ([]OllamaName, error) {
root := filepath.Join(dir, "manifests")
var out []OllamaName
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil || d.IsDir() {
return err
}
rel, err := filepath.Rel(root, path)
if err != nil {
return err
}
parts := strings.Split(rel, string(filepath.Separator))
if len(parts) != 4 || strings.HasPrefix(parts[3], ".") {
return nil // not a manifest: something else left in Ollama's directory
}
out = append(out, OllamaName{Host: parts[0], Namespace: parts[1], Model: parts[2], Tag: parts[3]})
return nil
})
if err != nil {
return nil, fmt.Errorf("reading Ollama's models in %s: %w", dir, err)
}
sort.Slice(out, func(i, j int) bool { return out[i].String() < out[j].String() })
return out, nil
}
// OllamaLayer is one layer of an Ollama manifest.
type OllamaLayer struct {
MediaType string `json:"mediaType"`
Digest string `json:"digest"`
Size int64 `json:"size"`
}
// ollamaManifest is the manifest's shape: Docker's image manifest, with
// Ollama's media types for the layers.
type ollamaManifest struct {
SchemaVersion int `json:"schemaVersion"`
Layers []OllamaLayer `json:"layers"`
}
// OllamaModel is what an import takes from one of Ollama's models.
type OllamaModel struct {
Name OllamaName
// Model is the weights, a GGUF.
Model OllamaLayer
// Projectors are a vision model's image encoders, GGUFs beside it.
Projectors []OllamaLayer
// ContextLength is the num_ctx Ollama was told to run it with, 0 when
// it was not: worth carrying over, since llama.cpp's own default differs.
ContextLength int
// Left says what of the Ollama model an import does not carry.
Left []string
}
var digestPattern = regexp.MustCompile(`^sha256:([0-9a-f]{64})$`)
// ollamaBlob is where a layer's file is, and the sha256 its name promises.
func ollamaBlob(dir, digest string) (path, sum string, err error) {
m := digestPattern.FindStringSubmatch(digest)
if m == nil {
return "", "", fmt.Errorf("the manifest names a layer by %q, which is not a sha256 digest", digest)
}
return filepath.Join(dir, "blobs", "sha256-"+m[1]), m[1], nil
}
// ReadOllama reads one model's manifest and the layers an import takes.
func ReadOllama(dir string, name OllamaName) (OllamaModel, error) {
b, err := os.ReadFile(name.manifestPath(dir)) //nolint:gosec // a manifest under Ollama's directory
if errors.Is(err, os.ErrNotExist) {
return OllamaModel{}, fmt.Errorf("no %s among the Ollama models in %s; `llmgw mirror import-ollama` with no name lists them",
name, dir)
}
if err != nil {
return OllamaModel{}, err
}
var mf ollamaManifest
if err := json.Unmarshal(b, &mf); err != nil {
return OllamaModel{}, fmt.Errorf("%s's manifest cannot be read: %w", name, err)
}
m := OllamaModel{Name: name}
var models []OllamaLayer
for _, l := range mf.Layers {
switch l.MediaType {
case ollamaModelLayer:
models = append(models, l)
case ollamaProjectorLayer:
m.Projectors = append(m.Projectors, l)
case ollamaTensorLayer:
return OllamaModel{}, fmt.Errorf("%s is kept by Ollama as separate tensors rather than a GGUF; "+
"llama.cpp cannot run that form: mirror the model's GGUF from its publisher instead", name)
case ollamaParamsLayer:
m.ContextLength = ollamaContext(dir, l)
case "application/vnd.ollama.image.template", "application/vnd.ollama.image.prompt":
m.Left = append(m.Left, "its prompt template: llama.cpp renders the chat template inside the GGUF")
case "application/vnd.ollama.image.system":
m.Left = append(m.Left, "its system prompt")
case "application/vnd.ollama.image.adapter":
m.Left = append(m.Left, "a LoRA adapter: give it to the model as one of its adapters")
case "application/vnd.ollama.image.draft":
m.Left = append(m.Left, "its draft model for speculative decoding")
}
}
switch len(models) {
case 0:
return OllamaModel{}, fmt.Errorf("%s has no GGUF layer (%s) to import", name, ollamaModelLayer)
case 1:
m.Model = models[0]
default:
return OllamaModel{}, fmt.Errorf("%s is made of %d weight layers, which an import does not join yet; "+
"mirror the model's GGUF from its publisher instead", name, len(models))
}
return m, nil
}
// ollamaContext is the num_ctx in a params layer, 0 when there is none or it
// cannot be read: a hint, not worth failing an import over.
func ollamaContext(dir string, l OllamaLayer) int {
path, _, err := ollamaBlob(dir, l.Digest)
if err != nil {
return 0
}
b, err := os.ReadFile(path) //nolint:gosec // a blob under Ollama's directory
if err != nil {
return 0
}
var p struct {
NumCtx int `json:"num_ctx"`
}
_ = json.Unmarshal(b, &p)
return p.NumCtx
}
// OllamaEntries are where an Ollama model's files go in the mirror, by
// default: the model's own name, and a version that is its tag with the start
// of its digest — a tag such as "latest" moves in Ollama, and a mirror
// version must not.
func OllamaEntries(m OllamaModel, model, version string) (weights Entry, projectors []Entry) {
if model == "" {
model = m.Name.Model
}
_, sum, _ := ollamaBlob("", m.Model.Digest)
if version == "" {
version = m.Name.Tag + "-" + sum[:min(12, len(sum))]
}
base := m.Name.Model + "-" + m.Name.Tag
weights = Entry{Model: model, Version: version, File: base + ".gguf"}
for i := range m.Projectors {
// "mmproj" in the name is what llama.cpp's engine knows a projector by,
// beside the model in one directory.
file := "mmproj-" + base + ".gguf"
if i > 0 {
file = fmt.Sprintf("mmproj-%s-%d.gguf", base, i+1)
}
projectors = append(projectors, Entry{Model: model, Version: version, File: file})
}
return weights, projectors
}
// ImportOllama copies an Ollama model's GGUF, and its projectors, into one
// version of the mirror, each checked against the digest Ollama named it by.
// progress is told which file it is about.
//
// All or nothing: a version left holding a projector and no model is one a
// node would fetch and fail to start, so the files already copied are taken
// out again when a later one fails.
func ImportOllama(ctx context.Context, root, dir string, m OllamaModel, model, version string,
progress func(file string, done, total int64),
) ([]Added, error) {
weights, projectors := OllamaEntries(m, model, version)
type job struct {
layer OllamaLayer
entry Entry
}
jobs := []job{{m.Model, weights}}
for i, l := range m.Projectors {
jobs = append(jobs, job{l, projectors[i]})
}
// Refused before anything is copied, rather than after the first file.
if _, err := os.Stat(filepath.Join(root, weights.Model, weights.Version)); err == nil {
return nil, fmt.Errorf("%s/%s: %w", weights.Model, weights.Version, ErrExists)
}
source := "ollama://" + m.Name.Host + "/" + m.Name.Namespace + "/" + m.Name.Model + ":" + m.Name.Tag
var done []Added
for _, j := range jobs {
a, err := importLayer(ctx, root, dir, j.layer, j.entry, source, progress)
if err != nil {
for _, prev := range done {
_ = Remove(root, prev.Entry)
}
return nil, fmt.Errorf("%s: %w", m.Name, err)
}
done = append(done, a)
}
return done, nil
}
func importLayer(ctx context.Context, root, dir string, l OllamaLayer, e Entry, source string,
progress func(file string, done, total int64),
) (Added, error) {
src, sum, err := ollamaBlob(dir, l.Digest)
if err != nil {
return Added{}, err
}
if _, err := os.Stat(src); err != nil {
return Added{}, fmt.Errorf("its layer %s is not in %s: %w", l.Digest, filepath.Join(dir, "blobs"), err)
}
var report func(int64, int64)
if progress != nil {
report = func(d, t int64) { progress(e.File, d, t) }
}
return Import(ctx, root, src, e, sum, source, report)
}
+369
View File
@@ -0,0 +1,369 @@
package mirror
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
)
// ollamaFixture is an Ollama models directory laid out as Ollama writes it:
// blobs/sha256-<hex> named by their content's digest, and a manifest per
// model and tag under manifests/<host>/<namespace>/<model>/<tag>.
type ollamaFixture struct {
t *testing.T
dir string
}
func newOllamaFixture(t *testing.T) *ollamaFixture {
t.Helper()
dir := filepath.Join(t.TempDir(), ".ollama", "models")
for _, d := range []string{"blobs", "manifests"} {
if err := os.MkdirAll(filepath.Join(dir, d), 0o755); err != nil {
t.Fatal(err)
}
}
return &ollamaFixture{t: t, dir: dir}
}
// blob stores content as Ollama does and returns the layer naming it.
func (f *ollamaFixture) blob(mediaType string, content []byte) OllamaLayer {
f.t.Helper()
sum := sha256.Sum256(content)
hexsum := hex.EncodeToString(sum[:])
if err := os.WriteFile(filepath.Join(f.dir, "blobs", "sha256-"+hexsum), content, 0o644); err != nil {
f.t.Fatal(err)
}
return OllamaLayer{MediaType: mediaType, Digest: "sha256:" + hexsum, Size: int64(len(content))}
}
func (f *ollamaFixture) manifest(name string, layers ...OllamaLayer) {
f.t.Helper()
n, err := ParseOllamaName(name)
if err != nil {
f.t.Fatal(err)
}
config := f.blob("application/vnd.docker.container.image.v1+json",
[]byte(`{"model_format":"gguf","model_family":"llama","file_type":"Q4_K_M"}`))
b, _ := json.Marshal(map[string]any{
"schemaVersion": 2,
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"config": config,
"layers": layers,
})
p := n.manifestPath(f.dir)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
f.t.Fatal(err)
}
if err := os.WriteFile(p, b, 0o644); err != nil {
f.t.Fatal(err)
}
}
// gguf is a small file that begins as every GGUF does.
func gguf(body string) []byte { return []byte("GGUF\x03\x00\x00\x00" + body) }
// llama32 is llama3.2:3b as Ollama pulls it: the weights, a template, a
// license and parameters.
func (f *ollamaFixture) llama32() OllamaLayer {
weights := f.blob(ollamaModelLayer, gguf("llama 3.2 3b weights"))
f.manifest("llama3.2:3b",
weights,
f.blob("application/vnd.ollama.image.template", []byte(`<|start_header_id|>{{ .Prompt }}`)),
f.blob("application/vnd.ollama.image.license", []byte("LLAMA 3.2 COMMUNITY LICENSE")),
f.blob(ollamaParamsLayer, []byte(`{"num_ctx":8192,"stop":["<|eot_id|>"]}`)),
)
return weights
}
func TestOllamaNamesAreReadAsOllamaReadsThem(t *testing.T) {
for in, want := range map[string]OllamaName{
"llama3.2": {ollamaHost, ollamaNamespace, "llama3.2", "latest"},
"llama3.2:3b": {ollamaHost, ollamaNamespace, "llama3.2", "3b"},
"jmorgan/llava:7b-q4": {ollamaHost, "jmorgan", "llava", "7b-q4"},
"hf.co/bartowski/Qwen-GGUF:Q4": {"hf.co", "bartowski", "Qwen-GGUF", "Q4"},
"localhost:5000/me/m:v1": {"localhost:5000", "me", "m", "v1"},
"qwen3:8b@sha256:abcd": {ollamaHost, ollamaNamespace, "qwen3", "8b"},
} {
got, err := ParseOllamaName(in)
if err != nil || got != want {
t.Errorf("%q: got %+v, %v; want %+v", in, got, err, want)
}
}
for _, bad := range []string{"", "a/b/c/d", "../etc:passwd", "x:.."} {
if _, err := ParseOllamaName(bad); err == nil {
t.Errorf("%q was taken for a name", bad)
}
}
// Shown the way Ollama shows them: without its defaults.
for in, want := range map[string]string{
"llama3.2": "llama3.2:latest", "jmorgan/llava:7b": "jmorgan/llava:7b",
"hf.co/bartowski/Qwen-GGUF:Q4": "hf.co/bartowski/Qwen-GGUF:Q4",
} {
n, _ := ParseOllamaName(in)
if n.String() != want {
t.Errorf("%q shows as %q", in, n.String())
}
}
}
func TestOllamasDirectoryIsFoundWhereOllamaKeepsIt(t *testing.T) {
f := newOllamaFixture(t)
// Given as the models directory, or as the ~/.ollama it is in.
for _, given := range []string{f.dir, filepath.Dir(f.dir)} {
if got, err := OllamaDir(given); err != nil || got != f.dir {
t.Errorf("%s: got %q, %v", given, got, err)
}
}
t.Setenv("OLLAMA_MODELS", f.dir)
if got, err := OllamaDir(""); err != nil || got != f.dir {
t.Errorf("OLLAMA_MODELS: got %q, %v", got, err)
}
if _, err := OllamaDir(t.TempDir()); err == nil {
t.Error("a directory with no manifests was taken for Ollama's")
}
}
func TestOllamasModelsAreListedFromItsManifests(t *testing.T) {
f := newOllamaFixture(t)
f.llama32()
f.manifest("hf.co/bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", f.blob(ollamaModelLayer, gguf("qwen")))
// Ollama's own leftovers are not models.
_ = os.WriteFile(filepath.Join(f.dir, "manifests", ".DS_Store"), nil, 0o644)
got, err := ListOllama(f.dir)
if err != nil {
t.Fatal(err)
}
var names []string
for _, n := range got {
names = append(names, n.String())
}
if !slices.Equal(names, []string{"hf.co/bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", "llama3.2:3b"}) {
t.Errorf("listed %v", names)
}
}
func TestAnOllamaManifestYieldsItsGGUFAndWhatIsLeftBehind(t *testing.T) {
f := newOllamaFixture(t)
weights := f.llama32()
m, err := ReadOllama(f.dir, mustName(t, "llama3.2:3b"))
if err != nil {
t.Fatal(err)
}
if m.Model != weights {
t.Errorf("model layer %+v, want %+v", m.Model, weights)
}
// What Ollama ran it with, to carry over as --ctx-size.
if m.ContextLength != 8192 {
t.Errorf("num_ctx=%d", m.ContextLength)
}
// The template is Ollama's own language; saying it is left is the honest
// part, since llama.cpp will use the GGUF's.
if len(m.Left) != 1 || !strings.Contains(m.Left[0], "template") {
t.Errorf("left: %v", m.Left)
}
}
func TestOllamaModelsLlamaCppCannotRunAreRefused(t *testing.T) {
f := newOllamaFixture(t)
// Ollama's newer layout keeps some models as separate tensors.
f.manifest("gpt-oss:20b", f.blob(ollamaTensorLayer, []byte("tensor")), f.blob(ollamaTensorLayer, []byte("other")))
// Two weight layers make a model this import does not join.
f.manifest("split:1", f.blob(ollamaModelLayer, gguf("a")), f.blob(ollamaModelLayer, gguf("b")))
f.manifest("empty:1", f.blob("application/vnd.ollama.image.license", []byte("MIT")))
for name, want := range map[string]string{
"gpt-oss:20b": "separate tensors", "split:1": "2 weight layers", "empty:1": "no GGUF layer",
"absent:1": "no absent:1",
} {
_, err := ReadOllama(f.dir, mustName(t, name))
if err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("%s: expected %q, got %v", name, want, err)
}
}
}
func TestAnOllamaModelIsImportedIntoTheMirrorWithItsChecksum(t *testing.T) {
f := newOllamaFixture(t)
weights := f.llama32()
root := t.TempDir()
m, err := ReadOllama(f.dir, mustName(t, "llama3.2:3b"))
if err != nil {
t.Fatal(err)
}
var reported []string
added, err := ImportOllama(t.Context(), root, f.dir, m, "", "", func(file string, _, _ int64) {
reported = append(reported, file)
})
if err != nil {
t.Fatal(err)
}
// The tag alone moves in Ollama ("latest" is whatever was pulled last);
// the start of the digest is what pins the version.
sum := strings.TrimPrefix(weights.Digest, "sha256:")
want := Entry{Model: "llama3.2", Version: "3b-" + sum[:12], File: "llama3.2-3b.gguf", Bytes: weights.Size}
if len(added) != 1 || added[0].Entry != want {
t.Fatalf("added %+v, want %+v", added, want)
}
if added[0].SHA256 != sum {
t.Errorf("sha256 %s, want the digest Ollama named it by, %s", added[0].SHA256, sum)
}
if len(reported) == 0 || reported[0] != "llama3.2-3b.gguf" {
t.Errorf("progress said %v", reported)
}
// What a node is given: the file, with the checksum it checks it against.
mf, err := ManifestFor(root, want.Model, want.Version)
if err != nil {
t.Fatal(err)
}
if len(mf.Files) != 1 || mf.Files[0].SHA256 != sum {
t.Errorf("manifest %+v", mf)
}
b, _ := os.ReadFile(want.Path(root))
if string(b) != string(gguf("llama 3.2 3b weights")) {
t.Error("the mirrored file is not the blob")
}
if got := readSource(want.Path(root)); got != "ollama://registry.ollama.ai/library/llama3.2:3b" {
t.Errorf("source recorded as %q", got)
}
// The same model again is the same version, and the mirror never
// replaces one.
if _, err := ImportOllama(t.Context(), root, f.dir, m, "", "", nil); !errors.Is(err, ErrExists) {
t.Errorf("a second import was not refused: %v", err)
}
// Under names the operator chose, it goes where they say.
added, err = ImportOllama(t.Context(), root, f.dir, m, "llama-3.2-3b-instruct", "2026-10", nil)
if err != nil || added[0].Entry.Path(root) != filepath.Join(root, "llama-3.2-3b-instruct", "2026-10", "llama3.2-3b.gguf") {
t.Errorf("named import: %+v, %v", added, err)
}
}
func TestAVisionModelsProjectorGoesBesideItForLlamaCpp(t *testing.T) {
f := newOllamaFixture(t)
f.manifest("llava:7b",
f.blob(ollamaModelLayer, gguf("llava weights")),
f.blob(ollamaProjectorLayer, gguf("clip projector")),
)
root := t.TempDir()
m, err := ReadOllama(f.dir, mustName(t, "llava:7b"))
if err != nil {
t.Fatal(err)
}
added, err := ImportOllama(t.Context(), root, f.dir, m, "", "v1", nil)
if err != nil {
t.Fatal(err)
}
if len(added) != 2 || added[1].Entry.File != "mmproj-llava-7b.gguf" {
t.Fatalf("added %+v", added)
}
// The version directory is the model's source, and llama.cpp's engine
// finds in it the model to load and the projector to load beside it.
dir := filepath.Join(root, "llava", "v1")
s := engines.ShardSpec{HTTPPort: 20000, Profile: engines.Profile{Name: "cpu", Engine: "llamacpp", ModelPath: dir}}
cmds, err := (&engines.LlamaCpp{}).Commands(s)
if err != nil {
t.Fatal(err)
}
args := cmds[0].Args
if got := args[slices.Index(args, "--model")+1]; got != filepath.Join(dir, "llava-7b.gguf") {
t.Errorf("--model %s", got)
}
if got := args[slices.Index(args, "--mmproj")+1]; got != filepath.Join(dir, "mmproj-llava-7b.gguf") {
t.Errorf("--mmproj %s", got)
}
}
func TestADamagedOllamaBlobIsNotMirrored(t *testing.T) {
f := newOllamaFixture(t)
f.manifest("llava:7b",
f.blob(ollamaModelLayer, gguf("llava weights")),
f.blob(ollamaProjectorLayer, gguf("clip projector")),
)
m, err := ReadOllama(f.dir, mustName(t, "llava:7b"))
if err != nil {
t.Fatal(err)
}
// A bit gone wrong on Ollama's disk: the projector no longer hashes to
// the name it was stored under.
path, _, _ := ollamaBlob(f.dir, m.Projectors[0].Digest)
if err := os.WriteFile(path, gguf("clip projectoR"), 0o644); err != nil {
t.Fatal(err)
}
root := t.TempDir()
_, err = ImportOllama(t.Context(), root, f.dir, m, "", "v1", nil)
if err == nil || !strings.Contains(err.Error(), "damaged") {
t.Fatalf("expected the damage to be found, got %v", err)
}
// All or nothing: the weights copied before it are taken out again, so
// no node is offered a vision model without its projector.
if entries, _ := List(root); len(entries) != 0 {
t.Errorf("left in the mirror: %+v", entries)
}
if _, err := os.Stat(filepath.Join(root, "llava")); !errors.Is(err, os.ErrNotExist) {
t.Errorf("the version's directory was left behind: %v", err)
}
// A blob gone altogether says which.
_ = os.Remove(path)
if _, err := ImportOllama(t.Context(), root, f.dir, m, "", "v1", nil); err == nil ||
!strings.Contains(err.Error(), m.Projectors[0].Digest) {
t.Errorf("a missing blob: %v", err)
}
}
func TestOnlyAGGUFIsImportedAsOne(t *testing.T) {
// An HTML error page saved as a .gguf would reach every node otherwise.
src := filepath.Join(t.TempDir(), "model.gguf")
_ = os.WriteFile(src, []byte("<html>rate limited</html>"), 0o644)
_, err := Import(t.Context(), t.TempDir(), src, Entry{Model: "m", Version: "v", File: "model.gguf"}, "", "file://"+src, nil)
if err == nil || !strings.Contains(err.Error(), "not a GGUF") {
t.Errorf("expected a refusal, got %v", err)
}
}
func TestALocalFileIsImportedWithItsChecksum(t *testing.T) {
src := filepath.Join(t.TempDir(), "q4_k_m.gguf")
content := gguf("converted by hand")
_ = os.WriteFile(src, content, 0o644)
root := t.TempDir()
e := Entry{Model: "qwen", Version: "2026-10-04", File: "q4_k_m.gguf"}
a, err := Import(t.Context(), root, src, e, "", "file://"+src, nil)
if err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(content)
if a.SHA256 != hex.EncodeToString(sum[:]) || readChecksum(e.Path(root)) != a.SHA256 {
t.Errorf("checksum %s, recorded %s", a.SHA256, readChecksum(e.Path(root)))
}
// A checksum the operator was given is checked too.
e.Version = "again"
if _, err := Import(t.Context(), root, src, e, strings.Repeat("0", 64), "file://"+src, nil); err == nil {
t.Error("a file that does not hash to the sum given was imported")
}
if _, err := os.Stat(e.Path(root) + partSuffix); !errors.Is(err, os.ErrNotExist) {
t.Error("a refused import left its partial file")
}
}
func mustName(t *testing.T, s string) OllamaName {
t.Helper()
n, err := ParseOllamaName(s)
if err != nil {
t.Fatal(err)
}
return n
}