TensorRT-LLM as an engine, and models from Ollama into the mirror
Engine trtllm runs trtllm-serve on one node, through a launcher of llmgw's that demands the replica's key and turns on the Prometheus endpoint its queue and cache gauges are read from; the router reads it at /prometheus/metrics. Brought by the operator (engines.trtllm.python), not installed; checked by ctl engine-check trtllm. Written against the documented CLI, server and metrics and tested with stand-ins: not yet run on a datacenter GPU. mirror import-ollama copies a model's GGUF, and a vision model's projector, from Ollama's directory into the mirror, checked against the digest Ollama named each blob by; mirror import does the same for any local file. The console's Mirror view imports from Ollama with I. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
30f7f3474a
commit
7940e8593c
28 files changed
+2376
-18
No files matched your search
@@ -0,0 +1,131 @@
|
||||
package mirror
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Import copies a file on this machine into the mirror at model/version/file:
|
||||
// weights somebody already has — a GGUF they converted, Ollama's blob of
|
||||
// one — put where every node can fetch them, with the checksum nodes check
|
||||
// them against.
|
||||
//
|
||||
// The same discipline as Add: written under a temporary name and renamed
|
||||
// when whole, the checksum and the source beside it first, and an existing
|
||||
// version never replaced. wantSHA256, when given, is what the file must hash
|
||||
// to — Ollama names each blob by its digest — and a file that does not is
|
||||
// refused rather than copied to every node.
|
||||
//
|
||||
// source is what is recorded as where the file came from.
|
||||
func Import(ctx context.Context, root, src string, e Entry, wantSHA256, source string,
|
||||
progress func(done, total int64),
|
||||
) (Added, error) {
|
||||
if e.Model == "" || e.Version == "" || e.File == "" {
|
||||
return Added{}, fmt.Errorf("a model, a version and a filename are all required")
|
||||
}
|
||||
for _, n := range []string{e.Model, e.Version, e.File} {
|
||||
if err := ValidName(n); err != nil {
|
||||
return Added{}, err
|
||||
}
|
||||
}
|
||||
in, err := os.Open(src) //nolint:gosec // a file the operator named
|
||||
if err != nil {
|
||||
return Added{}, err
|
||||
}
|
||||
defer in.Close()
|
||||
st, err := in.Stat()
|
||||
if err != nil {
|
||||
return Added{}, err
|
||||
}
|
||||
if !st.Mode().IsRegular() {
|
||||
return Added{}, fmt.Errorf("%s is not a file", src)
|
||||
}
|
||||
if strings.HasSuffix(strings.ToLower(e.File), ".gguf") {
|
||||
if err := isGGUF(in, src); err != nil {
|
||||
return Added{}, err
|
||||
}
|
||||
}
|
||||
|
||||
final := e.Path(root)
|
||||
if _, err := os.Stat(final); err == nil {
|
||||
return Added{}, fmt.Errorf("%s/%s/%s: %w", e.Model, e.Version, e.File, ErrExists)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(final), 0o755); err != nil { //nolint:gosec // served to every node
|
||||
return Added{}, err
|
||||
}
|
||||
|
||||
part := final + partSuffix
|
||||
// Truncated rather than refused when there: a .part is what an import cut
|
||||
// short left, and the next attempt is the one that should finish it.
|
||||
out, err := os.OpenFile(part, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) //nolint:gosec // inside the mirror root
|
||||
if err != nil {
|
||||
return Added{}, err
|
||||
}
|
||||
// Hashed on the way through: a forty gigabyte file is read once, not
|
||||
// copied and then read again.
|
||||
h := sha256.New()
|
||||
written, err := copyWithProgress(io.MultiWriter(out, h), ctxReader{ctx, in}, st.Size(), progress)
|
||||
closeErr := out.Close()
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err == nil && written != st.Size() {
|
||||
err = fmt.Errorf("%s: copied %d bytes of %d; it changed while being read", src, written, st.Size())
|
||||
}
|
||||
sum := hex.EncodeToString(h.Sum(nil))
|
||||
if err == nil && wantSHA256 != "" && !strings.EqualFold(sum, wantSHA256) {
|
||||
err = fmt.Errorf("%s is damaged: it should hash to sha256 %s and hashes to %s; "+
|
||||
"nothing was added", src, wantSHA256, sum)
|
||||
}
|
||||
if err == nil {
|
||||
err = writeChecksum(final, sum)
|
||||
}
|
||||
if err == nil {
|
||||
err = writeSource(final, source)
|
||||
}
|
||||
if err == nil {
|
||||
err = os.Rename(part, final)
|
||||
}
|
||||
if err != nil {
|
||||
_ = os.Remove(part)
|
||||
return Added{}, err
|
||||
}
|
||||
e.Bytes = written
|
||||
return Added{Entry: e, SHA256: sum}, nil
|
||||
}
|
||||
|
||||
// ggufMagic opens every GGUF file.
|
||||
const ggufMagic = "GGUF"
|
||||
|
||||
// isGGUF checks a file named .gguf is one, and leaves it read from the start.
|
||||
// A file that is not — an HTML error page saved under the name, Ollama's
|
||||
// newer safetensors-based layers — would otherwise reach every node and fail
|
||||
// there, in llama.cpp's words.
|
||||
func isGGUF(f *os.File, name string) error {
|
||||
head := make([]byte, len(ggufMagic))
|
||||
if _, err := io.ReadFull(f, head); err != nil || string(head) != ggufMagic {
|
||||
return fmt.Errorf("%s is not a GGUF file: it does not begin with %q", name, ggufMagic)
|
||||
}
|
||||
_, err := f.Seek(0, io.SeekStart)
|
||||
return err
|
||||
}
|
||||
|
||||
// ctxReader stops a copy when its context ends: an operator's ^C on a long
|
||||
// import leaves no partial file behind.
|
||||
type ctxReader struct {
|
||||
ctx context.Context
|
||||
r io.Reader
|
||||
}
|
||||
|
||||
func (c ctxReader) Read(p []byte) (int, error) {
|
||||
if err := c.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return c.r.Read(p)
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
package mirror
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Ollama keeps a model as a manifest and the blobs it names. The manifest is
|
||||
// at manifests/<host>/<namespace>/<model>/<tag> under its models directory —
|
||||
// registry.ollama.ai/library/llama3.2/3b for "llama3.2:3b" — and each layer
|
||||
// is a file blobs/sha256-<hex>, named by its own digest. The layer whose media
|
||||
// type is application/vnd.ollama.image.model is the weights as a GGUF, which
|
||||
// is what llama.cpp runs; a vision model has its projector in a second GGUF.
|
||||
//
|
||||
// What an operator already has in Ollama is weights they chose and have
|
||||
// tried. Importing them puts that file in the mirror, checked against the
|
||||
// digest Ollama named it by, rather than downloading it again from somewhere
|
||||
// that may not be the same file.
|
||||
|
||||
// Ollama's defaults for the parts of a name left out.
|
||||
const (
|
||||
ollamaHost = "registry.ollama.ai"
|
||||
ollamaNamespace = "library"
|
||||
ollamaTag = "latest"
|
||||
)
|
||||
|
||||
// The layer media types an import reads, or says it leaves.
|
||||
const (
|
||||
ollamaModelLayer = "application/vnd.ollama.image.model"
|
||||
ollamaProjectorLayer = "application/vnd.ollama.image.projector"
|
||||
ollamaParamsLayer = "application/vnd.ollama.image.params"
|
||||
ollamaTensorLayer = "application/vnd.ollama.image.tensor"
|
||||
)
|
||||
|
||||
// OllamaName is a model as Ollama names it.
|
||||
type OllamaName struct {
|
||||
Host, Namespace, Model, Tag string
|
||||
}
|
||||
|
||||
// ParseOllamaName reads a name as Ollama does: model, namespace/model or
|
||||
// host/namespace/model, with an optional :tag, and its defaults for whatever
|
||||
// is left out — "llama3.2" is registry.ollama.ai/library/llama3.2:latest.
|
||||
func ParseOllamaName(s string) (OllamaName, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
var n OllamaName
|
||||
// A digest pins a name in Ollama; the manifest on disk is by tag.
|
||||
s, _, _ = strings.Cut(s, "@")
|
||||
// "/" cannot be in a tag, so a colon after the last slash is the tag's.
|
||||
if i := strings.LastIndex(s, ":"); i > strings.LastIndex(s, "/") {
|
||||
s, n.Tag = s[:i], s[i+1:]
|
||||
}
|
||||
parts := strings.Split(s, "/")
|
||||
switch len(parts) {
|
||||
case 1:
|
||||
n.Model = parts[0]
|
||||
case 2:
|
||||
n.Namespace, n.Model = parts[0], parts[1]
|
||||
case 3:
|
||||
n.Host, n.Namespace, n.Model = parts[0], parts[1], parts[2]
|
||||
default:
|
||||
return OllamaName{}, fmt.Errorf("%q is not an Ollama model name: model[:tag], namespace/model[:tag] "+
|
||||
"or host/namespace/model[:tag]", s)
|
||||
}
|
||||
if n.Host == "" {
|
||||
n.Host = ollamaHost
|
||||
}
|
||||
if n.Namespace == "" {
|
||||
n.Namespace = ollamaNamespace
|
||||
}
|
||||
if n.Tag == "" {
|
||||
n.Tag = ollamaTag
|
||||
}
|
||||
for _, p := range []string{n.Host, n.Namespace, n.Model, n.Tag} {
|
||||
// Each part is a directory of Ollama's; one that is not a single path
|
||||
// element would read a manifest from somewhere else.
|
||||
if p == "" || p == "." || p == ".." || strings.ContainsAny(p, `\`+"\x00") {
|
||||
return OllamaName{}, fmt.Errorf("%q is not an Ollama model name", s)
|
||||
}
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// String is the name the way Ollama shows it: without the parts that are its
|
||||
// defaults.
|
||||
func (n OllamaName) String() string {
|
||||
name := n.Model + ":" + n.Tag
|
||||
switch {
|
||||
case !strings.EqualFold(n.Host, ollamaHost):
|
||||
return n.Host + "/" + n.Namespace + "/" + name
|
||||
case !strings.EqualFold(n.Namespace, ollamaNamespace):
|
||||
return n.Namespace + "/" + name
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func (n OllamaName) manifestPath(dir string) string {
|
||||
return filepath.Join(dir, "manifests", n.Host, n.Namespace, n.Model, n.Tag)
|
||||
}
|
||||
|
||||
// OllamaDir finds Ollama's models directory: the one given (or its models
|
||||
// subdirectory, for a ~/.ollama given whole), else OLLAMA_MODELS, else the
|
||||
// places Ollama keeps it — this user's ~/.ollama/models, and on Linux the
|
||||
// home of the ollama user its installer makes.
|
||||
func OllamaDir(given string) (string, error) {
|
||||
var candidates []string
|
||||
if given != "" {
|
||||
candidates = []string{given, filepath.Join(given, "models")}
|
||||
} else {
|
||||
if env := os.Getenv("OLLAMA_MODELS"); env != "" {
|
||||
candidates = append(candidates, env)
|
||||
}
|
||||
if home, err := os.UserHomeDir(); err == nil {
|
||||
candidates = append(candidates, filepath.Join(home, ".ollama", "models"))
|
||||
}
|
||||
candidates = append(candidates,
|
||||
"/usr/share/ollama/.ollama/models", // Ollama's Linux installer
|
||||
"/var/lib/ollama/.ollama/models", // some distributions' packages
|
||||
)
|
||||
}
|
||||
for _, c := range candidates {
|
||||
if st, err := os.Stat(filepath.Join(c, "manifests")); err == nil && st.IsDir() {
|
||||
return c, nil
|
||||
}
|
||||
}
|
||||
if given != "" {
|
||||
return "", fmt.Errorf("%s holds no Ollama models: there is no manifests directory in it or in its models", given)
|
||||
}
|
||||
return "", fmt.Errorf("no Ollama models found in %s; name the directory with --ollama-dir",
|
||||
strings.Join(candidates, ", "))
|
||||
}
|
||||
|
||||
// ListOllama lists the models in an Ollama models directory.
|
||||
func ListOllama(dir string) ([]OllamaName, error) {
|
||||
root := filepath.Join(dir, "manifests")
|
||||
var out []OllamaName
|
||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() {
|
||||
return err
|
||||
}
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
parts := strings.Split(rel, string(filepath.Separator))
|
||||
if len(parts) != 4 || strings.HasPrefix(parts[3], ".") {
|
||||
return nil // not a manifest: something else left in Ollama's directory
|
||||
}
|
||||
out = append(out, OllamaName{Host: parts[0], Namespace: parts[1], Model: parts[2], Tag: parts[3]})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading Ollama's models in %s: %w", dir, err)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].String() < out[j].String() })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// OllamaLayer is one layer of an Ollama manifest.
|
||||
type OllamaLayer struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
Digest string `json:"digest"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
// ollamaManifest is the manifest's shape: Docker's image manifest, with
|
||||
// Ollama's media types for the layers.
|
||||
type ollamaManifest struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
Layers []OllamaLayer `json:"layers"`
|
||||
}
|
||||
|
||||
// OllamaModel is what an import takes from one of Ollama's models.
|
||||
type OllamaModel struct {
|
||||
Name OllamaName
|
||||
// Model is the weights, a GGUF.
|
||||
Model OllamaLayer
|
||||
// Projectors are a vision model's image encoders, GGUFs beside it.
|
||||
Projectors []OllamaLayer
|
||||
// ContextLength is the num_ctx Ollama was told to run it with, 0 when
|
||||
// it was not: worth carrying over, since llama.cpp's own default differs.
|
||||
ContextLength int
|
||||
// Left says what of the Ollama model an import does not carry.
|
||||
Left []string
|
||||
}
|
||||
|
||||
var digestPattern = regexp.MustCompile(`^sha256:([0-9a-f]{64})$`)
|
||||
|
||||
// ollamaBlob is where a layer's file is, and the sha256 its name promises.
|
||||
func ollamaBlob(dir, digest string) (path, sum string, err error) {
|
||||
m := digestPattern.FindStringSubmatch(digest)
|
||||
if m == nil {
|
||||
return "", "", fmt.Errorf("the manifest names a layer by %q, which is not a sha256 digest", digest)
|
||||
}
|
||||
return filepath.Join(dir, "blobs", "sha256-"+m[1]), m[1], nil
|
||||
}
|
||||
|
||||
// ReadOllama reads one model's manifest and the layers an import takes.
|
||||
func ReadOllama(dir string, name OllamaName) (OllamaModel, error) {
|
||||
b, err := os.ReadFile(name.manifestPath(dir)) //nolint:gosec // a manifest under Ollama's directory
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return OllamaModel{}, fmt.Errorf("no %s among the Ollama models in %s; `llmgw mirror import-ollama` with no name lists them",
|
||||
name, dir)
|
||||
}
|
||||
if err != nil {
|
||||
return OllamaModel{}, err
|
||||
}
|
||||
var mf ollamaManifest
|
||||
if err := json.Unmarshal(b, &mf); err != nil {
|
||||
return OllamaModel{}, fmt.Errorf("%s's manifest cannot be read: %w", name, err)
|
||||
}
|
||||
|
||||
m := OllamaModel{Name: name}
|
||||
var models []OllamaLayer
|
||||
for _, l := range mf.Layers {
|
||||
switch l.MediaType {
|
||||
case ollamaModelLayer:
|
||||
models = append(models, l)
|
||||
case ollamaProjectorLayer:
|
||||
m.Projectors = append(m.Projectors, l)
|
||||
case ollamaTensorLayer:
|
||||
return OllamaModel{}, fmt.Errorf("%s is kept by Ollama as separate tensors rather than a GGUF; "+
|
||||
"llama.cpp cannot run that form: mirror the model's GGUF from its publisher instead", name)
|
||||
case ollamaParamsLayer:
|
||||
m.ContextLength = ollamaContext(dir, l)
|
||||
case "application/vnd.ollama.image.template", "application/vnd.ollama.image.prompt":
|
||||
m.Left = append(m.Left, "its prompt template: llama.cpp renders the chat template inside the GGUF")
|
||||
case "application/vnd.ollama.image.system":
|
||||
m.Left = append(m.Left, "its system prompt")
|
||||
case "application/vnd.ollama.image.adapter":
|
||||
m.Left = append(m.Left, "a LoRA adapter: give it to the model as one of its adapters")
|
||||
case "application/vnd.ollama.image.draft":
|
||||
m.Left = append(m.Left, "its draft model for speculative decoding")
|
||||
}
|
||||
}
|
||||
switch len(models) {
|
||||
case 0:
|
||||
return OllamaModel{}, fmt.Errorf("%s has no GGUF layer (%s) to import", name, ollamaModelLayer)
|
||||
case 1:
|
||||
m.Model = models[0]
|
||||
default:
|
||||
return OllamaModel{}, fmt.Errorf("%s is made of %d weight layers, which an import does not join yet; "+
|
||||
"mirror the model's GGUF from its publisher instead", name, len(models))
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// ollamaContext is the num_ctx in a params layer, 0 when there is none or it
|
||||
// cannot be read: a hint, not worth failing an import over.
|
||||
func ollamaContext(dir string, l OllamaLayer) int {
|
||||
path, _, err := ollamaBlob(dir, l.Digest)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
b, err := os.ReadFile(path) //nolint:gosec // a blob under Ollama's directory
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
var p struct {
|
||||
NumCtx int `json:"num_ctx"`
|
||||
}
|
||||
_ = json.Unmarshal(b, &p)
|
||||
return p.NumCtx
|
||||
}
|
||||
|
||||
// OllamaEntries are where an Ollama model's files go in the mirror, by
|
||||
// default: the model's own name, and a version that is its tag with the start
|
||||
// of its digest — a tag such as "latest" moves in Ollama, and a mirror
|
||||
// version must not.
|
||||
func OllamaEntries(m OllamaModel, model, version string) (weights Entry, projectors []Entry) {
|
||||
if model == "" {
|
||||
model = m.Name.Model
|
||||
}
|
||||
_, sum, _ := ollamaBlob("", m.Model.Digest)
|
||||
if version == "" {
|
||||
version = m.Name.Tag + "-" + sum[:min(12, len(sum))]
|
||||
}
|
||||
base := m.Name.Model + "-" + m.Name.Tag
|
||||
weights = Entry{Model: model, Version: version, File: base + ".gguf"}
|
||||
for i := range m.Projectors {
|
||||
// "mmproj" in the name is what llama.cpp's engine knows a projector by,
|
||||
// beside the model in one directory.
|
||||
file := "mmproj-" + base + ".gguf"
|
||||
if i > 0 {
|
||||
file = fmt.Sprintf("mmproj-%s-%d.gguf", base, i+1)
|
||||
}
|
||||
projectors = append(projectors, Entry{Model: model, Version: version, File: file})
|
||||
}
|
||||
return weights, projectors
|
||||
}
|
||||
|
||||
// ImportOllama copies an Ollama model's GGUF, and its projectors, into one
|
||||
// version of the mirror, each checked against the digest Ollama named it by.
|
||||
// progress is told which file it is about.
|
||||
//
|
||||
// All or nothing: a version left holding a projector and no model is one a
|
||||
// node would fetch and fail to start, so the files already copied are taken
|
||||
// out again when a later one fails.
|
||||
func ImportOllama(ctx context.Context, root, dir string, m OllamaModel, model, version string,
|
||||
progress func(file string, done, total int64),
|
||||
) ([]Added, error) {
|
||||
weights, projectors := OllamaEntries(m, model, version)
|
||||
type job struct {
|
||||
layer OllamaLayer
|
||||
entry Entry
|
||||
}
|
||||
jobs := []job{{m.Model, weights}}
|
||||
for i, l := range m.Projectors {
|
||||
jobs = append(jobs, job{l, projectors[i]})
|
||||
}
|
||||
// Refused before anything is copied, rather than after the first file.
|
||||
if _, err := os.Stat(filepath.Join(root, weights.Model, weights.Version)); err == nil {
|
||||
return nil, fmt.Errorf("%s/%s: %w", weights.Model, weights.Version, ErrExists)
|
||||
}
|
||||
|
||||
source := "ollama://" + m.Name.Host + "/" + m.Name.Namespace + "/" + m.Name.Model + ":" + m.Name.Tag
|
||||
var done []Added
|
||||
for _, j := range jobs {
|
||||
a, err := importLayer(ctx, root, dir, j.layer, j.entry, source, progress)
|
||||
if err != nil {
|
||||
for _, prev := range done {
|
||||
_ = Remove(root, prev.Entry)
|
||||
}
|
||||
return nil, fmt.Errorf("%s: %w", m.Name, err)
|
||||
}
|
||||
done = append(done, a)
|
||||
}
|
||||
return done, nil
|
||||
}
|
||||
|
||||
func importLayer(ctx context.Context, root, dir string, l OllamaLayer, e Entry, source string,
|
||||
progress func(file string, done, total int64),
|
||||
) (Added, error) {
|
||||
src, sum, err := ollamaBlob(dir, l.Digest)
|
||||
if err != nil {
|
||||
return Added{}, err
|
||||
}
|
||||
if _, err := os.Stat(src); err != nil {
|
||||
return Added{}, fmt.Errorf("its layer %s is not in %s: %w", l.Digest, filepath.Join(dir, "blobs"), err)
|
||||
}
|
||||
var report func(int64, int64)
|
||||
if progress != nil {
|
||||
report = func(d, t int64) { progress(e.File, d, t) }
|
||||
}
|
||||
return Import(ctx, root, src, e, sum, source, report)
|
||||
}
|
||||
@@ -0,0 +1,369 @@
|
||||
package mirror
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
)
|
||||
|
||||
// ollamaFixture is an Ollama models directory laid out as Ollama writes it:
|
||||
// blobs/sha256-<hex> named by their content's digest, and a manifest per
|
||||
// model and tag under manifests/<host>/<namespace>/<model>/<tag>.
|
||||
type ollamaFixture struct {
|
||||
t *testing.T
|
||||
dir string
|
||||
}
|
||||
|
||||
func newOllamaFixture(t *testing.T) *ollamaFixture {
|
||||
t.Helper()
|
||||
dir := filepath.Join(t.TempDir(), ".ollama", "models")
|
||||
for _, d := range []string{"blobs", "manifests"} {
|
||||
if err := os.MkdirAll(filepath.Join(dir, d), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return &ollamaFixture{t: t, dir: dir}
|
||||
}
|
||||
|
||||
// blob stores content as Ollama does and returns the layer naming it.
|
||||
func (f *ollamaFixture) blob(mediaType string, content []byte) OllamaLayer {
|
||||
f.t.Helper()
|
||||
sum := sha256.Sum256(content)
|
||||
hexsum := hex.EncodeToString(sum[:])
|
||||
if err := os.WriteFile(filepath.Join(f.dir, "blobs", "sha256-"+hexsum), content, 0o644); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return OllamaLayer{MediaType: mediaType, Digest: "sha256:" + hexsum, Size: int64(len(content))}
|
||||
}
|
||||
|
||||
func (f *ollamaFixture) manifest(name string, layers ...OllamaLayer) {
|
||||
f.t.Helper()
|
||||
n, err := ParseOllamaName(name)
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
config := f.blob("application/vnd.docker.container.image.v1+json",
|
||||
[]byte(`{"model_format":"gguf","model_family":"llama","file_type":"Q4_K_M"}`))
|
||||
b, _ := json.Marshal(map[string]any{
|
||||
"schemaVersion": 2,
|
||||
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
|
||||
"config": config,
|
||||
"layers": layers,
|
||||
})
|
||||
p := n.manifestPath(f.dir)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, b, 0o644); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// gguf is a small file that begins as every GGUF does.
|
||||
func gguf(body string) []byte { return []byte("GGUF\x03\x00\x00\x00" + body) }
|
||||
|
||||
// llama32 is llama3.2:3b as Ollama pulls it: the weights, a template, a
|
||||
// license and parameters.
|
||||
func (f *ollamaFixture) llama32() OllamaLayer {
|
||||
weights := f.blob(ollamaModelLayer, gguf("llama 3.2 3b weights"))
|
||||
f.manifest("llama3.2:3b",
|
||||
weights,
|
||||
f.blob("application/vnd.ollama.image.template", []byte(`<|start_header_id|>{{ .Prompt }}`)),
|
||||
f.blob("application/vnd.ollama.image.license", []byte("LLAMA 3.2 COMMUNITY LICENSE")),
|
||||
f.blob(ollamaParamsLayer, []byte(`{"num_ctx":8192,"stop":["<|eot_id|>"]}`)),
|
||||
)
|
||||
return weights
|
||||
}
|
||||
|
||||
func TestOllamaNamesAreReadAsOllamaReadsThem(t *testing.T) {
|
||||
for in, want := range map[string]OllamaName{
|
||||
"llama3.2": {ollamaHost, ollamaNamespace, "llama3.2", "latest"},
|
||||
"llama3.2:3b": {ollamaHost, ollamaNamespace, "llama3.2", "3b"},
|
||||
"jmorgan/llava:7b-q4": {ollamaHost, "jmorgan", "llava", "7b-q4"},
|
||||
"hf.co/bartowski/Qwen-GGUF:Q4": {"hf.co", "bartowski", "Qwen-GGUF", "Q4"},
|
||||
"localhost:5000/me/m:v1": {"localhost:5000", "me", "m", "v1"},
|
||||
"qwen3:8b@sha256:abcd": {ollamaHost, ollamaNamespace, "qwen3", "8b"},
|
||||
} {
|
||||
got, err := ParseOllamaName(in)
|
||||
if err != nil || got != want {
|
||||
t.Errorf("%q: got %+v, %v; want %+v", in, got, err, want)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"", "a/b/c/d", "../etc:passwd", "x:.."} {
|
||||
if _, err := ParseOllamaName(bad); err == nil {
|
||||
t.Errorf("%q was taken for a name", bad)
|
||||
}
|
||||
}
|
||||
// Shown the way Ollama shows them: without its defaults.
|
||||
for in, want := range map[string]string{
|
||||
"llama3.2": "llama3.2:latest", "jmorgan/llava:7b": "jmorgan/llava:7b",
|
||||
"hf.co/bartowski/Qwen-GGUF:Q4": "hf.co/bartowski/Qwen-GGUF:Q4",
|
||||
} {
|
||||
n, _ := ParseOllamaName(in)
|
||||
if n.String() != want {
|
||||
t.Errorf("%q shows as %q", in, n.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOllamasDirectoryIsFoundWhereOllamaKeepsIt(t *testing.T) {
|
||||
f := newOllamaFixture(t)
|
||||
// Given as the models directory, or as the ~/.ollama it is in.
|
||||
for _, given := range []string{f.dir, filepath.Dir(f.dir)} {
|
||||
if got, err := OllamaDir(given); err != nil || got != f.dir {
|
||||
t.Errorf("%s: got %q, %v", given, got, err)
|
||||
}
|
||||
}
|
||||
t.Setenv("OLLAMA_MODELS", f.dir)
|
||||
if got, err := OllamaDir(""); err != nil || got != f.dir {
|
||||
t.Errorf("OLLAMA_MODELS: got %q, %v", got, err)
|
||||
}
|
||||
if _, err := OllamaDir(t.TempDir()); err == nil {
|
||||
t.Error("a directory with no manifests was taken for Ollama's")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOllamasModelsAreListedFromItsManifests(t *testing.T) {
|
||||
f := newOllamaFixture(t)
|
||||
f.llama32()
|
||||
f.manifest("hf.co/bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", f.blob(ollamaModelLayer, gguf("qwen")))
|
||||
// Ollama's own leftovers are not models.
|
||||
_ = os.WriteFile(filepath.Join(f.dir, "manifests", ".DS_Store"), nil, 0o644)
|
||||
|
||||
got, err := ListOllama(f.dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var names []string
|
||||
for _, n := range got {
|
||||
names = append(names, n.String())
|
||||
}
|
||||
if !slices.Equal(names, []string{"hf.co/bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", "llama3.2:3b"}) {
|
||||
t.Errorf("listed %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOllamaManifestYieldsItsGGUFAndWhatIsLeftBehind(t *testing.T) {
|
||||
f := newOllamaFixture(t)
|
||||
weights := f.llama32()
|
||||
m, err := ReadOllama(f.dir, mustName(t, "llama3.2:3b"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.Model != weights {
|
||||
t.Errorf("model layer %+v, want %+v", m.Model, weights)
|
||||
}
|
||||
// What Ollama ran it with, to carry over as --ctx-size.
|
||||
if m.ContextLength != 8192 {
|
||||
t.Errorf("num_ctx=%d", m.ContextLength)
|
||||
}
|
||||
// The template is Ollama's own language; saying it is left is the honest
|
||||
// part, since llama.cpp will use the GGUF's.
|
||||
if len(m.Left) != 1 || !strings.Contains(m.Left[0], "template") {
|
||||
t.Errorf("left: %v", m.Left)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOllamaModelsLlamaCppCannotRunAreRefused(t *testing.T) {
|
||||
f := newOllamaFixture(t)
|
||||
// Ollama's newer layout keeps some models as separate tensors.
|
||||
f.manifest("gpt-oss:20b", f.blob(ollamaTensorLayer, []byte("tensor")), f.blob(ollamaTensorLayer, []byte("other")))
|
||||
// Two weight layers make a model this import does not join.
|
||||
f.manifest("split:1", f.blob(ollamaModelLayer, gguf("a")), f.blob(ollamaModelLayer, gguf("b")))
|
||||
f.manifest("empty:1", f.blob("application/vnd.ollama.image.license", []byte("MIT")))
|
||||
|
||||
for name, want := range map[string]string{
|
||||
"gpt-oss:20b": "separate tensors", "split:1": "2 weight layers", "empty:1": "no GGUF layer",
|
||||
"absent:1": "no absent:1",
|
||||
} {
|
||||
_, err := ReadOllama(f.dir, mustName(t, name))
|
||||
if err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("%s: expected %q, got %v", name, want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOllamaModelIsImportedIntoTheMirrorWithItsChecksum(t *testing.T) {
|
||||
f := newOllamaFixture(t)
|
||||
weights := f.llama32()
|
||||
root := t.TempDir()
|
||||
m, err := ReadOllama(f.dir, mustName(t, "llama3.2:3b"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var reported []string
|
||||
added, err := ImportOllama(t.Context(), root, f.dir, m, "", "", func(file string, _, _ int64) {
|
||||
reported = append(reported, file)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The tag alone moves in Ollama ("latest" is whatever was pulled last);
|
||||
// the start of the digest is what pins the version.
|
||||
sum := strings.TrimPrefix(weights.Digest, "sha256:")
|
||||
want := Entry{Model: "llama3.2", Version: "3b-" + sum[:12], File: "llama3.2-3b.gguf", Bytes: weights.Size}
|
||||
if len(added) != 1 || added[0].Entry != want {
|
||||
t.Fatalf("added %+v, want %+v", added, want)
|
||||
}
|
||||
if added[0].SHA256 != sum {
|
||||
t.Errorf("sha256 %s, want the digest Ollama named it by, %s", added[0].SHA256, sum)
|
||||
}
|
||||
if len(reported) == 0 || reported[0] != "llama3.2-3b.gguf" {
|
||||
t.Errorf("progress said %v", reported)
|
||||
}
|
||||
|
||||
// What a node is given: the file, with the checksum it checks it against.
|
||||
mf, err := ManifestFor(root, want.Model, want.Version)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(mf.Files) != 1 || mf.Files[0].SHA256 != sum {
|
||||
t.Errorf("manifest %+v", mf)
|
||||
}
|
||||
b, _ := os.ReadFile(want.Path(root))
|
||||
if string(b) != string(gguf("llama 3.2 3b weights")) {
|
||||
t.Error("the mirrored file is not the blob")
|
||||
}
|
||||
if got := readSource(want.Path(root)); got != "ollama://registry.ollama.ai/library/llama3.2:3b" {
|
||||
t.Errorf("source recorded as %q", got)
|
||||
}
|
||||
|
||||
// The same model again is the same version, and the mirror never
|
||||
// replaces one.
|
||||
if _, err := ImportOllama(t.Context(), root, f.dir, m, "", "", nil); !errors.Is(err, ErrExists) {
|
||||
t.Errorf("a second import was not refused: %v", err)
|
||||
}
|
||||
// Under names the operator chose, it goes where they say.
|
||||
added, err = ImportOllama(t.Context(), root, f.dir, m, "llama-3.2-3b-instruct", "2026-10", nil)
|
||||
if err != nil || added[0].Entry.Path(root) != filepath.Join(root, "llama-3.2-3b-instruct", "2026-10", "llama3.2-3b.gguf") {
|
||||
t.Errorf("named import: %+v, %v", added, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAVisionModelsProjectorGoesBesideItForLlamaCpp(t *testing.T) {
|
||||
f := newOllamaFixture(t)
|
||||
f.manifest("llava:7b",
|
||||
f.blob(ollamaModelLayer, gguf("llava weights")),
|
||||
f.blob(ollamaProjectorLayer, gguf("clip projector")),
|
||||
)
|
||||
root := t.TempDir()
|
||||
m, err := ReadOllama(f.dir, mustName(t, "llava:7b"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
added, err := ImportOllama(t.Context(), root, f.dir, m, "", "v1", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(added) != 2 || added[1].Entry.File != "mmproj-llava-7b.gguf" {
|
||||
t.Fatalf("added %+v", added)
|
||||
}
|
||||
|
||||
// The version directory is the model's source, and llama.cpp's engine
|
||||
// finds in it the model to load and the projector to load beside it.
|
||||
dir := filepath.Join(root, "llava", "v1")
|
||||
s := engines.ShardSpec{HTTPPort: 20000, Profile: engines.Profile{Name: "cpu", Engine: "llamacpp", ModelPath: dir}}
|
||||
cmds, err := (&engines.LlamaCpp{}).Commands(s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
args := cmds[0].Args
|
||||
if got := args[slices.Index(args, "--model")+1]; got != filepath.Join(dir, "llava-7b.gguf") {
|
||||
t.Errorf("--model %s", got)
|
||||
}
|
||||
if got := args[slices.Index(args, "--mmproj")+1]; got != filepath.Join(dir, "mmproj-llava-7b.gguf") {
|
||||
t.Errorf("--mmproj %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADamagedOllamaBlobIsNotMirrored(t *testing.T) {
|
||||
f := newOllamaFixture(t)
|
||||
f.manifest("llava:7b",
|
||||
f.blob(ollamaModelLayer, gguf("llava weights")),
|
||||
f.blob(ollamaProjectorLayer, gguf("clip projector")),
|
||||
)
|
||||
m, err := ReadOllama(f.dir, mustName(t, "llava:7b"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A bit gone wrong on Ollama's disk: the projector no longer hashes to
|
||||
// the name it was stored under.
|
||||
path, _, _ := ollamaBlob(f.dir, m.Projectors[0].Digest)
|
||||
if err := os.WriteFile(path, gguf("clip projectoR"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
root := t.TempDir()
|
||||
_, err = ImportOllama(t.Context(), root, f.dir, m, "", "v1", nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "damaged") {
|
||||
t.Fatalf("expected the damage to be found, got %v", err)
|
||||
}
|
||||
// All or nothing: the weights copied before it are taken out again, so
|
||||
// no node is offered a vision model without its projector.
|
||||
if entries, _ := List(root); len(entries) != 0 {
|
||||
t.Errorf("left in the mirror: %+v", entries)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "llava")); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Errorf("the version's directory was left behind: %v", err)
|
||||
}
|
||||
|
||||
// A blob gone altogether says which.
|
||||
_ = os.Remove(path)
|
||||
if _, err := ImportOllama(t.Context(), root, f.dir, m, "", "v1", nil); err == nil ||
|
||||
!strings.Contains(err.Error(), m.Projectors[0].Digest) {
|
||||
t.Errorf("a missing blob: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyAGGUFIsImportedAsOne(t *testing.T) {
|
||||
// An HTML error page saved as a .gguf would reach every node otherwise.
|
||||
src := filepath.Join(t.TempDir(), "model.gguf")
|
||||
_ = os.WriteFile(src, []byte("<html>rate limited</html>"), 0o644)
|
||||
_, err := Import(t.Context(), t.TempDir(), src, Entry{Model: "m", Version: "v", File: "model.gguf"}, "", "file://"+src, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "not a GGUF") {
|
||||
t.Errorf("expected a refusal, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALocalFileIsImportedWithItsChecksum(t *testing.T) {
|
||||
src := filepath.Join(t.TempDir(), "q4_k_m.gguf")
|
||||
content := gguf("converted by hand")
|
||||
_ = os.WriteFile(src, content, 0o644)
|
||||
root := t.TempDir()
|
||||
e := Entry{Model: "qwen", Version: "2026-10-04", File: "q4_k_m.gguf"}
|
||||
|
||||
a, err := Import(t.Context(), root, src, e, "", "file://"+src, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(content)
|
||||
if a.SHA256 != hex.EncodeToString(sum[:]) || readChecksum(e.Path(root)) != a.SHA256 {
|
||||
t.Errorf("checksum %s, recorded %s", a.SHA256, readChecksum(e.Path(root)))
|
||||
}
|
||||
// A checksum the operator was given is checked too.
|
||||
e.Version = "again"
|
||||
if _, err := Import(t.Context(), root, src, e, strings.Repeat("0", 64), "file://"+src, nil); err == nil {
|
||||
t.Error("a file that does not hash to the sum given was imported")
|
||||
}
|
||||
if _, err := os.Stat(e.Path(root) + partSuffix); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Error("a refused import left its partial file")
|
||||
}
|
||||
}
|
||||
|
||||
func mustName(t *testing.T, s string) OllamaName {
|
||||
t.Helper()
|
||||
n, err := ParseOllamaName(s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
Reference in new issue
Block a user