kev-0.5b declared architecture "ggmlc", which llama.cpp does not know. It was fetched, catalogued and placed, and failed on every node — while the answer was in the first kilobytes of the file. internal/gguf reads a GGUF header (v1-v3) up to general.architecture, over HTTP with a Range request, and judges it against the architectures llama.cpp b10991 lists in LLM_ARCH_NAMES (152, copied from the source at that tag); a vision projector is not a model on its own. An unknown build is not refused, only not vouched for, and a header that cannot be fetched does not refuse anything. Checked in three places: the registry listing (from the architecture Hugging Face reports with expand[]=gguf, so one request per repository) shows it and withholds the fetch button; a fetch reads the header and refuses unless "fetch anyway"; and the model service refuses to add such a model, which covers the serve form and ctl model add. Tried on real files: the mirror's qwen2.5-1.5b (qwen2) and bge-m3 (bert), and Qwen2.5-0.5B on Hugging Face through its redirect. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
214 lines
5.5 KiB
Go
214 lines
5.5 KiB
Go
// Package gguf reads what a GGUF file says about itself, from its header.
|
|
//
|
|
// Enough to ask one question before anything is downloaded or served: can the
|
|
// engine load this? A file downloaded, catalogued and placed, and then refused
|
|
// by llama.cpp with "unknown model architecture", costs a download and a
|
|
// replica failing on every node it is tried on — and the answer was in the
|
|
// first few kilobytes all along.
|
|
package gguf
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/binary"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"math"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Header is the part of a GGUF header this package reads.
|
|
type Header struct {
|
|
Version uint32
|
|
Architecture string
|
|
Name string
|
|
}
|
|
|
|
// ErrNotGGUF means the bytes are not a GGUF file.
|
|
var ErrNotGGUF = errors.New("not a GGUF file")
|
|
|
|
// Value types, from the GGUF specification.
|
|
const (
|
|
tUint8 = iota
|
|
tInt8
|
|
tUint16
|
|
tInt16
|
|
tUint32
|
|
tInt32
|
|
tFloat32
|
|
tBool
|
|
tString
|
|
tArray
|
|
tUint64
|
|
tInt64
|
|
tFloat64
|
|
)
|
|
|
|
// Read parses a header up to general.architecture and general.name.
|
|
func Read(r io.Reader) (Header, error) {
|
|
br := bufio.NewReader(r)
|
|
var h Header
|
|
magic := make([]byte, 4)
|
|
if _, err := io.ReadFull(br, magic); err != nil || string(magic) != "GGUF" {
|
|
return h, ErrNotGGUF
|
|
}
|
|
if err := binary.Read(br, binary.LittleEndian, &h.Version); err != nil {
|
|
return h, err
|
|
}
|
|
if h.Version < 1 || h.Version > 3 {
|
|
return h, fmt.Errorf("GGUF version %d is not one this reads", h.Version)
|
|
}
|
|
wide := h.Version >= 2 // v1 used 32-bit counts and lengths
|
|
count := func() (uint64, error) {
|
|
if wide {
|
|
var n uint64
|
|
return n, binary.Read(br, binary.LittleEndian, &n)
|
|
}
|
|
var n uint32
|
|
err := binary.Read(br, binary.LittleEndian, &n)
|
|
return uint64(n), err
|
|
}
|
|
str := func() (string, error) {
|
|
n, err := count()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if n > 1<<20 {
|
|
return "", fmt.Errorf("a %d-byte string in a GGUF header", n)
|
|
}
|
|
b := make([]byte, n)
|
|
_, err = io.ReadFull(br, b)
|
|
return string(b), err
|
|
}
|
|
skip := func(n int64) error {
|
|
_, err := io.CopyN(io.Discard, br, n)
|
|
return err
|
|
}
|
|
var value func(t uint32) (string, error)
|
|
value = func(t uint32) (string, error) {
|
|
switch t {
|
|
case tUint8, tInt8, tBool:
|
|
return "", skip(1)
|
|
case tUint16, tInt16:
|
|
return "", skip(2)
|
|
case tUint32, tInt32, tFloat32:
|
|
return "", skip(4)
|
|
case tUint64, tInt64, tFloat64:
|
|
return "", skip(8)
|
|
case tString:
|
|
return str()
|
|
case tArray:
|
|
var et uint32
|
|
if err := binary.Read(br, binary.LittleEndian, &et); err != nil {
|
|
return "", err
|
|
}
|
|
n, err := count()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if n > math.MaxInt32 {
|
|
return "", fmt.Errorf("a %d-element array in a GGUF header", n)
|
|
}
|
|
for i := uint64(0); i < n; i++ {
|
|
if _, err := value(et); err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
return "", nil
|
|
}
|
|
return "", fmt.Errorf("GGUF value type %d is unknown", t)
|
|
}
|
|
|
|
if _, err := count(); err != nil { // tensors
|
|
return h, err
|
|
}
|
|
kvs, err := count()
|
|
if err != nil {
|
|
return h, err
|
|
}
|
|
for i := uint64(0); i < kvs; i++ {
|
|
key, err := str()
|
|
if err != nil {
|
|
return h, err
|
|
}
|
|
var t uint32
|
|
if err := binary.Read(br, binary.LittleEndian, &t); err != nil {
|
|
return h, err
|
|
}
|
|
v, err := value(t)
|
|
if err != nil {
|
|
return h, err
|
|
}
|
|
switch key {
|
|
case "general.architecture":
|
|
h.Architecture = v
|
|
case "general.name":
|
|
h.Name = v
|
|
}
|
|
if h.Architecture != "" && h.Name != "" {
|
|
break
|
|
}
|
|
}
|
|
if h.Architecture == "" {
|
|
return h, errors.New("the GGUF header names no architecture")
|
|
}
|
|
return h, nil
|
|
}
|
|
|
|
// headerBytes is how much of a file is fetched to read its header. The
|
|
// architecture is among the first keys; the tokenizer's arrays, which can be
|
|
// megabytes, come after it.
|
|
const headerBytes = 8 << 20
|
|
|
|
// FromURL reads a GGUF header over HTTP, fetching only the start of the file.
|
|
func FromURL(ctx context.Context, client *http.Client, url string) (Header, error) {
|
|
if client == nil {
|
|
client = &http.Client{Timeout: 30 * time.Second}
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return Header{}, err
|
|
}
|
|
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", headerBytes-1))
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return Header{}, err
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
|
return Header{}, fmt.Errorf("%s answered %s", url, resp.Status)
|
|
}
|
|
return Read(io.LimitReader(resp.Body, headerBytes))
|
|
}
|
|
|
|
// IsGGUF reports whether a source names a GGUF file, or a directory of them.
|
|
func IsGGUF(source string) bool {
|
|
s := strings.ToLower(strings.TrimSuffix(source, "/"))
|
|
return strings.HasSuffix(s, ".gguf")
|
|
}
|
|
|
|
// Check reads a source's header and judges it against a llama.cpp build.
|
|
//
|
|
// Only a definite answer refuses: a file that is not GGUF, or whose
|
|
// architecture the build cannot load. A header that cannot be fetched — the
|
|
// network, a directory of shards — is not a reason to refuse, so the verdict
|
|
// then says it was not checked.
|
|
func Check(ctx context.Context, url, llamaVersion string) Verdict {
|
|
if strings.HasSuffix(url, "/") {
|
|
return Verdict{Servable: true, Reason: "a set of files; its header was not read"}
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
|
defer cancel()
|
|
h, err := FromURL(ctx, nil, url)
|
|
if errors.Is(err, ErrNotGGUF) {
|
|
return Verdict{Checked: true, Reason: "it is not a GGUF file"}
|
|
}
|
|
if err != nil {
|
|
return Verdict{Servable: true, Reason: "its header could not be read: " + err.Error()}
|
|
}
|
|
return LlamaCpp(llamaVersion, h.Architecture)
|
|
}
|