Refuse weights the nodes' llama.cpp cannot load, before fetching them

kev-0.5b declared architecture "ggmlc", which llama.cpp does not know.
It was fetched, catalogued and placed, and failed on every node — while
the answer was in the first kilobytes of the file.

internal/gguf reads a GGUF header (v1-v3) up to general.architecture,
over HTTP with a Range request, and judges it against the architectures
llama.cpp b10991 lists in LLM_ARCH_NAMES (152, copied from the source at
that tag); a vision projector is not a model on its own. An unknown
build is not refused, only not vouched for, and a header that cannot be
fetched does not refuse anything.

Checked in three places: the registry listing (from the architecture
Hugging Face reports with expand[]=gguf, so one request per repository)
shows it and withholds the fetch button; a fetch reads the header and
refuses unless "fetch anyway"; and the model service refuses to add
such a model, which covers the serve form and ctl model add.

Tried on real files: the mirror's qwen2.5-1.5b (qwen2) and bge-m3
(bert), and Qwen2.5-0.5B on Hugging Face through its redirect.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-28 20:44:36 +02:00
1 parent c609805e4b
commit f13b18149a
12 files changed
+782 -3

No files matched your search

+17
View File
@@ -37,6 +37,23 @@ form asks the same: on demand loads the model when somebody asks for it and
stops it after 15 minutes unused. A model declared in `server.yaml` shows its
mode and says to change it there, because the file is read on every start.
### Weights the nodes cannot load are not fetched
`kev-0.5b` was a GGUF whose architecture, `ggmlc`, llama.cpp does not know. It
was downloaded, catalogued and placed, and failed on every node it was tried on.
The architecture was in the first kilobytes of the file all along. Now:
- The mirror page's registry listing shows each file's architecture and, for
one the nodes' llama.cpp cannot load, says so instead of offering to fetch
it. A vision projector (`mmproj`) is marked as not a model on its own.
- A fetch reads the file's header first — a range request, not a download —
and refuses one that cannot be served, unless "fetch anyway" is ticked.
- Serving from the mirror and `llmgw ctl model add` refuse the same.
The list of architectures is llama.cpp's own (`LLM_ARCH_NAMES`) at the build
llmgw pins, b10991: 152 of them. A different build configured in `engines:` is
not refused, only not checked.
### Fixes
- Usage by team, project or organization, and the spend on the quotas page,
+52
View File
@@ -0,0 +1,52 @@
package cli
import (
"bytes"
"context"
"encoding/binary"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
func ggufAt(t *testing.T, arch string) string {
t.Helper()
var b bytes.Buffer
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
b.WriteString("GGUF")
w(uint32(3))
w(uint64(0))
w(uint64(1))
str("general.architecture")
w(uint32(8))
str(arch)
body := b.Bytes()
srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
http.ServeContent(rw, r, "m.gguf", time.Time{}, bytes.NewReader(body))
}))
t.Cleanup(srv.Close)
return srv.URL + "/m.gguf"
}
func TestAModelTheNodesCannotLoadIsNotCatalogued(t *testing.T) {
check := servableCheck("b10991")
llama := []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}
err := check(context.Background(), catalog.Model{Name: "kev", Source: ggufAt(t, "ggmlc"), Profiles: llama})
if err == nil || !strings.Contains(err.Error(), "ggmlc") {
t.Errorf("a ggmlc model was accepted: %v", err)
}
if err := check(context.Background(), catalog.Model{Name: "q", Source: ggufAt(t, "qwen2"), Profiles: llama}); err != nil {
t.Errorf("a qwen2 model was refused: %v", err)
}
// Not llama.cpp's to judge: vLLM reads a repository, not a GGUF header.
vllm := []catalog.Profile{{Name: "gpu", Engine: "vllm"}}
if err := check(context.Background(), catalog.Model{Name: "v", Source: ggufAt(t, "ggmlc"), Profiles: vllm}); err != nil {
t.Errorf("a vLLM profile was judged by llama.cpp's list: %v", err)
}
}
+27
View File
@@ -5,6 +5,8 @@ import (
"crypto/tls"
"errors"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
"log/slog"
"net"
"net/http"
@@ -150,6 +152,7 @@ func newServerCmd() *cobra.Command {
// rather than accepting a model that vanishes on restart.
if st := rt.Store(); st != nil {
svc := modelsvc.New(cfg.Models, st, admin, log)
svc.SetCheck(servableCheck(cfg.WantedEngines()["llamacpp"]))
// Rebuild the catalog from configuration plus what was
// stored, and put that in force — otherwise models added
// before the last restart would be silently absent until
@@ -535,3 +538,27 @@ func parseEndpointFlag(spec string) (config.Endpoint, error) {
Weight: 1,
}, nil
}
// servableCheck refuses a model whose llama.cpp weights the nodes' build
// cannot load, read from the weights' own header before anything is placed.
func servableCheck(llamaVersion string) func(context.Context, catalog.Model) error {
return func(ctx context.Context, m catalog.Model) error {
if llamaVersion == "" {
return nil
}
usesLlama := false
for _, p := range m.Profiles {
if p.Engine == "llamacpp" {
usesLlama = true
}
}
url, ok := sourcecheck.HTTPSource(m.Source)
if !usesLlama || !ok || !gguf.IsGGUF(url) {
return nil
}
if v := gguf.Check(ctx, url, llamaVersion); !v.Servable {
return fmt.Errorf("%s cannot be served: %s", m.Name, v.Reason)
}
return nil
}
}
+14
View File
@@ -46,6 +46,10 @@ type Service struct {
apply Applier
log *slog.Logger
// check refuses a model that cannot run before it is added; see
// SetCheck.
check func(context.Context, catalog.Model) error
// mu serialises rebuilds. They are rare — a person adding a model — and two
// at once could apply catalogs in either order, leaving the one in force
// disagreeing with the database that outlives it.
@@ -100,6 +104,11 @@ func (s *Service) Add(ctx context.Context, m catalog.Model, by string) error {
if s.Declared(m.Name) {
return fmt.Errorf("model %q is declared in configuration; edit that rather than adding a second definition", m.Name)
}
if s.check != nil {
if err := s.check(ctx, m); err != nil {
return err
}
}
stored, err := s.storedModels(ctx)
if err != nil {
@@ -299,3 +308,8 @@ func (s *Service) storedModels(ctx context.Context) ([]catalog.Model, error) {
}
return out, nil
}
// SetCheck runs before a model is added, and refuses it by returning an error.
// Used to catch weights the engine cannot load, which otherwise are placed and
// then fail on every node they are tried on.
func (s *Service) SetCheck(check func(context.Context, catalog.Model) error) { s.check = check }
+213
View File
@@ -0,0 +1,213 @@
// Package gguf reads what a GGUF file says about itself, from its header.
//
// Enough to ask one question before anything is downloaded or served: can the
// engine load this? A file downloaded, catalogued and placed, and then refused
// by llama.cpp with "unknown model architecture", costs a download and a
// replica failing on every node it is tried on — and the answer was in the
// first few kilobytes all along.
package gguf
import (
"bufio"
"context"
"encoding/binary"
"errors"
"fmt"
"io"
"math"
"net/http"
"strings"
"time"
)
// Header is the part of a GGUF header this package reads.
type Header struct {
Version uint32
Architecture string
Name string
}
// ErrNotGGUF means the bytes are not a GGUF file.
var ErrNotGGUF = errors.New("not a GGUF file")
// Value types, from the GGUF specification.
const (
tUint8 = iota
tInt8
tUint16
tInt16
tUint32
tInt32
tFloat32
tBool
tString
tArray
tUint64
tInt64
tFloat64
)
// Read parses a header up to general.architecture and general.name.
func Read(r io.Reader) (Header, error) {
br := bufio.NewReader(r)
var h Header
magic := make([]byte, 4)
if _, err := io.ReadFull(br, magic); err != nil || string(magic) != "GGUF" {
return h, ErrNotGGUF
}
if err := binary.Read(br, binary.LittleEndian, &h.Version); err != nil {
return h, err
}
if h.Version < 1 || h.Version > 3 {
return h, fmt.Errorf("GGUF version %d is not one this reads", h.Version)
}
wide := h.Version >= 2 // v1 used 32-bit counts and lengths
count := func() (uint64, error) {
if wide {
var n uint64
return n, binary.Read(br, binary.LittleEndian, &n)
}
var n uint32
err := binary.Read(br, binary.LittleEndian, &n)
return uint64(n), err
}
str := func() (string, error) {
n, err := count()
if err != nil {
return "", err
}
if n > 1<<20 {
return "", fmt.Errorf("a %d-byte string in a GGUF header", n)
}
b := make([]byte, n)
_, err = io.ReadFull(br, b)
return string(b), err
}
skip := func(n int64) error {
_, err := io.CopyN(io.Discard, br, n)
return err
}
var value func(t uint32) (string, error)
value = func(t uint32) (string, error) {
switch t {
case tUint8, tInt8, tBool:
return "", skip(1)
case tUint16, tInt16:
return "", skip(2)
case tUint32, tInt32, tFloat32:
return "", skip(4)
case tUint64, tInt64, tFloat64:
return "", skip(8)
case tString:
return str()
case tArray:
var et uint32
if err := binary.Read(br, binary.LittleEndian, &et); err != nil {
return "", err
}
n, err := count()
if err != nil {
return "", err
}
if n > math.MaxInt32 {
return "", fmt.Errorf("a %d-element array in a GGUF header", n)
}
for i := uint64(0); i < n; i++ {
if _, err := value(et); err != nil {
return "", err
}
}
return "", nil
}
return "", fmt.Errorf("GGUF value type %d is unknown", t)
}
if _, err := count(); err != nil { // tensors
return h, err
}
kvs, err := count()
if err != nil {
return h, err
}
for i := uint64(0); i < kvs; i++ {
key, err := str()
if err != nil {
return h, err
}
var t uint32
if err := binary.Read(br, binary.LittleEndian, &t); err != nil {
return h, err
}
v, err := value(t)
if err != nil {
return h, err
}
switch key {
case "general.architecture":
h.Architecture = v
case "general.name":
h.Name = v
}
if h.Architecture != "" && h.Name != "" {
break
}
}
if h.Architecture == "" {
return h, errors.New("the GGUF header names no architecture")
}
return h, nil
}
// headerBytes is how much of a file is fetched to read its header. The
// architecture is among the first keys; the tokenizer's arrays, which can be
// megabytes, come after it.
const headerBytes = 8 << 20
// FromURL reads a GGUF header over HTTP, fetching only the start of the file.
func FromURL(ctx context.Context, client *http.Client, url string) (Header, error) {
if client == nil {
client = &http.Client{Timeout: 30 * time.Second}
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return Header{}, err
}
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", headerBytes-1))
resp, err := client.Do(req)
if err != nil {
return Header{}, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
return Header{}, fmt.Errorf("%s answered %s", url, resp.Status)
}
return Read(io.LimitReader(resp.Body, headerBytes))
}
// IsGGUF reports whether a source names a GGUF file, or a directory of them.
func IsGGUF(source string) bool {
s := strings.ToLower(strings.TrimSuffix(source, "/"))
return strings.HasSuffix(s, ".gguf")
}
// Check reads a source's header and judges it against a llama.cpp build.
//
// Only a definite answer refuses: a file that is not GGUF, or whose
// architecture the build cannot load. A header that cannot be fetched — the
// network, a directory of shards — is not a reason to refuse, so the verdict
// then says it was not checked.
func Check(ctx context.Context, url, llamaVersion string) Verdict {
if strings.HasSuffix(url, "/") {
return Verdict{Servable: true, Reason: "a set of files; its header was not read"}
}
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
defer cancel()
h, err := FromURL(ctx, nil, url)
if errors.Is(err, ErrNotGGUF) {
return Verdict{Checked: true, Reason: "it is not a GGUF file"}
}
if err != nil {
return Verdict{Servable: true, Reason: "its header could not be read: " + err.Error()}
}
return LlamaCpp(llamaVersion, h.Architecture)
}
+94
View File
@@ -0,0 +1,94 @@
package gguf
import (
"bytes"
"context"
"encoding/binary"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
// header builds a GGUF v3 header: an array before the architecture, the way
// real files put tokenizer data, so skipping values is exercised.
func header(arch string) []byte {
var b bytes.Buffer
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
b.WriteString("GGUF")
w(uint32(3))
w(uint64(0)) // tensors
w(uint64(4)) // kvs
str("general.alignment")
w(uint32(tUint32))
w(uint32(32))
str("some.list")
w(uint32(tArray))
w(uint32(tString))
w(uint64(2))
str("a")
str("bc")
str("general.architecture")
w(uint32(tString))
str(arch)
str("general.name")
w(uint32(tString))
str("A Model")
return b.Bytes()
}
func TestTheArchitectureIsReadFromTheHeader(t *testing.T) {
h, err := Read(bytes.NewReader(header("ggmlc")))
if err != nil {
t.Fatal(err)
}
if h.Architecture != "ggmlc" || h.Name != "A Model" || h.Version != 3 {
t.Errorf("header = %+v", h)
}
}
func TestSomethingElseIsNotAGGUF(t *testing.T) {
if _, err := Read(strings.NewReader("<html>not found</html>")); !errors.Is(err, ErrNotGGUF) {
t.Errorf("an HTML page read as %v", err)
}
}
func TestOnlyTheStartOfARemoteFileIsFetched(t *testing.T) {
body := append(header("qwen2"), make([]byte, 1<<20)...)
var rng string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rng = r.Header.Get("Range")
http.ServeContent(w, r, "m.gguf", timeZero, bytes.NewReader(body))
}))
defer srv.Close()
h, err := FromURL(context.Background(), nil, srv.URL+"/m.gguf")
if err != nil || h.Architecture != "qwen2" {
t.Fatalf("%+v, %v", h, err)
}
if !strings.HasPrefix(rng, "bytes=0-") {
t.Errorf("asked for %q rather than the start of the file", rng)
}
}
func TestLlamaCppKnowsWhatItCanLoad(t *testing.T) {
if v := LlamaCpp("b10991", "qwen2"); !v.Servable || !v.Checked {
t.Errorf("qwen2: %+v", v)
}
// The file that started this: downloaded, catalogued, placed, and then
// refused by every node with "unknown model architecture: 'ggmlc'".
if v := LlamaCpp("b10991", "ggmlc"); v.Servable || !strings.Contains(v.Reason, "cannot load") {
t.Errorf("ggmlc: %+v", v)
}
if v := LlamaCpp("b10991", "clip"); v.Servable {
t.Errorf("a vision projector alone: %+v", v)
}
// A build llmgw has no list for is not refused, only not vouched for.
if v := LlamaCpp("b99999", "ggmlc"); !v.Servable || v.Checked {
t.Errorf("an unknown build: %+v", v)
}
}
var timeZero = time.Time{}
+204
View File
@@ -0,0 +1,204 @@
package gguf
// llamaCppArchitectures is every model architecture a llama.cpp build can
// load, by the build llmgw pins (config.DefaultEngines).
//
// Taken from LLM_ARCH_NAMES in src/llama-arch.cpp at that tag, rather than
// discovered from a node: the check has to happen before anything is
// downloaded, on the control plane, which runs no engine. A build other than
// these is not refused, only not vouched for.
var llamaCppArchitectures = map[string]map[string]bool{
"b10991": {
"clip": true,
"llama": true,
"llama4": true,
"deci": true,
"falcon": true,
"grok": true,
"gpt2": true,
"gptj": true,
"gptneox": true,
"mpt": true,
"baichuan": true,
"starcoder": true,
"refact": true,
"bert": true,
"modern-bert": true,
"nomic-bert": true,
"nomic-bert-moe": true,
"neo-bert": true,
"jina-bert-v2": true,
"jina-bert-v3": true,
"eurobert": true,
"bloom": true,
"stablelm": true,
"qwen": true,
"qwen2": true,
"qwen2moe": true,
"qwen2vl": true,
"qwen3": true,
"qwen3moe": true,
"qwen3next": true,
"qwen3vl": true,
"qwen3vlmoe": true,
"qwen35": true,
"qwen35moe": true,
"qwen4exp": true,
"phi2": true,
"phi3": true,
"phimoe": true,
"plamo": true,
"plamo2": true,
"plamo3": true,
"codeshell": true,
"orion": true,
"internlm2": true,
"minicpm": true,
"minicpm3": true,
"gemma": true,
"gemma2": true,
"gemma3": true,
"gemma3n": true,
"gemma4": true,
"gemma4-assistant": true,
"gemma-embedding": true,
"starcoder2": true,
"mamba": true,
"mamba2": true,
"maple": true,
"jamba": true,
"falcon-h1": true,
"xverse": true,
"command-r": true,
"cohere2": true,
"cohere2moe": true,
"dbrx": true,
"olmo": true,
"olmo2": true,
"olmoe": true,
"muse-glimmer": true,
"openelm": true,
"arctic": true,
"deepseek": true,
"deepseek2": true,
"deepseek2-ocr": true,
"deepseek32": true,
"deepseek4": true,
"chatglm": true,
"glm4": true,
"glm4moe": true,
"glm-dsa": true,
"bitnet": true,
"t5": true,
"t5encoder": true,
"jais": true,
"jais2": true,
"nemotron": true,
"nemotron_h": true,
"nemotron_h_moe": true,
"exaone": true,
"exaone4": true,
"exaone-moe": true,
"rwkv6": true,
"rwkv6qwen2": true,
"rwkv7": true,
"arwkv7": true,
"granite": true,
"granitemoe": true,
"granitehybrid": true,
"graniteswitch": true,
"granite_swa": true,
"chameleon": true,
"wavtokenizer-dec": true,
"plm": true,
"bailingmoe": true,
"bailingmoe2": true,
"bailingmoe3": true,
"dots1": true,
"dots3note": true,
"arcee": true,
"afmoe": true,
"laguna": true,
"ernie4_5": true,
"ernie4_5-moe": true,
"hunyuan-moe": true,
"hunyuan-dense": true,
"hunyuan_vl": true,
"hy_v3": true,
"hy_v4": true,
"smollm3": true,
"gpt-oss": true,
"lfm2": true,
"lfm2moe": true,
"dream": true,
"smallthinker": true,
"llada": true,
"llada-moe": true,
"seed_oss": true,
"grovemoe": true,
"apertus": true,
"minimax-01": true,
"minimax-m2": true,
"minimax-m3": true,
"cogvlm": true,
"rnd1": true,
"pangu-embedded": true,
"mistral3": true,
"eagle3": true,
"dflash": true,
"mistral4": true,
"paddleocr": true,
"mimo2": true,
"step35": true,
"spark2_5": true,
"llama-embed": true,
"maincoder": true,
"kimi-linear": true,
"kimi-k3": true,
"talkie": true,
"mellum": true,
"nanbeige": true,
"qwen3tts": true,
"pockettts": true,
"(unknown)": true,
},
}
// notAModel are architectures llama.cpp knows that are not a model on their
// own: a vision projector is loaded beside a language model with --mmproj.
var notAModel = map[string]string{
"clip": "a vision projector, loaded beside a language model rather than served alone",
}
// Verdict is whether a llama.cpp build can serve a file.
type Verdict struct {
Architecture string
// Servable is false only when the build is known not to load it.
Servable bool
// Checked is false when the build is not one whose architectures llmgw
// knows; Servable is then true by default, and Reason says it was not
// checked.
Checked bool
Reason string
}
// LlamaCpp judges an architecture against a llama.cpp build.
func LlamaCpp(version, arch string) Verdict {
v := Verdict{Architecture: arch, Servable: true}
if why, ok := notAModel[arch]; ok {
v.Servable, v.Checked = false, true
v.Reason = "its architecture is \"" + arch + "\", " + why
return v
}
known, ok := llamaCppArchitectures[version]
if !ok {
v.Reason = "llama.cpp " + version + " is not a build whose architectures llmgw knows; not checked"
return v
}
v.Checked = true
if !known[arch] {
v.Servable = false
v.Reason = "its architecture is \"" + arch + "\", which llama.cpp " + version + " cannot load"
}
return v
}
+13 -1
View File
@@ -148,6 +148,10 @@ type Repo struct {
Revision string
Gated Gated
Files []File
// Architecture is the GGUF architecture the registry read from the
// repository's weights, empty for a repository with none. What decides
// whether llama.cpp can load it, known before anything is downloaded.
Architecture string
}
// ShortRevision is the revision as a person would write it.
@@ -169,8 +173,13 @@ func (c *Client) Repo(ctx context.Context, id string) (Repo, error) {
ID string `json:"id"`
SHA string `json:"sha"`
Gated Gated `json:"gated"`
GGUF *struct {
Architecture string `json:"architecture"`
} `json:"gguf"`
}
if err := c.get(ctx, "/api/models/"+id, &meta); err != nil {
// Expanded to what this reads: asking for gguf alone would leave out the
// revision and the gate.
if err := c.get(ctx, "/api/models/"+id+"?expand[]=sha&expand[]=gated&expand[]=gguf", &meta); err != nil {
return Repo{}, err
}
@@ -194,6 +203,9 @@ func (c *Client) Repo(ctx context.Context, id string) (Repo, error) {
}
r := Repo{ID: meta.ID, Revision: meta.SHA, Gated: meta.Gated}
if meta.GGUF != nil {
r.Architecture = meta.GGUF.Architecture
}
if r.ID == "" {
r.ID = id
}
+6
View File
@@ -67,6 +67,9 @@ type AccountPages struct {
personalDefaults *store.Limits
// orgs is set when /org is mounted, so the menu can offer it to managers.
orgs OrgViewer
// llamaVersion is the llama.cpp build nodes install, for judging whether a
// GGUF can be served before it is fetched or catalogued.
llamaVersion string
// priced says whether a budget can ever be spent against.
priced Priced
// registry browses a model registry, or is nil where there is no way out.
@@ -602,3 +605,6 @@ func (p *AccountPages) renderWithCSRF(w http.ResponseWriter, r *http.Request,
p.fillPlaces(r.Context(), user, &d)
render(w, "profile", d)
}
// SetLlamaCppVersion tells the pages which llama.cpp build the nodes run.
func (p *AccountPages) SetLlamaCppVersion(v string) { p.llamaVersion = v }
+38 -1
View File
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"net/http"
"sort"
"strconv"
@@ -41,6 +42,10 @@ type Registry interface {
// repoSet is one thing worth fetching: a file, or every shard of one.
type repoSet struct {
// Verdict is whether the nodes' llama.cpp can load it, from the
// architecture the registry reports; a set that cannot be served is
// shown with the reason and no fetch button.
Verdict gguf.Verdict
hf.FileSet
// Sources are every file in the set, in the order the registry listed them.
// A shard set goes as one action — a fifteen-shard model is not fifteen
@@ -179,6 +184,15 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
model := strings.TrimSpace(r.FormValue("model"))
version := strings.TrimSpace(r.FormValue("version"))
// Read before anything is downloaded: the header says whether the
// nodes can load it, and a file that cannot be served is an hour
// of download and a replica failing on every node.
if r.FormValue("force") == "" {
if v := p.checkSource(ctx, strings.TrimSpace(sources[0])); !v.Servable {
return "", fmt.Errorf("not fetched: %s. Tick \"fetch anyway\" to take it regardless", v.Reason)
}
}
var started int
for i, src := range sources {
src = strings.TrimSpace(src)
@@ -379,7 +393,7 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
continue
}
form := planServe(d.Rows[i].Status,
byVersion[d.Rows[i].Model+"/"+d.Rows[i].Version], got.Base, p.fleet, p.catalog)
byVersion[d.Rows[i].Model+"/"+d.Rows[i].Version], got.Base, p.fleet, p.catalog, func(src string) gguf.Verdict { return p.checkSource(r.Context(), src) })
d.Rows[i].Serve = &form
break
}
@@ -467,6 +481,7 @@ func (p *AccountPages) explore(r *http.Request, d *adminMirrorData) {
// the same branch lands somewhere else.
Version: got.ShortRevision(),
}
rs.Verdict = p.verdictFor(got.Architecture, set.Name)
for _, f := range set.Files {
rs.Sources = append(rs.Sources,
repoSource{URL: got.DownloadURL("", f.Path), Path: f.Path})
@@ -498,3 +513,25 @@ func defaultModelName(repoID string) string {
name = strings.TrimSuffix(name, "-gguf")
return strings.ToLower(name)
}
// verdictFor judges a registry file set by the repository's architecture. A
// vision projector is its own architecture whatever the repository says.
func (p *AccountPages) verdictFor(arch, name string) gguf.Verdict {
if strings.Contains(strings.ToLower(name), "mmproj") {
return gguf.LlamaCpp(p.llamaVersion, "clip")
}
if arch == "" || !gguf.IsGGUF(name) {
return gguf.Verdict{Servable: true}
}
return gguf.LlamaCpp(p.llamaVersion, arch)
}
// checkSource reads a GGUF source's header and judges it; anything else, or a
// header that cannot be read, passes.
func (p *AccountPages) checkSource(ctx context.Context, source string) gguf.Verdict {
url, ok := sourcecheck.HTTPSource(source)
if !ok || !gguf.IsGGUF(url) || p.llamaVersion == "" {
return gguf.Verdict{Servable: true}
}
return gguf.Check(ctx, url, p.llamaVersion)
}
+8 -1
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"strconv"
"strings"
"time"
@@ -60,7 +61,7 @@ type serveForm struct {
// planServe works out what serving this version would mean.
func planServe(st mirror.Status, files []mirror.Status, base string,
fleet Source, cat CatalogAdmin,
fleet Source, cat CatalogAdmin, check func(source string) gguf.Verdict,
) serveForm {
f := serveForm{
Model: st.Model,
@@ -99,6 +100,12 @@ func planServe(st mirror.Status, files []mirror.Status, base string,
return f
case "llamacpp":
f.Profile = "cpu"
if check != nil {
if v := check(f.Source); !v.Servable {
f.Refusal = "llama.cpp on the nodes cannot run this: " + v.Reason
return f
}
}
default:
f.Profile = "gpu"
}
+96
View File
@@ -0,0 +1,96 @@
package web
import (
"bytes"
"encoding/binary"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// ggufServer serves a GGUF whose header names an architecture.
func ggufServer(t *testing.T, arch string) string {
t.Helper()
var b bytes.Buffer
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
b.WriteString("GGUF")
w(uint32(3))
w(uint64(0))
w(uint64(2))
str("general.architecture")
w(uint32(8))
str(arch)
str("general.name")
w(uint32(8))
str("m")
body := b.Bytes()
srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
http.ServeContent(rw, r, "m.gguf", time.Time{}, bytes.NewReader(body))
}))
t.Cleanup(srv.Close)
return srv.URL + "/m.gguf"
}
func mirrorPageLlama(t *testing.T, d MirrorDeps) *http.ServeMux {
t.Helper()
a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}}
p := NewAccountPages(a, nil, nil, "v0", slog.New(slog.NewTextHandler(io.Discard, nil)))
p.SetLlamaCppVersion("b10991")
mux := http.NewServeMux()
p.RegisterAdminMirror(mux, d)
return mux
}
// The file that started this: kev-0.5b, architecture "ggmlc", downloaded,
// catalogued, and refused by llama.cpp on every node it was placed on.
func TestAFileTheNodesCannotLoadIsNotFetched(t *testing.T) {
m := &fakeMirror{}
mux := mirrorPageLlama(t, MirrorDeps{Mirror: m})
src := ggufServer(t, "ggmlc")
body := mirrorPost(t, mux, "/admin/mirror/fetch", url.Values{"csrf": {"tok"},
"source": {src}, "model": {"kev"}, "version": {"v1"}}).Body.String()
if len(m.all) != 0 {
t.Fatal("a file llama.cpp cannot load was sent to the mirror")
}
if !strings.Contains(body, "ggmlc") || !strings.Contains(body, "cannot load") {
t.Errorf("the refusal should say why; got %s", body)
}
// Asked for regardless, it is fetched.
mirrorPost(t, mux, "/admin/mirror/fetch", url.Values{"csrf": {"tok"},
"source": {src}, "model": {"kev"}, "version": {"v1"}, "force": {"1"}})
if len(m.all) != 1 {
t.Error("fetch anyway did not fetch")
}
// And one it can load goes straight through.
m.all = nil
mirrorPost(t, mux, "/admin/mirror/fetch", url.Values{"csrf": {"tok"},
"source": {ggufServer(t, "qwen2")}, "model": {"q"}, "version": {"v1"}})
if len(m.all) != 1 {
t.Error("a qwen2 GGUF was refused")
}
}
func TestTheRegistryListingSaysWhatCannotBeServed(t *testing.T) {
reg := &fakeRegistry{repo: hf.Repo{ID: "someone/kev-gguf", Revision: "abc12345", Architecture: "ggmlc",
Files: []hf.File{{Path: "kev-q4_k_m.gguf", Bytes: 500 << 20}}}}
body := mirrorGet(t, mirrorPageLlama(t, MirrorDeps{Mirror: &fakeMirror{}, Registry: reg}),
"/admin/mirror?q=kev&repo=someone/kev-gguf").Body.String()
if !strings.Contains(body, "cannot be served") {
t.Error("the listing does not say the file cannot be served")
}
if strings.Contains(body, `value="`+"https://huggingface.co") && strings.Contains(body, ">Fetch</button>") {
t.Error("the listing still offers to fetch it")
}
}