Refuse weights the nodes' llama.cpp cannot load, before fetching them

kev-0.5b declared architecture "ggmlc", which llama.cpp does not know.
It was fetched, catalogued and placed, and failed on every node — while
the answer was in the first kilobytes of the file.

internal/gguf reads a GGUF header (v1-v3) up to general.architecture,
over HTTP with a Range request, and judges it against the architectures
llama.cpp b10991 lists in LLM_ARCH_NAMES (152, copied from the source at
that tag); a vision projector is not a model on its own. An unknown
build is not refused, only not vouched for, and a header that cannot be
fetched does not refuse anything.

Checked in three places: the registry listing (from the architecture
Hugging Face reports with expand[]=gguf, so one request per repository)
shows it and withholds the fetch button; a fetch reads the header and
refuses unless "fetch anyway"; and the model service refuses to add
such a model, which covers the serve form and ctl model add.

Tried on real files: the mirror's qwen2.5-1.5b (qwen2) and bge-m3
(bert), and Qwen2.5-0.5B on Hugging Face through its redirect.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-28 20:44:36 +02:00
1 parent c609805e4b
commit f13b18149a
12 files changed
+782 -3

No files matched your search

+52
View File
@@ -0,0 +1,52 @@
package cli
import (
"bytes"
"context"
"encoding/binary"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
func ggufAt(t *testing.T, arch string) string {
t.Helper()
var b bytes.Buffer
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
b.WriteString("GGUF")
w(uint32(3))
w(uint64(0))
w(uint64(1))
str("general.architecture")
w(uint32(8))
str(arch)
body := b.Bytes()
srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
http.ServeContent(rw, r, "m.gguf", time.Time{}, bytes.NewReader(body))
}))
t.Cleanup(srv.Close)
return srv.URL + "/m.gguf"
}
func TestAModelTheNodesCannotLoadIsNotCatalogued(t *testing.T) {
check := servableCheck("b10991")
llama := []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}
err := check(context.Background(), catalog.Model{Name: "kev", Source: ggufAt(t, "ggmlc"), Profiles: llama})
if err == nil || !strings.Contains(err.Error(), "ggmlc") {
t.Errorf("a ggmlc model was accepted: %v", err)
}
if err := check(context.Background(), catalog.Model{Name: "q", Source: ggufAt(t, "qwen2"), Profiles: llama}); err != nil {
t.Errorf("a qwen2 model was refused: %v", err)
}
// Not llama.cpp's to judge: vLLM reads a repository, not a GGUF header.
vllm := []catalog.Profile{{Name: "gpu", Engine: "vllm"}}
if err := check(context.Background(), catalog.Model{Name: "v", Source: ggufAt(t, "ggmlc"), Profiles: vllm}); err != nil {
t.Errorf("a vLLM profile was judged by llama.cpp's list: %v", err)
}
}
+27
View File
@@ -5,6 +5,8 @@ import (
"crypto/tls"
"errors"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
"log/slog"
"net"
"net/http"
@@ -150,6 +152,7 @@ func newServerCmd() *cobra.Command {
// rather than accepting a model that vanishes on restart.
if st := rt.Store(); st != nil {
svc := modelsvc.New(cfg.Models, st, admin, log)
svc.SetCheck(servableCheck(cfg.WantedEngines()["llamacpp"]))
// Rebuild the catalog from configuration plus what was
// stored, and put that in force — otherwise models added
// before the last restart would be silently absent until
@@ -535,3 +538,27 @@ func parseEndpointFlag(spec string) (config.Endpoint, error) {
Weight: 1,
}, nil
}
// servableCheck refuses a model whose llama.cpp weights the nodes' build
// cannot load, read from the weights' own header before anything is placed.
func servableCheck(llamaVersion string) func(context.Context, catalog.Model) error {
return func(ctx context.Context, m catalog.Model) error {
if llamaVersion == "" {
return nil
}
usesLlama := false
for _, p := range m.Profiles {
if p.Engine == "llamacpp" {
usesLlama = true
}
}
url, ok := sourcecheck.HTTPSource(m.Source)
if !usesLlama || !ok || !gguf.IsGGUF(url) {
return nil
}
if v := gguf.Check(ctx, url, llamaVersion); !v.Servable {
return fmt.Errorf("%s cannot be served: %s", m.Name, v.Reason)
}
return nil
}
}
+14
View File
@@ -46,6 +46,10 @@ type Service struct {
apply Applier
log *slog.Logger
// check refuses a model that cannot run before it is added; see
// SetCheck.
check func(context.Context, catalog.Model) error
// mu serialises rebuilds. They are rare — a person adding a model — and two
// at once could apply catalogs in either order, leaving the one in force
// disagreeing with the database that outlives it.
@@ -100,6 +104,11 @@ func (s *Service) Add(ctx context.Context, m catalog.Model, by string) error {
if s.Declared(m.Name) {
return fmt.Errorf("model %q is declared in configuration; edit that rather than adding a second definition", m.Name)
}
if s.check != nil {
if err := s.check(ctx, m); err != nil {
return err
}
}
stored, err := s.storedModels(ctx)
if err != nil {
@@ -299,3 +308,8 @@ func (s *Service) storedModels(ctx context.Context) ([]catalog.Model, error) {
}
return out, nil
}
// SetCheck runs before a model is added, and refuses it by returning an error.
// Used to catch weights the engine cannot load, which otherwise are placed and
// then fail on every node they are tried on.
func (s *Service) SetCheck(check func(context.Context, catalog.Model) error) { s.check = check }
+213
View File
@@ -0,0 +1,213 @@
// Package gguf reads what a GGUF file says about itself, from its header.
//
// Enough to ask one question before anything is downloaded or served: can the
// engine load this? A file downloaded, catalogued and placed, and then refused
// by llama.cpp with "unknown model architecture", costs a download and a
// replica failing on every node it is tried on — and the answer was in the
// first few kilobytes all along.
package gguf
import (
"bufio"
"context"
"encoding/binary"
"errors"
"fmt"
"io"
"math"
"net/http"
"strings"
"time"
)
// Header is the part of a GGUF header this package reads.
type Header struct {
Version uint32
Architecture string
Name string
}
// ErrNotGGUF means the bytes are not a GGUF file.
var ErrNotGGUF = errors.New("not a GGUF file")
// Value types, from the GGUF specification.
const (
tUint8 = iota
tInt8
tUint16
tInt16
tUint32
tInt32
tFloat32
tBool
tString
tArray
tUint64
tInt64
tFloat64
)
// Read parses a header up to general.architecture and general.name.
func Read(r io.Reader) (Header, error) {
br := bufio.NewReader(r)
var h Header
magic := make([]byte, 4)
if _, err := io.ReadFull(br, magic); err != nil || string(magic) != "GGUF" {
return h, ErrNotGGUF
}
if err := binary.Read(br, binary.LittleEndian, &h.Version); err != nil {
return h, err
}
if h.Version < 1 || h.Version > 3 {
return h, fmt.Errorf("GGUF version %d is not one this reads", h.Version)
}
wide := h.Version >= 2 // v1 used 32-bit counts and lengths
count := func() (uint64, error) {
if wide {
var n uint64
return n, binary.Read(br, binary.LittleEndian, &n)
}
var n uint32
err := binary.Read(br, binary.LittleEndian, &n)
return uint64(n), err
}
str := func() (string, error) {
n, err := count()
if err != nil {
return "", err
}
if n > 1<<20 {
return "", fmt.Errorf("a %d-byte string in a GGUF header", n)
}
b := make([]byte, n)
_, err = io.ReadFull(br, b)
return string(b), err
}
skip := func(n int64) error {
_, err := io.CopyN(io.Discard, br, n)
return err
}
var value func(t uint32) (string, error)
value = func(t uint32) (string, error) {
switch t {
case tUint8, tInt8, tBool:
return "", skip(1)
case tUint16, tInt16:
return "", skip(2)
case tUint32, tInt32, tFloat32:
return "", skip(4)
case tUint64, tInt64, tFloat64:
return "", skip(8)
case tString:
return str()
case tArray:
var et uint32
if err := binary.Read(br, binary.LittleEndian, &et); err != nil {
return "", err
}
n, err := count()
if err != nil {
return "", err
}
if n > math.MaxInt32 {
return "", fmt.Errorf("a %d-element array in a GGUF header", n)
}
for i := uint64(0); i < n; i++ {
if _, err := value(et); err != nil {
return "", err
}
}
return "", nil
}
return "", fmt.Errorf("GGUF value type %d is unknown", t)
}
if _, err := count(); err != nil { // tensors
return h, err
}
kvs, err := count()
if err != nil {
return h, err
}
for i := uint64(0); i < kvs; i++ {
key, err := str()
if err != nil {
return h, err
}
var t uint32
if err := binary.Read(br, binary.LittleEndian, &t); err != nil {
return h, err
}
v, err := value(t)
if err != nil {
return h, err
}
switch key {
case "general.architecture":
h.Architecture = v
case "general.name":
h.Name = v
}
if h.Architecture != "" && h.Name != "" {
break
}
}
if h.Architecture == "" {
return h, errors.New("the GGUF header names no architecture")
}
return h, nil
}
// headerBytes is how much of a file is fetched to read its header. The
// architecture is among the first keys; the tokenizer's arrays, which can be
// megabytes, come after it.
const headerBytes = 8 << 20
// FromURL reads a GGUF header over HTTP, fetching only the start of the file.
func FromURL(ctx context.Context, client *http.Client, url string) (Header, error) {
if client == nil {
client = &http.Client{Timeout: 30 * time.Second}
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return Header{}, err
}
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", headerBytes-1))
resp, err := client.Do(req)
if err != nil {
return Header{}, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
return Header{}, fmt.Errorf("%s answered %s", url, resp.Status)
}
return Read(io.LimitReader(resp.Body, headerBytes))
}
// IsGGUF reports whether a source names a GGUF file, or a directory of them.
func IsGGUF(source string) bool {
s := strings.ToLower(strings.TrimSuffix(source, "/"))
return strings.HasSuffix(s, ".gguf")
}
// Check reads a source's header and judges it against a llama.cpp build.
//
// Only a definite answer refuses: a file that is not GGUF, or whose
// architecture the build cannot load. A header that cannot be fetched — the
// network, a directory of shards — is not a reason to refuse, so the verdict
// then says it was not checked.
func Check(ctx context.Context, url, llamaVersion string) Verdict {
if strings.HasSuffix(url, "/") {
return Verdict{Servable: true, Reason: "a set of files; its header was not read"}
}
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
defer cancel()
h, err := FromURL(ctx, nil, url)
if errors.Is(err, ErrNotGGUF) {
return Verdict{Checked: true, Reason: "it is not a GGUF file"}
}
if err != nil {
return Verdict{Servable: true, Reason: "its header could not be read: " + err.Error()}
}
return LlamaCpp(llamaVersion, h.Architecture)
}
+94
View File
@@ -0,0 +1,94 @@
package gguf
import (
"bytes"
"context"
"encoding/binary"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
// header builds a GGUF v3 header: an array before the architecture, the way
// real files put tokenizer data, so skipping values is exercised.
func header(arch string) []byte {
var b bytes.Buffer
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
b.WriteString("GGUF")
w(uint32(3))
w(uint64(0)) // tensors
w(uint64(4)) // kvs
str("general.alignment")
w(uint32(tUint32))
w(uint32(32))
str("some.list")
w(uint32(tArray))
w(uint32(tString))
w(uint64(2))
str("a")
str("bc")
str("general.architecture")
w(uint32(tString))
str(arch)
str("general.name")
w(uint32(tString))
str("A Model")
return b.Bytes()
}
func TestTheArchitectureIsReadFromTheHeader(t *testing.T) {
h, err := Read(bytes.NewReader(header("ggmlc")))
if err != nil {
t.Fatal(err)
}
if h.Architecture != "ggmlc" || h.Name != "A Model" || h.Version != 3 {
t.Errorf("header = %+v", h)
}
}
func TestSomethingElseIsNotAGGUF(t *testing.T) {
if _, err := Read(strings.NewReader("<html>not found</html>")); !errors.Is(err, ErrNotGGUF) {
t.Errorf("an HTML page read as %v", err)
}
}
func TestOnlyTheStartOfARemoteFileIsFetched(t *testing.T) {
body := append(header("qwen2"), make([]byte, 1<<20)...)
var rng string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rng = r.Header.Get("Range")
http.ServeContent(w, r, "m.gguf", timeZero, bytes.NewReader(body))
}))
defer srv.Close()
h, err := FromURL(context.Background(), nil, srv.URL+"/m.gguf")
if err != nil || h.Architecture != "qwen2" {
t.Fatalf("%+v, %v", h, err)
}
if !strings.HasPrefix(rng, "bytes=0-") {
t.Errorf("asked for %q rather than the start of the file", rng)
}
}
func TestLlamaCppKnowsWhatItCanLoad(t *testing.T) {
if v := LlamaCpp("b10991", "qwen2"); !v.Servable || !v.Checked {
t.Errorf("qwen2: %+v", v)
}
// The file that started this: downloaded, catalogued, placed, and then
// refused by every node with "unknown model architecture: 'ggmlc'".
if v := LlamaCpp("b10991", "ggmlc"); v.Servable || !strings.Contains(v.Reason, "cannot load") {
t.Errorf("ggmlc: %+v", v)
}
if v := LlamaCpp("b10991", "clip"); v.Servable {
t.Errorf("a vision projector alone: %+v", v)
}
// A build llmgw has no list for is not refused, only not vouched for.
if v := LlamaCpp("b99999", "ggmlc"); !v.Servable || v.Checked {
t.Errorf("an unknown build: %+v", v)
}
}
var timeZero = time.Time{}
+204
View File
@@ -0,0 +1,204 @@
package gguf
// llamaCppArchitectures is every model architecture a llama.cpp build can
// load, by the build llmgw pins (config.DefaultEngines).
//
// Taken from LLM_ARCH_NAMES in src/llama-arch.cpp at that tag, rather than
// discovered from a node: the check has to happen before anything is
// downloaded, on the control plane, which runs no engine. A build other than
// these is not refused, only not vouched for.
var llamaCppArchitectures = map[string]map[string]bool{
"b10991": {
"clip": true,
"llama": true,
"llama4": true,
"deci": true,
"falcon": true,
"grok": true,
"gpt2": true,
"gptj": true,
"gptneox": true,
"mpt": true,
"baichuan": true,
"starcoder": true,
"refact": true,
"bert": true,
"modern-bert": true,
"nomic-bert": true,
"nomic-bert-moe": true,
"neo-bert": true,
"jina-bert-v2": true,
"jina-bert-v3": true,
"eurobert": true,
"bloom": true,
"stablelm": true,
"qwen": true,
"qwen2": true,
"qwen2moe": true,
"qwen2vl": true,
"qwen3": true,
"qwen3moe": true,
"qwen3next": true,
"qwen3vl": true,
"qwen3vlmoe": true,
"qwen35": true,
"qwen35moe": true,
"qwen4exp": true,
"phi2": true,
"phi3": true,
"phimoe": true,
"plamo": true,
"plamo2": true,
"plamo3": true,
"codeshell": true,
"orion": true,
"internlm2": true,
"minicpm": true,
"minicpm3": true,
"gemma": true,
"gemma2": true,
"gemma3": true,
"gemma3n": true,
"gemma4": true,
"gemma4-assistant": true,
"gemma-embedding": true,
"starcoder2": true,
"mamba": true,
"mamba2": true,
"maple": true,
"jamba": true,
"falcon-h1": true,
"xverse": true,
"command-r": true,
"cohere2": true,
"cohere2moe": true,
"dbrx": true,
"olmo": true,
"olmo2": true,
"olmoe": true,
"muse-glimmer": true,
"openelm": true,
"arctic": true,
"deepseek": true,
"deepseek2": true,
"deepseek2-ocr": true,
"deepseek32": true,
"deepseek4": true,
"chatglm": true,
"glm4": true,
"glm4moe": true,
"glm-dsa": true,
"bitnet": true,
"t5": true,
"t5encoder": true,
"jais": true,
"jais2": true,
"nemotron": true,
"nemotron_h": true,
"nemotron_h_moe": true,
"exaone": true,
"exaone4": true,
"exaone-moe": true,
"rwkv6": true,
"rwkv6qwen2": true,
"rwkv7": true,
"arwkv7": true,
"granite": true,
"granitemoe": true,
"granitehybrid": true,
"graniteswitch": true,
"granite_swa": true,
"chameleon": true,
"wavtokenizer-dec": true,
"plm": true,
"bailingmoe": true,
"bailingmoe2": true,
"bailingmoe3": true,
"dots1": true,
"dots3note": true,
"arcee": true,
"afmoe": true,
"laguna": true,
"ernie4_5": true,
"ernie4_5-moe": true,
"hunyuan-moe": true,
"hunyuan-dense": true,
"hunyuan_vl": true,
"hy_v3": true,
"hy_v4": true,
"smollm3": true,
"gpt-oss": true,
"lfm2": true,
"lfm2moe": true,
"dream": true,
"smallthinker": true,
"llada": true,
"llada-moe": true,
"seed_oss": true,
"grovemoe": true,
"apertus": true,
"minimax-01": true,
"minimax-m2": true,
"minimax-m3": true,
"cogvlm": true,
"rnd1": true,
"pangu-embedded": true,
"mistral3": true,
"eagle3": true,
"dflash": true,
"mistral4": true,
"paddleocr": true,
"mimo2": true,
"step35": true,
"spark2_5": true,
"llama-embed": true,
"maincoder": true,
"kimi-linear": true,
"kimi-k3": true,
"talkie": true,
"mellum": true,
"nanbeige": true,
"qwen3tts": true,
"pockettts": true,
"(unknown)": true,
},
}
// notAModel are architectures llama.cpp knows that are not a model on their
// own: a vision projector is loaded beside a language model with --mmproj.
var notAModel = map[string]string{
"clip": "a vision projector, loaded beside a language model rather than served alone",
}
// Verdict is whether a llama.cpp build can serve a file.
type Verdict struct {
Architecture string
// Servable is false only when the build is known not to load it.
Servable bool
// Checked is false when the build is not one whose architectures llmgw
// knows; Servable is then true by default, and Reason says it was not
// checked.
Checked bool
Reason string
}
// LlamaCpp judges an architecture against a llama.cpp build.
func LlamaCpp(version, arch string) Verdict {
v := Verdict{Architecture: arch, Servable: true}
if why, ok := notAModel[arch]; ok {
v.Servable, v.Checked = false, true
v.Reason = "its architecture is \"" + arch + "\", " + why
return v
}
known, ok := llamaCppArchitectures[version]
if !ok {
v.Reason = "llama.cpp " + version + " is not a build whose architectures llmgw knows; not checked"
return v
}
v.Checked = true
if !known[arch] {
v.Servable = false
v.Reason = "its architecture is \"" + arch + "\", which llama.cpp " + version + " cannot load"
}
return v
}
+13 -1
View File
@@ -148,6 +148,10 @@ type Repo struct {
Revision string
Gated Gated
Files []File
// Architecture is the GGUF architecture the registry read from the
// repository's weights, empty for a repository with none. What decides
// whether llama.cpp can load it, known before anything is downloaded.
Architecture string
}
// ShortRevision is the revision as a person would write it.
@@ -169,8 +173,13 @@ func (c *Client) Repo(ctx context.Context, id string) (Repo, error) {
ID string `json:"id"`
SHA string `json:"sha"`
Gated Gated `json:"gated"`
GGUF *struct {
Architecture string `json:"architecture"`
} `json:"gguf"`
}
if err := c.get(ctx, "/api/models/"+id, &meta); err != nil {
// Expanded to what this reads: asking for gguf alone would leave out the
// revision and the gate.
if err := c.get(ctx, "/api/models/"+id+"?expand[]=sha&expand[]=gated&expand[]=gguf", &meta); err != nil {
return Repo{}, err
}
@@ -194,6 +203,9 @@ func (c *Client) Repo(ctx context.Context, id string) (Repo, error) {
}
r := Repo{ID: meta.ID, Revision: meta.SHA, Gated: meta.Gated}
if meta.GGUF != nil {
r.Architecture = meta.GGUF.Architecture
}
if r.ID == "" {
r.ID = id
}
+6
View File
@@ -67,6 +67,9 @@ type AccountPages struct {
personalDefaults *store.Limits
// orgs is set when /org is mounted, so the menu can offer it to managers.
orgs OrgViewer
// llamaVersion is the llama.cpp build nodes install, for judging whether a
// GGUF can be served before it is fetched or catalogued.
llamaVersion string
// priced says whether a budget can ever be spent against.
priced Priced
// registry browses a model registry, or is nil where there is no way out.
@@ -602,3 +605,6 @@ func (p *AccountPages) renderWithCSRF(w http.ResponseWriter, r *http.Request,
p.fillPlaces(r.Context(), user, &d)
render(w, "profile", d)
}
// SetLlamaCppVersion tells the pages which llama.cpp build the nodes run.
func (p *AccountPages) SetLlamaCppVersion(v string) { p.llamaVersion = v }
+38 -1
View File
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"net/http"
"sort"
"strconv"
@@ -41,6 +42,10 @@ type Registry interface {
// repoSet is one thing worth fetching: a file, or every shard of one.
type repoSet struct {
// Verdict is whether the nodes' llama.cpp can load it, from the
// architecture the registry reports; a set that cannot be served is
// shown with the reason and no fetch button.
Verdict gguf.Verdict
hf.FileSet
// Sources are every file in the set, in the order the registry listed them.
// A shard set goes as one action — a fifteen-shard model is not fifteen
@@ -179,6 +184,15 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
model := strings.TrimSpace(r.FormValue("model"))
version := strings.TrimSpace(r.FormValue("version"))
// Read before anything is downloaded: the header says whether the
// nodes can load it, and a file that cannot be served is an hour
// of download and a replica failing on every node.
if r.FormValue("force") == "" {
if v := p.checkSource(ctx, strings.TrimSpace(sources[0])); !v.Servable {
return "", fmt.Errorf("not fetched: %s. Tick \"fetch anyway\" to take it regardless", v.Reason)
}
}
var started int
for i, src := range sources {
src = strings.TrimSpace(src)
@@ -379,7 +393,7 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
continue
}
form := planServe(d.Rows[i].Status,
byVersion[d.Rows[i].Model+"/"+d.Rows[i].Version], got.Base, p.fleet, p.catalog)
byVersion[d.Rows[i].Model+"/"+d.Rows[i].Version], got.Base, p.fleet, p.catalog, func(src string) gguf.Verdict { return p.checkSource(r.Context(), src) })
d.Rows[i].Serve = &form
break
}
@@ -467,6 +481,7 @@ func (p *AccountPages) explore(r *http.Request, d *adminMirrorData) {
// the same branch lands somewhere else.
Version: got.ShortRevision(),
}
rs.Verdict = p.verdictFor(got.Architecture, set.Name)
for _, f := range set.Files {
rs.Sources = append(rs.Sources,
repoSource{URL: got.DownloadURL("", f.Path), Path: f.Path})
@@ -498,3 +513,25 @@ func defaultModelName(repoID string) string {
name = strings.TrimSuffix(name, "-gguf")
return strings.ToLower(name)
}
// verdictFor judges a registry file set by the repository's architecture. A
// vision projector is its own architecture whatever the repository says.
func (p *AccountPages) verdictFor(arch, name string) gguf.Verdict {
if strings.Contains(strings.ToLower(name), "mmproj") {
return gguf.LlamaCpp(p.llamaVersion, "clip")
}
if arch == "" || !gguf.IsGGUF(name) {
return gguf.Verdict{Servable: true}
}
return gguf.LlamaCpp(p.llamaVersion, arch)
}
// checkSource reads a GGUF source's header and judges it; anything else, or a
// header that cannot be read, passes.
func (p *AccountPages) checkSource(ctx context.Context, source string) gguf.Verdict {
url, ok := sourcecheck.HTTPSource(source)
if !ok || !gguf.IsGGUF(url) || p.llamaVersion == "" {
return gguf.Verdict{Servable: true}
}
return gguf.Check(ctx, url, p.llamaVersion)
}
+8 -1
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"strconv"
"strings"
"time"
@@ -60,7 +61,7 @@ type serveForm struct {
// planServe works out what serving this version would mean.
func planServe(st mirror.Status, files []mirror.Status, base string,
fleet Source, cat CatalogAdmin,
fleet Source, cat CatalogAdmin, check func(source string) gguf.Verdict,
) serveForm {
f := serveForm{
Model: st.Model,
@@ -99,6 +100,12 @@ func planServe(st mirror.Status, files []mirror.Status, base string,
return f
case "llamacpp":
f.Profile = "cpu"
if check != nil {
if v := check(f.Source); !v.Servable {
f.Refusal = "llama.cpp on the nodes cannot run this: " + v.Reason
return f
}
}
default:
f.Profile = "gpu"
}
+96
View File
@@ -0,0 +1,96 @@
package web
import (
"bytes"
"encoding/binary"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// ggufServer serves a GGUF whose header names an architecture.
func ggufServer(t *testing.T, arch string) string {
t.Helper()
var b bytes.Buffer
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
b.WriteString("GGUF")
w(uint32(3))
w(uint64(0))
w(uint64(2))
str("general.architecture")
w(uint32(8))
str(arch)
str("general.name")
w(uint32(8))
str("m")
body := b.Bytes()
srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
http.ServeContent(rw, r, "m.gguf", time.Time{}, bytes.NewReader(body))
}))
t.Cleanup(srv.Close)
return srv.URL + "/m.gguf"
}
func mirrorPageLlama(t *testing.T, d MirrorDeps) *http.ServeMux {
t.Helper()
a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}}
p := NewAccountPages(a, nil, nil, "v0", slog.New(slog.NewTextHandler(io.Discard, nil)))
p.SetLlamaCppVersion("b10991")
mux := http.NewServeMux()
p.RegisterAdminMirror(mux, d)
return mux
}
// The file that started this: kev-0.5b, architecture "ggmlc", downloaded,
// catalogued, and refused by llama.cpp on every node it was placed on.
func TestAFileTheNodesCannotLoadIsNotFetched(t *testing.T) {
m := &fakeMirror{}
mux := mirrorPageLlama(t, MirrorDeps{Mirror: m})
src := ggufServer(t, "ggmlc")
body := mirrorPost(t, mux, "/admin/mirror/fetch", url.Values{"csrf": {"tok"},
"source": {src}, "model": {"kev"}, "version": {"v1"}}).Body.String()
if len(m.all) != 0 {
t.Fatal("a file llama.cpp cannot load was sent to the mirror")
}
if !strings.Contains(body, "ggmlc") || !strings.Contains(body, "cannot load") {
t.Errorf("the refusal should say why; got %s", body)
}
// Asked for regardless, it is fetched.
mirrorPost(t, mux, "/admin/mirror/fetch", url.Values{"csrf": {"tok"},
"source": {src}, "model": {"kev"}, "version": {"v1"}, "force": {"1"}})
if len(m.all) != 1 {
t.Error("fetch anyway did not fetch")
}
// And one it can load goes straight through.
m.all = nil
mirrorPost(t, mux, "/admin/mirror/fetch", url.Values{"csrf": {"tok"},
"source": {ggufServer(t, "qwen2")}, "model": {"q"}, "version": {"v1"}})
if len(m.all) != 1 {
t.Error("a qwen2 GGUF was refused")
}
}
func TestTheRegistryListingSaysWhatCannotBeServed(t *testing.T) {
reg := &fakeRegistry{repo: hf.Repo{ID: "someone/kev-gguf", Revision: "abc12345", Architecture: "ggmlc",
Files: []hf.File{{Path: "kev-q4_k_m.gguf", Bytes: 500 << 20}}}}
body := mirrorGet(t, mirrorPageLlama(t, MirrorDeps{Mirror: &fakeMirror{}, Registry: reg}),
"/admin/mirror?q=kev&repo=someone/kev-gguf").Body.String()
if !strings.Contains(body, "cannot be served") {
t.Error("the listing does not say the file cannot be served")
}
if strings.Contains(body, `value="`+"https://huggingface.co") && strings.Contains(body, ">Fetch</button>") {
t.Error("the listing still offers to fetch it")
}
}