Refuse weights the nodes' llama.cpp cannot load, before fetching them
kev-0.5b declared architecture "ggmlc", which llama.cpp does not know. It was fetched, catalogued and placed, and failed on every node — while the answer was in the first kilobytes of the file. internal/gguf reads a GGUF header (v1-v3) up to general.architecture, over HTTP with a Range request, and judges it against the architectures llama.cpp b10991 lists in LLM_ARCH_NAMES (152, copied from the source at that tag); a vision projector is not a model on its own. An unknown build is not refused, only not vouched for, and a header that cannot be fetched does not refuse anything. Checked in three places: the registry listing (from the architecture Hugging Face reports with expand[]=gguf, so one request per repository) shows it and withholds the fetch button; a fetch reads the header and refuses unless "fetch anyway"; and the model service refuses to add such a model, which covers the serve form and ctl model add. Tried on real files: the mirror's qwen2.5-1.5b (qwen2) and bge-m3 (bert), and Qwen2.5-0.5B on Hugging Face through its redirect. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
c609805e4b
commit
f13b18149a
12 files changed
+782
-3
No files matched your search
@@ -37,6 +37,23 @@ form asks the same: on demand loads the model when somebody asks for it and
|
||||
stops it after 15 minutes unused. A model declared in `server.yaml` shows its
|
||||
mode and says to change it there, because the file is read on every start.
|
||||
|
||||
### Weights the nodes cannot load are not fetched
|
||||
|
||||
`kev-0.5b` was a GGUF whose architecture, `ggmlc`, llama.cpp does not know. It
|
||||
was downloaded, catalogued and placed, and failed on every node it was tried on.
|
||||
The architecture was in the first kilobytes of the file all along. Now:
|
||||
|
||||
- The mirror page's registry listing shows each file's architecture and, for
|
||||
one the nodes' llama.cpp cannot load, says so instead of offering to fetch
|
||||
it. A vision projector (`mmproj`) is marked as not a model on its own.
|
||||
- A fetch reads the file's header first — a range request, not a download —
|
||||
and refuses one that cannot be served, unless "fetch anyway" is ticked.
|
||||
- Serving from the mirror and `llmgw ctl model add` refuse the same.
|
||||
|
||||
The list of architectures is llama.cpp's own (`LLM_ARCH_NAMES`) at the build
|
||||
llmgw pins, b10991: 152 of them. A different build configured in `engines:` is
|
||||
not refused, only not checked.
|
||||
|
||||
### Fixes
|
||||
|
||||
- Usage by team, project or organization, and the spend on the quotas page,
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
)
|
||||
|
||||
func ggufAt(t *testing.T, arch string) string {
|
||||
t.Helper()
|
||||
var b bytes.Buffer
|
||||
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
|
||||
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
|
||||
b.WriteString("GGUF")
|
||||
w(uint32(3))
|
||||
w(uint64(0))
|
||||
w(uint64(1))
|
||||
str("general.architecture")
|
||||
w(uint32(8))
|
||||
str(arch)
|
||||
body := b.Bytes()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
|
||||
http.ServeContent(rw, r, "m.gguf", time.Time{}, bytes.NewReader(body))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv.URL + "/m.gguf"
|
||||
}
|
||||
|
||||
func TestAModelTheNodesCannotLoadIsNotCatalogued(t *testing.T) {
|
||||
check := servableCheck("b10991")
|
||||
llama := []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}
|
||||
|
||||
err := check(context.Background(), catalog.Model{Name: "kev", Source: ggufAt(t, "ggmlc"), Profiles: llama})
|
||||
if err == nil || !strings.Contains(err.Error(), "ggmlc") {
|
||||
t.Errorf("a ggmlc model was accepted: %v", err)
|
||||
}
|
||||
if err := check(context.Background(), catalog.Model{Name: "q", Source: ggufAt(t, "qwen2"), Profiles: llama}); err != nil {
|
||||
t.Errorf("a qwen2 model was refused: %v", err)
|
||||
}
|
||||
// Not llama.cpp's to judge: vLLM reads a repository, not a GGUF header.
|
||||
vllm := []catalog.Profile{{Name: "gpu", Engine: "vllm"}}
|
||||
if err := check(context.Background(), catalog.Model{Name: "v", Source: ggufAt(t, "ggmlc"), Profiles: vllm}); err != nil {
|
||||
t.Errorf("a vLLM profile was judged by llama.cpp's list: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -150,6 +152,7 @@ func newServerCmd() *cobra.Command {
|
||||
// rather than accepting a model that vanishes on restart.
|
||||
if st := rt.Store(); st != nil {
|
||||
svc := modelsvc.New(cfg.Models, st, admin, log)
|
||||
svc.SetCheck(servableCheck(cfg.WantedEngines()["llamacpp"]))
|
||||
// Rebuild the catalog from configuration plus what was
|
||||
// stored, and put that in force — otherwise models added
|
||||
// before the last restart would be silently absent until
|
||||
@@ -535,3 +538,27 @@ func parseEndpointFlag(spec string) (config.Endpoint, error) {
|
||||
Weight: 1,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// servableCheck refuses a model whose llama.cpp weights the nodes' build
|
||||
// cannot load, read from the weights' own header before anything is placed.
|
||||
func servableCheck(llamaVersion string) func(context.Context, catalog.Model) error {
|
||||
return func(ctx context.Context, m catalog.Model) error {
|
||||
if llamaVersion == "" {
|
||||
return nil
|
||||
}
|
||||
usesLlama := false
|
||||
for _, p := range m.Profiles {
|
||||
if p.Engine == "llamacpp" {
|
||||
usesLlama = true
|
||||
}
|
||||
}
|
||||
url, ok := sourcecheck.HTTPSource(m.Source)
|
||||
if !usesLlama || !ok || !gguf.IsGGUF(url) {
|
||||
return nil
|
||||
}
|
||||
if v := gguf.Check(ctx, url, llamaVersion); !v.Servable {
|
||||
return fmt.Errorf("%s cannot be served: %s", m.Name, v.Reason)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -46,6 +46,10 @@ type Service struct {
|
||||
apply Applier
|
||||
log *slog.Logger
|
||||
|
||||
// check refuses a model that cannot run before it is added; see
|
||||
// SetCheck.
|
||||
check func(context.Context, catalog.Model) error
|
||||
|
||||
// mu serialises rebuilds. They are rare — a person adding a model — and two
|
||||
// at once could apply catalogs in either order, leaving the one in force
|
||||
// disagreeing with the database that outlives it.
|
||||
@@ -100,6 +104,11 @@ func (s *Service) Add(ctx context.Context, m catalog.Model, by string) error {
|
||||
if s.Declared(m.Name) {
|
||||
return fmt.Errorf("model %q is declared in configuration; edit that rather than adding a second definition", m.Name)
|
||||
}
|
||||
if s.check != nil {
|
||||
if err := s.check(ctx, m); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
stored, err := s.storedModels(ctx)
|
||||
if err != nil {
|
||||
@@ -299,3 +308,8 @@ func (s *Service) storedModels(ctx context.Context) ([]catalog.Model, error) {
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SetCheck runs before a model is added, and refuses it by returning an error.
|
||||
// Used to catch weights the engine cannot load, which otherwise are placed and
|
||||
// then fail on every node they are tried on.
|
||||
func (s *Service) SetCheck(check func(context.Context, catalog.Model) error) { s.check = check }
|
||||
@@ -0,0 +1,213 @@
|
||||
// Package gguf reads what a GGUF file says about itself, from its header.
|
||||
//
|
||||
// Enough to ask one question before anything is downloaded or served: can the
|
||||
// engine load this? A file downloaded, catalogued and placed, and then refused
|
||||
// by llama.cpp with "unknown model architecture", costs a download and a
|
||||
// replica failing on every node it is tried on — and the answer was in the
|
||||
// first few kilobytes all along.
|
||||
package gguf
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Header is the part of a GGUF header this package reads.
|
||||
type Header struct {
|
||||
Version uint32
|
||||
Architecture string
|
||||
Name string
|
||||
}
|
||||
|
||||
// ErrNotGGUF means the bytes are not a GGUF file.
|
||||
var ErrNotGGUF = errors.New("not a GGUF file")
|
||||
|
||||
// Value types, from the GGUF specification.
|
||||
const (
|
||||
tUint8 = iota
|
||||
tInt8
|
||||
tUint16
|
||||
tInt16
|
||||
tUint32
|
||||
tInt32
|
||||
tFloat32
|
||||
tBool
|
||||
tString
|
||||
tArray
|
||||
tUint64
|
||||
tInt64
|
||||
tFloat64
|
||||
)
|
||||
|
||||
// Read parses a header up to general.architecture and general.name.
|
||||
func Read(r io.Reader) (Header, error) {
|
||||
br := bufio.NewReader(r)
|
||||
var h Header
|
||||
magic := make([]byte, 4)
|
||||
if _, err := io.ReadFull(br, magic); err != nil || string(magic) != "GGUF" {
|
||||
return h, ErrNotGGUF
|
||||
}
|
||||
if err := binary.Read(br, binary.LittleEndian, &h.Version); err != nil {
|
||||
return h, err
|
||||
}
|
||||
if h.Version < 1 || h.Version > 3 {
|
||||
return h, fmt.Errorf("GGUF version %d is not one this reads", h.Version)
|
||||
}
|
||||
wide := h.Version >= 2 // v1 used 32-bit counts and lengths
|
||||
count := func() (uint64, error) {
|
||||
if wide {
|
||||
var n uint64
|
||||
return n, binary.Read(br, binary.LittleEndian, &n)
|
||||
}
|
||||
var n uint32
|
||||
err := binary.Read(br, binary.LittleEndian, &n)
|
||||
return uint64(n), err
|
||||
}
|
||||
str := func() (string, error) {
|
||||
n, err := count()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if n > 1<<20 {
|
||||
return "", fmt.Errorf("a %d-byte string in a GGUF header", n)
|
||||
}
|
||||
b := make([]byte, n)
|
||||
_, err = io.ReadFull(br, b)
|
||||
return string(b), err
|
||||
}
|
||||
skip := func(n int64) error {
|
||||
_, err := io.CopyN(io.Discard, br, n)
|
||||
return err
|
||||
}
|
||||
var value func(t uint32) (string, error)
|
||||
value = func(t uint32) (string, error) {
|
||||
switch t {
|
||||
case tUint8, tInt8, tBool:
|
||||
return "", skip(1)
|
||||
case tUint16, tInt16:
|
||||
return "", skip(2)
|
||||
case tUint32, tInt32, tFloat32:
|
||||
return "", skip(4)
|
||||
case tUint64, tInt64, tFloat64:
|
||||
return "", skip(8)
|
||||
case tString:
|
||||
return str()
|
||||
case tArray:
|
||||
var et uint32
|
||||
if err := binary.Read(br, binary.LittleEndian, &et); err != nil {
|
||||
return "", err
|
||||
}
|
||||
n, err := count()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if n > math.MaxInt32 {
|
||||
return "", fmt.Errorf("a %d-element array in a GGUF header", n)
|
||||
}
|
||||
for i := uint64(0); i < n; i++ {
|
||||
if _, err := value(et); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
return "", fmt.Errorf("GGUF value type %d is unknown", t)
|
||||
}
|
||||
|
||||
if _, err := count(); err != nil { // tensors
|
||||
return h, err
|
||||
}
|
||||
kvs, err := count()
|
||||
if err != nil {
|
||||
return h, err
|
||||
}
|
||||
for i := uint64(0); i < kvs; i++ {
|
||||
key, err := str()
|
||||
if err != nil {
|
||||
return h, err
|
||||
}
|
||||
var t uint32
|
||||
if err := binary.Read(br, binary.LittleEndian, &t); err != nil {
|
||||
return h, err
|
||||
}
|
||||
v, err := value(t)
|
||||
if err != nil {
|
||||
return h, err
|
||||
}
|
||||
switch key {
|
||||
case "general.architecture":
|
||||
h.Architecture = v
|
||||
case "general.name":
|
||||
h.Name = v
|
||||
}
|
||||
if h.Architecture != "" && h.Name != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
if h.Architecture == "" {
|
||||
return h, errors.New("the GGUF header names no architecture")
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// headerBytes is how much of a file is fetched to read its header. The
|
||||
// architecture is among the first keys; the tokenizer's arrays, which can be
|
||||
// megabytes, come after it.
|
||||
const headerBytes = 8 << 20
|
||||
|
||||
// FromURL reads a GGUF header over HTTP, fetching only the start of the file.
|
||||
func FromURL(ctx context.Context, client *http.Client, url string) (Header, error) {
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 30 * time.Second}
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return Header{}, err
|
||||
}
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", headerBytes-1))
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return Header{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
||||
return Header{}, fmt.Errorf("%s answered %s", url, resp.Status)
|
||||
}
|
||||
return Read(io.LimitReader(resp.Body, headerBytes))
|
||||
}
|
||||
|
||||
// IsGGUF reports whether a source names a GGUF file, or a directory of them.
|
||||
func IsGGUF(source string) bool {
|
||||
s := strings.ToLower(strings.TrimSuffix(source, "/"))
|
||||
return strings.HasSuffix(s, ".gguf")
|
||||
}
|
||||
|
||||
// Check reads a source's header and judges it against a llama.cpp build.
|
||||
//
|
||||
// Only a definite answer refuses: a file that is not GGUF, or whose
|
||||
// architecture the build cannot load. A header that cannot be fetched — the
|
||||
// network, a directory of shards — is not a reason to refuse, so the verdict
|
||||
// then says it was not checked.
|
||||
func Check(ctx context.Context, url, llamaVersion string) Verdict {
|
||||
if strings.HasSuffix(url, "/") {
|
||||
return Verdict{Servable: true, Reason: "a set of files; its header was not read"}
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
defer cancel()
|
||||
h, err := FromURL(ctx, nil, url)
|
||||
if errors.Is(err, ErrNotGGUF) {
|
||||
return Verdict{Checked: true, Reason: "it is not a GGUF file"}
|
||||
}
|
||||
if err != nil {
|
||||
return Verdict{Servable: true, Reason: "its header could not be read: " + err.Error()}
|
||||
}
|
||||
return LlamaCpp(llamaVersion, h.Architecture)
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package gguf
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// header builds a GGUF v3 header: an array before the architecture, the way
|
||||
// real files put tokenizer data, so skipping values is exercised.
|
||||
func header(arch string) []byte {
|
||||
var b bytes.Buffer
|
||||
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
|
||||
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
|
||||
b.WriteString("GGUF")
|
||||
w(uint32(3))
|
||||
w(uint64(0)) // tensors
|
||||
w(uint64(4)) // kvs
|
||||
str("general.alignment")
|
||||
w(uint32(tUint32))
|
||||
w(uint32(32))
|
||||
str("some.list")
|
||||
w(uint32(tArray))
|
||||
w(uint32(tString))
|
||||
w(uint64(2))
|
||||
str("a")
|
||||
str("bc")
|
||||
str("general.architecture")
|
||||
w(uint32(tString))
|
||||
str(arch)
|
||||
str("general.name")
|
||||
w(uint32(tString))
|
||||
str("A Model")
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func TestTheArchitectureIsReadFromTheHeader(t *testing.T) {
|
||||
h, err := Read(bytes.NewReader(header("ggmlc")))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h.Architecture != "ggmlc" || h.Name != "A Model" || h.Version != 3 {
|
||||
t.Errorf("header = %+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSomethingElseIsNotAGGUF(t *testing.T) {
|
||||
if _, err := Read(strings.NewReader("<html>not found</html>")); !errors.Is(err, ErrNotGGUF) {
|
||||
t.Errorf("an HTML page read as %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyTheStartOfARemoteFileIsFetched(t *testing.T) {
|
||||
body := append(header("qwen2"), make([]byte, 1<<20)...)
|
||||
var rng string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
rng = r.Header.Get("Range")
|
||||
http.ServeContent(w, r, "m.gguf", timeZero, bytes.NewReader(body))
|
||||
}))
|
||||
defer srv.Close()
|
||||
h, err := FromURL(context.Background(), nil, srv.URL+"/m.gguf")
|
||||
if err != nil || h.Architecture != "qwen2" {
|
||||
t.Fatalf("%+v, %v", h, err)
|
||||
}
|
||||
if !strings.HasPrefix(rng, "bytes=0-") {
|
||||
t.Errorf("asked for %q rather than the start of the file", rng)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLlamaCppKnowsWhatItCanLoad(t *testing.T) {
|
||||
if v := LlamaCpp("b10991", "qwen2"); !v.Servable || !v.Checked {
|
||||
t.Errorf("qwen2: %+v", v)
|
||||
}
|
||||
// The file that started this: downloaded, catalogued, placed, and then
|
||||
// refused by every node with "unknown model architecture: 'ggmlc'".
|
||||
if v := LlamaCpp("b10991", "ggmlc"); v.Servable || !strings.Contains(v.Reason, "cannot load") {
|
||||
t.Errorf("ggmlc: %+v", v)
|
||||
}
|
||||
if v := LlamaCpp("b10991", "clip"); v.Servable {
|
||||
t.Errorf("a vision projector alone: %+v", v)
|
||||
}
|
||||
// A build llmgw has no list for is not refused, only not vouched for.
|
||||
if v := LlamaCpp("b99999", "ggmlc"); !v.Servable || v.Checked {
|
||||
t.Errorf("an unknown build: %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
var timeZero = time.Time{}
|
||||
@@ -0,0 +1,204 @@
|
||||
package gguf
|
||||
|
||||
// llamaCppArchitectures is every model architecture a llama.cpp build can
|
||||
// load, by the build llmgw pins (config.DefaultEngines).
|
||||
//
|
||||
// Taken from LLM_ARCH_NAMES in src/llama-arch.cpp at that tag, rather than
|
||||
// discovered from a node: the check has to happen before anything is
|
||||
// downloaded, on the control plane, which runs no engine. A build other than
|
||||
// these is not refused, only not vouched for.
|
||||
var llamaCppArchitectures = map[string]map[string]bool{
|
||||
"b10991": {
|
||||
"clip": true,
|
||||
"llama": true,
|
||||
"llama4": true,
|
||||
"deci": true,
|
||||
"falcon": true,
|
||||
"grok": true,
|
||||
"gpt2": true,
|
||||
"gptj": true,
|
||||
"gptneox": true,
|
||||
"mpt": true,
|
||||
"baichuan": true,
|
||||
"starcoder": true,
|
||||
"refact": true,
|
||||
"bert": true,
|
||||
"modern-bert": true,
|
||||
"nomic-bert": true,
|
||||
"nomic-bert-moe": true,
|
||||
"neo-bert": true,
|
||||
"jina-bert-v2": true,
|
||||
"jina-bert-v3": true,
|
||||
"eurobert": true,
|
||||
"bloom": true,
|
||||
"stablelm": true,
|
||||
"qwen": true,
|
||||
"qwen2": true,
|
||||
"qwen2moe": true,
|
||||
"qwen2vl": true,
|
||||
"qwen3": true,
|
||||
"qwen3moe": true,
|
||||
"qwen3next": true,
|
||||
"qwen3vl": true,
|
||||
"qwen3vlmoe": true,
|
||||
"qwen35": true,
|
||||
"qwen35moe": true,
|
||||
"qwen4exp": true,
|
||||
"phi2": true,
|
||||
"phi3": true,
|
||||
"phimoe": true,
|
||||
"plamo": true,
|
||||
"plamo2": true,
|
||||
"plamo3": true,
|
||||
"codeshell": true,
|
||||
"orion": true,
|
||||
"internlm2": true,
|
||||
"minicpm": true,
|
||||
"minicpm3": true,
|
||||
"gemma": true,
|
||||
"gemma2": true,
|
||||
"gemma3": true,
|
||||
"gemma3n": true,
|
||||
"gemma4": true,
|
||||
"gemma4-assistant": true,
|
||||
"gemma-embedding": true,
|
||||
"starcoder2": true,
|
||||
"mamba": true,
|
||||
"mamba2": true,
|
||||
"maple": true,
|
||||
"jamba": true,
|
||||
"falcon-h1": true,
|
||||
"xverse": true,
|
||||
"command-r": true,
|
||||
"cohere2": true,
|
||||
"cohere2moe": true,
|
||||
"dbrx": true,
|
||||
"olmo": true,
|
||||
"olmo2": true,
|
||||
"olmoe": true,
|
||||
"muse-glimmer": true,
|
||||
"openelm": true,
|
||||
"arctic": true,
|
||||
"deepseek": true,
|
||||
"deepseek2": true,
|
||||
"deepseek2-ocr": true,
|
||||
"deepseek32": true,
|
||||
"deepseek4": true,
|
||||
"chatglm": true,
|
||||
"glm4": true,
|
||||
"glm4moe": true,
|
||||
"glm-dsa": true,
|
||||
"bitnet": true,
|
||||
"t5": true,
|
||||
"t5encoder": true,
|
||||
"jais": true,
|
||||
"jais2": true,
|
||||
"nemotron": true,
|
||||
"nemotron_h": true,
|
||||
"nemotron_h_moe": true,
|
||||
"exaone": true,
|
||||
"exaone4": true,
|
||||
"exaone-moe": true,
|
||||
"rwkv6": true,
|
||||
"rwkv6qwen2": true,
|
||||
"rwkv7": true,
|
||||
"arwkv7": true,
|
||||
"granite": true,
|
||||
"granitemoe": true,
|
||||
"granitehybrid": true,
|
||||
"graniteswitch": true,
|
||||
"granite_swa": true,
|
||||
"chameleon": true,
|
||||
"wavtokenizer-dec": true,
|
||||
"plm": true,
|
||||
"bailingmoe": true,
|
||||
"bailingmoe2": true,
|
||||
"bailingmoe3": true,
|
||||
"dots1": true,
|
||||
"dots3note": true,
|
||||
"arcee": true,
|
||||
"afmoe": true,
|
||||
"laguna": true,
|
||||
"ernie4_5": true,
|
||||
"ernie4_5-moe": true,
|
||||
"hunyuan-moe": true,
|
||||
"hunyuan-dense": true,
|
||||
"hunyuan_vl": true,
|
||||
"hy_v3": true,
|
||||
"hy_v4": true,
|
||||
"smollm3": true,
|
||||
"gpt-oss": true,
|
||||
"lfm2": true,
|
||||
"lfm2moe": true,
|
||||
"dream": true,
|
||||
"smallthinker": true,
|
||||
"llada": true,
|
||||
"llada-moe": true,
|
||||
"seed_oss": true,
|
||||
"grovemoe": true,
|
||||
"apertus": true,
|
||||
"minimax-01": true,
|
||||
"minimax-m2": true,
|
||||
"minimax-m3": true,
|
||||
"cogvlm": true,
|
||||
"rnd1": true,
|
||||
"pangu-embedded": true,
|
||||
"mistral3": true,
|
||||
"eagle3": true,
|
||||
"dflash": true,
|
||||
"mistral4": true,
|
||||
"paddleocr": true,
|
||||
"mimo2": true,
|
||||
"step35": true,
|
||||
"spark2_5": true,
|
||||
"llama-embed": true,
|
||||
"maincoder": true,
|
||||
"kimi-linear": true,
|
||||
"kimi-k3": true,
|
||||
"talkie": true,
|
||||
"mellum": true,
|
||||
"nanbeige": true,
|
||||
"qwen3tts": true,
|
||||
"pockettts": true,
|
||||
"(unknown)": true,
|
||||
},
|
||||
}
|
||||
|
||||
// notAModel are architectures llama.cpp knows that are not a model on their
|
||||
// own: a vision projector is loaded beside a language model with --mmproj.
|
||||
var notAModel = map[string]string{
|
||||
"clip": "a vision projector, loaded beside a language model rather than served alone",
|
||||
}
|
||||
|
||||
// Verdict is whether a llama.cpp build can serve a file.
|
||||
type Verdict struct {
|
||||
Architecture string
|
||||
// Servable is false only when the build is known not to load it.
|
||||
Servable bool
|
||||
// Checked is false when the build is not one whose architectures llmgw
|
||||
// knows; Servable is then true by default, and Reason says it was not
|
||||
// checked.
|
||||
Checked bool
|
||||
Reason string
|
||||
}
|
||||
|
||||
// LlamaCpp judges an architecture against a llama.cpp build.
|
||||
func LlamaCpp(version, arch string) Verdict {
|
||||
v := Verdict{Architecture: arch, Servable: true}
|
||||
if why, ok := notAModel[arch]; ok {
|
||||
v.Servable, v.Checked = false, true
|
||||
v.Reason = "its architecture is \"" + arch + "\", " + why
|
||||
return v
|
||||
}
|
||||
known, ok := llamaCppArchitectures[version]
|
||||
if !ok {
|
||||
v.Reason = "llama.cpp " + version + " is not a build whose architectures llmgw knows; not checked"
|
||||
return v
|
||||
}
|
||||
v.Checked = true
|
||||
if !known[arch] {
|
||||
v.Servable = false
|
||||
v.Reason = "its architecture is \"" + arch + "\", which llama.cpp " + version + " cannot load"
|
||||
}
|
||||
return v
|
||||
}
|
||||
+13
-1
@@ -148,6 +148,10 @@ type Repo struct {
|
||||
Revision string
|
||||
Gated Gated
|
||||
Files []File
|
||||
// Architecture is the GGUF architecture the registry read from the
|
||||
// repository's weights, empty for a repository with none. What decides
|
||||
// whether llama.cpp can load it, known before anything is downloaded.
|
||||
Architecture string
|
||||
}
|
||||
|
||||
// ShortRevision is the revision as a person would write it.
|
||||
@@ -169,8 +173,13 @@ func (c *Client) Repo(ctx context.Context, id string) (Repo, error) {
|
||||
ID string `json:"id"`
|
||||
SHA string `json:"sha"`
|
||||
Gated Gated `json:"gated"`
|
||||
GGUF *struct {
|
||||
Architecture string `json:"architecture"`
|
||||
} `json:"gguf"`
|
||||
}
|
||||
if err := c.get(ctx, "/api/models/"+id, &meta); err != nil {
|
||||
// Expanded to what this reads: asking for gguf alone would leave out the
|
||||
// revision and the gate.
|
||||
if err := c.get(ctx, "/api/models/"+id+"?expand[]=sha&expand[]=gated&expand[]=gguf", &meta); err != nil {
|
||||
return Repo{}, err
|
||||
}
|
||||
|
||||
@@ -194,6 +203,9 @@ func (c *Client) Repo(ctx context.Context, id string) (Repo, error) {
|
||||
}
|
||||
|
||||
r := Repo{ID: meta.ID, Revision: meta.SHA, Gated: meta.Gated}
|
||||
if meta.GGUF != nil {
|
||||
r.Architecture = meta.GGUF.Architecture
|
||||
}
|
||||
if r.ID == "" {
|
||||
r.ID = id
|
||||
}
|
||||
|
||||
@@ -67,6 +67,9 @@ type AccountPages struct {
|
||||
personalDefaults *store.Limits
|
||||
// orgs is set when /org is mounted, so the menu can offer it to managers.
|
||||
orgs OrgViewer
|
||||
// llamaVersion is the llama.cpp build nodes install, for judging whether a
|
||||
// GGUF can be served before it is fetched or catalogued.
|
||||
llamaVersion string
|
||||
// priced says whether a budget can ever be spent against.
|
||||
priced Priced
|
||||
// registry browses a model registry, or is nil where there is no way out.
|
||||
@@ -602,3 +605,6 @@ func (p *AccountPages) renderWithCSRF(w http.ResponseWriter, r *http.Request,
|
||||
p.fillPlaces(r.Context(), user, &d)
|
||||
render(w, "profile", d)
|
||||
}
|
||||
|
||||
// SetLlamaCppVersion tells the pages which llama.cpp build the nodes run.
|
||||
func (p *AccountPages) SetLlamaCppVersion(v string) { p.llamaVersion = v }
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -41,6 +42,10 @@ type Registry interface {
|
||||
|
||||
// repoSet is one thing worth fetching: a file, or every shard of one.
|
||||
type repoSet struct {
|
||||
// Verdict is whether the nodes' llama.cpp can load it, from the
|
||||
// architecture the registry reports; a set that cannot be served is
|
||||
// shown with the reason and no fetch button.
|
||||
Verdict gguf.Verdict
|
||||
hf.FileSet
|
||||
// Sources are every file in the set, in the order the registry listed them.
|
||||
// A shard set goes as one action — a fifteen-shard model is not fifteen
|
||||
@@ -179,6 +184,15 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
|
||||
model := strings.TrimSpace(r.FormValue("model"))
|
||||
version := strings.TrimSpace(r.FormValue("version"))
|
||||
|
||||
// Read before anything is downloaded: the header says whether the
|
||||
// nodes can load it, and a file that cannot be served is an hour
|
||||
// of download and a replica failing on every node.
|
||||
if r.FormValue("force") == "" {
|
||||
if v := p.checkSource(ctx, strings.TrimSpace(sources[0])); !v.Servable {
|
||||
return "", fmt.Errorf("not fetched: %s. Tick \"fetch anyway\" to take it regardless", v.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
var started int
|
||||
for i, src := range sources {
|
||||
src = strings.TrimSpace(src)
|
||||
@@ -379,7 +393,7 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
|
||||
continue
|
||||
}
|
||||
form := planServe(d.Rows[i].Status,
|
||||
byVersion[d.Rows[i].Model+"/"+d.Rows[i].Version], got.Base, p.fleet, p.catalog)
|
||||
byVersion[d.Rows[i].Model+"/"+d.Rows[i].Version], got.Base, p.fleet, p.catalog, func(src string) gguf.Verdict { return p.checkSource(r.Context(), src) })
|
||||
d.Rows[i].Serve = &form
|
||||
break
|
||||
}
|
||||
@@ -467,6 +481,7 @@ func (p *AccountPages) explore(r *http.Request, d *adminMirrorData) {
|
||||
// the same branch lands somewhere else.
|
||||
Version: got.ShortRevision(),
|
||||
}
|
||||
rs.Verdict = p.verdictFor(got.Architecture, set.Name)
|
||||
for _, f := range set.Files {
|
||||
rs.Sources = append(rs.Sources,
|
||||
repoSource{URL: got.DownloadURL("", f.Path), Path: f.Path})
|
||||
@@ -498,3 +513,25 @@ func defaultModelName(repoID string) string {
|
||||
name = strings.TrimSuffix(name, "-gguf")
|
||||
return strings.ToLower(name)
|
||||
}
|
||||
|
||||
// verdictFor judges a registry file set by the repository's architecture. A
|
||||
// vision projector is its own architecture whatever the repository says.
|
||||
func (p *AccountPages) verdictFor(arch, name string) gguf.Verdict {
|
||||
if strings.Contains(strings.ToLower(name), "mmproj") {
|
||||
return gguf.LlamaCpp(p.llamaVersion, "clip")
|
||||
}
|
||||
if arch == "" || !gguf.IsGGUF(name) {
|
||||
return gguf.Verdict{Servable: true}
|
||||
}
|
||||
return gguf.LlamaCpp(p.llamaVersion, arch)
|
||||
}
|
||||
|
||||
// checkSource reads a GGUF source's header and judges it; anything else, or a
|
||||
// header that cannot be read, passes.
|
||||
func (p *AccountPages) checkSource(ctx context.Context, source string) gguf.Verdict {
|
||||
url, ok := sourcecheck.HTTPSource(source)
|
||||
if !ok || !gguf.IsGGUF(url) || p.llamaVersion == "" {
|
||||
return gguf.Verdict{Servable: true}
|
||||
}
|
||||
return gguf.Check(ctx, url, p.llamaVersion)
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -60,7 +61,7 @@ type serveForm struct {
|
||||
|
||||
// planServe works out what serving this version would mean.
|
||||
func planServe(st mirror.Status, files []mirror.Status, base string,
|
||||
fleet Source, cat CatalogAdmin,
|
||||
fleet Source, cat CatalogAdmin, check func(source string) gguf.Verdict,
|
||||
) serveForm {
|
||||
f := serveForm{
|
||||
Model: st.Model,
|
||||
@@ -99,6 +100,12 @@ func planServe(st mirror.Status, files []mirror.Status, base string,
|
||||
return f
|
||||
case "llamacpp":
|
||||
f.Profile = "cpu"
|
||||
if check != nil {
|
||||
if v := check(f.Source); !v.Servable {
|
||||
f.Refusal = "llama.cpp on the nodes cannot run this: " + v.Reason
|
||||
return f
|
||||
}
|
||||
}
|
||||
default:
|
||||
f.Profile = "gpu"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// ggufServer serves a GGUF whose header names an architecture.
|
||||
func ggufServer(t *testing.T, arch string) string {
|
||||
t.Helper()
|
||||
var b bytes.Buffer
|
||||
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
|
||||
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
|
||||
b.WriteString("GGUF")
|
||||
w(uint32(3))
|
||||
w(uint64(0))
|
||||
w(uint64(2))
|
||||
str("general.architecture")
|
||||
w(uint32(8))
|
||||
str(arch)
|
||||
str("general.name")
|
||||
w(uint32(8))
|
||||
str("m")
|
||||
body := b.Bytes()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
|
||||
http.ServeContent(rw, r, "m.gguf", time.Time{}, bytes.NewReader(body))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv.URL + "/m.gguf"
|
||||
}
|
||||
|
||||
func mirrorPageLlama(t *testing.T, d MirrorDeps) *http.ServeMux {
|
||||
t.Helper()
|
||||
a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}}
|
||||
p := NewAccountPages(a, nil, nil, "v0", slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
p.SetLlamaCppVersion("b10991")
|
||||
mux := http.NewServeMux()
|
||||
p.RegisterAdminMirror(mux, d)
|
||||
return mux
|
||||
}
|
||||
|
||||
// The file that started this: kev-0.5b, architecture "ggmlc", downloaded,
|
||||
// catalogued, and refused by llama.cpp on every node it was placed on.
|
||||
func TestAFileTheNodesCannotLoadIsNotFetched(t *testing.T) {
|
||||
m := &fakeMirror{}
|
||||
mux := mirrorPageLlama(t, MirrorDeps{Mirror: m})
|
||||
src := ggufServer(t, "ggmlc")
|
||||
|
||||
body := mirrorPost(t, mux, "/admin/mirror/fetch", url.Values{"csrf": {"tok"},
|
||||
"source": {src}, "model": {"kev"}, "version": {"v1"}}).Body.String()
|
||||
if len(m.all) != 0 {
|
||||
t.Fatal("a file llama.cpp cannot load was sent to the mirror")
|
||||
}
|
||||
if !strings.Contains(body, "ggmlc") || !strings.Contains(body, "cannot load") {
|
||||
t.Errorf("the refusal should say why; got %s", body)
|
||||
}
|
||||
|
||||
// Asked for regardless, it is fetched.
|
||||
mirrorPost(t, mux, "/admin/mirror/fetch", url.Values{"csrf": {"tok"},
|
||||
"source": {src}, "model": {"kev"}, "version": {"v1"}, "force": {"1"}})
|
||||
if len(m.all) != 1 {
|
||||
t.Error("fetch anyway did not fetch")
|
||||
}
|
||||
|
||||
// And one it can load goes straight through.
|
||||
m.all = nil
|
||||
mirrorPost(t, mux, "/admin/mirror/fetch", url.Values{"csrf": {"tok"},
|
||||
"source": {ggufServer(t, "qwen2")}, "model": {"q"}, "version": {"v1"}})
|
||||
if len(m.all) != 1 {
|
||||
t.Error("a qwen2 GGUF was refused")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheRegistryListingSaysWhatCannotBeServed(t *testing.T) {
|
||||
reg := &fakeRegistry{repo: hf.Repo{ID: "someone/kev-gguf", Revision: "abc12345", Architecture: "ggmlc",
|
||||
Files: []hf.File{{Path: "kev-q4_k_m.gguf", Bytes: 500 << 20}}}}
|
||||
body := mirrorGet(t, mirrorPageLlama(t, MirrorDeps{Mirror: &fakeMirror{}, Registry: reg}),
|
||||
"/admin/mirror?q=kev&repo=someone/kev-gguf").Body.String()
|
||||
if !strings.Contains(body, "cannot be served") {
|
||||
t.Error("the listing does not say the file cannot be served")
|
||||
}
|
||||
if strings.Contains(body, `value="`+"https://huggingface.co") && strings.Contains(body, ">Fetch</button>") {
|
||||
t.Error("the listing still offers to fetch it")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user