diff --git a/CHANGELOG.md b/CHANGELOG.md index 459ca0a..8496286 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,23 @@ form asks the same: on demand loads the model when somebody asks for it and stops it after 15 minutes unused. A model declared in `server.yaml` shows its mode and says to change it there, because the file is read on every start. +### Weights the nodes cannot load are not fetched + +`kev-0.5b` was a GGUF whose architecture, `ggmlc`, llama.cpp does not know. It +was downloaded, catalogued and placed, and failed on every node it was tried on. +The architecture was in the first kilobytes of the file all along. Now: + +- The mirror page's registry listing shows each file's architecture and, for + one the nodes' llama.cpp cannot load, says so instead of offering to fetch + it. A vision projector (`mmproj`) is marked as not a model on its own. +- A fetch reads the file's header first — a range request, not a download — + and refuses one that cannot be served, unless "fetch anyway" is ticked. +- Serving from the mirror and `llmgw ctl model add` refuse the same. + +The list of architectures is llama.cpp's own (`LLM_ARCH_NAMES`) at the build +llmgw pins, b10991: 152 of them. A different build configured in `engines:` is +not refused, only not checked. + ### Fixes - Usage by team, project or organization, and the spend on the quotas page, diff --git a/internal/cli/servable_test.go b/internal/cli/servable_test.go new file mode 100644 index 0000000..0c9d0a9 --- /dev/null +++ b/internal/cli/servable_test.go @@ -0,0 +1,52 @@ +package cli + +import ( + "bytes" + "context" + "encoding/binary" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" +) + +func ggufAt(t *testing.T, arch string) string { + t.Helper() + var b bytes.Buffer + w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) } + str := func(s string) { w(uint64(len(s))); b.WriteString(s) } + b.WriteString("GGUF") + w(uint32(3)) + w(uint64(0)) + w(uint64(1)) + str("general.architecture") + w(uint32(8)) + str(arch) + body := b.Bytes() + srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + http.ServeContent(rw, r, "m.gguf", time.Time{}, bytes.NewReader(body)) + })) + t.Cleanup(srv.Close) + return srv.URL + "/m.gguf" +} + +func TestAModelTheNodesCannotLoadIsNotCatalogued(t *testing.T) { + check := servableCheck("b10991") + llama := []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}} + + err := check(context.Background(), catalog.Model{Name: "kev", Source: ggufAt(t, "ggmlc"), Profiles: llama}) + if err == nil || !strings.Contains(err.Error(), "ggmlc") { + t.Errorf("a ggmlc model was accepted: %v", err) + } + if err := check(context.Background(), catalog.Model{Name: "q", Source: ggufAt(t, "qwen2"), Profiles: llama}); err != nil { + t.Errorf("a qwen2 model was refused: %v", err) + } + // Not llama.cpp's to judge: vLLM reads a repository, not a GGUF header. + vllm := []catalog.Profile{{Name: "gpu", Engine: "vllm"}} + if err := check(context.Background(), catalog.Model{Name: "v", Source: ggufAt(t, "ggmlc"), Profiles: vllm}); err != nil { + t.Errorf("a vLLM profile was judged by llama.cpp's list: %v", err) + } +} diff --git a/internal/cli/server.go b/internal/cli/server.go index c10df8c..736095e 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -5,6 +5,8 @@ import ( "crypto/tls" "errors" "fmt" + "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" + "gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck" "log/slog" "net" "net/http" @@ -150,6 +152,7 @@ func newServerCmd() *cobra.Command { // rather than accepting a model that vanishes on restart. if st := rt.Store(); st != nil { svc := modelsvc.New(cfg.Models, st, admin, log) + svc.SetCheck(servableCheck(cfg.WantedEngines()["llamacpp"])) // Rebuild the catalog from configuration plus what was // stored, and put that in force — otherwise models added // before the last restart would be silently absent until @@ -535,3 +538,27 @@ func parseEndpointFlag(spec string) (config.Endpoint, error) { Weight: 1, }, nil } + +// servableCheck refuses a model whose llama.cpp weights the nodes' build +// cannot load, read from the weights' own header before anything is placed. +func servableCheck(llamaVersion string) func(context.Context, catalog.Model) error { + return func(ctx context.Context, m catalog.Model) error { + if llamaVersion == "" { + return nil + } + usesLlama := false + for _, p := range m.Profiles { + if p.Engine == "llamacpp" { + usesLlama = true + } + } + url, ok := sourcecheck.HTTPSource(m.Source) + if !usesLlama || !ok || !gguf.IsGGUF(url) { + return nil + } + if v := gguf.Check(ctx, url, llamaVersion); !v.Servable { + return fmt.Errorf("%s cannot be served: %s", m.Name, v.Reason) + } + return nil + } +} diff --git a/internal/control/modelsvc/modelsvc.go b/internal/control/modelsvc/modelsvc.go index f9024d6..94c7621 100644 --- a/internal/control/modelsvc/modelsvc.go +++ b/internal/control/modelsvc/modelsvc.go @@ -46,6 +46,10 @@ type Service struct { apply Applier log *slog.Logger + // check refuses a model that cannot run before it is added; see + // SetCheck. + check func(context.Context, catalog.Model) error + // mu serialises rebuilds. They are rare — a person adding a model — and two // at once could apply catalogs in either order, leaving the one in force // disagreeing with the database that outlives it. @@ -100,6 +104,11 @@ func (s *Service) Add(ctx context.Context, m catalog.Model, by string) error { if s.Declared(m.Name) { return fmt.Errorf("model %q is declared in configuration; edit that rather than adding a second definition", m.Name) } + if s.check != nil { + if err := s.check(ctx, m); err != nil { + return err + } + } stored, err := s.storedModels(ctx) if err != nil { @@ -299,3 +308,8 @@ func (s *Service) storedModels(ctx context.Context) ([]catalog.Model, error) { } return out, nil } + +// SetCheck runs before a model is added, and refuses it by returning an error. +// Used to catch weights the engine cannot load, which otherwise are placed and +// then fail on every node they are tried on. +func (s *Service) SetCheck(check func(context.Context, catalog.Model) error) { s.check = check } diff --git a/internal/gguf/gguf.go b/internal/gguf/gguf.go new file mode 100644 index 0000000..08083e2 --- /dev/null +++ b/internal/gguf/gguf.go @@ -0,0 +1,213 @@ +// Package gguf reads what a GGUF file says about itself, from its header. +// +// Enough to ask one question before anything is downloaded or served: can the +// engine load this? A file downloaded, catalogued and placed, and then refused +// by llama.cpp with "unknown model architecture", costs a download and a +// replica failing on every node it is tried on — and the answer was in the +// first few kilobytes all along. +package gguf + +import ( + "bufio" + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "math" + "net/http" + "strings" + "time" +) + +// Header is the part of a GGUF header this package reads. +type Header struct { + Version uint32 + Architecture string + Name string +} + +// ErrNotGGUF means the bytes are not a GGUF file. +var ErrNotGGUF = errors.New("not a GGUF file") + +// Value types, from the GGUF specification. +const ( + tUint8 = iota + tInt8 + tUint16 + tInt16 + tUint32 + tInt32 + tFloat32 + tBool + tString + tArray + tUint64 + tInt64 + tFloat64 +) + +// Read parses a header up to general.architecture and general.name. +func Read(r io.Reader) (Header, error) { + br := bufio.NewReader(r) + var h Header + magic := make([]byte, 4) + if _, err := io.ReadFull(br, magic); err != nil || string(magic) != "GGUF" { + return h, ErrNotGGUF + } + if err := binary.Read(br, binary.LittleEndian, &h.Version); err != nil { + return h, err + } + if h.Version < 1 || h.Version > 3 { + return h, fmt.Errorf("GGUF version %d is not one this reads", h.Version) + } + wide := h.Version >= 2 // v1 used 32-bit counts and lengths + count := func() (uint64, error) { + if wide { + var n uint64 + return n, binary.Read(br, binary.LittleEndian, &n) + } + var n uint32 + err := binary.Read(br, binary.LittleEndian, &n) + return uint64(n), err + } + str := func() (string, error) { + n, err := count() + if err != nil { + return "", err + } + if n > 1<<20 { + return "", fmt.Errorf("a %d-byte string in a GGUF header", n) + } + b := make([]byte, n) + _, err = io.ReadFull(br, b) + return string(b), err + } + skip := func(n int64) error { + _, err := io.CopyN(io.Discard, br, n) + return err + } + var value func(t uint32) (string, error) + value = func(t uint32) (string, error) { + switch t { + case tUint8, tInt8, tBool: + return "", skip(1) + case tUint16, tInt16: + return "", skip(2) + case tUint32, tInt32, tFloat32: + return "", skip(4) + case tUint64, tInt64, tFloat64: + return "", skip(8) + case tString: + return str() + case tArray: + var et uint32 + if err := binary.Read(br, binary.LittleEndian, &et); err != nil { + return "", err + } + n, err := count() + if err != nil { + return "", err + } + if n > math.MaxInt32 { + return "", fmt.Errorf("a %d-element array in a GGUF header", n) + } + for i := uint64(0); i < n; i++ { + if _, err := value(et); err != nil { + return "", err + } + } + return "", nil + } + return "", fmt.Errorf("GGUF value type %d is unknown", t) + } + + if _, err := count(); err != nil { // tensors + return h, err + } + kvs, err := count() + if err != nil { + return h, err + } + for i := uint64(0); i < kvs; i++ { + key, err := str() + if err != nil { + return h, err + } + var t uint32 + if err := binary.Read(br, binary.LittleEndian, &t); err != nil { + return h, err + } + v, err := value(t) + if err != nil { + return h, err + } + switch key { + case "general.architecture": + h.Architecture = v + case "general.name": + h.Name = v + } + if h.Architecture != "" && h.Name != "" { + break + } + } + if h.Architecture == "" { + return h, errors.New("the GGUF header names no architecture") + } + return h, nil +} + +// headerBytes is how much of a file is fetched to read its header. The +// architecture is among the first keys; the tokenizer's arrays, which can be +// megabytes, come after it. +const headerBytes = 8 << 20 + +// FromURL reads a GGUF header over HTTP, fetching only the start of the file. +func FromURL(ctx context.Context, client *http.Client, url string) (Header, error) { + if client == nil { + client = &http.Client{Timeout: 30 * time.Second} + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return Header{}, err + } + req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", headerBytes-1)) + resp, err := client.Do(req) + if err != nil { + return Header{}, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { + return Header{}, fmt.Errorf("%s answered %s", url, resp.Status) + } + return Read(io.LimitReader(resp.Body, headerBytes)) +} + +// IsGGUF reports whether a source names a GGUF file, or a directory of them. +func IsGGUF(source string) bool { + s := strings.ToLower(strings.TrimSuffix(source, "/")) + return strings.HasSuffix(s, ".gguf") +} + +// Check reads a source's header and judges it against a llama.cpp build. +// +// Only a definite answer refuses: a file that is not GGUF, or whose +// architecture the build cannot load. A header that cannot be fetched — the +// network, a directory of shards — is not a reason to refuse, so the verdict +// then says it was not checked. +func Check(ctx context.Context, url, llamaVersion string) Verdict { + if strings.HasSuffix(url, "/") { + return Verdict{Servable: true, Reason: "a set of files; its header was not read"} + } + ctx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + h, err := FromURL(ctx, nil, url) + if errors.Is(err, ErrNotGGUF) { + return Verdict{Checked: true, Reason: "it is not a GGUF file"} + } + if err != nil { + return Verdict{Servable: true, Reason: "its header could not be read: " + err.Error()} + } + return LlamaCpp(llamaVersion, h.Architecture) +} diff --git a/internal/gguf/gguf_test.go b/internal/gguf/gguf_test.go new file mode 100644 index 0000000..16f853c --- /dev/null +++ b/internal/gguf/gguf_test.go @@ -0,0 +1,94 @@ +package gguf + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// header builds a GGUF v3 header: an array before the architecture, the way +// real files put tokenizer data, so skipping values is exercised. +func header(arch string) []byte { + var b bytes.Buffer + w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) } + str := func(s string) { w(uint64(len(s))); b.WriteString(s) } + b.WriteString("GGUF") + w(uint32(3)) + w(uint64(0)) // tensors + w(uint64(4)) // kvs + str("general.alignment") + w(uint32(tUint32)) + w(uint32(32)) + str("some.list") + w(uint32(tArray)) + w(uint32(tString)) + w(uint64(2)) + str("a") + str("bc") + str("general.architecture") + w(uint32(tString)) + str(arch) + str("general.name") + w(uint32(tString)) + str("A Model") + return b.Bytes() +} + +func TestTheArchitectureIsReadFromTheHeader(t *testing.T) { + h, err := Read(bytes.NewReader(header("ggmlc"))) + if err != nil { + t.Fatal(err) + } + if h.Architecture != "ggmlc" || h.Name != "A Model" || h.Version != 3 { + t.Errorf("header = %+v", h) + } +} + +func TestSomethingElseIsNotAGGUF(t *testing.T) { + if _, err := Read(strings.NewReader("not found")); !errors.Is(err, ErrNotGGUF) { + t.Errorf("an HTML page read as %v", err) + } +} + +func TestOnlyTheStartOfARemoteFileIsFetched(t *testing.T) { + body := append(header("qwen2"), make([]byte, 1<<20)...) + var rng string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rng = r.Header.Get("Range") + http.ServeContent(w, r, "m.gguf", timeZero, bytes.NewReader(body)) + })) + defer srv.Close() + h, err := FromURL(context.Background(), nil, srv.URL+"/m.gguf") + if err != nil || h.Architecture != "qwen2" { + t.Fatalf("%+v, %v", h, err) + } + if !strings.HasPrefix(rng, "bytes=0-") { + t.Errorf("asked for %q rather than the start of the file", rng) + } +} + +func TestLlamaCppKnowsWhatItCanLoad(t *testing.T) { + if v := LlamaCpp("b10991", "qwen2"); !v.Servable || !v.Checked { + t.Errorf("qwen2: %+v", v) + } + // The file that started this: downloaded, catalogued, placed, and then + // refused by every node with "unknown model architecture: 'ggmlc'". + if v := LlamaCpp("b10991", "ggmlc"); v.Servable || !strings.Contains(v.Reason, "cannot load") { + t.Errorf("ggmlc: %+v", v) + } + if v := LlamaCpp("b10991", "clip"); v.Servable { + t.Errorf("a vision projector alone: %+v", v) + } + // A build llmgw has no list for is not refused, only not vouched for. + if v := LlamaCpp("b99999", "ggmlc"); !v.Servable || v.Checked { + t.Errorf("an unknown build: %+v", v) + } +} + +var timeZero = time.Time{} diff --git a/internal/gguf/llamacpp.go b/internal/gguf/llamacpp.go new file mode 100644 index 0000000..be31cf3 --- /dev/null +++ b/internal/gguf/llamacpp.go @@ -0,0 +1,204 @@ +package gguf + +// llamaCppArchitectures is every model architecture a llama.cpp build can +// load, by the build llmgw pins (config.DefaultEngines). +// +// Taken from LLM_ARCH_NAMES in src/llama-arch.cpp at that tag, rather than +// discovered from a node: the check has to happen before anything is +// downloaded, on the control plane, which runs no engine. A build other than +// these is not refused, only not vouched for. +var llamaCppArchitectures = map[string]map[string]bool{ + "b10991": { + "clip": true, + "llama": true, + "llama4": true, + "deci": true, + "falcon": true, + "grok": true, + "gpt2": true, + "gptj": true, + "gptneox": true, + "mpt": true, + "baichuan": true, + "starcoder": true, + "refact": true, + "bert": true, + "modern-bert": true, + "nomic-bert": true, + "nomic-bert-moe": true, + "neo-bert": true, + "jina-bert-v2": true, + "jina-bert-v3": true, + "eurobert": true, + "bloom": true, + "stablelm": true, + "qwen": true, + "qwen2": true, + "qwen2moe": true, + "qwen2vl": true, + "qwen3": true, + "qwen3moe": true, + "qwen3next": true, + "qwen3vl": true, + "qwen3vlmoe": true, + "qwen35": true, + "qwen35moe": true, + "qwen4exp": true, + "phi2": true, + "phi3": true, + "phimoe": true, + "plamo": true, + "plamo2": true, + "plamo3": true, + "codeshell": true, + "orion": true, + "internlm2": true, + "minicpm": true, + "minicpm3": true, + "gemma": true, + "gemma2": true, + "gemma3": true, + "gemma3n": true, + "gemma4": true, + "gemma4-assistant": true, + "gemma-embedding": true, + "starcoder2": true, + "mamba": true, + "mamba2": true, + "maple": true, + "jamba": true, + "falcon-h1": true, + "xverse": true, + "command-r": true, + "cohere2": true, + "cohere2moe": true, + "dbrx": true, + "olmo": true, + "olmo2": true, + "olmoe": true, + "muse-glimmer": true, + "openelm": true, + "arctic": true, + "deepseek": true, + "deepseek2": true, + "deepseek2-ocr": true, + "deepseek32": true, + "deepseek4": true, + "chatglm": true, + "glm4": true, + "glm4moe": true, + "glm-dsa": true, + "bitnet": true, + "t5": true, + "t5encoder": true, + "jais": true, + "jais2": true, + "nemotron": true, + "nemotron_h": true, + "nemotron_h_moe": true, + "exaone": true, + "exaone4": true, + "exaone-moe": true, + "rwkv6": true, + "rwkv6qwen2": true, + "rwkv7": true, + "arwkv7": true, + "granite": true, + "granitemoe": true, + "granitehybrid": true, + "graniteswitch": true, + "granite_swa": true, + "chameleon": true, + "wavtokenizer-dec": true, + "plm": true, + "bailingmoe": true, + "bailingmoe2": true, + "bailingmoe3": true, + "dots1": true, + "dots3note": true, + "arcee": true, + "afmoe": true, + "laguna": true, + "ernie4_5": true, + "ernie4_5-moe": true, + "hunyuan-moe": true, + "hunyuan-dense": true, + "hunyuan_vl": true, + "hy_v3": true, + "hy_v4": true, + "smollm3": true, + "gpt-oss": true, + "lfm2": true, + "lfm2moe": true, + "dream": true, + "smallthinker": true, + "llada": true, + "llada-moe": true, + "seed_oss": true, + "grovemoe": true, + "apertus": true, + "minimax-01": true, + "minimax-m2": true, + "minimax-m3": true, + "cogvlm": true, + "rnd1": true, + "pangu-embedded": true, + "mistral3": true, + "eagle3": true, + "dflash": true, + "mistral4": true, + "paddleocr": true, + "mimo2": true, + "step35": true, + "spark2_5": true, + "llama-embed": true, + "maincoder": true, + "kimi-linear": true, + "kimi-k3": true, + "talkie": true, + "mellum": true, + "nanbeige": true, + "qwen3tts": true, + "pockettts": true, + "(unknown)": true, + }, +} + +// notAModel are architectures llama.cpp knows that are not a model on their +// own: a vision projector is loaded beside a language model with --mmproj. +var notAModel = map[string]string{ + "clip": "a vision projector, loaded beside a language model rather than served alone", +} + +// Verdict is whether a llama.cpp build can serve a file. +type Verdict struct { + Architecture string + // Servable is false only when the build is known not to load it. + Servable bool + // Checked is false when the build is not one whose architectures llmgw + // knows; Servable is then true by default, and Reason says it was not + // checked. + Checked bool + Reason string +} + +// LlamaCpp judges an architecture against a llama.cpp build. +func LlamaCpp(version, arch string) Verdict { + v := Verdict{Architecture: arch, Servable: true} + if why, ok := notAModel[arch]; ok { + v.Servable, v.Checked = false, true + v.Reason = "its architecture is \"" + arch + "\", " + why + return v + } + known, ok := llamaCppArchitectures[version] + if !ok { + v.Reason = "llama.cpp " + version + " is not a build whose architectures llmgw knows; not checked" + return v + } + v.Checked = true + if !known[arch] { + v.Servable = false + v.Reason = "its architecture is \"" + arch + "\", which llama.cpp " + version + " cannot load" + } + return v +} diff --git a/internal/hf/hf.go b/internal/hf/hf.go index cdfa846..d274cdd 100644 --- a/internal/hf/hf.go +++ b/internal/hf/hf.go @@ -148,6 +148,10 @@ type Repo struct { Revision string Gated Gated Files []File + // Architecture is the GGUF architecture the registry read from the + // repository's weights, empty for a repository with none. What decides + // whether llama.cpp can load it, known before anything is downloaded. + Architecture string } // ShortRevision is the revision as a person would write it. @@ -169,8 +173,13 @@ func (c *Client) Repo(ctx context.Context, id string) (Repo, error) { ID string `json:"id"` SHA string `json:"sha"` Gated Gated `json:"gated"` + GGUF *struct { + Architecture string `json:"architecture"` + } `json:"gguf"` } - if err := c.get(ctx, "/api/models/"+id, &meta); err != nil { + // Expanded to what this reads: asking for gguf alone would leave out the + // revision and the gate. + if err := c.get(ctx, "/api/models/"+id+"?expand[]=sha&expand[]=gated&expand[]=gguf", &meta); err != nil { return Repo{}, err } @@ -194,6 +203,9 @@ func (c *Client) Repo(ctx context.Context, id string) (Repo, error) { } r := Repo{ID: meta.ID, Revision: meta.SHA, Gated: meta.Gated} + if meta.GGUF != nil { + r.Architecture = meta.GGUF.Architecture + } if r.ID == "" { r.ID = id } diff --git a/internal/web/account.go b/internal/web/account.go index 8ae7363..c7d0b63 100644 --- a/internal/web/account.go +++ b/internal/web/account.go @@ -67,6 +67,9 @@ type AccountPages struct { personalDefaults *store.Limits // orgs is set when /org is mounted, so the menu can offer it to managers. orgs OrgViewer + // llamaVersion is the llama.cpp build nodes install, for judging whether a + // GGUF can be served before it is fetched or catalogued. + llamaVersion string // priced says whether a budget can ever be spent against. priced Priced // registry browses a model registry, or is nil where there is no way out. @@ -602,3 +605,6 @@ func (p *AccountPages) renderWithCSRF(w http.ResponseWriter, r *http.Request, p.fillPlaces(r.Context(), user, &d) render(w, "profile", d) } + +// SetLlamaCppVersion tells the pages which llama.cpp build the nodes run. +func (p *AccountPages) SetLlamaCppVersion(v string) { p.llamaVersion = v } diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go index 5ed4434..41926b6 100644 --- a/internal/web/admin_mirror.go +++ b/internal/web/admin_mirror.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" "net/http" "sort" "strconv" @@ -41,6 +42,10 @@ type Registry interface { // repoSet is one thing worth fetching: a file, or every shard of one. type repoSet struct { + // Verdict is whether the nodes' llama.cpp can load it, from the + // architecture the registry reports; a set that cannot be served is + // shown with the reason and no fetch button. + Verdict gguf.Verdict hf.FileSet // Sources are every file in the set, in the order the registry listed them. // A shard set goes as one action — a fifteen-shard model is not fifteen @@ -179,6 +184,15 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) { model := strings.TrimSpace(r.FormValue("model")) version := strings.TrimSpace(r.FormValue("version")) + // Read before anything is downloaded: the header says whether the + // nodes can load it, and a file that cannot be served is an hour + // of download and a replica failing on every node. + if r.FormValue("force") == "" { + if v := p.checkSource(ctx, strings.TrimSpace(sources[0])); !v.Servable { + return "", fmt.Errorf("not fetched: %s. Tick \"fetch anyway\" to take it regardless", v.Reason) + } + } + var started int for i, src := range sources { src = strings.TrimSpace(src) @@ -379,7 +393,7 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request, continue } form := planServe(d.Rows[i].Status, - byVersion[d.Rows[i].Model+"/"+d.Rows[i].Version], got.Base, p.fleet, p.catalog) + byVersion[d.Rows[i].Model+"/"+d.Rows[i].Version], got.Base, p.fleet, p.catalog, func(src string) gguf.Verdict { return p.checkSource(r.Context(), src) }) d.Rows[i].Serve = &form break } @@ -467,6 +481,7 @@ func (p *AccountPages) explore(r *http.Request, d *adminMirrorData) { // the same branch lands somewhere else. Version: got.ShortRevision(), } + rs.Verdict = p.verdictFor(got.Architecture, set.Name) for _, f := range set.Files { rs.Sources = append(rs.Sources, repoSource{URL: got.DownloadURL("", f.Path), Path: f.Path}) @@ -498,3 +513,25 @@ func defaultModelName(repoID string) string { name = strings.TrimSuffix(name, "-gguf") return strings.ToLower(name) } + +// verdictFor judges a registry file set by the repository's architecture. A +// vision projector is its own architecture whatever the repository says. +func (p *AccountPages) verdictFor(arch, name string) gguf.Verdict { + if strings.Contains(strings.ToLower(name), "mmproj") { + return gguf.LlamaCpp(p.llamaVersion, "clip") + } + if arch == "" || !gguf.IsGGUF(name) { + return gguf.Verdict{Servable: true} + } + return gguf.LlamaCpp(p.llamaVersion, arch) +} + +// checkSource reads a GGUF source's header and judges it; anything else, or a +// header that cannot be read, passes. +func (p *AccountPages) checkSource(ctx context.Context, source string) gguf.Verdict { + url, ok := sourcecheck.HTTPSource(source) + if !ok || !gguf.IsGGUF(url) || p.llamaVersion == "" { + return gguf.Verdict{Servable: true} + } + return gguf.Check(ctx, url, p.llamaVersion) +} diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go index 74371c0..2ff667f 100644 --- a/internal/web/admin_serve.go +++ b/internal/web/admin_serve.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" "strconv" "strings" "time" @@ -60,7 +61,7 @@ type serveForm struct { // planServe works out what serving this version would mean. func planServe(st mirror.Status, files []mirror.Status, base string, - fleet Source, cat CatalogAdmin, + fleet Source, cat CatalogAdmin, check func(source string) gguf.Verdict, ) serveForm { f := serveForm{ Model: st.Model, @@ -99,6 +100,12 @@ func planServe(st mirror.Status, files []mirror.Status, base string, return f case "llamacpp": f.Profile = "cpu" + if check != nil { + if v := check(f.Source); !v.Servable { + f.Refusal = "llama.cpp on the nodes cannot run this: " + v.Reason + return f + } + } default: f.Profile = "gpu" } diff --git a/internal/web/servable_test.go b/internal/web/servable_test.go new file mode 100644 index 0000000..094071a --- /dev/null +++ b/internal/web/servable_test.go @@ -0,0 +1,96 @@ +package web + +import ( + "bytes" + "encoding/binary" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/hf" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// ggufServer serves a GGUF whose header names an architecture. +func ggufServer(t *testing.T, arch string) string { + t.Helper() + var b bytes.Buffer + w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) } + str := func(s string) { w(uint64(len(s))); b.WriteString(s) } + b.WriteString("GGUF") + w(uint32(3)) + w(uint64(0)) + w(uint64(2)) + str("general.architecture") + w(uint32(8)) + str(arch) + str("general.name") + w(uint32(8)) + str("m") + body := b.Bytes() + srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + http.ServeContent(rw, r, "m.gguf", time.Time{}, bytes.NewReader(body)) + })) + t.Cleanup(srv.Close) + return srv.URL + "/m.gguf" +} + +func mirrorPageLlama(t *testing.T, d MirrorDeps) *http.ServeMux { + t.Helper() + a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}} + p := NewAccountPages(a, nil, nil, "v0", slog.New(slog.NewTextHandler(io.Discard, nil))) + p.SetLlamaCppVersion("b10991") + mux := http.NewServeMux() + p.RegisterAdminMirror(mux, d) + return mux +} + +// The file that started this: kev-0.5b, architecture "ggmlc", downloaded, +// catalogued, and refused by llama.cpp on every node it was placed on. +func TestAFileTheNodesCannotLoadIsNotFetched(t *testing.T) { + m := &fakeMirror{} + mux := mirrorPageLlama(t, MirrorDeps{Mirror: m}) + src := ggufServer(t, "ggmlc") + + body := mirrorPost(t, mux, "/admin/mirror/fetch", url.Values{"csrf": {"tok"}, + "source": {src}, "model": {"kev"}, "version": {"v1"}}).Body.String() + if len(m.all) != 0 { + t.Fatal("a file llama.cpp cannot load was sent to the mirror") + } + if !strings.Contains(body, "ggmlc") || !strings.Contains(body, "cannot load") { + t.Errorf("the refusal should say why; got %s", body) + } + + // Asked for regardless, it is fetched. + mirrorPost(t, mux, "/admin/mirror/fetch", url.Values{"csrf": {"tok"}, + "source": {src}, "model": {"kev"}, "version": {"v1"}, "force": {"1"}}) + if len(m.all) != 1 { + t.Error("fetch anyway did not fetch") + } + + // And one it can load goes straight through. + m.all = nil + mirrorPost(t, mux, "/admin/mirror/fetch", url.Values{"csrf": {"tok"}, + "source": {ggufServer(t, "qwen2")}, "model": {"q"}, "version": {"v1"}}) + if len(m.all) != 1 { + t.Error("a qwen2 GGUF was refused") + } +} + +func TestTheRegistryListingSaysWhatCannotBeServed(t *testing.T) { + reg := &fakeRegistry{repo: hf.Repo{ID: "someone/kev-gguf", Revision: "abc12345", Architecture: "ggmlc", + Files: []hf.File{{Path: "kev-q4_k_m.gguf", Bytes: 500 << 20}}}} + body := mirrorGet(t, mirrorPageLlama(t, MirrorDeps{Mirror: &fakeMirror{}, Registry: reg}), + "/admin/mirror?q=kev&repo=someone/kev-gguf").Body.String() + if !strings.Contains(body, "cannot be served") { + t.Error("the listing does not say the file cannot be served") + } + if strings.Contains(body, `value="`+"https://huggingface.co") && strings.Contains(body, ">Fetch") { + t.Error("the listing still offers to fetch it") + } +}