Ilker ManapandClaude Opus 5.5
4a652fdcc1
The serve form asked for weights plus half again plus two gigabytes. That ignores the one thing that varies most: the KV cache. At the same 16k pool, the models on the live mirror need from 0.44 GB of cache (Qwen2.5-1.5B, two KV heads) to 5.25 GB (Gemma-2-9B, 256-wide heads) — so the flat guess overcounted the small models, which on 7 GB nodes means refusing placements that fit, and undercounted the ones with wide or many KV heads, which means an engine killed for memory. The GGUF header already carries the shape. The reader now also takes each architecture's block_count, head counts, embedding length and key/value lengths, reading past general.* until the tokenizer — and a header whose tokenizer comes first and runs past the fetched bytes still returns what it has rather than failing. The estimate is weights, plus the f16 KV cache for the context pool (llama.cpp's --ctx-size, which --parallel divides rather than multiplies), plus a gigabyte and five percent for the engine. It reuses the header the servability check already fetched, and the form says what it is made of. Checked against all twelve GGUF models on the live mirror, whose shapes read back as their published architectures. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
208 lines
6.0 KiB
Go
208 lines
6.0 KiB
Go
package gguf
|
|
|
|
// llamaCppArchitectures is every model architecture a llama.cpp build can
|
|
// load, by the build llmgw pins (config.DefaultEngines).
|
|
//
|
|
// Taken from LLM_ARCH_NAMES in src/llama-arch.cpp at that tag, rather than
|
|
// discovered from a node: the check has to happen before anything is
|
|
// downloaded, on the control plane, which runs no engine. A build other than
|
|
// these is not refused, only not vouched for.
|
|
var llamaCppArchitectures = map[string]map[string]bool{
|
|
"b10991": {
|
|
"clip": true,
|
|
"llama": true,
|
|
"llama4": true,
|
|
"deci": true,
|
|
"falcon": true,
|
|
"grok": true,
|
|
"gpt2": true,
|
|
"gptj": true,
|
|
"gptneox": true,
|
|
"mpt": true,
|
|
"baichuan": true,
|
|
"starcoder": true,
|
|
"refact": true,
|
|
"bert": true,
|
|
"modern-bert": true,
|
|
"nomic-bert": true,
|
|
"nomic-bert-moe": true,
|
|
"neo-bert": true,
|
|
"jina-bert-v2": true,
|
|
"jina-bert-v3": true,
|
|
"eurobert": true,
|
|
"bloom": true,
|
|
"stablelm": true,
|
|
"qwen": true,
|
|
"qwen2": true,
|
|
"qwen2moe": true,
|
|
"qwen2vl": true,
|
|
"qwen3": true,
|
|
"qwen3moe": true,
|
|
"qwen3next": true,
|
|
"qwen3vl": true,
|
|
"qwen3vlmoe": true,
|
|
"qwen35": true,
|
|
"qwen35moe": true,
|
|
"qwen4exp": true,
|
|
"phi2": true,
|
|
"phi3": true,
|
|
"phimoe": true,
|
|
"plamo": true,
|
|
"plamo2": true,
|
|
"plamo3": true,
|
|
"codeshell": true,
|
|
"orion": true,
|
|
"internlm2": true,
|
|
"minicpm": true,
|
|
"minicpm3": true,
|
|
"gemma": true,
|
|
"gemma2": true,
|
|
"gemma3": true,
|
|
"gemma3n": true,
|
|
"gemma4": true,
|
|
"gemma4-assistant": true,
|
|
"gemma-embedding": true,
|
|
"starcoder2": true,
|
|
"mamba": true,
|
|
"mamba2": true,
|
|
"maple": true,
|
|
"jamba": true,
|
|
"falcon-h1": true,
|
|
"xverse": true,
|
|
"command-r": true,
|
|
"cohere2": true,
|
|
"cohere2moe": true,
|
|
"dbrx": true,
|
|
"olmo": true,
|
|
"olmo2": true,
|
|
"olmoe": true,
|
|
"muse-glimmer": true,
|
|
"openelm": true,
|
|
"arctic": true,
|
|
"deepseek": true,
|
|
"deepseek2": true,
|
|
"deepseek2-ocr": true,
|
|
"deepseek32": true,
|
|
"deepseek4": true,
|
|
"chatglm": true,
|
|
"glm4": true,
|
|
"glm4moe": true,
|
|
"glm-dsa": true,
|
|
"bitnet": true,
|
|
"t5": true,
|
|
"t5encoder": true,
|
|
"jais": true,
|
|
"jais2": true,
|
|
"nemotron": true,
|
|
"nemotron_h": true,
|
|
"nemotron_h_moe": true,
|
|
"exaone": true,
|
|
"exaone4": true,
|
|
"exaone-moe": true,
|
|
"rwkv6": true,
|
|
"rwkv6qwen2": true,
|
|
"rwkv7": true,
|
|
"arwkv7": true,
|
|
"granite": true,
|
|
"granitemoe": true,
|
|
"granitehybrid": true,
|
|
"graniteswitch": true,
|
|
"granite_swa": true,
|
|
"chameleon": true,
|
|
"wavtokenizer-dec": true,
|
|
"plm": true,
|
|
"bailingmoe": true,
|
|
"bailingmoe2": true,
|
|
"bailingmoe3": true,
|
|
"dots1": true,
|
|
"dots3note": true,
|
|
"arcee": true,
|
|
"afmoe": true,
|
|
"laguna": true,
|
|
"ernie4_5": true,
|
|
"ernie4_5-moe": true,
|
|
"hunyuan-moe": true,
|
|
"hunyuan-dense": true,
|
|
"hunyuan_vl": true,
|
|
"hy_v3": true,
|
|
"hy_v4": true,
|
|
"smollm3": true,
|
|
"gpt-oss": true,
|
|
"lfm2": true,
|
|
"lfm2moe": true,
|
|
"dream": true,
|
|
"smallthinker": true,
|
|
"llada": true,
|
|
"llada-moe": true,
|
|
"seed_oss": true,
|
|
"grovemoe": true,
|
|
"apertus": true,
|
|
"minimax-01": true,
|
|
"minimax-m2": true,
|
|
"minimax-m3": true,
|
|
"cogvlm": true,
|
|
"rnd1": true,
|
|
"pangu-embedded": true,
|
|
"mistral3": true,
|
|
"eagle3": true,
|
|
"dflash": true,
|
|
"mistral4": true,
|
|
"paddleocr": true,
|
|
"mimo2": true,
|
|
"step35": true,
|
|
"spark2_5": true,
|
|
"llama-embed": true,
|
|
"maincoder": true,
|
|
"kimi-linear": true,
|
|
"kimi-k3": true,
|
|
"talkie": true,
|
|
"mellum": true,
|
|
"nanbeige": true,
|
|
"qwen3tts": true,
|
|
"pockettts": true,
|
|
"(unknown)": true,
|
|
},
|
|
}
|
|
|
|
// notAModel are architectures llama.cpp knows that are not a model on their
|
|
// own: a vision projector is loaded beside a language model with --mmproj.
|
|
var notAModel = map[string]string{
|
|
"clip": "a vision projector, loaded beside a language model rather than served alone",
|
|
}
|
|
|
|
// Verdict is whether a llama.cpp build can serve a file.
|
|
type Verdict struct {
|
|
Architecture string
|
|
// Servable is false only when the build is known not to load it.
|
|
Servable bool
|
|
// Checked is false when the build is not one whose architectures llmgw
|
|
// knows; Servable is then true by default, and Reason says it was not
|
|
// checked.
|
|
Checked bool
|
|
Reason string
|
|
// Header is what was read, zero when nothing was; its shape sizes the
|
|
// model's memory.
|
|
Header Header
|
|
}
|
|
|
|
// LlamaCpp judges an architecture against a llama.cpp build.
|
|
func LlamaCpp(version, arch string) Verdict {
|
|
v := Verdict{Architecture: arch, Servable: true}
|
|
if why, ok := notAModel[arch]; ok {
|
|
v.Servable, v.Checked = false, true
|
|
v.Reason = "its architecture is \"" + arch + "\", " + why
|
|
return v
|
|
}
|
|
known, ok := llamaCppArchitectures[version]
|
|
if !ok {
|
|
v.Reason = "llama.cpp " + version + " is not a build whose architectures llmgw knows; not checked"
|
|
return v
|
|
}
|
|
v.Checked = true
|
|
if !known[arch] {
|
|
v.Servable = false
|
|
v.Reason = "its architecture is \"" + arch + "\", which llama.cpp " + version + " cannot load"
|
|
}
|
|
return v
|
|
}
|