Files
llmgw/internal/web/admin_serve.go
T
Ilker ManapandClaude Opus 5 e0ea4f6be9 Fill in the memory a model needs, instead of leaving the field blank
Serving a model from the mirror page left "Needs RAM (GB)" optional and empty.
Empty writes no constraint, and a CPU profile has no card to be short of, so
every ready node became a candidate — and a 4.68 GB model went onto a 7.9 GB
machine and left it 3.5 GB for a KV pool. Placement was not wrong; it was never
told what the model needed.

The mirror knows the size of the weights, so the field arrives filled in:
weights plus half again for the KV pool, plus two for the engine. The half is
there because the cache grows with the same things the weights do, layers and
heads. It is an estimate and the field is still editable — what it is not is
blank.

A 1.12 GB model asks for 3 GB, so the small nodes stay usable. Pinning every
model to the one large machine would be a different way of getting placement
wrong.

An unknown size fills in nothing rather than the floor of two, which would
exclude no node while reading as though somebody had measured it.

And when the estimate is larger than any node in the fleet, the page says so,
with the largest node's memory. Better read there than discovered by a node
being OOM-killed twenty minutes into a load.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-27 16:38:49 +02:00

288 lines
8.6 KiB
Go

package web
import (
"context"
"errors"
"fmt"
"strconv"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
)
// CatalogAdmin adds a model to the catalogue in force.
type CatalogAdmin interface {
Add(ctx context.Context, m catalog.Model, by string) error
Declared(name string) bool
}
// serveForm is the offer to put one mirrored version into service.
//
// Everything a catalogue entry needs that the mirror already knows is filled
// in: the name, the source, and the engine that reads that kind of file. What
// is left is what nobody can infer — the context pool and how many replicas —
// and those are fields rather than guesses presented as facts.
type serveForm struct {
Model string
Version string
Source string
Engine string
// Profile is the profile name, which doubles as how a node_selector would
// be written later. Named for the hardware rather than the model, because
// a profile is about where a model runs.
Profile string
// MinMemoryGB is the host RAM to ask for, estimated from the weights.
//
// Filled in rather than left blank. Blank writes no constraint at all, and
// a CPU profile has no card to be short of, so every ready node becomes a
// candidate: a 4.7 GB model went onto a 7.9 GB machine and left it with
// 3.5 GB for a KV pool. Nothing was wrong with placement — it was never
// told what the model needed.
MinMemoryGB int
// Refusal is set when this version cannot be served as it stands, and says
// why. Offered and then failing at placement is worse than not offered.
Refusal string
// Warning is set when it can be served but probably will not run.
Warning string
// Already is true when a model of this name is in the catalogue.
Already bool
}
// planServe works out what serving this version would mean.
func planServe(st mirror.Status, files []mirror.Status, base string,
fleet Source, cat CatalogAdmin,
) serveForm {
f := serveForm{
Model: st.Model,
Version: st.Version,
Source: mirror.Entry{Model: st.Model, Version: st.Version, File: st.File}.
URL(base),
Engine: engineFor(st.File),
Profile: "cpu",
}
// A catalogue entry names one source. A version made of several files has
// no single one, so an entry for it would be an entry for one shard — which
// loads as a broken model rather than failing to load.
if len(files) > 1 {
f.Refusal = fmt.Sprintf(
"this version is %d files, and a catalogue entry names one source. "+
"Serve a single-file build (a GGUF), or give the engine the "+
"repository id so it fetches the shards itself",
len(files))
return f
}
switch f.Engine {
case "":
f.Refusal = "nothing here reads " + extOf(st.File) + " files"
return f
case "llamacpp":
f.Profile = "cpu"
default:
f.Profile = "gpu"
}
// Whether any node can actually run it. Agents report their installed
// engines now, so this is a fact rather than the guess it started as —
// which was cards, and would have passed a GPU node with no vLLM on it.
//
// A warning, not a refusal: an engine can be installed this afternoon, and
// a node that reports nothing is an older agent rather than an empty one.
if !runsAnywhere(fleet, f.Engine) {
f.Warning = "no node reports having " + f.Engine + " installed, and " +
extOf(st.File) + " weights need it"
if f.Engine != "llamacpp" && gpus(fleet) == 0 {
f.Warning += ". This fleet has no GPU either; a GGUF build of the " +
"same model runs on the CPU nodes"
}
}
f.MinMemoryGB = memoryFor(st.BytesTotal, st.BytesDone)
if biggest := biggestNode(fleet); biggest > 0 && f.MinMemoryGB > biggest {
f.Warning = strings.TrimSuffix(f.Warning, ".") +
fmt.Sprintf("%sthe largest node has %d GB and this needs about %d",
sep(f.Warning), biggest, f.MinMemoryGB)
}
if cat != nil && cat.Declared(st.Model) {
f.Already = true
f.Refusal = "a model called " + st.Model +
" is declared in configuration; edit it there rather than adding a second"
}
return f
}
// engineFor is the engine that reads this kind of weights file.
func engineFor(file string) string {
switch strings.ToLower(extOf(file)) {
case ".gguf":
return "llamacpp"
case ".safetensors", ".bin":
return "vllm"
}
return ""
}
func extOf(file string) string {
if i := strings.LastIndex(file, "."); i >= 0 {
return file[i:]
}
return file
}
// runsAnywhere reports whether some node has this engine.
//
// True when nothing is known, so a fleet of agents older than the engine probe
// is not told every model is unservable. That matches what placement does with
// the same absence, and the two disagreeing would be worse than either.
func runsAnywhere(fleet Source, engine string) bool {
if fleet == nil || engine == "" {
return true
}
anyReported := false
for _, n := range fleet.Nodes() {
if len(n.Engines) == 0 {
continue
}
anyReported = true
for _, e := range n.Engines {
if e == engine {
return true
}
}
}
return !anyReported
}
func gpus(fleet Source) int {
if fleet == nil {
return 0
}
n := 0
for _, node := range fleet.Nodes() {
n += node.GPUTotal
}
return n
}
// buildModel turns the submitted form into a catalogue entry.
func buildModel(get func(string) string) (catalog.Model, error) {
m := catalog.Model{
Name: strings.TrimSpace(get("name")),
Source: strings.TrimSpace(get("source")),
RoutingProfile: "chat",
}
if m.Name == "" || m.Source == "" {
return m, errors.New("a model needs a name and a source")
}
for _, a := range strings.Split(get("aliases"), ",") {
if a = strings.TrimSpace(a); a != "" {
m.Aliases = append(m.Aliases, a)
}
}
ctxLen, err := positive(get("context_length"), "the context length")
if err != nil {
return m, err
}
m.ContextLength = ctxLen
if v := strings.TrimSpace(get("max_output")); v != "" {
if m.MaxOutput, err = positive(v, "the output limit"); err != nil {
return m, err
}
}
min, err := positive(get("replicas"), "the replica count")
if err != nil {
return m, err
}
m.Scaling = catalog.Scaling{Min: min, Max: min}
engine := strings.TrimSpace(get("engine"))
if engine == "" {
return m, errors.New("which engine should run it?")
}
p := catalog.Profile{
Name: strings.TrimSpace(get("profile")), Engine: engine,
Weight: 1, Priority: 100,
}
if p.Name == "" {
p.Name = engine
}
// The pool, not the per-request limit: llama.cpp divides it across its
// parallel slots, so the number a client may use is smaller and is what
// context_length above says.
if v := strings.TrimSpace(get("ctx_size")); v != "" {
if _, err := positive(v, "the engine's context pool"); err != nil {
return m, err
}
p.Args = []string{"--ctx-size", v}
}
if v := strings.TrimSpace(get("min_memory_gb")); v != "" {
if p.MinMemoryGB, err = positive(v, "the memory requirement"); err != nil {
return m, err
}
}
if k, v := strings.TrimSpace(get("selector_key")), strings.TrimSpace(get("selector_value")); k != "" && v != "" {
// What keeps a large model off the small nodes. A CPU profile has no
// card to be short of, so without this every ready node looks like a
// candidate and the first one picked is the one that gets OOM-killed.
p.NodeSelector = map[string]string{k: v}
}
m.Profiles = []catalog.Profile{p}
return m, nil
}
func positive(v, what string) (int, error) {
n, err := strconv.Atoi(strings.TrimSpace(v))
if err != nil || n < 0 {
return 0, fmt.Errorf("%s must be a whole number", what)
}
return n, nil
}
// memoryFor estimates the host RAM a model needs, from the size of its weights.
//
// Weights plus half again, plus two. The half covers the KV pool, which grows
// with the same things the weights do — layers and heads — and the two are the
// engine's own footprint and the page cache it will not get back. It is an
// estimate and the field it fills is editable; what it is not is blank, which
// is what put a 4.7 GB model on a 7.9 GB node.
//
// The honest number comes from running it: `llmgw ctl bench` measures what a
// replica can actually do, and that is a different question from what it needs
// to start.
func memoryFor(total, done int64) int {
size := total
if size <= 0 {
size = done
}
if size <= 0 {
return 0
}
gb := float64(size) / (1 << 30)
return int(gb*1.5) + 2
}
// biggestNode is the most memory any single node reports, or 0 when none says.
func biggestNode(fleet Source) int {
if fleet == nil {
return 0
}
most := 0
for _, n := range fleet.Nodes() {
if n.MemoryGB > most {
most = n.MemoryGB
}
}
return most
}
// sep joins a second clause onto a warning that may be empty.
func sep(existing string) string {
if existing == "" {
return ""
}
return ", and "
}