package web import ( "context" "errors" "fmt" "strconv" "strings" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" ) // CatalogAdmin adds a model to the catalogue in force. type CatalogAdmin interface { Add(ctx context.Context, m catalog.Model, by string) error Declared(name string) bool } // serveForm is the offer to put one mirrored version into service. // // Everything a catalogue entry needs that the mirror already knows is filled // in: the name, the source, and the engine that reads that kind of file. What // is left is what nobody can infer — the context pool and how many replicas — // and those are fields rather than guesses presented as facts. type serveForm struct { Model string Version string Source string Engine string // Profile is the profile name, which doubles as how a node_selector would // be written later. Named for the hardware rather than the model, because // a profile is about where a model runs. Profile string // MinMemoryGB is the host RAM to ask for, estimated from the weights. // // Filled in rather than left blank. Blank writes no constraint at all, and // a CPU profile has no card to be short of, so every ready node becomes a // candidate: a 4.7 GB model went onto a 7.9 GB machine and left it with // 3.5 GB for a KV pool. Nothing was wrong with placement — it was never // told what the model needed. MinMemoryGB int // Refusal is set when this version cannot be served as it stands, and says // why. Offered and then failing at placement is worse than not offered. Refusal string // Warning is set when it can be served but probably will not run. Warning string // Already is true when a model of this name is in the catalogue. Already bool } // planServe works out what serving this version would mean. func planServe(st mirror.Status, files []mirror.Status, base string, fleet Source, cat CatalogAdmin, ) serveForm { f := serveForm{ Model: st.Model, Version: st.Version, Source: mirror.Entry{Model: st.Model, Version: st.Version, File: st.File}. URL(base), Engine: engineFor(st.File), Profile: "cpu", } // A catalogue entry names one source. A version made of several files has // no single one, so an entry for it would be an entry for one shard — which // loads as a broken model rather than failing to load. if len(files) > 1 { f.Refusal = fmt.Sprintf( "this version is %d files, and a catalogue entry names one source. "+ "Serve a single-file build (a GGUF), or give the engine the "+ "repository id so it fetches the shards itself", len(files)) return f } switch f.Engine { case "": f.Refusal = "nothing here reads " + extOf(st.File) + " files" return f case "llamacpp": f.Profile = "cpu" default: f.Profile = "gpu" } // Whether any node can actually run it. Agents report their installed // engines now, so this is a fact rather than the guess it started as — // which was cards, and would have passed a GPU node with no vLLM on it. // // A warning, not a refusal: an engine can be installed this afternoon, and // a node that reports nothing is an older agent rather than an empty one. if !runsAnywhere(fleet, f.Engine) { f.Warning = "no node reports having " + f.Engine + " installed, and " + extOf(st.File) + " weights need it" if f.Engine != "llamacpp" && gpus(fleet) == 0 { f.Warning += ". This fleet has no GPU either; a GGUF build of the " + "same model runs on the CPU nodes" } } f.MinMemoryGB = memoryFor(st.BytesTotal, st.BytesDone) if biggest := biggestNode(fleet); biggest > 0 && f.MinMemoryGB > biggest { f.Warning = strings.TrimSuffix(f.Warning, ".") + fmt.Sprintf("%sthe largest node has %d GB and this needs about %d", sep(f.Warning), biggest, f.MinMemoryGB) } if cat != nil && cat.Declared(st.Model) { f.Already = true f.Refusal = "a model called " + st.Model + " is declared in configuration; edit it there rather than adding a second" } return f } // engineFor is the engine that reads this kind of weights file. func engineFor(file string) string { switch strings.ToLower(extOf(file)) { case ".gguf": return "llamacpp" case ".safetensors", ".bin": return "vllm" } return "" } func extOf(file string) string { if i := strings.LastIndex(file, "."); i >= 0 { return file[i:] } return file } // runsAnywhere reports whether some node has this engine. // // True when nothing is known, so a fleet of agents older than the engine probe // is not told every model is unservable. That matches what placement does with // the same absence, and the two disagreeing would be worse than either. func runsAnywhere(fleet Source, engine string) bool { if fleet == nil || engine == "" { return true } anyReported := false for _, n := range fleet.Nodes() { if len(n.Engines) == 0 { continue } anyReported = true for _, e := range n.Engines { if e == engine { return true } } } return !anyReported } func gpus(fleet Source) int { if fleet == nil { return 0 } n := 0 for _, node := range fleet.Nodes() { n += node.GPUTotal } return n } // buildModel turns the submitted form into a catalogue entry. func buildModel(get func(string) string) (catalog.Model, error) { m := catalog.Model{ Name: strings.TrimSpace(get("name")), Description: strings.TrimSpace(get("description")), Source: strings.TrimSpace(get("source")), RoutingProfile: "chat", } if m.Name == "" || m.Source == "" { return m, errors.New("a model needs a name and a source") } for _, a := range strings.Split(get("aliases"), ",") { if a = strings.TrimSpace(a); a != "" { m.Aliases = append(m.Aliases, a) } } ctxLen, err := positive(get("context_length"), "the context length") if err != nil { return m, err } m.ContextLength = ctxLen if v := strings.TrimSpace(get("max_output")); v != "" { if m.MaxOutput, err = positive(v, "the output limit"); err != nil { return m, err } } min, err := positive(get("replicas"), "the replica count") if err != nil { return m, err } m.Scaling = catalog.Scaling{Min: min, Max: min} engine := strings.TrimSpace(get("engine")) if engine == "" { return m, errors.New("which engine should run it?") } p := catalog.Profile{ Name: strings.TrimSpace(get("profile")), Engine: engine, Weight: 1, Priority: 100, } if p.Name == "" { p.Name = engine } // The pool, not the per-request limit: llama.cpp divides it across its // parallel slots, so the number a client may use is smaller and is what // context_length above says. if v := strings.TrimSpace(get("ctx_size")); v != "" { if _, err := positive(v, "the engine's context pool"); err != nil { return m, err } p.Args = []string{"--ctx-size", v} } if v := strings.TrimSpace(get("min_memory_gb")); v != "" { if p.MinMemoryGB, err = positive(v, "the memory requirement"); err != nil { return m, err } } if k, v := strings.TrimSpace(get("selector_key")), strings.TrimSpace(get("selector_value")); k != "" && v != "" { // What keeps a large model off the small nodes. A CPU profile has no // card to be short of, so without this every ready node looks like a // candidate and the first one picked is the one that gets OOM-killed. p.NodeSelector = map[string]string{k: v} } m.Profiles = []catalog.Profile{p} return m, nil } func positive(v, what string) (int, error) { n, err := strconv.Atoi(strings.TrimSpace(v)) if err != nil || n < 0 { return 0, fmt.Errorf("%s must be a whole number", what) } return n, nil } // memoryFor estimates the host RAM a model needs, from the size of its weights. // // Weights plus half again, plus two. The half covers the KV pool, which grows // with the same things the weights do — layers and heads — and the two are the // engine's own footprint and the page cache it will not get back. It is an // estimate and the field it fills is editable; what it is not is blank, which // is what put a 4.7 GB model on a 7.9 GB node. // // The honest number comes from running it: `llmgw ctl bench` measures what a // replica can actually do, and that is a different question from what it needs // to start. func memoryFor(total, done int64) int { size := total if size <= 0 { size = done } if size <= 0 { return 0 } gb := float64(size) / (1 << 30) return int(gb*1.5) + 2 } // biggestNode is the most memory any single node reports, or 0 when none says. func biggestNode(fleet Source) int { if fleet == nil { return 0 } most := 0 for _, n := range fleet.Nodes() { if n.MemoryGB > most { most = n.MemoryGB } } return most } // sep joins a second clause onto a warning that may be empty. func sep(existing string) string { if existing == "" { return "" } return ", and " }