package web import ( "context" "errors" "fmt" "strconv" "strings" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" ) // CatalogAdmin adds a model to the catalogue in force. type CatalogAdmin interface { Add(ctx context.Context, m catalog.Model, by string) error Declared(name string) bool } // serveForm is the offer to put one mirrored version into service. // // Everything a catalogue entry needs that the mirror already knows is filled // in: the name, the source, and the engine that reads that kind of file. What // is left is what nobody can infer — the context pool and how many replicas — // and those are fields rather than guesses presented as facts. type serveForm struct { Model string Version string Source string Engine string // Profile is the profile name, which doubles as how a node_selector would // be written later. Named for the hardware rather than the model, because // a profile is about where a model runs. Profile string // Refusal is set when this version cannot be served as it stands, and says // why. Offered and then failing at placement is worse than not offered. Refusal string // Warning is set when it can be served but probably will not run. Warning string // Already is true when a model of this name is in the catalogue. Already bool } // planServe works out what serving this version would mean. func planServe(st mirror.Status, files []mirror.Status, base string, fleet Source, cat CatalogAdmin, ) serveForm { f := serveForm{ Model: st.Model, Version: st.Version, Source: mirror.Entry{Model: st.Model, Version: st.Version, File: st.File}. URL(base), Engine: engineFor(st.File), Profile: "cpu", } // A catalogue entry names one source. A version made of several files has // no single one, so an entry for it would be an entry for one shard — which // loads as a broken model rather than failing to load. if len(files) > 1 { f.Refusal = fmt.Sprintf( "this version is %d files, and a catalogue entry names one source. "+ "Serve a single-file build (a GGUF), or give the engine the "+ "repository id so it fetches the shards itself", len(files)) return f } switch f.Engine { case "": f.Refusal = "nothing here reads " + extOf(st.File) + " files" return f case "llamacpp": f.Profile = "cpu" default: f.Profile = "gpu" // The control plane is not told which engines a node has installed, so // this is the nearest thing it can check honestly. It is also the thing // that was wrong here: fifteen gigabytes of safetensors arrived before // anybody noticed the fleet had no card to run them on. if gpus(fleet) == 0 { f.Warning = "no node in this fleet reports a GPU, and " + extOf(st.File) + " weights need vLLM or SGLang, which do. " + "A GGUF build of the same model runs on the CPU nodes" } } if cat != nil && cat.Declared(st.Model) { f.Already = true f.Refusal = "a model called " + st.Model + " is declared in configuration; edit it there rather than adding a second" } return f } // engineFor is the engine that reads this kind of weights file. func engineFor(file string) string { switch strings.ToLower(extOf(file)) { case ".gguf": return "llamacpp" case ".safetensors", ".bin": return "vllm" } return "" } func extOf(file string) string { if i := strings.LastIndex(file, "."); i >= 0 { return file[i:] } return file } func gpus(fleet Source) int { if fleet == nil { return 0 } n := 0 for _, node := range fleet.Nodes() { n += node.GPUTotal } return n } // buildModel turns the submitted form into a catalogue entry. func buildModel(get func(string) string) (catalog.Model, error) { m := catalog.Model{ Name: strings.TrimSpace(get("name")), Source: strings.TrimSpace(get("source")), RoutingProfile: "chat", } if m.Name == "" || m.Source == "" { return m, errors.New("a model needs a name and a source") } for _, a := range strings.Split(get("aliases"), ",") { if a = strings.TrimSpace(a); a != "" { m.Aliases = append(m.Aliases, a) } } ctxLen, err := positive(get("context_length"), "the context length") if err != nil { return m, err } m.ContextLength = ctxLen if v := strings.TrimSpace(get("max_output")); v != "" { if m.MaxOutput, err = positive(v, "the output limit"); err != nil { return m, err } } min, err := positive(get("replicas"), "the replica count") if err != nil { return m, err } m.Scaling = catalog.Scaling{Min: min, Max: min} engine := strings.TrimSpace(get("engine")) if engine == "" { return m, errors.New("which engine should run it?") } p := catalog.Profile{ Name: strings.TrimSpace(get("profile")), Engine: engine, Weight: 1, Priority: 100, } if p.Name == "" { p.Name = engine } // The pool, not the per-request limit: llama.cpp divides it across its // parallel slots, so the number a client may use is smaller and is what // context_length above says. if v := strings.TrimSpace(get("ctx_size")); v != "" { if _, err := positive(v, "the engine's context pool"); err != nil { return m, err } p.Args = []string{"--ctx-size", v} } if v := strings.TrimSpace(get("min_memory_gb")); v != "" { if p.MinMemoryGB, err = positive(v, "the memory requirement"); err != nil { return m, err } } if k, v := strings.TrimSpace(get("selector_key")), strings.TrimSpace(get("selector_value")); k != "" && v != "" { // What keeps a large model off the small nodes. A CPU profile has no // card to be short of, so without this every ready node looks like a // candidate and the first one picked is the one that gets OOM-killed. p.NodeSelector = map[string]string{k: v} } m.Profiles = []catalog.Profile{p} return m, nil } func positive(v, what string) (int, error) { n, err := strconv.Atoi(strings.TrimSpace(v)) if err != nil || n < 0 { return 0, fmt.Errorf("%s must be a whole number", what) } return n, nil }