CI / test (push) Successful in 1m35s
Fetching a model put it on the mirror and left the operator to hand-write a catalogue entry and run `ctl model add` on the control plane — for a source the page was already displaying, with an engine decided by the file extension it had just shown them. A present row now offers to serve it. The form opens on that row with the name, the mirror's own URL and the engine filled in, and asks for the two things nobody can infer: the context pool and how many replicas. What it refuses is the point. A version of several files is refused with the reason. A catalogue entry names one source, so an entry for a sharded model is an entry for one shard, which loads as a broken model rather than failing to load. Nothing said so before — the mirror happily holds a four-shard model that no catalogue entry can describe. Safetensors on a fleet with no GPU is warned about before anything is added. This is the failure we hit today: fifteen gigabytes of shards arrived, and only then did looking at a node show no card, no vLLM and no SGLang. The control plane is not told which engines a node has installed, so the check is on cards, which it does know — a warning rather than a refusal, because a card can be added tomorrow. A name already declared in configuration is refused here too, matching what the model service would say anyway, but before the form is filled in rather than after. Without a store there is no serve action at all: a model added at runtime would not survive a restart, and offering it then refusing is worse than not offering. The mirror page's dependencies became a struct. Three are optional interfaces and two are the same shape, which at a call site is a swap nobody notices until a page shows the wrong thing. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
200 lines
5.8 KiB
Go
200 lines
5.8 KiB
Go
package web
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
|
)
|
|
|
|
// CatalogAdmin adds a model to the catalogue in force.
|
|
type CatalogAdmin interface {
|
|
Add(ctx context.Context, m catalog.Model, by string) error
|
|
Declared(name string) bool
|
|
}
|
|
|
|
// serveForm is the offer to put one mirrored version into service.
|
|
//
|
|
// Everything a catalogue entry needs that the mirror already knows is filled
|
|
// in: the name, the source, and the engine that reads that kind of file. What
|
|
// is left is what nobody can infer — the context pool and how many replicas —
|
|
// and those are fields rather than guesses presented as facts.
|
|
type serveForm struct {
|
|
Model string
|
|
Version string
|
|
Source string
|
|
Engine string
|
|
// Profile is the profile name, which doubles as how a node_selector would
|
|
// be written later. Named for the hardware rather than the model, because
|
|
// a profile is about where a model runs.
|
|
Profile string
|
|
|
|
// Refusal is set when this version cannot be served as it stands, and says
|
|
// why. Offered and then failing at placement is worse than not offered.
|
|
Refusal string
|
|
// Warning is set when it can be served but probably will not run.
|
|
Warning string
|
|
// Already is true when a model of this name is in the catalogue.
|
|
Already bool
|
|
}
|
|
|
|
// planServe works out what serving this version would mean.
|
|
func planServe(st mirror.Status, files []mirror.Status, base string,
|
|
fleet Source, cat CatalogAdmin,
|
|
) serveForm {
|
|
f := serveForm{
|
|
Model: st.Model,
|
|
Version: st.Version,
|
|
Source: mirror.Entry{Model: st.Model, Version: st.Version, File: st.File}.
|
|
URL(base),
|
|
Engine: engineFor(st.File),
|
|
Profile: "cpu",
|
|
}
|
|
|
|
// A catalogue entry names one source. A version made of several files has
|
|
// no single one, so an entry for it would be an entry for one shard — which
|
|
// loads as a broken model rather than failing to load.
|
|
if len(files) > 1 {
|
|
f.Refusal = fmt.Sprintf(
|
|
"this version is %d files, and a catalogue entry names one source. "+
|
|
"Serve a single-file build (a GGUF), or give the engine the "+
|
|
"repository id so it fetches the shards itself",
|
|
len(files))
|
|
return f
|
|
}
|
|
|
|
switch f.Engine {
|
|
case "":
|
|
f.Refusal = "nothing here reads " + extOf(st.File) + " files"
|
|
return f
|
|
case "llamacpp":
|
|
f.Profile = "cpu"
|
|
default:
|
|
f.Profile = "gpu"
|
|
// The control plane is not told which engines a node has installed, so
|
|
// this is the nearest thing it can check honestly. It is also the thing
|
|
// that was wrong here: fifteen gigabytes of safetensors arrived before
|
|
// anybody noticed the fleet had no card to run them on.
|
|
if gpus(fleet) == 0 {
|
|
f.Warning = "no node in this fleet reports a GPU, and " +
|
|
extOf(st.File) + " weights need vLLM or SGLang, which do. " +
|
|
"A GGUF build of the same model runs on the CPU nodes"
|
|
}
|
|
}
|
|
if cat != nil && cat.Declared(st.Model) {
|
|
f.Already = true
|
|
f.Refusal = "a model called " + st.Model +
|
|
" is declared in configuration; edit it there rather than adding a second"
|
|
}
|
|
return f
|
|
}
|
|
|
|
// engineFor is the engine that reads this kind of weights file.
|
|
func engineFor(file string) string {
|
|
switch strings.ToLower(extOf(file)) {
|
|
case ".gguf":
|
|
return "llamacpp"
|
|
case ".safetensors", ".bin":
|
|
return "vllm"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func extOf(file string) string {
|
|
if i := strings.LastIndex(file, "."); i >= 0 {
|
|
return file[i:]
|
|
}
|
|
return file
|
|
}
|
|
|
|
func gpus(fleet Source) int {
|
|
if fleet == nil {
|
|
return 0
|
|
}
|
|
n := 0
|
|
for _, node := range fleet.Nodes() {
|
|
n += node.GPUTotal
|
|
}
|
|
return n
|
|
}
|
|
|
|
// buildModel turns the submitted form into a catalogue entry.
|
|
func buildModel(get func(string) string) (catalog.Model, error) {
|
|
m := catalog.Model{
|
|
Name: strings.TrimSpace(get("name")),
|
|
Source: strings.TrimSpace(get("source")),
|
|
RoutingProfile: "chat",
|
|
}
|
|
if m.Name == "" || m.Source == "" {
|
|
return m, errors.New("a model needs a name and a source")
|
|
}
|
|
for _, a := range strings.Split(get("aliases"), ",") {
|
|
if a = strings.TrimSpace(a); a != "" {
|
|
m.Aliases = append(m.Aliases, a)
|
|
}
|
|
}
|
|
|
|
ctxLen, err := positive(get("context_length"), "the context length")
|
|
if err != nil {
|
|
return m, err
|
|
}
|
|
m.ContextLength = ctxLen
|
|
if v := strings.TrimSpace(get("max_output")); v != "" {
|
|
if m.MaxOutput, err = positive(v, "the output limit"); err != nil {
|
|
return m, err
|
|
}
|
|
}
|
|
|
|
min, err := positive(get("replicas"), "the replica count")
|
|
if err != nil {
|
|
return m, err
|
|
}
|
|
m.Scaling = catalog.Scaling{Min: min, Max: min}
|
|
|
|
engine := strings.TrimSpace(get("engine"))
|
|
if engine == "" {
|
|
return m, errors.New("which engine should run it?")
|
|
}
|
|
p := catalog.Profile{
|
|
Name: strings.TrimSpace(get("profile")), Engine: engine,
|
|
Weight: 1, Priority: 100,
|
|
}
|
|
if p.Name == "" {
|
|
p.Name = engine
|
|
}
|
|
// The pool, not the per-request limit: llama.cpp divides it across its
|
|
// parallel slots, so the number a client may use is smaller and is what
|
|
// context_length above says.
|
|
if v := strings.TrimSpace(get("ctx_size")); v != "" {
|
|
if _, err := positive(v, "the engine's context pool"); err != nil {
|
|
return m, err
|
|
}
|
|
p.Args = []string{"--ctx-size", v}
|
|
}
|
|
if v := strings.TrimSpace(get("min_memory_gb")); v != "" {
|
|
if p.MinMemoryGB, err = positive(v, "the memory requirement"); err != nil {
|
|
return m, err
|
|
}
|
|
}
|
|
if k, v := strings.TrimSpace(get("selector_key")), strings.TrimSpace(get("selector_value")); k != "" && v != "" {
|
|
// What keeps a large model off the small nodes. A CPU profile has no
|
|
// card to be short of, so without this every ready node looks like a
|
|
// candidate and the first one picked is the one that gets OOM-killed.
|
|
p.NodeSelector = map[string]string{k: v}
|
|
}
|
|
m.Profiles = []catalog.Profile{p}
|
|
return m, nil
|
|
}
|
|
|
|
func positive(v, what string) (int, error) {
|
|
n, err := strconv.Atoi(strings.TrimSpace(v))
|
|
if err != nil || n < 0 {
|
|
return 0, fmt.Errorf("%s must be a whole number", what)
|
|
}
|
|
return n, nil
|
|
}
|