Files
llmgw/internal/web/admin_serve.go
T
Ilker ManapandClaude Opus 5 2acd794022
CI / test (push) Successful in 1m42s
Stop placing engines on nodes that do not have them
A node's registry holds an adapter for every engine whether or not its binary
exists, the agent never said which were installed, and placement carried an
Engine field it had never once read. So a vLLM replica was as happily placed on
a node with no vLLM: the node registered healthy, the shard failed minutes
later with "executable file not found" in the engine's own words, and the
reconciler tried again.

That is how fifteen gigabytes of safetensors reached this cluster's mirror
before anybody noticed it had no card and no vLLM to run them with.

Agents now probe what is installed and report it — the protocol has carried
engine_versions since it was written and nothing ever filled it in. Placement
filters on the key set, `ctl nodes` has a column, and the mirror page's serve
warning is a fact instead of a guess about cards, which would have waved
through a fleet of eight-GPU nodes that never had vLLM.

Probing resolves binaries and runs none of them. The first version asked each
engine for --version, which is fine for an engine and not fine for the wrapper
scripts people put in front of one: a script that shifts past flags it does not
recognise and then execs the server will start an inference server, at agent
startup, on every node. The agent's own test suite installs exactly such a
script and caught it. So presence is a path lookup, and the version is left to
`ctl engine-check`, which is run by somebody who meant to run the engine.

SGLang is the exception and has to execute, because a module has no path to
look up — but the command is llmgw's own one-line import rather than the
engine's launcher. Its exit code is the answer: python3 is on almost every
machine, and treating a failed import as "installed" reported SGLang on every
node in a fleet that had none, which is the failure being fixed, inverted.

Probed once, not per reconnect, and bounded by the context that registration
waits on: an agent that cannot reach the control plane is worse than one
reporting an incomplete list.

A node that reports no engines at all is not excluded anywhere — by placement,
or by the page. That is what every agent older than this reports, and emptying
a fleet on a control plane upgrade is a worse failure than the one being fixed.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-27 16:11:55 +02:00

229 lines
6.7 KiB
Go

package web
import (
"context"
"errors"
"fmt"
"strconv"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
)
// CatalogAdmin adds a model to the catalogue in force.
type CatalogAdmin interface {
Add(ctx context.Context, m catalog.Model, by string) error
Declared(name string) bool
}
// serveForm is the offer to put one mirrored version into service.
//
// Everything a catalogue entry needs that the mirror already knows is filled
// in: the name, the source, and the engine that reads that kind of file. What
// is left is what nobody can infer — the context pool and how many replicas —
// and those are fields rather than guesses presented as facts.
type serveForm struct {
Model string
Version string
Source string
Engine string
// Profile is the profile name, which doubles as how a node_selector would
// be written later. Named for the hardware rather than the model, because
// a profile is about where a model runs.
Profile string
// Refusal is set when this version cannot be served as it stands, and says
// why. Offered and then failing at placement is worse than not offered.
Refusal string
// Warning is set when it can be served but probably will not run.
Warning string
// Already is true when a model of this name is in the catalogue.
Already bool
}
// planServe works out what serving this version would mean.
func planServe(st mirror.Status, files []mirror.Status, base string,
fleet Source, cat CatalogAdmin,
) serveForm {
f := serveForm{
Model: st.Model,
Version: st.Version,
Source: mirror.Entry{Model: st.Model, Version: st.Version, File: st.File}.
URL(base),
Engine: engineFor(st.File),
Profile: "cpu",
}
// A catalogue entry names one source. A version made of several files has
// no single one, so an entry for it would be an entry for one shard — which
// loads as a broken model rather than failing to load.
if len(files) > 1 {
f.Refusal = fmt.Sprintf(
"this version is %d files, and a catalogue entry names one source. "+
"Serve a single-file build (a GGUF), or give the engine the "+
"repository id so it fetches the shards itself",
len(files))
return f
}
switch f.Engine {
case "":
f.Refusal = "nothing here reads " + extOf(st.File) + " files"
return f
case "llamacpp":
f.Profile = "cpu"
default:
f.Profile = "gpu"
}
// Whether any node can actually run it. Agents report their installed
// engines now, so this is a fact rather than the guess it started as —
// which was cards, and would have passed a GPU node with no vLLM on it.
//
// A warning, not a refusal: an engine can be installed this afternoon, and
// a node that reports nothing is an older agent rather than an empty one.
if !runsAnywhere(fleet, f.Engine) {
f.Warning = "no node reports having " + f.Engine + " installed, and " +
extOf(st.File) + " weights need it"
if f.Engine != "llamacpp" && gpus(fleet) == 0 {
f.Warning += ". This fleet has no GPU either; a GGUF build of the " +
"same model runs on the CPU nodes"
}
}
if cat != nil && cat.Declared(st.Model) {
f.Already = true
f.Refusal = "a model called " + st.Model +
" is declared in configuration; edit it there rather than adding a second"
}
return f
}
// engineFor is the engine that reads this kind of weights file.
func engineFor(file string) string {
switch strings.ToLower(extOf(file)) {
case ".gguf":
return "llamacpp"
case ".safetensors", ".bin":
return "vllm"
}
return ""
}
func extOf(file string) string {
if i := strings.LastIndex(file, "."); i >= 0 {
return file[i:]
}
return file
}
// runsAnywhere reports whether some node has this engine.
//
// True when nothing is known, so a fleet of agents older than the engine probe
// is not told every model is unservable. That matches what placement does with
// the same absence, and the two disagreeing would be worse than either.
func runsAnywhere(fleet Source, engine string) bool {
if fleet == nil || engine == "" {
return true
}
anyReported := false
for _, n := range fleet.Nodes() {
if len(n.Engines) == 0 {
continue
}
anyReported = true
for _, e := range n.Engines {
if e == engine {
return true
}
}
}
return !anyReported
}
func gpus(fleet Source) int {
if fleet == nil {
return 0
}
n := 0
for _, node := range fleet.Nodes() {
n += node.GPUTotal
}
return n
}
// buildModel turns the submitted form into a catalogue entry.
func buildModel(get func(string) string) (catalog.Model, error) {
m := catalog.Model{
Name: strings.TrimSpace(get("name")),
Source: strings.TrimSpace(get("source")),
RoutingProfile: "chat",
}
if m.Name == "" || m.Source == "" {
return m, errors.New("a model needs a name and a source")
}
for _, a := range strings.Split(get("aliases"), ",") {
if a = strings.TrimSpace(a); a != "" {
m.Aliases = append(m.Aliases, a)
}
}
ctxLen, err := positive(get("context_length"), "the context length")
if err != nil {
return m, err
}
m.ContextLength = ctxLen
if v := strings.TrimSpace(get("max_output")); v != "" {
if m.MaxOutput, err = positive(v, "the output limit"); err != nil {
return m, err
}
}
min, err := positive(get("replicas"), "the replica count")
if err != nil {
return m, err
}
m.Scaling = catalog.Scaling{Min: min, Max: min}
engine := strings.TrimSpace(get("engine"))
if engine == "" {
return m, errors.New("which engine should run it?")
}
p := catalog.Profile{
Name: strings.TrimSpace(get("profile")), Engine: engine,
Weight: 1, Priority: 100,
}
if p.Name == "" {
p.Name = engine
}
// The pool, not the per-request limit: llama.cpp divides it across its
// parallel slots, so the number a client may use is smaller and is what
// context_length above says.
if v := strings.TrimSpace(get("ctx_size")); v != "" {
if _, err := positive(v, "the engine's context pool"); err != nil {
return m, err
}
p.Args = []string{"--ctx-size", v}
}
if v := strings.TrimSpace(get("min_memory_gb")); v != "" {
if p.MinMemoryGB, err = positive(v, "the memory requirement"); err != nil {
return m, err
}
}
if k, v := strings.TrimSpace(get("selector_key")), strings.TrimSpace(get("selector_value")); k != "" && v != "" {
// What keeps a large model off the small nodes. A CPU profile has no
// card to be short of, so without this every ready node looks like a
// candidate and the first one picked is the one that gets OOM-killed.
p.NodeSelector = map[string]string{k: v}
}
m.Profiles = []catalog.Profile{p}
return m, nil
}
func positive(v, what string) (int, error) {
n, err := strconv.Atoi(strings.TrimSpace(v))
if err != nil || n < 0 {
return 0, fmt.Errorf("%s must be a whole number", what)
}
return n, nil
}