Files
llmgw/internal/control/catalog/catalog.go
T
Ilker ManapandClaude Opus 5.5 d167686cf1 Stop a browser page, or a model definition, from running code on the nodes
The admin listener has no login. Reaching it is meant to mean holding the
control plane, so it binds to loopback and an operator reaches it over an
ssh tunnel — and so does every page open in that operator's browser. The
API decoded JSON whatever the Content-Type said, so a page could POST a
text/plain model definition, no preflight needed. A model's profile env
reached an engine the agent runs as root, unfiltered: LD_PRELOAD in a
definition was code execution on every node. A page that rebound its own
name to 127.0.0.1 could also read pprof's heap, which holds API keys.

The admin listener now answers only when Host is an address or localhost,
refuses requests a browser marks as cross-site (Origin, Sec-Fetch-Site),
and takes changes only as application/json, which no page can send across
origins without a preflight nobody grants. ctl sends that on every change,
bodiless DELETEs included; curl and ctl are otherwise unaffected.

Profile env that changes what runs rather than how — the dynamic loader's
LD_*, PATH, Python's PYTHON* start-up variables, BASH_ENV and their like —
is refused when a model is defined, and dropped again by the engine adapter
before the process starts, whatever sent the profile. None of the live
catalog's models sets any.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 00:29:52 +02:00

478 lines
17 KiB
Go

// Package catalog holds the desired state: which models exist, how each one can
// be run, and how many copies of it there should be (plan §6.1).
package catalog
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"slices"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
)
// Scaling is how many replicas a model should have.
type Scaling struct {
Min int `json:"min,omitempty" koanf:"min"`
Max int `json:"max,omitempty" koanf:"max"`
// IdleTimeout is how long an on-demand model may go unasked-for before its
// replicas are stopped. Zero takes the cluster default.
//
// Minutes rather than seconds, always. Stopping a replica throws away the
// KV cache prefix routing spent its effort filling, and a model that flaps
// in and out loses that entirely — so the saving has to be worth more than
// the cache, which it only is over a real idle period.
IdleTimeout time.Duration `json:"idle_timeout,omitempty" koanf:"idle_timeout"`
}
// OnDemand reports whether this model is loaded when someone asks for it.
//
// min 0 with a max above it, which needs no new field and reads correctly: keep
// none running, allow up to max. min 0 and max 0 is a model that is configured
// and deliberately not running, which is a different and equally useful thing —
// the difference between "load it when wanted" and "do not load it".
func (s Scaling) OnDemand() bool { return s.Min == 0 && s.Max > 0 }
// SLO is the latency target a model is expected to meet. `ctl bench` measures
// capacity against it (plan §6.2.1).
type SLO struct {
TTFTp99 time.Duration `koanf:"ttft_p99"`
TPOTp99 time.Duration `koanf:"tpot_p99"`
}
// Profile is how a model runs on one GPU class.
type Profile struct {
Name string `json:"name,omitempty" koanf:"name"`
Engine string `json:"engine,omitempty" koanf:"engine"`
GPUClass string `json:"gpu_class,omitempty" koanf:"gpu_class"`
TP int `json:"tp,omitempty" koanf:"tp"`
PP int `json:"pp,omitempty" koanf:"pp"`
// GPUsPerNode defaults to TP for a single-node profile.
GPUsPerNode int `json:"gpus_per_node,omitempty" koanf:"gpus_per_node"`
// Nodes above 1 makes this a multi-node profile, which requires a fabric.
Nodes int `json:"nodes,omitempty" koanf:"nodes"`
Args []string `json:"args,omitempty" koanf:"args"`
Env map[string]string `json:"env,omitempty" koanf:"env"`
// MinMemoryGB is the host RAM this profile needs to run.
//
// The number that matters is the model's working set: the weights, plus the
// KV cache for the whole context pool, plus the engine's own overhead. For
// a GPU profile that lives in the cards and this is small; for a CPU
// profile it is the whole thing, and getting it wrong means an OOM on a
// node that reported itself healthy.
//
// Zero means unconstrained, which is every profile that existed before
// this. A node that does not report its memory is not excluded either —
// an older agent must not have its nodes quietly emptied by an upgrade of
// the control plane.
MinMemoryGB int `json:"min_memory_gb,omitempty" koanf:"min_memory_gb"`
// MinDiskGB is the free space a node needs before it can hold these
// weights.
//
// Only checked when the node does not already have them: a node holding
// the file needs no room for it, and requiring the space twice would keep
// a model off the one node best placed to run it.
//
// Declared rather than derived, because the size of a model that has never
// been fetched anywhere is not knowable from here — and a wrong guess
// fails late, halfway through a download, which is the failure this exists
// to prevent. Zero means unconstrained, and a node that reports no storage
// is not excluded: unknown is not full.
MinDiskGB int `json:"min_disk_gb,omitempty" koanf:"min_disk_gb"`
// NodeSelector restricts this profile to nodes carrying every one of these
// labels, with these values.
//
// A CPU profile otherwise fits everywhere, because there is no card to be
// free — which is right for a 0.5B model and wrong for a 14B one. Nothing
// in the inventory says how much memory a node has, so the operator says
// which nodes are meant for a model instead of the scheduler inferring it.
// Agents have reported labels since the beginning; this is what reads them.
NodeSelector map[string]string `json:"node_selector,omitempty" koanf:"node_selector"`
// Weight is the router's capacity weight. It should come from ctl bench
// rather than a guess (plan §6.2.1).
Weight float64 `json:"weight,omitempty" koanf:"weight"`
// Priority orders profiles during placement; highest that fits wins.
Priority int `json:"priority,omitempty" koanf:"priority"`
// StartupTimeout bounds model loading. A 70B model across two nodes takes
// tens of minutes, so this is per profile rather than global.
StartupTimeout time.Duration `json:"startup_timeout,omitempty" koanf:"startup_timeout"`
}
// CPUOnly reports whether this profile needs no GPUs.
//
// Naming no GPU class is how a profile says so, and only engines that can
// actually serve without a card are allowed to leave it out — otherwise an
// omitted class would quietly become a replica placed on a machine with no
// hardware to run it.
func (p Profile) CPUOnly() bool { return p.GPUClass == "" && cpuCapable(p.Engine) }
// cpuCapable reports whether an engine can serve a model without a GPU.
//
// llama.cpp is the one that can, and it is what makes a cluster with no cards
// testable end to end.
func cpuCapable(engine string) bool { return engine == "llamacpp" }
// Model is a catalog entry as clients see it.
type Model struct {
Name string `json:"name,omitempty" koanf:"name"`
Aliases []string `json:"aliases,omitempty" koanf:"aliases"`
// Description is what this model is for, in a line.
//
// Written by whoever added it, because nobody else can say it. A registry
// offers a pipeline tag and a wall of README, and neither answers the
// question a person picking from a list is asking: which of these nine
// names should I send this request to.
//
// It reaches clients through /v1/models, which is the only view most of
// them ever see. A gateway serving nine models by name alone makes that
// choice by guesswork.
Description string `json:"description,omitempty" koanf:"description"`
// Source is the weights path or a HuggingFace id.
Source string `json:"source,omitempty" koanf:"source"`
Profiles []Profile `json:"profiles,omitempty" koanf:"profiles"`
Scaling Scaling `json:"scaling,omitempty" koanf:"scaling"`
Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"`
// RoutingProfile selects how the router scores endpoints for this model.
RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"`
SLO SLO `json:"slo,omitempty" koanf:"slo"`
// ContextLength is how many tokens one request may use, prompt and
// completion together, and MaxOutput bounds the completion alone. Zero
// means unknown, which is what every model was until now.
//
// The gateway knew neither, so an over-long prompt was not refused: it was
// admitted, given a replica, and failed at the engine, and a client sizing
// its prompts from /v1/models had nothing to read. Worse, the number is not
// the one an operator writes in the engine's arguments — llama.cpp divides
// --ctx-size across its parallel slots, so a pool of 8192 across four slots
// is 2048 a request, and nothing said so.
ContextLength int `json:"context_length,omitempty" koanf:"context_length"`
MaxOutput int `json:"max_output,omitempty" koanf:"max_output"`
// Capabilities is what this model can do beyond plain completion.
Capabilities Capabilities `json:"capabilities,omitempty" koanf:"capabilities"`
}
// Capabilities describes what a client may ask a model for.
//
// Declared by the operator rather than discovered, because there is no way to
// ask an engine what its model can do. That makes these claims, and a wrong
// claim is worse than none — which is why `ctl engine-check` verifies tool
// calling against a running engine rather than trusting this.
//
// The reason it exists: the gateway forwards a tools array to any model, and a
// model that cannot use them answers in prose. Every OpenAI-compatible client
// then behaves as though the model is broken, and the only way to find out was
// to try it. That cost an afternoon.
type Capabilities struct {
Tools bool `json:"tools,omitempty" koanf:"tools"`
Vision bool `json:"vision,omitempty" koanf:"vision"`
Embeddings bool `json:"embeddings,omitempty" koanf:"embeddings"`
}
// Catalog is the set of models the cluster should serve.
type Catalog struct {
models map[string]Model
aliases map[string]string
order []string
}
// New builds a catalog, rejecting definitions that would fail confusingly later.
func New(models []Model) (*Catalog, error) {
c := &Catalog{
models: make(map[string]Model, len(models)),
aliases: map[string]string{},
}
for i := range models {
m := models[i].normalize()
if err := m.validate(); err != nil {
return nil, err
}
if _, dup := c.models[m.Name]; dup {
return nil, fmt.Errorf("model %q is declared twice", m.Name)
}
c.models[m.Name] = m
c.order = append(c.order, m.Name)
}
// Aliases share a namespace with names. A collision would silently shadow a
// model at request time, so it has to fail at load.
for _, name := range c.order {
for _, a := range c.models[name].Aliases {
if _, clash := c.models[a]; clash {
return nil, fmt.Errorf("model %q: alias %q collides with another model name", name, a)
}
if prev, clash := c.aliases[a]; clash {
return nil, fmt.Errorf("alias %q is claimed by both %q and %q", a, prev, name)
}
c.aliases[a] = name
}
}
// Fallbacks are resolved now rather than at request time: a typo in a
// fallback should stop a deployment, not surface during an outage when the
// fallback is finally needed.
for _, name := range c.order {
for _, f := range c.models[name].Fallbacks {
if _, ok := c.Resolve(f); !ok {
return nil, fmt.Errorf("model %q: fallback %q is not in the catalog", name, f)
}
}
}
return c, nil
}
func (m Model) normalize() Model {
if m.Scaling.Min < 0 {
m.Scaling.Min = 0
}
// Max 0 used to be an omission to fill in, because a model with no maximum
// could not run. It is now also a statement: with min 0 it means configured
// and deliberately not running, which is a different and useful thing from
// "load it when wanted". Filling it in would silently turn every switched-
// off model into an on-demand one — including the ones already written that
// way in a running cluster's configuration.
if m.Scaling.Max == 0 && m.Scaling.Min > 0 {
m.Scaling.Max = m.Scaling.Min
}
for i := range m.Profiles {
p := &m.Profiles[i]
if p.Engine == "" {
p.Engine = "vllm"
}
if p.TP <= 0 {
p.TP = 1
}
if p.Nodes <= 0 {
p.Nodes = 1
}
switch {
case p.CPUOnly():
// Nothing to divide up: this profile asks for no cards at all.
p.GPUsPerNode = 0
case p.GPUsPerNode <= 0:
// The common case: a single-node profile uses TP GPUs on one node.
p.GPUsPerNode = p.TP / p.Nodes
if p.GPUsPerNode <= 0 {
p.GPUsPerNode = 1
}
}
if p.StartupTimeout <= 0 {
p.StartupTimeout = 20 * time.Minute
}
if p.Weight <= 0 {
p.Weight = 1
}
}
return m
}
func (m Model) validate() error {
if m.Name == "" {
return fmt.Errorf("a model has no name")
}
if m.Source == "" {
return fmt.Errorf("model %q: source is required", m.Name)
}
if len(m.Profiles) == 0 {
return fmt.Errorf("model %q: at least one profile is required", m.Name)
}
if m.Scaling.Max < m.Scaling.Min {
return fmt.Errorf("model %q: scaling max %d is below min %d", m.Name, m.Scaling.Max, m.Scaling.Min)
}
seen := map[string]struct{}{}
for _, p := range m.Profiles {
if p.Name == "" {
return fmt.Errorf("model %q: a profile has no name", m.Name)
}
if _, dup := seen[p.Name]; dup {
return fmt.Errorf("model %q: profile %q is declared twice", m.Name, p.Name)
}
seen[p.Name] = struct{}{}
for k := range p.Env {
if bad, why := engines.UnsafeEnv(k); bad {
return fmt.Errorf("model %q profile %q: env %s is not allowed: %s",
m.Name, p.Name, k, why)
}
}
if p.GPUClass == "" && !cpuCapable(p.Engine) {
return fmt.Errorf("model %q profile %q: gpu_class is required for engine %q",
m.Name, p.Name, p.Engine)
}
if p.CPUOnly() {
if p.Nodes > 1 {
return fmt.Errorf("model %q profile %q: a CPU profile cannot span %d nodes",
m.Name, p.Name, p.Nodes)
}
continue
}
// The arithmetic has to add up, or the engine is handed a shard spec it
// cannot satisfy and fails minutes later during model loading.
if want := p.GPUsPerNode * p.Nodes; want != p.TP*max(p.PP, 1) {
return fmt.Errorf("model %q profile %q: %d nodes x %d GPUs does not match TP %d x PP %d",
m.Name, p.Name, p.Nodes, p.GPUsPerNode, p.TP, max(p.PP, 1))
}
}
return nil
}
// Resolve maps a client-supplied name or alias onto a model.
func (c *Catalog) Resolve(name string) (Model, bool) {
if m, ok := c.models[name]; ok {
return m, true
}
if canonical, ok := c.aliases[name]; ok {
return c.models[canonical], true
}
return Model{}, false
}
// Models returns every model, in declaration order.
func (c *Catalog) Models() []Model {
out := make([]Model, 0, len(c.order))
for _, n := range c.order {
out = append(out, c.models[n])
}
return out
}
// Names returns the canonical model names, sorted.
func (c *Catalog) Names() []string {
out := slices.Clone(c.order)
slices.Sort(out)
return out
}
// AliasesOf returns the alternative names for a model.
func (c *Catalog) AliasesOf(name string) []string {
m, ok := c.Resolve(name)
if !ok {
return nil
}
return slices.Clone(m.Aliases)
}
// PlacementProfiles converts a model's profiles into placement's view.
func (m Model) PlacementProfiles() []placement.Profile {
out := make([]placement.Profile, 0, len(m.Profiles))
for _, p := range m.Profiles {
out = append(out, placement.Profile{
Name: p.Name,
Engine: p.Engine,
GPUClass: p.GPUClass,
GPUsPerNode: p.GPUsPerNode,
Nodes: p.Nodes,
Priority: p.Priority,
NodeSelector: p.NodeSelector,
MinMemoryGB: p.MinMemoryGB,
MinDiskGB: p.MinDiskGB,
// The model's source, so placement can prefer nodes that already
// hold the weights rather than one that must first move them.
Source: m.Source,
})
}
return out
}
// EngineProfile converts a catalog profile into the engine adapter's view.
func (m Model) EngineProfile(name string) (engines.Profile, bool) {
for _, p := range m.Profiles {
if p.Name != name {
continue
}
return engines.Profile{
Name: p.Name,
Engine: p.Engine,
GPUClass: p.GPUClass,
TP: p.TP,
PP: p.PP,
GPUsPerNode: p.GPUsPerNode,
Nodes: p.Nodes,
Args: slices.Clone(p.Args),
Env: p.Env,
ModelPath: m.Source,
ServedModelName: m.Name,
}, true
}
return engines.Profile{}, false
}
// ProfileByName returns one of a model's profiles.
func (m Model) ProfileByName(name string) (Profile, bool) {
for _, p := range m.Profiles {
if p.Name == name {
return p, true
}
}
return Profile{}, false
}
// String renders the catalog for `ctl models`.
func (c *Catalog) String() string {
var b strings.Builder
for _, m := range c.Models() {
fmt.Fprintf(&b, "%s (min %d, max %d)\n", m.Name, m.Scaling.Min, m.Scaling.Max)
for _, p := range m.Profiles {
fmt.Fprintf(&b, " %-20s %-10s %-12s tp=%d nodes=%d priority=%d\n",
p.Name, p.Engine, p.GPUClass, p.TP, p.Nodes, p.Priority)
}
}
return b.String()
}
// Simulated reports whether this model produces synthetic text rather than
// inference output.
//
// Derived from the profiles rather than configured, because an operator who
// added a sim profile and forgot the flag would be shipping placeholder text to
// users under a real model's name. A model counts as simulated only when every
// profile is simulated: one real profile means a request can land on a real
// engine, and a model that sometimes answers for real must not be labelled as
// though it never does.
func (m Model) Simulated() bool {
if len(m.Profiles) == 0 {
return false
}
for _, p := range m.Profiles {
if p.Engine != "sim" {
return false
}
}
return true
}
// Fingerprint identifies a profile's contents.
//
// The name is not enough. A profile can be edited — a memory requirement
// raised, a node label added, an engine argument changed — and keep its name,
// so a replica running the old definition is indistinguishable from one
// running the new. The reconciler does not move a running replica when its
// profile changes, which is right: churning a replica that is serving costs
// more than the imperfection. What was missing is any way to see that it had
// happened, and the answer to "why is this one on the wrong node" was in
// somebody's memory of what they edited an hour ago.
func (p Profile) Fingerprint() string {
b, err := json.Marshal(p)
if err != nil {
// A profile that will not marshal cannot be compared, and saying so by
// never matching is better than saying two differing ones are the same.
return "unmarshalable"
}
sum := sha256.Sum256(b)
return hex.EncodeToString(sum[:8])
}