Files
llmgw/internal/control/catalog/catalog.go
T
Ilker ManapandClaude Opus 5 72b03a1a1c
CI / test (push) Successful in 1m31s
Say which model is simulated instead of letting it look broken
big-model runs on the sim engine, whose whole vocabulary is the Greek
alphabet, cycled at a fixed rate until max_tokens. That is what it is for:
with no H100s it is the only thing exercising the TP8 two-node path, the
fabric invariant and rebuild-after-node-loss. The output is the point.

What was wrong is that nothing said so. /v1/models returned it with
owned_by "llmgw", identical to the real model, and the chat picker sorted
purely by name — so "big-model" led the list, was preselected by the
browser, and anyone who typed a question and pressed Send got Greek
letters back with no explanation. Reported as the model spitting out crap,
which is a fair reading of what it looks like from outside.

Simulated is derived from the profiles rather than configured: an operator
who adds a sim profile and forgets a flag would be shipping placeholder
text under a real model's name. A model counts as simulated only when
every profile is, because one real profile means the request can still
come back as inference.

The marker on /v1/models is owned_by "llmgw-sim", which is already what
the sim engine calls itself on its own /v1/models — the gateway was
flattening a distinction that existed one layer down.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 17:11:56 +02:00

337 lines
9.6 KiB
Go

// Package catalog holds the desired state: which models exist, how each one can
// be run, and how many copies of it there should be (plan §6.1).
package catalog
import (
"fmt"
"slices"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
)
// Scaling is how many replicas a model should have.
type Scaling struct {
Min int `koanf:"min"`
Max int `koanf:"max"`
}
// SLO is the latency target a model is expected to meet. `ctl bench` measures
// capacity against it (plan §6.2.1).
type SLO struct {
TTFTp99 time.Duration `koanf:"ttft_p99"`
TPOTp99 time.Duration `koanf:"tpot_p99"`
}
// Profile is how a model runs on one GPU class.
type Profile struct {
Name string `koanf:"name"`
Engine string `koanf:"engine"`
GPUClass string `koanf:"gpu_class"`
TP int `koanf:"tp"`
PP int `koanf:"pp"`
// GPUsPerNode defaults to TP for a single-node profile.
GPUsPerNode int `koanf:"gpus_per_node"`
// Nodes above 1 makes this a multi-node profile, which requires a fabric.
Nodes int `koanf:"nodes"`
Args []string `koanf:"args"`
Env map[string]string `koanf:"env"`
// Weight is the router's capacity weight. It should come from ctl bench
// rather than a guess (plan §6.2.1).
Weight float64 `koanf:"weight"`
// Priority orders profiles during placement; highest that fits wins.
Priority int `koanf:"priority"`
// StartupTimeout bounds model loading. A 70B model across two nodes takes
// tens of minutes, so this is per profile rather than global.
StartupTimeout time.Duration `koanf:"startup_timeout"`
}
// CPUOnly reports whether this profile needs no GPUs.
//
// Naming no GPU class is how a profile says so, and only engines that can
// actually serve without a card are allowed to leave it out — otherwise an
// omitted class would quietly become a replica placed on a machine with no
// hardware to run it.
func (p Profile) CPUOnly() bool { return p.GPUClass == "" && cpuCapable(p.Engine) }
// cpuCapable reports whether an engine can serve a model without a GPU.
//
// llama.cpp is the one that can, and it is what makes a cluster with no cards
// testable end to end.
func cpuCapable(engine string) bool { return engine == "llamacpp" }
// Model is a catalog entry as clients see it.
type Model struct {
Name string `koanf:"name"`
Aliases []string `koanf:"aliases"`
// Source is the weights path or a HuggingFace id.
Source string `koanf:"source"`
Profiles []Profile `koanf:"profiles"`
Scaling Scaling `koanf:"scaling"`
Fallbacks []string `koanf:"fallbacks"`
// RoutingProfile selects how the router scores endpoints for this model.
RoutingProfile string `koanf:"routing_profile"`
SLO SLO `koanf:"slo"`
}
// Catalog is the set of models the cluster should serve.
type Catalog struct {
models map[string]Model
aliases map[string]string
order []string
}
// New builds a catalog, rejecting definitions that would fail confusingly later.
func New(models []Model) (*Catalog, error) {
c := &Catalog{
models: make(map[string]Model, len(models)),
aliases: map[string]string{},
}
for i := range models {
m := models[i].normalize()
if err := m.validate(); err != nil {
return nil, err
}
if _, dup := c.models[m.Name]; dup {
return nil, fmt.Errorf("model %q is declared twice", m.Name)
}
c.models[m.Name] = m
c.order = append(c.order, m.Name)
}
// Aliases share a namespace with names. A collision would silently shadow a
// model at request time, so it has to fail at load.
for _, name := range c.order {
for _, a := range c.models[name].Aliases {
if _, clash := c.models[a]; clash {
return nil, fmt.Errorf("model %q: alias %q collides with another model name", name, a)
}
if prev, clash := c.aliases[a]; clash {
return nil, fmt.Errorf("alias %q is claimed by both %q and %q", a, prev, name)
}
c.aliases[a] = name
}
}
// Fallbacks are resolved now rather than at request time: a typo in a
// fallback should stop a deployment, not surface during an outage when the
// fallback is finally needed.
for _, name := range c.order {
for _, f := range c.models[name].Fallbacks {
if _, ok := c.Resolve(f); !ok {
return nil, fmt.Errorf("model %q: fallback %q is not in the catalog", name, f)
}
}
}
return c, nil
}
func (m Model) normalize() Model {
if m.Scaling.Min < 0 {
m.Scaling.Min = 0
}
if m.Scaling.Max == 0 {
m.Scaling.Max = max(m.Scaling.Min, 1)
}
for i := range m.Profiles {
p := &m.Profiles[i]
if p.Engine == "" {
p.Engine = "vllm"
}
if p.TP <= 0 {
p.TP = 1
}
if p.Nodes <= 0 {
p.Nodes = 1
}
switch {
case p.CPUOnly():
// Nothing to divide up: this profile asks for no cards at all.
p.GPUsPerNode = 0
case p.GPUsPerNode <= 0:
// The common case: a single-node profile uses TP GPUs on one node.
p.GPUsPerNode = p.TP / p.Nodes
if p.GPUsPerNode <= 0 {
p.GPUsPerNode = 1
}
}
if p.StartupTimeout <= 0 {
p.StartupTimeout = 20 * time.Minute
}
if p.Weight <= 0 {
p.Weight = 1
}
}
return m
}
func (m Model) validate() error {
if m.Name == "" {
return fmt.Errorf("a model has no name")
}
if m.Source == "" {
return fmt.Errorf("model %q: source is required", m.Name)
}
if len(m.Profiles) == 0 {
return fmt.Errorf("model %q: at least one profile is required", m.Name)
}
if m.Scaling.Max < m.Scaling.Min {
return fmt.Errorf("model %q: scaling max %d is below min %d", m.Name, m.Scaling.Max, m.Scaling.Min)
}
seen := map[string]struct{}{}
for _, p := range m.Profiles {
if p.Name == "" {
return fmt.Errorf("model %q: a profile has no name", m.Name)
}
if _, dup := seen[p.Name]; dup {
return fmt.Errorf("model %q: profile %q is declared twice", m.Name, p.Name)
}
seen[p.Name] = struct{}{}
if p.GPUClass == "" && !cpuCapable(p.Engine) {
return fmt.Errorf("model %q profile %q: gpu_class is required for engine %q",
m.Name, p.Name, p.Engine)
}
if p.CPUOnly() {
if p.Nodes > 1 {
return fmt.Errorf("model %q profile %q: a CPU profile cannot span %d nodes",
m.Name, p.Name, p.Nodes)
}
continue
}
// The arithmetic has to add up, or the engine is handed a shard spec it
// cannot satisfy and fails minutes later during model loading.
if want := p.GPUsPerNode * p.Nodes; want != p.TP*max(p.PP, 1) {
return fmt.Errorf("model %q profile %q: %d nodes x %d GPUs does not match TP %d x PP %d",
m.Name, p.Name, p.Nodes, p.GPUsPerNode, p.TP, max(p.PP, 1))
}
}
return nil
}
// Resolve maps a client-supplied name or alias onto a model.
func (c *Catalog) Resolve(name string) (Model, bool) {
if m, ok := c.models[name]; ok {
return m, true
}
if canonical, ok := c.aliases[name]; ok {
return c.models[canonical], true
}
return Model{}, false
}
// Models returns every model, in declaration order.
func (c *Catalog) Models() []Model {
out := make([]Model, 0, len(c.order))
for _, n := range c.order {
out = append(out, c.models[n])
}
return out
}
// Names returns the canonical model names, sorted.
func (c *Catalog) Names() []string {
out := slices.Clone(c.order)
slices.Sort(out)
return out
}
// AliasesOf returns the alternative names for a model.
func (c *Catalog) AliasesOf(name string) []string {
m, ok := c.Resolve(name)
if !ok {
return nil
}
return slices.Clone(m.Aliases)
}
// PlacementProfiles converts a model's profiles into placement's view.
func (m Model) PlacementProfiles() []placement.Profile {
out := make([]placement.Profile, 0, len(m.Profiles))
for _, p := range m.Profiles {
out = append(out, placement.Profile{
Name: p.Name,
Engine: p.Engine,
GPUClass: p.GPUClass,
GPUsPerNode: p.GPUsPerNode,
Nodes: p.Nodes,
Priority: p.Priority,
})
}
return out
}
// EngineProfile converts a catalog profile into the engine adapter's view.
func (m Model) EngineProfile(name string) (engines.Profile, bool) {
for _, p := range m.Profiles {
if p.Name != name {
continue
}
return engines.Profile{
Name: p.Name,
Engine: p.Engine,
GPUClass: p.GPUClass,
TP: p.TP,
PP: p.PP,
GPUsPerNode: p.GPUsPerNode,
Nodes: p.Nodes,
Args: slices.Clone(p.Args),
Env: p.Env,
ModelPath: m.Source,
ServedModelName: m.Name,
}, true
}
return engines.Profile{}, false
}
// ProfileByName returns one of a model's profiles.
func (m Model) ProfileByName(name string) (Profile, bool) {
for _, p := range m.Profiles {
if p.Name == name {
return p, true
}
}
return Profile{}, false
}
// String renders the catalog for `ctl models`.
func (c *Catalog) String() string {
var b strings.Builder
for _, m := range c.Models() {
fmt.Fprintf(&b, "%s (min %d, max %d)\n", m.Name, m.Scaling.Min, m.Scaling.Max)
for _, p := range m.Profiles {
fmt.Fprintf(&b, " %-20s %-10s %-12s tp=%d nodes=%d priority=%d\n",
p.Name, p.Engine, p.GPUClass, p.TP, p.Nodes, p.Priority)
}
}
return b.String()
}
// Simulated reports whether this model produces synthetic text rather than
// inference output.
//
// Derived from the profiles rather than configured, because an operator who
// added a sim profile and forgot the flag would be shipping placeholder text to
// users under a real model's name. A model counts as simulated only when every
// profile is simulated: one real profile means a request can land on a real
// engine, and a model that sometimes answers for real must not be labelled as
// though it never does.
func (m Model) Simulated() bool {
if len(m.Profiles) == 0 {
return false
}
for _, p := range m.Profiles {
if p.Engine != "sim" {
return false
}
}
return true
}