CI / test (push) Successful in 1m32s
Three things the gateway did not know about its own models, all of which surfaced this week as somebody else's bug. Context length. Nothing read it, /v1/models did not publish it, and an over-long prompt was not refused: it was admitted, scheduled, given a replica and a slot, and rejected by the engine in whatever shape that engine chose. A client sizing prompts from /v1/models had nothing to read. It is now declared per model, published when known, and checked before authentication work or a queue slot is spent — with a ten percent margin, because the estimate is bytes over four and being a few percent out must not turn a valid request into a refusal. Capabilities. The gateway forwards a tools array to any model, and one that cannot use them answers in prose; every OpenAI-compatible client then behaves as though the model is broken, and the only way to find out was to try it. Declared rather than discovered, because no engine will say — which is why these are claims, and why the next commit makes engine-check verify the tool-calling one instead of trusting it. And a bug found while wiring the first: the token estimate was taken from the routing prefix. That prefix stops at 2 KB by design, so every prompt above it estimated as 512 tokens however long it really was, and TPM limits undercounted exactly the requests that consume the most. Prefix routing wants a bounded head of the conversation; a token estimate wants all of it. They are now separate functions, and structured multimodal content counts rather than measuring zero. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
381 lines
12 KiB
Go
381 lines
12 KiB
Go
// Package catalog holds the desired state: which models exist, how each one can
|
|
// be run, and how many copies of it there should be (plan §6.1).
|
|
package catalog
|
|
|
|
import (
|
|
"fmt"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
|
)
|
|
|
|
// Scaling is how many replicas a model should have.
|
|
type Scaling struct {
|
|
Min int `koanf:"min"`
|
|
Max int `koanf:"max"`
|
|
}
|
|
|
|
// SLO is the latency target a model is expected to meet. `ctl bench` measures
|
|
// capacity against it (plan §6.2.1).
|
|
type SLO struct {
|
|
TTFTp99 time.Duration `koanf:"ttft_p99"`
|
|
TPOTp99 time.Duration `koanf:"tpot_p99"`
|
|
}
|
|
|
|
// Profile is how a model runs on one GPU class.
|
|
type Profile struct {
|
|
Name string `koanf:"name"`
|
|
Engine string `koanf:"engine"`
|
|
GPUClass string `koanf:"gpu_class"`
|
|
|
|
TP int `koanf:"tp"`
|
|
PP int `koanf:"pp"`
|
|
// GPUsPerNode defaults to TP for a single-node profile.
|
|
GPUsPerNode int `koanf:"gpus_per_node"`
|
|
// Nodes above 1 makes this a multi-node profile, which requires a fabric.
|
|
Nodes int `koanf:"nodes"`
|
|
|
|
Args []string `koanf:"args"`
|
|
Env map[string]string `koanf:"env"`
|
|
|
|
// NodeSelector restricts this profile to nodes carrying every one of these
|
|
// labels, with these values.
|
|
//
|
|
// A CPU profile otherwise fits everywhere, because there is no card to be
|
|
// free — which is right for a 0.5B model and wrong for a 14B one. Nothing
|
|
// in the inventory says how much memory a node has, so the operator says
|
|
// which nodes are meant for a model instead of the scheduler inferring it.
|
|
// Agents have reported labels since the beginning; this is what reads them.
|
|
NodeSelector map[string]string `koanf:"node_selector"`
|
|
|
|
// Weight is the router's capacity weight. It should come from ctl bench
|
|
// rather than a guess (plan §6.2.1).
|
|
Weight float64 `koanf:"weight"`
|
|
// Priority orders profiles during placement; highest that fits wins.
|
|
Priority int `koanf:"priority"`
|
|
// StartupTimeout bounds model loading. A 70B model across two nodes takes
|
|
// tens of minutes, so this is per profile rather than global.
|
|
StartupTimeout time.Duration `koanf:"startup_timeout"`
|
|
}
|
|
|
|
// CPUOnly reports whether this profile needs no GPUs.
|
|
//
|
|
// Naming no GPU class is how a profile says so, and only engines that can
|
|
// actually serve without a card are allowed to leave it out — otherwise an
|
|
// omitted class would quietly become a replica placed on a machine with no
|
|
// hardware to run it.
|
|
func (p Profile) CPUOnly() bool { return p.GPUClass == "" && cpuCapable(p.Engine) }
|
|
|
|
// cpuCapable reports whether an engine can serve a model without a GPU.
|
|
//
|
|
// llama.cpp is the one that can, and it is what makes a cluster with no cards
|
|
// testable end to end.
|
|
func cpuCapable(engine string) bool { return engine == "llamacpp" }
|
|
|
|
// Model is a catalog entry as clients see it.
|
|
type Model struct {
|
|
Name string `koanf:"name"`
|
|
Aliases []string `koanf:"aliases"`
|
|
// Source is the weights path or a HuggingFace id.
|
|
Source string `koanf:"source"`
|
|
Profiles []Profile `koanf:"profiles"`
|
|
Scaling Scaling `koanf:"scaling"`
|
|
Fallbacks []string `koanf:"fallbacks"`
|
|
// RoutingProfile selects how the router scores endpoints for this model.
|
|
RoutingProfile string `koanf:"routing_profile"`
|
|
SLO SLO `koanf:"slo"`
|
|
|
|
// ContextLength is how many tokens one request may use, prompt and
|
|
// completion together, and MaxOutput bounds the completion alone. Zero
|
|
// means unknown, which is what every model was until now.
|
|
//
|
|
// The gateway knew neither, so an over-long prompt was not refused: it was
|
|
// admitted, given a replica, and failed at the engine, and a client sizing
|
|
// its prompts from /v1/models had nothing to read. Worse, the number is not
|
|
// the one an operator writes in the engine's arguments — llama.cpp divides
|
|
// --ctx-size across its parallel slots, so a pool of 8192 across four slots
|
|
// is 2048 a request, and nothing said so.
|
|
ContextLength int `koanf:"context_length"`
|
|
MaxOutput int `koanf:"max_output"`
|
|
|
|
// Capabilities is what this model can do beyond plain completion.
|
|
Capabilities Capabilities `koanf:"capabilities"`
|
|
}
|
|
|
|
// Capabilities describes what a client may ask a model for.
|
|
//
|
|
// Declared by the operator rather than discovered, because there is no way to
|
|
// ask an engine what its model can do. That makes these claims, and a wrong
|
|
// claim is worse than none — which is why `ctl engine-check` verifies tool
|
|
// calling against a running engine rather than trusting this.
|
|
//
|
|
// The reason it exists: the gateway forwards a tools array to any model, and a
|
|
// model that cannot use them answers in prose. Every OpenAI-compatible client
|
|
// then behaves as though the model is broken, and the only way to find out was
|
|
// to try it. That cost an afternoon.
|
|
type Capabilities struct {
|
|
Tools bool `koanf:"tools"`
|
|
Vision bool `koanf:"vision"`
|
|
Embeddings bool `koanf:"embeddings"`
|
|
}
|
|
|
|
// Catalog is the set of models the cluster should serve.
|
|
type Catalog struct {
|
|
models map[string]Model
|
|
aliases map[string]string
|
|
order []string
|
|
}
|
|
|
|
// New builds a catalog, rejecting definitions that would fail confusingly later.
|
|
func New(models []Model) (*Catalog, error) {
|
|
c := &Catalog{
|
|
models: make(map[string]Model, len(models)),
|
|
aliases: map[string]string{},
|
|
}
|
|
|
|
for i := range models {
|
|
m := models[i].normalize()
|
|
if err := m.validate(); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, dup := c.models[m.Name]; dup {
|
|
return nil, fmt.Errorf("model %q is declared twice", m.Name)
|
|
}
|
|
c.models[m.Name] = m
|
|
c.order = append(c.order, m.Name)
|
|
}
|
|
|
|
// Aliases share a namespace with names. A collision would silently shadow a
|
|
// model at request time, so it has to fail at load.
|
|
for _, name := range c.order {
|
|
for _, a := range c.models[name].Aliases {
|
|
if _, clash := c.models[a]; clash {
|
|
return nil, fmt.Errorf("model %q: alias %q collides with another model name", name, a)
|
|
}
|
|
if prev, clash := c.aliases[a]; clash {
|
|
return nil, fmt.Errorf("alias %q is claimed by both %q and %q", a, prev, name)
|
|
}
|
|
c.aliases[a] = name
|
|
}
|
|
}
|
|
|
|
// Fallbacks are resolved now rather than at request time: a typo in a
|
|
// fallback should stop a deployment, not surface during an outage when the
|
|
// fallback is finally needed.
|
|
for _, name := range c.order {
|
|
for _, f := range c.models[name].Fallbacks {
|
|
if _, ok := c.Resolve(f); !ok {
|
|
return nil, fmt.Errorf("model %q: fallback %q is not in the catalog", name, f)
|
|
}
|
|
}
|
|
}
|
|
return c, nil
|
|
}
|
|
|
|
func (m Model) normalize() Model {
|
|
if m.Scaling.Min < 0 {
|
|
m.Scaling.Min = 0
|
|
}
|
|
if m.Scaling.Max == 0 {
|
|
m.Scaling.Max = max(m.Scaling.Min, 1)
|
|
}
|
|
for i := range m.Profiles {
|
|
p := &m.Profiles[i]
|
|
if p.Engine == "" {
|
|
p.Engine = "vllm"
|
|
}
|
|
if p.TP <= 0 {
|
|
p.TP = 1
|
|
}
|
|
if p.Nodes <= 0 {
|
|
p.Nodes = 1
|
|
}
|
|
switch {
|
|
case p.CPUOnly():
|
|
// Nothing to divide up: this profile asks for no cards at all.
|
|
p.GPUsPerNode = 0
|
|
case p.GPUsPerNode <= 0:
|
|
// The common case: a single-node profile uses TP GPUs on one node.
|
|
p.GPUsPerNode = p.TP / p.Nodes
|
|
if p.GPUsPerNode <= 0 {
|
|
p.GPUsPerNode = 1
|
|
}
|
|
}
|
|
if p.StartupTimeout <= 0 {
|
|
p.StartupTimeout = 20 * time.Minute
|
|
}
|
|
if p.Weight <= 0 {
|
|
p.Weight = 1
|
|
}
|
|
}
|
|
return m
|
|
}
|
|
|
|
func (m Model) validate() error {
|
|
if m.Name == "" {
|
|
return fmt.Errorf("a model has no name")
|
|
}
|
|
if m.Source == "" {
|
|
return fmt.Errorf("model %q: source is required", m.Name)
|
|
}
|
|
if len(m.Profiles) == 0 {
|
|
return fmt.Errorf("model %q: at least one profile is required", m.Name)
|
|
}
|
|
if m.Scaling.Max < m.Scaling.Min {
|
|
return fmt.Errorf("model %q: scaling max %d is below min %d", m.Name, m.Scaling.Max, m.Scaling.Min)
|
|
}
|
|
|
|
seen := map[string]struct{}{}
|
|
for _, p := range m.Profiles {
|
|
if p.Name == "" {
|
|
return fmt.Errorf("model %q: a profile has no name", m.Name)
|
|
}
|
|
if _, dup := seen[p.Name]; dup {
|
|
return fmt.Errorf("model %q: profile %q is declared twice", m.Name, p.Name)
|
|
}
|
|
seen[p.Name] = struct{}{}
|
|
|
|
if p.GPUClass == "" && !cpuCapable(p.Engine) {
|
|
return fmt.Errorf("model %q profile %q: gpu_class is required for engine %q",
|
|
m.Name, p.Name, p.Engine)
|
|
}
|
|
if p.CPUOnly() {
|
|
if p.Nodes > 1 {
|
|
return fmt.Errorf("model %q profile %q: a CPU profile cannot span %d nodes",
|
|
m.Name, p.Name, p.Nodes)
|
|
}
|
|
continue
|
|
}
|
|
// The arithmetic has to add up, or the engine is handed a shard spec it
|
|
// cannot satisfy and fails minutes later during model loading.
|
|
if want := p.GPUsPerNode * p.Nodes; want != p.TP*max(p.PP, 1) {
|
|
return fmt.Errorf("model %q profile %q: %d nodes x %d GPUs does not match TP %d x PP %d",
|
|
m.Name, p.Name, p.Nodes, p.GPUsPerNode, p.TP, max(p.PP, 1))
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Resolve maps a client-supplied name or alias onto a model.
|
|
func (c *Catalog) Resolve(name string) (Model, bool) {
|
|
if m, ok := c.models[name]; ok {
|
|
return m, true
|
|
}
|
|
if canonical, ok := c.aliases[name]; ok {
|
|
return c.models[canonical], true
|
|
}
|
|
return Model{}, false
|
|
}
|
|
|
|
// Models returns every model, in declaration order.
|
|
func (c *Catalog) Models() []Model {
|
|
out := make([]Model, 0, len(c.order))
|
|
for _, n := range c.order {
|
|
out = append(out, c.models[n])
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Names returns the canonical model names, sorted.
|
|
func (c *Catalog) Names() []string {
|
|
out := slices.Clone(c.order)
|
|
slices.Sort(out)
|
|
return out
|
|
}
|
|
|
|
// AliasesOf returns the alternative names for a model.
|
|
func (c *Catalog) AliasesOf(name string) []string {
|
|
m, ok := c.Resolve(name)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return slices.Clone(m.Aliases)
|
|
}
|
|
|
|
// PlacementProfiles converts a model's profiles into placement's view.
|
|
func (m Model) PlacementProfiles() []placement.Profile {
|
|
out := make([]placement.Profile, 0, len(m.Profiles))
|
|
for _, p := range m.Profiles {
|
|
out = append(out, placement.Profile{
|
|
Name: p.Name,
|
|
Engine: p.Engine,
|
|
GPUClass: p.GPUClass,
|
|
GPUsPerNode: p.GPUsPerNode,
|
|
Nodes: p.Nodes,
|
|
Priority: p.Priority,
|
|
NodeSelector: p.NodeSelector,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// EngineProfile converts a catalog profile into the engine adapter's view.
|
|
func (m Model) EngineProfile(name string) (engines.Profile, bool) {
|
|
for _, p := range m.Profiles {
|
|
if p.Name != name {
|
|
continue
|
|
}
|
|
return engines.Profile{
|
|
Name: p.Name,
|
|
Engine: p.Engine,
|
|
GPUClass: p.GPUClass,
|
|
TP: p.TP,
|
|
PP: p.PP,
|
|
GPUsPerNode: p.GPUsPerNode,
|
|
Nodes: p.Nodes,
|
|
Args: slices.Clone(p.Args),
|
|
Env: p.Env,
|
|
ModelPath: m.Source,
|
|
ServedModelName: m.Name,
|
|
}, true
|
|
}
|
|
return engines.Profile{}, false
|
|
}
|
|
|
|
// ProfileByName returns one of a model's profiles.
|
|
func (m Model) ProfileByName(name string) (Profile, bool) {
|
|
for _, p := range m.Profiles {
|
|
if p.Name == name {
|
|
return p, true
|
|
}
|
|
}
|
|
return Profile{}, false
|
|
}
|
|
|
|
// String renders the catalog for `ctl models`.
|
|
func (c *Catalog) String() string {
|
|
var b strings.Builder
|
|
for _, m := range c.Models() {
|
|
fmt.Fprintf(&b, "%s (min %d, max %d)\n", m.Name, m.Scaling.Min, m.Scaling.Max)
|
|
for _, p := range m.Profiles {
|
|
fmt.Fprintf(&b, " %-20s %-10s %-12s tp=%d nodes=%d priority=%d\n",
|
|
p.Name, p.Engine, p.GPUClass, p.TP, p.Nodes, p.Priority)
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// Simulated reports whether this model produces synthetic text rather than
|
|
// inference output.
|
|
//
|
|
// Derived from the profiles rather than configured, because an operator who
|
|
// added a sim profile and forgot the flag would be shipping placeholder text to
|
|
// users under a real model's name. A model counts as simulated only when every
|
|
// profile is simulated: one real profile means a request can land on a real
|
|
// engine, and a model that sometimes answers for real must not be labelled as
|
|
// though it never does.
|
|
func (m Model) Simulated() bool {
|
|
if len(m.Profiles) == 0 {
|
|
return false
|
|
}
|
|
for _, p := range m.Profiles {
|
|
if p.Engine != "sim" {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|