// Package catalog holds the desired state: which models exist, how each one can // be run, and how many copies of it there should be (plan §6.1). package catalog import ( "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "slices" "strings" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" "gitea.mynodes.xyz/ilker/llmgw/internal/control/placement" ) // Scaling is how many replicas a model should have. type Scaling struct { Min int `json:"min,omitempty" koanf:"min"` Max int `json:"max,omitempty" koanf:"max"` // IdleTimeout is how long an on-demand model may go unasked-for before its // replicas are stopped. Zero takes the cluster default. // // Minutes rather than seconds, always. Stopping a replica throws away the // KV cache prefix routing spent its effort filling, and a model that flaps // in and out loses that entirely — so the saving has to be worth more than // the cache, which it only is over a real idle period. IdleTimeout time.Duration `json:"idle_timeout,omitempty" koanf:"idle_timeout"` } // OnDemand reports whether this model is loaded when someone asks for it. // // min 0 with a max above it, which needs no new field and reads correctly: keep // none running, allow up to max. min 0 and max 0 is a model that is configured // and deliberately not running, which is a different and equally useful thing — // the difference between "load it when wanted" and "do not load it". func (s Scaling) OnDemand() bool { return s.Min == 0 && s.Max > 0 } // SLO is the latency target a model is expected to meet. `ctl bench` measures // capacity against it (plan §6.2.1). type SLO struct { TTFTp99 time.Duration `koanf:"ttft_p99"` TPOTp99 time.Duration `koanf:"tpot_p99"` } // Profile is how a model runs on one GPU class. type Profile struct { Name string `json:"name,omitempty" koanf:"name"` Engine string `json:"engine,omitempty" koanf:"engine"` GPUClass string `json:"gpu_class,omitempty" koanf:"gpu_class"` TP int `json:"tp,omitempty" koanf:"tp"` PP int `json:"pp,omitempty" koanf:"pp"` // GPUsPerNode defaults to TP for a single-node profile. GPUsPerNode int `json:"gpus_per_node,omitempty" koanf:"gpus_per_node"` // Nodes above 1 makes this a multi-node profile, which requires a fabric. Nodes int `json:"nodes,omitempty" koanf:"nodes"` Args []string `json:"args,omitempty" koanf:"args"` Env map[string]string `json:"env,omitempty" koanf:"env"` // MinMemoryGB is the host RAM this profile needs to run. // // The number that matters is the model's working set: the weights, plus the // KV cache for the whole context pool, plus the engine's own overhead. For // a GPU profile that lives in the cards and this is small; for a CPU // profile it is the whole thing, and getting it wrong means an OOM on a // node that reported itself healthy. // // Zero means unconstrained, which is every profile that existed before // this. A node that does not report its memory is not excluded either — // an older agent must not have its nodes quietly emptied by an upgrade of // the control plane. MinMemoryGB int `json:"min_memory_gb,omitempty" koanf:"min_memory_gb"` // MinDiskGB is the free space a node needs before it can hold these // weights. // // Only checked when the node does not already have them: a node holding // the file needs no room for it, and requiring the space twice would keep // a model off the one node best placed to run it. // // Declared rather than derived, because the size of a model that has never // been fetched anywhere is not knowable from here — and a wrong guess // fails late, halfway through a download, which is the failure this exists // to prevent. Zero means unconstrained, and a node that reports no storage // is not excluded: unknown is not full. MinDiskGB int `json:"min_disk_gb,omitempty" koanf:"min_disk_gb"` // NodeSelector restricts this profile to nodes carrying every one of these // labels, with these values. // // A CPU profile otherwise fits everywhere, because there is no card to be // free — which is right for a 0.5B model and wrong for a 14B one. Nothing // in the inventory says how much memory a node has, so the operator says // which nodes are meant for a model instead of the scheduler inferring it. // Agents have reported labels since the beginning; this is what reads them. NodeSelector map[string]string `json:"node_selector,omitempty" koanf:"node_selector"` // Weight is the router's capacity weight. It should come from ctl bench // rather than a guess (plan §6.2.1). Weight float64 `json:"weight,omitempty" koanf:"weight"` // Priority orders profiles during placement; highest that fits wins. Priority int `json:"priority,omitempty" koanf:"priority"` // StartupTimeout bounds model loading. A 70B model across two nodes takes // tens of minutes, so this is per profile rather than global. StartupTimeout time.Duration `json:"startup_timeout,omitempty" koanf:"startup_timeout"` } // CPUOnly reports whether this profile needs no GPUs. // // Naming no GPU class is how a profile says so, and only engines that can // actually serve without a card are allowed to leave it out — otherwise an // omitted class would quietly become a replica placed on a machine with no // hardware to run it. func (p Profile) CPUOnly() bool { return p.GPUClass == "" && cpuCapable(p.Engine) } // cpuCapable reports whether an engine can serve a model without a GPU. // // llama.cpp is the one that can, and it is what makes a cluster with no cards // testable end to end. func cpuCapable(engine string) bool { return engine == "llamacpp" } // Model is a catalog entry as clients see it. type Model struct { Name string `json:"name,omitempty" koanf:"name"` Aliases []string `json:"aliases,omitempty" koanf:"aliases"` // Description is what this model is for, in a line. // // Written by whoever added it, because nobody else can say it. A registry // offers a pipeline tag and a wall of README, and neither answers the // question a person picking from a list is asking: which of these nine // names should I send this request to. // // It reaches clients through /v1/models, which is the only view most of // them ever see. A gateway serving nine models by name alone makes that // choice by guesswork. Description string `json:"description,omitempty" koanf:"description"` // Source is the weights path or a HuggingFace id. Source string `json:"source,omitempty" koanf:"source"` Profiles []Profile `json:"profiles,omitempty" koanf:"profiles"` Scaling Scaling `json:"scaling,omitempty" koanf:"scaling"` Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"` // RoutingProfile selects how the router scores endpoints for this model. RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"` SLO SLO `json:"slo,omitempty" koanf:"slo"` // ContextLength is how many tokens one request may use, prompt and // completion together, and MaxOutput bounds the completion alone. Zero // means unknown, which is what every model was until now. // // The gateway knew neither, so an over-long prompt was not refused: it was // admitted, given a replica, and failed at the engine, and a client sizing // its prompts from /v1/models had nothing to read. Worse, the number is not // the one an operator writes in the engine's arguments — llama.cpp divides // --ctx-size across its parallel slots, so a pool of 8192 across four slots // is 2048 a request, and nothing said so. ContextLength int `json:"context_length,omitempty" koanf:"context_length"` MaxOutput int `json:"max_output,omitempty" koanf:"max_output"` // Capabilities is what this model can do beyond plain completion. Capabilities Capabilities `json:"capabilities,omitempty" koanf:"capabilities"` } // Capabilities describes what a client may ask a model for. // // Declared by the operator rather than discovered, because there is no way to // ask an engine what its model can do. That makes these claims, and a wrong // claim is worse than none — which is why `ctl engine-check` verifies tool // calling against a running engine rather than trusting this. // // The reason it exists: the gateway forwards a tools array to any model, and a // model that cannot use them answers in prose. Every OpenAI-compatible client // then behaves as though the model is broken, and the only way to find out was // to try it. That cost an afternoon. type Capabilities struct { Tools bool `json:"tools,omitempty" koanf:"tools"` Vision bool `json:"vision,omitempty" koanf:"vision"` Embeddings bool `json:"embeddings,omitempty" koanf:"embeddings"` } // Catalog is the set of models the cluster should serve. type Catalog struct { models map[string]Model aliases map[string]string order []string } // New builds a catalog, rejecting definitions that would fail confusingly later. func New(models []Model) (*Catalog, error) { c := &Catalog{ models: make(map[string]Model, len(models)), aliases: map[string]string{}, } for i := range models { m := models[i].normalize() if err := m.validate(); err != nil { return nil, err } if _, dup := c.models[m.Name]; dup { return nil, fmt.Errorf("model %q is declared twice", m.Name) } c.models[m.Name] = m c.order = append(c.order, m.Name) } // Aliases share a namespace with names. A collision would silently shadow a // model at request time, so it has to fail at load. for _, name := range c.order { for _, a := range c.models[name].Aliases { if _, clash := c.models[a]; clash { return nil, fmt.Errorf("model %q: alias %q collides with another model name", name, a) } if prev, clash := c.aliases[a]; clash { return nil, fmt.Errorf("alias %q is claimed by both %q and %q", a, prev, name) } c.aliases[a] = name } } // Fallbacks are resolved now rather than at request time: a typo in a // fallback should stop a deployment, not surface during an outage when the // fallback is finally needed. for _, name := range c.order { for _, f := range c.models[name].Fallbacks { if _, ok := c.Resolve(f); !ok { return nil, fmt.Errorf("model %q: fallback %q is not in the catalog", name, f) } } } return c, nil } func (m Model) normalize() Model { if m.Scaling.Min < 0 { m.Scaling.Min = 0 } // Max 0 used to be an omission to fill in, because a model with no maximum // could not run. It is now also a statement: with min 0 it means configured // and deliberately not running, which is a different and useful thing from // "load it when wanted". Filling it in would silently turn every switched- // off model into an on-demand one — including the ones already written that // way in a running cluster's configuration. if m.Scaling.Max == 0 && m.Scaling.Min > 0 { m.Scaling.Max = m.Scaling.Min } for i := range m.Profiles { p := &m.Profiles[i] if p.Engine == "" { p.Engine = "vllm" } if p.TP <= 0 { p.TP = 1 } if p.Nodes <= 0 { p.Nodes = 1 } switch { case p.CPUOnly(): // Nothing to divide up: this profile asks for no cards at all. p.GPUsPerNode = 0 case p.GPUsPerNode <= 0: // The common case: a single-node profile uses TP GPUs on one node. p.GPUsPerNode = p.TP / p.Nodes if p.GPUsPerNode <= 0 { p.GPUsPerNode = 1 } } if p.StartupTimeout <= 0 { p.StartupTimeout = 20 * time.Minute } if p.Weight <= 0 { p.Weight = 1 } } return m } func (m Model) validate() error { if m.Name == "" { return fmt.Errorf("a model has no name") } if m.Source == "" { return fmt.Errorf("model %q: source is required", m.Name) } if len(m.Profiles) == 0 { return fmt.Errorf("model %q: at least one profile is required", m.Name) } if m.Scaling.Max < m.Scaling.Min { return fmt.Errorf("model %q: scaling max %d is below min %d", m.Name, m.Scaling.Max, m.Scaling.Min) } seen := map[string]struct{}{} for _, p := range m.Profiles { if p.Name == "" { return fmt.Errorf("model %q: a profile has no name", m.Name) } if _, dup := seen[p.Name]; dup { return fmt.Errorf("model %q: profile %q is declared twice", m.Name, p.Name) } seen[p.Name] = struct{}{} if p.GPUClass == "" && !cpuCapable(p.Engine) { return fmt.Errorf("model %q profile %q: gpu_class is required for engine %q", m.Name, p.Name, p.Engine) } if p.CPUOnly() { if p.Nodes > 1 { return fmt.Errorf("model %q profile %q: a CPU profile cannot span %d nodes", m.Name, p.Name, p.Nodes) } continue } // The arithmetic has to add up, or the engine is handed a shard spec it // cannot satisfy and fails minutes later during model loading. if want := p.GPUsPerNode * p.Nodes; want != p.TP*max(p.PP, 1) { return fmt.Errorf("model %q profile %q: %d nodes x %d GPUs does not match TP %d x PP %d", m.Name, p.Name, p.Nodes, p.GPUsPerNode, p.TP, max(p.PP, 1)) } } return nil } // Resolve maps a client-supplied name or alias onto a model. func (c *Catalog) Resolve(name string) (Model, bool) { if m, ok := c.models[name]; ok { return m, true } if canonical, ok := c.aliases[name]; ok { return c.models[canonical], true } return Model{}, false } // Models returns every model, in declaration order. func (c *Catalog) Models() []Model { out := make([]Model, 0, len(c.order)) for _, n := range c.order { out = append(out, c.models[n]) } return out } // Names returns the canonical model names, sorted. func (c *Catalog) Names() []string { out := slices.Clone(c.order) slices.Sort(out) return out } // AliasesOf returns the alternative names for a model. func (c *Catalog) AliasesOf(name string) []string { m, ok := c.Resolve(name) if !ok { return nil } return slices.Clone(m.Aliases) } // PlacementProfiles converts a model's profiles into placement's view. func (m Model) PlacementProfiles() []placement.Profile { out := make([]placement.Profile, 0, len(m.Profiles)) for _, p := range m.Profiles { out = append(out, placement.Profile{ Name: p.Name, Engine: p.Engine, GPUClass: p.GPUClass, GPUsPerNode: p.GPUsPerNode, Nodes: p.Nodes, Priority: p.Priority, NodeSelector: p.NodeSelector, MinMemoryGB: p.MinMemoryGB, MinDiskGB: p.MinDiskGB, // The model's source, so placement can prefer nodes that already // hold the weights rather than one that must first move them. Source: m.Source, }) } return out } // EngineProfile converts a catalog profile into the engine adapter's view. func (m Model) EngineProfile(name string) (engines.Profile, bool) { for _, p := range m.Profiles { if p.Name != name { continue } return engines.Profile{ Name: p.Name, Engine: p.Engine, GPUClass: p.GPUClass, TP: p.TP, PP: p.PP, GPUsPerNode: p.GPUsPerNode, Nodes: p.Nodes, Args: slices.Clone(p.Args), Env: p.Env, ModelPath: m.Source, ServedModelName: m.Name, }, true } return engines.Profile{}, false } // ProfileByName returns one of a model's profiles. func (m Model) ProfileByName(name string) (Profile, bool) { for _, p := range m.Profiles { if p.Name == name { return p, true } } return Profile{}, false } // String renders the catalog for `ctl models`. func (c *Catalog) String() string { var b strings.Builder for _, m := range c.Models() { fmt.Fprintf(&b, "%s (min %d, max %d)\n", m.Name, m.Scaling.Min, m.Scaling.Max) for _, p := range m.Profiles { fmt.Fprintf(&b, " %-20s %-10s %-12s tp=%d nodes=%d priority=%d\n", p.Name, p.Engine, p.GPUClass, p.TP, p.Nodes, p.Priority) } } return b.String() } // Simulated reports whether this model produces synthetic text rather than // inference output. // // Derived from the profiles rather than configured, because an operator who // added a sim profile and forgot the flag would be shipping placeholder text to // users under a real model's name. A model counts as simulated only when every // profile is simulated: one real profile means a request can land on a real // engine, and a model that sometimes answers for real must not be labelled as // though it never does. func (m Model) Simulated() bool { if len(m.Profiles) == 0 { return false } for _, p := range m.Profiles { if p.Engine != "sim" { return false } } return true } // Fingerprint identifies a profile's contents. // // The name is not enough. A profile can be edited — a memory requirement // raised, a node label added, an engine argument changed — and keep its name, // so a replica running the old definition is indistinguishable from one // running the new. The reconciler does not move a running replica when its // profile changes, which is right: churning a replica that is serving costs // more than the imperfection. What was missing is any way to see that it had // happened, and the answer to "why is this one on the wrong node" was in // somebody's memory of what they edited an hour ago. func (p Profile) Fingerprint() string { b, err := json.Marshal(p) if err != nil { // A profile that will not marshal cannot be compared, and saying so by // never matching is better than saying two differing ones are the same. return "unmarshalable" } sum := sha256.Sum256(b) return hex.EncodeToString(sum[:8]) }