// Package catalog holds the desired state: which models exist, how each one can // be run, and how many copies of it there should be (plan §6.1). package catalog import ( "fmt" "slices" "strings" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" "gitea.mynodes.xyz/ilker/llmgw/internal/control/placement" ) // Scaling is how many replicas a model should have. type Scaling struct { Min int `json:"min,omitempty" koanf:"min"` Max int `json:"max,omitempty" koanf:"max"` } // SLO is the latency target a model is expected to meet. `ctl bench` measures // capacity against it (plan §6.2.1). type SLO struct { TTFTp99 time.Duration `koanf:"ttft_p99"` TPOTp99 time.Duration `koanf:"tpot_p99"` } // Profile is how a model runs on one GPU class. type Profile struct { Name string `json:"name,omitempty" koanf:"name"` Engine string `json:"engine,omitempty" koanf:"engine"` GPUClass string `json:"gpu_class,omitempty" koanf:"gpu_class"` TP int `json:"tp,omitempty" koanf:"tp"` PP int `json:"pp,omitempty" koanf:"pp"` // GPUsPerNode defaults to TP for a single-node profile. GPUsPerNode int `json:"gpus_per_node,omitempty" koanf:"gpus_per_node"` // Nodes above 1 makes this a multi-node profile, which requires a fabric. Nodes int `json:"nodes,omitempty" koanf:"nodes"` Args []string `json:"args,omitempty" koanf:"args"` Env map[string]string `json:"env,omitempty" koanf:"env"` // MinMemoryGB is the host RAM this profile needs to run. // // The number that matters is the model's working set: the weights, plus the // KV cache for the whole context pool, plus the engine's own overhead. For // a GPU profile that lives in the cards and this is small; for a CPU // profile it is the whole thing, and getting it wrong means an OOM on a // node that reported itself healthy. // // Zero means unconstrained, which is every profile that existed before // this. A node that does not report its memory is not excluded either — // an older agent must not have its nodes quietly emptied by an upgrade of // the control plane. MinMemoryGB int `json:"min_memory_gb,omitempty" koanf:"min_memory_gb"` // NodeSelector restricts this profile to nodes carrying every one of these // labels, with these values. // // A CPU profile otherwise fits everywhere, because there is no card to be // free — which is right for a 0.5B model and wrong for a 14B one. Nothing // in the inventory says how much memory a node has, so the operator says // which nodes are meant for a model instead of the scheduler inferring it. // Agents have reported labels since the beginning; this is what reads them. NodeSelector map[string]string `json:"node_selector,omitempty" koanf:"node_selector"` // Weight is the router's capacity weight. It should come from ctl bench // rather than a guess (plan §6.2.1). Weight float64 `json:"weight,omitempty" koanf:"weight"` // Priority orders profiles during placement; highest that fits wins. Priority int `json:"priority,omitempty" koanf:"priority"` // StartupTimeout bounds model loading. A 70B model across two nodes takes // tens of minutes, so this is per profile rather than global. StartupTimeout time.Duration `json:"startup_timeout,omitempty" koanf:"startup_timeout"` } // CPUOnly reports whether this profile needs no GPUs. // // Naming no GPU class is how a profile says so, and only engines that can // actually serve without a card are allowed to leave it out — otherwise an // omitted class would quietly become a replica placed on a machine with no // hardware to run it. func (p Profile) CPUOnly() bool { return p.GPUClass == "" && cpuCapable(p.Engine) } // cpuCapable reports whether an engine can serve a model without a GPU. // // llama.cpp is the one that can, and it is what makes a cluster with no cards // testable end to end. func cpuCapable(engine string) bool { return engine == "llamacpp" } // Model is a catalog entry as clients see it. type Model struct { Name string `json:"name,omitempty" koanf:"name"` Aliases []string `json:"aliases,omitempty" koanf:"aliases"` // Source is the weights path or a HuggingFace id. Source string `json:"source,omitempty" koanf:"source"` Profiles []Profile `json:"profiles,omitempty" koanf:"profiles"` Scaling Scaling `json:"scaling,omitempty" koanf:"scaling"` Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"` // RoutingProfile selects how the router scores endpoints for this model. RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"` SLO SLO `json:"slo,omitempty" koanf:"slo"` // ContextLength is how many tokens one request may use, prompt and // completion together, and MaxOutput bounds the completion alone. Zero // means unknown, which is what every model was until now. // // The gateway knew neither, so an over-long prompt was not refused: it was // admitted, given a replica, and failed at the engine, and a client sizing // its prompts from /v1/models had nothing to read. Worse, the number is not // the one an operator writes in the engine's arguments — llama.cpp divides // --ctx-size across its parallel slots, so a pool of 8192 across four slots // is 2048 a request, and nothing said so. ContextLength int `json:"context_length,omitempty" koanf:"context_length"` MaxOutput int `json:"max_output,omitempty" koanf:"max_output"` // Capabilities is what this model can do beyond plain completion. Capabilities Capabilities `json:"capabilities,omitempty" koanf:"capabilities"` } // Capabilities describes what a client may ask a model for. // // Declared by the operator rather than discovered, because there is no way to // ask an engine what its model can do. That makes these claims, and a wrong // claim is worse than none — which is why `ctl engine-check` verifies tool // calling against a running engine rather than trusting this. // // The reason it exists: the gateway forwards a tools array to any model, and a // model that cannot use them answers in prose. Every OpenAI-compatible client // then behaves as though the model is broken, and the only way to find out was // to try it. That cost an afternoon. type Capabilities struct { Tools bool `json:"tools,omitempty" koanf:"tools"` Vision bool `json:"vision,omitempty" koanf:"vision"` Embeddings bool `json:"embeddings,omitempty" koanf:"embeddings"` } // Catalog is the set of models the cluster should serve. type Catalog struct { models map[string]Model aliases map[string]string order []string } // New builds a catalog, rejecting definitions that would fail confusingly later. func New(models []Model) (*Catalog, error) { c := &Catalog{ models: make(map[string]Model, len(models)), aliases: map[string]string{}, } for i := range models { m := models[i].normalize() if err := m.validate(); err != nil { return nil, err } if _, dup := c.models[m.Name]; dup { return nil, fmt.Errorf("model %q is declared twice", m.Name) } c.models[m.Name] = m c.order = append(c.order, m.Name) } // Aliases share a namespace with names. A collision would silently shadow a // model at request time, so it has to fail at load. for _, name := range c.order { for _, a := range c.models[name].Aliases { if _, clash := c.models[a]; clash { return nil, fmt.Errorf("model %q: alias %q collides with another model name", name, a) } if prev, clash := c.aliases[a]; clash { return nil, fmt.Errorf("alias %q is claimed by both %q and %q", a, prev, name) } c.aliases[a] = name } } // Fallbacks are resolved now rather than at request time: a typo in a // fallback should stop a deployment, not surface during an outage when the // fallback is finally needed. for _, name := range c.order { for _, f := range c.models[name].Fallbacks { if _, ok := c.Resolve(f); !ok { return nil, fmt.Errorf("model %q: fallback %q is not in the catalog", name, f) } } } return c, nil } func (m Model) normalize() Model { if m.Scaling.Min < 0 { m.Scaling.Min = 0 } if m.Scaling.Max == 0 { m.Scaling.Max = max(m.Scaling.Min, 1) } for i := range m.Profiles { p := &m.Profiles[i] if p.Engine == "" { p.Engine = "vllm" } if p.TP <= 0 { p.TP = 1 } if p.Nodes <= 0 { p.Nodes = 1 } switch { case p.CPUOnly(): // Nothing to divide up: this profile asks for no cards at all. p.GPUsPerNode = 0 case p.GPUsPerNode <= 0: // The common case: a single-node profile uses TP GPUs on one node. p.GPUsPerNode = p.TP / p.Nodes if p.GPUsPerNode <= 0 { p.GPUsPerNode = 1 } } if p.StartupTimeout <= 0 { p.StartupTimeout = 20 * time.Minute } if p.Weight <= 0 { p.Weight = 1 } } return m } func (m Model) validate() error { if m.Name == "" { return fmt.Errorf("a model has no name") } if m.Source == "" { return fmt.Errorf("model %q: source is required", m.Name) } if len(m.Profiles) == 0 { return fmt.Errorf("model %q: at least one profile is required", m.Name) } if m.Scaling.Max < m.Scaling.Min { return fmt.Errorf("model %q: scaling max %d is below min %d", m.Name, m.Scaling.Max, m.Scaling.Min) } seen := map[string]struct{}{} for _, p := range m.Profiles { if p.Name == "" { return fmt.Errorf("model %q: a profile has no name", m.Name) } if _, dup := seen[p.Name]; dup { return fmt.Errorf("model %q: profile %q is declared twice", m.Name, p.Name) } seen[p.Name] = struct{}{} if p.GPUClass == "" && !cpuCapable(p.Engine) { return fmt.Errorf("model %q profile %q: gpu_class is required for engine %q", m.Name, p.Name, p.Engine) } if p.CPUOnly() { if p.Nodes > 1 { return fmt.Errorf("model %q profile %q: a CPU profile cannot span %d nodes", m.Name, p.Name, p.Nodes) } continue } // The arithmetic has to add up, or the engine is handed a shard spec it // cannot satisfy and fails minutes later during model loading. if want := p.GPUsPerNode * p.Nodes; want != p.TP*max(p.PP, 1) { return fmt.Errorf("model %q profile %q: %d nodes x %d GPUs does not match TP %d x PP %d", m.Name, p.Name, p.Nodes, p.GPUsPerNode, p.TP, max(p.PP, 1)) } } return nil } // Resolve maps a client-supplied name or alias onto a model. func (c *Catalog) Resolve(name string) (Model, bool) { if m, ok := c.models[name]; ok { return m, true } if canonical, ok := c.aliases[name]; ok { return c.models[canonical], true } return Model{}, false } // Models returns every model, in declaration order. func (c *Catalog) Models() []Model { out := make([]Model, 0, len(c.order)) for _, n := range c.order { out = append(out, c.models[n]) } return out } // Names returns the canonical model names, sorted. func (c *Catalog) Names() []string { out := slices.Clone(c.order) slices.Sort(out) return out } // AliasesOf returns the alternative names for a model. func (c *Catalog) AliasesOf(name string) []string { m, ok := c.Resolve(name) if !ok { return nil } return slices.Clone(m.Aliases) } // PlacementProfiles converts a model's profiles into placement's view. func (m Model) PlacementProfiles() []placement.Profile { out := make([]placement.Profile, 0, len(m.Profiles)) for _, p := range m.Profiles { out = append(out, placement.Profile{ Name: p.Name, Engine: p.Engine, GPUClass: p.GPUClass, GPUsPerNode: p.GPUsPerNode, Nodes: p.Nodes, Priority: p.Priority, NodeSelector: p.NodeSelector, MinMemoryGB: p.MinMemoryGB, }) } return out } // EngineProfile converts a catalog profile into the engine adapter's view. func (m Model) EngineProfile(name string) (engines.Profile, bool) { for _, p := range m.Profiles { if p.Name != name { continue } return engines.Profile{ Name: p.Name, Engine: p.Engine, GPUClass: p.GPUClass, TP: p.TP, PP: p.PP, GPUsPerNode: p.GPUsPerNode, Nodes: p.Nodes, Args: slices.Clone(p.Args), Env: p.Env, ModelPath: m.Source, ServedModelName: m.Name, }, true } return engines.Profile{}, false } // ProfileByName returns one of a model's profiles. func (m Model) ProfileByName(name string) (Profile, bool) { for _, p := range m.Profiles { if p.Name == name { return p, true } } return Profile{}, false } // String renders the catalog for `ctl models`. func (c *Catalog) String() string { var b strings.Builder for _, m := range c.Models() { fmt.Fprintf(&b, "%s (min %d, max %d)\n", m.Name, m.Scaling.Min, m.Scaling.Max) for _, p := range m.Profiles { fmt.Fprintf(&b, " %-20s %-10s %-12s tp=%d nodes=%d priority=%d\n", p.Name, p.Engine, p.GPUClass, p.TP, p.Nodes, p.Priority) } } return b.String() } // Simulated reports whether this model produces synthetic text rather than // inference output. // // Derived from the profiles rather than configured, because an operator who // added a sim profile and forgot the flag would be shipping placeholder text to // users under a real model's name. A model counts as simulated only when every // profile is simulated: one real profile means a request can land on a real // engine, and a model that sometimes answers for real must not be labelled as // though it never does. func (m Model) Simulated() bool { if len(m.Profiles) == 0 { return false } for _, p := range m.Profiles { if p.Engine != "sim" { return false } } return true }