llmgw/internal/control/catalog/catalog.go

// Package catalog holds the desired state: which models exist, how each one can
// be run, and how many copies of it there should be (plan §6.1).
package catalog

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"slices"
	"strings"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
)

// Scaling is how many replicas a model should have.
type Scaling struct {
	Min int `json:"min,omitempty" koanf:"min"`
	Max int `json:"max,omitempty" koanf:"max"`
	// IdleTimeout is how long an on-demand model may go unasked-for before its
	// replicas are stopped. Zero takes the cluster default.
	//
	// Minutes rather than seconds, always. Stopping a replica throws away the
	// KV cache prefix routing spent its effort filling, and a model that flaps
	// in and out loses that entirely — so the saving has to be worth more than
	// the cache, which it only is over a real idle period.
	IdleTimeout time.Duration `json:"idle_timeout,omitempty" koanf:"idle_timeout"`
}

// OnDemand reports whether this model is loaded when someone asks for it.
//
// min 0 with a max above it, which needs no new field and reads correctly: keep
// none running, allow up to max. min 0 and max 0 is a model that is configured
// and deliberately not running, which is a different and equally useful thing —
// the difference between "load it when wanted" and "do not load it".
func (s Scaling) OnDemand() bool { return s.Min == 0 && s.Max > 0 }

// SLO is the latency target a model is expected to meet. `ctl bench` measures
// capacity against it (plan §6.2.1).
type SLO struct {
	TTFTp99 time.Duration `koanf:"ttft_p99"`
	TPOTp99 time.Duration `koanf:"tpot_p99"`
}

// Profile is how a model runs on one GPU class.
type Profile struct {
	Name     string `json:"name,omitempty" koanf:"name"`
	Engine   string `json:"engine,omitempty" koanf:"engine"`
	GPUClass string `json:"gpu_class,omitempty" koanf:"gpu_class"`

	TP int `json:"tp,omitempty" koanf:"tp"`
	PP int `json:"pp,omitempty" koanf:"pp"`
	// GPUsPerNode defaults to TP for a single-node profile.
	GPUsPerNode int `json:"gpus_per_node,omitempty" koanf:"gpus_per_node"`
	// Nodes above 1 makes this a multi-node profile, which requires a fabric.
	Nodes int `json:"nodes,omitempty" koanf:"nodes"`

	Args []string          `json:"args,omitempty" koanf:"args"`
	Env  map[string]string `json:"env,omitempty" koanf:"env"`

	// MinMemoryGB is the host RAM this profile needs to run.
	//
	// The number that matters is the model's working set: the weights, plus the
	// KV cache for the whole context pool, plus the engine's own overhead. For
	// a GPU profile that lives in the cards and this is small; for a CPU
	// profile it is the whole thing, and getting it wrong means an OOM on a
	// node that reported itself healthy.
	//
	// Zero means unconstrained, which is every profile that existed before
	// this. A node that does not report its memory is not excluded either —
	// an older agent must not have its nodes quietly emptied by an upgrade of
	// the control plane.
	MinMemoryGB int `json:"min_memory_gb,omitempty" koanf:"min_memory_gb"`

	// MinDiskGB is the free space a node needs before it can hold these
	// weights.
	//
	// Only checked when the node does not already have them: a node holding
	// the file needs no room for it, and requiring the space twice would keep
	// a model off the one node best placed to run it.
	//
	// Declared rather than derived, because the size of a model that has never
	// been fetched anywhere is not knowable from here — and a wrong guess
	// fails late, halfway through a download, which is the failure this exists
	// to prevent. Zero means unconstrained, and a node that reports no storage
	// is not excluded: unknown is not full.
	MinDiskGB int `json:"min_disk_gb,omitempty" koanf:"min_disk_gb"`

	// NodeSelector restricts this profile to nodes carrying every one of these
	// labels, with these values.
	//
	// A CPU profile otherwise fits everywhere, because there is no card to be
	// free — which is right for a 0.5B model and wrong for a 14B one. Nothing
	// in the inventory says how much memory a node has, so the operator says
	// which nodes are meant for a model instead of the scheduler inferring it.
	// Agents have reported labels since the beginning; this is what reads them.
	NodeSelector map[string]string `json:"node_selector,omitempty" koanf:"node_selector"`

	// Weight is the router's capacity weight. It should come from ctl bench
	// rather than a guess (plan §6.2.1).
	Weight float64 `json:"weight,omitempty" koanf:"weight"`
	// Priority orders profiles during placement; highest that fits wins.
	Priority int `json:"priority,omitempty" koanf:"priority"`
	// StartupTimeout bounds model loading. A 70B model across two nodes takes
	// tens of minutes, so this is per profile rather than global.
	StartupTimeout time.Duration `json:"startup_timeout,omitempty" koanf:"startup_timeout"`
}

// CPUOnly reports whether this profile needs no GPUs.
//
// Naming no GPU class is how a profile says so, and only engines that can
// actually serve without a card are allowed to leave it out — otherwise an
// omitted class would quietly become a replica placed on a machine with no
// hardware to run it.
func (p Profile) CPUOnly() bool { return p.GPUClass == "" && cpuCapable(p.Engine) }

// cpuCapable reports whether an engine can serve a model without a GPU.
//
// llama.cpp is the one that can, and it is what makes a cluster with no cards
// testable end to end.
func cpuCapable(engine string) bool { return engine == "llamacpp" }

// Model is a catalog entry as clients see it.
type Model struct {
	Name    string   `json:"name,omitempty" koanf:"name"`
	Aliases []string `json:"aliases,omitempty" koanf:"aliases"`
	// Description is what this model is for, in a line.
	//
	// Written by whoever added it, because nobody else can say it. A registry
	// offers a pipeline tag and a wall of README, and neither answers the
	// question a person picking from a list is asking: which of these nine
	// names should I send this request to.
	//
	// It reaches clients through /v1/models, which is the only view most of
	// them ever see. A gateway serving nine models by name alone makes that
	// choice by guesswork.
	Description string `json:"description,omitempty" koanf:"description"`
	// Source is the weights path or a HuggingFace id.
	Source    string    `json:"source,omitempty" koanf:"source"`
	Profiles  []Profile `json:"profiles,omitempty" koanf:"profiles"`
	Scaling   Scaling   `json:"scaling,omitempty" koanf:"scaling"`
	Fallbacks []string  `json:"fallbacks,omitempty" koanf:"fallbacks"`
	// RoutingProfile selects how the router scores endpoints for this model.
	RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"`
	SLO            SLO    `json:"slo,omitempty" koanf:"slo"`

	// ContextLength is how many tokens one request may use, prompt and
	// completion together, and MaxOutput bounds the completion alone. Zero
	// means unknown, which is what every model was until now.
	//
	// The gateway knew neither, so an over-long prompt was not refused: it was
	// admitted, given a replica, and failed at the engine, and a client sizing
	// its prompts from /v1/models had nothing to read. Worse, the number is not
	// the one an operator writes in the engine's arguments — llama.cpp divides
	// --ctx-size across its parallel slots, so a pool of 8192 across four slots
	// is 2048 a request, and nothing said so.
	ContextLength int `json:"context_length,omitempty" koanf:"context_length"`
	MaxOutput     int `json:"max_output,omitempty" koanf:"max_output"`

	// Capabilities is what this model can do beyond plain completion.
	Capabilities Capabilities `json:"capabilities,omitempty" koanf:"capabilities"`
}

// Capabilities describes what a client may ask a model for.
//
// Declared by the operator rather than discovered, because there is no way to
// ask an engine what its model can do. That makes these claims, and a wrong
// claim is worse than none — which is why `ctl engine-check` verifies tool
// calling against a running engine rather than trusting this.
//
// The reason it exists: the gateway forwards a tools array to any model, and a
// model that cannot use them answers in prose. Every OpenAI-compatible client
// then behaves as though the model is broken, and the only way to find out was
// to try it. That cost an afternoon.
type Capabilities struct {
	Tools      bool `json:"tools,omitempty" koanf:"tools"`
	Vision     bool `json:"vision,omitempty" koanf:"vision"`
	Embeddings bool `json:"embeddings,omitempty" koanf:"embeddings"`
}

// Catalog is the set of models the cluster should serve.
type Catalog struct {
	models  map[string]Model
	aliases map[string]string
	order   []string
}

// New builds a catalog, rejecting definitions that would fail confusingly later.
func New(models []Model) (*Catalog, error) {
	c := &Catalog{
		models:  make(map[string]Model, len(models)),
		aliases: map[string]string{},
	}

	for i := range models {
		m := models[i].normalize()
		if err := m.validate(); err != nil {
			return nil, err
		}
		if _, dup := c.models[m.Name]; dup {
			return nil, fmt.Errorf("model %q is declared twice", m.Name)
		}
		c.models[m.Name] = m
		c.order = append(c.order, m.Name)
	}

	// Aliases share a namespace with names. A collision would silently shadow a
	// model at request time, so it has to fail at load.
	for _, name := range c.order {
		for _, a := range c.models[name].Aliases {
			if _, clash := c.models[a]; clash {
				return nil, fmt.Errorf("model %q: alias %q collides with another model name", name, a)
			}
			if prev, clash := c.aliases[a]; clash {
				return nil, fmt.Errorf("alias %q is claimed by both %q and %q", a, prev, name)
			}
			c.aliases[a] = name
		}
	}

	// Fallbacks are resolved now rather than at request time: a typo in a
	// fallback should stop a deployment, not surface during an outage when the
	// fallback is finally needed.
	for _, name := range c.order {
		for _, f := range c.models[name].Fallbacks {
			if _, ok := c.Resolve(f); !ok {
				return nil, fmt.Errorf("model %q: fallback %q is not in the catalog", name, f)
			}
		}
	}
	return c, nil
}

func (m Model) normalize() Model {
	if m.Scaling.Min < 0 {
		m.Scaling.Min = 0
	}
	// Max 0 used to be an omission to fill in, because a model with no maximum
	// could not run. It is now also a statement: with min 0 it means configured
	// and deliberately not running, which is a different and useful thing from
	// "load it when wanted". Filling it in would silently turn every switched-
	// off model into an on-demand one — including the ones already written that
	// way in a running cluster's configuration.
	if m.Scaling.Max == 0 && m.Scaling.Min > 0 {
		m.Scaling.Max = m.Scaling.Min
	}
	for i := range m.Profiles {
		p := &m.Profiles[i]
		if p.Engine == "" {
			p.Engine = "vllm"
		}
		if p.TP <= 0 {
			p.TP = 1
		}
		if p.Nodes <= 0 {
			p.Nodes = 1
		}
		switch {
		case p.CPUOnly():
			// Nothing to divide up: this profile asks for no cards at all.
			p.GPUsPerNode = 0
		case p.GPUsPerNode <= 0:
			// The common case: a single-node profile uses TP GPUs on one node.
			p.GPUsPerNode = p.TP / p.Nodes
			if p.GPUsPerNode <= 0 {
				p.GPUsPerNode = 1
			}
		}
		if p.StartupTimeout <= 0 {
			p.StartupTimeout = 20 * time.Minute
		}
		if p.Weight <= 0 {
			p.Weight = 1
		}
	}
	return m
}

func (m Model) validate() error {
	if m.Name == "" {
		return fmt.Errorf("a model has no name")
	}
	if m.Source == "" {
		return fmt.Errorf("model %q: source is required", m.Name)
	}
	if len(m.Profiles) == 0 {
		return fmt.Errorf("model %q: at least one profile is required", m.Name)
	}
	if m.Scaling.Max < m.Scaling.Min {
		return fmt.Errorf("model %q: scaling max %d is below min %d", m.Name, m.Scaling.Max, m.Scaling.Min)
	}

	seen := map[string]struct{}{}
	for _, p := range m.Profiles {
		if p.Name == "" {
			return fmt.Errorf("model %q: a profile has no name", m.Name)
		}
		if _, dup := seen[p.Name]; dup {
			return fmt.Errorf("model %q: profile %q is declared twice", m.Name, p.Name)
		}
		seen[p.Name] = struct{}{}

		if p.GPUClass == "" && !cpuCapable(p.Engine) {
			return fmt.Errorf("model %q profile %q: gpu_class is required for engine %q",
				m.Name, p.Name, p.Engine)
		}
		if p.CPUOnly() {
			if p.Nodes > 1 {
				return fmt.Errorf("model %q profile %q: a CPU profile cannot span %d nodes",
					m.Name, p.Name, p.Nodes)
			}
			continue
		}
		// The arithmetic has to add up, or the engine is handed a shard spec it
		// cannot satisfy and fails minutes later during model loading.
		if want := p.GPUsPerNode * p.Nodes; want != p.TP*max(p.PP, 1) {
			return fmt.Errorf("model %q profile %q: %d nodes x %d GPUs does not match TP %d x PP %d",
				m.Name, p.Name, p.Nodes, p.GPUsPerNode, p.TP, max(p.PP, 1))
		}
	}
	return nil
}

// Resolve maps a client-supplied name or alias onto a model.
func (c *Catalog) Resolve(name string) (Model, bool) {
	if m, ok := c.models[name]; ok {
		return m, true
	}
	if canonical, ok := c.aliases[name]; ok {
		return c.models[canonical], true
	}
	return Model{}, false
}

// Models returns every model, in declaration order.
func (c *Catalog) Models() []Model {
	out := make([]Model, 0, len(c.order))
	for _, n := range c.order {
		out = append(out, c.models[n])
	}
	return out
}

// Names returns the canonical model names, sorted.
func (c *Catalog) Names() []string {
	out := slices.Clone(c.order)
	slices.Sort(out)
	return out
}

// AliasesOf returns the alternative names for a model.
func (c *Catalog) AliasesOf(name string) []string {
	m, ok := c.Resolve(name)
	if !ok {
		return nil
	}
	return slices.Clone(m.Aliases)
}

// PlacementProfiles converts a model's profiles into placement's view.
func (m Model) PlacementProfiles() []placement.Profile {
	out := make([]placement.Profile, 0, len(m.Profiles))
	for _, p := range m.Profiles {
		out = append(out, placement.Profile{
			Name:         p.Name,
			Engine:       p.Engine,
			GPUClass:     p.GPUClass,
			GPUsPerNode:  p.GPUsPerNode,
			Nodes:        p.Nodes,
			Priority:     p.Priority,
			NodeSelector: p.NodeSelector,
			MinMemoryGB:  p.MinMemoryGB,
			MinDiskGB:    p.MinDiskGB,
			// The model's source, so placement can prefer nodes that already
			// hold the weights rather than one that must first move them.
			Source: m.Source,
		})
	}
	return out
}

// EngineProfile converts a catalog profile into the engine adapter's view.
func (m Model) EngineProfile(name string) (engines.Profile, bool) {
	for _, p := range m.Profiles {
		if p.Name != name {
			continue
		}
		return engines.Profile{
			Name:            p.Name,
			Engine:          p.Engine,
			GPUClass:        p.GPUClass,
			TP:              p.TP,
			PP:              p.PP,
			GPUsPerNode:     p.GPUsPerNode,
			Nodes:           p.Nodes,
			Args:            slices.Clone(p.Args),
			Env:             p.Env,
			ModelPath:       m.Source,
			ServedModelName: m.Name,
		}, true
	}
	return engines.Profile{}, false
}

// ProfileByName returns one of a model's profiles.
func (m Model) ProfileByName(name string) (Profile, bool) {
	for _, p := range m.Profiles {
		if p.Name == name {
			return p, true
		}
	}
	return Profile{}, false
}

// String renders the catalog for `ctl models`.
func (c *Catalog) String() string {
	var b strings.Builder
	for _, m := range c.Models() {
		fmt.Fprintf(&b, "%s (min %d, max %d)\n", m.Name, m.Scaling.Min, m.Scaling.Max)
		for _, p := range m.Profiles {
			fmt.Fprintf(&b, "  %-20s %-10s %-12s tp=%d nodes=%d priority=%d\n",
				p.Name, p.Engine, p.GPUClass, p.TP, p.Nodes, p.Priority)
		}
	}
	return b.String()
}

// Simulated reports whether this model produces synthetic text rather than
// inference output.
//
// Derived from the profiles rather than configured, because an operator who
// added a sim profile and forgot the flag would be shipping placeholder text to
// users under a real model's name. A model counts as simulated only when every
// profile is simulated: one real profile means a request can land on a real
// engine, and a model that sometimes answers for real must not be labelled as
// though it never does.
func (m Model) Simulated() bool {
	if len(m.Profiles) == 0 {
		return false
	}
	for _, p := range m.Profiles {
		if p.Engine != "sim" {
			return false
		}
	}
	return true
}

// Fingerprint identifies a profile's contents.
//
// The name is not enough. A profile can be edited — a memory requirement
// raised, a node label added, an engine argument changed — and keep its name,
// so a replica running the old definition is indistinguishable from one
// running the new. The reconciler does not move a running replica when its
// profile changes, which is right: churning a replica that is serving costs
// more than the imperfection. What was missing is any way to see that it had
// happened, and the answer to "why is this one on the wrong node" was in
// somebody's memory of what they edited an hour ago.
func (p Profile) Fingerprint() string {
	b, err := json.Marshal(p)
	if err != nil {
		// A profile that will not marshal cannot be compared, and saying so by
		// never matching is better than saying two differing ones are the same.
		return "unmarshalable"
	}
	sum := sha256.Sum256(b)
	return hex.EncodeToString(sum[:8])
}