Files
llmgw/internal/control/catalog/catalog.go
T
ilkerandClaude Opus 5 f0134a2f95
CI / test (push) Successful in 24s
Phase 3 (2/3): catalog and reconciler
The desired-versus-actual loop, with a Dispatcher interface where the agent
stream will go. The loop's logic is where the interesting failures live, not the
transport, so it is tested against a fake dispatcher and a synthetic clock.

- catalog: models, profiles, aliases, fallbacks, scaling bounds. Aliases and
  fallbacks resolve at load rather than at request time, because a typo in a
  fallback should stop a deployment instead of surfacing during the outage when
  the fallback is finally needed. The node/GPU/TP arithmetic is checked here
  too, so a bad profile fails at config load rather than minutes into loading.
- reconciler: one replica placed per pass, since placement depends on what is
  already allocated and several from one snapshot would be handed the same GPUs.
  Workers start before rank 0: SGLang's ranks rendezvous and vLLM under Ray
  needs its workers attached before the head serves.
- a failed shard fails the whole replica. The others are waiting on collectives
  that will never complete while holding GPUs, so every shard is stopped and the
  replica rebuilt, with a jittered exponential backoff carried on the model
  rather than on the replica that was just destroyed.
- node expiry has two stages, Unreachable then Lost, deliberately far apart: a
  network blip must not tear down replicas that are still serving, and
  rebuilding a multi-node replica costs tens of minutes of model loading.
- scale-down drains before stopping, so in-flight requests finish.

The phase 3 exit criterion is half met. TestLostNodeMovesReplicas covers the
reconciliation half end to end. The router continuing to serve throughout needs
the EndpointSource wiring and the agent stream.

Writing the tests caught a real property of the design worth recording: advancing
the clock makes every node look silent, so placement correctly stops. The test
harness now heartbeats nodes as it advances time, except where a node going
silent is the point.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 02:12:06 +02:00

290 lines
7.9 KiB
Go

// Package catalog holds the desired state: which models exist, how each one can
// be run, and how many copies of it there should be (plan §6.1).
package catalog
import (
"fmt"
"slices"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
)
// Scaling is how many replicas a model should have.
type Scaling struct {
Min int `koanf:"min"`
Max int `koanf:"max"`
}
// SLO is the latency target a model is expected to meet. `ctl bench` measures
// capacity against it (plan §6.2.1).
type SLO struct {
TTFTp99 time.Duration `koanf:"ttft_p99"`
TPOTp99 time.Duration `koanf:"tpot_p99"`
}
// Profile is how a model runs on one GPU class.
type Profile struct {
Name string `koanf:"name"`
Engine string `koanf:"engine"`
GPUClass string `koanf:"gpu_class"`
TP int `koanf:"tp"`
PP int `koanf:"pp"`
// GPUsPerNode defaults to TP for a single-node profile.
GPUsPerNode int `koanf:"gpus_per_node"`
// Nodes above 1 makes this a multi-node profile, which requires a fabric.
Nodes int `koanf:"nodes"`
Args []string `koanf:"args"`
Env map[string]string `koanf:"env"`
// Weight is the router's capacity weight. It should come from ctl bench
// rather than a guess (plan §6.2.1).
Weight float64 `koanf:"weight"`
// Priority orders profiles during placement; highest that fits wins.
Priority int `koanf:"priority"`
// StartupTimeout bounds model loading. A 70B model across two nodes takes
// tens of minutes, so this is per profile rather than global.
StartupTimeout time.Duration `koanf:"startup_timeout"`
}
// Model is a catalog entry as clients see it.
type Model struct {
Name string `koanf:"name"`
Aliases []string `koanf:"aliases"`
// Source is the weights path or a HuggingFace id.
Source string `koanf:"source"`
Profiles []Profile `koanf:"profiles"`
Scaling Scaling `koanf:"scaling"`
Fallbacks []string `koanf:"fallbacks"`
// RoutingProfile selects how the router scores endpoints for this model.
RoutingProfile string `koanf:"routing_profile"`
SLO SLO `koanf:"slo"`
}
// Catalog is the set of models the cluster should serve.
type Catalog struct {
models map[string]Model
aliases map[string]string
order []string
}
// New builds a catalog, rejecting definitions that would fail confusingly later.
func New(models []Model) (*Catalog, error) {
c := &Catalog{
models: make(map[string]Model, len(models)),
aliases: map[string]string{},
}
for i := range models {
m := models[i].normalize()
if err := m.validate(); err != nil {
return nil, err
}
if _, dup := c.models[m.Name]; dup {
return nil, fmt.Errorf("model %q is declared twice", m.Name)
}
c.models[m.Name] = m
c.order = append(c.order, m.Name)
}
// Aliases share a namespace with names. A collision would silently shadow a
// model at request time, so it has to fail at load.
for _, name := range c.order {
for _, a := range c.models[name].Aliases {
if _, clash := c.models[a]; clash {
return nil, fmt.Errorf("model %q: alias %q collides with another model name", name, a)
}
if prev, clash := c.aliases[a]; clash {
return nil, fmt.Errorf("alias %q is claimed by both %q and %q", a, prev, name)
}
c.aliases[a] = name
}
}
// Fallbacks are resolved now rather than at request time: a typo in a
// fallback should stop a deployment, not surface during an outage when the
// fallback is finally needed.
for _, name := range c.order {
for _, f := range c.models[name].Fallbacks {
if _, ok := c.Resolve(f); !ok {
return nil, fmt.Errorf("model %q: fallback %q is not in the catalog", name, f)
}
}
}
return c, nil
}
func (m Model) normalize() Model {
if m.Scaling.Min < 0 {
m.Scaling.Min = 0
}
if m.Scaling.Max == 0 {
m.Scaling.Max = max(m.Scaling.Min, 1)
}
for i := range m.Profiles {
p := &m.Profiles[i]
if p.Engine == "" {
p.Engine = "vllm"
}
if p.TP <= 0 {
p.TP = 1
}
if p.Nodes <= 0 {
p.Nodes = 1
}
if p.GPUsPerNode <= 0 {
// The common case: a single-node profile uses TP GPUs on one node.
p.GPUsPerNode = p.TP / p.Nodes
if p.GPUsPerNode <= 0 {
p.GPUsPerNode = 1
}
}
if p.StartupTimeout <= 0 {
p.StartupTimeout = 20 * time.Minute
}
if p.Weight <= 0 {
p.Weight = 1
}
}
return m
}
func (m Model) validate() error {
if m.Name == "" {
return fmt.Errorf("a model has no name")
}
if m.Source == "" {
return fmt.Errorf("model %q: source is required", m.Name)
}
if len(m.Profiles) == 0 {
return fmt.Errorf("model %q: at least one profile is required", m.Name)
}
if m.Scaling.Max < m.Scaling.Min {
return fmt.Errorf("model %q: scaling max %d is below min %d", m.Name, m.Scaling.Max, m.Scaling.Min)
}
seen := map[string]struct{}{}
for _, p := range m.Profiles {
if p.Name == "" {
return fmt.Errorf("model %q: a profile has no name", m.Name)
}
if _, dup := seen[p.Name]; dup {
return fmt.Errorf("model %q: profile %q is declared twice", m.Name, p.Name)
}
seen[p.Name] = struct{}{}
if p.GPUClass == "" {
return fmt.Errorf("model %q profile %q: gpu_class is required", m.Name, p.Name)
}
// The arithmetic has to add up, or the engine is handed a shard spec it
// cannot satisfy and fails minutes later during model loading.
if want := p.GPUsPerNode * p.Nodes; want != p.TP*max(p.PP, 1) {
return fmt.Errorf("model %q profile %q: %d nodes x %d GPUs does not match TP %d x PP %d",
m.Name, p.Name, p.Nodes, p.GPUsPerNode, p.TP, max(p.PP, 1))
}
}
return nil
}
// Resolve maps a client-supplied name or alias onto a model.
func (c *Catalog) Resolve(name string) (Model, bool) {
if m, ok := c.models[name]; ok {
return m, true
}
if canonical, ok := c.aliases[name]; ok {
return c.models[canonical], true
}
return Model{}, false
}
// Models returns every model, in declaration order.
func (c *Catalog) Models() []Model {
out := make([]Model, 0, len(c.order))
for _, n := range c.order {
out = append(out, c.models[n])
}
return out
}
// Names returns the canonical model names, sorted.
func (c *Catalog) Names() []string {
out := slices.Clone(c.order)
slices.Sort(out)
return out
}
// AliasesOf returns the alternative names for a model.
func (c *Catalog) AliasesOf(name string) []string {
m, ok := c.Resolve(name)
if !ok {
return nil
}
return slices.Clone(m.Aliases)
}
// PlacementProfiles converts a model's profiles into placement's view.
func (m Model) PlacementProfiles() []placement.Profile {
out := make([]placement.Profile, 0, len(m.Profiles))
for _, p := range m.Profiles {
out = append(out, placement.Profile{
Name: p.Name,
Engine: p.Engine,
GPUClass: p.GPUClass,
GPUsPerNode: p.GPUsPerNode,
Nodes: p.Nodes,
Priority: p.Priority,
})
}
return out
}
// EngineProfile converts a catalog profile into the engine adapter's view.
func (m Model) EngineProfile(name string) (engines.Profile, bool) {
for _, p := range m.Profiles {
if p.Name != name {
continue
}
return engines.Profile{
Name: p.Name,
Engine: p.Engine,
GPUClass: p.GPUClass,
TP: p.TP,
PP: p.PP,
GPUsPerNode: p.GPUsPerNode,
Nodes: p.Nodes,
Args: slices.Clone(p.Args),
Env: p.Env,
ModelPath: m.Source,
ServedModelName: m.Name,
}, true
}
return engines.Profile{}, false
}
// ProfileByName returns one of a model's profiles.
func (m Model) ProfileByName(name string) (Profile, bool) {
for _, p := range m.Profiles {
if p.Name == name {
return p, true
}
}
return Profile{}, false
}
// String renders the catalog for `ctl models`.
func (c *Catalog) String() string {
var b strings.Builder
for _, m := range c.Models() {
fmt.Fprintf(&b, "%s (min %d, max %d)\n", m.Name, m.Scaling.Min, m.Scaling.Max)
for _, p := range m.Profiles {
fmt.Fprintf(&b, " %-20s %-10s %-12s tp=%d nodes=%d priority=%d\n",
p.Name, p.Engine, p.GPUClass, p.TP, p.Nodes, p.Priority)
}
}
return b.String()
}