Llama Guard 3 1B could not tell ordinary coding requests from harmful ones: on the PoC it scored "continue" 0.51, "continue the code above" 0.89 and "how do I kill all child processes in bash" 0.92 unsafe, against 0.97 to 1.00 for harmful requests, at Q4_K_M and Q8_0 alike, with its own template. A site had "continue" refused. Qwen3Guard-Gen 0.6B gave the ordinary ones 0.15 at the most and the harmful ones 1.00, in 0.8 GB on CPUs. - format qwen3guard: its prompt written out as its template writes it, for requests and answers, and its Safe / Unsafe / Controversial verdict read; controversial counts as the policy's judge.controversial says, safe by default. - judge.models: judges asked in turn, the first that answers deciding; a failure is recorded against the judge that failed. guard judge use sets the order, replacing categories none of the judges names. - known judges (Qwen3Guard 0.6B/4B/8B, Llama Guard 1B/8B): guard judge enable serves any of them, --gpu the 4B on the roomiest GPU class as a standard model; the leader fetches and serves one a policy names that the catalog lacks. - server init installs qwen3guard-0.6b with the baseline's categories. - a judge reads a long text by its start and end, which fit its context, rather than overflowing it and refusing the request under fail: closed. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1221 lines
47 KiB
Go
1221 lines
47 KiB
Go
// Package catalog holds the desired state: which models exist, how each one can
|
|
// be run, and how many copies of it there should be (plan §6.1).
|
|
package catalog
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/url"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
|
)
|
|
|
|
// Scaling is how many replicas a model should have.
|
|
type Scaling struct {
|
|
Min int `json:"min,omitempty" koanf:"min"`
|
|
Max int `json:"max,omitempty" koanf:"max"`
|
|
// IdleTimeout is how long an on-demand model may go unasked-for before its
|
|
// replicas are stopped. Zero takes the cluster default.
|
|
//
|
|
// Minutes rather than seconds, always. Stopping a replica throws away the
|
|
// KV cache prefix routing spent its effort filling, and a model that flaps
|
|
// in and out loses that entirely — so the saving has to be worth more than
|
|
// the cache, which it only is over a real idle period.
|
|
IdleTimeout time.Duration `json:"idle_timeout,omitempty" koanf:"idle_timeout"`
|
|
// Schedule changes min and max by the time of day, in the control
|
|
// plane's control.timezone: more replicas kept through working hours,
|
|
// fewer at night, an on-demand model kept loaded while people are at
|
|
// work. The first window that holds now wins; outside them all, Min and
|
|
// Max apply.
|
|
Schedule []ScaleWindow `json:"schedule,omitempty" koanf:"schedule"`
|
|
// Prewarm starts a window this long before it opens, so its replicas
|
|
// have loaded by then: loading a model is minutes, and the people who
|
|
// arrive at nine are the reason for the window.
|
|
Prewarm time.Duration `json:"prewarm,omitempty" koanf:"prewarm"`
|
|
}
|
|
|
|
// ScaleWindow is a part of the week with its own min and max.
|
|
type ScaleWindow struct {
|
|
// Days are mon … sun, or weekdays, weekends; empty is every day.
|
|
Days []string `json:"days,omitempty" koanf:"days"`
|
|
// From and To are HH:MM; a window whose To is before its From runs
|
|
// past midnight.
|
|
From string `json:"from" koanf:"from"`
|
|
To string `json:"to" koanf:"to"`
|
|
Min int `json:"min" koanf:"min"`
|
|
// Max, when set, is the window's ceiling for load scaling; zero keeps
|
|
// the model's own.
|
|
Max int `json:"max,omitempty" koanf:"max"`
|
|
}
|
|
|
|
// clock parses HH:MM into minutes past midnight.
|
|
func clock(s string) (int, bool) {
|
|
var h, m int
|
|
if n, err := fmt.Sscanf(s, "%d:%d", &h, &m); err != nil || n != 2 || h < 0 || h > 23 || m < 0 || m > 59 || len(s) != 5 {
|
|
return 0, false
|
|
}
|
|
return h*60 + m, true
|
|
}
|
|
|
|
var weekdays = map[string][]time.Weekday{
|
|
"mon": {time.Monday}, "tue": {time.Tuesday}, "wed": {time.Wednesday}, "thu": {time.Thursday},
|
|
"fri": {time.Friday}, "sat": {time.Saturday}, "sun": {time.Sunday},
|
|
"weekdays": {time.Monday, time.Tuesday, time.Wednesday, time.Thursday, time.Friday},
|
|
"weekends": {time.Saturday, time.Sunday},
|
|
}
|
|
|
|
func (w ScaleWindow) validate() error {
|
|
from, ok1 := clock(w.From)
|
|
to, ok2 := clock(w.To)
|
|
if !ok1 || !ok2 {
|
|
return fmt.Errorf("from and to are HH:MM, like 08:00")
|
|
}
|
|
if from == to {
|
|
return fmt.Errorf("from and to are the same time")
|
|
}
|
|
for _, d := range w.Days {
|
|
if _, ok := weekdays[strings.ToLower(d)]; !ok {
|
|
return fmt.Errorf("day %q is not mon … sun, weekdays or weekends", d)
|
|
}
|
|
}
|
|
if w.Min < 0 || w.Max < 0 || w.Max > 0 && w.Max < w.Min {
|
|
return fmt.Errorf("min %d and max %d: max is 0 or at least min", w.Min, w.Max)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// holds says whether t falls in the window, opened early by lead.
|
|
func (w ScaleWindow) holds(t time.Time, lead time.Duration) bool {
|
|
from, _ := clock(w.From)
|
|
to, _ := clock(w.To)
|
|
span := to - from
|
|
if span <= 0 {
|
|
span += 24 * 60 // past midnight
|
|
}
|
|
midnight := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
|
|
// The window as it starts yesterday, today and tomorrow — tomorrow's
|
|
// can open today, pulled forward by the lead.
|
|
for d := -1; d <= 1; d++ {
|
|
day := midnight.AddDate(0, 0, d)
|
|
if !w.on(day.Weekday()) {
|
|
continue
|
|
}
|
|
start := day.Add(time.Duration(from)*time.Minute - lead)
|
|
end := day.Add(time.Duration(from+span) * time.Minute)
|
|
if !t.Before(start) && t.Before(end) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// on says whether the window starts on a day.
|
|
func (w ScaleWindow) on(wd time.Weekday) bool {
|
|
if len(w.Days) == 0 {
|
|
return true
|
|
}
|
|
for _, d := range w.Days {
|
|
for _, x := range weekdays[strings.ToLower(d)] {
|
|
if x == wd {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Window is the window in force at t, in its own time zone, and false
|
|
// outside them all.
|
|
func (s Scaling) Window(t time.Time) (ScaleWindow, bool) {
|
|
for _, w := range s.Schedule {
|
|
if w.holds(t, s.Prewarm) {
|
|
return w, true
|
|
}
|
|
}
|
|
return ScaleWindow{}, false
|
|
}
|
|
|
|
// OnDemand reports whether this model is loaded when someone asks for it.
|
|
//
|
|
// min 0 with a max above it, which needs no new field and reads correctly: keep
|
|
// none running, allow up to max. min 0 and max 0 is a model that is configured
|
|
// and deliberately not running, which is a different and equally useful thing —
|
|
// the difference between "load it when wanted" and "do not load it".
|
|
func (s Scaling) OnDemand() bool { return s.Min == 0 && s.Max > 0 }
|
|
|
|
// SLO is the latency target a model is expected to meet. `ctl bench` measures
|
|
// capacity against it (plan §6.2.1), and admission holds the lower bands
|
|
// back while the model's recent p99 is past it.
|
|
type SLO struct {
|
|
// TTFTp99 is the time to the first token 99 requests in 100 are within.
|
|
TTFTp99 time.Duration `koanf:"ttft_p99"`
|
|
// TPOTp99 is the time per output token, after the first, 99 streamed
|
|
// requests in 100 are within.
|
|
TPOTp99 time.Duration `koanf:"tpot_p99"`
|
|
}
|
|
|
|
// Profile is how a model runs on one GPU class.
|
|
type Profile struct {
|
|
Name string `json:"name,omitempty" koanf:"name"`
|
|
Engine string `json:"engine,omitempty" koanf:"engine"`
|
|
// EngineVersion restricts the profile to nodes whose installed engine's
|
|
// version starts with it: "0.11" takes vLLM 0.11.0 and 0.11.1, not
|
|
// 0.10.2. Empty takes whatever version the node has.
|
|
EngineVersion string `json:"engine_version,omitempty" koanf:"engine_version"`
|
|
GPUClass string `json:"gpu_class,omitempty" koanf:"gpu_class"`
|
|
|
|
TP int `json:"tp,omitempty" koanf:"tp"`
|
|
PP int `json:"pp,omitempty" koanf:"pp"`
|
|
// GPUsPerNode defaults to TP for a single-node profile.
|
|
GPUsPerNode int `json:"gpus_per_node,omitempty" koanf:"gpus_per_node"`
|
|
// Nodes above 1 makes this a multi-node profile, which requires a fabric.
|
|
Nodes int `json:"nodes,omitempty" koanf:"nodes"`
|
|
|
|
Args []string `json:"args,omitempty" koanf:"args"`
|
|
Env map[string]string `json:"env,omitempty" koanf:"env"`
|
|
|
|
// MinMemoryGB is the host RAM this profile needs to run.
|
|
//
|
|
// The number that matters is the model's working set: the weights, plus the
|
|
// KV cache for the whole context pool, plus the engine's own overhead. For
|
|
// a GPU profile that lives in the cards and this is small; for a CPU
|
|
// profile it is the whole thing, and getting it wrong means an OOM on a
|
|
// node that reported itself healthy.
|
|
//
|
|
// Zero means unconstrained, which is every profile that existed before
|
|
// this. A node that does not report its memory is not excluded either —
|
|
// an older agent must not have its nodes quietly emptied by an upgrade of
|
|
// the control plane.
|
|
MinMemoryGB int `json:"min_memory_gb,omitempty" koanf:"min_memory_gb"`
|
|
|
|
// CPUCores is the cores a replica of a profile with no GPU is given:
|
|
// reserved out of its node's CPUs by placement, as memory is, and what
|
|
// its engine computes on (llama.cpp's --threads).
|
|
//
|
|
// Reserved because an engine takes every core it is told about, and two
|
|
// on one node that each think they have all of them spin against each
|
|
// other: on an 8-vCPU node, two 7B replicas fell from 10 tokens a second
|
|
// to under one each. Zero reserves nothing and leaves the thread count to
|
|
// the node, which is every profile from before this.
|
|
CPUCores int `json:"cpu_cores,omitempty" koanf:"cpu_cores"`
|
|
|
|
// MinDiskGB is the free space a node needs before it can hold these
|
|
// weights.
|
|
//
|
|
// Only checked when the node does not already have them: a node holding
|
|
// the file needs no room for it, and requiring the space twice would keep
|
|
// a model off the one node best placed to run it.
|
|
//
|
|
// Declared rather than derived, because the size of a model that has never
|
|
// been fetched anywhere is not knowable from here — and a wrong guess
|
|
// fails late, halfway through a download, which is the failure this exists
|
|
// to prevent. Zero means unconstrained, and a node that reports no storage
|
|
// is not excluded: unknown is not full.
|
|
MinDiskGB int `json:"min_disk_gb,omitempty" koanf:"min_disk_gb"`
|
|
|
|
// NodeSelector restricts this profile to nodes carrying every one of these
|
|
// labels, with these values.
|
|
//
|
|
// A CPU profile otherwise fits everywhere, because there is no card to be
|
|
// free — which is right for a 0.5B model and wrong for a 14B one. Nothing
|
|
// in the inventory says how much memory a node has, so the operator says
|
|
// which nodes are meant for a model instead of the scheduler inferring it.
|
|
// Agents have reported labels since the beginning; this is what reads them.
|
|
NodeSelector map[string]string `json:"node_selector,omitempty" koanf:"node_selector"`
|
|
|
|
// PowerWatts is what one replica of this profile draws, in watts, split
|
|
// evenly over its nodes, counted against the power cap of every rack,
|
|
// PDU or other domain its nodes are in (facility.budgets in server.yaml).
|
|
// Zero derives it: each card at the power limit it reports, each
|
|
// reserved core at facility.cpu_core_watts. Only matters where a domain
|
|
// has a cap.
|
|
PowerWatts float64 `json:"power_watts,omitempty" koanf:"power_watts"`
|
|
|
|
// KeepWithin keeps a multi-node replica's nodes inside one rack, row,
|
|
// room or site, read from their llmgw.io/rack (and so on) labels: for a
|
|
// fabric whose leaves do not reach across racks, or cables that do not.
|
|
// Nodes without the label are passed over for such a profile. Empty is
|
|
// no constraint.
|
|
KeepWithin string `json:"keep_within,omitempty" koanf:"keep_within"`
|
|
|
|
// Weight is the router's capacity weight. It should come from ctl bench
|
|
// rather than a guess (plan §6.2.1).
|
|
Weight float64 `json:"weight,omitempty" koanf:"weight"`
|
|
// Priority orders profiles during placement; highest that fits wins.
|
|
Priority int `json:"priority,omitempty" koanf:"priority"`
|
|
// StartupTimeout bounds model loading. A 70B model across two nodes takes
|
|
// tens of minutes, so this is per profile rather than global.
|
|
StartupTimeout time.Duration `json:"startup_timeout,omitempty" koanf:"startup_timeout"`
|
|
|
|
// KVEvents has vLLM publish what its prefix cache holds, and the router
|
|
// route this model's requests by it: to the replica holding the most of
|
|
// a prompt's leading blocks (plan §7.5 tier 2). vLLM only.
|
|
//
|
|
// Off by default, because it costs something on every request: the
|
|
// router asks a replica to tokenize the prompt before placing it, a
|
|
// round trip and the chat template run twice. It pays where prompts
|
|
// share long prefixes that the approximate tree guesses wrong — many
|
|
// replicas, conversations that move between them, evictions. The engine
|
|
// is started with sha256_cbor block hashing and a fixed PYTHONHASHSEED,
|
|
// which the router's hashing assumes; arguments overriding either leave
|
|
// the index empty, and the agent says so in its log.
|
|
KVEvents bool `json:"kv_events,omitempty" koanf:"kv_events"`
|
|
}
|
|
|
|
// CPUOnly reports whether this profile needs no GPUs.
|
|
//
|
|
// Naming no GPU class is how a profile says so, and only engines that can
|
|
// actually serve without a card are allowed to leave it out — otherwise an
|
|
// omitted class would quietly become a replica placed on a machine with no
|
|
// hardware to run it.
|
|
func (p Profile) CPUOnly() bool { return p.GPUClass == "" && cpuCapable(p.Engine) }
|
|
|
|
// cpuCapable reports whether an engine can serve a model without a GPU.
|
|
//
|
|
// llama.cpp is the one that can, and it is what makes a cluster with no cards
|
|
// testable end to end.
|
|
func cpuCapable(engine string) bool { return engine == "llamacpp" }
|
|
|
|
// Model is a catalog entry as clients see it.
|
|
type Model struct {
|
|
Name string `json:"name,omitempty" koanf:"name"`
|
|
Aliases []string `json:"aliases,omitempty" koanf:"aliases"`
|
|
// Description is what this model is for, in a line.
|
|
//
|
|
// Written by whoever added it, because nobody else can say it. A registry
|
|
// offers a pipeline tag and a wall of README, and neither answers the
|
|
// question a person picking from a list is asking: which of these nine
|
|
// names should I send this request to.
|
|
//
|
|
// It reaches clients through /v1/models, which is the only view most of
|
|
// them ever see. A gateway serving nine models by name alone makes that
|
|
// choice by guesswork.
|
|
Description string `json:"description,omitempty" koanf:"description"`
|
|
// Source is the weights path or a HuggingFace id.
|
|
Source string `json:"source,omitempty" koanf:"source"`
|
|
Profiles []Profile `json:"profiles,omitempty" koanf:"profiles"`
|
|
Scaling Scaling `json:"scaling,omitempty" koanf:"scaling"`
|
|
// Class is how much this model's capacity is worth against another's,
|
|
// when there is not enough for both: critical, standard or opportunistic.
|
|
// Empty is standard, or opportunistic for a model loaded on demand. See
|
|
// EffectiveClass.
|
|
Class string `json:"class,omitempty" koanf:"class"`
|
|
// Owner is the part of the tenancy tree whose capacity quota this model
|
|
// counts against, written org:ID, team:ID or project:ID. Empty is shared:
|
|
// held by the cluster, counted against nobody.
|
|
Owner string `json:"owner,omitempty" koanf:"owner"`
|
|
// External is a model served by someone else — an OpenAI-compatible API
|
|
// outside the cluster — rather than by replicas placed here. Such a model
|
|
// has no profiles and nothing is placed for it; the router sends its
|
|
// requests to the provider. Most useful named in another model's
|
|
// fallbacks: used only when the cluster has no capacity left. Its
|
|
// prompts leave the cluster, so which keys may name it is worth an
|
|
// allowlist.
|
|
External *External `json:"external,omitempty" koanf:"external"`
|
|
// Adapters are LoRA adapters served on this model's replicas, each asked
|
|
// for by its own name as though it were a model. Every replica loads
|
|
// every adapter, applying one only to a request that names it — unless
|
|
// MaxLoadedAdapters is set.
|
|
Adapters []Adapter `json:"adapters,omitempty" koanf:"adapters"`
|
|
// MaxLoadedAdapters, above zero, has each replica hold at most this many
|
|
// adapters at once, loaded when a request first asks for one and unloaded
|
|
// least recently used first to make room. The router sends a request for
|
|
// an adapter to a replica that already holds it while that one has room
|
|
// to take it. vLLM profiles only: llama-server loads adapters at start
|
|
// and cannot load one afterwards. Zero loads every adapter on every
|
|
// replica at start.
|
|
MaxLoadedAdapters int `json:"max_loaded_adapters,omitempty" koanf:"max_loaded_adapters"`
|
|
Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"`
|
|
// Canary sends a share of this model's requests to another catalog
|
|
// model — the same model at new weights — to compare the two before
|
|
// moving to it (`llmgw ctl model canary`).
|
|
Canary *Canary `json:"canary,omitempty" koanf:"canary"`
|
|
// Eval is a quality suite new weights must pass before they replace
|
|
// this model's: `llmgw ctl model update` and `model promote` run it
|
|
// against them first, and stop below its score.
|
|
Eval *EvalGate `json:"eval,omitempty" koanf:"eval"`
|
|
// RoutingProfile selects how the router scores endpoints for this model.
|
|
RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"`
|
|
SLO SLO `json:"slo,omitempty" koanf:"slo"`
|
|
|
|
// ContextLength is how many tokens one request may use, prompt and
|
|
// completion together, and MaxOutput bounds the completion alone. Zero
|
|
// means unknown, which is what every model was until now.
|
|
//
|
|
// The gateway knew neither, so an over-long prompt was not refused: it was
|
|
// admitted, given a replica, and failed at the engine, and a client sizing
|
|
// its prompts from /v1/models had nothing to read. Worse, the number is not
|
|
// the one an operator writes in the engine's arguments — llama.cpp divides
|
|
// --ctx-size across its parallel slots, so a pool of 8192 across four slots
|
|
// is 2048 a request, and nothing said so.
|
|
ContextLength int `json:"context_length,omitempty" koanf:"context_length"`
|
|
MaxOutput int `json:"max_output,omitempty" koanf:"max_output"`
|
|
|
|
// Capabilities is what this model can do beyond plain completion.
|
|
Capabilities Capabilities `json:"capabilities,omitempty" koanf:"capabilities"`
|
|
|
|
// Role "guard" is a model that judges the gateway's traffic for the
|
|
// guardrails rather than serving clients: hidden from /v1/models and the
|
|
// chat page, refused to any key that names it, critical by default, and
|
|
// always running. Guard says how its answers are read.
|
|
Role string `json:"role,omitempty" koanf:"role"`
|
|
Guard *GuardModel `json:"guard,omitempty" koanf:"guard"`
|
|
|
|
// Origin is the registry repository the weights came from, at which
|
|
// commit, and each file's checksum there. Written when a model is served
|
|
// from the registry; for one that was not, worked out from the mirror's
|
|
// record of where it fetched the file. It is what lets the control plane
|
|
// say that the repository has published different weights since, and
|
|
// move the model to them.
|
|
Origin *Origin `json:"origin,omitempty" koanf:"origin"`
|
|
}
|
|
|
|
// RoleGuard is a judge model's role.
|
|
const RoleGuard = "guard"
|
|
|
|
// IsGuard reports whether a model judges traffic rather than serving it.
|
|
func (m Model) IsGuard() bool { return m.Role == RoleGuard }
|
|
|
|
// GuardModel is how a judge model's answers are read.
|
|
type GuardModel struct {
|
|
// Format is qwen3guard (Qwen3Guard-Gen: "Safety: Safe", "Unsafe" or
|
|
// "Controversial", and the categories), llamaguard3 (Meta's Llama Guard
|
|
// 3: "safe", or "unsafe" and the categories), shieldgemma (Google's
|
|
// ShieldGemma: Yes or No), or json ({"safe": false, "categories":
|
|
// ["..."]}).
|
|
Format string `json:"format" koanf:"format"`
|
|
// Categories the model is asked about, for shieldgemma — each a policy
|
|
// sentence — and the names reported for llamaguard3's S1, S2, ….
|
|
Categories []string `json:"categories,omitempty" koanf:"categories"`
|
|
}
|
|
|
|
// Origin is where a model's weights were published.
|
|
type Origin struct {
|
|
// Repo is the repository, owner/name.
|
|
Repo string `json:"repo" koanf:"repo"`
|
|
// Revision is the commit the files were fetched at.
|
|
Revision string `json:"revision" koanf:"revision"`
|
|
// Files are the weights fetched from it, with what the registry
|
|
// published as each one's sha256. A new commit that leaves them alone —
|
|
// a README edited — is not an update.
|
|
Files []OriginFile `json:"files,omitempty" koanf:"files"`
|
|
// Previous is what the last update replaced, kept so it can be gone back
|
|
// to while its files are still on the mirror.
|
|
Previous *PreviousOrigin `json:"previous,omitempty" koanf:"previous"`
|
|
}
|
|
|
|
// OriginFile is one weights file of an origin.
|
|
type OriginFile struct {
|
|
Path string `json:"path" koanf:"path"`
|
|
SHA256 string `json:"sha256,omitempty" koanf:"sha256"`
|
|
Bytes int64 `json:"bytes,omitempty" koanf:"bytes"`
|
|
}
|
|
|
|
// PreviousOrigin is a model's source before an update, and where it came from.
|
|
type PreviousOrigin struct {
|
|
Source string `json:"source" koanf:"source"`
|
|
Revision string `json:"revision" koanf:"revision"`
|
|
Files []OriginFile `json:"files,omitempty" koanf:"files"`
|
|
}
|
|
|
|
// ShortRevision is a commit as a person writes it.
|
|
func ShortRevision(rev string) string {
|
|
if len(rev) > 8 {
|
|
return rev[:8]
|
|
}
|
|
return rev
|
|
}
|
|
|
|
// Capabilities describes what a client may ask a model for.
|
|
//
|
|
// Declared by the operator rather than discovered, because there is no way to
|
|
// ask an engine what its model can do. That makes these claims, and a wrong
|
|
// claim is worse than none — which is why `ctl engine-check` verifies tool
|
|
// calling against a running engine rather than trusting this.
|
|
//
|
|
// The reason it exists: the gateway forwards a tools array to any model, and a
|
|
// model that cannot use them answers in prose. Every OpenAI-compatible client
|
|
// then behaves as though the model is broken, and the only way to find out was
|
|
// to try it. That cost an afternoon.
|
|
type Capabilities struct {
|
|
Tools bool `json:"tools,omitempty" koanf:"tools"`
|
|
Vision bool `json:"vision,omitempty" koanf:"vision"`
|
|
Embeddings bool `json:"embeddings,omitempty" koanf:"embeddings"`
|
|
}
|
|
|
|
// Catalog is the set of models the cluster should serve.
|
|
type Catalog struct {
|
|
models map[string]Model
|
|
aliases map[string]string
|
|
// adapters maps an adapter's name to the model it runs on.
|
|
adapters map[string]string
|
|
order []string
|
|
}
|
|
|
|
// New builds a catalog, rejecting definitions that would fail confusingly later.
|
|
func New(models []Model) (*Catalog, error) {
|
|
c := &Catalog{
|
|
models: make(map[string]Model, len(models)),
|
|
aliases: map[string]string{},
|
|
adapters: map[string]string{},
|
|
}
|
|
|
|
for i := range models {
|
|
m := models[i].normalize()
|
|
if err := m.validate(); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, dup := c.models[m.Name]; dup {
|
|
return nil, fmt.Errorf("model %q is declared twice", m.Name)
|
|
}
|
|
c.models[m.Name] = m
|
|
c.order = append(c.order, m.Name)
|
|
}
|
|
|
|
// Aliases share a namespace with names. A collision would silently shadow a
|
|
// model at request time, so it has to fail at load.
|
|
for _, name := range c.order {
|
|
for _, a := range c.models[name].Aliases {
|
|
if _, clash := c.models[a]; clash {
|
|
return nil, fmt.Errorf("model %q: alias %q collides with another model name", name, a)
|
|
}
|
|
if prev, clash := c.aliases[a]; clash {
|
|
return nil, fmt.Errorf("alias %q is claimed by both %q and %q", a, prev, name)
|
|
}
|
|
c.aliases[a] = name
|
|
}
|
|
}
|
|
|
|
// Adapters are named like models, so they share the namespace too.
|
|
for _, name := range c.order {
|
|
for _, a := range c.models[name].Adapters {
|
|
_, isModel := c.models[a.Name]
|
|
_, isAlias := c.aliases[a.Name]
|
|
if _, isAdapter := c.adapters[a.Name]; isModel || isAlias || isAdapter {
|
|
return nil, fmt.Errorf("model %q: adapter %q collides with a model, an alias or another adapter", name, a.Name)
|
|
}
|
|
c.adapters[a.Name] = name
|
|
}
|
|
}
|
|
|
|
// Fallbacks are resolved now rather than at request time: a typo in a
|
|
// fallback should stop a deployment, not surface during an outage when the
|
|
// fallback is finally needed.
|
|
for _, name := range c.order {
|
|
for _, f := range c.models[name].Fallbacks {
|
|
if _, ok := c.Resolve(f); !ok {
|
|
return nil, fmt.Errorf("model %q: fallback %q is not in the catalog", name, f)
|
|
}
|
|
}
|
|
}
|
|
// A canary is another model in the catalog, a candidate for this one's
|
|
// place: not itself, not a judge, not a canary of its own, and the
|
|
// candidate of one model only.
|
|
canaryOf := map[string]string{}
|
|
for _, name := range c.order {
|
|
cn := c.models[name].Canary
|
|
if cn == nil {
|
|
continue
|
|
}
|
|
target, ok := c.Resolve(cn.Model)
|
|
switch {
|
|
case !ok:
|
|
return nil, fmt.Errorf("model %q: canary %q is not in the catalog", name, cn.Model)
|
|
case target.Name == name:
|
|
return nil, fmt.Errorf("model %q: a model cannot be its own canary", name)
|
|
case target.IsGuard():
|
|
return nil, fmt.Errorf("model %q: canary %q is a guard model", name, cn.Model)
|
|
case target.Canary != nil:
|
|
return nil, fmt.Errorf("model %q: canary %q has a canary of its own", name, cn.Model)
|
|
case canaryOf[target.Name] != "":
|
|
return nil, fmt.Errorf("model %q: %q is already the canary of %q", name, cn.Model, canaryOf[target.Name])
|
|
case cn.Percent < 0 || cn.Percent > 100:
|
|
return nil, fmt.Errorf("model %q: canary percent %d is not 0 to 100", name, cn.Percent)
|
|
}
|
|
canaryOf[target.Name] = name
|
|
}
|
|
return c, nil
|
|
}
|
|
|
|
// Canary is a share of a model's requests sent to a candidate, and when to
|
|
// give up on it.
|
|
type Canary struct {
|
|
// Model is the candidate: a catalog model, usually this one at newer
|
|
// weights, which clients cannot name themselves.
|
|
Model string `json:"model" koanf:"model"`
|
|
// Percent of requests go to it, 0 to 100.
|
|
Percent int `json:"percent" koanf:"percent"`
|
|
// The candidate is taken out — rolled back — when, over at least
|
|
// MinRequests of its own, its error rate is MaxErrorIncrease above
|
|
// this model's (0.05 is five points), or its mean time to the first
|
|
// token is MaxTTFTRatio times this model's. Zero is 50, 0.05 and 1.5.
|
|
MinRequests int `json:"min_requests,omitempty" koanf:"min_requests"`
|
|
MaxErrorIncrease float64 `json:"max_error_increase,omitempty" koanf:"max_error_increase"`
|
|
MaxTTFTRatio float64 `json:"max_ttft_ratio,omitempty" koanf:"max_ttft_ratio"`
|
|
}
|
|
|
|
// EvalGate is a quality suite a model's new weights must pass.
|
|
type EvalGate struct {
|
|
// Suite is a suite's name in control.eval_dir, or one the binary
|
|
// carries (arithmetic, json-format).
|
|
Suite string `json:"suite" koanf:"suite"`
|
|
// MinScore is the score, 0 to 1, to pass at; zero is the suite's own
|
|
// threshold.
|
|
MinScore float64 `json:"min_score,omitempty" koanf:"min_score"`
|
|
// Judge grades the suite's rubric cases, instead of the judge it names.
|
|
Judge string `json:"judge,omitempty" koanf:"judge"`
|
|
}
|
|
|
|
// Limits are the rollback thresholds with their defaults filled in.
|
|
func (c Canary) Limits() (minRequests int, maxErrorIncrease, maxTTFTRatio float64) {
|
|
minRequests, maxErrorIncrease, maxTTFTRatio = c.MinRequests, c.MaxErrorIncrease, c.MaxTTFTRatio
|
|
if minRequests <= 0 {
|
|
minRequests = 50
|
|
}
|
|
if maxErrorIncrease <= 0 {
|
|
maxErrorIncrease = 0.05
|
|
}
|
|
if maxTTFTRatio <= 0 {
|
|
maxTTFTRatio = 1.5
|
|
}
|
|
return
|
|
}
|
|
|
|
// CanaryOf is the model whose canary name is, or "".
|
|
func (c *Catalog) CanaryOf(name string) string {
|
|
for _, n := range c.order {
|
|
if cn := c.models[n].Canary; cn != nil {
|
|
if t, ok := c.Resolve(cn.Model); ok && t.Name == name {
|
|
return n
|
|
}
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func (m Model) normalize() Model {
|
|
if m.Scaling.Min < 0 {
|
|
m.Scaling.Min = 0
|
|
}
|
|
// Max 0 used to be an omission to fill in, because a model with no maximum
|
|
// could not run. It is now also a statement: with min 0 it means configured
|
|
// and deliberately not running, which is a different and useful thing from
|
|
// "load it when wanted". Filling it in would silently turn every switched-
|
|
// off model into an on-demand one — including the ones already written that
|
|
// way in a running cluster's configuration.
|
|
if m.Scaling.Max == 0 && m.Scaling.Min > 0 {
|
|
m.Scaling.Max = m.Scaling.Min
|
|
}
|
|
for i := range m.Profiles {
|
|
p := &m.Profiles[i]
|
|
if p.Engine == "" {
|
|
p.Engine = "vllm"
|
|
}
|
|
if p.TP <= 0 {
|
|
p.TP = 1
|
|
}
|
|
if p.Nodes <= 0 {
|
|
p.Nodes = 1
|
|
}
|
|
// tp on a llama.cpp GPU profile, written before gpus_per_node meant
|
|
// anything for it, is read as the width it was meant as: N cards of
|
|
// one node. Refusing it would have dropped a catalog's model on the
|
|
// upgrade that made the width work. Only both given, disagreeing, is
|
|
// refused below, as ambiguous.
|
|
if p.Engine == "llamacpp" && p.GPUClass != "" && (p.TP > 1 || p.PP > 1) && p.Nodes == 1 {
|
|
if n := p.TP * max(p.PP, 1); p.GPUsPerNode == 0 || p.GPUsPerNode == n {
|
|
p.GPUsPerNode, p.TP, p.PP = n, 1, 0
|
|
}
|
|
}
|
|
switch {
|
|
case p.CPUOnly():
|
|
// Nothing to divide up: this profile asks for no cards at all.
|
|
p.GPUsPerNode = 0
|
|
case p.GPUsPerNode <= 0:
|
|
// The common case: a single-node profile uses TP GPUs on one node.
|
|
p.GPUsPerNode = p.TP / p.Nodes
|
|
if p.GPUsPerNode <= 0 {
|
|
p.GPUsPerNode = 1
|
|
}
|
|
}
|
|
if p.StartupTimeout <= 0 {
|
|
p.StartupTimeout = 20 * time.Minute
|
|
}
|
|
if p.Weight <= 0 {
|
|
p.Weight = 1
|
|
}
|
|
}
|
|
// A Hugging Face repository page for a model vLLM or SGLang runs is the
|
|
// repository it names: the engine downloads a repository id itself,
|
|
// where the page was fetched as one file of HTML and failed at start.
|
|
if repo, ok := repositoryPage(m.Source); ok && len(m.Profiles) > 0 &&
|
|
!slices.ContainsFunc(m.Profiles, func(p Profile) bool { return p.Engine == "llamacpp" }) {
|
|
m.Source = repo
|
|
}
|
|
return m
|
|
}
|
|
|
|
func (m Model) validate() error {
|
|
if m.Name == "" {
|
|
return fmt.Errorf("a model has no name")
|
|
}
|
|
if m.Source == "" {
|
|
return fmt.Errorf("model %q: source is required", m.Name)
|
|
}
|
|
if repo, ok := repositoryPage(m.Source); ok {
|
|
// A page, not weights: a node fetches the HTML as one file and the
|
|
// engine fails at start, on a path that looks right. For vLLM it was
|
|
// read as the repository id when the model was made; only a model
|
|
// llama.cpp runs is left here, which needs a file.
|
|
return fmt.Errorf("model %q: source %s is a Hugging Face repository page, not a file; "+
|
|
"llama.cpp needs a .gguf: mirror one and use its source (repository %s)",
|
|
m.Name, m.Source, repo)
|
|
}
|
|
if err := validateAdapters(m); err != nil {
|
|
return err
|
|
}
|
|
if m.External != nil {
|
|
if err := m.External.validate(m.Name, len(m.Profiles)); err != nil {
|
|
return err
|
|
}
|
|
} else if len(m.Profiles) == 0 {
|
|
return fmt.Errorf("model %q: at least one profile is required", m.Name)
|
|
}
|
|
if m.Owner != "" && !validOwner(m.Owner) {
|
|
return fmt.Errorf("model %q: owner %q is not org:ID, team:ID or project:ID", m.Name, m.Owner)
|
|
}
|
|
if !validClass(m.Class) {
|
|
return fmt.Errorf("model %q: class %q is not one of critical, standard, opportunistic", m.Name, m.Class)
|
|
}
|
|
if g := m.Eval; g != nil {
|
|
if strings.TrimSpace(g.Suite) == "" {
|
|
return fmt.Errorf("model %q: eval names no suite", m.Name)
|
|
}
|
|
if g.MinScore < 0 || g.MinScore > 1 {
|
|
return fmt.Errorf("model %q: eval min_score %v is not 0 to 1", m.Name, g.MinScore)
|
|
}
|
|
}
|
|
switch m.Role {
|
|
case "":
|
|
if m.Guard != nil {
|
|
return fmt.Errorf("model %q: guard: is for a model with role: guard", m.Name)
|
|
}
|
|
case RoleGuard:
|
|
if m.Guard == nil {
|
|
return fmt.Errorf("model %q: a guard model needs guard.format: %s", m.Name, strings.Join(guard.Formats, ", "))
|
|
}
|
|
if !guard.KnownFormat(m.Guard.Format) {
|
|
return fmt.Errorf("model %q: guard.format %q is not one of %s", m.Name, m.Guard.Format, strings.Join(guard.Formats, ", "))
|
|
}
|
|
if m.Scaling.OnDemand() {
|
|
// A judge loaded on demand makes the first request of every
|
|
// idle period wait for it, or be judged by nothing.
|
|
return fmt.Errorf("model %q: a guard model must always run: scaling.min 1 or more", m.Name)
|
|
}
|
|
if m.IsExternal() {
|
|
return fmt.Errorf("model %q: a guard model judges in the cluster; it cannot be external", m.Name)
|
|
}
|
|
default:
|
|
return fmt.Errorf("model %q: role %q is not guard", m.Name, m.Role)
|
|
}
|
|
for i, w := range m.Scaling.Schedule {
|
|
if err := w.validate(); err != nil {
|
|
return fmt.Errorf("model %q: scaling.schedule[%d]: %w", m.Name, i, err)
|
|
}
|
|
}
|
|
if m.Scaling.Prewarm < 0 || m.Scaling.Prewarm > 12*time.Hour {
|
|
return fmt.Errorf("model %q: scaling.prewarm is between 0 and 12h", m.Name)
|
|
}
|
|
if m.Scaling.Max < m.Scaling.Min {
|
|
return fmt.Errorf("model %q: scaling max %d is below min %d", m.Name, m.Scaling.Max, m.Scaling.Min)
|
|
}
|
|
|
|
seen := map[string]struct{}{}
|
|
for _, p := range m.Profiles {
|
|
if p.Name == "" {
|
|
return fmt.Errorf("model %q: a profile has no name", m.Name)
|
|
}
|
|
if _, dup := seen[p.Name]; dup {
|
|
return fmt.Errorf("model %q: profile %q is declared twice", m.Name, p.Name)
|
|
}
|
|
seen[p.Name] = struct{}{}
|
|
|
|
for k := range p.Env {
|
|
if bad, why := engines.UnsafeEnv(k); bad {
|
|
return fmt.Errorf("model %q profile %q: env %s is not allowed: %s",
|
|
m.Name, p.Name, k, why)
|
|
}
|
|
}
|
|
if err := engines.UnsafeArgs(p.Engine, p.Args); err != nil {
|
|
return fmt.Errorf("model %q profile %q: %w", m.Name, p.Name, err)
|
|
}
|
|
if p.KVEvents && p.Engine != "vllm" {
|
|
return fmt.Errorf("model %q profile %q: kv_events is vLLM's; engine %q publishes no cache events",
|
|
m.Name, p.Name, p.Engine)
|
|
}
|
|
|
|
if p.PowerWatts < 0 {
|
|
return fmt.Errorf("model %q profile %q: power_watts %v is negative", m.Name, p.Name, p.PowerWatts)
|
|
}
|
|
if p.KeepWithin != "" {
|
|
if _, err := facility.ParseLevel(p.KeepWithin); err != nil {
|
|
return fmt.Errorf("model %q profile %q: keep_within: %w", m.Name, p.Name, err)
|
|
}
|
|
}
|
|
if p.GPUClass == "" && !cpuCapable(p.Engine) {
|
|
return fmt.Errorf("model %q profile %q: gpu_class is required for engine %q",
|
|
m.Name, p.Name, p.Engine)
|
|
}
|
|
if p.CPUOnly() {
|
|
if p.Nodes > 1 {
|
|
return fmt.Errorf("model %q profile %q: a CPU profile cannot span %d nodes",
|
|
m.Name, p.Name, p.Nodes)
|
|
}
|
|
continue
|
|
}
|
|
// A MIG instance is a device of its own to CUDA, but a process sees
|
|
// only one, and instances have no NVLink or peer-to-peer between
|
|
// them: a group of them is neither tensor- nor layer-parallel, it is
|
|
// a replica that fails at start. One instance, one node, refused here
|
|
// rather than placed and found out.
|
|
if discovery.IsMIGClass(p.GPUClass) && (p.GPUsPerNode > 1 || p.Nodes > 1 || p.TP > 1 || p.PP > 1) {
|
|
return fmt.Errorf("model %q profile %q: %s is a MIG instance, and a replica runs on exactly one: "+
|
|
"a process can open only one MIG device, and instances have no NVLink between them. "+
|
|
"Give it tp 1 and one GPU on one node, or use a whole-card class or a larger MIG profile",
|
|
m.Name, p.Name, p.GPUClass)
|
|
}
|
|
// llama.cpp on GPUs splits a model across one node's cards by layer,
|
|
// and how many is gpus_per_node. Not tp: llama.cpp has no tensor
|
|
// parallelism, and a tp it was given was a width nothing placed — the
|
|
// replica stayed unplaced, or ran on one card while four were meant.
|
|
if p.Engine == "llamacpp" {
|
|
switch {
|
|
case p.Nodes > 1:
|
|
return fmt.Errorf("model %q profile %q: llama.cpp does not span nodes; use vllm or sglang for a %d-node replica",
|
|
m.Name, p.Name, p.Nodes)
|
|
case p.TP > 1 || p.PP > 1:
|
|
return fmt.Errorf("model %q profile %q: llama.cpp splits a model across a node's GPUs by layer, and "+
|
|
"tp %d does not agree with gpus_per_node %d: give gpus_per_node alone", m.Name, p.Name,
|
|
max(p.TP, 1)*max(p.PP, 1), p.GPUsPerNode)
|
|
}
|
|
continue
|
|
}
|
|
// TensorRT-LLM spans machines only under MPI started from one of them,
|
|
// which the agents, each starting its own node, do not do; and it
|
|
// loads a Hugging Face checkpoint, not llama.cpp's format.
|
|
if p.Engine == "trtllm" {
|
|
if p.Nodes > 1 {
|
|
return fmt.Errorf("model %q profile %q: trtllm runs a replica on one node; "+
|
|
"use vllm or sglang for a %d-node replica", m.Name, p.Name, p.Nodes)
|
|
}
|
|
if strings.HasSuffix(strings.ToLower(strings.TrimSuffix(m.Source, "/")), ".gguf") {
|
|
return fmt.Errorf("model %q profile %q: trtllm loads a Hugging Face checkpoint, not a .gguf; "+
|
|
"serve a GGUF with llamacpp", m.Name, p.Name)
|
|
}
|
|
}
|
|
// The arithmetic has to add up, or the engine is handed a shard spec it
|
|
// cannot satisfy and fails minutes later during model loading.
|
|
if want := p.GPUsPerNode * p.Nodes; want != p.TP*max(p.PP, 1) {
|
|
return fmt.Errorf("model %q profile %q: %d nodes x %d GPUs does not match TP %d x PP %d",
|
|
m.Name, p.Name, p.Nodes, p.GPUsPerNode, p.TP, max(p.PP, 1))
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Resolve maps a client-supplied name or alias onto a model.
|
|
// ResolveAdapter finds an adapter by its name: the model it runs on, the
|
|
// adapter, and its index among the model's adapters — which is the id the
|
|
// engine knows it by.
|
|
func (c *Catalog) ResolveAdapter(name string) (Model, Adapter, int, bool) {
|
|
base, ok := c.adapters[name]
|
|
if !ok {
|
|
return Model{}, Adapter{}, 0, false
|
|
}
|
|
m := c.models[base]
|
|
for i, a := range m.Adapters {
|
|
if a.Name == name {
|
|
return m, a, i, true
|
|
}
|
|
}
|
|
return Model{}, Adapter{}, 0, false
|
|
}
|
|
|
|
func (c *Catalog) Resolve(name string) (Model, bool) {
|
|
if m, ok := c.models[name]; ok {
|
|
return m, true
|
|
}
|
|
if canonical, ok := c.aliases[name]; ok {
|
|
return c.models[canonical], true
|
|
}
|
|
return Model{}, false
|
|
}
|
|
|
|
// Models returns every model, in declaration order.
|
|
func (c *Catalog) Models() []Model {
|
|
out := make([]Model, 0, len(c.order))
|
|
for _, n := range c.order {
|
|
out = append(out, c.models[n])
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Names returns the canonical model names, sorted.
|
|
func (c *Catalog) Names() []string {
|
|
out := slices.Clone(c.order)
|
|
slices.Sort(out)
|
|
return out
|
|
}
|
|
|
|
// AliasesOf returns the alternative names for a model.
|
|
func (c *Catalog) AliasesOf(name string) []string {
|
|
m, ok := c.Resolve(name)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return slices.Clone(m.Aliases)
|
|
}
|
|
|
|
// PlacementProfiles converts a model's profiles into placement's view.
|
|
func (m Model) PlacementProfiles() []placement.Profile {
|
|
out := make([]placement.Profile, 0, len(m.Profiles))
|
|
for _, p := range m.Profiles {
|
|
out = append(out, placement.Profile{
|
|
Name: p.Name,
|
|
Engine: p.Engine,
|
|
EngineVersion: p.EngineVersion,
|
|
GPUClass: p.GPUClass,
|
|
GPUsPerNode: p.GPUsPerNode,
|
|
Nodes: p.Nodes,
|
|
Priority: p.Priority,
|
|
NodeSelector: p.NodeSelector,
|
|
MinMemoryGB: p.MinMemoryGB,
|
|
MinDiskGB: p.MinDiskGB,
|
|
CPUCores: p.CPUCores,
|
|
PowerWatts: p.PowerWatts,
|
|
KeepWithin: facility.Level(strings.ToLower(strings.TrimSpace(p.KeepWithin))),
|
|
// The model's source, so placement can prefer nodes that already
|
|
// hold the weights rather than one that must first move them.
|
|
Source: m.Source,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// EngineProfile converts a catalog profile into the engine adapter's view.
|
|
func (m Model) EngineProfile(name string) (engines.Profile, bool) {
|
|
for _, p := range m.Profiles {
|
|
if p.Name != name {
|
|
continue
|
|
}
|
|
return engines.Profile{
|
|
Name: p.Name,
|
|
Engine: p.Engine,
|
|
GPUClass: p.GPUClass,
|
|
TP: p.TP,
|
|
PP: p.PP,
|
|
GPUsPerNode: p.GPUsPerNode,
|
|
Nodes: p.Nodes,
|
|
Args: slices.Clone(p.Args),
|
|
Env: p.Env,
|
|
ModelPath: m.Source,
|
|
ServedModelName: m.Name,
|
|
Adapters: m.engineAdapters(),
|
|
KVEvents: p.KVEvents,
|
|
}.WithRuntimeLoRA(m.MaxLoadedAdapters), true
|
|
}
|
|
return engines.Profile{}, false
|
|
}
|
|
|
|
// KVIndex reports whether the router should route this model by the KV
|
|
// cache its replicas report: whether any profile publishes events. A replica
|
|
// of another profile is simply one the index has not heard from.
|
|
func (m Model) KVIndex() bool {
|
|
for _, p := range m.Profiles {
|
|
if p.KVEvents {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// ProfileByName returns one of a model's profiles.
|
|
func (m Model) ProfileByName(name string) (Profile, bool) {
|
|
for _, p := range m.Profiles {
|
|
if p.Name == name {
|
|
return p, true
|
|
}
|
|
}
|
|
return Profile{}, false
|
|
}
|
|
|
|
// String renders the catalog for `ctl models`.
|
|
func (c *Catalog) String() string {
|
|
var b strings.Builder
|
|
for _, m := range c.Models() {
|
|
fmt.Fprintf(&b, "%s (min %d, max %d)\n", m.Name, m.Scaling.Min, m.Scaling.Max)
|
|
for _, p := range m.Profiles {
|
|
fmt.Fprintf(&b, " %-20s %-10s %-12s tp=%d nodes=%d priority=%d\n",
|
|
p.Name, p.Engine, p.GPUClass, p.TP, p.Nodes, p.Priority)
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// Simulated reports whether this model produces synthetic text rather than
|
|
// inference output.
|
|
//
|
|
// Derived from the profiles rather than configured, because an operator who
|
|
// added a sim profile and forgot the flag would be shipping placeholder text to
|
|
// users under a real model's name. A model counts as simulated only when every
|
|
// profile is simulated: one real profile means a request can land on a real
|
|
// engine, and a model that sometimes answers for real must not be labelled as
|
|
// though it never does.
|
|
func (m Model) Simulated() bool {
|
|
if len(m.Profiles) == 0 {
|
|
return false
|
|
}
|
|
for _, p := range m.Profiles {
|
|
if p.Engine != "sim" {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// Fingerprint identifies a profile's contents.
|
|
//
|
|
// The name is not enough. A profile can be edited — a memory requirement
|
|
// raised, a node label added, an engine argument changed — and keep its name,
|
|
// so a replica running the old definition is indistinguishable from one
|
|
// running the new. The reconciler does not move a running replica when its
|
|
// profile changes, which is right: churning a replica that is serving costs
|
|
// more than the imperfection. What was missing is any way to see that it had
|
|
// happened, and the answer to "why is this one on the wrong node" was in
|
|
// somebody's memory of what they edited an hour ago.
|
|
func (p Profile) Fingerprint() string {
|
|
b, err := json.Marshal(p)
|
|
if err != nil {
|
|
// A profile that will not marshal cannot be compared, and saying so by
|
|
// never matching is better than saying two differing ones are the same.
|
|
return "unmarshalable"
|
|
}
|
|
sum := sha256.Sum256(b)
|
|
return hex.EncodeToString(sum[:8])
|
|
}
|
|
|
|
// SuggestCores is the cores to give a replica with no GPU on nodes of these
|
|
// CPU counts: half of the smallest, so two replicas share any of them without
|
|
// competing. Zero when no node says, which reserves nothing.
|
|
func SuggestCores(cpuCounts []int) int {
|
|
smallest := 0
|
|
for _, c := range cpuCounts {
|
|
if c > 0 && (smallest == 0 || c < smallest) {
|
|
smallest = c
|
|
}
|
|
}
|
|
if smallest == 0 {
|
|
return 0
|
|
}
|
|
return max(smallest/2, 1)
|
|
}
|
|
|
|
// The capacity classes, lowest first.
|
|
const (
|
|
ClassOpportunistic = "opportunistic"
|
|
ClassStandard = "standard"
|
|
ClassCritical = "critical"
|
|
)
|
|
|
|
func validClass(c string) bool {
|
|
return c == "" || c == ClassOpportunistic || c == ClassStandard || c == ClassCritical
|
|
}
|
|
|
|
// EffectiveClass is the class a model is treated as: what it says, or, when
|
|
// it says nothing, opportunistic for one loaded on demand — it holds capacity
|
|
// only while asked for, and has always been the first to give it up — and
|
|
// standard for everything else.
|
|
func (m Model) EffectiveClass() string {
|
|
if m.Class != "" {
|
|
return m.Class
|
|
}
|
|
if m.IsGuard() {
|
|
// Every judged request waits on it.
|
|
return ClassCritical
|
|
}
|
|
if m.Scaling.OnDemand() {
|
|
return ClassOpportunistic
|
|
}
|
|
return ClassStandard
|
|
}
|
|
|
|
// ClassRank orders classes: a higher rank may take capacity from a lower one.
|
|
func ClassRank(c string) int {
|
|
switch c {
|
|
case ClassCritical:
|
|
return 3
|
|
case ClassStandard, "":
|
|
return 2
|
|
default:
|
|
return 1
|
|
}
|
|
}
|
|
|
|
func validOwner(o string) bool {
|
|
for _, p := range []string{"org:", "team:", "project:"} {
|
|
if strings.HasPrefix(o, p) && len(o) > len(p) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// External is a provider outside the cluster that serves a model.
|
|
type External struct {
|
|
// URL is the provider's base address, without /v1: https://api.openai.com.
|
|
URL string `json:"url" koanf:"url"`
|
|
// Model is what the provider calls it; requests are rewritten to it.
|
|
Model string `json:"model" koanf:"model"`
|
|
// APIKeyFile holds the provider's key, on the control plane. A file
|
|
// rather than the key itself, so the key is in neither the catalog, the
|
|
// store nor the audit log; empty for a provider that asks for none.
|
|
APIKeyFile string `json:"api_key_file,omitempty" koanf:"api_key_file"`
|
|
// Peer names the federated llmgw cluster this model is offered by, for
|
|
// a model that federation added (`llmgw ctl federation add`) rather
|
|
// than one written by hand. Such a model is named PEER/MODEL, comes and
|
|
// goes with the peer's own list, and is sent only while the peer is up.
|
|
Peer string `json:"peer,omitempty" koanf:"peer"`
|
|
}
|
|
|
|
// IsPeer reports whether a model is offered by a federated cluster.
|
|
func (m Model) IsPeer() bool { return m.External != nil && m.External.Peer != "" }
|
|
|
|
func (e *External) validate(model string, profiles int) error {
|
|
if profiles > 0 {
|
|
return fmt.Errorf("model %q: an external model is served elsewhere and has no profiles", model)
|
|
}
|
|
u, err := url.Parse(e.URL)
|
|
if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" {
|
|
return fmt.Errorf("model %q: external url %q is not an http(s) address", model, e.URL)
|
|
}
|
|
if strings.TrimSpace(e.Model) == "" {
|
|
return fmt.Errorf("model %q: external model needs the name the provider knows it by", model)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// IsExternal reports whether a model is served outside the cluster.
|
|
func (m Model) IsExternal() bool { return m.External != nil }
|
|
|
|
// Adapter is a LoRA adapter served on a model's replicas.
|
|
type Adapter struct {
|
|
// Name is what clients ask for, like a model name.
|
|
Name string `json:"name" koanf:"name"`
|
|
// Source is the adapter: a GGUF file for llama.cpp; for vLLM a PEFT
|
|
// adapter directory (a mirror version ending in "/", a path on the node,
|
|
// or a Hugging Face repository id). Fetched by each node as the model's
|
|
// weights are.
|
|
Source string `json:"source" koanf:"source"`
|
|
// Scale is how strongly it is applied; zero takes 1.
|
|
Scale float64 `json:"scale,omitempty" koanf:"scale"`
|
|
}
|
|
|
|
// EffectiveScale is the scale an adapter is applied at.
|
|
func (a Adapter) EffectiveScale() float64 {
|
|
if a.Scale <= 0 {
|
|
return 1
|
|
}
|
|
return a.Scale
|
|
}
|
|
|
|
func validateAdapters(m Model) error {
|
|
if m.MaxLoadedAdapters < 0 {
|
|
return fmt.Errorf("model %q: max_loaded_adapters cannot be negative", m.Name)
|
|
}
|
|
if m.MaxLoadedAdapters > 0 {
|
|
// Loading on demand is vLLM's runtime LoRA API. llama-server can
|
|
// only change the scale of adapters it loaded at start, so a cap
|
|
// there would be a cap on nothing.
|
|
for _, p := range m.Profiles {
|
|
if p.Engine != "vllm" {
|
|
return fmt.Errorf("model %q profile %q: max_loaded_adapters needs vllm profiles; "+
|
|
"llama.cpp loads every adapter at start", m.Name, p.Name)
|
|
}
|
|
}
|
|
}
|
|
if len(m.Adapters) == 0 {
|
|
return nil
|
|
}
|
|
if m.External != nil {
|
|
return fmt.Errorf("model %q: an external model cannot carry adapters", m.Name)
|
|
}
|
|
for _, p := range m.Profiles {
|
|
// llama.cpp loads adapters at start and applies them per request;
|
|
// vLLM loads them at start or on demand. The other engines are not
|
|
// driven either way yet.
|
|
if p.Engine != "llamacpp" && p.Engine != "vllm" && p.Engine != "sim" {
|
|
return fmt.Errorf("model %q profile %q: adapters are served by llamacpp and vllm profiles only", m.Name, p.Name)
|
|
}
|
|
// The variable is what switches vLLM to loading on demand, and the
|
|
// router loads adapters only for a model with a cap: set by hand, it
|
|
// would start replicas holding no adapter that nothing ever loads.
|
|
if _, set := p.Env[engines.RuntimeLoRAEnv]; set && m.MaxLoadedAdapters == 0 {
|
|
return fmt.Errorf("model %q profile %q: set max_loaded_adapters rather than %s",
|
|
m.Name, p.Name, engines.RuntimeLoRAEnv)
|
|
}
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, a := range m.Adapters {
|
|
switch {
|
|
case a.Name == "" || a.Source == "":
|
|
return fmt.Errorf("model %q: an adapter needs a name and a source", m.Name)
|
|
case a.Name == m.Name:
|
|
return fmt.Errorf("model %q: an adapter cannot have the model's own name", m.Name)
|
|
case seen[a.Name]:
|
|
return fmt.Errorf("model %q: adapter %q is declared twice", m.Name, a.Name)
|
|
case a.Scale < 0:
|
|
return fmt.Errorf("model %q: adapter %q has a negative scale", m.Name, a.Name)
|
|
case m.MaxLoadedAdapters > 0 && (strings.ContainsAny(a.Name, `/\`) || a.Name == "." || a.Name == ".."):
|
|
// Loaded on demand, the name is a file's name on the node.
|
|
return fmt.Errorf("model %q: adapter %q cannot contain a slash when adapters load on demand", m.Name, a.Name)
|
|
}
|
|
seen[a.Name] = true
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (m Model) engineAdapters() []engines.Adapter {
|
|
var out []engines.Adapter
|
|
for _, a := range m.Adapters {
|
|
out = append(out, engines.Adapter{Name: a.Name, Path: a.Source})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// repositoryPage reports whether source is a Hugging Face repository's page
|
|
// (https://huggingface.co/org/repo, or one of its tree views) rather than a
|
|
// file under /resolve/, and the repository's id.
|
|
func repositoryPage(source string) (string, bool) {
|
|
for _, host := range []string{"https://huggingface.co/", "http://huggingface.co/", "https://hf.co/"} {
|
|
rest, ok := strings.CutPrefix(source, host)
|
|
if !ok {
|
|
continue
|
|
}
|
|
parts := strings.Split(strings.Trim(rest, "/"), "/")
|
|
if len(parts) < 2 || (len(parts) > 2 && parts[2] == "resolve") {
|
|
return "", false
|
|
}
|
|
return parts[0] + "/" + parts[1], true
|
|
}
|
|
return "", false
|
|
}
|