Ilker ManapandClaude Opus 5.5
314cc5e6a3
Found on a fresh HA install with a judge machine: the leader saw the policy's judge missing while init was still fetching it, served one itself, and its entry replaced init's, so a second judge ran off the judge machine. The keeper now waits ten minutes after it starts, and a serve job it starts gives way to a model added meanwhile (AddNew). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1757 lines
68 KiB
Go
1757 lines
68 KiB
Go
// Package catalog holds the desired state: which models exist, how each one can
|
|
// be run, and how many copies of it there should be (plan §6.1).
|
|
package catalog
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"maps"
|
|
"net/url"
|
|
"path"
|
|
"regexp"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/comfy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
|
)
|
|
|
|
// ErrExists is a model added only if absent that was there already.
|
|
var ErrExists = errors.New("already in the catalog")
|
|
|
|
// Scaling is how many replicas a model should have.
|
|
type Scaling struct {
|
|
Min int `json:"min,omitempty" koanf:"min"`
|
|
Max int `json:"max,omitempty" koanf:"max"`
|
|
// IdleTimeout is how long an on-demand model may go unasked-for before its
|
|
// replicas are stopped. Zero takes the cluster default.
|
|
//
|
|
// Minutes rather than seconds, always. Stopping a replica throws away the
|
|
// KV cache prefix routing spent its effort filling, and a model that flaps
|
|
// in and out loses that entirely — so the saving has to be worth more than
|
|
// the cache, which it only is over a real idle period.
|
|
IdleTimeout time.Duration `json:"idle_timeout,omitempty" koanf:"idle_timeout"`
|
|
// Schedule changes min and max by the time of day, in the control
|
|
// plane's control.timezone: more replicas kept through working hours,
|
|
// fewer at night, an on-demand model kept loaded while people are at
|
|
// work. The first window that holds now wins; outside them all, Min and
|
|
// Max apply.
|
|
Schedule []ScaleWindow `json:"schedule,omitempty" koanf:"schedule"`
|
|
// Prewarm starts a window this long before it opens, so its replicas
|
|
// have loaded by then: loading a model is minutes, and the people who
|
|
// arrive at nine are the reason for the window.
|
|
Prewarm time.Duration `json:"prewarm,omitempty" koanf:"prewarm"`
|
|
}
|
|
|
|
// ScaleWindow is a part of the week with its own min and max.
|
|
type ScaleWindow struct {
|
|
// Days are mon … sun, or weekdays, weekends; empty is every day.
|
|
Days []string `json:"days,omitempty" koanf:"days"`
|
|
// From and To are HH:MM; a window whose To is before its From runs
|
|
// past midnight.
|
|
From string `json:"from" koanf:"from"`
|
|
To string `json:"to" koanf:"to"`
|
|
Min int `json:"min" koanf:"min"`
|
|
// Max, when set, is the window's ceiling for load scaling; zero keeps
|
|
// the model's own.
|
|
Max int `json:"max,omitempty" koanf:"max"`
|
|
}
|
|
|
|
// clock parses HH:MM into minutes past midnight.
|
|
func clock(s string) (int, bool) {
|
|
var h, m int
|
|
if n, err := fmt.Sscanf(s, "%d:%d", &h, &m); err != nil || n != 2 || h < 0 || h > 23 || m < 0 || m > 59 || len(s) != 5 {
|
|
return 0, false
|
|
}
|
|
return h*60 + m, true
|
|
}
|
|
|
|
var weekdays = map[string][]time.Weekday{
|
|
"mon": {time.Monday}, "tue": {time.Tuesday}, "wed": {time.Wednesday}, "thu": {time.Thursday},
|
|
"fri": {time.Friday}, "sat": {time.Saturday}, "sun": {time.Sunday},
|
|
"weekdays": {time.Monday, time.Tuesday, time.Wednesday, time.Thursday, time.Friday},
|
|
"weekends": {time.Saturday, time.Sunday},
|
|
}
|
|
|
|
func (w ScaleWindow) validate() error {
|
|
from, ok1 := clock(w.From)
|
|
to, ok2 := clock(w.To)
|
|
if !ok1 || !ok2 {
|
|
return fmt.Errorf("from and to are HH:MM, like 08:00")
|
|
}
|
|
if from == to {
|
|
return fmt.Errorf("from and to are the same time")
|
|
}
|
|
for _, d := range w.Days {
|
|
if _, ok := weekdays[strings.ToLower(d)]; !ok {
|
|
return fmt.Errorf("day %q is not mon … sun, weekdays or weekends", d)
|
|
}
|
|
}
|
|
if w.Min < 0 || w.Max < 0 || w.Max > 0 && w.Max < w.Min {
|
|
return fmt.Errorf("min %d and max %d: max is 0 or at least min", w.Min, w.Max)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// holds says whether t falls in the window, opened early by lead.
|
|
func (w ScaleWindow) holds(t time.Time, lead time.Duration) bool {
|
|
from, _ := clock(w.From)
|
|
to, _ := clock(w.To)
|
|
span := to - from
|
|
if span <= 0 {
|
|
span += 24 * 60 // past midnight
|
|
}
|
|
midnight := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
|
|
// The window as it starts yesterday, today and tomorrow — tomorrow's
|
|
// can open today, pulled forward by the lead.
|
|
for d := -1; d <= 1; d++ {
|
|
day := midnight.AddDate(0, 0, d)
|
|
if !w.on(day.Weekday()) {
|
|
continue
|
|
}
|
|
start := day.Add(time.Duration(from)*time.Minute - lead)
|
|
end := day.Add(time.Duration(from+span) * time.Minute)
|
|
if !t.Before(start) && t.Before(end) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// on says whether the window starts on a day.
|
|
func (w ScaleWindow) on(wd time.Weekday) bool {
|
|
if len(w.Days) == 0 {
|
|
return true
|
|
}
|
|
for _, d := range w.Days {
|
|
for _, x := range weekdays[strings.ToLower(d)] {
|
|
if x == wd {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Window is the window in force at t, in its own time zone, and false
|
|
// outside them all.
|
|
func (s Scaling) Window(t time.Time) (ScaleWindow, bool) {
|
|
for _, w := range s.Schedule {
|
|
if w.holds(t, s.Prewarm) {
|
|
return w, true
|
|
}
|
|
}
|
|
return ScaleWindow{}, false
|
|
}
|
|
|
|
// OnDemand reports whether this model is loaded when someone asks for it.
|
|
//
|
|
// min 0 with a max above it, which needs no new field and reads correctly: keep
|
|
// none running, allow up to max. min 0 and max 0 is a model that is configured
|
|
// and deliberately not running, which is a different and equally useful thing —
|
|
// the difference between "load it when wanted" and "do not load it".
|
|
func (s Scaling) OnDemand() bool { return s.Min == 0 && s.Max > 0 }
|
|
|
|
// SLO is the latency target a model is expected to meet. `ctl bench` measures
|
|
// capacity against it (plan §6.2.1), and admission holds the lower bands
|
|
// back while the model's recent p99 is past it.
|
|
type SLO struct {
|
|
// TTFTp99 is the time to the first token 99 requests in 100 are within.
|
|
TTFTp99 time.Duration `koanf:"ttft_p99"`
|
|
// TPOTp99 is the time per output token, after the first, 99 streamed
|
|
// requests in 100 are within.
|
|
TPOTp99 time.Duration `koanf:"tpot_p99"`
|
|
}
|
|
|
|
// Profile is how a model runs on one GPU class.
|
|
type Profile struct {
|
|
Name string `json:"name,omitempty" koanf:"name"`
|
|
Engine string `json:"engine,omitempty" koanf:"engine"`
|
|
// EngineVersion restricts the profile to nodes whose installed engine's
|
|
// version starts with it: "0.11" takes vLLM 0.11.0 and 0.11.1, not
|
|
// 0.10.2. Empty takes whatever version the node has.
|
|
EngineVersion string `json:"engine_version,omitempty" koanf:"engine_version"`
|
|
GPUClass string `json:"gpu_class,omitempty" koanf:"gpu_class"`
|
|
// Source is the weights this profile loads, when not the model's: an
|
|
// embedding model served by vLLM from its repository on a card and by
|
|
// llama.cpp from a GGUF on CPUs is one model with two sources. Empty
|
|
// loads the model's. Updates and canaries move the model's source; a
|
|
// profile with its own keeps it.
|
|
Source string `json:"source,omitempty" koanf:"source"`
|
|
|
|
TP int `json:"tp,omitempty" koanf:"tp"`
|
|
PP int `json:"pp,omitempty" koanf:"pp"`
|
|
// GPUsPerNode defaults to TP for a single-node profile.
|
|
GPUsPerNode int `json:"gpus_per_node,omitempty" koanf:"gpus_per_node"`
|
|
// Nodes above 1 makes this a multi-node profile, which requires a fabric.
|
|
Nodes int `json:"nodes,omitempty" koanf:"nodes"`
|
|
|
|
Args []string `json:"args,omitempty" koanf:"args"`
|
|
Env map[string]string `json:"env,omitempty" koanf:"env"`
|
|
|
|
// MinMemoryGB is the host RAM this profile needs to run.
|
|
//
|
|
// The number that matters is the model's working set: the weights, plus the
|
|
// KV cache for the whole context pool, plus the engine's own overhead. For
|
|
// a GPU profile that lives in the cards and this is small; for a CPU
|
|
// profile it is the whole thing, and getting it wrong means an OOM on a
|
|
// node that reported itself healthy.
|
|
//
|
|
// Zero means unconstrained, which is every profile that existed before
|
|
// this. A node that does not report its memory is not excluded either —
|
|
// an older agent must not have its nodes quietly emptied by an upgrade of
|
|
// the control plane.
|
|
MinMemoryGB int `json:"min_memory_gb,omitempty" koanf:"min_memory_gb"`
|
|
|
|
// CPUCores is the cores a replica of a profile with no GPU is given:
|
|
// reserved out of its node's CPUs by placement, as memory is, and what
|
|
// its engine computes on (llama.cpp's --threads).
|
|
//
|
|
// Reserved because an engine takes every core it is told about, and two
|
|
// on one node that each think they have all of them spin against each
|
|
// other: on an 8-vCPU node, two 7B replicas fell from 10 tokens a second
|
|
// to under one each. Zero reserves nothing and leaves the thread count to
|
|
// the node, which is every profile from before this.
|
|
CPUCores int `json:"cpu_cores,omitempty" koanf:"cpu_cores"`
|
|
|
|
// MinDiskGB is the free space a node needs before it can hold these
|
|
// weights.
|
|
//
|
|
// Only checked when the node does not already have them: a node holding
|
|
// the file needs no room for it, and requiring the space twice would keep
|
|
// a model off the one node best placed to run it.
|
|
//
|
|
// Declared rather than derived, because the size of a model that has never
|
|
// been fetched anywhere is not knowable from here — and a wrong guess
|
|
// fails late, halfway through a download, which is the failure this exists
|
|
// to prevent. Zero means unconstrained, and a node that reports no storage
|
|
// is not excluded: unknown is not full.
|
|
MinDiskGB int `json:"min_disk_gb,omitempty" koanf:"min_disk_gb"`
|
|
|
|
// NodeSelector restricts this profile to nodes carrying every one of these
|
|
// labels, with these values.
|
|
//
|
|
// A CPU profile otherwise fits everywhere, because there is no card to be
|
|
// free — which is right for a 0.5B model and wrong for a 14B one. Nothing
|
|
// in the inventory says how much memory a node has, so the operator says
|
|
// which nodes are meant for a model instead of the scheduler inferring it.
|
|
// Agents have reported labels since the beginning; this is what reads them.
|
|
NodeSelector map[string]string `json:"node_selector,omitempty" koanf:"node_selector"`
|
|
|
|
// PowerWatts is what one replica of this profile draws, in watts, split
|
|
// evenly over its nodes, counted against the power cap of every rack,
|
|
// PDU or other domain its nodes are in (facility.budgets in server.yaml).
|
|
// Zero derives it: each card at the power limit it reports, each
|
|
// reserved core at facility.cpu_core_watts. Only matters where a domain
|
|
// has a cap.
|
|
PowerWatts float64 `json:"power_watts,omitempty" koanf:"power_watts"`
|
|
|
|
// KeepWithin keeps a multi-node replica's nodes inside one rack, row,
|
|
// room or site, read from their llmgw.io/rack (and so on) labels: for a
|
|
// fabric whose leaves do not reach across racks, or cables that do not.
|
|
// Nodes without the label are passed over for such a profile. Empty is
|
|
// no constraint.
|
|
KeepWithin string `json:"keep_within,omitempty" koanf:"keep_within"`
|
|
|
|
// Weight is the router's capacity weight. It should come from ctl bench
|
|
// rather than a guess (plan §6.2.1).
|
|
Weight float64 `json:"weight,omitempty" koanf:"weight"`
|
|
// Priority orders profiles during placement; highest that fits wins.
|
|
Priority int `json:"priority,omitempty" koanf:"priority"`
|
|
// StartupTimeout bounds model loading. A 70B model across two nodes takes
|
|
// tens of minutes, so this is per profile rather than global.
|
|
StartupTimeout time.Duration `json:"startup_timeout,omitempty" koanf:"startup_timeout"`
|
|
|
|
// KVEvents has vLLM publish what its prefix cache holds, and the router
|
|
// route this model's requests by it: to the replica holding the most of
|
|
// a prompt's leading blocks (plan §7.5 tier 2). vLLM only.
|
|
//
|
|
// Off by default, because it costs something on every request: the
|
|
// router asks a replica to tokenize the prompt before placing it, a
|
|
// round trip and the chat template run twice. It pays where prompts
|
|
// share long prefixes that the approximate tree guesses wrong — many
|
|
// replicas, conversations that move between them, evictions. The engine
|
|
// is started with sha256_cbor block hashing and a fixed PYTHONHASHSEED,
|
|
// which the router's hashing assumes; arguments overriding either leave
|
|
// the index empty, and the agent says so in its log.
|
|
KVEvents bool `json:"kv_events,omitempty" koanf:"kv_events"`
|
|
|
|
// GPU says what card the profile needs, in place of naming its class:
|
|
// it stands for every class in the cluster that meets it. A catalog
|
|
// entry written once then places sensibly on any site's cards.
|
|
GPU *GPUNeed `json:"gpu,omitempty" koanf:"gpu"`
|
|
// ComfyUI is this profile's own workflow, checkpoint and nodes, over the
|
|
// model's: an fp8 workflow where the card has fp8, fp16 where it has not.
|
|
ComfyUI *ProfileComfyUI `json:"comfyui,omitempty" koanf:"comfyui"`
|
|
}
|
|
|
|
// CPUOnly reports whether this profile needs no GPUs.
|
|
//
|
|
// Naming no GPU class is how a profile says so, and only engines that can
|
|
// actually serve without a card are allowed to leave it out — otherwise an
|
|
// omitted class would quietly become a replica placed on a machine with no
|
|
// hardware to run it.
|
|
func (p Profile) CPUOnly() bool { return p.GPUClass == "" && p.GPU == nil && cpuCapable(p.Engine) }
|
|
|
|
// cpuCapable reports whether an engine can serve a model without a GPU.
|
|
//
|
|
// llama.cpp is the one that can, and it is what makes a cluster with no cards
|
|
// testable end to end.
|
|
func cpuCapable(engine string) bool { return engine == "llamacpp" }
|
|
|
|
// Model is a catalog entry as clients see it.
|
|
type Model struct {
|
|
Name string `json:"name,omitempty" koanf:"name"`
|
|
Aliases []string `json:"aliases,omitempty" koanf:"aliases"`
|
|
// Description is what this model is for, in a line.
|
|
//
|
|
// Written by whoever added it, because nobody else can say it. A registry
|
|
// offers a pipeline tag and a wall of README, and neither answers the
|
|
// question a person picking from a list is asking: which of these nine
|
|
// names should I send this request to.
|
|
//
|
|
// It reaches clients through /v1/models, which is the only view most of
|
|
// them ever see. A gateway serving nine models by name alone makes that
|
|
// choice by guesswork.
|
|
Description string `json:"description,omitempty" koanf:"description"`
|
|
// Source is the weights path or a HuggingFace id.
|
|
Source string `json:"source,omitempty" koanf:"source"`
|
|
Profiles []Profile `json:"profiles,omitempty" koanf:"profiles"`
|
|
Scaling Scaling `json:"scaling,omitempty" koanf:"scaling"`
|
|
// Class is how much this model's capacity is worth against another's,
|
|
// when there is not enough for both: critical, standard or opportunistic.
|
|
// Empty is standard, or opportunistic for a model loaded on demand. See
|
|
// EffectiveClass.
|
|
Class string `json:"class,omitempty" koanf:"class"`
|
|
// Owner is the part of the tenancy tree whose capacity quota this model
|
|
// counts against, written org:ID, team:ID or project:ID. Empty is shared:
|
|
// held by the cluster, counted against nobody.
|
|
Owner string `json:"owner,omitempty" koanf:"owner"`
|
|
// External is a model served by someone else — an OpenAI-compatible API
|
|
// outside the cluster — rather than by replicas placed here. Such a model
|
|
// has no profiles and nothing is placed for it; the router sends its
|
|
// requests to the provider. Most useful named in another model's
|
|
// fallbacks: used only when the cluster has no capacity left. Its
|
|
// prompts leave the cluster, so which keys may name it is worth an
|
|
// allowlist.
|
|
External *External `json:"external,omitempty" koanf:"external"`
|
|
// ComfyUI is how a model whose profiles run engine comfyui makes its
|
|
// pictures: the workflow, and the file it loads.
|
|
ComfyUI *ComfyUI `json:"comfyui,omitempty" koanf:"comfyui"`
|
|
// License is the terms of what the model makes, for a model whose
|
|
// output somebody ships: shown with every result and written into a
|
|
// mesh.
|
|
License *License `json:"license,omitempty" koanf:"license"`
|
|
// Adapters are LoRA adapters served on this model's replicas, each asked
|
|
// for by its own name as though it were a model. Every replica loads
|
|
// every adapter, applying one only to a request that names it — unless
|
|
// MaxLoadedAdapters is set.
|
|
Adapters []Adapter `json:"adapters,omitempty" koanf:"adapters"`
|
|
// MaxLoadedAdapters, above zero, has each replica hold at most this many
|
|
// adapters at once, loaded when a request first asks for one and unloaded
|
|
// least recently used first to make room. The router sends a request for
|
|
// an adapter to a replica that already holds it while that one has room
|
|
// to take it. vLLM profiles only: llama-server loads adapters at start
|
|
// and cannot load one afterwards. Zero loads every adapter on every
|
|
// replica at start.
|
|
MaxLoadedAdapters int `json:"max_loaded_adapters,omitempty" koanf:"max_loaded_adapters"`
|
|
Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"`
|
|
// Canary sends a share of this model's requests to another catalog
|
|
// model — the same model at new weights — to compare the two before
|
|
// moving to it (`llmgw ctl model canary`).
|
|
Canary *Canary `json:"canary,omitempty" koanf:"canary"`
|
|
// Eval is a quality suite new weights must pass before they replace
|
|
// this model's: `llmgw ctl model update` and `model promote` run it
|
|
// against them first, and stop below its score.
|
|
Eval *EvalGate `json:"eval,omitempty" koanf:"eval"`
|
|
// RoutingProfile selects how the router scores endpoints for this model.
|
|
RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"`
|
|
SLO SLO `json:"slo,omitempty" koanf:"slo"`
|
|
|
|
// ContextLength is how many tokens one request may use, prompt and
|
|
// completion together, and MaxOutput bounds the completion alone. Zero
|
|
// means unknown, which is what every model was until now.
|
|
//
|
|
// The gateway knew neither, so an over-long prompt was not refused: it was
|
|
// admitted, given a replica, and failed at the engine, and a client sizing
|
|
// its prompts from /v1/models had nothing to read. Worse, the number is not
|
|
// the one an operator writes in the engine's arguments — llama.cpp divides
|
|
// --ctx-size across its parallel slots, so a pool of 8192 across four slots
|
|
// is 2048 a request, and nothing said so.
|
|
ContextLength int `json:"context_length,omitempty" koanf:"context_length"`
|
|
MaxOutput int `json:"max_output,omitempty" koanf:"max_output"`
|
|
|
|
// Capabilities is what this model can do beyond plain completion.
|
|
Capabilities Capabilities `json:"capabilities,omitempty" koanf:"capabilities"`
|
|
|
|
// QueryPrefix is put before a question an embedding model embeds for a
|
|
// knowledge-base search; documents are embedded without it. Models
|
|
// trained with an instruction on the query side want it, Qwen3-Embedding
|
|
// "Instruct: Given a web search query, retrieve relevant passages that
|
|
// answer the query\nQuery: " for one.
|
|
QueryPrefix string `json:"query_prefix,omitempty" koanf:"query_prefix"`
|
|
|
|
// Role "guard" is a model that judges the gateway's traffic for the
|
|
// guardrails rather than serving clients: hidden from /v1/models and the
|
|
// chat page, refused to any key that names it, critical by default, and
|
|
// always running. Guard says how its answers are read.
|
|
Role string `json:"role,omitempty" koanf:"role"`
|
|
Guard *GuardModel `json:"guard,omitempty" koanf:"guard"`
|
|
|
|
// Origin is the registry repository the weights came from, at which
|
|
// commit, and each file's checksum there. Written when a model is served
|
|
// from the registry; for one that was not, worked out from the mirror's
|
|
// record of where it fetched the file. It is what lets the control plane
|
|
// say that the repository has published different weights since, and
|
|
// move the model to them.
|
|
Origin *Origin `json:"origin,omitempty" koanf:"origin"`
|
|
|
|
// Shared is set on an environment's hidden entry only: the members its
|
|
// replica answers for. Made by the catalog, never written.
|
|
Shared *SharedComfy `json:"shared,omitempty" koanf:"-"`
|
|
}
|
|
|
|
// RoleGuard is a judge model's role.
|
|
const RoleGuard = "guard"
|
|
|
|
// IsGuard reports whether a model judges traffic rather than serving it.
|
|
func (m Model) IsGuard() bool { return m.Role == RoleGuard }
|
|
|
|
// GuardModel is how a judge model's answers are read.
|
|
type GuardModel struct {
|
|
// Format is qwen3guard (Qwen3Guard-Gen: "Safety: Safe", "Unsafe" or
|
|
// "Controversial", and the categories), llamaguard3 (Meta's Llama Guard
|
|
// 3: "safe", or "unsafe" and the categories), shieldgemma (Google's
|
|
// ShieldGemma: Yes or No), or json ({"safe": false, "categories":
|
|
// ["..."]}).
|
|
Format string `json:"format" koanf:"format"`
|
|
// Categories the model is asked about, for shieldgemma — each a policy
|
|
// sentence — and the names reported for llamaguard3's S1, S2, ….
|
|
Categories []string `json:"categories,omitempty" koanf:"categories"`
|
|
}
|
|
|
|
// Origin is where a model's weights were published.
|
|
type Origin struct {
|
|
// Repo is the repository, owner/name.
|
|
Repo string `json:"repo" koanf:"repo"`
|
|
// Revision is the commit the files were fetched at.
|
|
Revision string `json:"revision" koanf:"revision"`
|
|
// Files are the weights fetched from it, with what the registry
|
|
// published as each one's sha256. A new commit that leaves them alone —
|
|
// a README edited — is not an update.
|
|
Files []OriginFile `json:"files,omitempty" koanf:"files"`
|
|
// Previous is what the last update replaced, kept so it can be gone back
|
|
// to while its files are still on the mirror.
|
|
Previous *PreviousOrigin `json:"previous,omitempty" koanf:"previous"`
|
|
}
|
|
|
|
// OriginFile is one weights file of an origin.
|
|
type OriginFile struct {
|
|
Path string `json:"path" koanf:"path"`
|
|
SHA256 string `json:"sha256,omitempty" koanf:"sha256"`
|
|
Bytes int64 `json:"bytes,omitempty" koanf:"bytes"`
|
|
}
|
|
|
|
// PreviousOrigin is a model's source before an update, and where it came from.
|
|
type PreviousOrigin struct {
|
|
Source string `json:"source" koanf:"source"`
|
|
Revision string `json:"revision" koanf:"revision"`
|
|
Files []OriginFile `json:"files,omitempty" koanf:"files"`
|
|
}
|
|
|
|
// ShortRevision is a commit as a person writes it.
|
|
func ShortRevision(rev string) string {
|
|
if len(rev) > 8 {
|
|
return rev[:8]
|
|
}
|
|
return rev
|
|
}
|
|
|
|
// Capabilities describes what a client may ask a model for.
|
|
//
|
|
// Declared by the operator rather than discovered, because there is no way to
|
|
// ask an engine what its model can do. That makes these claims, and a wrong
|
|
// claim is worse than none — which is why `ctl engine-check` verifies tool
|
|
// calling against a running engine rather than trusting this.
|
|
//
|
|
// The reason it exists: the gateway forwards a tools array to any model, and a
|
|
// model that cannot use them answers in prose. Every OpenAI-compatible client
|
|
// then behaves as though the model is broken, and the only way to find out was
|
|
// to try it. That cost an afternoon.
|
|
type Capabilities struct {
|
|
Tools bool `json:"tools,omitempty" koanf:"tools"`
|
|
Vision bool `json:"vision,omitempty" koanf:"vision"`
|
|
Embeddings bool `json:"embeddings,omitempty" koanf:"embeddings"`
|
|
// Images is a model that makes pictures, asked for at
|
|
// /v1/images/generations. Implied by an external model whose API is
|
|
// ComfyUI's.
|
|
Images bool `json:"images,omitempty" koanf:"images"`
|
|
// Meshes is a model that makes 3D models from a picture, asked for as a
|
|
// job at /v1/3d/generations. Implied by a ComfyUI model whose output is
|
|
// meshes.
|
|
Meshes bool `json:"meshes,omitempty" koanf:"meshes"`
|
|
}
|
|
|
|
// The kinds of thing a ComfyUI model makes.
|
|
const (
|
|
OutputPictures = "pictures"
|
|
OutputMeshes = "meshes"
|
|
)
|
|
|
|
// comfyOutput is what a ComfyUI model makes: pictures unless it says.
|
|
func (m Model) comfyOutput() string {
|
|
out := ""
|
|
switch {
|
|
case m.External != nil:
|
|
out = m.External.Output
|
|
case m.ComfyUI != nil:
|
|
out = m.ComfyUI.Output
|
|
}
|
|
if out == "" {
|
|
return OutputPictures
|
|
}
|
|
return out
|
|
}
|
|
|
|
// MakesImages reports whether a model answers /v1/images/generations.
|
|
func (m Model) MakesImages() bool {
|
|
return m.Capabilities.Images || m.ServesComfyUI() && m.comfyOutput() == OutputPictures
|
|
}
|
|
|
|
// MakesMeshes reports whether a model answers /v1/3d/generations.
|
|
func (m Model) MakesMeshes() bool {
|
|
return m.Capabilities.Meshes || m.ServesComfyUI() && m.comfyOutput() == OutputMeshes
|
|
}
|
|
|
|
// License is the terms what a model makes comes under: said beside every
|
|
// result, and written into a mesh, which travels without the API.
|
|
type License struct {
|
|
// ID names the licence for a machine: an SPDX identifier where there is
|
|
// one (MIT, Apache-2.0, CC-BY-NC-4.0), else a lowercase name of the
|
|
// licence's own (stability-community, tencent-hunyuan-community). What a
|
|
// person's or a site's choice of licences matches on. Name is for people.
|
|
ID string `json:"id,omitempty" koanf:"id"`
|
|
// Name is the licence's own: MIT, Apache-2.0.
|
|
Name string `json:"name" koanf:"name"`
|
|
URL string `json:"url,omitempty" koanf:"url"`
|
|
// Notice is what must accompany what was made: attribution, terms on
|
|
// the output, what the weights were trained on.
|
|
Notice string `json:"notice,omitempty" koanf:"notice"`
|
|
// Commercial says whether what is made may be used commercially: nil
|
|
// is not said, which is not the same as yes.
|
|
Commercial *bool `json:"commercial,omitempty" koanf:"commercial"`
|
|
// ExcludedRegions are where the licence grants nothing: ISO 3166 codes,
|
|
// and EU for the union (Hunyuan's: EU, UK, KR).
|
|
ExcludedRegions []string `json:"excluded_regions,omitempty" koanf:"excluded_regions"`
|
|
// RevenueCapUSD is the yearly revenue above which the licence asks for
|
|
// another agreement (Stability's community licence: 1000000). Zero is
|
|
// none.
|
|
RevenueCapUSD int64 `json:"revenue_cap_usd,omitempty" koanf:"revenue_cap_usd"`
|
|
// Attribution is what a product using the output must show: "Powered by
|
|
// Stability AI".
|
|
Attribution string `json:"attribution,omitempty" koanf:"attribution"`
|
|
}
|
|
|
|
var (
|
|
licenseID = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9.+-]{0,63}$`)
|
|
regionID = regexp.MustCompile(`^[A-Z]{2}$`)
|
|
)
|
|
|
|
func (l License) validate(model string) error {
|
|
if strings.TrimSpace(l.Name) == "" {
|
|
return fmt.Errorf("model %q: license.name is required: a licence nobody can name is none", model)
|
|
}
|
|
if l.ID != "" && !licenseID.MatchString(l.ID) {
|
|
return fmt.Errorf("model %q: license.id %q is not an SPDX-style identifier (letters, digits, '.', '+', '-')", model, l.ID)
|
|
}
|
|
for _, r := range l.ExcludedRegions {
|
|
if !regionID.MatchString(r) {
|
|
return fmt.Errorf("model %q: license.excluded_regions %q is not an ISO 3166 code in capitals (EU for the union)", model, r)
|
|
}
|
|
}
|
|
if l.RevenueCapUSD < 0 {
|
|
return fmt.Errorf("model %q: license.revenue_cap_usd is negative", model)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// EngineComfyUI is the engine that runs ComfyUI on a node.
|
|
const EngineComfyUI = "comfyui"
|
|
|
|
// ServesComfyUI reports whether a model is made by ComfyUI, on the
|
|
// cluster's own nodes or on a server outside it.
|
|
func (m Model) ServesComfyUI() bool {
|
|
if m.External != nil {
|
|
return m.External.API == APIComfyUI
|
|
}
|
|
return len(m.Profiles) > 0 && m.Profiles[0].Engine == EngineComfyUI
|
|
}
|
|
|
|
// ComfyUI is a ComfyUI model's workflow and checkpoint.
|
|
type ComfyUI struct {
|
|
// Workflow is a built-in's name (flux-schnell, the default, or
|
|
// flux-dev) or a JSON file on the control plane, in ComfyUI's API
|
|
// format.
|
|
Workflow string `json:"workflow,omitempty" koanf:"workflow"`
|
|
// Checkpoint is the file the workflow's {{checkpoint}} names: the
|
|
// source's own name unless set, which a source that is a set of files
|
|
// must.
|
|
Checkpoint string `json:"checkpoint,omitempty" koanf:"checkpoint"`
|
|
// Output is what the workflow makes: pictures (the default), asked for
|
|
// at /v1/images/generations; meshes, asked for as a job at
|
|
// /v1/3d/generations from a picture; or video, audio or files. Every
|
|
// kind is asked for as a job at /v1/workflows/runs too.
|
|
Output string `json:"output,omitempty" koanf:"output"`
|
|
// Inputs are what the workflow takes from a request at
|
|
// /v1/workflows/runs, each the placeholder of its name.
|
|
Inputs []Input `json:"inputs,omitempty" koanf:"inputs"`
|
|
// Nodes are custom nodes the workflow needs, put in each replica's own
|
|
// custom_nodes at the commit named.
|
|
Nodes []CustomNode `json:"nodes,omitempty" koanf:"nodes"`
|
|
// HuggingFaceRepo is the repository a custom node loads the weights by
|
|
// name from (Stable Fast 3D's loader asks for stabilityai/stable-fast-3d):
|
|
// the replica's weights are shown to the node there, in the Hugging Face
|
|
// cache of the engines' user, so nothing is fetched and no token is
|
|
// needed when it runs.
|
|
HuggingFaceRepo string `json:"huggingface_repo,omitempty" koanf:"huggingface_repo"`
|
|
// Environment names the models that may share one ComfyUI replica: every
|
|
// model of an environment agrees on the custom nodes they both name, so
|
|
// one replica can carry all of them and ComfyUI moves between their
|
|
// weights itself. Empty is an environment of the model alone.
|
|
Environment string `json:"environment,omitempty" koanf:"environment"`
|
|
}
|
|
|
|
// CustomNode is a ComfyUI node from a git repository.
|
|
type CustomNode struct {
|
|
// Repo is the repository, https only.
|
|
Repo string `json:"repo" koanf:"repo"`
|
|
// Commit is the full commit the node is used at: a branch moves, and a
|
|
// node that changed under a model is a model that changed unasked.
|
|
Commit string `json:"commit" koanf:"commit"`
|
|
// Requirements are the PyPI packages the node needs, by name and version
|
|
// range only. They are installed beside the replica, never into
|
|
// ComfyUI's environment: a package named here at the range named, and
|
|
// every other one at the version ComfyUI already runs. One named here
|
|
// is first on ComfyUI's path, so its range must suit ComfyUI too.
|
|
Requirements []string `json:"requirements,omitempty" koanf:"requirements"`
|
|
// Build are directories of the repository that are Python packages to
|
|
// be compiled rather than downloaded: C++ and CUDA extensions no index
|
|
// has (Stable Fast 3D's texture_baker and uv_unwrapper). Each is built
|
|
// as a wheel with ComfyUI's Python for the card the replica runs on,
|
|
// kept on the node for the next replica on the same kind of card, and
|
|
// installed beside the replica like its requirements.
|
|
Build []string `json:"build,omitempty" koanf:"build"`
|
|
}
|
|
|
|
// NodesEnv carries a ComfyUI model's custom nodes to the agent, which hands
|
|
// them to the gate.
|
|
const NodesEnv = "LLMGW_COMFYUI_NODES"
|
|
|
|
// HFRepoEnv carries the repository a node loads the weights by name from.
|
|
const HFRepoEnv = "LLMGW_COMFYUI_HF_REPO"
|
|
|
|
// SharedModelsEnv carries a shared replica's members — name, weights source,
|
|
// repository — to the agent, which fetches each and hands the gate where
|
|
// they landed.
|
|
const SharedModelsEnv = "LLMGW_COMFYUI_MODELS"
|
|
|
|
var hfRepo = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
|
|
|
var (
|
|
fullCommit = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
|
packageSpec = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*(\[[A-Za-z0-9,._-]+\])?\s*([<>=!~]=?\s*[A-Za-z0-9.*+!_-]+\s*,?\s*)*$`)
|
|
)
|
|
|
|
func (n CustomNode) validate(model string) error {
|
|
u, err := url.Parse(n.Repo)
|
|
if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil {
|
|
return fmt.Errorf("model %q: custom node %q is not an https repository", model, n.Repo)
|
|
}
|
|
if !fullCommit.MatchString(n.Commit) {
|
|
return fmt.Errorf("model %q: custom node %s needs its full commit (40 hex), not %q: a branch moves",
|
|
model, n.Repo, n.Commit)
|
|
}
|
|
for _, b := range n.Build {
|
|
clean := path.Clean(b)
|
|
if b == "" || path.IsAbs(b) || clean == "." || clean == ".." || strings.HasPrefix(clean, "../") || strings.ContainsAny(b, "\\ ") {
|
|
return fmt.Errorf("model %q: custom node %s build %q is not a directory inside the repository", model, n.Repo, b)
|
|
}
|
|
}
|
|
for _, r := range n.Requirements {
|
|
// A package and a version range: an option or a URL would install
|
|
// from somewhere other than PyPI.
|
|
if !packageSpec.MatchString(strings.TrimSpace(r)) {
|
|
return fmt.Errorf("model %q: custom node %s requirement %q is not a package with a version range", model, n.Repo, r)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ComfyWorkflow is what a ComfyUI model's replicas are sent: the workflow
|
|
// as the catalog names it, and the checkpoint it loads.
|
|
func (m Model) ComfyWorkflow() (workflow, checkpoint string) {
|
|
if m.External != nil {
|
|
return m.External.Workflow, m.External.Model
|
|
}
|
|
if m.ComfyUI != nil {
|
|
workflow, checkpoint = m.ComfyUI.Workflow, m.ComfyUI.Checkpoint
|
|
}
|
|
if checkpoint == "" && !strings.HasSuffix(m.Source, "/") {
|
|
checkpoint = path.Base(strings.TrimPrefix(m.Source, "hf:"))
|
|
}
|
|
return workflow, checkpoint
|
|
}
|
|
|
|
// checkWorkflow refuses a workflow that is neither a built-in nor a file.
|
|
func checkWorkflow(model, w string) error {
|
|
// A file is read where the router runs; a name is checked here.
|
|
if w != "" && !strings.ContainsAny(w, "/.") && !slices.Contains(comfy.Builtins(), w) {
|
|
return fmt.Errorf("model %q: no built-in workflow %q (there are %s; or name a JSON file)",
|
|
model, w, strings.Join(comfy.Builtins(), ", "))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (m Model) validateComfyUI() error {
|
|
comfyProfiles := 0
|
|
for _, p := range m.Profiles {
|
|
if p.Engine == EngineComfyUI {
|
|
comfyProfiles++
|
|
}
|
|
}
|
|
if comfyProfiles == 0 {
|
|
if m.ComfyUI != nil {
|
|
return fmt.Errorf("model %q: comfyui: is for a model whose profiles run engine comfyui", m.Name)
|
|
}
|
|
return nil
|
|
}
|
|
if comfyProfiles != len(m.Profiles) {
|
|
return fmt.Errorf("model %q: every profile runs engine comfyui or none does: "+
|
|
"a request for pictures cannot be answered by an engine that writes text", m.Name)
|
|
}
|
|
if len(m.Adapters) > 0 {
|
|
return fmt.Errorf("model %q: ComfyUI serves no adapters", m.Name)
|
|
}
|
|
var w string
|
|
if m.ComfyUI != nil {
|
|
w = m.ComfyUI.Workflow
|
|
}
|
|
if err := checkWorkflow(m.Name, w); err != nil {
|
|
return err
|
|
}
|
|
if m.ComfyUI != nil {
|
|
for _, n := range m.ComfyUI.Nodes {
|
|
if err := n.validate(m.Name); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
if err := validOutput(m.Name, m.comfyOutput()); err != nil {
|
|
return err
|
|
}
|
|
if m.ComfyUI != nil && m.ComfyUI.HuggingFaceRepo != "" && !hfRepo.MatchString(m.ComfyUI.HuggingFaceRepo) {
|
|
return fmt.Errorf("model %q: comfyui.huggingface_repo %q is not owner/name", m.Name, m.ComfyUI.HuggingFaceRepo)
|
|
}
|
|
if out := m.comfyOutput(); out != OutputPictures && w == "" {
|
|
return fmt.Errorf("model %q: a model that makes %s names its workflow: no built-in makes them", m.Name, out)
|
|
}
|
|
if err := validateInputs(m.Name, m.ComfyInputs()); err != nil {
|
|
return err
|
|
}
|
|
if _, ckpt := m.ComfyWorkflow(); ckpt == "" {
|
|
return fmt.Errorf("model %q: its source is a set of files, so comfyui.checkpoint must name the one the workflow loads",
|
|
m.Name)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Catalog is the set of models the cluster should serve.
|
|
type Catalog struct {
|
|
models map[string]Model
|
|
aliases map[string]string
|
|
// adapters maps an adapter's name to the model it runs on.
|
|
adapters map[string]string
|
|
order []string
|
|
}
|
|
|
|
// New builds a catalog, rejecting definitions that would fail confusingly later.
|
|
func New(models []Model) (*Catalog, error) {
|
|
c := &Catalog{
|
|
models: make(map[string]Model, len(models)),
|
|
aliases: map[string]string{},
|
|
adapters: map[string]string{},
|
|
}
|
|
|
|
for i := range models {
|
|
if models[i].IsEnvironment() {
|
|
continue // the catalog's own, made again below
|
|
}
|
|
m := models[i].normalize()
|
|
if err := m.validate(); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, dup := c.models[m.Name]; dup {
|
|
return nil, fmt.Errorf("model %q is declared twice", m.Name)
|
|
}
|
|
c.models[m.Name] = m
|
|
c.order = append(c.order, m.Name)
|
|
}
|
|
|
|
// Aliases share a namespace with names. A collision would silently shadow a
|
|
// model at request time, so it has to fail at load.
|
|
for _, name := range c.order {
|
|
for _, a := range c.models[name].Aliases {
|
|
if _, clash := c.models[a]; clash {
|
|
return nil, fmt.Errorf("model %q: alias %q collides with another model name", name, a)
|
|
}
|
|
if prev, clash := c.aliases[a]; clash {
|
|
return nil, fmt.Errorf("alias %q is claimed by both %q and %q", a, prev, name)
|
|
}
|
|
c.aliases[a] = name
|
|
}
|
|
}
|
|
|
|
// Adapters are named like models, so they share the namespace too.
|
|
for _, name := range c.order {
|
|
for _, a := range c.models[name].Adapters {
|
|
_, isModel := c.models[a.Name]
|
|
_, isAlias := c.aliases[a.Name]
|
|
if _, isAdapter := c.adapters[a.Name]; isModel || isAlias || isAdapter {
|
|
return nil, fmt.Errorf("model %q: adapter %q collides with a model, an alias or another adapter", name, a.Name)
|
|
}
|
|
c.adapters[a.Name] = name
|
|
}
|
|
}
|
|
|
|
// Fallbacks are resolved now rather than at request time: a typo in a
|
|
// fallback should stop a deployment, not surface during an outage when the
|
|
// fallback is finally needed.
|
|
for _, name := range c.order {
|
|
for _, f := range c.models[name].Fallbacks {
|
|
if _, ok := c.Resolve(f); !ok {
|
|
return nil, fmt.Errorf("model %q: fallback %q is not in the catalog", name, f)
|
|
}
|
|
}
|
|
}
|
|
// A canary is another model in the catalog, a candidate for this one's
|
|
// place: not itself, not a judge, not a canary of its own, and the
|
|
// candidate of one model only.
|
|
canaryOf := map[string]string{}
|
|
for _, name := range c.order {
|
|
cn := c.models[name].Canary
|
|
if cn == nil {
|
|
continue
|
|
}
|
|
target, ok := c.Resolve(cn.Model)
|
|
switch {
|
|
case !ok:
|
|
return nil, fmt.Errorf("model %q: canary %q is not in the catalog", name, cn.Model)
|
|
case target.Name == name:
|
|
return nil, fmt.Errorf("model %q: a model cannot be its own canary", name)
|
|
case target.IsGuard():
|
|
return nil, fmt.Errorf("model %q: canary %q is a guard model", name, cn.Model)
|
|
case target.Canary != nil:
|
|
return nil, fmt.Errorf("model %q: canary %q has a canary of its own", name, cn.Model)
|
|
case canaryOf[target.Name] != "":
|
|
return nil, fmt.Errorf("model %q: %q is already the canary of %q", name, cn.Model, canaryOf[target.Name])
|
|
case cn.Percent < 0 || cn.Percent > 100:
|
|
return nil, fmt.Errorf("model %q: canary percent %d is not 0 to 100", name, cn.Percent)
|
|
}
|
|
canaryOf[target.Name] = name
|
|
}
|
|
envs, err := c.environments()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, e := range envs {
|
|
if len(e.Models) < 2 {
|
|
continue
|
|
}
|
|
var members []Model
|
|
for _, n := range e.Models {
|
|
members = append(members, c.models[n])
|
|
}
|
|
em := envModel(e, members)
|
|
if _, clash := c.models[em.Name]; clash {
|
|
return nil, fmt.Errorf("model %q: the name is the hidden entry of environment %q", em.Name, e.Name)
|
|
}
|
|
c.models[em.Name] = em
|
|
c.order = append(c.order, em.Name)
|
|
}
|
|
return c, nil
|
|
}
|
|
|
|
// Canary is a share of a model's requests sent to a candidate, and when to
|
|
// give up on it.
|
|
type Canary struct {
|
|
// Model is the candidate: a catalog model, usually this one at newer
|
|
// weights, which clients cannot name themselves.
|
|
Model string `json:"model" koanf:"model"`
|
|
// Percent of requests go to it, 0 to 100.
|
|
Percent int `json:"percent" koanf:"percent"`
|
|
// The candidate is taken out — rolled back — when, over at least
|
|
// MinRequests of its own, its error rate is MaxErrorIncrease above
|
|
// this model's (0.05 is five points), or its mean time to the first
|
|
// token is MaxTTFTRatio times this model's. Zero is 50, 0.05 and 1.5.
|
|
MinRequests int `json:"min_requests,omitempty" koanf:"min_requests"`
|
|
MaxErrorIncrease float64 `json:"max_error_increase,omitempty" koanf:"max_error_increase"`
|
|
MaxTTFTRatio float64 `json:"max_ttft_ratio,omitempty" koanf:"max_ttft_ratio"`
|
|
}
|
|
|
|
// EvalGate is a quality suite a model's new weights must pass.
|
|
type EvalGate struct {
|
|
// Suite is a suite's name in control.eval_dir, or one the binary
|
|
// carries (arithmetic, json-format).
|
|
Suite string `json:"suite" koanf:"suite"`
|
|
// MinScore is the score, 0 to 1, to pass at; zero is the suite's own
|
|
// threshold.
|
|
MinScore float64 `json:"min_score,omitempty" koanf:"min_score"`
|
|
// Judge grades the suite's rubric cases, instead of the judge it names.
|
|
Judge string `json:"judge,omitempty" koanf:"judge"`
|
|
}
|
|
|
|
// Limits are the rollback thresholds with their defaults filled in.
|
|
func (c Canary) Limits() (minRequests int, maxErrorIncrease, maxTTFTRatio float64) {
|
|
minRequests, maxErrorIncrease, maxTTFTRatio = c.MinRequests, c.MaxErrorIncrease, c.MaxTTFTRatio
|
|
if minRequests <= 0 {
|
|
minRequests = 50
|
|
}
|
|
if maxErrorIncrease <= 0 {
|
|
maxErrorIncrease = 0.05
|
|
}
|
|
if maxTTFTRatio <= 0 {
|
|
maxTTFTRatio = 1.5
|
|
}
|
|
return
|
|
}
|
|
|
|
// CanaryOf is the model whose canary name is, or "".
|
|
func (c *Catalog) CanaryOf(name string) string {
|
|
for _, n := range c.order {
|
|
if cn := c.models[n].Canary; cn != nil {
|
|
if t, ok := c.Resolve(cn.Model); ok && t.Name == name {
|
|
return n
|
|
}
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func (m Model) normalize() Model {
|
|
if m.Scaling.Min < 0 {
|
|
m.Scaling.Min = 0
|
|
}
|
|
// Max 0 used to be an omission to fill in, because a model with no maximum
|
|
// could not run. It is now also a statement: with min 0 it means configured
|
|
// and deliberately not running, which is a different and useful thing from
|
|
// "load it when wanted". Filling it in would silently turn every switched-
|
|
// off model into an on-demand one — including the ones already written that
|
|
// way in a running cluster's configuration.
|
|
if m.Scaling.Max == 0 && m.Scaling.Min > 0 {
|
|
m.Scaling.Max = m.Scaling.Min
|
|
}
|
|
for i := range m.Profiles {
|
|
p := &m.Profiles[i]
|
|
if p.Engine == "" {
|
|
p.Engine = "vllm"
|
|
}
|
|
if p.TP <= 0 {
|
|
p.TP = 1
|
|
}
|
|
if p.Nodes <= 0 {
|
|
p.Nodes = 1
|
|
}
|
|
// tp on a llama.cpp GPU profile, written before gpus_per_node meant
|
|
// anything for it, is read as the width it was meant as: N cards of
|
|
// one node. Refusing it would have dropped a catalog's model on the
|
|
// upgrade that made the width work. Only both given, disagreeing, is
|
|
// refused below, as ambiguous.
|
|
if p.Engine == "llamacpp" && p.GPUClass != "" && (p.TP > 1 || p.PP > 1) && p.Nodes == 1 {
|
|
if n := p.TP * max(p.PP, 1); p.GPUsPerNode == 0 || p.GPUsPerNode == n {
|
|
p.GPUsPerNode, p.TP, p.PP = n, 1, 0
|
|
}
|
|
}
|
|
switch {
|
|
case p.CPUOnly():
|
|
// Nothing to divide up: this profile asks for no cards at all.
|
|
p.GPUsPerNode = 0
|
|
case p.GPUsPerNode <= 0:
|
|
// The common case: a single-node profile uses TP GPUs on one node.
|
|
p.GPUsPerNode = p.TP / p.Nodes
|
|
if p.GPUsPerNode <= 0 {
|
|
p.GPUsPerNode = 1
|
|
}
|
|
}
|
|
if p.StartupTimeout <= 0 {
|
|
p.StartupTimeout = 20 * time.Minute
|
|
}
|
|
if p.Weight <= 0 {
|
|
p.Weight = 1
|
|
}
|
|
}
|
|
// A Hugging Face repository page for a model vLLM or SGLang runs is the
|
|
// repository it names: the engine downloads a repository id itself,
|
|
// where the page was fetched as one file of HTML and failed at start.
|
|
if repo, ok := repositoryPage(m.Source); ok && len(m.Profiles) > 0 &&
|
|
!slices.ContainsFunc(m.Profiles, func(p Profile) bool { return p.Engine == "llamacpp" && p.Source == "" }) {
|
|
m.Source = repo
|
|
}
|
|
for i := range m.Profiles {
|
|
p := &m.Profiles[i]
|
|
if repo, ok := repositoryPage(p.Source); ok && p.Engine != "llamacpp" {
|
|
p.Source = repo
|
|
}
|
|
}
|
|
return m
|
|
}
|
|
|
|
func (m Model) validate() error {
|
|
if m.Name == "" {
|
|
return fmt.Errorf("a model has no name")
|
|
}
|
|
if m.Source == "" {
|
|
return fmt.Errorf("model %q: source is required", m.Name)
|
|
}
|
|
if m.License != nil {
|
|
if err := m.License.validate(m.Name); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// A page, not weights: a node fetches the HTML as one file and the
|
|
// engine fails at start, on a path that looks right. For vLLM it was
|
|
// read as the repository id when the model was made; only a source
|
|
// llama.cpp loads is left here, which needs a file.
|
|
if repo, ok := repositoryPage(m.Source); ok && (len(m.Profiles) == 0 ||
|
|
slices.ContainsFunc(m.Profiles, func(p Profile) bool { return p.Engine == "llamacpp" && p.Source == "" })) {
|
|
return fmt.Errorf("model %q: source %s is a Hugging Face repository page, not a file; "+
|
|
"llama.cpp needs a .gguf: mirror one and use its source (repository %s)",
|
|
m.Name, m.Source, repo)
|
|
}
|
|
for _, p := range m.Profiles {
|
|
if repo, ok := repositoryPage(p.Source); ok {
|
|
return fmt.Errorf("model %q profile %q: source %s is a Hugging Face repository page, not a file; "+
|
|
"llama.cpp needs a .gguf: mirror one and use its source (repository %s)",
|
|
m.Name, p.Name, p.Source, repo)
|
|
}
|
|
if p.Source != "" && m.External != nil {
|
|
return fmt.Errorf("model %q profile %q: an external model has no profiles to give a source", m.Name, p.Name)
|
|
}
|
|
}
|
|
if err := validateAdapters(m); err != nil {
|
|
return err
|
|
}
|
|
if m.External != nil {
|
|
if err := m.External.validate(m.Name, len(m.Profiles)); err != nil {
|
|
return err
|
|
}
|
|
} else if len(m.Profiles) == 0 {
|
|
return fmt.Errorf("model %q: at least one profile is required", m.Name)
|
|
}
|
|
if err := m.validateComfyUI(); err != nil {
|
|
return err
|
|
}
|
|
if m.Owner != "" && !validOwner(m.Owner) {
|
|
return fmt.Errorf("model %q: owner %q is not org:ID, team:ID or project:ID", m.Name, m.Owner)
|
|
}
|
|
if !validClass(m.Class) {
|
|
return fmt.Errorf("model %q: class %q is not one of critical, standard, opportunistic", m.Name, m.Class)
|
|
}
|
|
if g := m.Eval; g != nil {
|
|
if strings.TrimSpace(g.Suite) == "" {
|
|
return fmt.Errorf("model %q: eval names no suite", m.Name)
|
|
}
|
|
if g.MinScore < 0 || g.MinScore > 1 {
|
|
return fmt.Errorf("model %q: eval min_score %v is not 0 to 1", m.Name, g.MinScore)
|
|
}
|
|
}
|
|
switch m.Role {
|
|
case "":
|
|
if m.Guard != nil {
|
|
return fmt.Errorf("model %q: guard: is for a model with role: guard", m.Name)
|
|
}
|
|
case RoleGuard:
|
|
if m.Guard == nil {
|
|
return fmt.Errorf("model %q: a guard model needs guard.format: %s", m.Name, strings.Join(guard.Formats, ", "))
|
|
}
|
|
if !guard.KnownFormat(m.Guard.Format) {
|
|
return fmt.Errorf("model %q: guard.format %q is not one of %s", m.Name, m.Guard.Format, strings.Join(guard.Formats, ", "))
|
|
}
|
|
if m.Scaling.OnDemand() {
|
|
// A judge loaded on demand makes the first request of every
|
|
// idle period wait for it, or be judged by nothing.
|
|
return fmt.Errorf("model %q: a guard model must always run: scaling.min 1 or more", m.Name)
|
|
}
|
|
if m.IsExternal() {
|
|
return fmt.Errorf("model %q: a guard model judges in the cluster; it cannot be external", m.Name)
|
|
}
|
|
default:
|
|
return fmt.Errorf("model %q: role %q is not guard", m.Name, m.Role)
|
|
}
|
|
for i, w := range m.Scaling.Schedule {
|
|
if err := w.validate(); err != nil {
|
|
return fmt.Errorf("model %q: scaling.schedule[%d]: %w", m.Name, i, err)
|
|
}
|
|
}
|
|
if m.Scaling.Prewarm < 0 || m.Scaling.Prewarm > 12*time.Hour {
|
|
return fmt.Errorf("model %q: scaling.prewarm is between 0 and 12h", m.Name)
|
|
}
|
|
if m.Scaling.Max < m.Scaling.Min {
|
|
return fmt.Errorf("model %q: scaling max %d is below min %d", m.Name, m.Scaling.Max, m.Scaling.Min)
|
|
}
|
|
|
|
seen := map[string]struct{}{}
|
|
for _, p := range m.Profiles {
|
|
if p.Name == "" {
|
|
return fmt.Errorf("model %q: a profile has no name", m.Name)
|
|
}
|
|
if _, dup := seen[p.Name]; dup {
|
|
return fmt.Errorf("model %q: profile %q is declared twice", m.Name, p.Name)
|
|
}
|
|
seen[p.Name] = struct{}{}
|
|
|
|
for k := range p.Env {
|
|
if bad, why := engines.UnsafeEnv(k); bad {
|
|
return fmt.Errorf("model %q profile %q: env %s is not allowed: %s",
|
|
m.Name, p.Name, k, why)
|
|
}
|
|
}
|
|
if err := engines.UnsafeArgs(p.Engine, p.Args); err != nil {
|
|
return fmt.Errorf("model %q profile %q: %w", m.Name, p.Name, err)
|
|
}
|
|
if p.KVEvents && p.Engine != "vllm" {
|
|
return fmt.Errorf("model %q profile %q: kv_events is vLLM's; engine %q publishes no cache events",
|
|
m.Name, p.Name, p.Engine)
|
|
}
|
|
|
|
if p.PowerWatts < 0 {
|
|
return fmt.Errorf("model %q profile %q: power_watts %v is negative", m.Name, p.Name, p.PowerWatts)
|
|
}
|
|
if p.KeepWithin != "" {
|
|
if _, err := facility.ParseLevel(p.KeepWithin); err != nil {
|
|
return fmt.Errorf("model %q profile %q: keep_within: %w", m.Name, p.Name, err)
|
|
}
|
|
}
|
|
if err := p.validateHardware(m); err != nil {
|
|
return err
|
|
}
|
|
if p.GPUClass == "" && p.GPU == nil && !cpuCapable(p.Engine) {
|
|
return fmt.Errorf("model %q profile %q: gpu_class is required for engine %q, or gpu: the card it needs",
|
|
m.Name, p.Name, p.Engine)
|
|
}
|
|
if p.GPU != nil {
|
|
// Its class is known only when it is placed; what can be checked
|
|
// of a class waits for then.
|
|
continue
|
|
}
|
|
if p.CPUOnly() {
|
|
if p.Nodes > 1 {
|
|
return fmt.Errorf("model %q profile %q: a CPU profile cannot span %d nodes",
|
|
m.Name, p.Name, p.Nodes)
|
|
}
|
|
continue
|
|
}
|
|
// A MIG instance is a device of its own to CUDA, but a process sees
|
|
// only one, and instances have no NVLink or peer-to-peer between
|
|
// them: a group of them is neither tensor- nor layer-parallel, it is
|
|
// a replica that fails at start. One instance, one node, refused here
|
|
// rather than placed and found out.
|
|
if discovery.IsMIGClass(p.GPUClass) && (p.GPUsPerNode > 1 || p.Nodes > 1 || p.TP > 1 || p.PP > 1) {
|
|
return fmt.Errorf("model %q profile %q: %s is a MIG instance, and a replica runs on exactly one: "+
|
|
"a process can open only one MIG device, and instances have no NVLink between them. "+
|
|
"Give it tp 1 and one GPU on one node, or use a whole-card class or a larger MIG profile",
|
|
m.Name, p.Name, p.GPUClass)
|
|
}
|
|
// llama.cpp on GPUs splits a model across one node's cards by layer,
|
|
// and how many is gpus_per_node. Not tp: llama.cpp has no tensor
|
|
// parallelism, and a tp it was given was a width nothing placed — the
|
|
// replica stayed unplaced, or ran on one card while four were meant.
|
|
if p.Engine == "llamacpp" {
|
|
switch {
|
|
case p.Nodes > 1:
|
|
return fmt.Errorf("model %q profile %q: llama.cpp does not span nodes; use vllm or sglang for a %d-node replica",
|
|
m.Name, p.Name, p.Nodes)
|
|
case p.TP > 1 || p.PP > 1:
|
|
return fmt.Errorf("model %q profile %q: llama.cpp splits a model across a node's GPUs by layer, and "+
|
|
"tp %d does not agree with gpus_per_node %d: give gpus_per_node alone", m.Name, p.Name,
|
|
max(p.TP, 1)*max(p.PP, 1), p.GPUsPerNode)
|
|
}
|
|
continue
|
|
}
|
|
// TensorRT-LLM spans machines only under MPI started from one of them,
|
|
// which the agents, each starting its own node, do not do; and it
|
|
// loads a Hugging Face checkpoint, not llama.cpp's format.
|
|
if p.Engine == "trtllm" {
|
|
if p.Nodes > 1 {
|
|
return fmt.Errorf("model %q profile %q: trtllm runs a replica on one node; "+
|
|
"use vllm or sglang for a %d-node replica", m.Name, p.Name, p.Nodes)
|
|
}
|
|
if strings.HasSuffix(strings.ToLower(strings.TrimSuffix(m.SourceOf(p), "/")), ".gguf") {
|
|
return fmt.Errorf("model %q profile %q: trtllm loads a Hugging Face checkpoint, not a .gguf; "+
|
|
"serve a GGUF with llamacpp", m.Name, p.Name)
|
|
}
|
|
}
|
|
// The arithmetic has to add up, or the engine is handed a shard spec it
|
|
// cannot satisfy and fails minutes later during model loading.
|
|
if want := p.GPUsPerNode * p.Nodes; want != p.TP*max(p.PP, 1) {
|
|
return fmt.Errorf("model %q profile %q: %d nodes x %d GPUs does not match TP %d x PP %d",
|
|
m.Name, p.Name, p.Nodes, p.GPUsPerNode, p.TP, max(p.PP, 1))
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Resolve maps a client-supplied name or alias onto a model.
|
|
// ResolveAdapter finds an adapter by its name: the model it runs on, the
|
|
// adapter, and its index among the model's adapters — which is the id the
|
|
// engine knows it by.
|
|
func (c *Catalog) ResolveAdapter(name string) (Model, Adapter, int, bool) {
|
|
base, ok := c.adapters[name]
|
|
if !ok {
|
|
return Model{}, Adapter{}, 0, false
|
|
}
|
|
m := c.models[base]
|
|
for i, a := range m.Adapters {
|
|
if a.Name == name {
|
|
return m, a, i, true
|
|
}
|
|
}
|
|
return Model{}, Adapter{}, 0, false
|
|
}
|
|
|
|
func (c *Catalog) Resolve(name string) (Model, bool) {
|
|
if m, ok := c.models[name]; ok {
|
|
return m, true
|
|
}
|
|
if canonical, ok := c.aliases[name]; ok {
|
|
return c.models[canonical], true
|
|
}
|
|
return Model{}, false
|
|
}
|
|
|
|
// Models returns every model, in declaration order.
|
|
func (c *Catalog) Models() []Model {
|
|
out := make([]Model, 0, len(c.order))
|
|
for _, n := range c.order {
|
|
out = append(out, c.models[n])
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Names returns the canonical model names, sorted.
|
|
func (c *Catalog) Names() []string {
|
|
out := slices.Clone(c.order)
|
|
slices.Sort(out)
|
|
return out
|
|
}
|
|
|
|
// AliasesOf returns the alternative names for a model.
|
|
func (c *Catalog) AliasesOf(name string) []string {
|
|
m, ok := c.Resolve(name)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return slices.Clone(m.Aliases)
|
|
}
|
|
|
|
// PlacementProfiles converts a model's profiles into placement's view. A
|
|
// profile that names its card by need is left out: with no class it would
|
|
// read to placement as one that needs no GPU. PlacementProfilesFor gives
|
|
// it the classes it stands for.
|
|
func (m Model) PlacementProfiles() []placement.Profile {
|
|
out := make([]placement.Profile, 0, len(m.Profiles))
|
|
for _, p := range m.Profiles {
|
|
if p.GPU != nil {
|
|
continue
|
|
}
|
|
out = append(out, placement.Profile{
|
|
Name: p.Name,
|
|
Engine: p.Engine,
|
|
EngineVersion: p.EngineVersion,
|
|
GPUClass: p.GPUClass,
|
|
GPUsPerNode: p.GPUsPerNode,
|
|
Nodes: p.Nodes,
|
|
Priority: p.Priority,
|
|
NodeSelector: p.NodeSelector,
|
|
MinMemoryGB: p.MinMemoryGB,
|
|
MinDiskGB: p.MinDiskGB,
|
|
CPUCores: p.CPUCores,
|
|
PowerWatts: p.PowerWatts,
|
|
KeepWithin: facility.Level(strings.ToLower(strings.TrimSpace(p.KeepWithin))),
|
|
// The profile's source, so placement can prefer nodes that already
|
|
// hold the weights rather than one that must first move them.
|
|
Source: m.SourceOf(p),
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// SourceOf is the weights a profile loads: its own, or the model's.
|
|
func (m Model) SourceOf(p Profile) string {
|
|
if p.Source != "" {
|
|
return p.Source
|
|
}
|
|
return m.Source
|
|
}
|
|
|
|
// WeightSources are every source a model's replicas may load: the model's
|
|
// and each profile's of its own, once each. What a node must not have
|
|
// collected from under it.
|
|
func (m Model) WeightSources() []string {
|
|
out := []string{}
|
|
if m.Source != "" {
|
|
out = append(out, m.Source)
|
|
}
|
|
for _, p := range m.Profiles {
|
|
if p.Source != "" && !slices.Contains(out, p.Source) {
|
|
out = append(out, p.Source)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// EngineProfile converts a catalog profile into the engine adapter's view.
|
|
func (m Model) EngineProfile(name string) (engines.Profile, bool) {
|
|
if p, ok := m.ProfileByName(name); ok {
|
|
env := p.Env
|
|
if nodes := m.ComfyNodesFor(p); p.Engine == EngineComfyUI && len(nodes) > 0 {
|
|
nodes, _ := json.Marshal(nodes)
|
|
env = maps.Clone(env)
|
|
if env == nil {
|
|
env = map[string]string{}
|
|
}
|
|
env[NodesEnv] = string(nodes)
|
|
}
|
|
if p.Engine == EngineComfyUI && m.Shared != nil {
|
|
// Every member's weights, for the node to fetch and the gate to
|
|
// link: one replica answers for all of them.
|
|
type member struct {
|
|
Name string `json:"name"`
|
|
Source string `json:"source"`
|
|
HFRepo string `json:"hf_repo,omitempty"`
|
|
}
|
|
var ms []member
|
|
for _, sm := range m.Shared.Members {
|
|
ms = append(ms, member{Name: sm.Name, Source: sm.Source, HFRepo: sm.HFRepo})
|
|
}
|
|
b, _ := json.Marshal(ms)
|
|
env = maps.Clone(env)
|
|
if env == nil {
|
|
env = map[string]string{}
|
|
}
|
|
env[SharedModelsEnv] = string(b)
|
|
}
|
|
if p.Engine == EngineComfyUI && m.ComfyUI != nil && m.ComfyUI.HuggingFaceRepo != "" {
|
|
env = maps.Clone(env)
|
|
if env == nil {
|
|
env = map[string]string{}
|
|
}
|
|
env[HFRepoEnv] = m.ComfyUI.HuggingFaceRepo
|
|
}
|
|
return engines.Profile{
|
|
Name: p.Name,
|
|
Engine: p.Engine,
|
|
GPUClass: p.GPUClass,
|
|
TP: p.TP,
|
|
PP: p.PP,
|
|
GPUsPerNode: p.GPUsPerNode,
|
|
Nodes: p.Nodes,
|
|
Args: slices.Clone(p.Args),
|
|
Env: env,
|
|
ModelPath: m.SourceOf(p),
|
|
ServedModelName: m.Name,
|
|
Adapters: m.engineAdapters(),
|
|
KVEvents: p.KVEvents,
|
|
}.WithRuntimeLoRA(m.MaxLoadedAdapters), true
|
|
}
|
|
return engines.Profile{}, false
|
|
}
|
|
|
|
// KVIndex reports whether the router should route this model by the KV
|
|
// cache its replicas report: whether any profile publishes events. A replica
|
|
// of another profile is simply one the index has not heard from.
|
|
func (m Model) KVIndex() bool {
|
|
for _, p := range m.Profiles {
|
|
if p.KVEvents {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// ProfileByName returns one of a model's profiles, or one a profile that
|
|
// names its GPU by need stands for: "<profile>@<class>", with the class.
|
|
func (m Model) ProfileByName(name string) (Profile, bool) {
|
|
for _, p := range m.Profiles {
|
|
if p.Name == name {
|
|
return p, true
|
|
}
|
|
}
|
|
base, class, ok := strings.Cut(name, "@")
|
|
if !ok || class == "" {
|
|
return Profile{}, false
|
|
}
|
|
for _, p := range m.Profiles {
|
|
if p.Name == base && p.GPU != nil {
|
|
return p.forClass(class), true
|
|
}
|
|
}
|
|
return Profile{}, false
|
|
}
|
|
|
|
// String renders the catalog for `ctl models`.
|
|
func (c *Catalog) String() string {
|
|
var b strings.Builder
|
|
for _, m := range c.Models() {
|
|
fmt.Fprintf(&b, "%s (min %d, max %d)\n", m.Name, m.Scaling.Min, m.Scaling.Max)
|
|
for _, p := range m.Profiles {
|
|
fmt.Fprintf(&b, " %-20s %-10s %-12s tp=%d nodes=%d priority=%d\n",
|
|
p.Name, p.Engine, p.GPUClass, p.TP, p.Nodes, p.Priority)
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// Simulated reports whether this model produces synthetic text rather than
|
|
// inference output.
|
|
//
|
|
// Derived from the profiles rather than configured, because an operator who
|
|
// added a sim profile and forgot the flag would be shipping placeholder text to
|
|
// users under a real model's name. A model counts as simulated only when every
|
|
// profile is simulated: one real profile means a request can land on a real
|
|
// engine, and a model that sometimes answers for real must not be labelled as
|
|
// though it never does.
|
|
func (m Model) Simulated() bool {
|
|
if len(m.Profiles) == 0 {
|
|
return false
|
|
}
|
|
for _, p := range m.Profiles {
|
|
if p.Engine != "sim" {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// Fingerprint identifies a profile's contents.
|
|
//
|
|
// The name is not enough. A profile can be edited — a memory requirement
|
|
// raised, a node label added, an engine argument changed — and keep its name,
|
|
// so a replica running the old definition is indistinguishable from one
|
|
// running the new. The reconciler does not move a running replica when its
|
|
// profile changes, which is right: churning a replica that is serving costs
|
|
// more than the imperfection. What was missing is any way to see that it had
|
|
// happened, and the answer to "why is this one on the wrong node" was in
|
|
// somebody's memory of what they edited an hour ago.
|
|
func (p Profile) Fingerprint() string {
|
|
b, err := json.Marshal(p)
|
|
if err != nil {
|
|
// A profile that will not marshal cannot be compared, and saying so by
|
|
// never matching is better than saying two differing ones are the same.
|
|
return "unmarshalable"
|
|
}
|
|
sum := sha256.Sum256(b)
|
|
return hex.EncodeToString(sum[:8])
|
|
}
|
|
|
|
// SuggestCores is the cores to give a replica with no GPU on nodes of these
|
|
// CPU counts: half of the smallest, so two replicas share any of them without
|
|
// competing. Zero when no node says, which reserves nothing.
|
|
func SuggestCores(cpuCounts []int) int {
|
|
smallest := 0
|
|
for _, c := range cpuCounts {
|
|
if c > 0 && (smallest == 0 || c < smallest) {
|
|
smallest = c
|
|
}
|
|
}
|
|
if smallest == 0 {
|
|
return 0
|
|
}
|
|
return max(smallest/2, 1)
|
|
}
|
|
|
|
// The capacity classes, lowest first.
|
|
const (
|
|
ClassOpportunistic = "opportunistic"
|
|
ClassStandard = "standard"
|
|
ClassCritical = "critical"
|
|
)
|
|
|
|
func validClass(c string) bool {
|
|
return c == "" || c == ClassOpportunistic || c == ClassStandard || c == ClassCritical
|
|
}
|
|
|
|
// EffectiveClass is the class a model is treated as: what it says, or, when
|
|
// it says nothing, opportunistic for one loaded on demand — it holds capacity
|
|
// only while asked for, and has always been the first to give it up — and
|
|
// standard for everything else.
|
|
func (m Model) EffectiveClass() string {
|
|
if m.Class != "" {
|
|
return m.Class
|
|
}
|
|
if m.IsGuard() {
|
|
// Every judged request waits on it.
|
|
return ClassCritical
|
|
}
|
|
if m.Scaling.OnDemand() {
|
|
return ClassOpportunistic
|
|
}
|
|
return ClassStandard
|
|
}
|
|
|
|
// ClassRank orders classes: a higher rank may take capacity from a lower one.
|
|
func ClassRank(c string) int {
|
|
switch c {
|
|
case ClassCritical:
|
|
return 3
|
|
case ClassStandard, "":
|
|
return 2
|
|
default:
|
|
return 1
|
|
}
|
|
}
|
|
|
|
func validOwner(o string) bool {
|
|
for _, p := range []string{"org:", "team:", "project:"} {
|
|
if strings.HasPrefix(o, p) && len(o) > len(p) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// External is a provider outside the cluster that serves a model.
|
|
type External struct {
|
|
// URL is the provider's base address, without /v1: https://api.openai.com.
|
|
URL string `json:"url" koanf:"url"`
|
|
// Model is what the provider calls it; requests are rewritten to it.
|
|
Model string `json:"model" koanf:"model"`
|
|
// APIKeyFile holds the provider's key, on the control plane. A file
|
|
// rather than the key itself, so the key is in neither the catalog, the
|
|
// store nor the audit log; empty for a provider that asks for none.
|
|
APIKeyFile string `json:"api_key_file,omitempty" koanf:"api_key_file"`
|
|
// APIKeyFiles are several keys for the provider, in place of
|
|
// APIKeyFile: one endpoint each (external:MODEL#1, #2…), so a key that
|
|
// answers 429 or 401 is passed over while its breaker is open and the
|
|
// request goes to another. A provider's limits are per key.
|
|
APIKeyFiles []ExternalKey `json:"api_key_files,omitempty" koanf:"api_key_files"`
|
|
// Peer names the federated llmgw cluster this model is offered by, for
|
|
// a model that federation added (`llmgw ctl federation add`) rather
|
|
// than one written by hand. Such a model is named PEER/MODEL, comes and
|
|
// goes with the peer's own list, and is sent only while the peer is up.
|
|
Peer string `json:"peer,omitempty" koanf:"peer"`
|
|
// API is what the provider speaks: OpenAI's, unless this says comfyui —
|
|
// a ComfyUI server, asked for pictures at /v1/images/generations and
|
|
// sent the workflow below with the request's prompt in it. Model is then
|
|
// the checkpoint file the workflow loads, as ComfyUI lists it.
|
|
API string `json:"api,omitempty" koanf:"api"`
|
|
// Workflow is the ComfyUI workflow, in its API format: a built-in's name
|
|
// (flux-schnell, the default, or flux-dev) or a JSON file on the control
|
|
// plane.
|
|
Workflow string `json:"workflow,omitempty" koanf:"workflow"`
|
|
// Output is what it makes: pictures, or meshes (see ComfyUI.Output).
|
|
Output string `json:"output,omitempty" koanf:"output"`
|
|
}
|
|
|
|
// ExternalKey is one of a provider's keys.
|
|
type ExternalKey struct {
|
|
// File holds the key, on the control plane, as APIKeyFile does.
|
|
File string `json:"file" koanf:"file"`
|
|
// Weight is its share of the requests against the others'; zero is 1.
|
|
Weight float64 `json:"weight,omitempty" koanf:"weight"`
|
|
}
|
|
|
|
// APIComfyUI is External.API for a ComfyUI server.
|
|
const APIComfyUI = "comfyui"
|
|
|
|
// IsPeer reports whether a model is offered by a federated cluster.
|
|
func (m Model) IsPeer() bool { return m.External != nil && m.External.Peer != "" }
|
|
|
|
func (e *External) validate(model string, profiles int) error {
|
|
if profiles > 0 {
|
|
return fmt.Errorf("model %q: an external model is served elsewhere and has no profiles", model)
|
|
}
|
|
u, err := url.Parse(e.URL)
|
|
if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" {
|
|
return fmt.Errorf("model %q: external url %q is not an http(s) address", model, e.URL)
|
|
}
|
|
if strings.TrimSpace(e.Model) == "" {
|
|
return fmt.Errorf("model %q: external model needs the name the provider knows it by", model)
|
|
}
|
|
if e.APIKeyFile != "" && len(e.APIKeyFiles) > 0 {
|
|
return fmt.Errorf("model %q: api_key_file or api_key_files, not both", model)
|
|
}
|
|
for i, k := range e.APIKeyFiles {
|
|
if strings.TrimSpace(k.File) == "" {
|
|
return fmt.Errorf("model %q: api_key_files[%d] names no file", model, i)
|
|
}
|
|
if k.Weight < 0 {
|
|
return fmt.Errorf("model %q: api_key_files[%d] has a negative weight", model, i)
|
|
}
|
|
}
|
|
if len(e.APIKeyFiles) > 0 && e.Peer != "" {
|
|
return fmt.Errorf("model %q: a federated model is reached with its peer's key, not api_key_files", model)
|
|
}
|
|
switch e.API {
|
|
case "", "openai":
|
|
if e.Workflow != "" || e.Output != "" {
|
|
return fmt.Errorf("model %q: a workflow and an output are for api: comfyui", model)
|
|
}
|
|
case APIComfyUI:
|
|
if e.Peer != "" {
|
|
return fmt.Errorf("model %q: a federated model speaks OpenAI's API, not ComfyUI's", model)
|
|
}
|
|
if err := checkWorkflow(model, e.Workflow); err != nil {
|
|
return err
|
|
}
|
|
if e.Output != "" {
|
|
if err := validOutput(model, e.Output); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if e.Output != "" && e.Output != OutputPictures && e.Workflow == "" {
|
|
return fmt.Errorf("model %q: a model that makes %s names its workflow: no built-in makes them", model, e.Output)
|
|
}
|
|
default:
|
|
return fmt.Errorf("model %q: external api %q is not openai or comfyui", model, e.API)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// IsExternal reports whether a model is served outside the cluster.
|
|
func (m Model) IsExternal() bool { return m.External != nil }
|
|
|
|
// Adapter is a LoRA adapter served on a model's replicas.
|
|
type Adapter struct {
|
|
// Name is what clients ask for, like a model name.
|
|
Name string `json:"name" koanf:"name"`
|
|
// Source is the adapter: a GGUF file for llama.cpp; for vLLM a PEFT
|
|
// adapter directory (a mirror version ending in "/", a path on the node,
|
|
// or a Hugging Face repository id). Fetched by each node as the model's
|
|
// weights are.
|
|
Source string `json:"source" koanf:"source"`
|
|
// Scale is how strongly it is applied; zero takes 1.
|
|
Scale float64 `json:"scale,omitempty" koanf:"scale"`
|
|
// Owner is the part of the tenancy tree the adapter belongs to —
|
|
// org:ID, team:ID, project:ID, user:ID or key:ID — and only callers in
|
|
// it see it: to anyone else it is a model that does not exist. An
|
|
// adapter trained on an organisation's documents carries some of what
|
|
// they say in its weights. Empty is every caller the model allows.
|
|
Owner string `json:"owner,omitempty" koanf:"owner"`
|
|
}
|
|
|
|
// ValidAdapterOwner checks an adapter's owner is scope:ID.
|
|
func ValidAdapterOwner(owner string) error {
|
|
if owner == "" {
|
|
return nil
|
|
}
|
|
scope, id, ok := strings.Cut(owner, ":")
|
|
switch {
|
|
case !ok || id == "":
|
|
return fmt.Errorf("an adapter's owner is scope:ID, %q is not", owner)
|
|
case scope != "org" && scope != "team" && scope != "project" && scope != "user" && scope != "key":
|
|
return fmt.Errorf("an adapter's owner is an org, team, project, user or key, not %q", scope)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// EffectiveScale is the scale an adapter is applied at.
|
|
func (a Adapter) EffectiveScale() float64 {
|
|
if a.Scale <= 0 {
|
|
return 1
|
|
}
|
|
return a.Scale
|
|
}
|
|
|
|
func validateAdapters(m Model) error {
|
|
if m.MaxLoadedAdapters < 0 {
|
|
return fmt.Errorf("model %q: max_loaded_adapters cannot be negative", m.Name)
|
|
}
|
|
if m.MaxLoadedAdapters > 0 {
|
|
// Loading on demand is vLLM's runtime LoRA API. llama-server can
|
|
// only change the scale of adapters it loaded at start, so a cap
|
|
// there would be a cap on nothing.
|
|
for _, p := range m.Profiles {
|
|
if p.Engine != "vllm" {
|
|
return fmt.Errorf("model %q profile %q: max_loaded_adapters needs vllm profiles; "+
|
|
"llama.cpp loads every adapter at start", m.Name, p.Name)
|
|
}
|
|
}
|
|
}
|
|
if len(m.Adapters) == 0 {
|
|
return nil
|
|
}
|
|
if m.External != nil {
|
|
return fmt.Errorf("model %q: an external model cannot carry adapters", m.Name)
|
|
}
|
|
for _, p := range m.Profiles {
|
|
// llama.cpp loads adapters at start and applies them per request;
|
|
// vLLM loads them at start or on demand. The other engines are not
|
|
// driven either way yet.
|
|
if p.Engine != "llamacpp" && p.Engine != "vllm" && p.Engine != "sim" {
|
|
return fmt.Errorf("model %q profile %q: adapters are served by llamacpp and vllm profiles only", m.Name, p.Name)
|
|
}
|
|
// The variable is what switches vLLM to loading on demand, and the
|
|
// router loads adapters only for a model with a cap: set by hand, it
|
|
// would start replicas holding no adapter that nothing ever loads.
|
|
if _, set := p.Env[engines.RuntimeLoRAEnv]; set && m.MaxLoadedAdapters == 0 {
|
|
return fmt.Errorf("model %q profile %q: set max_loaded_adapters rather than %s",
|
|
m.Name, p.Name, engines.RuntimeLoRAEnv)
|
|
}
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, a := range m.Adapters {
|
|
if err := ValidAdapterOwner(a.Owner); err != nil {
|
|
return fmt.Errorf("model %q adapter %q: %w", m.Name, a.Name, err)
|
|
}
|
|
switch {
|
|
case a.Name == "" || a.Source == "":
|
|
return fmt.Errorf("model %q: an adapter needs a name and a source", m.Name)
|
|
case a.Name == m.Name:
|
|
return fmt.Errorf("model %q: an adapter cannot have the model's own name", m.Name)
|
|
case seen[a.Name]:
|
|
return fmt.Errorf("model %q: adapter %q is declared twice", m.Name, a.Name)
|
|
case a.Scale < 0:
|
|
return fmt.Errorf("model %q: adapter %q has a negative scale", m.Name, a.Name)
|
|
case m.MaxLoadedAdapters > 0 && (strings.ContainsAny(a.Name, `/\`) || a.Name == "." || a.Name == ".."):
|
|
// Loaded on demand, the name is a file's name on the node.
|
|
return fmt.Errorf("model %q: adapter %q cannot contain a slash when adapters load on demand", m.Name, a.Name)
|
|
}
|
|
seen[a.Name] = true
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (m Model) engineAdapters() []engines.Adapter {
|
|
var out []engines.Adapter
|
|
for _, a := range m.Adapters {
|
|
out = append(out, engines.Adapter{Name: a.Name, Path: a.Source})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// repositoryPage reports whether source is a Hugging Face repository's page
|
|
// (https://huggingface.co/org/repo, or one of its tree views) rather than a
|
|
// file under /resolve/, and the repository's id.
|
|
func repositoryPage(source string) (string, bool) {
|
|
for _, host := range []string{"https://huggingface.co/", "http://huggingface.co/", "https://hf.co/"} {
|
|
rest, ok := strings.CutPrefix(source, host)
|
|
if !ok {
|
|
continue
|
|
}
|
|
parts := strings.Split(strings.Trim(rest, "/"), "/")
|
|
if len(parts) < 2 || (len(parts) > 2 && parts[2] == "resolve") {
|
|
return "", false
|
|
}
|
|
return parts[0] + "/" + parts[1], true
|
|
}
|
|
return "", false
|
|
}
|