Ilker ManapandClaude Opus 5.5
ac4360b3fd
Stable Fast 3D's texture_baker and uv_unwrapper are C++/CUDA packages no index has; on the PoC they were built and installed into ComfyUI's environment by hand. nodes[].build names such directories, and the gate builds each as a wheel with ComfyUI's Python, for the compute capability torch reports for the replica's own card, keeps it on the node by the node's commit, the architecture and the directory, and installs it beside the replica with the node's requirements. A copy in ComfyUI's environment does not pin it. A node with no CUDA toolkit says so. comfyui.huggingface_repo shows the replica's weights to a node that loads them by repository name, in the engines' Hugging Face cache, so a gated repository needs no token on the node. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1679 lines
65 KiB
Go
1679 lines
65 KiB
Go
// Package catalog holds the desired state: which models exist, how each one can
|
|
// be run, and how many copies of it there should be (plan §6.1).
|
|
package catalog
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"maps"
|
|
"net/url"
|
|
"path"
|
|
"regexp"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/comfy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
|
)
|
|
|
|
// Scaling is how many replicas a model should have.
|
|
type Scaling struct {
|
|
Min int `json:"min,omitempty" koanf:"min"`
|
|
Max int `json:"max,omitempty" koanf:"max"`
|
|
// IdleTimeout is how long an on-demand model may go unasked-for before its
|
|
// replicas are stopped. Zero takes the cluster default.
|
|
//
|
|
// Minutes rather than seconds, always. Stopping a replica throws away the
|
|
// KV cache prefix routing spent its effort filling, and a model that flaps
|
|
// in and out loses that entirely — so the saving has to be worth more than
|
|
// the cache, which it only is over a real idle period.
|
|
IdleTimeout time.Duration `json:"idle_timeout,omitempty" koanf:"idle_timeout"`
|
|
// Schedule changes min and max by the time of day, in the control
|
|
// plane's control.timezone: more replicas kept through working hours,
|
|
// fewer at night, an on-demand model kept loaded while people are at
|
|
// work. The first window that holds now wins; outside them all, Min and
|
|
// Max apply.
|
|
Schedule []ScaleWindow `json:"schedule,omitempty" koanf:"schedule"`
|
|
// Prewarm starts a window this long before it opens, so its replicas
|
|
// have loaded by then: loading a model is minutes, and the people who
|
|
// arrive at nine are the reason for the window.
|
|
Prewarm time.Duration `json:"prewarm,omitempty" koanf:"prewarm"`
|
|
}
|
|
|
|
// ScaleWindow is a part of the week with its own min and max.
|
|
type ScaleWindow struct {
|
|
// Days are mon … sun, or weekdays, weekends; empty is every day.
|
|
Days []string `json:"days,omitempty" koanf:"days"`
|
|
// From and To are HH:MM; a window whose To is before its From runs
|
|
// past midnight.
|
|
From string `json:"from" koanf:"from"`
|
|
To string `json:"to" koanf:"to"`
|
|
Min int `json:"min" koanf:"min"`
|
|
// Max, when set, is the window's ceiling for load scaling; zero keeps
|
|
// the model's own.
|
|
Max int `json:"max,omitempty" koanf:"max"`
|
|
}
|
|
|
|
// clock parses HH:MM into minutes past midnight.
|
|
func clock(s string) (int, bool) {
|
|
var h, m int
|
|
if n, err := fmt.Sscanf(s, "%d:%d", &h, &m); err != nil || n != 2 || h < 0 || h > 23 || m < 0 || m > 59 || len(s) != 5 {
|
|
return 0, false
|
|
}
|
|
return h*60 + m, true
|
|
}
|
|
|
|
var weekdays = map[string][]time.Weekday{
|
|
"mon": {time.Monday}, "tue": {time.Tuesday}, "wed": {time.Wednesday}, "thu": {time.Thursday},
|
|
"fri": {time.Friday}, "sat": {time.Saturday}, "sun": {time.Sunday},
|
|
"weekdays": {time.Monday, time.Tuesday, time.Wednesday, time.Thursday, time.Friday},
|
|
"weekends": {time.Saturday, time.Sunday},
|
|
}
|
|
|
|
func (w ScaleWindow) validate() error {
|
|
from, ok1 := clock(w.From)
|
|
to, ok2 := clock(w.To)
|
|
if !ok1 || !ok2 {
|
|
return fmt.Errorf("from and to are HH:MM, like 08:00")
|
|
}
|
|
if from == to {
|
|
return fmt.Errorf("from and to are the same time")
|
|
}
|
|
for _, d := range w.Days {
|
|
if _, ok := weekdays[strings.ToLower(d)]; !ok {
|
|
return fmt.Errorf("day %q is not mon … sun, weekdays or weekends", d)
|
|
}
|
|
}
|
|
if w.Min < 0 || w.Max < 0 || w.Max > 0 && w.Max < w.Min {
|
|
return fmt.Errorf("min %d and max %d: max is 0 or at least min", w.Min, w.Max)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// holds says whether t falls in the window, opened early by lead.
|
|
func (w ScaleWindow) holds(t time.Time, lead time.Duration) bool {
|
|
from, _ := clock(w.From)
|
|
to, _ := clock(w.To)
|
|
span := to - from
|
|
if span <= 0 {
|
|
span += 24 * 60 // past midnight
|
|
}
|
|
midnight := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
|
|
// The window as it starts yesterday, today and tomorrow — tomorrow's
|
|
// can open today, pulled forward by the lead.
|
|
for d := -1; d <= 1; d++ {
|
|
day := midnight.AddDate(0, 0, d)
|
|
if !w.on(day.Weekday()) {
|
|
continue
|
|
}
|
|
start := day.Add(time.Duration(from)*time.Minute - lead)
|
|
end := day.Add(time.Duration(from+span) * time.Minute)
|
|
if !t.Before(start) && t.Before(end) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// on says whether the window starts on a day.
|
|
func (w ScaleWindow) on(wd time.Weekday) bool {
|
|
if len(w.Days) == 0 {
|
|
return true
|
|
}
|
|
for _, d := range w.Days {
|
|
for _, x := range weekdays[strings.ToLower(d)] {
|
|
if x == wd {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Window is the window in force at t, in its own time zone, and false
|
|
// outside them all.
|
|
func (s Scaling) Window(t time.Time) (ScaleWindow, bool) {
|
|
for _, w := range s.Schedule {
|
|
if w.holds(t, s.Prewarm) {
|
|
return w, true
|
|
}
|
|
}
|
|
return ScaleWindow{}, false
|
|
}
|
|
|
|
// OnDemand reports whether this model is loaded when someone asks for it.
|
|
//
|
|
// min 0 with a max above it, which needs no new field and reads correctly: keep
|
|
// none running, allow up to max. min 0 and max 0 is a model that is configured
|
|
// and deliberately not running, which is a different and equally useful thing —
|
|
// the difference between "load it when wanted" and "do not load it".
|
|
func (s Scaling) OnDemand() bool { return s.Min == 0 && s.Max > 0 }
|
|
|
|
// SLO is the latency target a model is expected to meet. `ctl bench` measures
|
|
// capacity against it (plan §6.2.1), and admission holds the lower bands
|
|
// back while the model's recent p99 is past it.
|
|
type SLO struct {
|
|
// TTFTp99 is the time to the first token 99 requests in 100 are within.
|
|
TTFTp99 time.Duration `koanf:"ttft_p99"`
|
|
// TPOTp99 is the time per output token, after the first, 99 streamed
|
|
// requests in 100 are within.
|
|
TPOTp99 time.Duration `koanf:"tpot_p99"`
|
|
}
|
|
|
|
// Profile is how a model runs on one GPU class.
|
|
type Profile struct {
|
|
Name string `json:"name,omitempty" koanf:"name"`
|
|
Engine string `json:"engine,omitempty" koanf:"engine"`
|
|
// EngineVersion restricts the profile to nodes whose installed engine's
|
|
// version starts with it: "0.11" takes vLLM 0.11.0 and 0.11.1, not
|
|
// 0.10.2. Empty takes whatever version the node has.
|
|
EngineVersion string `json:"engine_version,omitempty" koanf:"engine_version"`
|
|
GPUClass string `json:"gpu_class,omitempty" koanf:"gpu_class"`
|
|
// Source is the weights this profile loads, when not the model's: an
|
|
// embedding model served by vLLM from its repository on a card and by
|
|
// llama.cpp from a GGUF on CPUs is one model with two sources. Empty
|
|
// loads the model's. Updates and canaries move the model's source; a
|
|
// profile with its own keeps it.
|
|
Source string `json:"source,omitempty" koanf:"source"`
|
|
|
|
TP int `json:"tp,omitempty" koanf:"tp"`
|
|
PP int `json:"pp,omitempty" koanf:"pp"`
|
|
// GPUsPerNode defaults to TP for a single-node profile.
|
|
GPUsPerNode int `json:"gpus_per_node,omitempty" koanf:"gpus_per_node"`
|
|
// Nodes above 1 makes this a multi-node profile, which requires a fabric.
|
|
Nodes int `json:"nodes,omitempty" koanf:"nodes"`
|
|
|
|
Args []string `json:"args,omitempty" koanf:"args"`
|
|
Env map[string]string `json:"env,omitempty" koanf:"env"`
|
|
|
|
// MinMemoryGB is the host RAM this profile needs to run.
|
|
//
|
|
// The number that matters is the model's working set: the weights, plus the
|
|
// KV cache for the whole context pool, plus the engine's own overhead. For
|
|
// a GPU profile that lives in the cards and this is small; for a CPU
|
|
// profile it is the whole thing, and getting it wrong means an OOM on a
|
|
// node that reported itself healthy.
|
|
//
|
|
// Zero means unconstrained, which is every profile that existed before
|
|
// this. A node that does not report its memory is not excluded either —
|
|
// an older agent must not have its nodes quietly emptied by an upgrade of
|
|
// the control plane.
|
|
MinMemoryGB int `json:"min_memory_gb,omitempty" koanf:"min_memory_gb"`
|
|
|
|
// CPUCores is the cores a replica of a profile with no GPU is given:
|
|
// reserved out of its node's CPUs by placement, as memory is, and what
|
|
// its engine computes on (llama.cpp's --threads).
|
|
//
|
|
// Reserved because an engine takes every core it is told about, and two
|
|
// on one node that each think they have all of them spin against each
|
|
// other: on an 8-vCPU node, two 7B replicas fell from 10 tokens a second
|
|
// to under one each. Zero reserves nothing and leaves the thread count to
|
|
// the node, which is every profile from before this.
|
|
CPUCores int `json:"cpu_cores,omitempty" koanf:"cpu_cores"`
|
|
|
|
// MinDiskGB is the free space a node needs before it can hold these
|
|
// weights.
|
|
//
|
|
// Only checked when the node does not already have them: a node holding
|
|
// the file needs no room for it, and requiring the space twice would keep
|
|
// a model off the one node best placed to run it.
|
|
//
|
|
// Declared rather than derived, because the size of a model that has never
|
|
// been fetched anywhere is not knowable from here — and a wrong guess
|
|
// fails late, halfway through a download, which is the failure this exists
|
|
// to prevent. Zero means unconstrained, and a node that reports no storage
|
|
// is not excluded: unknown is not full.
|
|
MinDiskGB int `json:"min_disk_gb,omitempty" koanf:"min_disk_gb"`
|
|
|
|
// NodeSelector restricts this profile to nodes carrying every one of these
|
|
// labels, with these values.
|
|
//
|
|
// A CPU profile otherwise fits everywhere, because there is no card to be
|
|
// free — which is right for a 0.5B model and wrong for a 14B one. Nothing
|
|
// in the inventory says how much memory a node has, so the operator says
|
|
// which nodes are meant for a model instead of the scheduler inferring it.
|
|
// Agents have reported labels since the beginning; this is what reads them.
|
|
NodeSelector map[string]string `json:"node_selector,omitempty" koanf:"node_selector"`
|
|
|
|
// PowerWatts is what one replica of this profile draws, in watts, split
|
|
// evenly over its nodes, counted against the power cap of every rack,
|
|
// PDU or other domain its nodes are in (facility.budgets in server.yaml).
|
|
// Zero derives it: each card at the power limit it reports, each
|
|
// reserved core at facility.cpu_core_watts. Only matters where a domain
|
|
// has a cap.
|
|
PowerWatts float64 `json:"power_watts,omitempty" koanf:"power_watts"`
|
|
|
|
// KeepWithin keeps a multi-node replica's nodes inside one rack, row,
|
|
// room or site, read from their llmgw.io/rack (and so on) labels: for a
|
|
// fabric whose leaves do not reach across racks, or cables that do not.
|
|
// Nodes without the label are passed over for such a profile. Empty is
|
|
// no constraint.
|
|
KeepWithin string `json:"keep_within,omitempty" koanf:"keep_within"`
|
|
|
|
// Weight is the router's capacity weight. It should come from ctl bench
|
|
// rather than a guess (plan §6.2.1).
|
|
Weight float64 `json:"weight,omitempty" koanf:"weight"`
|
|
// Priority orders profiles during placement; highest that fits wins.
|
|
Priority int `json:"priority,omitempty" koanf:"priority"`
|
|
// StartupTimeout bounds model loading. A 70B model across two nodes takes
|
|
// tens of minutes, so this is per profile rather than global.
|
|
StartupTimeout time.Duration `json:"startup_timeout,omitempty" koanf:"startup_timeout"`
|
|
|
|
// KVEvents has vLLM publish what its prefix cache holds, and the router
|
|
// route this model's requests by it: to the replica holding the most of
|
|
// a prompt's leading blocks (plan §7.5 tier 2). vLLM only.
|
|
//
|
|
// Off by default, because it costs something on every request: the
|
|
// router asks a replica to tokenize the prompt before placing it, a
|
|
// round trip and the chat template run twice. It pays where prompts
|
|
// share long prefixes that the approximate tree guesses wrong — many
|
|
// replicas, conversations that move between them, evictions. The engine
|
|
// is started with sha256_cbor block hashing and a fixed PYTHONHASHSEED,
|
|
// which the router's hashing assumes; arguments overriding either leave
|
|
// the index empty, and the agent says so in its log.
|
|
KVEvents bool `json:"kv_events,omitempty" koanf:"kv_events"`
|
|
|
|
// GPU says what card the profile needs, in place of naming its class:
|
|
// it stands for every class in the cluster that meets it. A catalog
|
|
// entry written once then places sensibly on any site's cards.
|
|
GPU *GPUNeed `json:"gpu,omitempty" koanf:"gpu"`
|
|
// ComfyUI is this profile's own workflow, checkpoint and nodes, over the
|
|
// model's: an fp8 workflow where the card has fp8, fp16 where it has not.
|
|
ComfyUI *ProfileComfyUI `json:"comfyui,omitempty" koanf:"comfyui"`
|
|
}
|
|
|
|
// CPUOnly reports whether this profile needs no GPUs.
|
|
//
|
|
// Naming no GPU class is how a profile says so, and only engines that can
|
|
// actually serve without a card are allowed to leave it out — otherwise an
|
|
// omitted class would quietly become a replica placed on a machine with no
|
|
// hardware to run it.
|
|
func (p Profile) CPUOnly() bool { return p.GPUClass == "" && p.GPU == nil && cpuCapable(p.Engine) }
|
|
|
|
// cpuCapable reports whether an engine can serve a model without a GPU.
|
|
//
|
|
// llama.cpp is the one that can, and it is what makes a cluster with no cards
|
|
// testable end to end.
|
|
func cpuCapable(engine string) bool { return engine == "llamacpp" }
|
|
|
|
// Model is a catalog entry as clients see it.
|
|
type Model struct {
|
|
Name string `json:"name,omitempty" koanf:"name"`
|
|
Aliases []string `json:"aliases,omitempty" koanf:"aliases"`
|
|
// Description is what this model is for, in a line.
|
|
//
|
|
// Written by whoever added it, because nobody else can say it. A registry
|
|
// offers a pipeline tag and a wall of README, and neither answers the
|
|
// question a person picking from a list is asking: which of these nine
|
|
// names should I send this request to.
|
|
//
|
|
// It reaches clients through /v1/models, which is the only view most of
|
|
// them ever see. A gateway serving nine models by name alone makes that
|
|
// choice by guesswork.
|
|
Description string `json:"description,omitempty" koanf:"description"`
|
|
// Source is the weights path or a HuggingFace id.
|
|
Source string `json:"source,omitempty" koanf:"source"`
|
|
Profiles []Profile `json:"profiles,omitempty" koanf:"profiles"`
|
|
Scaling Scaling `json:"scaling,omitempty" koanf:"scaling"`
|
|
// Class is how much this model's capacity is worth against another's,
|
|
// when there is not enough for both: critical, standard or opportunistic.
|
|
// Empty is standard, or opportunistic for a model loaded on demand. See
|
|
// EffectiveClass.
|
|
Class string `json:"class,omitempty" koanf:"class"`
|
|
// Owner is the part of the tenancy tree whose capacity quota this model
|
|
// counts against, written org:ID, team:ID or project:ID. Empty is shared:
|
|
// held by the cluster, counted against nobody.
|
|
Owner string `json:"owner,omitempty" koanf:"owner"`
|
|
// External is a model served by someone else — an OpenAI-compatible API
|
|
// outside the cluster — rather than by replicas placed here. Such a model
|
|
// has no profiles and nothing is placed for it; the router sends its
|
|
// requests to the provider. Most useful named in another model's
|
|
// fallbacks: used only when the cluster has no capacity left. Its
|
|
// prompts leave the cluster, so which keys may name it is worth an
|
|
// allowlist.
|
|
External *External `json:"external,omitempty" koanf:"external"`
|
|
// ComfyUI is how a model whose profiles run engine comfyui makes its
|
|
// pictures: the workflow, and the file it loads.
|
|
ComfyUI *ComfyUI `json:"comfyui,omitempty" koanf:"comfyui"`
|
|
// License is the terms of what the model makes, for a model whose
|
|
// output somebody ships: shown with every result and written into a
|
|
// mesh.
|
|
License *License `json:"license,omitempty" koanf:"license"`
|
|
// Adapters are LoRA adapters served on this model's replicas, each asked
|
|
// for by its own name as though it were a model. Every replica loads
|
|
// every adapter, applying one only to a request that names it — unless
|
|
// MaxLoadedAdapters is set.
|
|
Adapters []Adapter `json:"adapters,omitempty" koanf:"adapters"`
|
|
// MaxLoadedAdapters, above zero, has each replica hold at most this many
|
|
// adapters at once, loaded when a request first asks for one and unloaded
|
|
// least recently used first to make room. The router sends a request for
|
|
// an adapter to a replica that already holds it while that one has room
|
|
// to take it. vLLM profiles only: llama-server loads adapters at start
|
|
// and cannot load one afterwards. Zero loads every adapter on every
|
|
// replica at start.
|
|
MaxLoadedAdapters int `json:"max_loaded_adapters,omitempty" koanf:"max_loaded_adapters"`
|
|
Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"`
|
|
// Canary sends a share of this model's requests to another catalog
|
|
// model — the same model at new weights — to compare the two before
|
|
// moving to it (`llmgw ctl model canary`).
|
|
Canary *Canary `json:"canary,omitempty" koanf:"canary"`
|
|
// Eval is a quality suite new weights must pass before they replace
|
|
// this model's: `llmgw ctl model update` and `model promote` run it
|
|
// against them first, and stop below its score.
|
|
Eval *EvalGate `json:"eval,omitempty" koanf:"eval"`
|
|
// RoutingProfile selects how the router scores endpoints for this model.
|
|
RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"`
|
|
SLO SLO `json:"slo,omitempty" koanf:"slo"`
|
|
|
|
// ContextLength is how many tokens one request may use, prompt and
|
|
// completion together, and MaxOutput bounds the completion alone. Zero
|
|
// means unknown, which is what every model was until now.
|
|
//
|
|
// The gateway knew neither, so an over-long prompt was not refused: it was
|
|
// admitted, given a replica, and failed at the engine, and a client sizing
|
|
// its prompts from /v1/models had nothing to read. Worse, the number is not
|
|
// the one an operator writes in the engine's arguments — llama.cpp divides
|
|
// --ctx-size across its parallel slots, so a pool of 8192 across four slots
|
|
// is 2048 a request, and nothing said so.
|
|
ContextLength int `json:"context_length,omitempty" koanf:"context_length"`
|
|
MaxOutput int `json:"max_output,omitempty" koanf:"max_output"`
|
|
|
|
// Capabilities is what this model can do beyond plain completion.
|
|
Capabilities Capabilities `json:"capabilities,omitempty" koanf:"capabilities"`
|
|
|
|
// QueryPrefix is put before a question an embedding model embeds for a
|
|
// knowledge-base search; documents are embedded without it. Models
|
|
// trained with an instruction on the query side want it, Qwen3-Embedding
|
|
// "Instruct: Given a web search query, retrieve relevant passages that
|
|
// answer the query\nQuery: " for one.
|
|
QueryPrefix string `json:"query_prefix,omitempty" koanf:"query_prefix"`
|
|
|
|
// Role "guard" is a model that judges the gateway's traffic for the
|
|
// guardrails rather than serving clients: hidden from /v1/models and the
|
|
// chat page, refused to any key that names it, critical by default, and
|
|
// always running. Guard says how its answers are read.
|
|
Role string `json:"role,omitempty" koanf:"role"`
|
|
Guard *GuardModel `json:"guard,omitempty" koanf:"guard"`
|
|
|
|
// Origin is the registry repository the weights came from, at which
|
|
// commit, and each file's checksum there. Written when a model is served
|
|
// from the registry; for one that was not, worked out from the mirror's
|
|
// record of where it fetched the file. It is what lets the control plane
|
|
// say that the repository has published different weights since, and
|
|
// move the model to them.
|
|
Origin *Origin `json:"origin,omitempty" koanf:"origin"`
|
|
}
|
|
|
|
// RoleGuard is a judge model's role.
|
|
const RoleGuard = "guard"
|
|
|
|
// IsGuard reports whether a model judges traffic rather than serving it.
|
|
func (m Model) IsGuard() bool { return m.Role == RoleGuard }
|
|
|
|
// GuardModel is how a judge model's answers are read.
|
|
type GuardModel struct {
|
|
// Format is qwen3guard (Qwen3Guard-Gen: "Safety: Safe", "Unsafe" or
|
|
// "Controversial", and the categories), llamaguard3 (Meta's Llama Guard
|
|
// 3: "safe", or "unsafe" and the categories), shieldgemma (Google's
|
|
// ShieldGemma: Yes or No), or json ({"safe": false, "categories":
|
|
// ["..."]}).
|
|
Format string `json:"format" koanf:"format"`
|
|
// Categories the model is asked about, for shieldgemma — each a policy
|
|
// sentence — and the names reported for llamaguard3's S1, S2, ….
|
|
Categories []string `json:"categories,omitempty" koanf:"categories"`
|
|
}
|
|
|
|
// Origin is where a model's weights were published.
|
|
type Origin struct {
|
|
// Repo is the repository, owner/name.
|
|
Repo string `json:"repo" koanf:"repo"`
|
|
// Revision is the commit the files were fetched at.
|
|
Revision string `json:"revision" koanf:"revision"`
|
|
// Files are the weights fetched from it, with what the registry
|
|
// published as each one's sha256. A new commit that leaves them alone —
|
|
// a README edited — is not an update.
|
|
Files []OriginFile `json:"files,omitempty" koanf:"files"`
|
|
// Previous is what the last update replaced, kept so it can be gone back
|
|
// to while its files are still on the mirror.
|
|
Previous *PreviousOrigin `json:"previous,omitempty" koanf:"previous"`
|
|
}
|
|
|
|
// OriginFile is one weights file of an origin.
|
|
type OriginFile struct {
|
|
Path string `json:"path" koanf:"path"`
|
|
SHA256 string `json:"sha256,omitempty" koanf:"sha256"`
|
|
Bytes int64 `json:"bytes,omitempty" koanf:"bytes"`
|
|
}
|
|
|
|
// PreviousOrigin is a model's source before an update, and where it came from.
|
|
type PreviousOrigin struct {
|
|
Source string `json:"source" koanf:"source"`
|
|
Revision string `json:"revision" koanf:"revision"`
|
|
Files []OriginFile `json:"files,omitempty" koanf:"files"`
|
|
}
|
|
|
|
// ShortRevision is a commit as a person writes it.
|
|
func ShortRevision(rev string) string {
|
|
if len(rev) > 8 {
|
|
return rev[:8]
|
|
}
|
|
return rev
|
|
}
|
|
|
|
// Capabilities describes what a client may ask a model for.
|
|
//
|
|
// Declared by the operator rather than discovered, because there is no way to
|
|
// ask an engine what its model can do. That makes these claims, and a wrong
|
|
// claim is worse than none — which is why `ctl engine-check` verifies tool
|
|
// calling against a running engine rather than trusting this.
|
|
//
|
|
// The reason it exists: the gateway forwards a tools array to any model, and a
|
|
// model that cannot use them answers in prose. Every OpenAI-compatible client
|
|
// then behaves as though the model is broken, and the only way to find out was
|
|
// to try it. That cost an afternoon.
|
|
type Capabilities struct {
|
|
Tools bool `json:"tools,omitempty" koanf:"tools"`
|
|
Vision bool `json:"vision,omitempty" koanf:"vision"`
|
|
Embeddings bool `json:"embeddings,omitempty" koanf:"embeddings"`
|
|
// Images is a model that makes pictures, asked for at
|
|
// /v1/images/generations. Implied by an external model whose API is
|
|
// ComfyUI's.
|
|
Images bool `json:"images,omitempty" koanf:"images"`
|
|
// Meshes is a model that makes 3D models from a picture, asked for as a
|
|
// job at /v1/3d/generations. Implied by a ComfyUI model whose output is
|
|
// meshes.
|
|
Meshes bool `json:"meshes,omitempty" koanf:"meshes"`
|
|
}
|
|
|
|
// The kinds of thing a ComfyUI model makes.
|
|
const (
|
|
OutputPictures = "pictures"
|
|
OutputMeshes = "meshes"
|
|
)
|
|
|
|
// comfyOutput is what a ComfyUI model makes: pictures unless it says.
|
|
func (m Model) comfyOutput() string {
|
|
out := ""
|
|
switch {
|
|
case m.External != nil:
|
|
out = m.External.Output
|
|
case m.ComfyUI != nil:
|
|
out = m.ComfyUI.Output
|
|
}
|
|
if out == "" {
|
|
return OutputPictures
|
|
}
|
|
return out
|
|
}
|
|
|
|
// MakesImages reports whether a model answers /v1/images/generations.
|
|
func (m Model) MakesImages() bool {
|
|
return m.Capabilities.Images || m.ServesComfyUI() && m.comfyOutput() == OutputPictures
|
|
}
|
|
|
|
// MakesMeshes reports whether a model answers /v1/3d/generations.
|
|
func (m Model) MakesMeshes() bool {
|
|
return m.Capabilities.Meshes || m.ServesComfyUI() && m.comfyOutput() == OutputMeshes
|
|
}
|
|
|
|
// License is the terms what a model makes comes under: said beside every
|
|
// result, and written into a mesh, which travels without the API.
|
|
type License struct {
|
|
// ID names the licence for a machine: an SPDX identifier where there is
|
|
// one (MIT, Apache-2.0, CC-BY-NC-4.0), else a lowercase name of the
|
|
// licence's own (stability-community, tencent-hunyuan-community). What a
|
|
// person's or a site's choice of licences matches on. Name is for people.
|
|
ID string `json:"id,omitempty" koanf:"id"`
|
|
// Name is the licence's own: MIT, Apache-2.0.
|
|
Name string `json:"name" koanf:"name"`
|
|
URL string `json:"url,omitempty" koanf:"url"`
|
|
// Notice is what must accompany what was made: attribution, terms on
|
|
// the output, what the weights were trained on.
|
|
Notice string `json:"notice,omitempty" koanf:"notice"`
|
|
// Commercial says whether what is made may be used commercially: nil
|
|
// is not said, which is not the same as yes.
|
|
Commercial *bool `json:"commercial,omitempty" koanf:"commercial"`
|
|
// ExcludedRegions are where the licence grants nothing: ISO 3166 codes,
|
|
// and EU for the union (Hunyuan's: EU, UK, KR).
|
|
ExcludedRegions []string `json:"excluded_regions,omitempty" koanf:"excluded_regions"`
|
|
// RevenueCapUSD is the yearly revenue above which the licence asks for
|
|
// another agreement (Stability's community licence: 1000000). Zero is
|
|
// none.
|
|
RevenueCapUSD int64 `json:"revenue_cap_usd,omitempty" koanf:"revenue_cap_usd"`
|
|
// Attribution is what a product using the output must show: "Powered by
|
|
// Stability AI".
|
|
Attribution string `json:"attribution,omitempty" koanf:"attribution"`
|
|
}
|
|
|
|
var (
|
|
licenseID = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9.+-]{0,63}$`)
|
|
regionID = regexp.MustCompile(`^[A-Z]{2}$`)
|
|
)
|
|
|
|
func (l License) validate(model string) error {
|
|
if strings.TrimSpace(l.Name) == "" {
|
|
return fmt.Errorf("model %q: license.name is required: a licence nobody can name is none", model)
|
|
}
|
|
if l.ID != "" && !licenseID.MatchString(l.ID) {
|
|
return fmt.Errorf("model %q: license.id %q is not an SPDX-style identifier (letters, digits, '.', '+', '-')", model, l.ID)
|
|
}
|
|
for _, r := range l.ExcludedRegions {
|
|
if !regionID.MatchString(r) {
|
|
return fmt.Errorf("model %q: license.excluded_regions %q is not an ISO 3166 code in capitals (EU for the union)", model, r)
|
|
}
|
|
}
|
|
if l.RevenueCapUSD < 0 {
|
|
return fmt.Errorf("model %q: license.revenue_cap_usd is negative", model)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// EngineComfyUI is the engine that runs ComfyUI on a node.
|
|
const EngineComfyUI = "comfyui"
|
|
|
|
// ServesComfyUI reports whether a model is made by ComfyUI, on the
|
|
// cluster's own nodes or on a server outside it.
|
|
func (m Model) ServesComfyUI() bool {
|
|
if m.External != nil {
|
|
return m.External.API == APIComfyUI
|
|
}
|
|
return len(m.Profiles) > 0 && m.Profiles[0].Engine == EngineComfyUI
|
|
}
|
|
|
|
// ComfyUI is a ComfyUI model's workflow and checkpoint.
|
|
type ComfyUI struct {
|
|
// Workflow is a built-in's name (flux-schnell, the default, or
|
|
// flux-dev) or a JSON file on the control plane, in ComfyUI's API
|
|
// format.
|
|
Workflow string `json:"workflow,omitempty" koanf:"workflow"`
|
|
// Checkpoint is the file the workflow's {{checkpoint}} names: the
|
|
// source's own name unless set, which a source that is a set of files
|
|
// must.
|
|
Checkpoint string `json:"checkpoint,omitempty" koanf:"checkpoint"`
|
|
// Output is what the workflow makes: pictures (the default), asked for
|
|
// at /v1/images/generations; meshes, asked for as a job at
|
|
// /v1/3d/generations from a picture; or video, audio or files. Every
|
|
// kind is asked for as a job at /v1/workflows/runs too.
|
|
Output string `json:"output,omitempty" koanf:"output"`
|
|
// Inputs are what the workflow takes from a request at
|
|
// /v1/workflows/runs, each the placeholder of its name.
|
|
Inputs []Input `json:"inputs,omitempty" koanf:"inputs"`
|
|
// Nodes are custom nodes the workflow needs, put in each replica's own
|
|
// custom_nodes at the commit named.
|
|
Nodes []CustomNode `json:"nodes,omitempty" koanf:"nodes"`
|
|
// HuggingFaceRepo is the repository a custom node loads the weights by
|
|
// name from (Stable Fast 3D's loader asks for stabilityai/stable-fast-3d):
|
|
// the replica's weights are shown to the node there, in the Hugging Face
|
|
// cache of the engines' user, so nothing is fetched and no token is
|
|
// needed when it runs.
|
|
HuggingFaceRepo string `json:"huggingface_repo,omitempty" koanf:"huggingface_repo"`
|
|
// Environment names the models that may share one ComfyUI replica: every
|
|
// model of an environment agrees on the custom nodes they both name, so
|
|
// one replica can carry all of them and ComfyUI moves between their
|
|
// weights itself. Empty is an environment of the model alone.
|
|
Environment string `json:"environment,omitempty" koanf:"environment"`
|
|
}
|
|
|
|
// CustomNode is a ComfyUI node from a git repository.
|
|
type CustomNode struct {
|
|
// Repo is the repository, https only.
|
|
Repo string `json:"repo" koanf:"repo"`
|
|
// Commit is the full commit the node is used at: a branch moves, and a
|
|
// node that changed under a model is a model that changed unasked.
|
|
Commit string `json:"commit" koanf:"commit"`
|
|
// Requirements are the PyPI packages the node needs, by name and version
|
|
// range only. They are installed beside the replica, never into
|
|
// ComfyUI's environment: a package named here at the range named, and
|
|
// every other one at the version ComfyUI already runs. One named here
|
|
// is first on ComfyUI's path, so its range must suit ComfyUI too.
|
|
Requirements []string `json:"requirements,omitempty" koanf:"requirements"`
|
|
// Build are directories of the repository that are Python packages to
|
|
// be compiled rather than downloaded: C++ and CUDA extensions no index
|
|
// has (Stable Fast 3D's texture_baker and uv_unwrapper). Each is built
|
|
// as a wheel with ComfyUI's Python for the card the replica runs on,
|
|
// kept on the node for the next replica on the same kind of card, and
|
|
// installed beside the replica like its requirements.
|
|
Build []string `json:"build,omitempty" koanf:"build"`
|
|
}
|
|
|
|
// NodesEnv carries a ComfyUI model's custom nodes to the agent, which hands
|
|
// them to the gate.
|
|
const NodesEnv = "LLMGW_COMFYUI_NODES"
|
|
|
|
// HFRepoEnv carries the repository a node loads the weights by name from.
|
|
const HFRepoEnv = "LLMGW_COMFYUI_HF_REPO"
|
|
|
|
var hfRepo = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
|
|
|
var (
|
|
fullCommit = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
|
packageSpec = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*(\[[A-Za-z0-9,._-]+\])?\s*([<>=!~]=?\s*[A-Za-z0-9.*+!_-]+\s*,?\s*)*$`)
|
|
)
|
|
|
|
func (n CustomNode) validate(model string) error {
|
|
u, err := url.Parse(n.Repo)
|
|
if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil {
|
|
return fmt.Errorf("model %q: custom node %q is not an https repository", model, n.Repo)
|
|
}
|
|
if !fullCommit.MatchString(n.Commit) {
|
|
return fmt.Errorf("model %q: custom node %s needs its full commit (40 hex), not %q: a branch moves",
|
|
model, n.Repo, n.Commit)
|
|
}
|
|
for _, b := range n.Build {
|
|
clean := path.Clean(b)
|
|
if b == "" || path.IsAbs(b) || clean == "." || clean == ".." || strings.HasPrefix(clean, "../") || strings.ContainsAny(b, "\\ ") {
|
|
return fmt.Errorf("model %q: custom node %s build %q is not a directory inside the repository", model, n.Repo, b)
|
|
}
|
|
}
|
|
for _, r := range n.Requirements {
|
|
// A package and a version range: an option or a URL would install
|
|
// from somewhere other than PyPI.
|
|
if !packageSpec.MatchString(strings.TrimSpace(r)) {
|
|
return fmt.Errorf("model %q: custom node %s requirement %q is not a package with a version range", model, n.Repo, r)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ComfyWorkflow is what a ComfyUI model's replicas are sent: the workflow
|
|
// as the catalog names it, and the checkpoint it loads.
|
|
func (m Model) ComfyWorkflow() (workflow, checkpoint string) {
|
|
if m.External != nil {
|
|
return m.External.Workflow, m.External.Model
|
|
}
|
|
if m.ComfyUI != nil {
|
|
workflow, checkpoint = m.ComfyUI.Workflow, m.ComfyUI.Checkpoint
|
|
}
|
|
if checkpoint == "" && !strings.HasSuffix(m.Source, "/") {
|
|
checkpoint = path.Base(strings.TrimPrefix(m.Source, "hf:"))
|
|
}
|
|
return workflow, checkpoint
|
|
}
|
|
|
|
// checkWorkflow refuses a workflow that is neither a built-in nor a file.
|
|
func checkWorkflow(model, w string) error {
|
|
// A file is read where the router runs; a name is checked here.
|
|
if w != "" && !strings.ContainsAny(w, "/.") && !slices.Contains(comfy.Builtins(), w) {
|
|
return fmt.Errorf("model %q: no built-in workflow %q (there are %s; or name a JSON file)",
|
|
model, w, strings.Join(comfy.Builtins(), ", "))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (m Model) validateComfyUI() error {
|
|
comfyProfiles := 0
|
|
for _, p := range m.Profiles {
|
|
if p.Engine == EngineComfyUI {
|
|
comfyProfiles++
|
|
}
|
|
}
|
|
if comfyProfiles == 0 {
|
|
if m.ComfyUI != nil {
|
|
return fmt.Errorf("model %q: comfyui: is for a model whose profiles run engine comfyui", m.Name)
|
|
}
|
|
return nil
|
|
}
|
|
if comfyProfiles != len(m.Profiles) {
|
|
return fmt.Errorf("model %q: every profile runs engine comfyui or none does: "+
|
|
"a request for pictures cannot be answered by an engine that writes text", m.Name)
|
|
}
|
|
if len(m.Adapters) > 0 {
|
|
return fmt.Errorf("model %q: ComfyUI serves no adapters", m.Name)
|
|
}
|
|
var w string
|
|
if m.ComfyUI != nil {
|
|
w = m.ComfyUI.Workflow
|
|
}
|
|
if err := checkWorkflow(m.Name, w); err != nil {
|
|
return err
|
|
}
|
|
if m.ComfyUI != nil {
|
|
for _, n := range m.ComfyUI.Nodes {
|
|
if err := n.validate(m.Name); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
if err := validOutput(m.Name, m.comfyOutput()); err != nil {
|
|
return err
|
|
}
|
|
if m.ComfyUI != nil && m.ComfyUI.HuggingFaceRepo != "" && !hfRepo.MatchString(m.ComfyUI.HuggingFaceRepo) {
|
|
return fmt.Errorf("model %q: comfyui.huggingface_repo %q is not owner/name", m.Name, m.ComfyUI.HuggingFaceRepo)
|
|
}
|
|
if out := m.comfyOutput(); out != OutputPictures && w == "" {
|
|
return fmt.Errorf("model %q: a model that makes %s names its workflow: no built-in makes them", m.Name, out)
|
|
}
|
|
if err := validateInputs(m.Name, m.ComfyInputs()); err != nil {
|
|
return err
|
|
}
|
|
if _, ckpt := m.ComfyWorkflow(); ckpt == "" {
|
|
return fmt.Errorf("model %q: its source is a set of files, so comfyui.checkpoint must name the one the workflow loads",
|
|
m.Name)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Catalog is the set of models the cluster should serve.
|
|
type Catalog struct {
|
|
models map[string]Model
|
|
aliases map[string]string
|
|
// adapters maps an adapter's name to the model it runs on.
|
|
adapters map[string]string
|
|
order []string
|
|
}
|
|
|
|
// New builds a catalog, rejecting definitions that would fail confusingly later.
|
|
func New(models []Model) (*Catalog, error) {
|
|
c := &Catalog{
|
|
models: make(map[string]Model, len(models)),
|
|
aliases: map[string]string{},
|
|
adapters: map[string]string{},
|
|
}
|
|
|
|
for i := range models {
|
|
m := models[i].normalize()
|
|
if err := m.validate(); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, dup := c.models[m.Name]; dup {
|
|
return nil, fmt.Errorf("model %q is declared twice", m.Name)
|
|
}
|
|
c.models[m.Name] = m
|
|
c.order = append(c.order, m.Name)
|
|
}
|
|
|
|
// Aliases share a namespace with names. A collision would silently shadow a
|
|
// model at request time, so it has to fail at load.
|
|
for _, name := range c.order {
|
|
for _, a := range c.models[name].Aliases {
|
|
if _, clash := c.models[a]; clash {
|
|
return nil, fmt.Errorf("model %q: alias %q collides with another model name", name, a)
|
|
}
|
|
if prev, clash := c.aliases[a]; clash {
|
|
return nil, fmt.Errorf("alias %q is claimed by both %q and %q", a, prev, name)
|
|
}
|
|
c.aliases[a] = name
|
|
}
|
|
}
|
|
|
|
// Adapters are named like models, so they share the namespace too.
|
|
for _, name := range c.order {
|
|
for _, a := range c.models[name].Adapters {
|
|
_, isModel := c.models[a.Name]
|
|
_, isAlias := c.aliases[a.Name]
|
|
if _, isAdapter := c.adapters[a.Name]; isModel || isAlias || isAdapter {
|
|
return nil, fmt.Errorf("model %q: adapter %q collides with a model, an alias or another adapter", name, a.Name)
|
|
}
|
|
c.adapters[a.Name] = name
|
|
}
|
|
}
|
|
|
|
// Fallbacks are resolved now rather than at request time: a typo in a
|
|
// fallback should stop a deployment, not surface during an outage when the
|
|
// fallback is finally needed.
|
|
for _, name := range c.order {
|
|
for _, f := range c.models[name].Fallbacks {
|
|
if _, ok := c.Resolve(f); !ok {
|
|
return nil, fmt.Errorf("model %q: fallback %q is not in the catalog", name, f)
|
|
}
|
|
}
|
|
}
|
|
// A canary is another model in the catalog, a candidate for this one's
|
|
// place: not itself, not a judge, not a canary of its own, and the
|
|
// candidate of one model only.
|
|
canaryOf := map[string]string{}
|
|
for _, name := range c.order {
|
|
cn := c.models[name].Canary
|
|
if cn == nil {
|
|
continue
|
|
}
|
|
target, ok := c.Resolve(cn.Model)
|
|
switch {
|
|
case !ok:
|
|
return nil, fmt.Errorf("model %q: canary %q is not in the catalog", name, cn.Model)
|
|
case target.Name == name:
|
|
return nil, fmt.Errorf("model %q: a model cannot be its own canary", name)
|
|
case target.IsGuard():
|
|
return nil, fmt.Errorf("model %q: canary %q is a guard model", name, cn.Model)
|
|
case target.Canary != nil:
|
|
return nil, fmt.Errorf("model %q: canary %q has a canary of its own", name, cn.Model)
|
|
case canaryOf[target.Name] != "":
|
|
return nil, fmt.Errorf("model %q: %q is already the canary of %q", name, cn.Model, canaryOf[target.Name])
|
|
case cn.Percent < 0 || cn.Percent > 100:
|
|
return nil, fmt.Errorf("model %q: canary percent %d is not 0 to 100", name, cn.Percent)
|
|
}
|
|
canaryOf[target.Name] = name
|
|
}
|
|
if _, err := c.environments(); err != nil {
|
|
return nil, err
|
|
}
|
|
return c, nil
|
|
}
|
|
|
|
// Canary is a share of a model's requests sent to a candidate, and when to
|
|
// give up on it.
|
|
type Canary struct {
|
|
// Model is the candidate: a catalog model, usually this one at newer
|
|
// weights, which clients cannot name themselves.
|
|
Model string `json:"model" koanf:"model"`
|
|
// Percent of requests go to it, 0 to 100.
|
|
Percent int `json:"percent" koanf:"percent"`
|
|
// The candidate is taken out — rolled back — when, over at least
|
|
// MinRequests of its own, its error rate is MaxErrorIncrease above
|
|
// this model's (0.05 is five points), or its mean time to the first
|
|
// token is MaxTTFTRatio times this model's. Zero is 50, 0.05 and 1.5.
|
|
MinRequests int `json:"min_requests,omitempty" koanf:"min_requests"`
|
|
MaxErrorIncrease float64 `json:"max_error_increase,omitempty" koanf:"max_error_increase"`
|
|
MaxTTFTRatio float64 `json:"max_ttft_ratio,omitempty" koanf:"max_ttft_ratio"`
|
|
}
|
|
|
|
// EvalGate is a quality suite a model's new weights must pass.
|
|
type EvalGate struct {
|
|
// Suite is a suite's name in control.eval_dir, or one the binary
|
|
// carries (arithmetic, json-format).
|
|
Suite string `json:"suite" koanf:"suite"`
|
|
// MinScore is the score, 0 to 1, to pass at; zero is the suite's own
|
|
// threshold.
|
|
MinScore float64 `json:"min_score,omitempty" koanf:"min_score"`
|
|
// Judge grades the suite's rubric cases, instead of the judge it names.
|
|
Judge string `json:"judge,omitempty" koanf:"judge"`
|
|
}
|
|
|
|
// Limits are the rollback thresholds with their defaults filled in.
|
|
func (c Canary) Limits() (minRequests int, maxErrorIncrease, maxTTFTRatio float64) {
|
|
minRequests, maxErrorIncrease, maxTTFTRatio = c.MinRequests, c.MaxErrorIncrease, c.MaxTTFTRatio
|
|
if minRequests <= 0 {
|
|
minRequests = 50
|
|
}
|
|
if maxErrorIncrease <= 0 {
|
|
maxErrorIncrease = 0.05
|
|
}
|
|
if maxTTFTRatio <= 0 {
|
|
maxTTFTRatio = 1.5
|
|
}
|
|
return
|
|
}
|
|
|
|
// CanaryOf is the model whose canary name is, or "".
|
|
func (c *Catalog) CanaryOf(name string) string {
|
|
for _, n := range c.order {
|
|
if cn := c.models[n].Canary; cn != nil {
|
|
if t, ok := c.Resolve(cn.Model); ok && t.Name == name {
|
|
return n
|
|
}
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func (m Model) normalize() Model {
|
|
if m.Scaling.Min < 0 {
|
|
m.Scaling.Min = 0
|
|
}
|
|
// Max 0 used to be an omission to fill in, because a model with no maximum
|
|
// could not run. It is now also a statement: with min 0 it means configured
|
|
// and deliberately not running, which is a different and useful thing from
|
|
// "load it when wanted". Filling it in would silently turn every switched-
|
|
// off model into an on-demand one — including the ones already written that
|
|
// way in a running cluster's configuration.
|
|
if m.Scaling.Max == 0 && m.Scaling.Min > 0 {
|
|
m.Scaling.Max = m.Scaling.Min
|
|
}
|
|
for i := range m.Profiles {
|
|
p := &m.Profiles[i]
|
|
if p.Engine == "" {
|
|
p.Engine = "vllm"
|
|
}
|
|
if p.TP <= 0 {
|
|
p.TP = 1
|
|
}
|
|
if p.Nodes <= 0 {
|
|
p.Nodes = 1
|
|
}
|
|
// tp on a llama.cpp GPU profile, written before gpus_per_node meant
|
|
// anything for it, is read as the width it was meant as: N cards of
|
|
// one node. Refusing it would have dropped a catalog's model on the
|
|
// upgrade that made the width work. Only both given, disagreeing, is
|
|
// refused below, as ambiguous.
|
|
if p.Engine == "llamacpp" && p.GPUClass != "" && (p.TP > 1 || p.PP > 1) && p.Nodes == 1 {
|
|
if n := p.TP * max(p.PP, 1); p.GPUsPerNode == 0 || p.GPUsPerNode == n {
|
|
p.GPUsPerNode, p.TP, p.PP = n, 1, 0
|
|
}
|
|
}
|
|
switch {
|
|
case p.CPUOnly():
|
|
// Nothing to divide up: this profile asks for no cards at all.
|
|
p.GPUsPerNode = 0
|
|
case p.GPUsPerNode <= 0:
|
|
// The common case: a single-node profile uses TP GPUs on one node.
|
|
p.GPUsPerNode = p.TP / p.Nodes
|
|
if p.GPUsPerNode <= 0 {
|
|
p.GPUsPerNode = 1
|
|
}
|
|
}
|
|
if p.StartupTimeout <= 0 {
|
|
p.StartupTimeout = 20 * time.Minute
|
|
}
|
|
if p.Weight <= 0 {
|
|
p.Weight = 1
|
|
}
|
|
}
|
|
// A Hugging Face repository page for a model vLLM or SGLang runs is the
|
|
// repository it names: the engine downloads a repository id itself,
|
|
// where the page was fetched as one file of HTML and failed at start.
|
|
if repo, ok := repositoryPage(m.Source); ok && len(m.Profiles) > 0 &&
|
|
!slices.ContainsFunc(m.Profiles, func(p Profile) bool { return p.Engine == "llamacpp" && p.Source == "" }) {
|
|
m.Source = repo
|
|
}
|
|
for i := range m.Profiles {
|
|
p := &m.Profiles[i]
|
|
if repo, ok := repositoryPage(p.Source); ok && p.Engine != "llamacpp" {
|
|
p.Source = repo
|
|
}
|
|
}
|
|
return m
|
|
}
|
|
|
|
func (m Model) validate() error {
|
|
if m.Name == "" {
|
|
return fmt.Errorf("a model has no name")
|
|
}
|
|
if m.Source == "" {
|
|
return fmt.Errorf("model %q: source is required", m.Name)
|
|
}
|
|
if m.License != nil {
|
|
if err := m.License.validate(m.Name); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// A page, not weights: a node fetches the HTML as one file and the
|
|
// engine fails at start, on a path that looks right. For vLLM it was
|
|
// read as the repository id when the model was made; only a source
|
|
// llama.cpp loads is left here, which needs a file.
|
|
if repo, ok := repositoryPage(m.Source); ok && (len(m.Profiles) == 0 ||
|
|
slices.ContainsFunc(m.Profiles, func(p Profile) bool { return p.Engine == "llamacpp" && p.Source == "" })) {
|
|
return fmt.Errorf("model %q: source %s is a Hugging Face repository page, not a file; "+
|
|
"llama.cpp needs a .gguf: mirror one and use its source (repository %s)",
|
|
m.Name, m.Source, repo)
|
|
}
|
|
for _, p := range m.Profiles {
|
|
if repo, ok := repositoryPage(p.Source); ok {
|
|
return fmt.Errorf("model %q profile %q: source %s is a Hugging Face repository page, not a file; "+
|
|
"llama.cpp needs a .gguf: mirror one and use its source (repository %s)",
|
|
m.Name, p.Name, p.Source, repo)
|
|
}
|
|
if p.Source != "" && m.External != nil {
|
|
return fmt.Errorf("model %q profile %q: an external model has no profiles to give a source", m.Name, p.Name)
|
|
}
|
|
}
|
|
if err := validateAdapters(m); err != nil {
|
|
return err
|
|
}
|
|
if m.External != nil {
|
|
if err := m.External.validate(m.Name, len(m.Profiles)); err != nil {
|
|
return err
|
|
}
|
|
} else if len(m.Profiles) == 0 {
|
|
return fmt.Errorf("model %q: at least one profile is required", m.Name)
|
|
}
|
|
if err := m.validateComfyUI(); err != nil {
|
|
return err
|
|
}
|
|
if m.Owner != "" && !validOwner(m.Owner) {
|
|
return fmt.Errorf("model %q: owner %q is not org:ID, team:ID or project:ID", m.Name, m.Owner)
|
|
}
|
|
if !validClass(m.Class) {
|
|
return fmt.Errorf("model %q: class %q is not one of critical, standard, opportunistic", m.Name, m.Class)
|
|
}
|
|
if g := m.Eval; g != nil {
|
|
if strings.TrimSpace(g.Suite) == "" {
|
|
return fmt.Errorf("model %q: eval names no suite", m.Name)
|
|
}
|
|
if g.MinScore < 0 || g.MinScore > 1 {
|
|
return fmt.Errorf("model %q: eval min_score %v is not 0 to 1", m.Name, g.MinScore)
|
|
}
|
|
}
|
|
switch m.Role {
|
|
case "":
|
|
if m.Guard != nil {
|
|
return fmt.Errorf("model %q: guard: is for a model with role: guard", m.Name)
|
|
}
|
|
case RoleGuard:
|
|
if m.Guard == nil {
|
|
return fmt.Errorf("model %q: a guard model needs guard.format: %s", m.Name, strings.Join(guard.Formats, ", "))
|
|
}
|
|
if !guard.KnownFormat(m.Guard.Format) {
|
|
return fmt.Errorf("model %q: guard.format %q is not one of %s", m.Name, m.Guard.Format, strings.Join(guard.Formats, ", "))
|
|
}
|
|
if m.Scaling.OnDemand() {
|
|
// A judge loaded on demand makes the first request of every
|
|
// idle period wait for it, or be judged by nothing.
|
|
return fmt.Errorf("model %q: a guard model must always run: scaling.min 1 or more", m.Name)
|
|
}
|
|
if m.IsExternal() {
|
|
return fmt.Errorf("model %q: a guard model judges in the cluster; it cannot be external", m.Name)
|
|
}
|
|
default:
|
|
return fmt.Errorf("model %q: role %q is not guard", m.Name, m.Role)
|
|
}
|
|
for i, w := range m.Scaling.Schedule {
|
|
if err := w.validate(); err != nil {
|
|
return fmt.Errorf("model %q: scaling.schedule[%d]: %w", m.Name, i, err)
|
|
}
|
|
}
|
|
if m.Scaling.Prewarm < 0 || m.Scaling.Prewarm > 12*time.Hour {
|
|
return fmt.Errorf("model %q: scaling.prewarm is between 0 and 12h", m.Name)
|
|
}
|
|
if m.Scaling.Max < m.Scaling.Min {
|
|
return fmt.Errorf("model %q: scaling max %d is below min %d", m.Name, m.Scaling.Max, m.Scaling.Min)
|
|
}
|
|
|
|
seen := map[string]struct{}{}
|
|
for _, p := range m.Profiles {
|
|
if p.Name == "" {
|
|
return fmt.Errorf("model %q: a profile has no name", m.Name)
|
|
}
|
|
if _, dup := seen[p.Name]; dup {
|
|
return fmt.Errorf("model %q: profile %q is declared twice", m.Name, p.Name)
|
|
}
|
|
seen[p.Name] = struct{}{}
|
|
|
|
for k := range p.Env {
|
|
if bad, why := engines.UnsafeEnv(k); bad {
|
|
return fmt.Errorf("model %q profile %q: env %s is not allowed: %s",
|
|
m.Name, p.Name, k, why)
|
|
}
|
|
}
|
|
if err := engines.UnsafeArgs(p.Engine, p.Args); err != nil {
|
|
return fmt.Errorf("model %q profile %q: %w", m.Name, p.Name, err)
|
|
}
|
|
if p.KVEvents && p.Engine != "vllm" {
|
|
return fmt.Errorf("model %q profile %q: kv_events is vLLM's; engine %q publishes no cache events",
|
|
m.Name, p.Name, p.Engine)
|
|
}
|
|
|
|
if p.PowerWatts < 0 {
|
|
return fmt.Errorf("model %q profile %q: power_watts %v is negative", m.Name, p.Name, p.PowerWatts)
|
|
}
|
|
if p.KeepWithin != "" {
|
|
if _, err := facility.ParseLevel(p.KeepWithin); err != nil {
|
|
return fmt.Errorf("model %q profile %q: keep_within: %w", m.Name, p.Name, err)
|
|
}
|
|
}
|
|
if err := p.validateHardware(m); err != nil {
|
|
return err
|
|
}
|
|
if p.GPUClass == "" && p.GPU == nil && !cpuCapable(p.Engine) {
|
|
return fmt.Errorf("model %q profile %q: gpu_class is required for engine %q, or gpu: the card it needs",
|
|
m.Name, p.Name, p.Engine)
|
|
}
|
|
if p.GPU != nil {
|
|
// Its class is known only when it is placed; what can be checked
|
|
// of a class waits for then.
|
|
continue
|
|
}
|
|
if p.CPUOnly() {
|
|
if p.Nodes > 1 {
|
|
return fmt.Errorf("model %q profile %q: a CPU profile cannot span %d nodes",
|
|
m.Name, p.Name, p.Nodes)
|
|
}
|
|
continue
|
|
}
|
|
// A MIG instance is a device of its own to CUDA, but a process sees
|
|
// only one, and instances have no NVLink or peer-to-peer between
|
|
// them: a group of them is neither tensor- nor layer-parallel, it is
|
|
// a replica that fails at start. One instance, one node, refused here
|
|
// rather than placed and found out.
|
|
if discovery.IsMIGClass(p.GPUClass) && (p.GPUsPerNode > 1 || p.Nodes > 1 || p.TP > 1 || p.PP > 1) {
|
|
return fmt.Errorf("model %q profile %q: %s is a MIG instance, and a replica runs on exactly one: "+
|
|
"a process can open only one MIG device, and instances have no NVLink between them. "+
|
|
"Give it tp 1 and one GPU on one node, or use a whole-card class or a larger MIG profile",
|
|
m.Name, p.Name, p.GPUClass)
|
|
}
|
|
// llama.cpp on GPUs splits a model across one node's cards by layer,
|
|
// and how many is gpus_per_node. Not tp: llama.cpp has no tensor
|
|
// parallelism, and a tp it was given was a width nothing placed — the
|
|
// replica stayed unplaced, or ran on one card while four were meant.
|
|
if p.Engine == "llamacpp" {
|
|
switch {
|
|
case p.Nodes > 1:
|
|
return fmt.Errorf("model %q profile %q: llama.cpp does not span nodes; use vllm or sglang for a %d-node replica",
|
|
m.Name, p.Name, p.Nodes)
|
|
case p.TP > 1 || p.PP > 1:
|
|
return fmt.Errorf("model %q profile %q: llama.cpp splits a model across a node's GPUs by layer, and "+
|
|
"tp %d does not agree with gpus_per_node %d: give gpus_per_node alone", m.Name, p.Name,
|
|
max(p.TP, 1)*max(p.PP, 1), p.GPUsPerNode)
|
|
}
|
|
continue
|
|
}
|
|
// TensorRT-LLM spans machines only under MPI started from one of them,
|
|
// which the agents, each starting its own node, do not do; and it
|
|
// loads a Hugging Face checkpoint, not llama.cpp's format.
|
|
if p.Engine == "trtllm" {
|
|
if p.Nodes > 1 {
|
|
return fmt.Errorf("model %q profile %q: trtllm runs a replica on one node; "+
|
|
"use vllm or sglang for a %d-node replica", m.Name, p.Name, p.Nodes)
|
|
}
|
|
if strings.HasSuffix(strings.ToLower(strings.TrimSuffix(m.SourceOf(p), "/")), ".gguf") {
|
|
return fmt.Errorf("model %q profile %q: trtllm loads a Hugging Face checkpoint, not a .gguf; "+
|
|
"serve a GGUF with llamacpp", m.Name, p.Name)
|
|
}
|
|
}
|
|
// The arithmetic has to add up, or the engine is handed a shard spec it
|
|
// cannot satisfy and fails minutes later during model loading.
|
|
if want := p.GPUsPerNode * p.Nodes; want != p.TP*max(p.PP, 1) {
|
|
return fmt.Errorf("model %q profile %q: %d nodes x %d GPUs does not match TP %d x PP %d",
|
|
m.Name, p.Name, p.Nodes, p.GPUsPerNode, p.TP, max(p.PP, 1))
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Resolve maps a client-supplied name or alias onto a model.
|
|
// ResolveAdapter finds an adapter by its name: the model it runs on, the
|
|
// adapter, and its index among the model's adapters — which is the id the
|
|
// engine knows it by.
|
|
func (c *Catalog) ResolveAdapter(name string) (Model, Adapter, int, bool) {
|
|
base, ok := c.adapters[name]
|
|
if !ok {
|
|
return Model{}, Adapter{}, 0, false
|
|
}
|
|
m := c.models[base]
|
|
for i, a := range m.Adapters {
|
|
if a.Name == name {
|
|
return m, a, i, true
|
|
}
|
|
}
|
|
return Model{}, Adapter{}, 0, false
|
|
}
|
|
|
|
func (c *Catalog) Resolve(name string) (Model, bool) {
|
|
if m, ok := c.models[name]; ok {
|
|
return m, true
|
|
}
|
|
if canonical, ok := c.aliases[name]; ok {
|
|
return c.models[canonical], true
|
|
}
|
|
return Model{}, false
|
|
}
|
|
|
|
// Models returns every model, in declaration order.
|
|
func (c *Catalog) Models() []Model {
|
|
out := make([]Model, 0, len(c.order))
|
|
for _, n := range c.order {
|
|
out = append(out, c.models[n])
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Names returns the canonical model names, sorted.
|
|
func (c *Catalog) Names() []string {
|
|
out := slices.Clone(c.order)
|
|
slices.Sort(out)
|
|
return out
|
|
}
|
|
|
|
// AliasesOf returns the alternative names for a model.
|
|
func (c *Catalog) AliasesOf(name string) []string {
|
|
m, ok := c.Resolve(name)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return slices.Clone(m.Aliases)
|
|
}
|
|
|
|
// PlacementProfiles converts a model's profiles into placement's view. A
|
|
// profile that names its card by need is left out: with no class it would
|
|
// read to placement as one that needs no GPU. PlacementProfilesFor gives
|
|
// it the classes it stands for.
|
|
func (m Model) PlacementProfiles() []placement.Profile {
|
|
out := make([]placement.Profile, 0, len(m.Profiles))
|
|
for _, p := range m.Profiles {
|
|
if p.GPU != nil {
|
|
continue
|
|
}
|
|
out = append(out, placement.Profile{
|
|
Name: p.Name,
|
|
Engine: p.Engine,
|
|
EngineVersion: p.EngineVersion,
|
|
GPUClass: p.GPUClass,
|
|
GPUsPerNode: p.GPUsPerNode,
|
|
Nodes: p.Nodes,
|
|
Priority: p.Priority,
|
|
NodeSelector: p.NodeSelector,
|
|
MinMemoryGB: p.MinMemoryGB,
|
|
MinDiskGB: p.MinDiskGB,
|
|
CPUCores: p.CPUCores,
|
|
PowerWatts: p.PowerWatts,
|
|
KeepWithin: facility.Level(strings.ToLower(strings.TrimSpace(p.KeepWithin))),
|
|
// The profile's source, so placement can prefer nodes that already
|
|
// hold the weights rather than one that must first move them.
|
|
Source: m.SourceOf(p),
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// SourceOf is the weights a profile loads: its own, or the model's.
|
|
func (m Model) SourceOf(p Profile) string {
|
|
if p.Source != "" {
|
|
return p.Source
|
|
}
|
|
return m.Source
|
|
}
|
|
|
|
// WeightSources are every source a model's replicas may load: the model's
|
|
// and each profile's of its own, once each. What a node must not have
|
|
// collected from under it.
|
|
func (m Model) WeightSources() []string {
|
|
out := []string{}
|
|
if m.Source != "" {
|
|
out = append(out, m.Source)
|
|
}
|
|
for _, p := range m.Profiles {
|
|
if p.Source != "" && !slices.Contains(out, p.Source) {
|
|
out = append(out, p.Source)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// EngineProfile converts a catalog profile into the engine adapter's view.
|
|
func (m Model) EngineProfile(name string) (engines.Profile, bool) {
|
|
if p, ok := m.ProfileByName(name); ok {
|
|
env := p.Env
|
|
if nodes := m.ComfyNodesFor(p); p.Engine == EngineComfyUI && len(nodes) > 0 {
|
|
nodes, _ := json.Marshal(nodes)
|
|
env = maps.Clone(env)
|
|
if env == nil {
|
|
env = map[string]string{}
|
|
}
|
|
env[NodesEnv] = string(nodes)
|
|
}
|
|
if p.Engine == EngineComfyUI && m.ComfyUI != nil && m.ComfyUI.HuggingFaceRepo != "" {
|
|
env = maps.Clone(env)
|
|
if env == nil {
|
|
env = map[string]string{}
|
|
}
|
|
env[HFRepoEnv] = m.ComfyUI.HuggingFaceRepo
|
|
}
|
|
return engines.Profile{
|
|
Name: p.Name,
|
|
Engine: p.Engine,
|
|
GPUClass: p.GPUClass,
|
|
TP: p.TP,
|
|
PP: p.PP,
|
|
GPUsPerNode: p.GPUsPerNode,
|
|
Nodes: p.Nodes,
|
|
Args: slices.Clone(p.Args),
|
|
Env: env,
|
|
ModelPath: m.SourceOf(p),
|
|
ServedModelName: m.Name,
|
|
Adapters: m.engineAdapters(),
|
|
KVEvents: p.KVEvents,
|
|
}.WithRuntimeLoRA(m.MaxLoadedAdapters), true
|
|
}
|
|
return engines.Profile{}, false
|
|
}
|
|
|
|
// KVIndex reports whether the router should route this model by the KV
|
|
// cache its replicas report: whether any profile publishes events. A replica
|
|
// of another profile is simply one the index has not heard from.
|
|
func (m Model) KVIndex() bool {
|
|
for _, p := range m.Profiles {
|
|
if p.KVEvents {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// ProfileByName returns one of a model's profiles, or one a profile that
|
|
// names its GPU by need stands for: "<profile>@<class>", with the class.
|
|
func (m Model) ProfileByName(name string) (Profile, bool) {
|
|
for _, p := range m.Profiles {
|
|
if p.Name == name {
|
|
return p, true
|
|
}
|
|
}
|
|
base, class, ok := strings.Cut(name, "@")
|
|
if !ok || class == "" {
|
|
return Profile{}, false
|
|
}
|
|
for _, p := range m.Profiles {
|
|
if p.Name == base && p.GPU != nil {
|
|
return p.forClass(class), true
|
|
}
|
|
}
|
|
return Profile{}, false
|
|
}
|
|
|
|
// String renders the catalog for `ctl models`.
|
|
func (c *Catalog) String() string {
|
|
var b strings.Builder
|
|
for _, m := range c.Models() {
|
|
fmt.Fprintf(&b, "%s (min %d, max %d)\n", m.Name, m.Scaling.Min, m.Scaling.Max)
|
|
for _, p := range m.Profiles {
|
|
fmt.Fprintf(&b, " %-20s %-10s %-12s tp=%d nodes=%d priority=%d\n",
|
|
p.Name, p.Engine, p.GPUClass, p.TP, p.Nodes, p.Priority)
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// Simulated reports whether this model produces synthetic text rather than
|
|
// inference output.
|
|
//
|
|
// Derived from the profiles rather than configured, because an operator who
|
|
// added a sim profile and forgot the flag would be shipping placeholder text to
|
|
// users under a real model's name. A model counts as simulated only when every
|
|
// profile is simulated: one real profile means a request can land on a real
|
|
// engine, and a model that sometimes answers for real must not be labelled as
|
|
// though it never does.
|
|
func (m Model) Simulated() bool {
|
|
if len(m.Profiles) == 0 {
|
|
return false
|
|
}
|
|
for _, p := range m.Profiles {
|
|
if p.Engine != "sim" {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// Fingerprint identifies a profile's contents.
|
|
//
|
|
// The name is not enough. A profile can be edited — a memory requirement
|
|
// raised, a node label added, an engine argument changed — and keep its name,
|
|
// so a replica running the old definition is indistinguishable from one
|
|
// running the new. The reconciler does not move a running replica when its
|
|
// profile changes, which is right: churning a replica that is serving costs
|
|
// more than the imperfection. What was missing is any way to see that it had
|
|
// happened, and the answer to "why is this one on the wrong node" was in
|
|
// somebody's memory of what they edited an hour ago.
|
|
func (p Profile) Fingerprint() string {
|
|
b, err := json.Marshal(p)
|
|
if err != nil {
|
|
// A profile that will not marshal cannot be compared, and saying so by
|
|
// never matching is better than saying two differing ones are the same.
|
|
return "unmarshalable"
|
|
}
|
|
sum := sha256.Sum256(b)
|
|
return hex.EncodeToString(sum[:8])
|
|
}
|
|
|
|
// SuggestCores is the cores to give a replica with no GPU on nodes of these
|
|
// CPU counts: half of the smallest, so two replicas share any of them without
|
|
// competing. Zero when no node says, which reserves nothing.
|
|
func SuggestCores(cpuCounts []int) int {
|
|
smallest := 0
|
|
for _, c := range cpuCounts {
|
|
if c > 0 && (smallest == 0 || c < smallest) {
|
|
smallest = c
|
|
}
|
|
}
|
|
if smallest == 0 {
|
|
return 0
|
|
}
|
|
return max(smallest/2, 1)
|
|
}
|
|
|
|
// The capacity classes, lowest first.
|
|
const (
|
|
ClassOpportunistic = "opportunistic"
|
|
ClassStandard = "standard"
|
|
ClassCritical = "critical"
|
|
)
|
|
|
|
func validClass(c string) bool {
|
|
return c == "" || c == ClassOpportunistic || c == ClassStandard || c == ClassCritical
|
|
}
|
|
|
|
// EffectiveClass is the class a model is treated as: what it says, or, when
|
|
// it says nothing, opportunistic for one loaded on demand — it holds capacity
|
|
// only while asked for, and has always been the first to give it up — and
|
|
// standard for everything else.
|
|
func (m Model) EffectiveClass() string {
|
|
if m.Class != "" {
|
|
return m.Class
|
|
}
|
|
if m.IsGuard() {
|
|
// Every judged request waits on it.
|
|
return ClassCritical
|
|
}
|
|
if m.Scaling.OnDemand() {
|
|
return ClassOpportunistic
|
|
}
|
|
return ClassStandard
|
|
}
|
|
|
|
// ClassRank orders classes: a higher rank may take capacity from a lower one.
|
|
func ClassRank(c string) int {
|
|
switch c {
|
|
case ClassCritical:
|
|
return 3
|
|
case ClassStandard, "":
|
|
return 2
|
|
default:
|
|
return 1
|
|
}
|
|
}
|
|
|
|
func validOwner(o string) bool {
|
|
for _, p := range []string{"org:", "team:", "project:"} {
|
|
if strings.HasPrefix(o, p) && len(o) > len(p) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// External is a provider outside the cluster that serves a model.
|
|
type External struct {
|
|
// URL is the provider's base address, without /v1: https://api.openai.com.
|
|
URL string `json:"url" koanf:"url"`
|
|
// Model is what the provider calls it; requests are rewritten to it.
|
|
Model string `json:"model" koanf:"model"`
|
|
// APIKeyFile holds the provider's key, on the control plane. A file
|
|
// rather than the key itself, so the key is in neither the catalog, the
|
|
// store nor the audit log; empty for a provider that asks for none.
|
|
APIKeyFile string `json:"api_key_file,omitempty" koanf:"api_key_file"`
|
|
// Peer names the federated llmgw cluster this model is offered by, for
|
|
// a model that federation added (`llmgw ctl federation add`) rather
|
|
// than one written by hand. Such a model is named PEER/MODEL, comes and
|
|
// goes with the peer's own list, and is sent only while the peer is up.
|
|
Peer string `json:"peer,omitempty" koanf:"peer"`
|
|
// API is what the provider speaks: OpenAI's, unless this says comfyui —
|
|
// a ComfyUI server, asked for pictures at /v1/images/generations and
|
|
// sent the workflow below with the request's prompt in it. Model is then
|
|
// the checkpoint file the workflow loads, as ComfyUI lists it.
|
|
API string `json:"api,omitempty" koanf:"api"`
|
|
// Workflow is the ComfyUI workflow, in its API format: a built-in's name
|
|
// (flux-schnell, the default, or flux-dev) or a JSON file on the control
|
|
// plane.
|
|
Workflow string `json:"workflow,omitempty" koanf:"workflow"`
|
|
// Output is what it makes: pictures, or meshes (see ComfyUI.Output).
|
|
Output string `json:"output,omitempty" koanf:"output"`
|
|
}
|
|
|
|
// APIComfyUI is External.API for a ComfyUI server.
|
|
const APIComfyUI = "comfyui"
|
|
|
|
// IsPeer reports whether a model is offered by a federated cluster.
|
|
func (m Model) IsPeer() bool { return m.External != nil && m.External.Peer != "" }
|
|
|
|
func (e *External) validate(model string, profiles int) error {
|
|
if profiles > 0 {
|
|
return fmt.Errorf("model %q: an external model is served elsewhere and has no profiles", model)
|
|
}
|
|
u, err := url.Parse(e.URL)
|
|
if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" {
|
|
return fmt.Errorf("model %q: external url %q is not an http(s) address", model, e.URL)
|
|
}
|
|
if strings.TrimSpace(e.Model) == "" {
|
|
return fmt.Errorf("model %q: external model needs the name the provider knows it by", model)
|
|
}
|
|
switch e.API {
|
|
case "", "openai":
|
|
if e.Workflow != "" || e.Output != "" {
|
|
return fmt.Errorf("model %q: a workflow and an output are for api: comfyui", model)
|
|
}
|
|
case APIComfyUI:
|
|
if e.Peer != "" {
|
|
return fmt.Errorf("model %q: a federated model speaks OpenAI's API, not ComfyUI's", model)
|
|
}
|
|
if err := checkWorkflow(model, e.Workflow); err != nil {
|
|
return err
|
|
}
|
|
if e.Output != "" {
|
|
if err := validOutput(model, e.Output); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if e.Output != "" && e.Output != OutputPictures && e.Workflow == "" {
|
|
return fmt.Errorf("model %q: a model that makes %s names its workflow: no built-in makes them", model, e.Output)
|
|
}
|
|
default:
|
|
return fmt.Errorf("model %q: external api %q is not openai or comfyui", model, e.API)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// IsExternal reports whether a model is served outside the cluster.
|
|
func (m Model) IsExternal() bool { return m.External != nil }
|
|
|
|
// Adapter is a LoRA adapter served on a model's replicas.
|
|
type Adapter struct {
|
|
// Name is what clients ask for, like a model name.
|
|
Name string `json:"name" koanf:"name"`
|
|
// Source is the adapter: a GGUF file for llama.cpp; for vLLM a PEFT
|
|
// adapter directory (a mirror version ending in "/", a path on the node,
|
|
// or a Hugging Face repository id). Fetched by each node as the model's
|
|
// weights are.
|
|
Source string `json:"source" koanf:"source"`
|
|
// Scale is how strongly it is applied; zero takes 1.
|
|
Scale float64 `json:"scale,omitempty" koanf:"scale"`
|
|
// Owner is the part of the tenancy tree the adapter belongs to —
|
|
// org:ID, team:ID, project:ID, user:ID or key:ID — and only callers in
|
|
// it see it: to anyone else it is a model that does not exist. An
|
|
// adapter trained on an organisation's documents carries some of what
|
|
// they say in its weights. Empty is every caller the model allows.
|
|
Owner string `json:"owner,omitempty" koanf:"owner"`
|
|
}
|
|
|
|
// ValidAdapterOwner checks an adapter's owner is scope:ID.
|
|
func ValidAdapterOwner(owner string) error {
|
|
if owner == "" {
|
|
return nil
|
|
}
|
|
scope, id, ok := strings.Cut(owner, ":")
|
|
switch {
|
|
case !ok || id == "":
|
|
return fmt.Errorf("an adapter's owner is scope:ID, %q is not", owner)
|
|
case scope != "org" && scope != "team" && scope != "project" && scope != "user" && scope != "key":
|
|
return fmt.Errorf("an adapter's owner is an org, team, project, user or key, not %q", scope)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// EffectiveScale is the scale an adapter is applied at.
|
|
func (a Adapter) EffectiveScale() float64 {
|
|
if a.Scale <= 0 {
|
|
return 1
|
|
}
|
|
return a.Scale
|
|
}
|
|
|
|
func validateAdapters(m Model) error {
|
|
if m.MaxLoadedAdapters < 0 {
|
|
return fmt.Errorf("model %q: max_loaded_adapters cannot be negative", m.Name)
|
|
}
|
|
if m.MaxLoadedAdapters > 0 {
|
|
// Loading on demand is vLLM's runtime LoRA API. llama-server can
|
|
// only change the scale of adapters it loaded at start, so a cap
|
|
// there would be a cap on nothing.
|
|
for _, p := range m.Profiles {
|
|
if p.Engine != "vllm" {
|
|
return fmt.Errorf("model %q profile %q: max_loaded_adapters needs vllm profiles; "+
|
|
"llama.cpp loads every adapter at start", m.Name, p.Name)
|
|
}
|
|
}
|
|
}
|
|
if len(m.Adapters) == 0 {
|
|
return nil
|
|
}
|
|
if m.External != nil {
|
|
return fmt.Errorf("model %q: an external model cannot carry adapters", m.Name)
|
|
}
|
|
for _, p := range m.Profiles {
|
|
// llama.cpp loads adapters at start and applies them per request;
|
|
// vLLM loads them at start or on demand. The other engines are not
|
|
// driven either way yet.
|
|
if p.Engine != "llamacpp" && p.Engine != "vllm" && p.Engine != "sim" {
|
|
return fmt.Errorf("model %q profile %q: adapters are served by llamacpp and vllm profiles only", m.Name, p.Name)
|
|
}
|
|
// The variable is what switches vLLM to loading on demand, and the
|
|
// router loads adapters only for a model with a cap: set by hand, it
|
|
// would start replicas holding no adapter that nothing ever loads.
|
|
if _, set := p.Env[engines.RuntimeLoRAEnv]; set && m.MaxLoadedAdapters == 0 {
|
|
return fmt.Errorf("model %q profile %q: set max_loaded_adapters rather than %s",
|
|
m.Name, p.Name, engines.RuntimeLoRAEnv)
|
|
}
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, a := range m.Adapters {
|
|
if err := ValidAdapterOwner(a.Owner); err != nil {
|
|
return fmt.Errorf("model %q adapter %q: %w", m.Name, a.Name, err)
|
|
}
|
|
switch {
|
|
case a.Name == "" || a.Source == "":
|
|
return fmt.Errorf("model %q: an adapter needs a name and a source", m.Name)
|
|
case a.Name == m.Name:
|
|
return fmt.Errorf("model %q: an adapter cannot have the model's own name", m.Name)
|
|
case seen[a.Name]:
|
|
return fmt.Errorf("model %q: adapter %q is declared twice", m.Name, a.Name)
|
|
case a.Scale < 0:
|
|
return fmt.Errorf("model %q: adapter %q has a negative scale", m.Name, a.Name)
|
|
case m.MaxLoadedAdapters > 0 && (strings.ContainsAny(a.Name, `/\`) || a.Name == "." || a.Name == ".."):
|
|
// Loaded on demand, the name is a file's name on the node.
|
|
return fmt.Errorf("model %q: adapter %q cannot contain a slash when adapters load on demand", m.Name, a.Name)
|
|
}
|
|
seen[a.Name] = true
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (m Model) engineAdapters() []engines.Adapter {
|
|
var out []engines.Adapter
|
|
for _, a := range m.Adapters {
|
|
out = append(out, engines.Adapter{Name: a.Name, Path: a.Source})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// repositoryPage reports whether source is a Hugging Face repository's page
|
|
// (https://huggingface.co/org/repo, or one of its tree views) rather than a
|
|
// file under /resolve/, and the repository's id.
|
|
func repositoryPage(source string) (string, bool) {
|
|
for _, host := range []string{"https://huggingface.co/", "http://huggingface.co/", "https://hf.co/"} {
|
|
rest, ok := strings.CutPrefix(source, host)
|
|
if !ok {
|
|
continue
|
|
}
|
|
parts := strings.Split(strings.Trim(rest, "/"), "/")
|
|
if len(parts) < 2 || (len(parts) > 2 && parts[2] == "resolve") {
|
|
return "", false
|
|
}
|
|
return parts[0] + "/" + parts[1], true
|
|
}
|
|
return "", false
|
|
}
|