// Package catalog holds the desired state: which models exist, how each one can // be run, and how many copies of it there should be (plan §6.1). package catalog import ( "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "net/url" "path" "slices" "strings" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" "gitea.mynodes.xyz/ilker/llmgw/internal/comfy" "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" "gitea.mynodes.xyz/ilker/llmgw/internal/control/placement" "gitea.mynodes.xyz/ilker/llmgw/internal/guard" ) // Scaling is how many replicas a model should have. type Scaling struct { Min int `json:"min,omitempty" koanf:"min"` Max int `json:"max,omitempty" koanf:"max"` // IdleTimeout is how long an on-demand model may go unasked-for before its // replicas are stopped. Zero takes the cluster default. // // Minutes rather than seconds, always. Stopping a replica throws away the // KV cache prefix routing spent its effort filling, and a model that flaps // in and out loses that entirely — so the saving has to be worth more than // the cache, which it only is over a real idle period. IdleTimeout time.Duration `json:"idle_timeout,omitempty" koanf:"idle_timeout"` // Schedule changes min and max by the time of day, in the control // plane's control.timezone: more replicas kept through working hours, // fewer at night, an on-demand model kept loaded while people are at // work. The first window that holds now wins; outside them all, Min and // Max apply. Schedule []ScaleWindow `json:"schedule,omitempty" koanf:"schedule"` // Prewarm starts a window this long before it opens, so its replicas // have loaded by then: loading a model is minutes, and the people who // arrive at nine are the reason for the window. Prewarm time.Duration `json:"prewarm,omitempty" koanf:"prewarm"` } // ScaleWindow is a part of the week with its own min and max. type ScaleWindow struct { // Days are mon … sun, or weekdays, weekends; empty is every day. Days []string `json:"days,omitempty" koanf:"days"` // From and To are HH:MM; a window whose To is before its From runs // past midnight. From string `json:"from" koanf:"from"` To string `json:"to" koanf:"to"` Min int `json:"min" koanf:"min"` // Max, when set, is the window's ceiling for load scaling; zero keeps // the model's own. Max int `json:"max,omitempty" koanf:"max"` } // clock parses HH:MM into minutes past midnight. func clock(s string) (int, bool) { var h, m int if n, err := fmt.Sscanf(s, "%d:%d", &h, &m); err != nil || n != 2 || h < 0 || h > 23 || m < 0 || m > 59 || len(s) != 5 { return 0, false } return h*60 + m, true } var weekdays = map[string][]time.Weekday{ "mon": {time.Monday}, "tue": {time.Tuesday}, "wed": {time.Wednesday}, "thu": {time.Thursday}, "fri": {time.Friday}, "sat": {time.Saturday}, "sun": {time.Sunday}, "weekdays": {time.Monday, time.Tuesday, time.Wednesday, time.Thursday, time.Friday}, "weekends": {time.Saturday, time.Sunday}, } func (w ScaleWindow) validate() error { from, ok1 := clock(w.From) to, ok2 := clock(w.To) if !ok1 || !ok2 { return fmt.Errorf("from and to are HH:MM, like 08:00") } if from == to { return fmt.Errorf("from and to are the same time") } for _, d := range w.Days { if _, ok := weekdays[strings.ToLower(d)]; !ok { return fmt.Errorf("day %q is not mon … sun, weekdays or weekends", d) } } if w.Min < 0 || w.Max < 0 || w.Max > 0 && w.Max < w.Min { return fmt.Errorf("min %d and max %d: max is 0 or at least min", w.Min, w.Max) } return nil } // holds says whether t falls in the window, opened early by lead. func (w ScaleWindow) holds(t time.Time, lead time.Duration) bool { from, _ := clock(w.From) to, _ := clock(w.To) span := to - from if span <= 0 { span += 24 * 60 // past midnight } midnight := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) // The window as it starts yesterday, today and tomorrow — tomorrow's // can open today, pulled forward by the lead. for d := -1; d <= 1; d++ { day := midnight.AddDate(0, 0, d) if !w.on(day.Weekday()) { continue } start := day.Add(time.Duration(from)*time.Minute - lead) end := day.Add(time.Duration(from+span) * time.Minute) if !t.Before(start) && t.Before(end) { return true } } return false } // on says whether the window starts on a day. func (w ScaleWindow) on(wd time.Weekday) bool { if len(w.Days) == 0 { return true } for _, d := range w.Days { for _, x := range weekdays[strings.ToLower(d)] { if x == wd { return true } } } return false } // Window is the window in force at t, in its own time zone, and false // outside them all. func (s Scaling) Window(t time.Time) (ScaleWindow, bool) { for _, w := range s.Schedule { if w.holds(t, s.Prewarm) { return w, true } } return ScaleWindow{}, false } // OnDemand reports whether this model is loaded when someone asks for it. // // min 0 with a max above it, which needs no new field and reads correctly: keep // none running, allow up to max. min 0 and max 0 is a model that is configured // and deliberately not running, which is a different and equally useful thing — // the difference between "load it when wanted" and "do not load it". func (s Scaling) OnDemand() bool { return s.Min == 0 && s.Max > 0 } // SLO is the latency target a model is expected to meet. `ctl bench` measures // capacity against it (plan §6.2.1), and admission holds the lower bands // back while the model's recent p99 is past it. type SLO struct { // TTFTp99 is the time to the first token 99 requests in 100 are within. TTFTp99 time.Duration `koanf:"ttft_p99"` // TPOTp99 is the time per output token, after the first, 99 streamed // requests in 100 are within. TPOTp99 time.Duration `koanf:"tpot_p99"` } // Profile is how a model runs on one GPU class. type Profile struct { Name string `json:"name,omitempty" koanf:"name"` Engine string `json:"engine,omitempty" koanf:"engine"` // EngineVersion restricts the profile to nodes whose installed engine's // version starts with it: "0.11" takes vLLM 0.11.0 and 0.11.1, not // 0.10.2. Empty takes whatever version the node has. EngineVersion string `json:"engine_version,omitempty" koanf:"engine_version"` GPUClass string `json:"gpu_class,omitempty" koanf:"gpu_class"` TP int `json:"tp,omitempty" koanf:"tp"` PP int `json:"pp,omitempty" koanf:"pp"` // GPUsPerNode defaults to TP for a single-node profile. GPUsPerNode int `json:"gpus_per_node,omitempty" koanf:"gpus_per_node"` // Nodes above 1 makes this a multi-node profile, which requires a fabric. Nodes int `json:"nodes,omitempty" koanf:"nodes"` Args []string `json:"args,omitempty" koanf:"args"` Env map[string]string `json:"env,omitempty" koanf:"env"` // MinMemoryGB is the host RAM this profile needs to run. // // The number that matters is the model's working set: the weights, plus the // KV cache for the whole context pool, plus the engine's own overhead. For // a GPU profile that lives in the cards and this is small; for a CPU // profile it is the whole thing, and getting it wrong means an OOM on a // node that reported itself healthy. // // Zero means unconstrained, which is every profile that existed before // this. A node that does not report its memory is not excluded either — // an older agent must not have its nodes quietly emptied by an upgrade of // the control plane. MinMemoryGB int `json:"min_memory_gb,omitempty" koanf:"min_memory_gb"` // CPUCores is the cores a replica of a profile with no GPU is given: // reserved out of its node's CPUs by placement, as memory is, and what // its engine computes on (llama.cpp's --threads). // // Reserved because an engine takes every core it is told about, and two // on one node that each think they have all of them spin against each // other: on an 8-vCPU node, two 7B replicas fell from 10 tokens a second // to under one each. Zero reserves nothing and leaves the thread count to // the node, which is every profile from before this. CPUCores int `json:"cpu_cores,omitempty" koanf:"cpu_cores"` // MinDiskGB is the free space a node needs before it can hold these // weights. // // Only checked when the node does not already have them: a node holding // the file needs no room for it, and requiring the space twice would keep // a model off the one node best placed to run it. // // Declared rather than derived, because the size of a model that has never // been fetched anywhere is not knowable from here — and a wrong guess // fails late, halfway through a download, which is the failure this exists // to prevent. Zero means unconstrained, and a node that reports no storage // is not excluded: unknown is not full. MinDiskGB int `json:"min_disk_gb,omitempty" koanf:"min_disk_gb"` // NodeSelector restricts this profile to nodes carrying every one of these // labels, with these values. // // A CPU profile otherwise fits everywhere, because there is no card to be // free — which is right for a 0.5B model and wrong for a 14B one. Nothing // in the inventory says how much memory a node has, so the operator says // which nodes are meant for a model instead of the scheduler inferring it. // Agents have reported labels since the beginning; this is what reads them. NodeSelector map[string]string `json:"node_selector,omitempty" koanf:"node_selector"` // PowerWatts is what one replica of this profile draws, in watts, split // evenly over its nodes, counted against the power cap of every rack, // PDU or other domain its nodes are in (facility.budgets in server.yaml). // Zero derives it: each card at the power limit it reports, each // reserved core at facility.cpu_core_watts. Only matters where a domain // has a cap. PowerWatts float64 `json:"power_watts,omitempty" koanf:"power_watts"` // KeepWithin keeps a multi-node replica's nodes inside one rack, row, // room or site, read from their llmgw.io/rack (and so on) labels: for a // fabric whose leaves do not reach across racks, or cables that do not. // Nodes without the label are passed over for such a profile. Empty is // no constraint. KeepWithin string `json:"keep_within,omitempty" koanf:"keep_within"` // Weight is the router's capacity weight. It should come from ctl bench // rather than a guess (plan §6.2.1). Weight float64 `json:"weight,omitempty" koanf:"weight"` // Priority orders profiles during placement; highest that fits wins. Priority int `json:"priority,omitempty" koanf:"priority"` // StartupTimeout bounds model loading. A 70B model across two nodes takes // tens of minutes, so this is per profile rather than global. StartupTimeout time.Duration `json:"startup_timeout,omitempty" koanf:"startup_timeout"` // KVEvents has vLLM publish what its prefix cache holds, and the router // route this model's requests by it: to the replica holding the most of // a prompt's leading blocks (plan §7.5 tier 2). vLLM only. // // Off by default, because it costs something on every request: the // router asks a replica to tokenize the prompt before placing it, a // round trip and the chat template run twice. It pays where prompts // share long prefixes that the approximate tree guesses wrong — many // replicas, conversations that move between them, evictions. The engine // is started with sha256_cbor block hashing and a fixed PYTHONHASHSEED, // which the router's hashing assumes; arguments overriding either leave // the index empty, and the agent says so in its log. KVEvents bool `json:"kv_events,omitempty" koanf:"kv_events"` } // CPUOnly reports whether this profile needs no GPUs. // // Naming no GPU class is how a profile says so, and only engines that can // actually serve without a card are allowed to leave it out — otherwise an // omitted class would quietly become a replica placed on a machine with no // hardware to run it. func (p Profile) CPUOnly() bool { return p.GPUClass == "" && cpuCapable(p.Engine) } // cpuCapable reports whether an engine can serve a model without a GPU. // // llama.cpp is the one that can, and it is what makes a cluster with no cards // testable end to end. func cpuCapable(engine string) bool { return engine == "llamacpp" } // Model is a catalog entry as clients see it. type Model struct { Name string `json:"name,omitempty" koanf:"name"` Aliases []string `json:"aliases,omitempty" koanf:"aliases"` // Description is what this model is for, in a line. // // Written by whoever added it, because nobody else can say it. A registry // offers a pipeline tag and a wall of README, and neither answers the // question a person picking from a list is asking: which of these nine // names should I send this request to. // // It reaches clients through /v1/models, which is the only view most of // them ever see. A gateway serving nine models by name alone makes that // choice by guesswork. Description string `json:"description,omitempty" koanf:"description"` // Source is the weights path or a HuggingFace id. Source string `json:"source,omitempty" koanf:"source"` Profiles []Profile `json:"profiles,omitempty" koanf:"profiles"` Scaling Scaling `json:"scaling,omitempty" koanf:"scaling"` // Class is how much this model's capacity is worth against another's, // when there is not enough for both: critical, standard or opportunistic. // Empty is standard, or opportunistic for a model loaded on demand. See // EffectiveClass. Class string `json:"class,omitempty" koanf:"class"` // Owner is the part of the tenancy tree whose capacity quota this model // counts against, written org:ID, team:ID or project:ID. Empty is shared: // held by the cluster, counted against nobody. Owner string `json:"owner,omitempty" koanf:"owner"` // External is a model served by someone else — an OpenAI-compatible API // outside the cluster — rather than by replicas placed here. Such a model // has no profiles and nothing is placed for it; the router sends its // requests to the provider. Most useful named in another model's // fallbacks: used only when the cluster has no capacity left. Its // prompts leave the cluster, so which keys may name it is worth an // allowlist. External *External `json:"external,omitempty" koanf:"external"` // ComfyUI is how a model whose profiles run engine comfyui makes its // pictures: the workflow, and the file it loads. ComfyUI *ComfyUI `json:"comfyui,omitempty" koanf:"comfyui"` // Adapters are LoRA adapters served on this model's replicas, each asked // for by its own name as though it were a model. Every replica loads // every adapter, applying one only to a request that names it — unless // MaxLoadedAdapters is set. Adapters []Adapter `json:"adapters,omitempty" koanf:"adapters"` // MaxLoadedAdapters, above zero, has each replica hold at most this many // adapters at once, loaded when a request first asks for one and unloaded // least recently used first to make room. The router sends a request for // an adapter to a replica that already holds it while that one has room // to take it. vLLM profiles only: llama-server loads adapters at start // and cannot load one afterwards. Zero loads every adapter on every // replica at start. MaxLoadedAdapters int `json:"max_loaded_adapters,omitempty" koanf:"max_loaded_adapters"` Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"` // Canary sends a share of this model's requests to another catalog // model — the same model at new weights — to compare the two before // moving to it (`llmgw ctl model canary`). Canary *Canary `json:"canary,omitempty" koanf:"canary"` // Eval is a quality suite new weights must pass before they replace // this model's: `llmgw ctl model update` and `model promote` run it // against them first, and stop below its score. Eval *EvalGate `json:"eval,omitempty" koanf:"eval"` // RoutingProfile selects how the router scores endpoints for this model. RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"` SLO SLO `json:"slo,omitempty" koanf:"slo"` // ContextLength is how many tokens one request may use, prompt and // completion together, and MaxOutput bounds the completion alone. Zero // means unknown, which is what every model was until now. // // The gateway knew neither, so an over-long prompt was not refused: it was // admitted, given a replica, and failed at the engine, and a client sizing // its prompts from /v1/models had nothing to read. Worse, the number is not // the one an operator writes in the engine's arguments — llama.cpp divides // --ctx-size across its parallel slots, so a pool of 8192 across four slots // is 2048 a request, and nothing said so. ContextLength int `json:"context_length,omitempty" koanf:"context_length"` MaxOutput int `json:"max_output,omitempty" koanf:"max_output"` // Capabilities is what this model can do beyond plain completion. Capabilities Capabilities `json:"capabilities,omitempty" koanf:"capabilities"` // QueryPrefix is put before a question an embedding model embeds for a // knowledge-base search; documents are embedded without it. Models // trained with an instruction on the query side want it, Qwen3-Embedding // "Instruct: Given a web search query, retrieve relevant passages that // answer the query\nQuery: " for one. QueryPrefix string `json:"query_prefix,omitempty" koanf:"query_prefix"` // Role "guard" is a model that judges the gateway's traffic for the // guardrails rather than serving clients: hidden from /v1/models and the // chat page, refused to any key that names it, critical by default, and // always running. Guard says how its answers are read. Role string `json:"role,omitempty" koanf:"role"` Guard *GuardModel `json:"guard,omitempty" koanf:"guard"` // Origin is the registry repository the weights came from, at which // commit, and each file's checksum there. Written when a model is served // from the registry; for one that was not, worked out from the mirror's // record of where it fetched the file. It is what lets the control plane // say that the repository has published different weights since, and // move the model to them. Origin *Origin `json:"origin,omitempty" koanf:"origin"` } // RoleGuard is a judge model's role. const RoleGuard = "guard" // IsGuard reports whether a model judges traffic rather than serving it. func (m Model) IsGuard() bool { return m.Role == RoleGuard } // GuardModel is how a judge model's answers are read. type GuardModel struct { // Format is qwen3guard (Qwen3Guard-Gen: "Safety: Safe", "Unsafe" or // "Controversial", and the categories), llamaguard3 (Meta's Llama Guard // 3: "safe", or "unsafe" and the categories), shieldgemma (Google's // ShieldGemma: Yes or No), or json ({"safe": false, "categories": // ["..."]}). Format string `json:"format" koanf:"format"` // Categories the model is asked about, for shieldgemma — each a policy // sentence — and the names reported for llamaguard3's S1, S2, …. Categories []string `json:"categories,omitempty" koanf:"categories"` } // Origin is where a model's weights were published. type Origin struct { // Repo is the repository, owner/name. Repo string `json:"repo" koanf:"repo"` // Revision is the commit the files were fetched at. Revision string `json:"revision" koanf:"revision"` // Files are the weights fetched from it, with what the registry // published as each one's sha256. A new commit that leaves them alone — // a README edited — is not an update. Files []OriginFile `json:"files,omitempty" koanf:"files"` // Previous is what the last update replaced, kept so it can be gone back // to while its files are still on the mirror. Previous *PreviousOrigin `json:"previous,omitempty" koanf:"previous"` } // OriginFile is one weights file of an origin. type OriginFile struct { Path string `json:"path" koanf:"path"` SHA256 string `json:"sha256,omitempty" koanf:"sha256"` Bytes int64 `json:"bytes,omitempty" koanf:"bytes"` } // PreviousOrigin is a model's source before an update, and where it came from. type PreviousOrigin struct { Source string `json:"source" koanf:"source"` Revision string `json:"revision" koanf:"revision"` Files []OriginFile `json:"files,omitempty" koanf:"files"` } // ShortRevision is a commit as a person writes it. func ShortRevision(rev string) string { if len(rev) > 8 { return rev[:8] } return rev } // Capabilities describes what a client may ask a model for. // // Declared by the operator rather than discovered, because there is no way to // ask an engine what its model can do. That makes these claims, and a wrong // claim is worse than none — which is why `ctl engine-check` verifies tool // calling against a running engine rather than trusting this. // // The reason it exists: the gateway forwards a tools array to any model, and a // model that cannot use them answers in prose. Every OpenAI-compatible client // then behaves as though the model is broken, and the only way to find out was // to try it. That cost an afternoon. type Capabilities struct { Tools bool `json:"tools,omitempty" koanf:"tools"` Vision bool `json:"vision,omitempty" koanf:"vision"` Embeddings bool `json:"embeddings,omitempty" koanf:"embeddings"` // Images is a model that makes pictures, asked for at // /v1/images/generations. Implied by an external model whose API is // ComfyUI's. Images bool `json:"images,omitempty" koanf:"images"` } // MakesImages reports whether a model answers /v1/images/generations. func (m Model) MakesImages() bool { return m.Capabilities.Images || m.ServesComfyUI() } // EngineComfyUI is the engine that runs ComfyUI on a node. const EngineComfyUI = "comfyui" // ServesComfyUI reports whether a model is made by ComfyUI, on the // cluster's own nodes or on a server outside it. func (m Model) ServesComfyUI() bool { if m.External != nil { return m.External.API == APIComfyUI } return len(m.Profiles) > 0 && m.Profiles[0].Engine == EngineComfyUI } // ComfyUI is a ComfyUI model's workflow and checkpoint. type ComfyUI struct { // Workflow is a built-in's name (flux-schnell, the default, or // flux-dev) or a JSON file on the control plane, in ComfyUI's API // format. Workflow string `json:"workflow,omitempty" koanf:"workflow"` // Checkpoint is the file the workflow's {{checkpoint}} names: the // source's own name unless set, which a source that is a set of files // must. Checkpoint string `json:"checkpoint,omitempty" koanf:"checkpoint"` } // ComfyWorkflow is what a ComfyUI model's replicas are sent: the workflow // as the catalog names it, and the checkpoint it loads. func (m Model) ComfyWorkflow() (workflow, checkpoint string) { if m.External != nil { return m.External.Workflow, m.External.Model } if m.ComfyUI != nil { workflow, checkpoint = m.ComfyUI.Workflow, m.ComfyUI.Checkpoint } if checkpoint == "" && !strings.HasSuffix(m.Source, "/") { checkpoint = path.Base(strings.TrimPrefix(m.Source, "hf:")) } return workflow, checkpoint } // checkWorkflow refuses a workflow that is neither a built-in nor a file. func checkWorkflow(model, w string) error { // A file is read where the router runs; a name is checked here. if w != "" && !strings.ContainsAny(w, "/.") && !slices.Contains(comfy.Builtins(), w) { return fmt.Errorf("model %q: no built-in workflow %q (there are %s; or name a JSON file)", model, w, strings.Join(comfy.Builtins(), ", ")) } return nil } func (m Model) validateComfyUI() error { comfyProfiles := 0 for _, p := range m.Profiles { if p.Engine == EngineComfyUI { comfyProfiles++ } } if comfyProfiles == 0 { if m.ComfyUI != nil { return fmt.Errorf("model %q: comfyui: is for a model whose profiles run engine comfyui", m.Name) } return nil } if comfyProfiles != len(m.Profiles) { return fmt.Errorf("model %q: every profile runs engine comfyui or none does: "+ "a request for pictures cannot be answered by an engine that writes text", m.Name) } if len(m.Adapters) > 0 { return fmt.Errorf("model %q: ComfyUI serves no adapters", m.Name) } var w string if m.ComfyUI != nil { w = m.ComfyUI.Workflow } if err := checkWorkflow(m.Name, w); err != nil { return err } if _, ckpt := m.ComfyWorkflow(); ckpt == "" { return fmt.Errorf("model %q: its source is a set of files, so comfyui.checkpoint must name the one the workflow loads", m.Name) } return nil } // Catalog is the set of models the cluster should serve. type Catalog struct { models map[string]Model aliases map[string]string // adapters maps an adapter's name to the model it runs on. adapters map[string]string order []string } // New builds a catalog, rejecting definitions that would fail confusingly later. func New(models []Model) (*Catalog, error) { c := &Catalog{ models: make(map[string]Model, len(models)), aliases: map[string]string{}, adapters: map[string]string{}, } for i := range models { m := models[i].normalize() if err := m.validate(); err != nil { return nil, err } if _, dup := c.models[m.Name]; dup { return nil, fmt.Errorf("model %q is declared twice", m.Name) } c.models[m.Name] = m c.order = append(c.order, m.Name) } // Aliases share a namespace with names. A collision would silently shadow a // model at request time, so it has to fail at load. for _, name := range c.order { for _, a := range c.models[name].Aliases { if _, clash := c.models[a]; clash { return nil, fmt.Errorf("model %q: alias %q collides with another model name", name, a) } if prev, clash := c.aliases[a]; clash { return nil, fmt.Errorf("alias %q is claimed by both %q and %q", a, prev, name) } c.aliases[a] = name } } // Adapters are named like models, so they share the namespace too. for _, name := range c.order { for _, a := range c.models[name].Adapters { _, isModel := c.models[a.Name] _, isAlias := c.aliases[a.Name] if _, isAdapter := c.adapters[a.Name]; isModel || isAlias || isAdapter { return nil, fmt.Errorf("model %q: adapter %q collides with a model, an alias or another adapter", name, a.Name) } c.adapters[a.Name] = name } } // Fallbacks are resolved now rather than at request time: a typo in a // fallback should stop a deployment, not surface during an outage when the // fallback is finally needed. for _, name := range c.order { for _, f := range c.models[name].Fallbacks { if _, ok := c.Resolve(f); !ok { return nil, fmt.Errorf("model %q: fallback %q is not in the catalog", name, f) } } } // A canary is another model in the catalog, a candidate for this one's // place: not itself, not a judge, not a canary of its own, and the // candidate of one model only. canaryOf := map[string]string{} for _, name := range c.order { cn := c.models[name].Canary if cn == nil { continue } target, ok := c.Resolve(cn.Model) switch { case !ok: return nil, fmt.Errorf("model %q: canary %q is not in the catalog", name, cn.Model) case target.Name == name: return nil, fmt.Errorf("model %q: a model cannot be its own canary", name) case target.IsGuard(): return nil, fmt.Errorf("model %q: canary %q is a guard model", name, cn.Model) case target.Canary != nil: return nil, fmt.Errorf("model %q: canary %q has a canary of its own", name, cn.Model) case canaryOf[target.Name] != "": return nil, fmt.Errorf("model %q: %q is already the canary of %q", name, cn.Model, canaryOf[target.Name]) case cn.Percent < 0 || cn.Percent > 100: return nil, fmt.Errorf("model %q: canary percent %d is not 0 to 100", name, cn.Percent) } canaryOf[target.Name] = name } return c, nil } // Canary is a share of a model's requests sent to a candidate, and when to // give up on it. type Canary struct { // Model is the candidate: a catalog model, usually this one at newer // weights, which clients cannot name themselves. Model string `json:"model" koanf:"model"` // Percent of requests go to it, 0 to 100. Percent int `json:"percent" koanf:"percent"` // The candidate is taken out — rolled back — when, over at least // MinRequests of its own, its error rate is MaxErrorIncrease above // this model's (0.05 is five points), or its mean time to the first // token is MaxTTFTRatio times this model's. Zero is 50, 0.05 and 1.5. MinRequests int `json:"min_requests,omitempty" koanf:"min_requests"` MaxErrorIncrease float64 `json:"max_error_increase,omitempty" koanf:"max_error_increase"` MaxTTFTRatio float64 `json:"max_ttft_ratio,omitempty" koanf:"max_ttft_ratio"` } // EvalGate is a quality suite a model's new weights must pass. type EvalGate struct { // Suite is a suite's name in control.eval_dir, or one the binary // carries (arithmetic, json-format). Suite string `json:"suite" koanf:"suite"` // MinScore is the score, 0 to 1, to pass at; zero is the suite's own // threshold. MinScore float64 `json:"min_score,omitempty" koanf:"min_score"` // Judge grades the suite's rubric cases, instead of the judge it names. Judge string `json:"judge,omitempty" koanf:"judge"` } // Limits are the rollback thresholds with their defaults filled in. func (c Canary) Limits() (minRequests int, maxErrorIncrease, maxTTFTRatio float64) { minRequests, maxErrorIncrease, maxTTFTRatio = c.MinRequests, c.MaxErrorIncrease, c.MaxTTFTRatio if minRequests <= 0 { minRequests = 50 } if maxErrorIncrease <= 0 { maxErrorIncrease = 0.05 } if maxTTFTRatio <= 0 { maxTTFTRatio = 1.5 } return } // CanaryOf is the model whose canary name is, or "". func (c *Catalog) CanaryOf(name string) string { for _, n := range c.order { if cn := c.models[n].Canary; cn != nil { if t, ok := c.Resolve(cn.Model); ok && t.Name == name { return n } } } return "" } func (m Model) normalize() Model { if m.Scaling.Min < 0 { m.Scaling.Min = 0 } // Max 0 used to be an omission to fill in, because a model with no maximum // could not run. It is now also a statement: with min 0 it means configured // and deliberately not running, which is a different and useful thing from // "load it when wanted". Filling it in would silently turn every switched- // off model into an on-demand one — including the ones already written that // way in a running cluster's configuration. if m.Scaling.Max == 0 && m.Scaling.Min > 0 { m.Scaling.Max = m.Scaling.Min } for i := range m.Profiles { p := &m.Profiles[i] if p.Engine == "" { p.Engine = "vllm" } if p.TP <= 0 { p.TP = 1 } if p.Nodes <= 0 { p.Nodes = 1 } // tp on a llama.cpp GPU profile, written before gpus_per_node meant // anything for it, is read as the width it was meant as: N cards of // one node. Refusing it would have dropped a catalog's model on the // upgrade that made the width work. Only both given, disagreeing, is // refused below, as ambiguous. if p.Engine == "llamacpp" && p.GPUClass != "" && (p.TP > 1 || p.PP > 1) && p.Nodes == 1 { if n := p.TP * max(p.PP, 1); p.GPUsPerNode == 0 || p.GPUsPerNode == n { p.GPUsPerNode, p.TP, p.PP = n, 1, 0 } } switch { case p.CPUOnly(): // Nothing to divide up: this profile asks for no cards at all. p.GPUsPerNode = 0 case p.GPUsPerNode <= 0: // The common case: a single-node profile uses TP GPUs on one node. p.GPUsPerNode = p.TP / p.Nodes if p.GPUsPerNode <= 0 { p.GPUsPerNode = 1 } } if p.StartupTimeout <= 0 { p.StartupTimeout = 20 * time.Minute } if p.Weight <= 0 { p.Weight = 1 } } // A Hugging Face repository page for a model vLLM or SGLang runs is the // repository it names: the engine downloads a repository id itself, // where the page was fetched as one file of HTML and failed at start. if repo, ok := repositoryPage(m.Source); ok && len(m.Profiles) > 0 && !slices.ContainsFunc(m.Profiles, func(p Profile) bool { return p.Engine == "llamacpp" }) { m.Source = repo } return m } func (m Model) validate() error { if m.Name == "" { return fmt.Errorf("a model has no name") } if m.Source == "" { return fmt.Errorf("model %q: source is required", m.Name) } if repo, ok := repositoryPage(m.Source); ok { // A page, not weights: a node fetches the HTML as one file and the // engine fails at start, on a path that looks right. For vLLM it was // read as the repository id when the model was made; only a model // llama.cpp runs is left here, which needs a file. return fmt.Errorf("model %q: source %s is a Hugging Face repository page, not a file; "+ "llama.cpp needs a .gguf: mirror one and use its source (repository %s)", m.Name, m.Source, repo) } if err := validateAdapters(m); err != nil { return err } if m.External != nil { if err := m.External.validate(m.Name, len(m.Profiles)); err != nil { return err } } else if len(m.Profiles) == 0 { return fmt.Errorf("model %q: at least one profile is required", m.Name) } if err := m.validateComfyUI(); err != nil { return err } if m.Owner != "" && !validOwner(m.Owner) { return fmt.Errorf("model %q: owner %q is not org:ID, team:ID or project:ID", m.Name, m.Owner) } if !validClass(m.Class) { return fmt.Errorf("model %q: class %q is not one of critical, standard, opportunistic", m.Name, m.Class) } if g := m.Eval; g != nil { if strings.TrimSpace(g.Suite) == "" { return fmt.Errorf("model %q: eval names no suite", m.Name) } if g.MinScore < 0 || g.MinScore > 1 { return fmt.Errorf("model %q: eval min_score %v is not 0 to 1", m.Name, g.MinScore) } } switch m.Role { case "": if m.Guard != nil { return fmt.Errorf("model %q: guard: is for a model with role: guard", m.Name) } case RoleGuard: if m.Guard == nil { return fmt.Errorf("model %q: a guard model needs guard.format: %s", m.Name, strings.Join(guard.Formats, ", ")) } if !guard.KnownFormat(m.Guard.Format) { return fmt.Errorf("model %q: guard.format %q is not one of %s", m.Name, m.Guard.Format, strings.Join(guard.Formats, ", ")) } if m.Scaling.OnDemand() { // A judge loaded on demand makes the first request of every // idle period wait for it, or be judged by nothing. return fmt.Errorf("model %q: a guard model must always run: scaling.min 1 or more", m.Name) } if m.IsExternal() { return fmt.Errorf("model %q: a guard model judges in the cluster; it cannot be external", m.Name) } default: return fmt.Errorf("model %q: role %q is not guard", m.Name, m.Role) } for i, w := range m.Scaling.Schedule { if err := w.validate(); err != nil { return fmt.Errorf("model %q: scaling.schedule[%d]: %w", m.Name, i, err) } } if m.Scaling.Prewarm < 0 || m.Scaling.Prewarm > 12*time.Hour { return fmt.Errorf("model %q: scaling.prewarm is between 0 and 12h", m.Name) } if m.Scaling.Max < m.Scaling.Min { return fmt.Errorf("model %q: scaling max %d is below min %d", m.Name, m.Scaling.Max, m.Scaling.Min) } seen := map[string]struct{}{} for _, p := range m.Profiles { if p.Name == "" { return fmt.Errorf("model %q: a profile has no name", m.Name) } if _, dup := seen[p.Name]; dup { return fmt.Errorf("model %q: profile %q is declared twice", m.Name, p.Name) } seen[p.Name] = struct{}{} for k := range p.Env { if bad, why := engines.UnsafeEnv(k); bad { return fmt.Errorf("model %q profile %q: env %s is not allowed: %s", m.Name, p.Name, k, why) } } if err := engines.UnsafeArgs(p.Engine, p.Args); err != nil { return fmt.Errorf("model %q profile %q: %w", m.Name, p.Name, err) } if p.KVEvents && p.Engine != "vllm" { return fmt.Errorf("model %q profile %q: kv_events is vLLM's; engine %q publishes no cache events", m.Name, p.Name, p.Engine) } if p.PowerWatts < 0 { return fmt.Errorf("model %q profile %q: power_watts %v is negative", m.Name, p.Name, p.PowerWatts) } if p.KeepWithin != "" { if _, err := facility.ParseLevel(p.KeepWithin); err != nil { return fmt.Errorf("model %q profile %q: keep_within: %w", m.Name, p.Name, err) } } if p.GPUClass == "" && !cpuCapable(p.Engine) { return fmt.Errorf("model %q profile %q: gpu_class is required for engine %q", m.Name, p.Name, p.Engine) } if p.CPUOnly() { if p.Nodes > 1 { return fmt.Errorf("model %q profile %q: a CPU profile cannot span %d nodes", m.Name, p.Name, p.Nodes) } continue } // A MIG instance is a device of its own to CUDA, but a process sees // only one, and instances have no NVLink or peer-to-peer between // them: a group of them is neither tensor- nor layer-parallel, it is // a replica that fails at start. One instance, one node, refused here // rather than placed and found out. if discovery.IsMIGClass(p.GPUClass) && (p.GPUsPerNode > 1 || p.Nodes > 1 || p.TP > 1 || p.PP > 1) { return fmt.Errorf("model %q profile %q: %s is a MIG instance, and a replica runs on exactly one: "+ "a process can open only one MIG device, and instances have no NVLink between them. "+ "Give it tp 1 and one GPU on one node, or use a whole-card class or a larger MIG profile", m.Name, p.Name, p.GPUClass) } // llama.cpp on GPUs splits a model across one node's cards by layer, // and how many is gpus_per_node. Not tp: llama.cpp has no tensor // parallelism, and a tp it was given was a width nothing placed — the // replica stayed unplaced, or ran on one card while four were meant. if p.Engine == "llamacpp" { switch { case p.Nodes > 1: return fmt.Errorf("model %q profile %q: llama.cpp does not span nodes; use vllm or sglang for a %d-node replica", m.Name, p.Name, p.Nodes) case p.TP > 1 || p.PP > 1: return fmt.Errorf("model %q profile %q: llama.cpp splits a model across a node's GPUs by layer, and "+ "tp %d does not agree with gpus_per_node %d: give gpus_per_node alone", m.Name, p.Name, max(p.TP, 1)*max(p.PP, 1), p.GPUsPerNode) } continue } // TensorRT-LLM spans machines only under MPI started from one of them, // which the agents, each starting its own node, do not do; and it // loads a Hugging Face checkpoint, not llama.cpp's format. if p.Engine == "trtllm" { if p.Nodes > 1 { return fmt.Errorf("model %q profile %q: trtllm runs a replica on one node; "+ "use vllm or sglang for a %d-node replica", m.Name, p.Name, p.Nodes) } if strings.HasSuffix(strings.ToLower(strings.TrimSuffix(m.Source, "/")), ".gguf") { return fmt.Errorf("model %q profile %q: trtllm loads a Hugging Face checkpoint, not a .gguf; "+ "serve a GGUF with llamacpp", m.Name, p.Name) } } // The arithmetic has to add up, or the engine is handed a shard spec it // cannot satisfy and fails minutes later during model loading. if want := p.GPUsPerNode * p.Nodes; want != p.TP*max(p.PP, 1) { return fmt.Errorf("model %q profile %q: %d nodes x %d GPUs does not match TP %d x PP %d", m.Name, p.Name, p.Nodes, p.GPUsPerNode, p.TP, max(p.PP, 1)) } } return nil } // Resolve maps a client-supplied name or alias onto a model. // ResolveAdapter finds an adapter by its name: the model it runs on, the // adapter, and its index among the model's adapters — which is the id the // engine knows it by. func (c *Catalog) ResolveAdapter(name string) (Model, Adapter, int, bool) { base, ok := c.adapters[name] if !ok { return Model{}, Adapter{}, 0, false } m := c.models[base] for i, a := range m.Adapters { if a.Name == name { return m, a, i, true } } return Model{}, Adapter{}, 0, false } func (c *Catalog) Resolve(name string) (Model, bool) { if m, ok := c.models[name]; ok { return m, true } if canonical, ok := c.aliases[name]; ok { return c.models[canonical], true } return Model{}, false } // Models returns every model, in declaration order. func (c *Catalog) Models() []Model { out := make([]Model, 0, len(c.order)) for _, n := range c.order { out = append(out, c.models[n]) } return out } // Names returns the canonical model names, sorted. func (c *Catalog) Names() []string { out := slices.Clone(c.order) slices.Sort(out) return out } // AliasesOf returns the alternative names for a model. func (c *Catalog) AliasesOf(name string) []string { m, ok := c.Resolve(name) if !ok { return nil } return slices.Clone(m.Aliases) } // PlacementProfiles converts a model's profiles into placement's view. func (m Model) PlacementProfiles() []placement.Profile { out := make([]placement.Profile, 0, len(m.Profiles)) for _, p := range m.Profiles { out = append(out, placement.Profile{ Name: p.Name, Engine: p.Engine, EngineVersion: p.EngineVersion, GPUClass: p.GPUClass, GPUsPerNode: p.GPUsPerNode, Nodes: p.Nodes, Priority: p.Priority, NodeSelector: p.NodeSelector, MinMemoryGB: p.MinMemoryGB, MinDiskGB: p.MinDiskGB, CPUCores: p.CPUCores, PowerWatts: p.PowerWatts, KeepWithin: facility.Level(strings.ToLower(strings.TrimSpace(p.KeepWithin))), // The model's source, so placement can prefer nodes that already // hold the weights rather than one that must first move them. Source: m.Source, }) } return out } // EngineProfile converts a catalog profile into the engine adapter's view. func (m Model) EngineProfile(name string) (engines.Profile, bool) { for _, p := range m.Profiles { if p.Name != name { continue } return engines.Profile{ Name: p.Name, Engine: p.Engine, GPUClass: p.GPUClass, TP: p.TP, PP: p.PP, GPUsPerNode: p.GPUsPerNode, Nodes: p.Nodes, Args: slices.Clone(p.Args), Env: p.Env, ModelPath: m.Source, ServedModelName: m.Name, Adapters: m.engineAdapters(), KVEvents: p.KVEvents, }.WithRuntimeLoRA(m.MaxLoadedAdapters), true } return engines.Profile{}, false } // KVIndex reports whether the router should route this model by the KV // cache its replicas report: whether any profile publishes events. A replica // of another profile is simply one the index has not heard from. func (m Model) KVIndex() bool { for _, p := range m.Profiles { if p.KVEvents { return true } } return false } // ProfileByName returns one of a model's profiles. func (m Model) ProfileByName(name string) (Profile, bool) { for _, p := range m.Profiles { if p.Name == name { return p, true } } return Profile{}, false } // String renders the catalog for `ctl models`. func (c *Catalog) String() string { var b strings.Builder for _, m := range c.Models() { fmt.Fprintf(&b, "%s (min %d, max %d)\n", m.Name, m.Scaling.Min, m.Scaling.Max) for _, p := range m.Profiles { fmt.Fprintf(&b, " %-20s %-10s %-12s tp=%d nodes=%d priority=%d\n", p.Name, p.Engine, p.GPUClass, p.TP, p.Nodes, p.Priority) } } return b.String() } // Simulated reports whether this model produces synthetic text rather than // inference output. // // Derived from the profiles rather than configured, because an operator who // added a sim profile and forgot the flag would be shipping placeholder text to // users under a real model's name. A model counts as simulated only when every // profile is simulated: one real profile means a request can land on a real // engine, and a model that sometimes answers for real must not be labelled as // though it never does. func (m Model) Simulated() bool { if len(m.Profiles) == 0 { return false } for _, p := range m.Profiles { if p.Engine != "sim" { return false } } return true } // Fingerprint identifies a profile's contents. // // The name is not enough. A profile can be edited — a memory requirement // raised, a node label added, an engine argument changed — and keep its name, // so a replica running the old definition is indistinguishable from one // running the new. The reconciler does not move a running replica when its // profile changes, which is right: churning a replica that is serving costs // more than the imperfection. What was missing is any way to see that it had // happened, and the answer to "why is this one on the wrong node" was in // somebody's memory of what they edited an hour ago. func (p Profile) Fingerprint() string { b, err := json.Marshal(p) if err != nil { // A profile that will not marshal cannot be compared, and saying so by // never matching is better than saying two differing ones are the same. return "unmarshalable" } sum := sha256.Sum256(b) return hex.EncodeToString(sum[:8]) } // SuggestCores is the cores to give a replica with no GPU on nodes of these // CPU counts: half of the smallest, so two replicas share any of them without // competing. Zero when no node says, which reserves nothing. func SuggestCores(cpuCounts []int) int { smallest := 0 for _, c := range cpuCounts { if c > 0 && (smallest == 0 || c < smallest) { smallest = c } } if smallest == 0 { return 0 } return max(smallest/2, 1) } // The capacity classes, lowest first. const ( ClassOpportunistic = "opportunistic" ClassStandard = "standard" ClassCritical = "critical" ) func validClass(c string) bool { return c == "" || c == ClassOpportunistic || c == ClassStandard || c == ClassCritical } // EffectiveClass is the class a model is treated as: what it says, or, when // it says nothing, opportunistic for one loaded on demand — it holds capacity // only while asked for, and has always been the first to give it up — and // standard for everything else. func (m Model) EffectiveClass() string { if m.Class != "" { return m.Class } if m.IsGuard() { // Every judged request waits on it. return ClassCritical } if m.Scaling.OnDemand() { return ClassOpportunistic } return ClassStandard } // ClassRank orders classes: a higher rank may take capacity from a lower one. func ClassRank(c string) int { switch c { case ClassCritical: return 3 case ClassStandard, "": return 2 default: return 1 } } func validOwner(o string) bool { for _, p := range []string{"org:", "team:", "project:"} { if strings.HasPrefix(o, p) && len(o) > len(p) { return true } } return false } // External is a provider outside the cluster that serves a model. type External struct { // URL is the provider's base address, without /v1: https://api.openai.com. URL string `json:"url" koanf:"url"` // Model is what the provider calls it; requests are rewritten to it. Model string `json:"model" koanf:"model"` // APIKeyFile holds the provider's key, on the control plane. A file // rather than the key itself, so the key is in neither the catalog, the // store nor the audit log; empty for a provider that asks for none. APIKeyFile string `json:"api_key_file,omitempty" koanf:"api_key_file"` // Peer names the federated llmgw cluster this model is offered by, for // a model that federation added (`llmgw ctl federation add`) rather // than one written by hand. Such a model is named PEER/MODEL, comes and // goes with the peer's own list, and is sent only while the peer is up. Peer string `json:"peer,omitempty" koanf:"peer"` // API is what the provider speaks: OpenAI's, unless this says comfyui — // a ComfyUI server, asked for pictures at /v1/images/generations and // sent the workflow below with the request's prompt in it. Model is then // the checkpoint file the workflow loads, as ComfyUI lists it. API string `json:"api,omitempty" koanf:"api"` // Workflow is the ComfyUI workflow, in its API format: a built-in's name // (flux-schnell, the default, or flux-dev) or a JSON file on the control // plane. Workflow string `json:"workflow,omitempty" koanf:"workflow"` } // APIComfyUI is External.API for a ComfyUI server. const APIComfyUI = "comfyui" // IsPeer reports whether a model is offered by a federated cluster. func (m Model) IsPeer() bool { return m.External != nil && m.External.Peer != "" } func (e *External) validate(model string, profiles int) error { if profiles > 0 { return fmt.Errorf("model %q: an external model is served elsewhere and has no profiles", model) } u, err := url.Parse(e.URL) if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" { return fmt.Errorf("model %q: external url %q is not an http(s) address", model, e.URL) } if strings.TrimSpace(e.Model) == "" { return fmt.Errorf("model %q: external model needs the name the provider knows it by", model) } switch e.API { case "", "openai": if e.Workflow != "" { return fmt.Errorf("model %q: a workflow is for api: comfyui", model) } case APIComfyUI: if e.Peer != "" { return fmt.Errorf("model %q: a federated model speaks OpenAI's API, not ComfyUI's", model) } if err := checkWorkflow(model, e.Workflow); err != nil { return err } default: return fmt.Errorf("model %q: external api %q is not openai or comfyui", model, e.API) } return nil } // IsExternal reports whether a model is served outside the cluster. func (m Model) IsExternal() bool { return m.External != nil } // Adapter is a LoRA adapter served on a model's replicas. type Adapter struct { // Name is what clients ask for, like a model name. Name string `json:"name" koanf:"name"` // Source is the adapter: a GGUF file for llama.cpp; for vLLM a PEFT // adapter directory (a mirror version ending in "/", a path on the node, // or a Hugging Face repository id). Fetched by each node as the model's // weights are. Source string `json:"source" koanf:"source"` // Scale is how strongly it is applied; zero takes 1. Scale float64 `json:"scale,omitempty" koanf:"scale"` // Owner is the part of the tenancy tree the adapter belongs to — // org:ID, team:ID, project:ID, user:ID or key:ID — and only callers in // it see it: to anyone else it is a model that does not exist. An // adapter trained on an organisation's documents carries some of what // they say in its weights. Empty is every caller the model allows. Owner string `json:"owner,omitempty" koanf:"owner"` } // ValidAdapterOwner checks an adapter's owner is scope:ID. func ValidAdapterOwner(owner string) error { if owner == "" { return nil } scope, id, ok := strings.Cut(owner, ":") switch { case !ok || id == "": return fmt.Errorf("an adapter's owner is scope:ID, %q is not", owner) case scope != "org" && scope != "team" && scope != "project" && scope != "user" && scope != "key": return fmt.Errorf("an adapter's owner is an org, team, project, user or key, not %q", scope) } return nil } // EffectiveScale is the scale an adapter is applied at. func (a Adapter) EffectiveScale() float64 { if a.Scale <= 0 { return 1 } return a.Scale } func validateAdapters(m Model) error { if m.MaxLoadedAdapters < 0 { return fmt.Errorf("model %q: max_loaded_adapters cannot be negative", m.Name) } if m.MaxLoadedAdapters > 0 { // Loading on demand is vLLM's runtime LoRA API. llama-server can // only change the scale of adapters it loaded at start, so a cap // there would be a cap on nothing. for _, p := range m.Profiles { if p.Engine != "vllm" { return fmt.Errorf("model %q profile %q: max_loaded_adapters needs vllm profiles; "+ "llama.cpp loads every adapter at start", m.Name, p.Name) } } } if len(m.Adapters) == 0 { return nil } if m.External != nil { return fmt.Errorf("model %q: an external model cannot carry adapters", m.Name) } for _, p := range m.Profiles { // llama.cpp loads adapters at start and applies them per request; // vLLM loads them at start or on demand. The other engines are not // driven either way yet. if p.Engine != "llamacpp" && p.Engine != "vllm" && p.Engine != "sim" { return fmt.Errorf("model %q profile %q: adapters are served by llamacpp and vllm profiles only", m.Name, p.Name) } // The variable is what switches vLLM to loading on demand, and the // router loads adapters only for a model with a cap: set by hand, it // would start replicas holding no adapter that nothing ever loads. if _, set := p.Env[engines.RuntimeLoRAEnv]; set && m.MaxLoadedAdapters == 0 { return fmt.Errorf("model %q profile %q: set max_loaded_adapters rather than %s", m.Name, p.Name, engines.RuntimeLoRAEnv) } } seen := map[string]bool{} for _, a := range m.Adapters { if err := ValidAdapterOwner(a.Owner); err != nil { return fmt.Errorf("model %q adapter %q: %w", m.Name, a.Name, err) } switch { case a.Name == "" || a.Source == "": return fmt.Errorf("model %q: an adapter needs a name and a source", m.Name) case a.Name == m.Name: return fmt.Errorf("model %q: an adapter cannot have the model's own name", m.Name) case seen[a.Name]: return fmt.Errorf("model %q: adapter %q is declared twice", m.Name, a.Name) case a.Scale < 0: return fmt.Errorf("model %q: adapter %q has a negative scale", m.Name, a.Name) case m.MaxLoadedAdapters > 0 && (strings.ContainsAny(a.Name, `/\`) || a.Name == "." || a.Name == ".."): // Loaded on demand, the name is a file's name on the node. return fmt.Errorf("model %q: adapter %q cannot contain a slash when adapters load on demand", m.Name, a.Name) } seen[a.Name] = true } return nil } func (m Model) engineAdapters() []engines.Adapter { var out []engines.Adapter for _, a := range m.Adapters { out = append(out, engines.Adapter{Name: a.Name, Path: a.Source}) } return out } // repositoryPage reports whether source is a Hugging Face repository's page // (https://huggingface.co/org/repo, or one of its tree views) rather than a // file under /resolve/, and the repository's id. func repositoryPage(source string) (string, bool) { for _, host := range []string{"https://huggingface.co/", "http://huggingface.co/", "https://hf.co/"} { rest, ok := strings.CutPrefix(source, host) if !ok { continue } parts := strings.Split(strings.Trim(rest, "/"), "/") if len(parts) < 2 || (len(parts) > 2 && parts[2] == "resolve") { return "", false } return parts[0] + "/" + parts[1], true } return "", false }