diff --git a/BACKLOG.md b/BACKLOG.md index b6e6b20..1429421 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -1328,7 +1328,11 @@ audio and files count as nothing in usage until a per-run price exists through `/v1/workflows/runs` with a 400 for a missing required input before any job is written. -### H2. Profiles that fit the hardware (M) +### H2. Profiles that fit the hardware (M) — built 2026-10-11 +Built as written. A profile by need is left out of placement until the +cluster's cards are known: with no class, placement would have read it as +a CPU profile. A profile's own `source` is loaded under its own file name +by the workflow's `{{checkpoint}}`. - A profile may say `gpu: {min_compute_capability, min_memory_gb}` in place of `gpu_class`. When the reconciler places a model, such a profile stands for every GPU class in the cluster that meets it, at diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a9ef78..e6f72e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,12 @@ APIs change between minors without a deprecation period. - **Environments**: `comfyui.environment` names ComfyUI models that agree on their custom nodes and may share one replica; the catalog refuses one that cannot. +- **Profiles that fit the hardware**: a profile may say `gpu: + {min_compute_capability, min_memory_gb}` instead of `gpu_class`, and + stands for every class in the cluster that meets it, as + `@`; a ComfyUI profile may carry its own `comfyui` + workflow, checkpoint and nodes, and any profile its own `source`. One + catalog entry then runs fp8 where the card has it and fp16 on Turing. - **A licence a machine can match**: `license.id`, `commercial`, `excluded_regions`, `revenue_cap_usd`, `attribution`. - **On-demand models in `/v1/models`**, with `loaded` saying whether one is diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index c604298..6afe658 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -8869,6 +8869,20 @@ } } }, + "catalog.GPUNeed": { + "type": "object", + "description": "GPUNeed is what card a profile needs.", + "properties": { + "min_compute_capability": { + "type": "number", + "description": "MinComputeCapability is the oldest architecture that runs it: 7.5 is Turing, 8.0 Ampere (bf16, flash-attention 2), 8.9 Ada (fp8), 9.0 Hopper." + }, + "min_memory_gb": { + "type": "integer", + "description": "MinMemoryGB is the card's own memory it needs." + } + } + }, "catalog.GuardModel": { "type": "object", "description": "GuardModel is how a judge model's answers are read.", @@ -9227,6 +9241,32 @@ "kv_events": { "type": "boolean", "description": "KVEvents has vLLM publish what its prefix cache holds, and the router route this model's requests by it: to the replica holding the most of a prompt's leading blocks (plan §7.5 tier 2). vLLM only.\n\nOff by default, because it costs something on every request: the router asks a replica to tokenize the prompt before placing it, a round trip and the chat template run twice. It pays where prompts share long prefixes that the approximate tree guesses wrong — many replicas, conversations that move between them, evictions. The engine is started with sha256_cbor block hashing and a fixed PYTHONHASHSEED, which the router's hashing assumes; arguments overriding either leave the index empty, and the agent says so in its log." + }, + "gpu": { + "$ref": "#/components/schemas/catalog.GPUNeed", + "description": "GPU says what card the profile needs, in place of naming its class: it stands for every class in the cluster that meets it. A catalog entry written once then places sensibly on any site's cards." + }, + "comfyui": { + "$ref": "#/components/schemas/catalog.ProfileComfyUI", + "description": "ComfyUI is this profile's own workflow, checkpoint and nodes, over the model's: an fp8 workflow where the card has fp8, fp16 where it has not." + } + } + }, + "catalog.ProfileComfyUI": { + "type": "object", + "description": "ProfileComfyUI is what a ComfyUI profile runs differently from the model: each field set replaces the model's.", + "properties": { + "workflow": { + "type": "string" + }, + "checkpoint": { + "type": "string" + }, + "nodes": { + "type": "array", + "items": { + "$ref": "#/components/schemas/catalog.CustomNode" + } } } }, diff --git a/docs/reference.md b/docs/reference.md index 14b16dd..d487fde 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -264,6 +264,8 @@ rather than repeated. | `priority` | `int` | [Profile](#profile) | | `startup_timeout` | `time.Duration` | [Profile](#profile) | | `kv_events` | `bool` | [Profile](#profile) | +| `gpu` | `*GPUNeed` | [Profile](#profile) | +| `comfyui` | `*ProfileComfyUI` | [Profile](#profile) | | `activation_wait` | `time.Duration` | [Router](#router) | | `scrape_interval` | `time.Duration` | [Router](#router) | | `scrape_timeout` | `time.Duration` | [Router](#router) | @@ -2306,6 +2308,21 @@ is started with sha256_cbor block hashing and a fixed PYTHONHASHSEED, which the router's hashing assumes; arguments overriding either leave the index empty, and the agent says so in its log. +### `gpu` + +`*GPUNeed` + +GPU says what card the profile needs, in place of naming its class: +it stands for every class in the cluster that meets it. A catalog +entry written once then places sensibly on any site's cards. + +### `comfyui` + +`*ProfileComfyUI` + +ComfyUI is this profile's own workflow, checkpoint and nodes, over the +model's: an fp8 workflow where the card has fp8, fp16 where it has not. + ## Router ### `activation_wait` diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index a428769..1673e8f 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -279,6 +279,14 @@ type Profile struct { // which the router's hashing assumes; arguments overriding either leave // the index empty, and the agent says so in its log. KVEvents bool `json:"kv_events,omitempty" koanf:"kv_events"` + + // GPU says what card the profile needs, in place of naming its class: + // it stands for every class in the cluster that meets it. A catalog + // entry written once then places sensibly on any site's cards. + GPU *GPUNeed `json:"gpu,omitempty" koanf:"gpu"` + // ComfyUI is this profile's own workflow, checkpoint and nodes, over the + // model's: an fp8 workflow where the card has fp8, fp16 where it has not. + ComfyUI *ProfileComfyUI `json:"comfyui,omitempty" koanf:"comfyui"` } // CPUOnly reports whether this profile needs no GPUs. @@ -287,7 +295,7 @@ type Profile struct { // actually serve without a card are allowed to leave it out — otherwise an // omitted class would quietly become a replica placed on a machine with no // hardware to run it. -func (p Profile) CPUOnly() bool { return p.GPUClass == "" && cpuCapable(p.Engine) } +func (p Profile) CPUOnly() bool { return p.GPUClass == "" && p.GPU == nil && cpuCapable(p.Engine) } // cpuCapable reports whether an engine can serve a model without a GPU. // @@ -1072,10 +1080,18 @@ func (m Model) validate() error { return fmt.Errorf("model %q profile %q: keep_within: %w", m.Name, p.Name, err) } } - if p.GPUClass == "" && !cpuCapable(p.Engine) { - return fmt.Errorf("model %q profile %q: gpu_class is required for engine %q", + if err := p.validateHardware(m); err != nil { + return err + } + if p.GPUClass == "" && p.GPU == nil && !cpuCapable(p.Engine) { + return fmt.Errorf("model %q profile %q: gpu_class is required for engine %q, or gpu: the card it needs", m.Name, p.Name, p.Engine) } + if p.GPU != nil { + // Its class is known only when it is placed; what can be checked + // of a class waits for then. + continue + } if p.CPUOnly() { if p.Nodes > 1 { return fmt.Errorf("model %q profile %q: a CPU profile cannot span %d nodes", @@ -1186,10 +1202,16 @@ func (c *Catalog) AliasesOf(name string) []string { return slices.Clone(m.Aliases) } -// PlacementProfiles converts a model's profiles into placement's view. +// PlacementProfiles converts a model's profiles into placement's view. A +// profile that names its card by need is left out: with no class it would +// read to placement as one that needs no GPU. PlacementProfilesFor gives +// it the classes it stands for. func (m Model) PlacementProfiles() []placement.Profile { out := make([]placement.Profile, 0, len(m.Profiles)) for _, p := range m.Profiles { + if p.GPU != nil { + continue + } out = append(out, placement.Profile{ Name: p.Name, Engine: p.Engine, @@ -1238,13 +1260,10 @@ func (m Model) WeightSources() []string { // EngineProfile converts a catalog profile into the engine adapter's view. func (m Model) EngineProfile(name string) (engines.Profile, bool) { - for _, p := range m.Profiles { - if p.Name != name { - continue - } + if p, ok := m.ProfileByName(name); ok { env := p.Env - if p.Engine == EngineComfyUI && m.ComfyUI != nil && len(m.ComfyUI.Nodes) > 0 { - nodes, _ := json.Marshal(m.ComfyUI.Nodes) + if nodes := m.ComfyNodesFor(p); p.Engine == EngineComfyUI && len(nodes) > 0 { + nodes, _ := json.Marshal(nodes) env = maps.Clone(p.Env) if env == nil { env = map[string]string{} @@ -1282,13 +1301,23 @@ func (m Model) KVIndex() bool { return false } -// ProfileByName returns one of a model's profiles. +// ProfileByName returns one of a model's profiles, or one a profile that +// names its GPU by need stands for: "@", with the class. func (m Model) ProfileByName(name string) (Profile, bool) { for _, p := range m.Profiles { if p.Name == name { return p, true } } + base, class, ok := strings.Cut(name, "@") + if !ok || class == "" { + return Profile{}, false + } + for _, p := range m.Profiles { + if p.Name == base && p.GPU != nil { + return p.forClass(class), true + } + } return Profile{}, false } diff --git a/internal/control/catalog/hardware.go b/internal/control/catalog/hardware.go new file mode 100644 index 0000000..be83d50 --- /dev/null +++ b/internal/control/catalog/hardware.go @@ -0,0 +1,188 @@ +package catalog + +import ( + "fmt" + "slices" + "strconv" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/placement" +) + +// Profiles that fit the hardware. +// +// A profile names its GPU class — 6000-24g, a100-80g — and placement matches +// it exactly. A catalog entry meant for many sites cannot know their class +// names: the PoC has one Turing card, a customer an A100, another an L40S. So +// a profile may say what card it needs instead — a compute capability, a +// memory — and stands for every class in the cluster that meets it. What +// differs between cards goes on the profile too: its own weights (Source), +// its own workflow and nodes (ComfyUI), its own engine arguments. The model +// then has, say, an fp8 profile for compute 8.9 and up at a higher priority +// and an fp16 one for anything with 16 GB, and each site gets the best its +// cards can run. + +// GPUNeed is what card a profile needs. +type GPUNeed struct { + // MinComputeCapability is the oldest architecture that runs it: 7.5 is + // Turing, 8.0 Ampere (bf16, flash-attention 2), 8.9 Ada (fp8), 9.0 Hopper. + MinComputeCapability float64 `json:"min_compute_capability,omitempty" koanf:"min_compute_capability"` + // MinMemoryGB is the card's own memory it needs. + MinMemoryGB int `json:"min_memory_gb,omitempty" koanf:"min_memory_gb"` +} + +// ProfileComfyUI is what a ComfyUI profile runs differently from the model: +// each field set replaces the model's. +type ProfileComfyUI struct { + Workflow string `json:"workflow,omitempty" koanf:"workflow"` + Checkpoint string `json:"checkpoint,omitempty" koanf:"checkpoint"` + Nodes []CustomNode `json:"nodes,omitempty" koanf:"nodes"` +} + +// GPUKind is one kind of card in the cluster. +type GPUKind struct { + Class string + ComputeCapability float64 + MemoryGB int +} + +// GPUKinds are the kinds of card among gpus, one each, by class. +func GPUKinds(gpus []discovery.GPU) []GPUKind { + seen := map[string]bool{} + var out []GPUKind + for _, g := range gpus { + c := g.Class() + if seen[c] { + continue + } + seen[c] = true + cc, _ := strconv.ParseFloat(strings.TrimSpace(g.ComputeCapability), 64) + // A MIG instance's memory is the instance's: what a process sees. + out = append(out, GPUKind{Class: c, ComputeCapability: cc, MemoryGB: (g.MemoryTotalMB + 512) / 1024}) + } + slices.SortFunc(out, func(a, b GPUKind) int { return strings.Compare(a.Class, b.Class) }) + return out +} + +// Fits reports whether a kind of card meets the need. +func (n GPUNeed) Fits(k GPUKind) bool { + // A card that does not say its capability is not taken for one that has + // it: an fp8 workflow on a card of unknown age fails at its first step. + if n.MinComputeCapability > 0 && k.ComputeCapability+1e-9 < n.MinComputeCapability { + return false + } + return n.MinMemoryGB <= 0 || k.MemoryGB >= n.MinMemoryGB +} + +func (n GPUNeed) String() string { + var parts []string + if n.MinComputeCapability > 0 { + parts = append(parts, fmt.Sprintf("compute %.1f+", n.MinComputeCapability)) + } + if n.MinMemoryGB > 0 { + parts = append(parts, fmt.Sprintf("%d GB+", n.MinMemoryGB)) + } + if len(parts) == 0 { + return "any GPU" + } + return strings.Join(parts, ", ") +} + +// forClass is a profile that names its GPU by need, as the one it stands for +// on a class. +func (p Profile) forClass(class string) Profile { + p.Name = p.Name + "@" + class + p.GPUClass = class + p.GPU = nil + return p +} + +func (p Profile) validateHardware(m Model) error { + if strings.Contains(p.Name, "@") { + return fmt.Errorf("model %q profile %q: a profile's name has no '@': it is how one standing for a class is named", + m.Name, p.Name) + } + if p.GPU != nil { + if p.GPUClass != "" { + return fmt.Errorf("model %q profile %q: gpu_class and gpu are one or the other: a class, or what card it needs", + m.Name, p.Name) + } + if p.GPU.MinComputeCapability < 0 || p.GPU.MinComputeCapability > 20 || p.GPU.MinMemoryGB < 0 { + return fmt.Errorf("model %q profile %q: gpu %+v is not a compute capability and a memory", m.Name, p.Name, *p.GPU) + } + } + if p.ComfyUI != nil { + if p.Engine != EngineComfyUI { + return fmt.Errorf("model %q profile %q: comfyui: is for a profile of engine comfyui", m.Name, p.Name) + } + if err := checkWorkflow(m.Name, p.ComfyUI.Workflow); err != nil { + return err + } + for _, n := range p.ComfyUI.Nodes { + if err := n.validate(m.Name); err != nil { + return err + } + } + } + return nil +} + +// PlacementProfilesFor is a model's profiles in placement's view, each one +// that names its GPU by need standing for every kind of card among kinds +// that meets it, at its own priority. One that no card meets stands for +// none, and is not tried. +func (m Model) PlacementProfilesFor(kinds []GPUKind) []placement.Profile { + var expanded []Profile + for _, p := range m.Profiles { + if p.GPU == nil { + expanded = append(expanded, p) + continue + } + for _, k := range kinds { + if p.GPU.Fits(k) { + expanded = append(expanded, p.forClass(k.Class)) + } + } + } + mm := m + mm.Profiles = expanded + return mm.PlacementProfiles() +} + +// ComfyWorkflowFor is what a profile's replicas are sent: the profile's own +// workflow and checkpoint where it has them, else the model's. +func (m Model) ComfyWorkflowFor(p Profile) (workflow, checkpoint string) { + workflow, checkpoint = m.ComfyWorkflow() + if p.ComfyUI != nil { + if p.ComfyUI.Workflow != "" { + workflow = p.ComfyUI.Workflow + } + if p.ComfyUI.Checkpoint != "" { + checkpoint = p.ComfyUI.Checkpoint + } + } + if p.Source != "" && (p.ComfyUI == nil || p.ComfyUI.Checkpoint == "") && !strings.HasSuffix(p.Source, "/") { + // A profile with weights of its own loads them by their own name. + checkpoint = lastPathPart(strings.TrimPrefix(p.Source, "hf:")) + } + return workflow, checkpoint +} + +// ComfyNodesFor are the custom nodes a profile's replicas carry. +func (m Model) ComfyNodesFor(p Profile) []CustomNode { + if p.ComfyUI != nil && p.ComfyUI.Nodes != nil { + return p.ComfyUI.Nodes + } + if m.ComfyUI != nil { + return m.ComfyUI.Nodes + } + return nil +} + +func lastPathPart(s string) string { + if i := strings.LastIndex(s, "/"); i >= 0 { + return s[i+1:] + } + return s +} diff --git a/internal/control/catalog/hardware_test.go b/internal/control/catalog/hardware_test.go new file mode 100644 index 0000000..e07c9e8 --- /dev/null +++ b/internal/control/catalog/hardware_test.go @@ -0,0 +1,103 @@ +package catalog + +import ( + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" +) + +// sf3dByCard is one entry for every site: fp16 on anything with 16 GB, and +// a bf16 workflow with its own weights where the card is Ampere or newer. +func sf3dByCard() Model { + return Model{Name: "sf3d", Source: "/w/sf3d/model.safetensors", + ComfyUI: &ComfyUI{Workflow: "/etc/llmgw/workflows/sf3d.json", Output: OutputMeshes}, + Profiles: []Profile{ + {Name: "ampere", Engine: EngineComfyUI, TP: 1, Priority: 10, GPU: &GPUNeed{MinComputeCapability: 8.0, MinMemoryGB: 16}, + Source: "/w/sf3d-bf16/model-bf16.safetensors", + ComfyUI: &ProfileComfyUI{Workflow: "/etc/llmgw/workflows/sf3d-bf16.json"}}, + {Name: "any", Engine: EngineComfyUI, TP: 1, GPU: &GPUNeed{MinMemoryGB: 16}}, + }} +} + +func TestAProfileThatNamesItsCardByNeedStandsForEveryClassThatMeetsIt(t *testing.T) { + cat, err := New([]Model{sf3dByCard()}) + if err != nil { + t.Fatal(err) + } + m, _ := cat.Resolve("sf3d") + turing := GPUKinds([]discovery.GPU{{Name: "Quadro RTX 6000", MemoryTotalMB: 24576, ComputeCapability: "7.5"}}) + if len(turing) != 1 || turing[0].Class != "6000-24g" || turing[0].ComputeCapability != 7.5 || turing[0].MemoryGB != 24 { + t.Fatalf("kinds %+v", turing) + } + ps := m.PlacementProfilesFor(turing) + if len(ps) != 1 || ps[0].Name != "any@6000-24g" || ps[0].GPUClass != "6000-24g" { + t.Fatalf("on Turing: %+v", ps) + } + + both := GPUKinds([]discovery.GPU{ + {Name: "Quadro RTX 6000", MemoryTotalMB: 24576, ComputeCapability: "7.5"}, + {Name: "NVIDIA A100-SXM4-80GB", MemoryTotalMB: 81920, ComputeCapability: "8.0"}, + {Name: "NVIDIA A100-SXM4-80GB", MemoryTotalMB: 81920, ComputeCapability: "8.0"}, + {Name: "Tesla T4", MemoryTotalMB: 15360, ComputeCapability: "7.5"}, + }) + var names []string + for _, p := range m.PlacementProfilesFor(both) { + names = append(names, p.Name+"/"+p.GPUClass) + } + if got := strings.Join(names, " "); got != "ampere@a100-80g/a100-80g any@6000-24g/6000-24g any@a100-80g/a100-80g" { + t.Errorf("on a mixed cluster: %s (the T4 has 15 GB, short of 16)", got) + } + if len(m.PlacementProfiles()) != 0 { + t.Error("with no cards known, a profile by need stood for something") + } + + // A replica placed as ampere@a100-80g is started with that profile's own + // weights, workflow and class. + p, ok := m.ProfileByName("ampere@a100-80g") + if !ok || p.GPUClass != "a100-80g" || p.GPU != nil || m.SourceOf(p) != "/w/sf3d-bf16/model-bf16.safetensors" { + t.Fatalf("profile %+v %v", p, ok) + } + if w, ckpt := m.ComfyWorkflowFor(p); w != "/etc/llmgw/workflows/sf3d-bf16.json" || ckpt != "model-bf16.safetensors" { + t.Errorf("workflow %q checkpoint %q", w, ckpt) + } + if ep, ok := m.EngineProfile("ampere@a100-80g"); !ok || ep.GPUClass != "a100-80g" || ep.ModelPath != "/w/sf3d-bf16/model-bf16.safetensors" { + t.Errorf("engine profile %+v", ep) + } + if w, ckpt := m.ComfyWorkflowFor(m.Profiles[1]); w != "/etc/llmgw/workflows/sf3d.json" || ckpt != "model.safetensors" { + t.Errorf("the fallback's workflow %q checkpoint %q", w, ckpt) + } + if _, ok := m.ProfileByName("nothing@a100-80g"); ok { + t.Error("a profile that is not there was found") + } +} + +func TestAProfileByNeedIsCheckedLikeAnyOther(t *testing.T) { + for name, c := range map[string]struct { + p Profile + want string + }{ + "both": {Profile{Name: "p", Engine: EngineComfyUI, TP: 1, GPUClass: "a100-80g", GPU: &GPUNeed{MinMemoryGB: 8}}, "one or the other"}, + "an @ in a name": {Profile{Name: "p@x", Engine: EngineComfyUI, TP: 1, GPUClass: "a100-80g"}, "no '@'"}, + "a capability of 80": {Profile{Name: "p", Engine: EngineComfyUI, TP: 1, GPU: &GPUNeed{MinComputeCapability: 80}}, + "compute capability"}, + "comfyui on vllm": {Profile{Name: "p", Engine: "vllm", TP: 1, GPU: &GPUNeed{}, ComfyUI: &ProfileComfyUI{}}, + "engine comfyui"}, + } { + m := sf3dByCard() + m.Profiles = []Profile{c.p} + if c.p.Engine == "vllm" { + m.ComfyUI = nil + } + if _, err := New([]Model{m}); err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("%s: %v", name, err) + } + } + need := GPUNeed{MinComputeCapability: 8.9} + if need.Fits(GPUKind{Class: "x", MemoryGB: 80}) { + t.Error("a card that does not say its capability was taken for an Ada") + } + if !need.Fits(GPUKind{ComputeCapability: 9.0}) || need.String() != "compute 8.9+" { + t.Error("a Hopper is not an Ada or newer") + } +} diff --git a/internal/control/control.go b/internal/control/control.go index ff9b12f..81b8eff 100644 --- a/internal/control/control.go +++ b/internal/control/control.go @@ -372,7 +372,7 @@ func (c *Controller) endpoints() []endpoint.Info { Priority: prof.Priority, } if prof.Engine == catalog.EngineComfyUI { - workflow, ckpt := m.ComfyWorkflow() + workflow, ckpt := m.ComfyWorkflowFor(prof) w, err := c.workflow(workflow) if err != nil { c.log.Warn("a ComfyUI model's workflow could not be read; its replica is not offered", diff --git a/internal/control/reconciler/hardware.go b/internal/control/reconciler/hardware.go new file mode 100644 index 0000000..c30e3a1 --- /dev/null +++ b/internal/control/reconciler/hardware.go @@ -0,0 +1,16 @@ +package reconciler + +import ( + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" +) + +// gpuKinds are the kinds of card the cluster's nodes have, for a profile that +// names its GPU by what it needs rather than by class to stand for. +func (r *Reconciler) gpuKinds() []catalog.GPUKind { + var gpus []discovery.GPU + for _, n := range r.inv.Nodes() { + gpus = append(gpus, n.GPUs...) + } + return catalog.GPUKinds(gpus) +} diff --git a/internal/control/reconciler/measured.go b/internal/control/reconciler/measured.go index ac8a5ca..80764c9 100644 --- a/internal/control/reconciler/measured.go +++ b/internal/control/reconciler/measured.go @@ -45,7 +45,7 @@ func (r *Reconciler) measured(model, profile string) (Measured, bool) { // Placement sorts by priority itself, so the order found here is written back // as priorities, descending, for it to keep. func (r *Reconciler) orderedProfiles(m catalog.Model) []placement.Profile { - ps := m.PlacementProfiles() + ps := m.PlacementProfilesFor(r.gpuKinds()) eff := make([]float64, len(ps)) for i, p := range ps { meas, ok := r.measured(m.Name, p.Name) diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index bcd061d..7227f0b 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -334,6 +334,8 @@ var Reference = []Block{ {Key: "priority", Type: "int", Doc: "Priority orders profiles during placement; highest that fits wins.", Nested: ""}, {Key: "startup_timeout", Type: "time.Duration", Doc: "StartupTimeout bounds model loading. A 70B model across two nodes takes\ntens of minutes, so this is per profile rather than global.", Nested: ""}, {Key: "kv_events", Type: "bool", Doc: "KVEvents has vLLM publish what its prefix cache holds, and the router\nroute this model's requests by it: to the replica holding the most of\na prompt's leading blocks (plan §7.5 tier 2). vLLM only.\n\nOff by default, because it costs something on every request: the\nrouter asks a replica to tokenize the prompt before placing it, a\nround trip and the chat template run twice. It pays where prompts\nshare long prefixes that the approximate tree guesses wrong — many\nreplicas, conversations that move between them, evictions. The engine\nis started with sha256_cbor block hashing and a fixed PYTHONHASHSEED,\nwhich the router's hashing assumes; arguments overriding either leave\nthe index empty, and the agent says so in its log.", Nested: ""}, + {Key: "gpu", Type: "*GPUNeed", Doc: "GPU says what card the profile needs, in place of naming its class:\nit stands for every class in the cluster that meets it. A catalog\nentry written once then places sensibly on any site's cards.", Nested: "catalog.GPUNeed"}, + {Key: "comfyui", Type: "*ProfileComfyUI", Doc: "ComfyUI is this profile's own workflow, checkpoint and nodes, over the\nmodel's: an fp8 workflow where the card has fp8, fp16 where it has not.", Nested: "catalog.ProfileComfyUI"}, }}, {Package: "config", Name: "Router", Title: "Router", Doc: "", Fields: []Field{ {Key: "activation_wait", Type: "time.Duration", Doc: "ActivationWait is how long a request waits for an on-demand model to\nload before the client is told to come back. Zero takes the default.\n\nThe one real decision in on-demand loading. Refusing outright makes the\nfeature useless, since the first request for every model would fail;\nwaiting without a bound turns a request into a job, and a client that\nbudgeted thirty seconds ends up holding a connection for ten minutes.", Nested: ""},