From 63198f27534e336958f30636538ce61adbf818b2 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sat, 10 Oct 2026 17:15:38 +0200 Subject: [PATCH] A profile may load weights of its own The catalog had one source a model, so an embedding model served by vLLM from its repository on a card and by llama.cpp from a GGUF on CPUs had its entry swapped by hand halfway through the metallurgy import. A profile's source is what its engine loads and placement looks for; the model's is the default, and what updates and canaries move. ctl model add checks every source, the GGUF check reads each llama.cpp profile's, and ctl weights collects none a profile loads. An empty source leaves a profile's fingerprint as it was, so running replicas do not read as stale. Co-Authored-By: Claude Opus 5.5 (1M context) --- BACKLOG.md | 9 +- CHANGELOG.md | 5 ++ docs/admin-api.openapi.json | 4 + docs/install.md | 21 +++++ docs/reference.md | 11 +++ internal/cli/model.go | 19 ++-- internal/cli/server.go | 20 ++--- internal/control/adminapi/weights.go | 2 +- internal/control/catalog/catalog.go | 67 +++++++++++--- .../control/catalog/profilesource_test.go | 90 +++++++++++++++++++ internal/control/modelsvc/modelsvc.go | 4 +- internal/docgen/reference_gen.go | 1 + 12 files changed, 218 insertions(+), 35 deletions(-) create mode 100644 internal/control/catalog/profilesource_test.go diff --git a/BACKLOG.md b/BACKLOG.md index b386008..db0fc73 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -1091,10 +1091,11 @@ at a third of its CPUs. FlashAttention and ubatch 1–2k; f16 no better. vLLM fp16 (`--runner pooling`) 6,200 direct, 4,700–4,900 through the gateway in the import, and its vectors match llama.cpp's (cosine 0.9995). The 103,579 chunks - took 3 h. Wanted: an embedding model's GPU profile on vLLM and CPU - profile on llama.cpp, which needs a source per profile (the catalog has - one per model; the import swapped the entry by hand), and an admission - queue limit that suits bulk imports (30 s turned some away). + took 3 h. Done (2026-10-10, branch `admission-bulk`): a source per + profile, so an embedding model's GPU profile runs vLLM and its CPU + profile llama.cpp; a band's own `queue_ttl` for bulk imports, and + `Retry-After` on a busy refusal, which is what had turned passages away + (llmgw-kb retries only what says when to). - Retrieval on 40 questions written from the corpus, document in the top 5: bge-m3 31/40 (Turkish questions 13/21), Qwen3-Embedding-4B 37/40 (18/21), with its query prefix 36/40 and first place 31/40 instead of diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f01ddc..b287382 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,11 @@ APIs change between minors without a deprecation period. ### Added +- **A profile's own `source`**: the weights it loads instead of the + model's, so an embedding model can run on vLLM from its repository on a + card and on llama.cpp from a GGUF on CPUs as one model. Placement prefers + nodes holding each profile's weights, `ctl model add` checks every + source, and `ctl weights` collects none a profile loads. - **A band's own `queue_ttl`** (`router.admission.bands[].queue_ttl`): how long its requests may wait for room, instead of the admission's 30 s. A band for bulk work — a knowledge-base import — waits behind people rather diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index 869d9dd..aec49aa 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -8837,6 +8837,10 @@ "gpu_class": { "type": "string" }, + "source": { + "type": "string", + "description": "Source is the weights this profile loads, when not the model's: an embedding model served by vLLM from its repository on a card and by llama.cpp from a GGUF on CPUs is one model with two sources. Empty loads the model's. Updates and canaries move the model's source; a profile with its own keeps it." + }, "tp": { "type": "integer" }, diff --git a/docs/install.md b/docs/install.md index 234275b..99c084b 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1038,6 +1038,27 @@ qwen2.5-coder-3b-instruct yes 1/1 ready coder No restart, and it survives one: models added this way are stored, while anything in `server.yaml` is declarative and returns on every start. +A profile may load weights of its own. An embedding model, for example, can +be served by vLLM from its repository on a card and by llama.cpp from a GGUF +on CPUs, as one model: + +```yaml +name: qwen3-embed-4b +source: Qwen/Qwen3-Embedding-4B # what a profile loads unless it says +capabilities: { embeddings: true } +profiles: + - name: gpu + engine: vllm + gpu_class: 6000-24g + args: ["--runner", "pooling"] + - name: cpu + engine: llamacpp + source: hf:Qwen/Qwen3-Embedding-4B-GGUF/Qwen3-Embedding-4B-Q8_0.gguf +``` + +Updates and canaries move the model's `source`; a profile with its own keeps +it. + ### Models that load when someone asks A cluster with more models than GPUs can hold every model in its catalog and diff --git a/docs/reference.md b/docs/reference.md index 467d8c7..62a960d 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -220,6 +220,7 @@ rather than repeated. | `engine` | `string` | [Profile](#profile) | | `engine_version` | `string` | [Profile](#profile) | | `gpu_class` | `string` | [Profile](#profile) | +| `source` | `string` | [Profile](#profile) | | `tp` | `int` | [Profile](#profile) | | `pp` | `int` | [Profile](#profile) | | `gpus_per_node` | `int` | [Profile](#profile) | @@ -1925,6 +1926,16 @@ version starts with it: "0.11" takes vLLM 0.11.0 and 0.11.1, not `string` +### `source` + +`string` + +Source is the weights this profile loads, when not the model's: an +embedding model served by vLLM from its repository on a card and by +llama.cpp from a GGUF on CPUs is one model with two sources. Empty +loads the model's. Updates and canaries move the model's source; a +profile with its own keeps it. + ### `tp` `int` diff --git a/internal/cli/model.go b/internal/cli/model.go index 29ad603..f8171b7 100644 --- a/internal/cli/model.go +++ b/internal/cli/model.go @@ -80,15 +80,18 @@ func newModelAddCmd(client func() *ctlClient) *cobra.Command { // resolve is otherwise discovered by a replica failing to start // minutes later on a node — with an error naming the node rather // than the typo. + // Every source the model's profiles load, not the model's alone. if !skipCheck { - size, err := sourcecheck.Check(cmd.Context(), m.Source) - if err != nil { - return fmt.Errorf("%w\n Pass --no-source-check to add it anyway, "+ - "which is right when the mirror is not up yet or the nodes "+ - "reach it by an address this machine does not.", err) - } - if _, isURL := sourcecheck.HTTPSource(m.Source); isURL { - fmt.Fprintln(cmd.OutOrStdout(), sourceNote(m.Source, size)) + for _, src := range m.WeightSources() { + size, err := sourcecheck.Check(cmd.Context(), src) + if err != nil { + return fmt.Errorf("%w\n Pass --no-source-check to add it anyway, "+ + "which is right when the mirror is not up yet or the nodes "+ + "reach it by an address this machine does not.", err) + } + if _, isURL := sourcecheck.HTTPSource(src); isURL { + fmt.Fprintln(cmd.OutOrStdout(), sourceNote(src, size)) + } } } diff --git a/internal/cli/server.go b/internal/cli/server.go index 78cc886..9b27618 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -1173,18 +1173,18 @@ func servableCheck(llamaVersion string) func(context.Context, catalog.Model) err if llamaVersion == "" { return nil } - usesLlama := false + // Each source a llama.cpp profile loads: its own, or the model's. for _, p := range m.Profiles { - if p.Engine == "llamacpp" { - usesLlama = true + if p.Engine != "llamacpp" { + continue + } + url, ok := sourcecheck.HTTPSource(m.SourceOf(p)) + if !ok || !gguf.IsGGUF(url) { + continue + } + if v := gguf.Check(ctx, url, llamaVersion); !v.Servable { + return fmt.Errorf("%s cannot be served: %s", m.Name, v.Reason) } - } - url, ok := sourcecheck.HTTPSource(m.Source) - if !usesLlama || !ok || !gguf.IsGGUF(url) { - return nil - } - if v := gguf.Check(ctx, url, llamaVersion); !v.Servable { - return fmt.Errorf("%s cannot be served: %s", m.Name, v.Reason) } return nil } diff --git a/internal/control/adminapi/weights.go b/internal/control/adminapi/weights.go index d7c4592..f171a88 100644 --- a/internal/control/adminapi/weights.go +++ b/internal/control/adminapi/weights.go @@ -30,7 +30,7 @@ type OrphanView struct { func (a *API) Orphans() []OrphanView { var wanted []string for _, m := range a.ctl.Catalog().Models() { - wanted = append(wanted, m.Source) + wanted = append(wanted, m.WeightSources()...) // A model's adapters are its weights too; collecting them would // leave replicas that cannot restart. for _, ad := range m.Adapters { diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index adf3c15..79cea98 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -174,6 +174,12 @@ type Profile struct { // 0.10.2. Empty takes whatever version the node has. EngineVersion string `json:"engine_version,omitempty" koanf:"engine_version"` GPUClass string `json:"gpu_class,omitempty" koanf:"gpu_class"` + // Source is the weights this profile loads, when not the model's: an + // embedding model served by vLLM from its repository on a card and by + // llama.cpp from a GGUF on CPUs is one model with two sources. Empty + // loads the model's. Updates and canaries move the model's source; a + // profile with its own keeps it. + Source string `json:"source,omitempty" koanf:"source"` TP int `json:"tp,omitempty" koanf:"tp"` PP int `json:"pp,omitempty" koanf:"pp"` @@ -762,9 +768,15 @@ func (m Model) normalize() Model { // repository it names: the engine downloads a repository id itself, // where the page was fetched as one file of HTML and failed at start. if repo, ok := repositoryPage(m.Source); ok && len(m.Profiles) > 0 && - !slices.ContainsFunc(m.Profiles, func(p Profile) bool { return p.Engine == "llamacpp" }) { + !slices.ContainsFunc(m.Profiles, func(p Profile) bool { return p.Engine == "llamacpp" && p.Source == "" }) { m.Source = repo } + for i := range m.Profiles { + p := &m.Profiles[i] + if repo, ok := repositoryPage(p.Source); ok && p.Engine != "llamacpp" { + p.Source = repo + } + } return m } @@ -775,15 +787,26 @@ func (m Model) validate() error { if m.Source == "" { return fmt.Errorf("model %q: source is required", m.Name) } - if repo, ok := repositoryPage(m.Source); ok { - // A page, not weights: a node fetches the HTML as one file and the - // engine fails at start, on a path that looks right. For vLLM it was - // read as the repository id when the model was made; only a model - // llama.cpp runs is left here, which needs a file. + // A page, not weights: a node fetches the HTML as one file and the + // engine fails at start, on a path that looks right. For vLLM it was + // read as the repository id when the model was made; only a source + // llama.cpp loads is left here, which needs a file. + if repo, ok := repositoryPage(m.Source); ok && (len(m.Profiles) == 0 || + slices.ContainsFunc(m.Profiles, func(p Profile) bool { return p.Engine == "llamacpp" && p.Source == "" })) { return fmt.Errorf("model %q: source %s is a Hugging Face repository page, not a file; "+ "llama.cpp needs a .gguf: mirror one and use its source (repository %s)", m.Name, m.Source, repo) } + for _, p := range m.Profiles { + if repo, ok := repositoryPage(p.Source); ok { + return fmt.Errorf("model %q profile %q: source %s is a Hugging Face repository page, not a file; "+ + "llama.cpp needs a .gguf: mirror one and use its source (repository %s)", + m.Name, p.Name, p.Source, repo) + } + if p.Source != "" && m.External != nil { + return fmt.Errorf("model %q profile %q: an external model has no profiles to give a source", m.Name, p.Name) + } + } if err := validateAdapters(m); err != nil { return err } @@ -924,7 +947,7 @@ func (m Model) validate() error { return fmt.Errorf("model %q profile %q: trtllm runs a replica on one node; "+ "use vllm or sglang for a %d-node replica", m.Name, p.Name, p.Nodes) } - if strings.HasSuffix(strings.ToLower(strings.TrimSuffix(m.Source, "/")), ".gguf") { + if strings.HasSuffix(strings.ToLower(strings.TrimSuffix(m.SourceOf(p), "/")), ".gguf") { return fmt.Errorf("model %q profile %q: trtllm loads a Hugging Face checkpoint, not a .gguf; "+ "serve a GGUF with llamacpp", m.Name, p.Name) } @@ -1010,14 +1033,38 @@ func (m Model) PlacementProfiles() []placement.Profile { CPUCores: p.CPUCores, PowerWatts: p.PowerWatts, KeepWithin: facility.Level(strings.ToLower(strings.TrimSpace(p.KeepWithin))), - // The model's source, so placement can prefer nodes that already + // The profile's source, so placement can prefer nodes that already // hold the weights rather than one that must first move them. - Source: m.Source, + Source: m.SourceOf(p), }) } return out } +// SourceOf is the weights a profile loads: its own, or the model's. +func (m Model) SourceOf(p Profile) string { + if p.Source != "" { + return p.Source + } + return m.Source +} + +// WeightSources are every source a model's replicas may load: the model's +// and each profile's of its own, once each. What a node must not have +// collected from under it. +func (m Model) WeightSources() []string { + out := []string{} + if m.Source != "" { + out = append(out, m.Source) + } + for _, p := range m.Profiles { + if p.Source != "" && !slices.Contains(out, p.Source) { + out = append(out, p.Source) + } + } + return out +} + // EngineProfile converts a catalog profile into the engine adapter's view. func (m Model) EngineProfile(name string) (engines.Profile, bool) { for _, p := range m.Profiles { @@ -1034,7 +1081,7 @@ func (m Model) EngineProfile(name string) (engines.Profile, bool) { Nodes: p.Nodes, Args: slices.Clone(p.Args), Env: p.Env, - ModelPath: m.Source, + ModelPath: m.SourceOf(p), ServedModelName: m.Name, Adapters: m.engineAdapters(), KVEvents: p.KVEvents, diff --git a/internal/control/catalog/profilesource_test.go b/internal/control/catalog/profilesource_test.go new file mode 100644 index 0000000..ece6040 --- /dev/null +++ b/internal/control/catalog/profilesource_test.go @@ -0,0 +1,90 @@ +package catalog + +import ( + "encoding/json" + "slices" + "strings" + "testing" +) + +// embedModel is an embedding model the way it was wanted on the PoC: vLLM +// on the card from the repository, llama.cpp on CPUs from a GGUF. +func embedModel() Model { + return Model{Name: "qwen3-embed-4b", Source: "Qwen/Qwen3-Embedding-4B", + Profiles: []Profile{ + {Name: "gpu", Engine: "vllm", GPUClass: "6000-24g", TP: 1}, + {Name: "cpu", Engine: "llamacpp", Source: "hf:Qwen/Qwen3-Embedding-4B-GGUF/Qwen3-Embedding-4B-Q8_0.gguf"}, + }} +} + +func TestEachProfileLoadsItsOwnSource(t *testing.T) { + // The catalog had one source a model, so serving an embedding model by + // vLLM on a card and by llama.cpp on CPUs meant swapping the entry by + // hand halfway through an import. + cat, err := New([]Model{embedModel()}) + if err != nil { + t.Fatal(err) + } + m, _ := cat.Resolve("qwen3-embed-4b") + gpu, _ := m.EngineProfile("gpu") + cpu, _ := m.EngineProfile("cpu") + if gpu.ModelPath != "Qwen/Qwen3-Embedding-4B" || !strings.HasSuffix(cpu.ModelPath, "Q8_0.gguf") { + t.Errorf("gpu loads %q, cpu %q", gpu.ModelPath, cpu.ModelPath) + } + for _, p := range m.PlacementProfiles() { + if want := map[string]string{"gpu": gpu.ModelPath, "cpu": cpu.ModelPath}[p.Name]; p.Source != want { + t.Errorf("placement for %s looks for %q, want %q", p.Name, p.Source, want) + } + } + if got := m.WeightSources(); len(got) != 2 || !slices.Contains(got, cpu.ModelPath) { + t.Errorf("weight sources %v: a profile's own must not be collected", got) + } +} + +func TestAProfileWithoutASourceKeepsItsFingerprint(t *testing.T) { + // Replicas are compared with their profile by fingerprint; a field added + // to every profile must not make every running replica look stale. + p := Profile{Name: "gpu", Engine: "vllm", GPUClass: "6000-24g", TP: 1} + before := p.Fingerprint() + if strings.Contains(mustJSON(t, p), "source") { + t.Error("an empty source is written into the profile") + } + p.Source = "Qwen/Other" + if p.Fingerprint() == before { + t.Error("a profile whose source changed reads as unchanged") + } +} + +func TestAProfileSourceIsCheckedAsTheModelsIs(t *testing.T) { + page := embedModel() + page.Profiles[1].Source = "https://huggingface.co/Qwen/Qwen3-Embedding-4B-GGUF" + if _, err := New([]Model{page}); err == nil || !strings.Contains(err.Error(), `profile "cpu"`) { + t.Errorf("a repository page for llama.cpp: %v", err) + } + // The model's own source may be a page vLLM reads as a repository, now + // that the llama.cpp profile loads something else. + repo := embedModel() + repo.Source = "https://huggingface.co/Qwen/Qwen3-Embedding-4B" + cat, err := New([]Model{repo}) + if err != nil { + t.Fatalf("a repository page for the vLLM profile beside a llama.cpp one with its own GGUF: %v", err) + } + m, _ := cat.Resolve("qwen3-embed-4b") + if m.Source != "Qwen/Qwen3-Embedding-4B" { + t.Errorf("the page was not read as the repository: %q", m.Source) + } + trt := embedModel() + trt.Profiles[0] = Profile{Name: "gpu", Engine: "trtllm", GPUClass: "6000-24g", TP: 1, Source: "hf:a/b/m.gguf"} + if _, err := New([]Model{trt}); err == nil || !strings.Contains(err.Error(), "not a .gguf") { + t.Errorf("trtllm given a GGUF by its profile: %v", err) + } +} + +func mustJSON(t *testing.T, v any) string { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return string(b) +} diff --git a/internal/control/modelsvc/modelsvc.go b/internal/control/modelsvc/modelsvc.go index 2c2b43b..8fa1240 100644 --- a/internal/control/modelsvc/modelsvc.go +++ b/internal/control/modelsvc/modelsvc.go @@ -87,8 +87,8 @@ func (s *Service) Sources(ctx context.Context) (map[string][]catalog.Model, erro } out := map[string][]catalog.Model{} for _, m := range c.Models() { - if m.Source != "" { - out[m.Source] = append(out[m.Source], m) + for _, src := range m.WeightSources() { + out[src] = append(out[src], m) } for _, ad := range m.Adapters { out[ad.Source] = append(out[ad.Source], m) diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index 285a438..805a4d3 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -284,6 +284,7 @@ var Reference = []Block{ {Key: "engine", Type: "string", Doc: "", Nested: ""}, {Key: "engine_version", Type: "string", Doc: "EngineVersion restricts the profile to nodes whose installed engine's\nversion starts with it: \"0.11\" takes vLLM 0.11.0 and 0.11.1, not\n0.10.2. Empty takes whatever version the node has.", Nested: ""}, {Key: "gpu_class", Type: "string", Doc: "", Nested: ""}, + {Key: "source", Type: "string", Doc: "Source is the weights this profile loads, when not the model's: an\nembedding model served by vLLM from its repository on a card and by\nllama.cpp from a GGUF on CPUs is one model with two sources. Empty\nloads the model's. Updates and canaries move the model's source; a\nprofile with its own keeps it.", Nested: ""}, {Key: "tp", Type: "int", Doc: "", Nested: ""}, {Key: "pp", Type: "int", Doc: "", Nested: ""}, {Key: "gpus_per_node", Type: "int", Doc: "GPUsPerNode defaults to TP for a single-node profile.", Nested: ""},