From 09ebe77513dbfc43a6838bab4643996bd61da500 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sat, 21 Nov 2026 13:43:40 +0100 Subject: [PATCH] A GGUF is servable when a llama.cpp that engine_overrides give some nodes loads it `ctl model serve`, its plan and model updates judged a file only against the cluster's own llama.cpp, so a GPU node given a newer build could not be served an architecture only that build knows: Inkling 2 Mini (inkling2_loop) was refused for b10991 with cuda01 on b12606. Co-Authored-By: Claude Opus 5.5 --- internal/cli/server.go | 2 +- internal/config/config.go | 12 +++++++++++ internal/config/engines_test.go | 15 +++++++++++++ internal/control/modelserve/modelserve.go | 26 +++++++++++++++++------ internal/control/modelserve/plan.go | 2 +- internal/control/modelserve/updates.go | 4 ++-- internal/gguf/gguf.go | 26 +++++++++++++++++++++++ internal/gguf/gguf_test.go | 22 +++++++++++++++++++ 8 files changed, 99 insertions(+), 10 deletions(-) diff --git a/internal/cli/server.go b/internal/cli/server.go index bcaa68c..1823d3c 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -679,7 +679,7 @@ func newServerCmd() *cobra.Command { if adminAPI != nil && catalogSvc != nil { serve := modelserve.New(modelserve.Deps{ Registry: registry, Mirror: mc, Catalog: catalogSvc, Fleet: admin, - LlamaVersion: cfg.WantedEngines()["llamacpp"], Log: log, + LlamaVersion: cfg.WantedEngines()["llamacpp"], LlamaOverrides: cfg.OverrideVersions("llamacpp"), Log: log, // Updates: the catalog in force, the service that // changes it, and what replaces replicas. Current: func() []catalog.Model { return admin.Catalog().Models() }, diff --git a/internal/config/config.go b/internal/config/config.go index 9912d20..be09013 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1164,6 +1164,18 @@ func (s Server) EnginesFor(node string, classes []string, labels map[string]stri return out } +// OverrideVersions are the versions of an engine that engine_overrides give +// some nodes, in the order they are set. +func (s Server) OverrideVersions(engine string) []string { + var out []string + for _, o := range s.EngineOverrides { + if v := o.Engines[engine]; v != "" && !slices.Contains(out, v) { + out = append(out, v) + } + } + return out +} + // WantedEngines is DefaultEngines with the configuration laid over it. func (s Server) WantedEngines() map[string]string { out := map[string]string{} diff --git a/internal/config/engines_test.go b/internal/config/engines_test.go index 2d39245..741374f 100644 --- a/internal/config/engines_test.go +++ b/internal/config/engines_test.go @@ -1,6 +1,7 @@ package config import ( + "slices" "strings" "testing" ) @@ -54,3 +55,17 @@ func TestOIDCSettingsAreCheckedBeforeAProviderIsAsked(t *testing.T) { } } } + +func TestOverrideVersionsAreTheEnginesNodesAreGivenInstead(t *testing.T) { + s := Server{EngineOverrides: []EngineOverride{ + {Nodes: []string{"cuda01"}, Engines: map[string]string{"strata": "0.2.7", "llamacpp": "b12606"}}, + {GPUClass: "v100-32g", Engines: map[string]string{"llamacpp": ""}}, + {Labels: map[string]string{"site": "b"}, Engines: map[string]string{"llamacpp": "b12606"}}, + }} + if got := s.OverrideVersions("llamacpp"); !slices.Equal(got, []string{"b12606"}) { + t.Errorf("llamacpp: %v", got) + } + if got := s.OverrideVersions("vllm"); got != nil { + t.Errorf("vllm: %v", got) + } +} diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index 79a57a3..1c294ee 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -68,10 +68,12 @@ type Deps struct { Catalog Catalog Fleet Fleet // LlamaVersion is the llama.cpp build the nodes install, which decides - // which architectures load. - LlamaVersion string - HTTP *http.Client - Log *slog.Logger + // which architectures load. LlamaOverrides are the builds some nodes are + // given instead (engine_overrides): a file one of them loads is servable. + LlamaVersion string + LlamaOverrides []string + HTTP *http.Client + Log *slog.Logger // Poll is how often the job looks at the mirror and the fleet. Poll time.Duration // ReadyTimeout bounds the wait for a replica: the node fetching the @@ -240,6 +242,18 @@ type Server struct { } // New builds a server. +// llamaVersions are the llama.cpp builds the nodes install, the cluster's own +// first. +func (s *Server) llamaVersions() []string { + out := []string{s.deps.LlamaVersion} + for _, v := range s.deps.LlamaOverrides { + if !slices.Contains(out, v) { + out = append(out, v) + } + } + return out +} + func New(d Deps) *Server { if d.Poll <= 0 { d.Poll = 2 * time.Second @@ -553,9 +567,9 @@ func (s *Server) run(j *Job) { s.step(j, "check", Running, "reading its header from the mirror") var header gguf.Header if u := headerURL(source, set); u != "" { - v := gguf.Check(ctx, u, s.deps.LlamaVersion) + v := gguf.CheckAny(ctx, u, s.llamaVersions()) if !v.Servable { - s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run this: %s", s.deps.LlamaVersion, v.Reason)) + s.fail(j, "check", fmt.Errorf("llama.cpp on the nodes cannot run this: %s", v.Reason)) return } header = v.Header diff --git a/internal/control/modelserve/plan.go b/internal/control/modelserve/plan.go index fa2c11a..5665dd7 100644 --- a/internal/control/modelserve/plan.go +++ b/internal/control/modelserve/plan.go @@ -147,7 +147,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan, if b, ok := s.deps.Registry.(interface{ Base() string }); ok { base = b.Base() } - v := gguf.Check(ctx, repo.DownloadURL(base, small.Files[0].Path), s.deps.LlamaVersion) + v := gguf.CheckAny(ctx, repo.DownloadURL(base, small.Files[0].Path), s.llamaVersions()) if v.Checked && !v.Servable { p.Servable, p.Reason = false, v.Reason } diff --git a/internal/control/modelserve/updates.go b/internal/control/modelserve/updates.go index 4ce09d1..5c1c9fe 100644 --- a/internal/control/modelserve/updates.go +++ b/internal/control/modelserve/updates.go @@ -585,9 +585,9 @@ func (s *Server) prepare(ctx context.Context, j *Job, name string) (prepared, bo s.step(j, "check", Running, "reading its header from the mirror") detail := "no header to read" if u := headerURL(source, set); u != "" { - v := gguf.Check(ctx, u, s.deps.LlamaVersion) + v := gguf.CheckAny(ctx, u, s.llamaVersions()) if !v.Servable { - s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run the new weights: %s", s.deps.LlamaVersion, v.Reason)) + s.fail(j, "check", fmt.Errorf("llama.cpp on the nodes cannot run the new weights: %s", v.Reason)) return prepared{}, false } detail = orUnknown(v.Header.Architecture) diff --git a/internal/gguf/gguf.go b/internal/gguf/gguf.go index de2b7e1..67cbfb7 100644 --- a/internal/gguf/gguf.go +++ b/internal/gguf/gguf.go @@ -404,6 +404,32 @@ func Check(ctx context.Context, url, llamaVersion string) Verdict { return v } +// CheckAny is Check against several llama.cpp builds, the header read once: +// servable when any of them loads it. A cluster whose GPU nodes are given a +// newer build than the rest (engine_overrides) can serve an architecture +// only that build knows. The first version is the cluster's own; its verdict +// is the one given when none loads it. +func CheckAny(ctx context.Context, url string, versions []string) Verdict { + if len(versions) == 0 { + versions = []string{""} + } + v := Check(ctx, url, versions[0]) + if v.Servable || v.Header.Architecture == "" { + return v + } + for _, other := range versions[1:] { + if o := LlamaCpp(other, v.Header.Architecture); o.Servable { + o.Header = v.Header + return o + } + } + if len(versions) > 1 { + v.Reason = "its architecture is \"" + v.Header.Architecture + "\", which no llama.cpp the nodes install (" + + strings.Join(versions, ", ") + ") can load" + } + return v +} + // CardUsable is the share of a card's memory a model may take: the rest is // CUDA's own and the engine's compute buffers. const CardUsable = 0.9 diff --git a/internal/gguf/gguf_test.go b/internal/gguf/gguf_test.go index 1481a29..402b472 100644 --- a/internal/gguf/gguf_test.go +++ b/internal/gguf/gguf_test.go @@ -221,3 +221,25 @@ func TestAModelOfExpertsReadsAShareOfItsWeights(t *testing.T) { t.Error("a dense model reads all its weights") } } + +func TestAnArchitectureOnlyAnOverriddenBuildKnowsIsServable(t *testing.T) { + // Found on the PoC: Inkling 2 Mini (inkling2_loop) refused by `ctl model + // serve` for the cluster's b10991, though the GPU node had been given + // b12606 through engine_overrides. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.ServeContent(w, r, "m.gguf", timeZero, bytes.NewReader(header("inkling2_loop"))) + })) + defer srv.Close() + url := srv.URL + "/m.gguf" + if v := CheckAny(context.Background(), url, []string{"b10991"}); v.Servable { + t.Errorf("b10991 alone: %+v", v) + } + if v := CheckAny(context.Background(), url, []string{"b10991", "b12606"}); !v.Servable || + v.Header.Architecture != "inkling2_loop" { + t.Errorf("with b12606 on some nodes: %+v", v) + } + if v := CheckAny(context.Background(), url, []string{"b10991", "b10991"}); v.Servable || + !strings.Contains(v.Reason, "no llama.cpp the nodes install (b10991, b10991)") { + t.Errorf("refused by every build: %+v", v) + } +}