A GGUF is servable when a llama.cpp that engine_overrides give some nodes loads it
CI / test (push) Failing after 9m58s

`ctl model serve`, its plan and model updates judged a file only against
the cluster's own llama.cpp, so a GPU node given a newer build could not
be served an architecture only that build knows: Inkling 2 Mini
(inkling2_loop) was refused for b10991 with cuda01 on b12606.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-21 13:43:40 +01:00
1 parent 0ffd633bf6
commit 09ebe77513
8 files changed
+99 -10

No files matched your search

+1 -1
View File
@@ -679,7 +679,7 @@ func newServerCmd() *cobra.Command {
if adminAPI != nil && catalogSvc != nil {
serve := modelserve.New(modelserve.Deps{
Registry: registry, Mirror: mc, Catalog: catalogSvc, Fleet: admin,
LlamaVersion: cfg.WantedEngines()["llamacpp"], Log: log,
LlamaVersion: cfg.WantedEngines()["llamacpp"], LlamaOverrides: cfg.OverrideVersions("llamacpp"), Log: log,
// Updates: the catalog in force, the service that
// changes it, and what replaces replicas.
Current: func() []catalog.Model { return admin.Catalog().Models() },
+12
View File
@@ -1164,6 +1164,18 @@ func (s Server) EnginesFor(node string, classes []string, labels map[string]stri
return out
}
// OverrideVersions are the versions of an engine that engine_overrides give
// some nodes, in the order they are set.
func (s Server) OverrideVersions(engine string) []string {
var out []string
for _, o := range s.EngineOverrides {
if v := o.Engines[engine]; v != "" && !slices.Contains(out, v) {
out = append(out, v)
}
}
return out
}
// WantedEngines is DefaultEngines with the configuration laid over it.
func (s Server) WantedEngines() map[string]string {
out := map[string]string{}
+15
View File
@@ -1,6 +1,7 @@
package config
import (
"slices"
"strings"
"testing"
)
@@ -54,3 +55,17 @@ func TestOIDCSettingsAreCheckedBeforeAProviderIsAsked(t *testing.T) {
}
}
}
func TestOverrideVersionsAreTheEnginesNodesAreGivenInstead(t *testing.T) {
s := Server{EngineOverrides: []EngineOverride{
{Nodes: []string{"cuda01"}, Engines: map[string]string{"strata": "0.2.7", "llamacpp": "b12606"}},
{GPUClass: "v100-32g", Engines: map[string]string{"llamacpp": ""}},
{Labels: map[string]string{"site": "b"}, Engines: map[string]string{"llamacpp": "b12606"}},
}}
if got := s.OverrideVersions("llamacpp"); !slices.Equal(got, []string{"b12606"}) {
t.Errorf("llamacpp: %v", got)
}
if got := s.OverrideVersions("vllm"); got != nil {
t.Errorf("vllm: %v", got)
}
}
+20 -6
View File
@@ -68,10 +68,12 @@ type Deps struct {
Catalog Catalog
Fleet Fleet
// LlamaVersion is the llama.cpp build the nodes install, which decides
// which architectures load.
LlamaVersion string
HTTP *http.Client
Log *slog.Logger
// which architectures load. LlamaOverrides are the builds some nodes are
// given instead (engine_overrides): a file one of them loads is servable.
LlamaVersion string
LlamaOverrides []string
HTTP *http.Client
Log *slog.Logger
// Poll is how often the job looks at the mirror and the fleet.
Poll time.Duration
// ReadyTimeout bounds the wait for a replica: the node fetching the
@@ -240,6 +242,18 @@ type Server struct {
}
// New builds a server.
// llamaVersions are the llama.cpp builds the nodes install, the cluster's own
// first.
func (s *Server) llamaVersions() []string {
out := []string{s.deps.LlamaVersion}
for _, v := range s.deps.LlamaOverrides {
if !slices.Contains(out, v) {
out = append(out, v)
}
}
return out
}
func New(d Deps) *Server {
if d.Poll <= 0 {
d.Poll = 2 * time.Second
@@ -553,9 +567,9 @@ func (s *Server) run(j *Job) {
s.step(j, "check", Running, "reading its header from the mirror")
var header gguf.Header
if u := headerURL(source, set); u != "" {
v := gguf.Check(ctx, u, s.deps.LlamaVersion)
v := gguf.CheckAny(ctx, u, s.llamaVersions())
if !v.Servable {
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run this: %s", s.deps.LlamaVersion, v.Reason))
s.fail(j, "check", fmt.Errorf("llama.cpp on the nodes cannot run this: %s", v.Reason))
return
}
header = v.Header
+1 -1
View File
@@ -147,7 +147,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan,
if b, ok := s.deps.Registry.(interface{ Base() string }); ok {
base = b.Base()
}
v := gguf.Check(ctx, repo.DownloadURL(base, small.Files[0].Path), s.deps.LlamaVersion)
v := gguf.CheckAny(ctx, repo.DownloadURL(base, small.Files[0].Path), s.llamaVersions())
if v.Checked && !v.Servable {
p.Servable, p.Reason = false, v.Reason
}
+2 -2
View File
@@ -585,9 +585,9 @@ func (s *Server) prepare(ctx context.Context, j *Job, name string) (prepared, bo
s.step(j, "check", Running, "reading its header from the mirror")
detail := "no header to read"
if u := headerURL(source, set); u != "" {
v := gguf.Check(ctx, u, s.deps.LlamaVersion)
v := gguf.CheckAny(ctx, u, s.llamaVersions())
if !v.Servable {
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run the new weights: %s", s.deps.LlamaVersion, v.Reason))
s.fail(j, "check", fmt.Errorf("llama.cpp on the nodes cannot run the new weights: %s", v.Reason))
return prepared{}, false
}
detail = orUnknown(v.Header.Architecture)
+26
View File
@@ -404,6 +404,32 @@ func Check(ctx context.Context, url, llamaVersion string) Verdict {
return v
}
// CheckAny is Check against several llama.cpp builds, the header read once:
// servable when any of them loads it. A cluster whose GPU nodes are given a
// newer build than the rest (engine_overrides) can serve an architecture
// only that build knows. The first version is the cluster's own; its verdict
// is the one given when none loads it.
func CheckAny(ctx context.Context, url string, versions []string) Verdict {
if len(versions) == 0 {
versions = []string{""}
}
v := Check(ctx, url, versions[0])
if v.Servable || v.Header.Architecture == "" {
return v
}
for _, other := range versions[1:] {
if o := LlamaCpp(other, v.Header.Architecture); o.Servable {
o.Header = v.Header
return o
}
}
if len(versions) > 1 {
v.Reason = "its architecture is \"" + v.Header.Architecture + "\", which no llama.cpp the nodes install (" +
strings.Join(versions, ", ") + ") can load"
}
return v
}
// CardUsable is the share of a card's memory a model may take: the rest is
// CUDA's own and the engine's compute buffers.
const CardUsable = 0.9
+22
View File
@@ -221,3 +221,25 @@ func TestAModelOfExpertsReadsAShareOfItsWeights(t *testing.T) {
t.Error("a dense model reads all its weights")
}
}
func TestAnArchitectureOnlyAnOverriddenBuildKnowsIsServable(t *testing.T) {
// Found on the PoC: Inkling 2 Mini (inkling2_loop) refused by `ctl model
// serve` for the cluster's b10991, though the GPU node had been given
// b12606 through engine_overrides.
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.ServeContent(w, r, "m.gguf", timeZero, bytes.NewReader(header("inkling2_loop")))
}))
defer srv.Close()
url := srv.URL + "/m.gguf"
if v := CheckAny(context.Background(), url, []string{"b10991"}); v.Servable {
t.Errorf("b10991 alone: %+v", v)
}
if v := CheckAny(context.Background(), url, []string{"b10991", "b12606"}); !v.Servable ||
v.Header.Architecture != "inkling2_loop" {
t.Errorf("with b12606 on some nodes: %+v", v)
}
if v := CheckAny(context.Background(), url, []string{"b10991", "b10991"}); v.Servable ||
!strings.Contains(v.Reason, "no llama.cpp the nodes install (b10991, b10991)") {
t.Errorf("refused by every build: %+v", v)
}
}