A GGUF is servable when a llama.cpp that engine_overrides give some nodes loads it
CI / test (push) Failing after 9m58s

`ctl model serve`, its plan and model updates judged a file only against
the cluster's own llama.cpp, so a GPU node given a newer build could not
be served an architecture only that build knows: Inkling 2 Mini
(inkling2_loop) was refused for b10991 with cuda01 on b12606.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-21 13:43:40 +01:00
1 parent 0ffd633bf6
commit 09ebe77513
8 files changed
+99 -10

No files matched your search

+20 -6
View File
@@ -68,10 +68,12 @@ type Deps struct {
Catalog Catalog
Fleet Fleet
// LlamaVersion is the llama.cpp build the nodes install, which decides
// which architectures load.
LlamaVersion string
HTTP *http.Client
Log *slog.Logger
// which architectures load. LlamaOverrides are the builds some nodes are
// given instead (engine_overrides): a file one of them loads is servable.
LlamaVersion string
LlamaOverrides []string
HTTP *http.Client
Log *slog.Logger
// Poll is how often the job looks at the mirror and the fleet.
Poll time.Duration
// ReadyTimeout bounds the wait for a replica: the node fetching the
@@ -240,6 +242,18 @@ type Server struct {
}
// New builds a server.
// llamaVersions are the llama.cpp builds the nodes install, the cluster's own
// first.
func (s *Server) llamaVersions() []string {
out := []string{s.deps.LlamaVersion}
for _, v := range s.deps.LlamaOverrides {
if !slices.Contains(out, v) {
out = append(out, v)
}
}
return out
}
func New(d Deps) *Server {
if d.Poll <= 0 {
d.Poll = 2 * time.Second
@@ -553,9 +567,9 @@ func (s *Server) run(j *Job) {
s.step(j, "check", Running, "reading its header from the mirror")
var header gguf.Header
if u := headerURL(source, set); u != "" {
v := gguf.Check(ctx, u, s.deps.LlamaVersion)
v := gguf.CheckAny(ctx, u, s.llamaVersions())
if !v.Servable {
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run this: %s", s.deps.LlamaVersion, v.Reason))
s.fail(j, "check", fmt.Errorf("llama.cpp on the nodes cannot run this: %s", v.Reason))
return
}
header = v.Header
+1 -1
View File
@@ -147,7 +147,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan,
if b, ok := s.deps.Registry.(interface{ Base() string }); ok {
base = b.Base()
}
v := gguf.Check(ctx, repo.DownloadURL(base, small.Files[0].Path), s.deps.LlamaVersion)
v := gguf.CheckAny(ctx, repo.DownloadURL(base, small.Files[0].Path), s.llamaVersions())
if v.Checked && !v.Servable {
p.Servable, p.Reason = false, v.Reason
}
+2 -2
View File
@@ -585,9 +585,9 @@ func (s *Server) prepare(ctx context.Context, j *Job, name string) (prepared, bo
s.step(j, "check", Running, "reading its header from the mirror")
detail := "no header to read"
if u := headerURL(source, set); u != "" {
v := gguf.Check(ctx, u, s.deps.LlamaVersion)
v := gguf.CheckAny(ctx, u, s.llamaVersions())
if !v.Servable {
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run the new weights: %s", s.deps.LlamaVersion, v.Reason))
s.fail(j, "check", fmt.Errorf("llama.cpp on the nodes cannot run the new weights: %s", v.Reason))
return prepared{}, false
}
detail = orUnknown(v.Header.Architecture)