A GGUF is servable when a llama.cpp that engine_overrides give some nodes loads it
CI / test (push) Failing after 9m58s
CI / test (push) Failing after 9m58s
`ctl model serve`, its plan and model updates judged a file only against the cluster's own llama.cpp, so a GPU node given a newer build could not be served an architecture only that build knows: Inkling 2 Mini (inkling2_loop) was refused for b10991 with cuda01 on b12606. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0ffd633bf6
commit
09ebe77513
8 files changed
+99
-10
No files matched your search
@@ -68,10 +68,12 @@ type Deps struct {
|
||||
Catalog Catalog
|
||||
Fleet Fleet
|
||||
// LlamaVersion is the llama.cpp build the nodes install, which decides
|
||||
// which architectures load.
|
||||
LlamaVersion string
|
||||
HTTP *http.Client
|
||||
Log *slog.Logger
|
||||
// which architectures load. LlamaOverrides are the builds some nodes are
|
||||
// given instead (engine_overrides): a file one of them loads is servable.
|
||||
LlamaVersion string
|
||||
LlamaOverrides []string
|
||||
HTTP *http.Client
|
||||
Log *slog.Logger
|
||||
// Poll is how often the job looks at the mirror and the fleet.
|
||||
Poll time.Duration
|
||||
// ReadyTimeout bounds the wait for a replica: the node fetching the
|
||||
@@ -240,6 +242,18 @@ type Server struct {
|
||||
}
|
||||
|
||||
// New builds a server.
|
||||
// llamaVersions are the llama.cpp builds the nodes install, the cluster's own
|
||||
// first.
|
||||
func (s *Server) llamaVersions() []string {
|
||||
out := []string{s.deps.LlamaVersion}
|
||||
for _, v := range s.deps.LlamaOverrides {
|
||||
if !slices.Contains(out, v) {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func New(d Deps) *Server {
|
||||
if d.Poll <= 0 {
|
||||
d.Poll = 2 * time.Second
|
||||
@@ -553,9 +567,9 @@ func (s *Server) run(j *Job) {
|
||||
s.step(j, "check", Running, "reading its header from the mirror")
|
||||
var header gguf.Header
|
||||
if u := headerURL(source, set); u != "" {
|
||||
v := gguf.Check(ctx, u, s.deps.LlamaVersion)
|
||||
v := gguf.CheckAny(ctx, u, s.llamaVersions())
|
||||
if !v.Servable {
|
||||
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run this: %s", s.deps.LlamaVersion, v.Reason))
|
||||
s.fail(j, "check", fmt.Errorf("llama.cpp on the nodes cannot run this: %s", v.Reason))
|
||||
return
|
||||
}
|
||||
header = v.Header
|
||||
|
||||
@@ -147,7 +147,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan,
|
||||
if b, ok := s.deps.Registry.(interface{ Base() string }); ok {
|
||||
base = b.Base()
|
||||
}
|
||||
v := gguf.Check(ctx, repo.DownloadURL(base, small.Files[0].Path), s.deps.LlamaVersion)
|
||||
v := gguf.CheckAny(ctx, repo.DownloadURL(base, small.Files[0].Path), s.llamaVersions())
|
||||
if v.Checked && !v.Servable {
|
||||
p.Servable, p.Reason = false, v.Reason
|
||||
}
|
||||
|
||||
@@ -585,9 +585,9 @@ func (s *Server) prepare(ctx context.Context, j *Job, name string) (prepared, bo
|
||||
s.step(j, "check", Running, "reading its header from the mirror")
|
||||
detail := "no header to read"
|
||||
if u := headerURL(source, set); u != "" {
|
||||
v := gguf.Check(ctx, u, s.deps.LlamaVersion)
|
||||
v := gguf.CheckAny(ctx, u, s.llamaVersions())
|
||||
if !v.Servable {
|
||||
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run the new weights: %s", s.deps.LlamaVersion, v.Reason))
|
||||
s.fail(j, "check", fmt.Errorf("llama.cpp on the nodes cannot run the new weights: %s", v.Reason))
|
||||
return prepared{}, false
|
||||
}
|
||||
detail = orUnknown(v.Header.Architecture)
|
||||
|
||||
Reference in new issue
Block a user