A GGUF is servable when a llama.cpp that engine_overrides give some nodes loads it
CI / test (push) Failing after 9m58s
CI / test (push) Failing after 9m58s
`ctl model serve`, its plan and model updates judged a file only against the cluster's own llama.cpp, so a GPU node given a newer build could not be served an architecture only that build knows: Inkling 2 Mini (inkling2_loop) was refused for b10991 with cuda01 on b12606. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0ffd633bf6
commit
09ebe77513
8 files changed
+99
-10
No files matched your search
@@ -679,7 +679,7 @@ func newServerCmd() *cobra.Command {
|
||||
if adminAPI != nil && catalogSvc != nil {
|
||||
serve := modelserve.New(modelserve.Deps{
|
||||
Registry: registry, Mirror: mc, Catalog: catalogSvc, Fleet: admin,
|
||||
LlamaVersion: cfg.WantedEngines()["llamacpp"], Log: log,
|
||||
LlamaVersion: cfg.WantedEngines()["llamacpp"], LlamaOverrides: cfg.OverrideVersions("llamacpp"), Log: log,
|
||||
// Updates: the catalog in force, the service that
|
||||
// changes it, and what replaces replicas.
|
||||
Current: func() []catalog.Model { return admin.Catalog().Models() },
|
||||
|
||||
@@ -1164,6 +1164,18 @@ func (s Server) EnginesFor(node string, classes []string, labels map[string]stri
|
||||
return out
|
||||
}
|
||||
|
||||
// OverrideVersions are the versions of an engine that engine_overrides give
|
||||
// some nodes, in the order they are set.
|
||||
func (s Server) OverrideVersions(engine string) []string {
|
||||
var out []string
|
||||
for _, o := range s.EngineOverrides {
|
||||
if v := o.Engines[engine]; v != "" && !slices.Contains(out, v) {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// WantedEngines is DefaultEngines with the configuration laid over it.
|
||||
func (s Server) WantedEngines() map[string]string {
|
||||
out := map[string]string{}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -54,3 +55,17 @@ func TestOIDCSettingsAreCheckedBeforeAProviderIsAsked(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverrideVersionsAreTheEnginesNodesAreGivenInstead(t *testing.T) {
|
||||
s := Server{EngineOverrides: []EngineOverride{
|
||||
{Nodes: []string{"cuda01"}, Engines: map[string]string{"strata": "0.2.7", "llamacpp": "b12606"}},
|
||||
{GPUClass: "v100-32g", Engines: map[string]string{"llamacpp": ""}},
|
||||
{Labels: map[string]string{"site": "b"}, Engines: map[string]string{"llamacpp": "b12606"}},
|
||||
}}
|
||||
if got := s.OverrideVersions("llamacpp"); !slices.Equal(got, []string{"b12606"}) {
|
||||
t.Errorf("llamacpp: %v", got)
|
||||
}
|
||||
if got := s.OverrideVersions("vllm"); got != nil {
|
||||
t.Errorf("vllm: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -68,10 +68,12 @@ type Deps struct {
|
||||
Catalog Catalog
|
||||
Fleet Fleet
|
||||
// LlamaVersion is the llama.cpp build the nodes install, which decides
|
||||
// which architectures load.
|
||||
LlamaVersion string
|
||||
HTTP *http.Client
|
||||
Log *slog.Logger
|
||||
// which architectures load. LlamaOverrides are the builds some nodes are
|
||||
// given instead (engine_overrides): a file one of them loads is servable.
|
||||
LlamaVersion string
|
||||
LlamaOverrides []string
|
||||
HTTP *http.Client
|
||||
Log *slog.Logger
|
||||
// Poll is how often the job looks at the mirror and the fleet.
|
||||
Poll time.Duration
|
||||
// ReadyTimeout bounds the wait for a replica: the node fetching the
|
||||
@@ -240,6 +242,18 @@ type Server struct {
|
||||
}
|
||||
|
||||
// New builds a server.
|
||||
// llamaVersions are the llama.cpp builds the nodes install, the cluster's own
|
||||
// first.
|
||||
func (s *Server) llamaVersions() []string {
|
||||
out := []string{s.deps.LlamaVersion}
|
||||
for _, v := range s.deps.LlamaOverrides {
|
||||
if !slices.Contains(out, v) {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func New(d Deps) *Server {
|
||||
if d.Poll <= 0 {
|
||||
d.Poll = 2 * time.Second
|
||||
@@ -553,9 +567,9 @@ func (s *Server) run(j *Job) {
|
||||
s.step(j, "check", Running, "reading its header from the mirror")
|
||||
var header gguf.Header
|
||||
if u := headerURL(source, set); u != "" {
|
||||
v := gguf.Check(ctx, u, s.deps.LlamaVersion)
|
||||
v := gguf.CheckAny(ctx, u, s.llamaVersions())
|
||||
if !v.Servable {
|
||||
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run this: %s", s.deps.LlamaVersion, v.Reason))
|
||||
s.fail(j, "check", fmt.Errorf("llama.cpp on the nodes cannot run this: %s", v.Reason))
|
||||
return
|
||||
}
|
||||
header = v.Header
|
||||
|
||||
@@ -147,7 +147,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan,
|
||||
if b, ok := s.deps.Registry.(interface{ Base() string }); ok {
|
||||
base = b.Base()
|
||||
}
|
||||
v := gguf.Check(ctx, repo.DownloadURL(base, small.Files[0].Path), s.deps.LlamaVersion)
|
||||
v := gguf.CheckAny(ctx, repo.DownloadURL(base, small.Files[0].Path), s.llamaVersions())
|
||||
if v.Checked && !v.Servable {
|
||||
p.Servable, p.Reason = false, v.Reason
|
||||
}
|
||||
|
||||
@@ -585,9 +585,9 @@ func (s *Server) prepare(ctx context.Context, j *Job, name string) (prepared, bo
|
||||
s.step(j, "check", Running, "reading its header from the mirror")
|
||||
detail := "no header to read"
|
||||
if u := headerURL(source, set); u != "" {
|
||||
v := gguf.Check(ctx, u, s.deps.LlamaVersion)
|
||||
v := gguf.CheckAny(ctx, u, s.llamaVersions())
|
||||
if !v.Servable {
|
||||
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run the new weights: %s", s.deps.LlamaVersion, v.Reason))
|
||||
s.fail(j, "check", fmt.Errorf("llama.cpp on the nodes cannot run the new weights: %s", v.Reason))
|
||||
return prepared{}, false
|
||||
}
|
||||
detail = orUnknown(v.Header.Architecture)
|
||||
|
||||
@@ -404,6 +404,32 @@ func Check(ctx context.Context, url, llamaVersion string) Verdict {
|
||||
return v
|
||||
}
|
||||
|
||||
// CheckAny is Check against several llama.cpp builds, the header read once:
|
||||
// servable when any of them loads it. A cluster whose GPU nodes are given a
|
||||
// newer build than the rest (engine_overrides) can serve an architecture
|
||||
// only that build knows. The first version is the cluster's own; its verdict
|
||||
// is the one given when none loads it.
|
||||
func CheckAny(ctx context.Context, url string, versions []string) Verdict {
|
||||
if len(versions) == 0 {
|
||||
versions = []string{""}
|
||||
}
|
||||
v := Check(ctx, url, versions[0])
|
||||
if v.Servable || v.Header.Architecture == "" {
|
||||
return v
|
||||
}
|
||||
for _, other := range versions[1:] {
|
||||
if o := LlamaCpp(other, v.Header.Architecture); o.Servable {
|
||||
o.Header = v.Header
|
||||
return o
|
||||
}
|
||||
}
|
||||
if len(versions) > 1 {
|
||||
v.Reason = "its architecture is \"" + v.Header.Architecture + "\", which no llama.cpp the nodes install (" +
|
||||
strings.Join(versions, ", ") + ") can load"
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// CardUsable is the share of a card's memory a model may take: the rest is
|
||||
// CUDA's own and the engine's compute buffers.
|
||||
const CardUsable = 0.9
|
||||
|
||||
@@ -221,3 +221,25 @@ func TestAModelOfExpertsReadsAShareOfItsWeights(t *testing.T) {
|
||||
t.Error("a dense model reads all its weights")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnArchitectureOnlyAnOverriddenBuildKnowsIsServable(t *testing.T) {
|
||||
// Found on the PoC: Inkling 2 Mini (inkling2_loop) refused by `ctl model
|
||||
// serve` for the cluster's b10991, though the GPU node had been given
|
||||
// b12606 through engine_overrides.
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.ServeContent(w, r, "m.gguf", timeZero, bytes.NewReader(header("inkling2_loop")))
|
||||
}))
|
||||
defer srv.Close()
|
||||
url := srv.URL + "/m.gguf"
|
||||
if v := CheckAny(context.Background(), url, []string{"b10991"}); v.Servable {
|
||||
t.Errorf("b10991 alone: %+v", v)
|
||||
}
|
||||
if v := CheckAny(context.Background(), url, []string{"b10991", "b12606"}); !v.Servable ||
|
||||
v.Header.Architecture != "inkling2_loop" {
|
||||
t.Errorf("with b12606 on some nodes: %+v", v)
|
||||
}
|
||||
if v := CheckAny(context.Background(), url, []string{"b10991", "b10991"}); v.Servable ||
|
||||
!strings.Contains(v.Reason, "no llama.cpp the nodes install (b10991, b10991)") {
|
||||
t.Errorf("refused by every build: %+v", v)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user