The judge init installs is no longer replaced by the leader's own
CI / test (push) Successful in 5m27s

Found on a fresh HA install with a judge machine: the leader saw the
policy's judge missing while init was still fetching it, served one
itself, and its entry replaced init's, so a second judge ran off the
judge machine. The keeper now waits ten minutes after it starts, and a
serve job it starts gives way to a model added meanwhile (AddNew).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-19 20:37:10 +02:00
1 parent 34764f5ef0
commit 314cc5e6a3
9 files changed
+131 -2

No files matched your search

+16 -1
View File
@@ -163,6 +163,10 @@ type Request struct {
// CPUCores is what a replica computes on; zero takes half the smallest
// node it fits on.
CPUCores int `json:"cpu_cores,omitempty"`
// IfAbsent adds the model only if nobody has meanwhile: the leader's own
// serving of a judge a policy names gives way to the one init or an
// operator added while it fetched, rather than replacing it.
IfAbsent bool `json:"-"`
// MemoryGB is what a replica on CPUs reserves, in place of the figure
// worked out from its weights and context: a judge's, whose own sizing
// is known.
@@ -691,7 +695,18 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
}
m.Class = req.Class
s.step(j, "catalog", Running, "adding "+name)
if err := s.deps.Catalog.Add(ctx, m, req.By); err != nil {
add := s.deps.Catalog.Add
if adder, ok := s.deps.Catalog.(interface {
AddNew(ctx context.Context, m catalog.Model, by string) error
}); ok && req.IfAbsent {
add = adder.AddNew
}
if err := add(ctx, m, req.By); err != nil {
if req.IfAbsent && errors.Is(err, catalog.ErrExists) {
s.step(j, "catalog", Skipped, name+" was added meanwhile; left as it is")
s.finish(j)
return
}
s.fail(j, "catalog", err)
return
}
@@ -452,3 +452,26 @@ func TestAJudgeIsServedWithTheMemoryAskedAndNoPromptCache(t *testing.T) {
t.Error("a model that is not a judge lost its prompt cache")
}
}
// takenCatalog has the model by the time a job comes to add it.
type takenCatalog struct{ fakeCatalog }
func (c *takenCatalog) AddNew(context.Context, catalog.Model, string) error {
return fmt.Errorf("model %q: %w", "x", catalog.ErrExists)
}
func TestAJobAddingOnlyIfAbsentGivesWayToAModelAddedMeanwhile(t *testing.T) {
files := []hf.File{{Path: "Qwen3Guard-Gen-0.6B-Q8_0.gguf", Bytes: 600 << 20, SHA256: "aa"}}
s, _, _, _ := setup(t, files, nil)
taken := &takenCatalog{}
s.deps.Catalog = taken
job, err := s.Start(Request{Repo: "example/Qwen3Guard-Gen-0.6B-GGUF", File: "Qwen3Guard-Gen-0.6B-Q8_0.gguf",
GuardFormat: "qwen3guard", CPU: true, CPUCores: 2, MemoryGB: 2, Replicas: 2, IfAbsent: true})
if err != nil {
t.Fatal(err)
}
j := wait(t, s, job.ID)
if j.State != Done || len(taken.added) != 0 {
t.Errorf("job %s, added %d: %+v", j.State, len(taken.added), j.Steps)
}
}