The judge init installs is no longer replaced by the leader's own
Found on a fresh HA install with a judge machine: the leader saw the policy's judge missing while init was still fetching it, served one itself, and its entry replaced init's, so a second judge ran off the judge machine. The keeper now waits ten minutes after it starts, and a serve job it starts gives way to a model added meanwhile (AddNew). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
34764f5ef0
commit
314cc5e6a3
9 files changed
+131
-2
No files matched your search
@@ -6,6 +6,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/url"
|
||||
@@ -23,6 +24,9 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
)
|
||||
|
||||
// ErrExists is a model added only if absent that was there already.
|
||||
var ErrExists = errors.New("already in the catalog")
|
||||
|
||||
// Scaling is how many replicas a model should have.
|
||||
type Scaling struct {
|
||||
Min int `json:"min,omitempty" koanf:"min"`
|
||||
|
||||
@@ -163,6 +163,10 @@ type Request struct {
|
||||
// CPUCores is what a replica computes on; zero takes half the smallest
|
||||
// node it fits on.
|
||||
CPUCores int `json:"cpu_cores,omitempty"`
|
||||
// IfAbsent adds the model only if nobody has meanwhile: the leader's own
|
||||
// serving of a judge a policy names gives way to the one init or an
|
||||
// operator added while it fetched, rather than replacing it.
|
||||
IfAbsent bool `json:"-"`
|
||||
// MemoryGB is what a replica on CPUs reserves, in place of the figure
|
||||
// worked out from its weights and context: a judge's, whose own sizing
|
||||
// is known.
|
||||
@@ -691,7 +695,18 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
|
||||
}
|
||||
m.Class = req.Class
|
||||
s.step(j, "catalog", Running, "adding "+name)
|
||||
if err := s.deps.Catalog.Add(ctx, m, req.By); err != nil {
|
||||
add := s.deps.Catalog.Add
|
||||
if adder, ok := s.deps.Catalog.(interface {
|
||||
AddNew(ctx context.Context, m catalog.Model, by string) error
|
||||
}); ok && req.IfAbsent {
|
||||
add = adder.AddNew
|
||||
}
|
||||
if err := add(ctx, m, req.By); err != nil {
|
||||
if req.IfAbsent && errors.Is(err, catalog.ErrExists) {
|
||||
s.step(j, "catalog", Skipped, name+" was added meanwhile; left as it is")
|
||||
s.finish(j)
|
||||
return
|
||||
}
|
||||
s.fail(j, "catalog", err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -452,3 +452,26 @@ func TestAJudgeIsServedWithTheMemoryAskedAndNoPromptCache(t *testing.T) {
|
||||
t.Error("a model that is not a judge lost its prompt cache")
|
||||
}
|
||||
}
|
||||
|
||||
// takenCatalog has the model by the time a job comes to add it.
|
||||
type takenCatalog struct{ fakeCatalog }
|
||||
|
||||
func (c *takenCatalog) AddNew(context.Context, catalog.Model, string) error {
|
||||
return fmt.Errorf("model %q: %w", "x", catalog.ErrExists)
|
||||
}
|
||||
|
||||
func TestAJobAddingOnlyIfAbsentGivesWayToAModelAddedMeanwhile(t *testing.T) {
|
||||
files := []hf.File{{Path: "Qwen3Guard-Gen-0.6B-Q8_0.gguf", Bytes: 600 << 20, SHA256: "aa"}}
|
||||
s, _, _, _ := setup(t, files, nil)
|
||||
taken := &takenCatalog{}
|
||||
s.deps.Catalog = taken
|
||||
job, err := s.Start(Request{Repo: "example/Qwen3Guard-Gen-0.6B-GGUF", File: "Qwen3Guard-Gen-0.6B-Q8_0.gguf",
|
||||
GuardFormat: "qwen3guard", CPU: true, CPUCores: 2, MemoryGB: 2, Replicas: 2, IfAbsent: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j := wait(t, s, job.ID)
|
||||
if j.State != Done || len(taken.added) != 0 {
|
||||
t.Errorf("job %s, added %d: %+v", j.State, len(taken.added), j.Steps)
|
||||
}
|
||||
}
|
||||
@@ -106,6 +106,17 @@ func (s *Service) Sources(ctx context.Context) (map[string][]catalog.Model, erro
|
||||
// than persisted and then skipped on every future start — which would be a
|
||||
// model that looks added and never appears.
|
||||
func (s *Service) Add(ctx context.Context, m catalog.Model, by string) error {
|
||||
return s.add(ctx, m, by, false)
|
||||
}
|
||||
|
||||
// AddNew is Add for a model nobody has added meanwhile: one that is in the
|
||||
// catalog by now is left as it is, and catalog.ErrExists said. The leader serving a
|
||||
// judge a policy names gives way so to the judge init added while it fetched.
|
||||
func (s *Service) AddNew(ctx context.Context, m catalog.Model, by string) error {
|
||||
return s.add(ctx, m, by, true)
|
||||
}
|
||||
|
||||
func (s *Service) add(ctx context.Context, m catalog.Model, by string, onlyNew bool) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
@@ -135,6 +146,9 @@ func (s *Service) Add(ctx context.Context, m catalog.Model, by string) error {
|
||||
next := s.base(stored)
|
||||
for _, e := range stored {
|
||||
if e.Name == m.Name {
|
||||
if onlyNew {
|
||||
return fmt.Errorf("model %q: %w", m.Name, catalog.ErrExists)
|
||||
}
|
||||
continue // this call replaces it
|
||||
}
|
||||
next = append(next, e)
|
||||
|
||||
@@ -3,6 +3,7 @@ package modelsvc
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -377,3 +378,25 @@ func TestResourcesAreSetOnAStoredModelAndSurviveARestart(t *testing.T) {
|
||||
t.Error("the cores were lost on reload")
|
||||
}
|
||||
}
|
||||
|
||||
// The leader serving a judge gives way to the one init added while it
|
||||
// fetched: AddNew leaves a model already there as it is.
|
||||
func TestAddNewLeavesAModelAddedMeanwhile(t *testing.T) {
|
||||
s, _, rec := newService(t)
|
||||
first := model("judge")
|
||||
first.Description = "init's"
|
||||
if err := s.Add(t.Context(), first, "init"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second := model("judge")
|
||||
second.Description = "the leader's"
|
||||
if err := s.AddNew(t.Context(), second, "llmgw"); !errors.Is(err, catalog.ErrExists) {
|
||||
t.Fatalf("err %v", err)
|
||||
}
|
||||
if m, _ := rec.last.Resolve("judge"); m.Description != "init's" {
|
||||
t.Errorf("replaced: %q", m.Description)
|
||||
}
|
||||
if err := s.AddNew(t.Context(), model("fresh"), "llmgw"); err != nil {
|
||||
t.Errorf("a new model: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user