The judge init installs is no longer replaced by the leader's own
CI / test (push) Successful in 5m27s

Found on a fresh HA install with a judge machine: the leader saw the
policy's judge missing while init was still fetching it, served one
itself, and its entry replaced init's, so a second judge ran off the
judge machine. The keeper now waits ten minutes after it starts, and a
serve job it starts gives way to a model added meanwhile (AddNew).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-19 20:37:10 +02:00
1 parent 34764f5ef0
commit 314cc5e6a3
9 files changed
+131 -2

No files matched your search

+17
View File
@@ -32,6 +32,12 @@ type judgeKeeper struct {
// tried is when each judge was last asked for, so one that cannot be
// fetched is not asked for again on every look.
tried map[string]time.Time
// grace is how long after its first look the keeper serves no judge
// itself: init adds its own judge just after the control plane starts,
// sized and placed as the install says, and one the leader added first
// was what init then kept. Zero waits for nothing.
grace time.Duration
first time.Time
// What refitting an earlier install's judge needs: the catalog to change
// its profile in, the replicas to restart, and the nodes' CPUs. Any nil
@@ -51,6 +57,9 @@ type judgeKeeper struct {
// asked for again.
const judgeRetry = 10 * time.Minute
// judgeGrace is the keeper's grace on a control plane just started.
const judgeGrace = 10 * time.Minute
func (k *judgeKeeper) run(ctx context.Context, every time.Duration) {
t := time.NewTicker(every)
defer t.Stop()
@@ -84,6 +93,12 @@ func (k *judgeKeeper) check() {
have[m.Name] = true
k.refit(m)
}
if k.first.IsZero() {
k.first = now()
}
if now().Sub(k.first) < k.grace {
return
}
busy := map[string]bool{}
for _, j := range k.serve.Jobs() {
if j.State == modelserve.Running {
@@ -128,6 +143,8 @@ func (k *judgeKeeper) check() {
}
req := kj.serveRequest(opts)
req.By = "llmgw (a guard policy names it)"
// init, or an operator, may add it while this fetches: theirs stays.
req.IfAbsent = true
job, err := k.serve.Start(req)
if err != nil {
k.log.Error("could not start serving a judge a policy names", "judge", name, "err", err)
+29 -1
View File
@@ -52,7 +52,8 @@ func TestAJudgeAPolicyNamesIsFetchedWhenTheCatalogLacksIt(t *testing.T) {
// The GPU judge alone: the CPU one is there, and one llmgw does not know
// is not guessed at.
if len(fs.started) != 1 || fs.started[0].Name != "qwen3guard-4b" || fs.started[0].GPUClass != "6000-24g" ||
fs.started[0].Class != catalog.ClassStandard || fs.started[0].GuardFormat != "qwen3guard" || fs.started[0].CPU {
fs.started[0].Class != catalog.ClassStandard || fs.started[0].GuardFormat != "qwen3guard" || fs.started[0].CPU ||
!fs.started[0].IfAbsent {
t.Fatalf("started %+v", fs.started)
}
@@ -207,3 +208,30 @@ func TestAJudgeIsRefittedToTheCoresInitWouldGive(t *testing.T) {
}
}
}
// Just after the control plane starts, the judge init is adding is left to
// init: the keeper serves one itself only after its grace.
func TestTheKeeperLeavesAJustStartedInstallsJudgeToInit(t *testing.T) {
lib := guard.Config{Policies: map[string]guard.Policy{"b": {Judge: &guard.JudgePolicy{Model: "qwen3guard-0.6b", Input: "block"}}}}
now := time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC)
fs := &fakeServe{}
k := &judgeKeeper{
library: func() guard.Config { return lib },
models: func() []catalog.Model { return nil },
serve: fs, grace: judgeGrace,
gpus: func() []adminapi.ClassUse { return nil },
log: slog.New(slog.DiscardHandler),
now: func() time.Time { return now },
}
k.check()
now = now.Add(judgeGrace / 2)
k.check()
if len(fs.started) != 0 {
t.Fatalf("served within its grace: %+v", fs.started)
}
now = now.Add(judgeGrace)
k.check()
if len(fs.started) != 1 {
t.Errorf("never served after its grace: %d", len(fs.started))
}
}
+1
View File
@@ -607,6 +607,7 @@ func newServerCmd() *cobra.Command {
// A judge a policy names and the catalog lacks is
// fetched and served, on the leader.
keeper := &judgeKeeper{
grace: judgeGrace,
library: gate.Library,
models: func() []catalog.Model { return admin.Catalog().Models() },
serve: serve,