An upgrade refits the judge an earlier server init installed
CI / test (push) Successful in 8m53s

A v0.14.6 install on fewer than eight cores refused every request until
its judge was given cores by hand, and upgrading did not change that. The
judge keeper, on the leader, now finds a judge whose entry is exactly what
init before v0.14.7 wrote: one llama.cpp CPU profile, an 8192 context and
nothing else, 2 or 4 GB, none to two cores. It gives the judge 2 GB,
--cache-ram 0 and cores by init's rule through modelsvc.SetProfiles, then
restarts its ready replicas. The replica count is left as it is. A judge
whose entry differs is left alone, and the log says so once.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-16 22:06:56 +02:00
1 parent b8274d6278
commit 2545fca174
6 files changed
+228 -4

No files matched your search

+99
View File
@@ -32,6 +32,19 @@ type judgeKeeper struct {
// tried is when each judge was last asked for, so one that cannot be
// fetched is not asked for again on every look.
tried map[string]time.Time
// What refitting an earlier install's judge needs: the catalog to change
// its profile in, the replicas to restart, and the nodes' CPUs. Any nil
// leaves judges as they are.
profiles interface {
SetProfiles(ctx context.Context, name string, profiles []catalog.Profile, by string) (catalog.Model, error)
}
restart func(ids ...string) error
replicas func(model string) []string
nodeCPUs func() []int
// looked is each judge already refitted or found changed by hand, so
// each is said once.
looked map[string]bool
}
// judgeRetry is how long a judge whose serving failed waits before it is
@@ -69,6 +82,7 @@ func (k *judgeKeeper) check() {
have := map[string]bool{}
for _, m := range k.models() {
have[m.Name] = true
k.refit(m)
}
busy := map[string]bool{}
for _, j := range k.serve.Jobs() {
@@ -114,3 +128,88 @@ func (k *judgeKeeper) check() {
"judge", name, "repo", kj.Repo, "file", kj.File, "job", job.ID)
}
}
// refit moves a judge an earlier `server init` installed to what init
// installs now, and restarts its replicas onto it. Those judges reserved no
// cores or one, 2 or 4 GB, and ran llama.cpp's prompt cache: on fewer than
// eight cores every judgment missed its 5 s budget and fail: closed refused
// every request, until fixed by hand. A judge whose profile differs in any
// other way was set by somebody, and is left as it is, said once.
func (k *judgeKeeper) refit(m catalog.Model) {
if k.profiles == nil || k.restart == nil || k.replicas == nil || m.Name != defaultJudge || !m.IsGuard() {
return
}
if k.looked == nil {
k.looked = map[string]bool{}
}
if k.looked[m.Name] {
return
}
k.looked[m.Name] = true
if !earlierInitJudge(m) {
if len(m.Profiles) != 1 || !slices.Contains(m.Profiles[0].Args, "--cache-ram") {
k.log.Info("the judge's profile is not one init wrote; left as it is",
"judge", m.Name, "see", "llmgw ctl model resources "+m.Name)
}
return
}
var cpus []int
if k.nodeCPUs != nil {
cpus = k.nodeCPUs()
}
p := m.Profiles[0]
p.MinMemoryGB = judgeMemoryGB
p.CPUCores = refitCores(m.Scaling.Min, cpus)
p.Args = []string{"--ctx-size", "8192", "--cache-ram", "0"}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if _, err := k.profiles.SetProfiles(ctx, m.Name, []catalog.Profile{p}, "llmgw (the judge an earlier init installed)"); err != nil {
k.log.Warn("could not refit the judge an earlier init installed", "judge", m.Name, "err", err)
k.looked[m.Name] = false
return
}
ids := k.replicas(m.Name)
if len(ids) > 0 {
if err := k.restart(ids...); err != nil {
k.log.Warn("refitted the judge, but could not restart it", "judge", m.Name, "err", err)
}
}
k.log.Info("refitted the judge an earlier init installed", "judge", m.Name,
"memory_gb", p.MinMemoryGB, "cpu_cores", p.CPUCores, "replicas_restarted", len(ids))
}
// earlierInitJudge is whether a judge's entry is exactly what `server init`
// before v0.14.7 wrote: one llama.cpp profile on CPUs, an 8192 context and
// no other args, 2 or 4 GB, none to two cores.
func earlierInitJudge(m catalog.Model) bool {
if len(m.Profiles) != 1 {
return false
}
p := m.Profiles[0]
return p.Name == "cpu" && p.Engine == "llamacpp" && p.GPUClass == "" && p.GPU == nil &&
len(p.NodeSelector) == 0 && slices.Equal(p.Args, []string{"--ctx-size", "8192"}) &&
(p.MinMemoryGB == 2 || p.MinMemoryGB == 4) && p.CPUCores >= 0 && p.CPUCores <= 2
}
// refitCores is a judge replica's cores by init's rule: up to four for the
// one judge of a cluster, two each where there are more, fewer where the
// smallest node cannot hold them all.
func refitCores(replicas int, cpus []int) int {
smallest := 0
for _, c := range cpus {
if c > 0 && (smallest == 0 || c < smallest) {
smallest = c
}
}
if replicas <= 1 {
if smallest == 0 {
return judgeCores
}
return min(smallest, judgeCoresAlone)
}
cores := judgeCores
if len(cpus) == 1 && smallest > 0 && replicas*cores > smallest {
cores = max(1, smallest/replicas)
}
return cores
}
+79
View File
@@ -1,6 +1,7 @@
package cli
import (
"context"
"errors"
"log/slog"
"slices"
@@ -124,3 +125,81 @@ func TestCategoriesFollowTheJudges(t *testing.T) {
t.Errorf("no categories (every one counts) became %v", got)
}
}
type fakeProfiles struct {
set map[string][]catalog.Profile
}
func (f *fakeProfiles) SetProfiles(_ context.Context, name string, ps []catalog.Profile, _ string) (catalog.Model, error) {
if f.set == nil {
f.set = map[string][]catalog.Profile{}
}
f.set[name] = ps
return catalog.Model{Name: name, Profiles: ps}, nil
}
// A judge that v0.14.6 or earlier's init installed is moved to today's
// figures, once, and its replicas restarted onto them; one somebody changed
// is left alone.
func TestAnEarlierInstallsJudgeIsRefittedOnce(t *testing.T) {
lib := guard.Config{Policies: map[string]guard.Policy{
"baseline": {Judge: &guard.JudgePolicy{Model: "qwen3guard-0.6b", Input: "block"}},
}}
earlier := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf", 2, 1)
earlier.Profiles[0].MinMemoryGB, earlier.Profiles[0].Args = 4, []string{"--ctx-size", "8192"}
inCatalog := []catalog.Model{earlier}
fp := &fakeProfiles{}
var restarted []string
k := &judgeKeeper{
library: func() guard.Config { return lib },
models: func() []catalog.Model { return inCatalog },
serve: &fakeServe{},
log: slog.New(slog.DiscardHandler),
profiles: fp,
restart: func(ids ...string) error { restarted = append(restarted, ids...); return nil },
replicas: func(string) []string { return []string{"j-1", "j-2"} },
nodeCPUs: func() []int { return []int{4} },
}
k.check()
got := fp.set["qwen3guard-0.6b"]
if len(got) != 1 || got[0].MinMemoryGB != 2 || got[0].CPUCores != 2 || !slices.Contains(got[0].Args, "--cache-ram") {
t.Fatalf("refitted to %+v", got)
}
if !slices.Equal(restarted, []string{"j-1", "j-2"}) {
t.Errorf("restarted %v", restarted)
}
k.check()
if len(restarted) != 2 {
t.Error("refitted twice")
}
// Changed by somebody: an extra arg. Left as it is.
mine := guardModel("http://x/g.gguf", 2, 3)
mine.Profiles[0].Args = []string{"--ctx-size", "16384"}
inCatalog = []catalog.Model{mine}
fp.set, k.looked = nil, nil
k.check()
if fp.set != nil {
t.Errorf("a judge set by hand was changed: %+v", fp.set)
}
}
func TestAJudgeIsRefittedToTheCoresInitWouldGive(t *testing.T) {
for _, c := range []struct {
replicas int
cpus []int
want int
}{
{1, []int{4}, 4},
{1, []int{2}, 2},
{1, []int{16}, 4},
{2, []int{4}, 2},
{2, []int{2}, 1},
{2, []int{4, 32}, 2},
{2, nil, 2},
} {
if got := refitCores(c.replicas, c.cpus); got != c.want {
t.Errorf("%d judges on %v: %d cores, want %d", c.replicas, c.cpus, got, c.want)
}
}
}
+19
View File
@@ -593,6 +593,25 @@ func newServerCmd() *cobra.Command {
},
gpus: func() []adminapi.ClassUse { return adminAPI.Summary().GPUs },
log: log,
// An earlier install's judge, moved to today's figures.
profiles: catalogSvc,
restart: admin.Reconciler().Restart,
replicas: func(model string) []string {
var ids []string
for _, r := range admin.Reconciler().ReadyReplicas() {
if r.Model == model {
ids = append(ids, r.ID)
}
}
return ids
},
nodeCPUs: func() []int {
var out []int
for _, n := range admin.Nodes() {
out = append(out, n.CPUCount)
}
return out
},
}
go keeper.run(ctx, 30*time.Second)
}