diff --git a/BACKLOG.md b/BACKLOG.md index 4c5b39b..5814524 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -21,9 +21,8 @@ What is left, most needed first. The detail is in the item each names. 1. ~~**J1**~~ — done 2026-10-16: a restart no longer refused for room another restart was giving back. -2. **J5** — an upgrade leaves an earlier install's judge as it was. A - v0.14.6 install on fewer than eight cores refuses every request until - fixed by hand. (S) +2. ~~**J5**~~ — done 2026-10-16: an upgrade refits an earlier install's + judge by itself. 3. **I3** — what llmgw adds to a request, measured, in the README. (S) 4. **I4** — several keys for one outside provider. (S) 5. **J2** — a judge the leader adds by itself is sized as init's is. (S) @@ -1601,7 +1600,15 @@ models the nodes already have keep serving and new ones wait for it. - Done when: with the first machine powered off, a model added then is fetched and served. -### J5. An upgrade corrects an earlier install's judge (S) +### J5. An upgrade corrects an earlier install's judge (S) — done 2026-10-16 +Built in the judge keeper (`internal/cli/judgekeeper.go`, `refit`), on the +leader: a judge whose entry is exactly what init before v0.14.7 wrote is +given 2 GB, `--cache-ram 0` and cores by init's rule (`refitCores`: up to +four for one judge, two each for more), through `modelsvc.SetProfiles`, +and its ready replicas are restarted (after J1). Its replica count is left +as it is. One that differs is left alone and said once in the log. The +original item follows. + v0.14.6 and earlier installed two judges reserving no cores or one, with 2 or 4 GB and the prompt cache on. On fewer than eight cores every judgment missed its 5 s budget and every request was refused. v0.14.7's notes say diff --git a/CHANGELOG.md b/CHANGELOG.md index b370294..f8fa556 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,12 @@ APIs change between minors without a deprecation period. ### Fixed +- **An upgrade refits the judge an earlier `server init` installed.** Upgrading to v0.14.7 left such a judge as it was: + no cores or one, 2 or 4 GB, the prompt cache on. On fewer than eight cores every request was refused until it was + fixed by hand. Now the leader finds a judge whose entry is exactly what an earlier init wrote and gives it 2 GB, no + prompt cache and cores by today's rule (up to four for one judge, two each for more). It then restarts the judge's + replicas onto that. The number of replicas is left as it is. A judge whose entry was changed by hand is left alone, + and the log says so once. - **A restart is no longer refused for room another restart is giving back.** With every core of a node reserved, two models restarted together, one to need more and one less (as after `ctl model resources` on a judge and a starter), had the first refused: "fits on no node even with this replica stopped", though the second was about to give back diff --git a/internal/cli/judgekeeper.go b/internal/cli/judgekeeper.go index de72f81..0eb4a45 100644 --- a/internal/cli/judgekeeper.go +++ b/internal/cli/judgekeeper.go @@ -32,6 +32,19 @@ type judgeKeeper struct { // tried is when each judge was last asked for, so one that cannot be // fetched is not asked for again on every look. tried map[string]time.Time + + // What refitting an earlier install's judge needs: the catalog to change + // its profile in, the replicas to restart, and the nodes' CPUs. Any nil + // leaves judges as they are. + profiles interface { + SetProfiles(ctx context.Context, name string, profiles []catalog.Profile, by string) (catalog.Model, error) + } + restart func(ids ...string) error + replicas func(model string) []string + nodeCPUs func() []int + // looked is each judge already refitted or found changed by hand, so + // each is said once. + looked map[string]bool } // judgeRetry is how long a judge whose serving failed waits before it is @@ -69,6 +82,7 @@ func (k *judgeKeeper) check() { have := map[string]bool{} for _, m := range k.models() { have[m.Name] = true + k.refit(m) } busy := map[string]bool{} for _, j := range k.serve.Jobs() { @@ -114,3 +128,88 @@ func (k *judgeKeeper) check() { "judge", name, "repo", kj.Repo, "file", kj.File, "job", job.ID) } } + +// refit moves a judge an earlier `server init` installed to what init +// installs now, and restarts its replicas onto it. Those judges reserved no +// cores or one, 2 or 4 GB, and ran llama.cpp's prompt cache: on fewer than +// eight cores every judgment missed its 5 s budget and fail: closed refused +// every request, until fixed by hand. A judge whose profile differs in any +// other way was set by somebody, and is left as it is, said once. +func (k *judgeKeeper) refit(m catalog.Model) { + if k.profiles == nil || k.restart == nil || k.replicas == nil || m.Name != defaultJudge || !m.IsGuard() { + return + } + if k.looked == nil { + k.looked = map[string]bool{} + } + if k.looked[m.Name] { + return + } + k.looked[m.Name] = true + if !earlierInitJudge(m) { + if len(m.Profiles) != 1 || !slices.Contains(m.Profiles[0].Args, "--cache-ram") { + k.log.Info("the judge's profile is not one init wrote; left as it is", + "judge", m.Name, "see", "llmgw ctl model resources "+m.Name) + } + return + } + var cpus []int + if k.nodeCPUs != nil { + cpus = k.nodeCPUs() + } + p := m.Profiles[0] + p.MinMemoryGB = judgeMemoryGB + p.CPUCores = refitCores(m.Scaling.Min, cpus) + p.Args = []string{"--ctx-size", "8192", "--cache-ram", "0"} + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if _, err := k.profiles.SetProfiles(ctx, m.Name, []catalog.Profile{p}, "llmgw (the judge an earlier init installed)"); err != nil { + k.log.Warn("could not refit the judge an earlier init installed", "judge", m.Name, "err", err) + k.looked[m.Name] = false + return + } + ids := k.replicas(m.Name) + if len(ids) > 0 { + if err := k.restart(ids...); err != nil { + k.log.Warn("refitted the judge, but could not restart it", "judge", m.Name, "err", err) + } + } + k.log.Info("refitted the judge an earlier init installed", "judge", m.Name, + "memory_gb", p.MinMemoryGB, "cpu_cores", p.CPUCores, "replicas_restarted", len(ids)) +} + +// earlierInitJudge is whether a judge's entry is exactly what `server init` +// before v0.14.7 wrote: one llama.cpp profile on CPUs, an 8192 context and +// no other args, 2 or 4 GB, none to two cores. +func earlierInitJudge(m catalog.Model) bool { + if len(m.Profiles) != 1 { + return false + } + p := m.Profiles[0] + return p.Name == "cpu" && p.Engine == "llamacpp" && p.GPUClass == "" && p.GPU == nil && + len(p.NodeSelector) == 0 && slices.Equal(p.Args, []string{"--ctx-size", "8192"}) && + (p.MinMemoryGB == 2 || p.MinMemoryGB == 4) && p.CPUCores >= 0 && p.CPUCores <= 2 +} + +// refitCores is a judge replica's cores by init's rule: up to four for the +// one judge of a cluster, two each where there are more, fewer where the +// smallest node cannot hold them all. +func refitCores(replicas int, cpus []int) int { + smallest := 0 + for _, c := range cpus { + if c > 0 && (smallest == 0 || c < smallest) { + smallest = c + } + } + if replicas <= 1 { + if smallest == 0 { + return judgeCores + } + return min(smallest, judgeCoresAlone) + } + cores := judgeCores + if len(cpus) == 1 && smallest > 0 && replicas*cores > smallest { + cores = max(1, smallest/replicas) + } + return cores +} diff --git a/internal/cli/judgekeeper_test.go b/internal/cli/judgekeeper_test.go index accd62e..9f4d0c5 100644 --- a/internal/cli/judgekeeper_test.go +++ b/internal/cli/judgekeeper_test.go @@ -1,6 +1,7 @@ package cli import ( + "context" "errors" "log/slog" "slices" @@ -124,3 +125,81 @@ func TestCategoriesFollowTheJudges(t *testing.T) { t.Errorf("no categories (every one counts) became %v", got) } } + +type fakeProfiles struct { + set map[string][]catalog.Profile +} + +func (f *fakeProfiles) SetProfiles(_ context.Context, name string, ps []catalog.Profile, _ string) (catalog.Model, error) { + if f.set == nil { + f.set = map[string][]catalog.Profile{} + } + f.set[name] = ps + return catalog.Model{Name: name, Profiles: ps}, nil +} + +// A judge that v0.14.6 or earlier's init installed is moved to today's +// figures, once, and its replicas restarted onto them; one somebody changed +// is left alone. +func TestAnEarlierInstallsJudgeIsRefittedOnce(t *testing.T) { + lib := guard.Config{Policies: map[string]guard.Policy{ + "baseline": {Judge: &guard.JudgePolicy{Model: "qwen3guard-0.6b", Input: "block"}}, + }} + earlier := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf", 2, 1) + earlier.Profiles[0].MinMemoryGB, earlier.Profiles[0].Args = 4, []string{"--ctx-size", "8192"} + inCatalog := []catalog.Model{earlier} + fp := &fakeProfiles{} + var restarted []string + k := &judgeKeeper{ + library: func() guard.Config { return lib }, + models: func() []catalog.Model { return inCatalog }, + serve: &fakeServe{}, + log: slog.New(slog.DiscardHandler), + profiles: fp, + restart: func(ids ...string) error { restarted = append(restarted, ids...); return nil }, + replicas: func(string) []string { return []string{"j-1", "j-2"} }, + nodeCPUs: func() []int { return []int{4} }, + } + k.check() + got := fp.set["qwen3guard-0.6b"] + if len(got) != 1 || got[0].MinMemoryGB != 2 || got[0].CPUCores != 2 || !slices.Contains(got[0].Args, "--cache-ram") { + t.Fatalf("refitted to %+v", got) + } + if !slices.Equal(restarted, []string{"j-1", "j-2"}) { + t.Errorf("restarted %v", restarted) + } + k.check() + if len(restarted) != 2 { + t.Error("refitted twice") + } + + // Changed by somebody: an extra arg. Left as it is. + mine := guardModel("http://x/g.gguf", 2, 3) + mine.Profiles[0].Args = []string{"--ctx-size", "16384"} + inCatalog = []catalog.Model{mine} + fp.set, k.looked = nil, nil + k.check() + if fp.set != nil { + t.Errorf("a judge set by hand was changed: %+v", fp.set) + } +} + +func TestAJudgeIsRefittedToTheCoresInitWouldGive(t *testing.T) { + for _, c := range []struct { + replicas int + cpus []int + want int + }{ + {1, []int{4}, 4}, + {1, []int{2}, 2}, + {1, []int{16}, 4}, + {2, []int{4}, 2}, + {2, []int{2}, 1}, + {2, []int{4, 32}, 2}, + {2, nil, 2}, + } { + if got := refitCores(c.replicas, c.cpus); got != c.want { + t.Errorf("%d judges on %v: %d cores, want %d", c.replicas, c.cpus, got, c.want) + } + } +} diff --git a/internal/cli/server.go b/internal/cli/server.go index 6e6b6e9..4542333 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -593,6 +593,25 @@ func newServerCmd() *cobra.Command { }, gpus: func() []adminapi.ClassUse { return adminAPI.Summary().GPUs }, log: log, + // An earlier install's judge, moved to today's figures. + profiles: catalogSvc, + restart: admin.Reconciler().Restart, + replicas: func(model string) []string { + var ids []string + for _, r := range admin.Reconciler().ReadyReplicas() { + if r.Model == model { + ids = append(ids, r.ID) + } + } + return ids + }, + nodeCPUs: func() []int { + var out []int + for _, n := range admin.Nodes() { + out = append(out, n.CPUCount) + } + return out + }, } go keeper.run(ctx, 30*time.Second) } diff --git a/internal/control/modelsvc/modelsvc.go b/internal/control/modelsvc/modelsvc.go index 8fa1240..c05e38d 100644 --- a/internal/control/modelsvc/modelsvc.go +++ b/internal/control/modelsvc/modelsvc.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "log/slog" + "slices" "sync" "time" @@ -248,6 +249,19 @@ func (s *Service) Resources(ctx context.Context, name, profile string, memoryGB, return final, err } +// SetProfiles replaces a model's profiles, as a model added with them +// would have: for a model whose engines are to run otherwise, its args among +// it. Running replicas keep what they were started with until restarted. +func (s *Service) SetProfiles(ctx context.Context, name string, profiles []catalog.Profile, by string) (catalog.Model, error) { + if len(profiles) == 0 { + return catalog.Model{}, errors.New("a model needs a profile") + } + return s.update(ctx, name, by, "replace its profiles", func(m *catalog.Model) error { + m.Profiles = slices.Clone(profiles) + return nil + }) +} + // SetClass sets a model's capacity class: critical, standard or // opportunistic, or empty to go back to what its scaling implies. func (s *Service) SetClass(ctx context.Context, name, class, by string) (catalog.Model, error) {