From 85f135311e408eff345a03366956e31a1754248b Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Tue, 20 Oct 2026 22:35:19 +0200 Subject: [PATCH] How fast the judge is, measured, and said at install ctl guard judge check, and POST /api/v1/guard/judges/check, time three judgments of a 280-token prompt, then a burst at once, against the policies' wait: ok, tight or slow, about how many requests a second can be judged, and what to do. server init runs it once the judge is up and says a machine too slow before the next steps; the wait is not changed without asking. BACKLOG reordered by what goes wrong where installed. Co-Authored-By: Claude Opus 5.5 (1M context) --- BACKLOG.md | 75 ++++++------ CHANGELOG.md | 6 + docs/admin-api.openapi.json | 111 ++++++++++++++++++ docs/install.md | 12 ++ internal/cli/guard_judge.go | 70 ++++++++++- internal/cli/serverinit.go | 9 ++ internal/cli/serverinit_judges.go | 43 +++++++ internal/control/adminapi/guard.go | 1 + internal/control/adminapi/judges.go | 141 +++++++++++++++++++++++ internal/control/adminapi/judges_test.go | 41 +++++++ internal/control/adminapi/routes.go | 2 + internal/guard/judge.go | 3 + 12 files changed, 475 insertions(+), 39 deletions(-) diff --git a/BACKLOG.md b/BACKLOG.md index 425367e..a30464f 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -15,49 +15,48 @@ Conventions this codebase keeps, which every item here must keep too: - `go test ./...`, `go vet ./...` and `govulncheck ./...` pass. No release is cut and nothing is deployed to the PoC cluster without being told to. -## Open, in order (2026-10-16, after v0.14.7) +## Open, in order (2026-10-20) What is left, most needed first. The detail is in the item each names. -1. ~~**J1**~~ — done 2026-10-16: a restart no longer refused for room - another restart was giving back. -2. ~~**J5**~~ — done 2026-10-16: an upgrade refits an earlier install's - judge by itself. -3. ~~**I3**~~ — done 2026-10-16: `ctl bench overhead`, figures in the - README. -4. ~~**I4**~~ — done 2026-10-17: several keys for one outside provider. -5. ~~**Active Directory and Entra ID**~~ — done 2026-10-18: `auth.ldap` - sign-in and sync, Entra's 200-group overflow kept from taking roles. - Still to do: try it against a real domain controller (a Samba AD DC - on Proxmox): LDAPS, nested groups, a disabled user suspended by the - sync. -6. ~~**J2**~~ — done 2026-10-19: a judge the leader adds by itself is - sized as init's is (2 GB, init's cores, no prompt cache). -7. **J3** — HA setup puts the judges on nodes of their own. (S) Code - done 2026-10-19 (setup's judge machines, `--judge-node`); the load test - on a fresh HA install is still to do. -8. **J8** — a judge budget that fits the machine, measured at install. (S) -9. **F5** — large imports limited by the embedding engine's slots. (M) -10. **I1** — answers to similar questions from the cache. (M) -11. **J4** — the model mirror on more than one machine. (M) -12. **I2** — the customer's MCP servers behind llmgw. (L) -13. Smaller leftovers: audit before/after values, a tenant-admin role and - forms hidden from viewers (C1); Parquet and a scheduled push for usage - export (C4); English errors on Turkish pages (C24); H1 tried on the PoC; - licences in video and audio (H5); nginx in front of several control - planes (J7). +**Now: what goes wrong where it is installed.** +1. ~~**J8**~~: done 2026-10-20. `ctl guard judge check`, and init times the judge. Left of **J3**: its load test + against judges on the control plane, which was stopped half way. Judges on a 4-core node of their own refused + 0/60 at 2 at once, 13/60 at 4 and 42/60 at 10. +2. **I2 P2**: the company tools' answers through the guardrails (prompt injection from a ticket), a rate limit per + tool, and a breaker. P1 is live on the PoC. +3. **K**: fixes from the fabric test the operator runs with docs/large-models.md. -14. **K** — large models on several cards and nodes (2026-10-20): code - done (expert parallelism, any number of nodes, memory per card, the KV - head rule, the startup timeout from the last download), and - docs/large-models.md. Left for the operator: trying it on a fabric with - the checklist there. Then fixes from what it shows. +**Soon: the customer's experience, and HA.** +4. **C24**: English errors on Turkish pages. (S) +5. **J4**: the model mirror on more than one machine, the last single point of an HA install. (M) +6. **F5**: large imports limited by the embedding engine's slots. (M) -Not code, the operator's to decide or do: the PoC on v0.14.7 (it runs a -v0.14.5 dev build; no hurry); where Alertmanager sends (its receiver is -"none"). Settled 2026-10-16: Apertus 8B is on demand (min 0); the HA test -VMs 157–159, 165, 166 and the e2e VMs 151, 153–156 (a v0.11 HA test from -2026-10-03, idle since, holding 10.0.0.160 with VRRP id 51) were removed. +**Governance.** +7. **C1**: a tenant-admin role, and forms hidden from viewers. +8. **Audit before and after**: what a change changed, and changes made with the CLI recorded too. + +**When there is room.** +9. **I1**: answers to similar questions from the cache. (M) +10. **H1**: tried on the PoC. +11. **C4**: Parquet, and a scheduled push for usage export. + +**When needed.** +12. **H5**: licences in video and audio. 13. **H3**: replicas shared, the catalog side. 14. **J7**: nginx in front +of several control planes. 15. **J6**: rate limits per process (a note). + +**Done since 2026-10-16:** +- J1, J5, I3, I4, J2. +- Active Directory and Entra ID, tried against a Samba DC, with the console's Sign-in screen. +- J3's code. +- K's code and docs/large-models.md. +- I2 P1 and the cluster's own read-only MCP server (llmgw:admin). +- The node console's Overview; a node's page showing its version and the models on each GPU. + +**Not code, the operator's to decide or do:** +- Where Alertmanager sends (its receiver is "none"): with a judge down, nobody hears. +- The fabric test for K. +- The PoC's control planes run a dev build of main (v0.14.9-11-g4eb49d0); its nodes run v0.14.8. --- diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ab73c1..5359dc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,12 @@ APIs change between minors without a deprecation period. ### Added +- **How fast the judge is, measured.** `llmgw ctl guard judge check` (and `POST /api/v1/guard/judges/check`) works + out whether the judge keeps up with requests. It times three judgments of a 280-token prompt, then a burst at once, + and sets both against the policies' wait. It says ok, tight or slow, about how many requests a second can be judged, + and what to do: more cores, more judges, a GPU judge, a longer wait, or `fail: open`. `server init` runs it once the + judge is up. If the machine is too slow it says so before the next steps, not after the first refused request. + - **The company's own MCP servers through llmgw's `/mcp`.** `llmgw ctl mcp add jira --url … --secret-file Authorization=FILE` registers one; `/mcp` then lists its tools as `jira.search` beside llmgw's own. Only keys granted them see them: `llmgw keys limits --tools 'jira.*'`, patterns at any level of the tenancy tree. None granted anywhere diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index 751d0fc..3dbbb1c 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -1284,6 +1284,48 @@ } } }, + "/api/v1/guard/judges/check": { + "post": { + "operationId": "postGuardJudgesCheck", + "summary": "How a judge keeps up: three judgments in turn, then a burst at once, against the budget", + "tags": [ + "guard" + ], + "parameters": [ + { + "$ref": "#/components/parameters/Actor" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JudgeCheckRequest" + } + } + } + }, + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JudgeCheckView" + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "default": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/api/v1/jobs": { "get": { "operationId": "getJobs", @@ -6755,6 +6797,70 @@ } } }, + "JudgeCheckRequest": { + "type": "object", + "description": "JudgeCheckRequest asks how a judge keeps up: Burst judgments at once after three in turn; zero is two for each ready replica.", + "properties": { + "model": { + "type": "string" + }, + "burst": { + "type": "integer" + } + } + }, + "JudgeCheckView": { + "type": "object", + "description": "JudgeCheckView is how long a judgment took, alone and among others, set against the time a request waits for one.", + "properties": { + "model": { + "type": "string" + }, + "budget_ms": { + "type": "integer", + "format": "int64" + }, + "ready_replicas": { + "type": "integer" + }, + "took_ms": { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + }, + "description": "TookMs are three judgments of the same text, one after another." + }, + "slowest_ms": { + "type": "integer", + "format": "int64" + }, + "burst": { + "type": "integer", + "description": "Burst judgments at once: the slowest of them, how many gave no verdict, and how many a second the judges read." + }, + "burst_slowest_ms": { + "type": "integer", + "format": "int64" + }, + "burst_failed": { + "type": "integer" + }, + "judgments_per_second": { + "type": "number" + }, + "verdict": { + "type": "string", + "description": "Verdict is ok, tight (past half the budget alone, or past it among others) or slow (past it alone); Advice what to do when not ok." + }, + "advice": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, "JudgeView": { "type": "object", "description": "JudgeView is one judge model the guardrail policies ask, and whether it can answer them.", @@ -6795,6 +6901,11 @@ "wanted_replicas": { "type": "integer" }, + "budget_ms": { + "type": "integer", + "format": "int64", + "description": "BudgetMs is how long a request waits for it: the shortest of its policies' latency budgets." + }, "last_error": { "type": "string", "description": "LastError is its last failure to give a verdict, if any." diff --git a/docs/install.md b/docs/install.md index 0f33647..ee5e6f9 100644 --- a/docs/install.md +++ b/docs/install.md @@ -201,6 +201,18 @@ first thing to fail. The choices there are a longer `latency_budget`, `fail: open` (serve the request, recorded as flagged), the judge on a GPU, or `--no-guard-model`. +init times the judge once it is up. If a judgment takes more than half the +wait, or more than the wait when several come at once, it says so before the +next steps, with what to do. The wait itself is not changed without being +asked. `llmgw ctl guard judge check` does the same at any time: + +``` +qwen3guard-0.6b, 2 replica(s) ready, a request waits 5.0 s for it + alone: 2.4, 2.3, 2.3 s + 4 at once: slowest 6.1 s; about 0.7 judgments a second, so about 0.3 requests a second with short answers +tight: a busy moment passes the budget +``` + In a three-machine install the judges are better on machines of their own. On a 4-core control plane that was also a node, 38 of 60 requests under load were refused for want of a judgment. With the judges on a node of their own diff --git a/internal/cli/guard_judge.go b/internal/cli/guard_judge.go index 6cfd5ef..9f9ab4c 100644 --- a/internal/cli/guard_judge.go +++ b/internal/cli/guard_judge.go @@ -1,6 +1,7 @@ package cli import ( + "context" "encoding/json" "errors" "fmt" @@ -33,7 +34,7 @@ func newGuardJudgeCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { "then fetches and serves by itself: " + knownJudgeNames() + ".", } c.AddCommand(newGuardJudgeStatusCmd(client, asJSON), newGuardJudgeEnableCmd(client), - newGuardJudgeUseCmd(client), newGuardJudgeAskCmd(client, asJSON)) + newGuardJudgeUseCmd(client), newGuardJudgeAskCmd(client, asJSON), newGuardJudgeCheckCmd(client, asJSON)) return c } @@ -383,3 +384,70 @@ func newGuardJudgeAskCmd(client func() *ctlClient, asJSON *bool) *cobra.Command c.Flags().StringVar(&answer, "answer", "", "an assistant's reply to TEXT, judged as output is") return c } + +func newGuardJudgeCheckCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { + var model string + var burst int + c := &cobra.Command{ + Use: "check", + Short: "How a judge keeps up: a judgment alone and among others, against the time a request waits", + Long: "Times three judgments of a prompt of about 280 tokens, one after another, then\n" + + "a burst of them at once (two for each ready replica unless --burst says),\n" + + "and sets both against the policies' latency budget. A judgment past the\n" + + "budget is refused under fail: closed; past half of it alone, or past it\n" + + "among others, a busy moment refuses requests. Each request is judged once\n" + + "for its prompt and again every 600 characters of its answer.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + v, err := checkJudge(cmd.Context(), client(), model, burst) + if err != nil { + return err + } + if *asJSON { + return emitJSON(v) + } + fmt.Fprint(cmd.OutOrStdout(), judgeCheckText(v)) + return nil + }, + } + c.Flags().StringVar(&model, "model", "", "the judge (default: the one the policies ask)") + c.Flags().IntVar(&burst, "burst", 0, "judgments at once (default: two for each ready replica)") + return c +} + +func checkJudge(ctx context.Context, c *ctlClient, model string, burst int) (adminapi.JudgeCheckView, error) { + body, _ := json.Marshal(adminapi.JudgeCheckRequest{Model: model, Burst: burst}) + var v adminapi.JudgeCheckView + err := c.patient(3*time.Minute).post(ctx, "/api/v1/guard/judges/check", body, &v) + return v, err +} + +// judgeCheckText says a check in a few lines, and what to do when it is not +// fine. +func judgeCheckText(v adminapi.JudgeCheckView) string { + var b strings.Builder + took := make([]string, len(v.TookMs)) + for i, t := range v.TookMs { + took[i] = fmt.Sprintf("%.1f", float64(t)/1000) + } + fmt.Fprintf(&b, "%s, %d replica(s) ready, a request waits %.1f s for it\n", v.Model, v.Ready, float64(v.BudgetMs)/1000) + fmt.Fprintf(&b, " alone: %s s\n", strings.Join(took, ", ")) + fmt.Fprintf(&b, " %d at once: slowest %.1f s", v.Burst, float64(v.BurstSlowestMs)/1000) + if v.BurstFailed > 0 { + fmt.Fprintf(&b, ", %d gave no verdict", v.BurstFailed) + } + fmt.Fprintf(&b, "; about %.1f judgments a second, so about %.1f requests a second with short answers\n", + v.PerSecond, v.PerSecond/2) + switch v.Verdict { + case "ok": + b.WriteString("ok: well inside the budget\n") + case "tight": + b.WriteString("tight: a busy moment passes the budget\n") + default: + b.WriteString("slow: a judgment alone passes the budget\n") + } + for _, a := range v.Advice { + b.WriteString(" - " + a + "\n") + } + return b.String() +} diff --git a/internal/cli/serverinit.go b/internal/cli/serverinit.go index 4c60e69..d6b0a14 100644 --- a/internal/cli/serverinit.go +++ b/internal/cli/serverinit.go @@ -84,6 +84,9 @@ type serverInit struct { // nodes labelled for them; judgeCPUs their cores, once added. judgeMachines []judgeMachine judgeCPUs []int + // judgeNote is what timing the judge found, said before the next + // steps when it is not fine. + judgeNote string // noMirror, noNode and noModel turn off the parts of a first install that // make it serve something: a mirror, this machine as a node, and a small // model on it. @@ -349,6 +352,9 @@ func (in *serverInit) run(ctx context.Context) error { return fmt.Errorf("standby control planes: %w", err) } } + if hasSystemd && in.judgeWanted() && in.configJudgesWith(guardModelName) { + in.checkJudge(ctx) + } in.nextSteps() return nil } @@ -913,6 +919,9 @@ func (in *serverInit) waitHealthy(ctx context.Context) error { } func (in *serverInit) nextSteps() { + if in.judgeNote != "" { + fmt.Fprint(in.out, "\n"+in.judgeNote) + } fmt.Fprint(in.out, ` The control plane is up. Next: diff --git a/internal/cli/serverinit_judges.go b/internal/cli/serverinit_judges.go index ff6d751..3fa763b 100644 --- a/internal/cli/serverinit_judges.go +++ b/internal/cli/serverinit_judges.go @@ -5,7 +5,9 @@ import ( "fmt" "strconv" "strings" + "time" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/placement" "gitea.mynodes.xyz/ilker/llmgw/internal/onboard" "gitea.mynodes.xyz/ilker/llmgw/internal/settings" @@ -105,3 +107,44 @@ func (f judgeMachineFlag) Set(v string) error { } func (judgeMachineFlag) Type() string { return "user@host" } + +// judgeReadyWait is how long init waits for its judge to load before timing +// it: the weights come from the mirror, and llama.cpp loads in seconds. +const judgeReadyWait = 15 * time.Minute + +// checkJudge times the judge once it is up, and keeps what it found for the +// closing text: a machine too slow for its judge is said before its first +// refused request, not after. The budget is not changed without asking. +func (in *serverInit) checkJudge(ctx context.Context) { + c := in.adminAPI() + deadline := time.Now().Add(judgeReadyWait) + for { + var judges []adminapi.JudgeView + if err := c.get(ctx, "/api/v1/guard/judges", &judges); err == nil { + ready := false + for _, j := range judges { + ready = ready || (j.Model == guardModelName && j.Ready > 0) + } + if ready { + break + } + } + if time.Now().After(deadline) || ctx.Err() != nil { + in.judgeNote = " !! the judge was not up to be timed; once it is: llmgw ctl guard judge check\n" + return + } + time.Sleep(10 * time.Second) + } + fmt.Fprintf(in.out, " .. timing the judge\n") + v, err := checkJudge(ctx, c, guardModelName, 0) + if err != nil { + in.judgeNote = " !! the judge could not be timed (" + err.Error() + "); later: llmgw ctl guard judge check\n" + return + } + if v.Verdict == "ok" { + in.done("the judge reads a prompt in %.1f s, of the %.1f s a request waits; about %.1f requests a second", + float64(v.SlowestMs)/1000, float64(v.BudgetMs)/1000, v.PerSecond/2) + return + } + in.judgeNote = "The judge is " + v.Verdict + " on this cluster:\n" + indent(judgeCheckText(v), " ") + "\n" +} diff --git a/internal/control/adminapi/guard.go b/internal/control/adminapi/guard.go index 1389d71..2c6c1f5 100644 --- a/internal/control/adminapi/guard.go +++ b/internal/control/adminapi/guard.go @@ -209,6 +209,7 @@ type GuardAttachmentRequest struct { func (a *API) registerGuard(mux Mux) { mux.HandleFunc("GET /api/v1/guard/judges", a.listJudges) mux.HandleFunc("POST /api/v1/guard/judges/ask", a.askJudge) + mux.HandleFunc("POST /api/v1/guard/judges/check", a.checkJudge) mux.HandleFunc("GET /api/v1/guard", func(w http.ResponseWriter, r *http.Request) { if a.guard == nil { writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard in the cluster's settings (llmgw ctl config edit)") diff --git a/internal/control/adminapi/judges.go b/internal/control/adminapi/judges.go index 777b210..84d1fe0 100644 --- a/internal/control/adminapi/judges.go +++ b/internal/control/adminapi/judges.go @@ -7,6 +7,7 @@ import ( "slices" "sort" "strings" + "sync" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" @@ -30,6 +31,9 @@ type JudgeView struct { Judge bool `json:"judge"` Ready int `json:"ready_replicas"` Wanted int `json:"wanted_replicas"` + // BudgetMs is how long a request waits for it: the shortest of its + // policies' latency budgets. + BudgetMs int64 `json:"budget_ms,omitempty"` // LastError is its last failure to give a verdict, if any. LastError string `json:"last_error,omitempty"` LastErrorAt *time.Time `json:"last_error_at,omitempty"` @@ -65,6 +69,9 @@ func (a *API) Judges() []JudgeView { if !slices.Contains(v.Policies, name) { v.Policies = append(v.Policies, name) } + if b := p.Judge.Budget().Milliseconds(); v.BudgetMs == 0 || b < v.BudgetMs { + v.BudgetMs = b + } if p.Fail == "closed" && p.Mode != "shadow" && !slices.Contains(v.Closed, name) { v.Closed = append(v.Closed, name) } @@ -198,3 +205,137 @@ func (a *API) askJudge(w http.ResponseWriter, r *http.Request) { } writeJSON(w, http.StatusOK, v) } + +// JudgeCheckRequest asks how a judge keeps up: Burst judgments at once after +// three in turn; zero is two for each ready replica. +type JudgeCheckRequest struct { + Model string `json:"model,omitempty"` + Burst int `json:"burst,omitempty"` +} + +// JudgeCheckView is how long a judgment took, alone and among others, set +// against the time a request waits for one. +type JudgeCheckView struct { + Model string `json:"model"` + BudgetMs int64 `json:"budget_ms"` + Ready int `json:"ready_replicas"` + // TookMs are three judgments of the same text, one after another. + TookMs []int64 `json:"took_ms"` + SlowestMs int64 `json:"slowest_ms"` + // Burst judgments at once: the slowest of them, how many gave no + // verdict, and how many a second the judges read. + Burst int `json:"burst"` + BurstSlowestMs int64 `json:"burst_slowest_ms"` + BurstFailed int `json:"burst_failed"` + PerSecond float64 `json:"judgments_per_second"` + // Verdict is ok, tight (past half the budget alone, or past it among + // others) or slow (past it alone); Advice what to do when not ok. + Verdict string `json:"verdict"` + Advice []string `json:"advice,omitempty"` +} + +// judgeCheckText is a prompt of about 280 tokens, the length the judges +// were measured at: ordinary work, so the verdict is safe and the time is +// what a request pays. +const judgeCheckText = "Our support team has been asked to write a short guide for new customers who are setting " + + "up the gateway for the first time. The guide should explain how to create an API key in the portal, how to " + + "choose a model from the list the portal shows, and how to send a first request with curl or with the OpenAI " + + "Python client by changing only the base URL and the key. It should also say what the usage page shows, how " + + "budgets and rate limits are set by an administrator for each team, and what a person should do when a " + + "request is refused because a limit was reached. Please keep the tone friendly and plain, use short " + + "paragraphs, and add a small table that lists the three most common error codes with what each means and " + + "what to try next. The audience is technical but new to the product, so avoid internal names, and end with " + + "a short checklist the reader can follow on their first day. Write it in English, about four hundred words." + +func (a *API) checkJudge(w http.ResponseWriter, r *http.Request) { + if a.guard == nil || a.guard.Ask == nil { + writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard in the cluster's settings (llmgw ctl config edit)") + return + } + var req JudgeCheckRequest + _ = json.NewDecoder(r.Body).Decode(&req) + var jv *JudgeView + for _, j := range a.Judges() { + if req.Model == "" || j.Model == req.Model { + j := j + jv = &j + break + } + } + if jv == nil { + writeError(w, http.StatusBadRequest, "no policy asks a judge of that name") + return + } + if jv.Ready == 0 { + writeError(w, http.StatusConflict, "the judge "+jv.Model+" has no replica ready: "+jv.Problem) + return + } + v := JudgeCheckView{Model: jv.Model, BudgetMs: jv.BudgetMs, Ready: jv.Ready, Burst: req.Burst} + if v.Burst <= 0 { + v.Burst = 2 * jv.Ready + } + v.Burst = min(v.Burst, 32) + for range 3 { + ans, err := a.guard.Ask(r.Context(), jv.Model, judgeCheckText, "") + if err != nil { + writeError(w, http.StatusBadGateway, fmt.Sprintf("the judge %s gave no verdict: %v", jv.Model, err)) + return + } + v.TookMs = append(v.TookMs, ans.Took.Milliseconds()) + v.SlowestMs = max(v.SlowestMs, ans.Took.Milliseconds()) + } + var mu sync.Mutex + var wg sync.WaitGroup + start := time.Now() + for range v.Burst { + wg.Add(1) + go func() { + defer wg.Done() + ans, err := a.guard.Ask(r.Context(), jv.Model, judgeCheckText, "") + mu.Lock() + defer mu.Unlock() + if err != nil { + v.BurstFailed++ + return + } + v.BurstSlowestMs = max(v.BurstSlowestMs, ans.Took.Milliseconds()) + }() + } + wg.Wait() + if wall := time.Since(start).Seconds(); wall > 0 { + v.PerSecond = float64(v.Burst-v.BurstFailed) / wall + } + v.Verdict, v.Advice = judgeVerdict(v, *jv) + writeJSON(w, http.StatusOK, v) +} + +// judgeVerdict reads a check: a judgment alone past the budget refuses every +// request under fail: closed; past half of it, or past it among others, a +// busy moment does. +func judgeVerdict(v JudgeCheckView, jv JudgeView) (string, []string) { + verdict := "ok" + switch { + case v.BudgetMs > 0 && v.SlowestMs > v.BudgetMs: + verdict = "slow" + case v.BudgetMs > 0 && (v.SlowestMs*2 > v.BudgetMs || v.BurstSlowestMs > v.BudgetMs || v.BurstFailed > 0): + verdict = "tight" + } + if verdict == "ok" { + return verdict, nil + } + refused := "served unjudged" + if len(jv.Closed) > 0 { + refused = "refused" + } + advice := []string{ + fmt.Sprintf("a judgment past %d ms is %s: each request is judged once for its prompt and again every 600 characters of its answer", v.BudgetMs, refused), + "more cores for each judge: llmgw ctl model resources " + v.Model + " --cores 4, then llmgw ctl restart --model " + v.Model, + "more judges, on machines of their own: llmgw ctl scale " + v.Model + " N, or judge machines at install", + "a judge on a GPU: llmgw ctl guard judge use qwen3guard-4b " + v.Model, + "a longer wait: guard.policies..judge.latency_budget in llmgw ctl config edit", + } + if len(jv.Closed) > 0 { + advice = append(advice, "or fail: open, which serves what the judge did not read, flagged") + } + return verdict, advice +} diff --git a/internal/control/adminapi/judges_test.go b/internal/control/adminapi/judges_test.go index 116fa45..b571e63 100644 --- a/internal/control/adminapi/judges_test.go +++ b/internal/control/adminapi/judges_test.go @@ -123,3 +123,44 @@ func TestANodeSaysWhetherItsFabricsAreUp(t *testing.T) { t.Errorf("no fabric: %q", got) } } + +// A judge alone past its budget is slow; past half of it, or past it among +// others, tight; and what to do is said, fail: open only where it is closed. +func TestAJudgeCheckIsReadAgainstTheBudget(t *testing.T) { + closed := JudgeView{Model: "qwen3guard-0.6b", Closed: []string{"baseline"}} + for _, c := range []struct { + alone, burst int64 + failed int + want string + }{{1200, 2400, 0, "ok"}, {2600, 3000, 0, "tight"}, {1200, 6000, 0, "tight"}, {1200, 2000, 1, "tight"}, {5200, 9000, 0, "slow"}} { + v := JudgeCheckView{Model: closed.Model, BudgetMs: 5000, SlowestMs: c.alone, BurstSlowestMs: c.burst, BurstFailed: c.failed} + got, advice := judgeVerdict(v, closed) + if got != c.want || (got == "ok") != (advice == nil) { + t.Errorf("%+v: %s %v", c, got, advice) + } + if got != "ok" && !strings.Contains(strings.Join(advice, " "), "fail: open") { + t.Errorf("a closed policy is not offered fail: open: %v", advice) + } + } + if _, advice := judgeVerdict(JudgeCheckView{BudgetMs: 5000, SlowestMs: 6000}, JudgeView{}); strings.Contains(strings.Join(advice, " "), "fail: open") { + t.Error("fail: open offered to a policy already open") + } +} + +func TestAJudgeWithNoReplicaIsNotTimed(t *testing.T) { + a, _ := setup(t, testModels()) + lib := guard.Config{Policies: map[string]guard.Policy{ + "baseline": {Judge: &guard.JudgePolicy{Model: "nowhere-judge", Input: "block", LatencyBudget: 5 * time.Second}}}} + a.SetGuard(&GuardAdmin{Library: func() guard.Config { return lib }, + Ask: func(context.Context, string, string, string) (guard.JudgeAnswer, error) { + return guard.JudgeAnswer{}, nil + }}) + if code := call(t, a, "POST", "/api/v1/guard/judges/check", `{}`, nil); code != 409 { + t.Errorf("a judge with no replica: %d", code) + } + var judges []JudgeView + call(t, a, "GET", "/api/v1/guard/judges", "", &judges) + if len(judges) != 1 || judges[0].BudgetMs != 5000 { + t.Errorf("judges %+v", judges) + } +} diff --git a/internal/control/adminapi/routes.go b/internal/control/adminapi/routes.go index 1d7e968..f3e7e37 100644 --- a/internal/control/adminapi/routes.go +++ b/internal/control/adminapi/routes.go @@ -282,6 +282,8 @@ var Routes = []Route{ Response: []JudgeView{}}, {Method: "POST", Path: "/api/v1/guard/judges/ask", Summary: "Have a judge read a text and say what it answered; nothing is kept", Request: JudgeAskRequest{}, Response: JudgeAskView{}}, + {Method: "POST", Path: "/api/v1/guard/judges/check", Summary: "How a judge keeps up: three judgments in turn, then a burst at once, against the budget", + Request: JudgeCheckRequest{}, Response: JudgeCheckView{}}, {Method: "POST", Path: "/api/v1/guard/attachments", Summary: "Attach a guardrail policy", Request: GuardAttachmentRequest{}, Response: guard.Attachment{}}, {Method: "DELETE", Path: "/api/v1/guard/attachments", Summary: "Detach a guardrail policy", diff --git a/internal/guard/judge.go b/internal/guard/judge.go index 5303abe..a371168 100644 --- a/internal/guard/judge.go +++ b/internal/guard/judge.go @@ -61,6 +61,9 @@ func (j *JudgePolicy) Name() string { return strings.Join(j.Judges(), ", else ") // not say. const DefaultJudgeBudget = 250 * time.Millisecond +// Budget is how long each judge is waited for. +func (j *JudgePolicy) Budget() time.Duration { return j.budget() } + func (j *JudgePolicy) budget() time.Duration { if j.LatencyBudget > 0 { return j.LatencyBudget