How fast the judge is, measured, and said at install

ctl guard judge check, and POST /api/v1/guard/judges/check, time three
judgments of a 280-token prompt, then a burst at once, against the
policies' wait: ok, tight or slow, about how many requests a second can
be judged, and what to do. server init runs it once the judge is up and
says a machine too slow before the next steps; the wait is not changed
without asking. BACKLOG reordered by what goes wrong where installed.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-20 22:35:19 +02:00
1 parent d1bf26528b
commit 85f135311e
12 files changed
+475 -39

No files matched your search

+1
View File
@@ -209,6 +209,7 @@ type GuardAttachmentRequest struct {
func (a *API) registerGuard(mux Mux) {
mux.HandleFunc("GET /api/v1/guard/judges", a.listJudges)
mux.HandleFunc("POST /api/v1/guard/judges/ask", a.askJudge)
mux.HandleFunc("POST /api/v1/guard/judges/check", a.checkJudge)
mux.HandleFunc("GET /api/v1/guard", func(w http.ResponseWriter, r *http.Request) {
if a.guard == nil {
writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard in the cluster's settings (llmgw ctl config edit)")
+141
View File
@@ -7,6 +7,7 @@ import (
"slices"
"sort"
"strings"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
@@ -30,6 +31,9 @@ type JudgeView struct {
Judge bool `json:"judge"`
Ready int `json:"ready_replicas"`
Wanted int `json:"wanted_replicas"`
// BudgetMs is how long a request waits for it: the shortest of its
// policies' latency budgets.
BudgetMs int64 `json:"budget_ms,omitempty"`
// LastError is its last failure to give a verdict, if any.
LastError string `json:"last_error,omitempty"`
LastErrorAt *time.Time `json:"last_error_at,omitempty"`
@@ -65,6 +69,9 @@ func (a *API) Judges() []JudgeView {
if !slices.Contains(v.Policies, name) {
v.Policies = append(v.Policies, name)
}
if b := p.Judge.Budget().Milliseconds(); v.BudgetMs == 0 || b < v.BudgetMs {
v.BudgetMs = b
}
if p.Fail == "closed" && p.Mode != "shadow" && !slices.Contains(v.Closed, name) {
v.Closed = append(v.Closed, name)
}
@@ -198,3 +205,137 @@ func (a *API) askJudge(w http.ResponseWriter, r *http.Request) {
}
writeJSON(w, http.StatusOK, v)
}
// JudgeCheckRequest asks how a judge keeps up: Burst judgments at once after
// three in turn; zero is two for each ready replica.
type JudgeCheckRequest struct {
Model string `json:"model,omitempty"`
Burst int `json:"burst,omitempty"`
}
// JudgeCheckView is how long a judgment took, alone and among others, set
// against the time a request waits for one.
type JudgeCheckView struct {
Model string `json:"model"`
BudgetMs int64 `json:"budget_ms"`
Ready int `json:"ready_replicas"`
// TookMs are three judgments of the same text, one after another.
TookMs []int64 `json:"took_ms"`
SlowestMs int64 `json:"slowest_ms"`
// Burst judgments at once: the slowest of them, how many gave no
// verdict, and how many a second the judges read.
Burst int `json:"burst"`
BurstSlowestMs int64 `json:"burst_slowest_ms"`
BurstFailed int `json:"burst_failed"`
PerSecond float64 `json:"judgments_per_second"`
// Verdict is ok, tight (past half the budget alone, or past it among
// others) or slow (past it alone); Advice what to do when not ok.
Verdict string `json:"verdict"`
Advice []string `json:"advice,omitempty"`
}
// judgeCheckText is a prompt of about 280 tokens, the length the judges
// were measured at: ordinary work, so the verdict is safe and the time is
// what a request pays.
const judgeCheckText = "Our support team has been asked to write a short guide for new customers who are setting " +
"up the gateway for the first time. The guide should explain how to create an API key in the portal, how to " +
"choose a model from the list the portal shows, and how to send a first request with curl or with the OpenAI " +
"Python client by changing only the base URL and the key. It should also say what the usage page shows, how " +
"budgets and rate limits are set by an administrator for each team, and what a person should do when a " +
"request is refused because a limit was reached. Please keep the tone friendly and plain, use short " +
"paragraphs, and add a small table that lists the three most common error codes with what each means and " +
"what to try next. The audience is technical but new to the product, so avoid internal names, and end with " +
"a short checklist the reader can follow on their first day. Write it in English, about four hundred words."
func (a *API) checkJudge(w http.ResponseWriter, r *http.Request) {
if a.guard == nil || a.guard.Ask == nil {
writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard in the cluster's settings (llmgw ctl config edit)")
return
}
var req JudgeCheckRequest
_ = json.NewDecoder(r.Body).Decode(&req)
var jv *JudgeView
for _, j := range a.Judges() {
if req.Model == "" || j.Model == req.Model {
j := j
jv = &j
break
}
}
if jv == nil {
writeError(w, http.StatusBadRequest, "no policy asks a judge of that name")
return
}
if jv.Ready == 0 {
writeError(w, http.StatusConflict, "the judge "+jv.Model+" has no replica ready: "+jv.Problem)
return
}
v := JudgeCheckView{Model: jv.Model, BudgetMs: jv.BudgetMs, Ready: jv.Ready, Burst: req.Burst}
if v.Burst <= 0 {
v.Burst = 2 * jv.Ready
}
v.Burst = min(v.Burst, 32)
for range 3 {
ans, err := a.guard.Ask(r.Context(), jv.Model, judgeCheckText, "")
if err != nil {
writeError(w, http.StatusBadGateway, fmt.Sprintf("the judge %s gave no verdict: %v", jv.Model, err))
return
}
v.TookMs = append(v.TookMs, ans.Took.Milliseconds())
v.SlowestMs = max(v.SlowestMs, ans.Took.Milliseconds())
}
var mu sync.Mutex
var wg sync.WaitGroup
start := time.Now()
for range v.Burst {
wg.Add(1)
go func() {
defer wg.Done()
ans, err := a.guard.Ask(r.Context(), jv.Model, judgeCheckText, "")
mu.Lock()
defer mu.Unlock()
if err != nil {
v.BurstFailed++
return
}
v.BurstSlowestMs = max(v.BurstSlowestMs, ans.Took.Milliseconds())
}()
}
wg.Wait()
if wall := time.Since(start).Seconds(); wall > 0 {
v.PerSecond = float64(v.Burst-v.BurstFailed) / wall
}
v.Verdict, v.Advice = judgeVerdict(v, *jv)
writeJSON(w, http.StatusOK, v)
}
// judgeVerdict reads a check: a judgment alone past the budget refuses every
// request under fail: closed; past half of it, or past it among others, a
// busy moment does.
func judgeVerdict(v JudgeCheckView, jv JudgeView) (string, []string) {
verdict := "ok"
switch {
case v.BudgetMs > 0 && v.SlowestMs > v.BudgetMs:
verdict = "slow"
case v.BudgetMs > 0 && (v.SlowestMs*2 > v.BudgetMs || v.BurstSlowestMs > v.BudgetMs || v.BurstFailed > 0):
verdict = "tight"
}
if verdict == "ok" {
return verdict, nil
}
refused := "served unjudged"
if len(jv.Closed) > 0 {
refused = "refused"
}
advice := []string{
fmt.Sprintf("a judgment past %d ms is %s: each request is judged once for its prompt and again every 600 characters of its answer", v.BudgetMs, refused),
"more cores for each judge: llmgw ctl model resources " + v.Model + " --cores 4, then llmgw ctl restart --model " + v.Model,
"more judges, on machines of their own: llmgw ctl scale " + v.Model + " N, or judge machines at install",
"a judge on a GPU: llmgw ctl guard judge use qwen3guard-4b " + v.Model,
"a longer wait: guard.policies.<policy>.judge.latency_budget in llmgw ctl config edit",
}
if len(jv.Closed) > 0 {
advice = append(advice, "or fail: open, which serves what the judge did not read, flagged")
}
return verdict, advice
}
+41
View File
@@ -123,3 +123,44 @@ func TestANodeSaysWhetherItsFabricsAreUp(t *testing.T) {
t.Errorf("no fabric: %q", got)
}
}
// A judge alone past its budget is slow; past half of it, or past it among
// others, tight; and what to do is said, fail: open only where it is closed.
func TestAJudgeCheckIsReadAgainstTheBudget(t *testing.T) {
closed := JudgeView{Model: "qwen3guard-0.6b", Closed: []string{"baseline"}}
for _, c := range []struct {
alone, burst int64
failed int
want string
}{{1200, 2400, 0, "ok"}, {2600, 3000, 0, "tight"}, {1200, 6000, 0, "tight"}, {1200, 2000, 1, "tight"}, {5200, 9000, 0, "slow"}} {
v := JudgeCheckView{Model: closed.Model, BudgetMs: 5000, SlowestMs: c.alone, BurstSlowestMs: c.burst, BurstFailed: c.failed}
got, advice := judgeVerdict(v, closed)
if got != c.want || (got == "ok") != (advice == nil) {
t.Errorf("%+v: %s %v", c, got, advice)
}
if got != "ok" && !strings.Contains(strings.Join(advice, " "), "fail: open") {
t.Errorf("a closed policy is not offered fail: open: %v", advice)
}
}
if _, advice := judgeVerdict(JudgeCheckView{BudgetMs: 5000, SlowestMs: 6000}, JudgeView{}); strings.Contains(strings.Join(advice, " "), "fail: open") {
t.Error("fail: open offered to a policy already open")
}
}
func TestAJudgeWithNoReplicaIsNotTimed(t *testing.T) {
a, _ := setup(t, testModels())
lib := guard.Config{Policies: map[string]guard.Policy{
"baseline": {Judge: &guard.JudgePolicy{Model: "nowhere-judge", Input: "block", LatencyBudget: 5 * time.Second}}}}
a.SetGuard(&GuardAdmin{Library: func() guard.Config { return lib },
Ask: func(context.Context, string, string, string) (guard.JudgeAnswer, error) {
return guard.JudgeAnswer{}, nil
}})
if code := call(t, a, "POST", "/api/v1/guard/judges/check", `{}`, nil); code != 409 {
t.Errorf("a judge with no replica: %d", code)
}
var judges []JudgeView
call(t, a, "GET", "/api/v1/guard/judges", "", &judges)
if len(judges) != 1 || judges[0].BudgetMs != 5000 {
t.Errorf("judges %+v", judges)
}
}
+2
View File
@@ -282,6 +282,8 @@ var Routes = []Route{
Response: []JudgeView{}},
{Method: "POST", Path: "/api/v1/guard/judges/ask", Summary: "Have a judge read a text and say what it answered; nothing is kept",
Request: JudgeAskRequest{}, Response: JudgeAskView{}},
{Method: "POST", Path: "/api/v1/guard/judges/check", Summary: "How a judge keeps up: three judgments in turn, then a burst at once, against the budget",
Request: JudgeCheckRequest{}, Response: JudgeCheckView{}},
{Method: "POST", Path: "/api/v1/guard/attachments", Summary: "Attach a guardrail policy",
Request: GuardAttachmentRequest{}, Response: guard.Attachment{}},
{Method: "DELETE", Path: "/api/v1/guard/attachments", Summary: "Detach a guardrail policy",