How fast the judge is, measured, and said at install
ctl guard judge check, and POST /api/v1/guard/judges/check, time three judgments of a 280-token prompt, then a burst at once, against the policies' wait: ok, tight or slow, about how many requests a second can be judged, and what to do. server init runs it once the judge is up and says a machine too slow before the next steps; the wait is not changed without asking. BACKLOG reordered by what goes wrong where installed. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d1bf26528b
commit
85f135311e
12 files changed
+475
-39
No files matched your search
@@ -209,6 +209,7 @@ type GuardAttachmentRequest struct {
|
||||
func (a *API) registerGuard(mux Mux) {
|
||||
mux.HandleFunc("GET /api/v1/guard/judges", a.listJudges)
|
||||
mux.HandleFunc("POST /api/v1/guard/judges/ask", a.askJudge)
|
||||
mux.HandleFunc("POST /api/v1/guard/judges/check", a.checkJudge)
|
||||
mux.HandleFunc("GET /api/v1/guard", func(w http.ResponseWriter, r *http.Request) {
|
||||
if a.guard == nil {
|
||||
writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard in the cluster's settings (llmgw ctl config edit)")
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
@@ -30,6 +31,9 @@ type JudgeView struct {
|
||||
Judge bool `json:"judge"`
|
||||
Ready int `json:"ready_replicas"`
|
||||
Wanted int `json:"wanted_replicas"`
|
||||
// BudgetMs is how long a request waits for it: the shortest of its
|
||||
// policies' latency budgets.
|
||||
BudgetMs int64 `json:"budget_ms,omitempty"`
|
||||
// LastError is its last failure to give a verdict, if any.
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
LastErrorAt *time.Time `json:"last_error_at,omitempty"`
|
||||
@@ -65,6 +69,9 @@ func (a *API) Judges() []JudgeView {
|
||||
if !slices.Contains(v.Policies, name) {
|
||||
v.Policies = append(v.Policies, name)
|
||||
}
|
||||
if b := p.Judge.Budget().Milliseconds(); v.BudgetMs == 0 || b < v.BudgetMs {
|
||||
v.BudgetMs = b
|
||||
}
|
||||
if p.Fail == "closed" && p.Mode != "shadow" && !slices.Contains(v.Closed, name) {
|
||||
v.Closed = append(v.Closed, name)
|
||||
}
|
||||
@@ -198,3 +205,137 @@ func (a *API) askJudge(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
}
|
||||
|
||||
// JudgeCheckRequest asks how a judge keeps up: Burst judgments at once after
|
||||
// three in turn; zero is two for each ready replica.
|
||||
type JudgeCheckRequest struct {
|
||||
Model string `json:"model,omitempty"`
|
||||
Burst int `json:"burst,omitempty"`
|
||||
}
|
||||
|
||||
// JudgeCheckView is how long a judgment took, alone and among others, set
|
||||
// against the time a request waits for one.
|
||||
type JudgeCheckView struct {
|
||||
Model string `json:"model"`
|
||||
BudgetMs int64 `json:"budget_ms"`
|
||||
Ready int `json:"ready_replicas"`
|
||||
// TookMs are three judgments of the same text, one after another.
|
||||
TookMs []int64 `json:"took_ms"`
|
||||
SlowestMs int64 `json:"slowest_ms"`
|
||||
// Burst judgments at once: the slowest of them, how many gave no
|
||||
// verdict, and how many a second the judges read.
|
||||
Burst int `json:"burst"`
|
||||
BurstSlowestMs int64 `json:"burst_slowest_ms"`
|
||||
BurstFailed int `json:"burst_failed"`
|
||||
PerSecond float64 `json:"judgments_per_second"`
|
||||
// Verdict is ok, tight (past half the budget alone, or past it among
|
||||
// others) or slow (past it alone); Advice what to do when not ok.
|
||||
Verdict string `json:"verdict"`
|
||||
Advice []string `json:"advice,omitempty"`
|
||||
}
|
||||
|
||||
// judgeCheckText is a prompt of about 280 tokens, the length the judges
|
||||
// were measured at: ordinary work, so the verdict is safe and the time is
|
||||
// what a request pays.
|
||||
const judgeCheckText = "Our support team has been asked to write a short guide for new customers who are setting " +
|
||||
"up the gateway for the first time. The guide should explain how to create an API key in the portal, how to " +
|
||||
"choose a model from the list the portal shows, and how to send a first request with curl or with the OpenAI " +
|
||||
"Python client by changing only the base URL and the key. It should also say what the usage page shows, how " +
|
||||
"budgets and rate limits are set by an administrator for each team, and what a person should do when a " +
|
||||
"request is refused because a limit was reached. Please keep the tone friendly and plain, use short " +
|
||||
"paragraphs, and add a small table that lists the three most common error codes with what each means and " +
|
||||
"what to try next. The audience is technical but new to the product, so avoid internal names, and end with " +
|
||||
"a short checklist the reader can follow on their first day. Write it in English, about four hundred words."
|
||||
|
||||
func (a *API) checkJudge(w http.ResponseWriter, r *http.Request) {
|
||||
if a.guard == nil || a.guard.Ask == nil {
|
||||
writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard in the cluster's settings (llmgw ctl config edit)")
|
||||
return
|
||||
}
|
||||
var req JudgeCheckRequest
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
var jv *JudgeView
|
||||
for _, j := range a.Judges() {
|
||||
if req.Model == "" || j.Model == req.Model {
|
||||
j := j
|
||||
jv = &j
|
||||
break
|
||||
}
|
||||
}
|
||||
if jv == nil {
|
||||
writeError(w, http.StatusBadRequest, "no policy asks a judge of that name")
|
||||
return
|
||||
}
|
||||
if jv.Ready == 0 {
|
||||
writeError(w, http.StatusConflict, "the judge "+jv.Model+" has no replica ready: "+jv.Problem)
|
||||
return
|
||||
}
|
||||
v := JudgeCheckView{Model: jv.Model, BudgetMs: jv.BudgetMs, Ready: jv.Ready, Burst: req.Burst}
|
||||
if v.Burst <= 0 {
|
||||
v.Burst = 2 * jv.Ready
|
||||
}
|
||||
v.Burst = min(v.Burst, 32)
|
||||
for range 3 {
|
||||
ans, err := a.guard.Ask(r.Context(), jv.Model, judgeCheckText, "")
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadGateway, fmt.Sprintf("the judge %s gave no verdict: %v", jv.Model, err))
|
||||
return
|
||||
}
|
||||
v.TookMs = append(v.TookMs, ans.Took.Milliseconds())
|
||||
v.SlowestMs = max(v.SlowestMs, ans.Took.Milliseconds())
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
start := time.Now()
|
||||
for range v.Burst {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
ans, err := a.guard.Ask(r.Context(), jv.Model, judgeCheckText, "")
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if err != nil {
|
||||
v.BurstFailed++
|
||||
return
|
||||
}
|
||||
v.BurstSlowestMs = max(v.BurstSlowestMs, ans.Took.Milliseconds())
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if wall := time.Since(start).Seconds(); wall > 0 {
|
||||
v.PerSecond = float64(v.Burst-v.BurstFailed) / wall
|
||||
}
|
||||
v.Verdict, v.Advice = judgeVerdict(v, *jv)
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
}
|
||||
|
||||
// judgeVerdict reads a check: a judgment alone past the budget refuses every
|
||||
// request under fail: closed; past half of it, or past it among others, a
|
||||
// busy moment does.
|
||||
func judgeVerdict(v JudgeCheckView, jv JudgeView) (string, []string) {
|
||||
verdict := "ok"
|
||||
switch {
|
||||
case v.BudgetMs > 0 && v.SlowestMs > v.BudgetMs:
|
||||
verdict = "slow"
|
||||
case v.BudgetMs > 0 && (v.SlowestMs*2 > v.BudgetMs || v.BurstSlowestMs > v.BudgetMs || v.BurstFailed > 0):
|
||||
verdict = "tight"
|
||||
}
|
||||
if verdict == "ok" {
|
||||
return verdict, nil
|
||||
}
|
||||
refused := "served unjudged"
|
||||
if len(jv.Closed) > 0 {
|
||||
refused = "refused"
|
||||
}
|
||||
advice := []string{
|
||||
fmt.Sprintf("a judgment past %d ms is %s: each request is judged once for its prompt and again every 600 characters of its answer", v.BudgetMs, refused),
|
||||
"more cores for each judge: llmgw ctl model resources " + v.Model + " --cores 4, then llmgw ctl restart --model " + v.Model,
|
||||
"more judges, on machines of their own: llmgw ctl scale " + v.Model + " N, or judge machines at install",
|
||||
"a judge on a GPU: llmgw ctl guard judge use qwen3guard-4b " + v.Model,
|
||||
"a longer wait: guard.policies.<policy>.judge.latency_budget in llmgw ctl config edit",
|
||||
}
|
||||
if len(jv.Closed) > 0 {
|
||||
advice = append(advice, "or fail: open, which serves what the judge did not read, flagged")
|
||||
}
|
||||
return verdict, advice
|
||||
}
|
||||
@@ -123,3 +123,44 @@ func TestANodeSaysWhetherItsFabricsAreUp(t *testing.T) {
|
||||
t.Errorf("no fabric: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A judge alone past its budget is slow; past half of it, or past it among
|
||||
// others, tight; and what to do is said, fail: open only where it is closed.
|
||||
func TestAJudgeCheckIsReadAgainstTheBudget(t *testing.T) {
|
||||
closed := JudgeView{Model: "qwen3guard-0.6b", Closed: []string{"baseline"}}
|
||||
for _, c := range []struct {
|
||||
alone, burst int64
|
||||
failed int
|
||||
want string
|
||||
}{{1200, 2400, 0, "ok"}, {2600, 3000, 0, "tight"}, {1200, 6000, 0, "tight"}, {1200, 2000, 1, "tight"}, {5200, 9000, 0, "slow"}} {
|
||||
v := JudgeCheckView{Model: closed.Model, BudgetMs: 5000, SlowestMs: c.alone, BurstSlowestMs: c.burst, BurstFailed: c.failed}
|
||||
got, advice := judgeVerdict(v, closed)
|
||||
if got != c.want || (got == "ok") != (advice == nil) {
|
||||
t.Errorf("%+v: %s %v", c, got, advice)
|
||||
}
|
||||
if got != "ok" && !strings.Contains(strings.Join(advice, " "), "fail: open") {
|
||||
t.Errorf("a closed policy is not offered fail: open: %v", advice)
|
||||
}
|
||||
}
|
||||
if _, advice := judgeVerdict(JudgeCheckView{BudgetMs: 5000, SlowestMs: 6000}, JudgeView{}); strings.Contains(strings.Join(advice, " "), "fail: open") {
|
||||
t.Error("fail: open offered to a policy already open")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAJudgeWithNoReplicaIsNotTimed(t *testing.T) {
|
||||
a, _ := setup(t, testModels())
|
||||
lib := guard.Config{Policies: map[string]guard.Policy{
|
||||
"baseline": {Judge: &guard.JudgePolicy{Model: "nowhere-judge", Input: "block", LatencyBudget: 5 * time.Second}}}}
|
||||
a.SetGuard(&GuardAdmin{Library: func() guard.Config { return lib },
|
||||
Ask: func(context.Context, string, string, string) (guard.JudgeAnswer, error) {
|
||||
return guard.JudgeAnswer{}, nil
|
||||
}})
|
||||
if code := call(t, a, "POST", "/api/v1/guard/judges/check", `{}`, nil); code != 409 {
|
||||
t.Errorf("a judge with no replica: %d", code)
|
||||
}
|
||||
var judges []JudgeView
|
||||
call(t, a, "GET", "/api/v1/guard/judges", "", &judges)
|
||||
if len(judges) != 1 || judges[0].BudgetMs != 5000 {
|
||||
t.Errorf("judges %+v", judges)
|
||||
}
|
||||
}
|
||||
@@ -282,6 +282,8 @@ var Routes = []Route{
|
||||
Response: []JudgeView{}},
|
||||
{Method: "POST", Path: "/api/v1/guard/judges/ask", Summary: "Have a judge read a text and say what it answered; nothing is kept",
|
||||
Request: JudgeAskRequest{}, Response: JudgeAskView{}},
|
||||
{Method: "POST", Path: "/api/v1/guard/judges/check", Summary: "How a judge keeps up: three judgments in turn, then a burst at once, against the budget",
|
||||
Request: JudgeCheckRequest{}, Response: JudgeCheckView{}},
|
||||
{Method: "POST", Path: "/api/v1/guard/attachments", Summary: "Attach a guardrail policy",
|
||||
Request: GuardAttachmentRequest{}, Response: guard.Attachment{}},
|
||||
{Method: "DELETE", Path: "/api/v1/guard/attachments", Summary: "Detach a guardrail policy",
|
||||
|
||||
Reference in new issue
Block a user