guard judge says controversial, and names a judge no policy asks as one
judge ask showed Qwen3Guard's controversial verdict as "safe (Violent)"; it now says controversial, which a policy's judge.controversial counts. The stray-judge note called a judge no policy asks any more "served as a model clients use"; a model with role: guard is now said to be a judge no policy asks. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
1fabb05fb2
commit
7f7e5a470e
3 files changed
+31
-16
No files matched your search
@@ -4535,7 +4535,7 @@
|
||||
},
|
||||
"verdict": {
|
||||
"type": "string",
|
||||
"description": "Verdict is safe or unsafe, as the gateway reads the answer."
|
||||
"description": "Verdict is safe, unsafe or controversial, as the gateway reads the answer; a policy's judge.controversial says what controversial counts as."
|
||||
},
|
||||
"categories": {
|
||||
"type": "array",
|
||||
|
||||
+21
-12
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -73,19 +74,16 @@ func newGuardJudgeStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Comma
|
||||
return err
|
||||
}
|
||||
strays, _ := strayJudges(cmd, client, js)
|
||||
for _, s := range strays {
|
||||
fmt.Fprintf(out, "\n%s looks like a judge's weights but is served as a model clients use, and no "+
|
||||
"policy asks it: llmgw ctl model remove %s\n", s, s)
|
||||
}
|
||||
sayStrays(out, strays)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// strayJudges are models whose weights are a judge's but which are in the
|
||||
// catalog as models clients use, under a name no policy asks: what serving
|
||||
// Llama Guard from the mirror page did before it knew better.
|
||||
func strayJudges(cmd *cobra.Command, client func() *ctlClient, js []adminapi.JudgeView) ([]string, error) {
|
||||
// strayJudges are models whose weights are a judge's, under a name no policy
|
||||
// asks: a judge the policies moved on from, or one served from the mirror
|
||||
// page as a model clients use, as before that page knew better.
|
||||
func strayJudges(cmd *cobra.Command, client func() *ctlClient, js []adminapi.JudgeView) ([]adminapi.ModelView, error) {
|
||||
var models []adminapi.ModelView
|
||||
if err := client().get(cmd.Context(), "/api/v1/models", &models); err != nil {
|
||||
return nil, err
|
||||
@@ -94,15 +92,27 @@ func strayJudges(cmd *cobra.Command, client func() *ctlClient, js []adminapi.Jud
|
||||
for _, j := range js {
|
||||
asked[j.Model] = true
|
||||
}
|
||||
var out []string
|
||||
var out []adminapi.ModelView
|
||||
for _, m := range models {
|
||||
if _, judge := guard.FormatOfWeights(m.Source); !asked[m.Name] && judge {
|
||||
out = append(out, m.Name)
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sayStrays says what each stray judge is and how to be rid of it.
|
||||
func sayStrays(out io.Writer, strays []adminapi.ModelView) {
|
||||
for _, m := range strays {
|
||||
if m.Role == catalog.RoleGuard {
|
||||
fmt.Fprintf(out, "\n%s is a judge no policy asks, still running: llmgw ctl model remove %s\n", m.Name, m.Name)
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(out, "\n%s looks like a judge's weights but is served as a model clients use, and no "+
|
||||
"policy asks it: llmgw ctl model remove %s\n", m.Name, m.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func newGuardJudgeEnableCmd(client func() *ctlClient) *cobra.Command {
|
||||
var (
|
||||
model, repo, file, format, gpuClass string
|
||||
@@ -220,8 +230,7 @@ func newGuardJudgeEnableCmd(client func() *ctlClient) *cobra.Command {
|
||||
fmt.Fprintln(out)
|
||||
}
|
||||
if strays, _ := strayJudges(cmd, client, js); len(strays) > 0 {
|
||||
fmt.Fprintf(out, "\nno policy asks %s, served from a judge's weights as a model clients use: "+
|
||||
"llmgw ctl model remove %s\n", strings.Join(strays, ", "), strings.Join(strays, " "))
|
||||
sayStrays(out, slices.DeleteFunc(strays, func(m adminapi.ModelView) bool { return m.Name == name }))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
|
||||
@@ -154,7 +154,8 @@ type JudgeAskRequest struct {
|
||||
// JudgeAskView is what the judge answered.
|
||||
type JudgeAskView struct {
|
||||
Model string `json:"model"`
|
||||
// Verdict is safe or unsafe, as the gateway reads the answer.
|
||||
// Verdict is safe, unsafe or controversial, as the gateway reads the
|
||||
// answer; a policy's judge.controversial says what controversial counts as.
|
||||
Verdict string `json:"verdict"`
|
||||
Categories []string `json:"categories,omitempty"`
|
||||
// Raw is the judge's answer as it gave it.
|
||||
@@ -186,9 +187,14 @@ func (a *API) askJudge(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusBadGateway, fmt.Sprintf("the judge %s gave no verdict: %v (it answered %q)", req.Model, err, ans.Raw))
|
||||
return
|
||||
}
|
||||
v.Verdict = "safe"
|
||||
if ans.Unsafe {
|
||||
switch {
|
||||
case ans.Unsafe:
|
||||
v.Verdict = "unsafe"
|
||||
case ans.Controversial:
|
||||
// Qwen3Guard's middle verdict: what it counts as is the policy's.
|
||||
v.Verdict = "controversial"
|
||||
default:
|
||||
v.Verdict = "safe"
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
}
|
||||
Reference in new issue
Block a user