Qwen3Guard is the judge, and judges on a GPU and on CPUs are asked in turn
Llama Guard 3 1B could not tell ordinary coding requests from harmful ones: on the PoC it scored "continue" 0.51, "continue the code above" 0.89 and "how do I kill all child processes in bash" 0.92 unsafe, against 0.97 to 1.00 for harmful requests, at Q4_K_M and Q8_0 alike, with its own template. A site had "continue" refused. Qwen3Guard-Gen 0.6B gave the ordinary ones 0.15 at the most and the harmful ones 1.00, in 0.8 GB on CPUs. - format qwen3guard: its prompt written out as its template writes it, for requests and answers, and its Safe / Unsafe / Controversial verdict read; controversial counts as the policy's judge.controversial says, safe by default. - judge.models: judges asked in turn, the first that answers deciding; a failure is recorded against the judge that failed. guard judge use sets the order, replacing categories none of the judges names. - known judges (Qwen3Guard 0.6B/4B/8B, Llama Guard 1B/8B): guard judge enable serves any of them, --gpu the 4B on the roomiest GPU class as a standard model; the leader fetches and serves one a policy names that the catalog lacks. - server init installs qwen3guard-0.6b with the baseline's categories. - a judge reads a long text by its start and end, which fit its context, rather than overflowing it and refusing the request under fail: closed. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
63f7c8cfd7
commit
b0273ffb5f
24 files changed
+1313
-208
No files matched your search
@@ -431,10 +431,12 @@ type ModelView struct {
|
||||
Description string `json:"description,omitempty"`
|
||||
Aliases []string `json:"aliases,omitempty"`
|
||||
Source string `json:"source"`
|
||||
Min int `json:"min_replicas"`
|
||||
Max int `json:"max_replicas"`
|
||||
Ready int `json:"ready_replicas"`
|
||||
Total int `json:"total_replicas"`
|
||||
// Role is "guard" for a guardrails judge, empty for a model clients use.
|
||||
Role string `json:"role,omitempty"`
|
||||
Min int `json:"min_replicas"`
|
||||
Max int `json:"max_replicas"`
|
||||
Ready int `json:"ready_replicas"`
|
||||
Total int `json:"total_replicas"`
|
||||
// Desired is how many it should have now: Min, or for an on-demand
|
||||
// model one while it is being asked for and none otherwise.
|
||||
Desired int `json:"desired_replicas"`
|
||||
@@ -810,6 +812,7 @@ func (a *API) Models() []ModelView {
|
||||
Class: m.EffectiveClass(), Owner: m.Owner, Preempted: h.Preempted, HeldFor: h.HeldFor,
|
||||
External: externalLabel(m), Adapters: m.Adapters,
|
||||
Canary: m.Canary, CanaryOf: a.ctl.Catalog().CanaryOf(m.Name), Eval: m.Eval,
|
||||
Role: m.Role,
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
v.Profiles = append(v.Profiles, p.Name)
|
||||
|
||||
@@ -21,6 +21,9 @@ type JudgeView struct {
|
||||
// cannot read (fail: closed).
|
||||
Policies []string `json:"policies"`
|
||||
Closed []string `json:"closed,omitempty"`
|
||||
// Then are the judges its policies ask after it, when it gives no
|
||||
// verdict: a CPU judge behind a GPU one.
|
||||
Then []string `json:"then,omitempty"`
|
||||
// InCatalog is whether a model of that name is in the catalog, and
|
||||
// Judge whether it is there as a judge (role: guard).
|
||||
InCatalog bool `json:"in_catalog"`
|
||||
@@ -52,17 +55,24 @@ func (a *API) Judges() []JudgeView {
|
||||
}
|
||||
byModel := map[string]*JudgeView{}
|
||||
for name, p := range lib.Policies {
|
||||
if p.Judge == nil || p.Judge.Model == "" {
|
||||
continue
|
||||
}
|
||||
v := byModel[p.Judge.Model]
|
||||
if v == nil {
|
||||
v = &JudgeView{Model: p.Judge.Model}
|
||||
byModel[p.Judge.Model] = v
|
||||
}
|
||||
v.Policies = append(v.Policies, name)
|
||||
if p.Fail == "closed" && p.Mode != "shadow" {
|
||||
v.Closed = append(v.Closed, name)
|
||||
judges := p.Judge.Judges()
|
||||
for i, model := range judges {
|
||||
v := byModel[model]
|
||||
if v == nil {
|
||||
v = &JudgeView{Model: model}
|
||||
byModel[model] = v
|
||||
}
|
||||
if !slices.Contains(v.Policies, name) {
|
||||
v.Policies = append(v.Policies, name)
|
||||
}
|
||||
if p.Fail == "closed" && p.Mode != "shadow" && !slices.Contains(v.Closed, name) {
|
||||
v.Closed = append(v.Closed, name)
|
||||
}
|
||||
for _, after := range judges[i+1:] {
|
||||
if !slices.Contains(v.Then, after) {
|
||||
v.Then = append(v.Then, after)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ready := map[string]int{}
|
||||
@@ -104,7 +114,10 @@ func (a *API) Judges() []JudgeView {
|
||||
|
||||
func judgeProblem(v JudgeView) string {
|
||||
consequence := "requests its policies judge are served unjudged"
|
||||
if len(v.Closed) > 0 {
|
||||
switch {
|
||||
case len(v.Then) > 0:
|
||||
consequence = "its policies ask " + strings.Join(v.Then, ", else ") + " instead"
|
||||
case len(v.Closed) > 0:
|
||||
consequence = "requests its policies judge are refused (guard_unavailable)"
|
||||
}
|
||||
switch {
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
)
|
||||
|
||||
// Scaling is how many replicas a model should have.
|
||||
@@ -386,9 +387,11 @@ func (m Model) IsGuard() bool { return m.Role == RoleGuard }
|
||||
|
||||
// GuardModel is how a judge model's answers are read.
|
||||
type GuardModel struct {
|
||||
// Format is llamaguard3 (Meta's Llama Guard 3: "safe", or "unsafe" and
|
||||
// the categories), shieldgemma (Google's ShieldGemma: Yes or No), or
|
||||
// json ({"safe": false, "categories": ["..."]}).
|
||||
// Format is qwen3guard (Qwen3Guard-Gen: "Safety: Safe", "Unsafe" or
|
||||
// "Controversial", and the categories), llamaguard3 (Meta's Llama Guard
|
||||
// 3: "safe", or "unsafe" and the categories), shieldgemma (Google's
|
||||
// ShieldGemma: Yes or No), or json ({"safe": false, "categories":
|
||||
// ["..."]}).
|
||||
Format string `json:"format" koanf:"format"`
|
||||
// Categories the model is asked about, for shieldgemma — each a policy
|
||||
// sentence — and the names reported for llamaguard3's S1, S2, ….
|
||||
@@ -708,12 +711,10 @@ func (m Model) validate() error {
|
||||
}
|
||||
case RoleGuard:
|
||||
if m.Guard == nil {
|
||||
return fmt.Errorf("model %q: a guard model needs guard.format: llamaguard3, shieldgemma or json", m.Name)
|
||||
return fmt.Errorf("model %q: a guard model needs guard.format: %s", m.Name, strings.Join(guard.Formats, ", "))
|
||||
}
|
||||
switch m.Guard.Format {
|
||||
case "llamaguard3", "shieldgemma", "json":
|
||||
default:
|
||||
return fmt.Errorf("model %q: guard.format %q is not llamaguard3, shieldgemma or json", m.Name, m.Guard.Format)
|
||||
if !guard.KnownFormat(m.Guard.Format) {
|
||||
return fmt.Errorf("model %q: guard.format %q is not one of %s", m.Name, m.Guard.Format, strings.Join(guard.Formats, ", "))
|
||||
}
|
||||
if m.Scaling.OnDemand() {
|
||||
// A judge loaded on demand makes the first request of every
|
||||
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
@@ -137,8 +138,12 @@ type Request struct {
|
||||
// Replicas to run; 0 makes it on demand, loaded when first asked for.
|
||||
Replicas int `json:"replicas"`
|
||||
// GuardFormat makes it a judge model for the guardrails (role: guard):
|
||||
// llamaguard3, shieldgemma or json. Empty serves clients.
|
||||
// llamaguard3, qwen3guard, shieldgemma or json. Empty serves clients.
|
||||
GuardFormat string `json:"guard_format,omitempty"`
|
||||
// Class is what it may take from others and they from it when there is
|
||||
// not room for both: critical, standard or opportunistic. Empty is the
|
||||
// catalog's default for its role.
|
||||
Class string `json:"class,omitempty"`
|
||||
// CtxPool is llama.cpp's --ctx-size; ContextLength what one request may
|
||||
// use. They default to 16384 and 4096, what four parallel slots divide;
|
||||
// llama.cpp is given as many slots as the pool holds requests.
|
||||
@@ -252,10 +257,13 @@ func (s *Server) Start(req Request) (Job, error) {
|
||||
default:
|
||||
return Job{}, fmt.Errorf("the format is gguf or safetensors, not %q", req.Format)
|
||||
}
|
||||
switch req.GuardFormat {
|
||||
case "", "llamaguard3", "shieldgemma", "json":
|
||||
if req.GuardFormat != "" && !guard.KnownFormat(req.GuardFormat) {
|
||||
return Job{}, fmt.Errorf("the guard format is one of %s, not %q", strings.Join(guard.Formats, ", "), req.GuardFormat)
|
||||
}
|
||||
switch req.Class {
|
||||
case "", catalog.ClassCritical, catalog.ClassStandard, catalog.ClassOpportunistic:
|
||||
default:
|
||||
return Job{}, fmt.Errorf("the guard format is llamaguard3, shieldgemma or json, not %q", req.GuardFormat)
|
||||
return Job{}, fmt.Errorf("the class is critical, standard or opportunistic, not %q", req.Class)
|
||||
}
|
||||
if req.GuardFormat != "" && req.Replicas == 0 {
|
||||
return Job{}, errors.New("a judge model must always run: give it --replicas 1 or more, not --on-demand")
|
||||
@@ -618,6 +626,7 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
|
||||
if req.GuardFormat != "" {
|
||||
m.Role, m.Guard, m.RoutingProfile = catalog.RoleGuard, &catalog.GuardModel{Format: req.GuardFormat}, ""
|
||||
}
|
||||
m.Class = req.Class
|
||||
s.step(j, "catalog", Running, "adding "+name)
|
||||
if err := s.deps.Catalog.Add(ctx, m, req.By); err != nil {
|
||||
s.fail(j, "catalog", err)
|
||||
|
||||
Reference in new issue
Block a user