Qwen3Guard is the judge, and judges on a GPU and on CPUs are asked in turn
Llama Guard 3 1B could not tell ordinary coding requests from harmful ones: on the PoC it scored "continue" 0.51, "continue the code above" 0.89 and "how do I kill all child processes in bash" 0.92 unsafe, against 0.97 to 1.00 for harmful requests, at Q4_K_M and Q8_0 alike, with its own template. A site had "continue" refused. Qwen3Guard-Gen 0.6B gave the ordinary ones 0.15 at the most and the harmful ones 1.00, in 0.8 GB on CPUs. - format qwen3guard: its prompt written out as its template writes it, for requests and answers, and its Safe / Unsafe / Controversial verdict read; controversial counts as the policy's judge.controversial says, safe by default. - judge.models: judges asked in turn, the first that answers deciding; a failure is recorded against the judge that failed. guard judge use sets the order, replacing categories none of the judges names. - known judges (Qwen3Guard 0.6B/4B/8B, Llama Guard 1B/8B): guard judge enable serves any of them, --gpu the 4B on the roomiest GPU class as a standard model; the leader fetches and serves one a policy names that the catalog lacks. - server init installs qwen3guard-0.6b with the baseline's categories. - a judge reads a long text by its start and end, which fit its context, rather than overflowing it and refusing the request under fail: closed. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
63f7c8cfd7
commit
b0273ffb5f
24 files changed
+1313
-208
No files matched your search
+205
-54
@@ -4,20 +4,17 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
)
|
||||
|
||||
// The judge the guardrails are installed with: Meta's Llama Guard 3 1B,
|
||||
// from the repository server init fetches it from.
|
||||
const (
|
||||
guardModelRepo = "QuantFactory/Llama-Guard-3-1B-GGUF"
|
||||
guardModelFile = "Llama-Guard-3-1B.Q4_K_M.gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
|
||||
)
|
||||
|
||||
func newGuardJudgeCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
@@ -27,9 +24,15 @@ func newGuardJudgeCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
Long: "A policy with a judge has a model read what patterns cannot. The judge must\n" +
|
||||
"be in the catalog under the name the policy gives, as a judge (role:\n" +
|
||||
"guard), with a replica ready; otherwise what the policy judges is refused\n" +
|
||||
"(fail: closed, guard_unavailable) or served unjudged (fail: open).",
|
||||
"(fail: closed, guard_unavailable) or served unjudged (fail: open).\n\n" +
|
||||
"A policy may name several, asked in turn (judge.models): a judge on a GPU\n" +
|
||||
"first and one on CPUs after it is the GPU's speed while it is there, and\n" +
|
||||
"the CPUs' when it is not.\n\n" +
|
||||
"Judges llmgw knows where to fetch, which a policy may name and the leader\n" +
|
||||
"then fetches and serves by itself: " + knownJudgeNames() + ".",
|
||||
}
|
||||
c.AddCommand(newGuardJudgeStatusCmd(client, asJSON), newGuardJudgeEnableCmd(client), newGuardJudgeAskCmd(client, asJSON))
|
||||
c.AddCommand(newGuardJudgeStatusCmd(client, asJSON), newGuardJudgeEnableCmd(client),
|
||||
newGuardJudgeUseCmd(client), newGuardJudgeAskCmd(client, asJSON))
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -61,6 +64,11 @@ func newGuardJudgeStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Comma
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d/%d\t%s\n", j.Model, strings.Join(j.Policies, ","),
|
||||
yesNo(j.InCatalog), yesNo(j.Judge), j.Ready, j.Wanted, state)
|
||||
}
|
||||
for _, j := range js {
|
||||
if len(j.Then) > 0 {
|
||||
fmt.Fprintf(w, "\n%s gives no verdict: %s is asked instead", j.Model, strings.Join(j.Then, ", else "))
|
||||
}
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -88,7 +96,7 @@ func strayJudges(cmd *cobra.Command, client func() *ctlClient, js []adminapi.Jud
|
||||
}
|
||||
var out []string
|
||||
for _, m := range models {
|
||||
if !asked[m.Name] && strings.Contains(strings.ToLower(m.Source), "llama-guard") {
|
||||
if _, judge := guard.FormatOfWeights(m.Source); !asked[m.Name] && judge {
|
||||
out = append(out, m.Name)
|
||||
}
|
||||
}
|
||||
@@ -97,22 +105,29 @@ func strayJudges(cmd *cobra.Command, client func() *ctlClient, js []adminapi.Jud
|
||||
|
||||
func newGuardJudgeEnableCmd(client func() *ctlClient) *cobra.Command {
|
||||
var (
|
||||
model, repo, file string
|
||||
replicas, cores int
|
||||
replace, detach bool
|
||||
model, repo, file, format, gpuClass string
|
||||
replicas, cores int
|
||||
gpu, replace, detach bool
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "enable",
|
||||
Short: "Serve the judge the policies ask, as a judge: Llama Guard 3 1B on CPUs",
|
||||
Long: "Fetches Llama Guard 3 1B into the mirror (once), adds it to the catalog under\n" +
|
||||
"the name the policies give its judge, as a judge (role: guard,\n" +
|
||||
"format llamaguard3), on CPUs so it takes no card, and waits for a replica\n" +
|
||||
"to answer. With no --model, the judge the policies ask that is missing.\n\n" +
|
||||
"A model of that name already in the catalog as a model clients use is\n" +
|
||||
"refused; --replace removes it first. An offline cluster gets its judge\n" +
|
||||
"from its bundle with server init.",
|
||||
Short: "Serve a judge: Qwen3Guard-Gen 0.6B on CPUs, or with --gpu the 4B on a GPU",
|
||||
Long: "Fetches a judge's weights into the mirror (once; what is there already, its\n" +
|
||||
"checksums matching, is not fetched again), adds it to the catalog as a judge\n" +
|
||||
"(role: guard) and waits for a replica to answer. Which judge: --model, or\n" +
|
||||
"with --gpu " + defaultGPUJudge + ", or else the first the policies ask that is\n" +
|
||||
"missing, or else " + defaultJudge + ".\n\n" +
|
||||
"A judge on CPUs takes no card. One on a GPU takes one card, as a standard\n" +
|
||||
"model: from nothing more important, so it waits while the cards are busy;\n" +
|
||||
"put a judge on CPUs after it in the policies (llmgw ctl guard judge use) so\n" +
|
||||
"there is always one to ask.\n\n" +
|
||||
"Known judges: " + knownJudgeNames() + ". Another is served with --repo,\n" +
|
||||
"--file and --format. A model of that name already in the catalog as a\n" +
|
||||
"model clients use is refused; --replace removes it first. An offline\n" +
|
||||
"cluster gets its judge from its bundle with server init.",
|
||||
Example: " llmgw ctl guard judge enable\n" +
|
||||
" llmgw ctl guard judge enable --replace",
|
||||
" llmgw ctl guard judge enable --gpu\n" +
|
||||
" llmgw ctl guard judge use " + defaultGPUJudge + " " + defaultJudge,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
ctx := cmd.Context()
|
||||
@@ -121,45 +136,72 @@ func newGuardJudgeEnableCmd(client func() *ctlClient) *cobra.Command {
|
||||
if err := client().get(ctx, "/api/v1/guard/judges", &js); err != nil {
|
||||
return err
|
||||
}
|
||||
var target *adminapi.JudgeView
|
||||
for i, j := range js {
|
||||
if (model != "" && j.Model == model) || (model == "" && (!j.InCatalog || !j.Judge)) {
|
||||
target = &js[i]
|
||||
break
|
||||
name := model
|
||||
if name == "" && gpu {
|
||||
name = defaultGPUJudge
|
||||
}
|
||||
if name == "" {
|
||||
for _, j := range js {
|
||||
if !j.InCatalog || !j.Judge {
|
||||
name = j.Model
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if name == "" {
|
||||
name = defaultJudge
|
||||
}
|
||||
|
||||
k, known := knownJudgeNamed(name)
|
||||
switch {
|
||||
case target == nil && model != "":
|
||||
target = &adminapi.JudgeView{Model: model}
|
||||
fmt.Fprintf(out, "no policy asks %s yet; it is served, and a policy's judge.model names it\n", model)
|
||||
case target == nil && len(js) == 0:
|
||||
return errors.New("no policy asks a judge: give one a judge (llmgw ctl config edit, guard.policies.<name>.judge.model)")
|
||||
case target == nil:
|
||||
fmt.Fprintln(out, "every judge the policies ask is in the catalog, as a judge")
|
||||
return nil
|
||||
case repo != "":
|
||||
f := format
|
||||
if f == "" {
|
||||
f, _ = guard.FormatOfWeights(repo + "/" + file)
|
||||
}
|
||||
if f == "" || !guard.KnownFormat(f) {
|
||||
return fmt.Errorf("say how its answers are read: --format %s", strings.Join(guard.Formats, "|"))
|
||||
}
|
||||
k = knownJudge{Name: name, Repo: repo, File: file, Format: f, GPU: gpu}
|
||||
case !known:
|
||||
return fmt.Errorf("%s is not a judge llmgw knows where to fetch (%s): give --repo, --file and --format", name, knownJudgeNames())
|
||||
}
|
||||
if target.InCatalog && target.Judge {
|
||||
fmt.Fprintf(out, "%s is in the catalog as a judge already; llmgw ctl guard judge status says how it is\n", target.Model)
|
||||
return nil
|
||||
if gpu {
|
||||
k.GPU = true
|
||||
}
|
||||
if target.InCatalog && !target.Judge {
|
||||
|
||||
var models []adminapi.ModelView
|
||||
if err := client().get(ctx, "/api/v1/models", &models); err != nil {
|
||||
return err
|
||||
}
|
||||
if i := slices.IndexFunc(models, func(m adminapi.ModelView) bool { return m.Name == name }); i >= 0 {
|
||||
if models[i].Role == catalog.RoleGuard {
|
||||
fmt.Fprintf(out, "%s is in the catalog as a judge already; llmgw ctl guard judge status says how it is\n", name)
|
||||
return nil
|
||||
}
|
||||
if !replace {
|
||||
return fmt.Errorf("%s is in the catalog as a model clients use, not as a judge: "+
|
||||
"--replace removes it and serves it again as a judge", target.Model)
|
||||
"--replace removes it and serves it again as a judge", name)
|
||||
}
|
||||
var res map[string]string
|
||||
if err := client().delete(ctx, "/api/v1/models/"+target.Model, &res); err != nil {
|
||||
return fmt.Errorf("removing %s: %w", target.Model, err)
|
||||
if err := client().delete(ctx, "/api/v1/models/"+name, &res); err != nil {
|
||||
return fmt.Errorf("removing %s: %w", name, err)
|
||||
}
|
||||
fmt.Fprintf(out, "%s removed\n", target.Model)
|
||||
fmt.Fprintf(out, "%s removed\n", name)
|
||||
}
|
||||
req := modelserve.Request{
|
||||
Repo: repo, File: file, Name: target.Model, GuardFormat: "llamaguard3",
|
||||
Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.",
|
||||
CPU: true, Replicas: replicas, CPUCores: cores,
|
||||
CtxPool: 8192, ContextLength: 4096,
|
||||
|
||||
opts := judgeServeOptions{Replicas: replicas, Cores: cores, GPUClass: gpuClass}
|
||||
if k.GPU && opts.GPUClass == "" {
|
||||
var s adminapi.Summary
|
||||
if err := client().get(ctx, "/api/v1/summary", &s); err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.GPUClass = roomiestGPUClass(s.GPUs); opts.GPUClass == "" {
|
||||
return fmt.Errorf("this cluster has no GPU for %s: serve %s on CPUs instead", name, defaultJudge)
|
||||
}
|
||||
}
|
||||
body, _ := json.Marshal(req)
|
||||
fmt.Fprintf(out, "serving %s\n", k)
|
||||
body, _ := json.Marshal(k.serveRequest(opts))
|
||||
var job modelserve.Job
|
||||
if err := client().patient(30*time.Second).post(ctx, "/api/v1/models/serve", body, &job); err != nil {
|
||||
return err
|
||||
@@ -170,6 +212,13 @@ func newGuardJudgeEnableCmd(client func() *ctlClient) *cobra.Command {
|
||||
if err := followJob(cmd, client, job, "is the judge"); err != nil {
|
||||
return err
|
||||
}
|
||||
if !slices.ContainsFunc(js, func(j adminapi.JudgeView) bool { return j.Model == name }) {
|
||||
fmt.Fprintf(out, "\nno policy asks %s yet: llmgw ctl guard judge use %s", name, name)
|
||||
if k.GPU {
|
||||
fmt.Fprintf(out, " %s", defaultJudge)
|
||||
}
|
||||
fmt.Fprintln(out)
|
||||
}
|
||||
if strays, _ := strayJudges(cmd, client, js); len(strays) > 0 {
|
||||
fmt.Fprintf(out, "\nno policy asks %s, served from a judge's weights as a model clients use: "+
|
||||
"llmgw ctl model remove %s\n", strings.Join(strays, ", "), strings.Join(strays, " "))
|
||||
@@ -178,16 +227,118 @@ func newGuardJudgeEnableCmd(client func() *ctlClient) *cobra.Command {
|
||||
},
|
||||
}
|
||||
f := c.Flags()
|
||||
f.StringVar(&model, "model", "", "the judge to serve (default: the one the policies ask that is missing)")
|
||||
f.StringVar(&repo, "repo", guardModelRepo, "the registry repository its weights come from")
|
||||
f.StringVar(&file, "file", guardModelFile, "the file in it")
|
||||
f.IntVar(&replicas, "replicas", 2, "replicas: every judged request waits on one")
|
||||
f.IntVar(&cores, "cores", 4, "CPU cores a replica computes on")
|
||||
f.StringVar(&model, "model", "", "the judge to serve: "+knownJudgeNames()+", or a name for --repo's")
|
||||
f.BoolVar(&gpu, "gpu", false, "serve it on a GPU ("+defaultGPUJudge+" when no --model)")
|
||||
f.StringVar(&gpuClass, "gpu-class", "", "the GPUs it runs on (default: the class with the most free cards)")
|
||||
f.StringVar(&repo, "repo", "", "the registry repository of a judge llmgw does not know")
|
||||
f.StringVar(&file, "file", "", "the GGUF file in --repo")
|
||||
f.StringVar(&format, "format", "", "how --repo's judge answers: "+strings.Join(guard.Formats, ", "))
|
||||
f.IntVar(&replicas, "replicas", 0, "replicas (default: 2 on CPUs, as every judged request waits on one; 1 on a GPU)")
|
||||
f.IntVar(&cores, "cores", 4, "CPU cores a replica on CPUs computes on")
|
||||
f.BoolVar(&replace, "replace", false, "remove a model of that name that is not a judge first")
|
||||
f.BoolVar(&detach, "detach", false, "start it and return; llmgw ctl jobs follows it")
|
||||
return c
|
||||
}
|
||||
|
||||
// roomiestGPUClass is the class with the most free cards, else the most
|
||||
// cards; empty when the cluster has none.
|
||||
func roomiestGPUClass(classes []adminapi.ClassUse) string {
|
||||
best := -1
|
||||
for i, c := range classes {
|
||||
if c.Total == 0 {
|
||||
continue
|
||||
}
|
||||
if best < 0 || c.Free() > classes[best].Free() || (c.Free() == classes[best].Free() && c.Total > classes[best].Total) {
|
||||
best = i
|
||||
}
|
||||
}
|
||||
if best < 0 {
|
||||
return ""
|
||||
}
|
||||
return classes[best].Class
|
||||
}
|
||||
|
||||
func newGuardJudgeUseCmd(client func() *ctlClient) *cobra.Command {
|
||||
var policies []string
|
||||
var note string
|
||||
c := &cobra.Command{
|
||||
Use: "use JUDGE...",
|
||||
Short: "Set the judges the policies ask, in the order they are asked",
|
||||
Long: "Sets judge.models of every policy that has a judge, or of those given with\n" +
|
||||
"--policy: the first judge that answers gives the verdict, and the next is\n" +
|
||||
"asked when one has no replica serving, fails, or does not answer within\n" +
|
||||
"the policy's latency_budget. A judge on a GPU first and one on CPUs after\n" +
|
||||
"it is the GPU's speed while it is there, and the CPUs' when it is not.\n\n" +
|
||||
"A judge llmgw knows (" + knownJudgeNames() + ")\n" +
|
||||
"that is not in the catalog is fetched and served by the leader; another\n" +
|
||||
"is served first with llmgw ctl guard judge enable --repo. In force on\n" +
|
||||
"every control plane within seconds.",
|
||||
Example: " llmgw ctl guard judge use " + defaultJudge + "\n" +
|
||||
" llmgw ctl guard judge use " + defaultGPUJudge + " " + defaultJudge + "\n" +
|
||||
" llmgw ctl guard judge use " + defaultGPUJudge + " " + defaultJudge + " --policy strict",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
for i, a := range args {
|
||||
if slices.Contains(args[:i], a) {
|
||||
return fmt.Errorf("%s is named twice", a)
|
||||
}
|
||||
}
|
||||
if note == "" {
|
||||
note = "judges: " + strings.Join(args, ", else ")
|
||||
}
|
||||
list := "[" + strings.Join(args, ", ") + "]"
|
||||
return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
|
||||
names := policies
|
||||
if len(names) == 0 {
|
||||
for _, p := range doc.MapKeys([]string{"guard", "policies"}) {
|
||||
if doc.Node([]string{"guard", "policies", p, "judge"}) != nil {
|
||||
names = append(names, p)
|
||||
}
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return errors.New("no policy has a judge: name one with --policy, and it is given one " +
|
||||
"that blocks what the judge finds unsafe in requests (judge.input: block)")
|
||||
}
|
||||
}
|
||||
for _, p := range names {
|
||||
base := []string{"guard", "policies", p}
|
||||
if doc.Node(base) == nil {
|
||||
return fmt.Errorf("there is no policy %s (llmgw ctl guard list)", p)
|
||||
}
|
||||
judge := append(slices.Clone(base), "judge")
|
||||
if doc.Node(judge) == nil {
|
||||
if err := doc.SetYAML(append(slices.Clone(judge), "input"), "block"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
doc.Unset(append(slices.Clone(judge), "model"))
|
||||
if err := doc.SetYAML(append(slices.Clone(judge), "models"), list); err != nil {
|
||||
return err
|
||||
}
|
||||
cats := append(slices.Clone(judge), "categories")
|
||||
var have []string
|
||||
if n := doc.Node(cats); n != nil {
|
||||
for _, c := range n.Content {
|
||||
have = append(have, c.Value)
|
||||
}
|
||||
}
|
||||
if next := categoriesFor(args, have); next != nil {
|
||||
if err := doc.SetYAML(cats, "["+strings.Join(next, ", ")+"]"); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s: categories [%s] are not what %s name; it counts [%s] instead\n",
|
||||
p, strings.Join(have, ", "), strings.Join(args, " or "), strings.Join(next, ", "))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
},
|
||||
}
|
||||
c.Flags().StringSliceVar(&policies, "policy", nil, "the policies to change (default: every one with a judge)")
|
||||
c.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version")
|
||||
return c
|
||||
}
|
||||
|
||||
func newGuardJudgeAskCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
var model, answer string
|
||||
c := &cobra.Command{
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
)
|
||||
|
||||
// judgeKeeper fetches and serves, on the leader, each judge llmgw knows that
|
||||
// a policy names and the catalog does not have: naming qwen3guard-4b in a
|
||||
// policy is all it takes to have one, and a judge removed by mistake comes
|
||||
// back. A judge it does not know is left to llmgw ctl guard judge enable
|
||||
// --repo, said in guard judge status.
|
||||
type judgeKeeper struct {
|
||||
library func() guard.Config
|
||||
models func() []catalog.Model
|
||||
serve interface {
|
||||
Start(modelserve.Request) (modelserve.Job, error)
|
||||
Jobs() []modelserve.Job
|
||||
}
|
||||
leader func() bool
|
||||
gpus func() []adminapi.ClassUse
|
||||
log *slog.Logger
|
||||
now func() time.Time
|
||||
|
||||
// tried is when each judge was last asked for, so one that cannot be
|
||||
// fetched is not asked for again on every look.
|
||||
tried map[string]time.Time
|
||||
}
|
||||
|
||||
// judgeRetry is how long a judge whose serving failed waits before it is
|
||||
// asked for again.
|
||||
const judgeRetry = 10 * time.Minute
|
||||
|
||||
func (k *judgeKeeper) run(ctx context.Context, every time.Duration) {
|
||||
t := time.NewTicker(every)
|
||||
defer t.Stop()
|
||||
for {
|
||||
k.check()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (k *judgeKeeper) check() {
|
||||
if k.leader != nil && !k.leader() {
|
||||
return
|
||||
}
|
||||
lib := k.library()
|
||||
if !lib.On() {
|
||||
return
|
||||
}
|
||||
if k.tried == nil {
|
||||
k.tried = map[string]time.Time{}
|
||||
}
|
||||
now := time.Now
|
||||
if k.now != nil {
|
||||
now = k.now
|
||||
}
|
||||
have := map[string]bool{}
|
||||
for _, m := range k.models() {
|
||||
have[m.Name] = true
|
||||
}
|
||||
busy := map[string]bool{}
|
||||
for _, j := range k.serve.Jobs() {
|
||||
if j.State == modelserve.Running {
|
||||
busy[j.Model] = true
|
||||
busy[j.Request.Name] = true
|
||||
}
|
||||
}
|
||||
var wanted []string
|
||||
for _, p := range lib.Policies {
|
||||
for _, j := range p.Judge.Judges() {
|
||||
if !slices.Contains(wanted, j) {
|
||||
wanted = append(wanted, j)
|
||||
}
|
||||
}
|
||||
}
|
||||
slices.Sort(wanted)
|
||||
for _, name := range wanted {
|
||||
kj, known := knownJudgeNamed(name)
|
||||
if have[name] || !known || busy[name] {
|
||||
continue
|
||||
}
|
||||
if at, ok := k.tried[name]; ok && now().Sub(at) < judgeRetry {
|
||||
continue
|
||||
}
|
||||
k.tried[name] = now()
|
||||
var opts judgeServeOptions
|
||||
if kj.GPU {
|
||||
if opts.GPUClass = roomiestGPUClass(k.gpus()); opts.GPUClass == "" {
|
||||
k.log.Warn("a policy names a judge that runs on a GPU, and this cluster has none",
|
||||
"judge", name, "instead", defaultJudge)
|
||||
continue
|
||||
}
|
||||
}
|
||||
req := kj.serveRequest(opts)
|
||||
req.By = "llmgw (a guard policy names it)"
|
||||
job, err := k.serve.Start(req)
|
||||
if err != nil {
|
||||
k.log.Error("could not start serving a judge a policy names", "judge", name, "err", err)
|
||||
continue
|
||||
}
|
||||
k.log.Info("serving a judge a policy names, which the catalog did not have",
|
||||
"judge", name, "repo", kj.Repo, "file", kj.File, "job", job.ID)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
)
|
||||
|
||||
type fakeServe struct {
|
||||
started []modelserve.Request
|
||||
running []modelserve.Job
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeServe) Start(r modelserve.Request) (modelserve.Job, error) {
|
||||
if f.err != nil {
|
||||
return modelserve.Job{}, f.err
|
||||
}
|
||||
f.started = append(f.started, r)
|
||||
return modelserve.Job{ID: "j", Model: r.Name, Request: r, State: modelserve.Running}, nil
|
||||
}
|
||||
func (f *fakeServe) Jobs() []modelserve.Job { return f.running }
|
||||
|
||||
func TestAJudgeAPolicyNamesIsFetchedWhenTheCatalogLacksIt(t *testing.T) {
|
||||
lib := guard.Config{Policies: map[string]guard.Policy{
|
||||
"baseline": {Judge: &guard.JudgePolicy{Models: []string{"qwen3guard-4b", "qwen3guard-0.6b"}, Input: "block"}},
|
||||
"custom": {Judge: &guard.JudgePolicy{Model: "our-own-judge", Input: "block"}},
|
||||
}}
|
||||
inCatalog := []catalog.Model{{Name: "qwen3guard-0.6b"}}
|
||||
now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||
fs := &fakeServe{}
|
||||
leader := true
|
||||
k := &judgeKeeper{
|
||||
library: func() guard.Config { return lib },
|
||||
models: func() []catalog.Model { return inCatalog },
|
||||
serve: fs,
|
||||
leader: func() bool { return leader },
|
||||
gpus: func() []adminapi.ClassUse { return []adminapi.ClassUse{{Class: "6000-24g", Total: 1, Used: 1}} },
|
||||
log: slog.New(slog.DiscardHandler),
|
||||
now: func() time.Time { return now },
|
||||
}
|
||||
|
||||
k.check()
|
||||
// The GPU judge alone: the CPU one is there, and one llmgw does not know
|
||||
// is not guessed at.
|
||||
if len(fs.started) != 1 || fs.started[0].Name != "qwen3guard-4b" || fs.started[0].GPUClass != "6000-24g" ||
|
||||
fs.started[0].Class != catalog.ClassStandard || fs.started[0].GuardFormat != "qwen3guard" || fs.started[0].CPU {
|
||||
t.Fatalf("started %+v", fs.started)
|
||||
}
|
||||
|
||||
// Being fetched: not asked for again.
|
||||
fs.running = []modelserve.Job{{Model: "qwen3guard-4b", State: modelserve.Running}}
|
||||
k.check()
|
||||
if len(fs.started) != 1 {
|
||||
t.Errorf("asked for again while it is being fetched: %d", len(fs.started))
|
||||
}
|
||||
// Failed: asked for again only after a while.
|
||||
fs.running = nil
|
||||
now = now.Add(time.Minute)
|
||||
k.check()
|
||||
if len(fs.started) != 1 {
|
||||
t.Error("asked for again at once after it failed")
|
||||
}
|
||||
now = now.Add(judgeRetry)
|
||||
k.check()
|
||||
if len(fs.started) != 2 {
|
||||
t.Error("never asked for again")
|
||||
}
|
||||
|
||||
// Only the leader fetches, and nothing while the guardrails are off.
|
||||
leader = false
|
||||
now = now.Add(judgeRetry)
|
||||
k.check()
|
||||
leader = true
|
||||
off := false
|
||||
lib.Enabled = &off
|
||||
k.check()
|
||||
if len(fs.started) != 2 {
|
||||
t.Errorf("a standby or the guardrails off fetched: %d", len(fs.started))
|
||||
}
|
||||
|
||||
// A CPU judge on a cluster without GPUs is served on CPUs, two of it.
|
||||
lib = guard.Config{Policies: map[string]guard.Policy{"b": {Judge: &guard.JudgePolicy{Model: "qwen3guard-0.6b", Input: "block"}}}}
|
||||
inCatalog = nil
|
||||
k.gpus = func() []adminapi.ClassUse { return nil }
|
||||
k.check()
|
||||
if last := fs.started[len(fs.started)-1]; last.Name != "qwen3guard-0.6b" || !last.CPU || last.Replicas != 2 {
|
||||
t.Errorf("CPU judge: %+v", last)
|
||||
}
|
||||
fs.err = errors.New("the mirror is down")
|
||||
lib = guard.Config{Policies: map[string]guard.Policy{"b": {Judge: &guard.JudgePolicy{Model: "qwen3guard-8b", Input: "block"}}}}
|
||||
k.check() // no GPU, so not started, and no panic on the error path either
|
||||
}
|
||||
|
||||
// A policy moved from Llama Guard's codes to Qwen3Guard would count nothing:
|
||||
// its categories become the judge's own.
|
||||
func TestCategoriesFollowTheJudges(t *testing.T) {
|
||||
lg := []string{"S1", "S2", "S10"}
|
||||
if got := categoriesFor([]string{"qwen3guard-0.6b"}, lg); !slices.Equal(got, baselineCategories["qwen3guard"]) {
|
||||
t.Errorf("Llama Guard's codes for Qwen3Guard: %v", got)
|
||||
}
|
||||
if got := categoriesFor([]string{"qwen3guard-4b", "qwen3guard-0.6b"}, []string{"Violent", "Jailbreak"}); got != nil {
|
||||
t.Errorf("its own names were replaced: %v", got)
|
||||
}
|
||||
if got := categoriesFor([]string{"llama-guard-3-1b"}, lg); got != nil {
|
||||
t.Errorf("Llama Guard's own codes were replaced: %v", got)
|
||||
}
|
||||
// Both kinds asked in turn: each must have something to count.
|
||||
got := categoriesFor([]string{"qwen3guard-4b", "llama-guard-3-1b"}, []string{"Violent"})
|
||||
if !slices.Contains(got, "Violent") || !slices.Contains(got, "S1") {
|
||||
t.Errorf("mixed: %v", got)
|
||||
}
|
||||
if got := categoriesFor([]string{"our-own-judge"}, lg); got != nil {
|
||||
t.Errorf("a judge llmgw does not know: %v", got)
|
||||
}
|
||||
if got := categoriesFor([]string{"qwen3guard-0.6b"}, nil); got != nil {
|
||||
t.Errorf("no categories (every one counts) became %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
)
|
||||
|
||||
// knownJudge is a judge model llmgw knows where to fetch and how to serve:
|
||||
// what a policy may name without anyone saying where its weights are.
|
||||
type knownJudge struct {
|
||||
Name, Repo, File, Format string
|
||||
// GPU is a judge too slow on CPUs to wait for: served on one card.
|
||||
GPU bool
|
||||
About string
|
||||
}
|
||||
|
||||
// knownJudges, the default first. Measured on the reference cluster against
|
||||
// ordinary coding requests ("continue", "kill the process on port 8080") and
|
||||
// harmful ones: Llama Guard 3 1B scored both alike, about one in two
|
||||
// ordinary ones refused; Qwen3Guard-Gen 0.6B kept them apart, 0.15 against
|
||||
// 1.00 at the worst, at about two seconds a check on two CPU threads.
|
||||
var knownJudges = []knownJudge{
|
||||
{Name: "qwen3guard-0.6b", Repo: "mradermacher/Qwen3Guard-Gen-0.6B-GGUF", File: "Qwen3Guard-Gen-0.6B.Q8_0.gguf",
|
||||
Format: "qwen3guard", About: "Qwen3Guard-Gen 0.6B on CPUs, 0.8 GB: the default"},
|
||||
{Name: "qwen3guard-4b", Repo: "mradermacher/Qwen3Guard-Gen-4B-GGUF", File: "Qwen3Guard-Gen-4B.Q4_K_M.gguf",
|
||||
Format: "qwen3guard", GPU: true, About: "Qwen3Guard-Gen 4B on a GPU, 2.5 GB"},
|
||||
{Name: "qwen3guard-8b", Repo: "mradermacher/Qwen3Guard-Gen-8B-GGUF", File: "Qwen3Guard-Gen-8B.Q4_K_M.gguf",
|
||||
Format: "qwen3guard", GPU: true, About: "Qwen3Guard-Gen 8B on a GPU, 4.7 GB"},
|
||||
{Name: "llama-guard-3-1b", Repo: "QuantFactory/Llama-Guard-3-1B-GGUF", File: "Llama-Guard-3-1B.Q4_K_M.gguf",
|
||||
Format: "llamaguard3", About: "Llama Guard 3 1B on CPUs, 0.9 GB: refuses ordinary coding requests"},
|
||||
{Name: "llama-guard-3-8b", Repo: "QuantFactory/Llama-Guard-3-8B-GGUF", File: "Llama-Guard-3-8B.Q4_K_M.gguf",
|
||||
Format: "llamaguard3", GPU: true, About: "Llama Guard 3 8B on a GPU, 4.6 GB"},
|
||||
}
|
||||
|
||||
const (
|
||||
defaultJudge = "qwen3guard-0.6b"
|
||||
defaultGPUJudge = "qwen3guard-4b"
|
||||
)
|
||||
|
||||
// baselineCategories are what the baseline policy counts of each format's
|
||||
// verdicts: not PII, which is the patterns', nor what refuses ordinary work
|
||||
// — copyright, unethical acts, politically sensitive topics; Llama Guard's
|
||||
// defamation, specialized advice, intellectual property and elections.
|
||||
var baselineCategories = map[string][]string{
|
||||
"qwen3guard": {"Violent", "Non-violent Illegal Acts", "Sexual Content or Sexual Acts", "Suicide & Self-Harm", "Jailbreak"},
|
||||
"llamaguard3": {"S1", "S2", "S3", "S4", "S7", "S9", "S10", "S11", "S12"},
|
||||
}
|
||||
|
||||
// reported is whether a judge of that format ever names category c: its own
|
||||
// names, or for Llama Guard its codes.
|
||||
func reported(format, c string) bool {
|
||||
if slices.ContainsFunc(guard.DefaultCategories(format), func(n string) bool { return strings.EqualFold(n, c) }) {
|
||||
return true
|
||||
}
|
||||
var n int
|
||||
_, err := fmt.Sscanf(c, "S%d", &n)
|
||||
return format == "llamaguard3" && err == nil && n >= 1 && n <= 14
|
||||
}
|
||||
|
||||
// categoriesFor is what a policy asking these judges should count, when the
|
||||
// categories it has are ones none of them names: a policy moved from Llama
|
||||
// Guard's S1 to Qwen3Guard, which says Violent, would otherwise count
|
||||
// nothing. Nil keeps what it has; so does a judge llmgw does not know.
|
||||
func categoriesFor(judges, have []string) []string {
|
||||
var formats []string
|
||||
for _, j := range judges {
|
||||
k, ok := knownJudgeNamed(j)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if !slices.Contains(formats, k.Format) {
|
||||
formats = append(formats, k.Format)
|
||||
}
|
||||
}
|
||||
if len(have) == 0 {
|
||||
return nil
|
||||
}
|
||||
stray := false
|
||||
for _, f := range formats {
|
||||
for _, c := range have {
|
||||
if !reported(f, c) && !slices.ContainsFunc(formats, func(o string) bool { return reported(o, c) }) {
|
||||
stray = true
|
||||
}
|
||||
}
|
||||
// Every judge must have something it counts.
|
||||
if !slices.ContainsFunc(have, func(c string) bool { return reported(f, c) }) {
|
||||
stray = true
|
||||
}
|
||||
}
|
||||
if !stray {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, f := range formats {
|
||||
out = append(out, baselineCategories[f]...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func knownJudgeNamed(name string) (knownJudge, bool) {
|
||||
i := slices.IndexFunc(knownJudges, func(k knownJudge) bool { return k.Name == name })
|
||||
if i < 0 {
|
||||
return knownJudge{}, false
|
||||
}
|
||||
return knownJudges[i], true
|
||||
}
|
||||
|
||||
func knownJudgeNames() string {
|
||||
var n []string
|
||||
for _, k := range knownJudges {
|
||||
n = append(n, k.Name)
|
||||
}
|
||||
return strings.Join(n, ", ")
|
||||
}
|
||||
|
||||
// judgeServeOptions are how a judge is served beyond which weights.
|
||||
type judgeServeOptions struct {
|
||||
Replicas, Cores int
|
||||
// GPUClass is the cards a GPU judge runs on; empty lets the plan choose.
|
||||
GPUClass string
|
||||
}
|
||||
|
||||
// serveRequest is the serve job for a judge. A judge on CPUs is critical,
|
||||
// as every judged request waits on it; one on a GPU is standard, so it takes
|
||||
// a card only from what may give one up, and the judge on CPUs after it in
|
||||
// the policies stands in while it has none.
|
||||
func (k knownJudge) serveRequest(o judgeServeOptions) modelserve.Request {
|
||||
req := modelserve.Request{
|
||||
Repo: k.Repo, File: k.File, Name: k.Name, GuardFormat: k.Format,
|
||||
Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.",
|
||||
Replicas: o.Replicas, CtxPool: 8192, ContextLength: 4096,
|
||||
}
|
||||
if k.GPU {
|
||||
req.GPUClass, req.GPUsPerNode, req.Class = o.GPUClass, 1, catalog.ClassStandard
|
||||
if req.Replicas == 0 {
|
||||
req.Replicas = 1
|
||||
}
|
||||
return req
|
||||
}
|
||||
req.CPU, req.CPUCores = true, o.Cores
|
||||
if req.Replicas == 0 {
|
||||
req.Replicas = 2
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
func (k knownJudge) String() string {
|
||||
where := "CPUs"
|
||||
if k.GPU {
|
||||
where = "a GPU"
|
||||
}
|
||||
return fmt.Sprintf("%s (%s, %s/%s, on %s)", k.Name, k.Format, k.Repo, k.File, where)
|
||||
}
|
||||
@@ -518,6 +518,21 @@ func newServerCmd() *cobra.Command {
|
||||
})
|
||||
go serve.WatchCanaries(ctx, 30*time.Second)
|
||||
adminAPI.SetServe(serve)
|
||||
if gate != nil {
|
||||
// A judge a policy names and the catalog lacks is
|
||||
// fetched and served, on the leader.
|
||||
keeper := &judgeKeeper{
|
||||
library: gate.Library,
|
||||
models: func() []catalog.Model { return admin.Catalog().Models() },
|
||||
serve: serve,
|
||||
leader: func() bool {
|
||||
return admin.Leadership() == nil || admin.Leadership().IsLeader()
|
||||
},
|
||||
gpus: func() []adminapi.ClassUse { return adminAPI.Summary().GPUs },
|
||||
log: log,
|
||||
}
|
||||
go keeper.run(ctx, 30*time.Second)
|
||||
}
|
||||
updateMetrics.Set(serve)
|
||||
go serve.WatchUpdates(ctx, cfg.Mirror.UpdateCheck)
|
||||
}
|
||||
|
||||
@@ -293,7 +293,7 @@ func (in *serverInit) configJudgesWith(model string) bool {
|
||||
return false
|
||||
}
|
||||
for _, p := range cfg.Guard.Policies {
|
||||
if p.Judge != nil && p.Judge.Model == model {
|
||||
if slices.Contains(p.Judge.Judges(), model) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -308,18 +308,21 @@ func guardSection(judge bool) string {
|
||||
# judge: {model: ` + guardModelName + `, input: block, output: block, ...}
|
||||
`
|
||||
if judge {
|
||||
j = ` # A judge model reads what patterns cannot: violence, weapons, abuse,
|
||||
# self-harm. Two replicas run on CPUs with AVX2, about half a second
|
||||
# for a short text; an answer is read every 600 characters as it
|
||||
# streams. With fail: closed, a request the judge cannot read within
|
||||
j = ` # A judge model reads what patterns cannot: violence, crime, sexual
|
||||
# content, self-harm, jailbreaks. Two replicas run on CPUs, about two
|
||||
# seconds for a short text; an answer is read every 600 characters as
|
||||
# it streams. With fail: closed, a request no judge can read within
|
||||
# latency_budget is refused (503); open serves it, flagged, unjudged.
|
||||
# Defamation (S5), specialized advice (S6), intellectual property (S8)
|
||||
# and elections (S13) are left out: they refuse too much ordinary work.
|
||||
# PII is the patterns' (above), and copyright, unethical acts and
|
||||
# politically sensitive topics are left out: they refuse too much
|
||||
# ordinary work. Controversial counts as safe; controversial: unsafe
|
||||
# makes it count. With a GPU: llmgw ctl guard judge enable --gpu, then
|
||||
# llmgw ctl guard judge use ` + defaultGPUJudge + ` ` + guardModelName + `.
|
||||
judge:
|
||||
model: ` + guardModelName + `
|
||||
models: [` + guardModelName + `]
|
||||
input: block
|
||||
output: block
|
||||
categories: [S1, S2, S3, S4, S7, S9, S10, S11, S12]
|
||||
categories: [Violent, Non-violent Illegal Acts, Sexual Content or Sexual Acts, Suicide & Self-Harm, Jailbreak]
|
||||
latency_budget: 5s
|
||||
`
|
||||
}
|
||||
|
||||
@@ -41,11 +41,13 @@ const (
|
||||
starterSource = "hf:Qwen/Qwen2.5-0.5B-Instruct-GGUF/qwen2.5-0.5b-instruct-q4_k_m.gguf"
|
||||
starterVersion = "starter"
|
||||
|
||||
// The guardrails' judge: Meta's Llama Guard 3 1B, which reads a prompt
|
||||
// or an answer and says whether it is unsafe and why. Q4_K_M is about
|
||||
// 900 MB and judges a short text in about half a second on a CPU.
|
||||
guardModelName = "llama-guard-3-1b"
|
||||
guardModelSource = "hf:QuantFactory/Llama-Guard-3-1B-GGUF/Llama-Guard-3-1B.Q4_K_M.gguf"
|
||||
// The guardrails' judge: Qwen3Guard-Gen 0.6B, which reads a prompt or an
|
||||
// answer and says whether it is safe, controversial or unsafe, and why.
|
||||
// Q8_0 is 0.8 GB and judges a short text in about two seconds on two CPU
|
||||
// threads. It replaced Llama Guard 3 1B, which refused about one in two
|
||||
// ordinary coding requests ("continue").
|
||||
guardModelName = defaultJudge
|
||||
guardModelSource = "hf:mradermacher/Qwen3Guard-Gen-0.6B-GGUF/Qwen3Guard-Gen-0.6B.Q8_0.gguf"
|
||||
guardModelVersion = "guard"
|
||||
|
||||
// localLabel marks the control plane's own node, so the starter model is
|
||||
@@ -443,12 +445,6 @@ func starterModel(source string, gpuClasses []string, pinned bool) catalog.Model
|
||||
}
|
||||
}
|
||||
|
||||
// llamaGuard3Categories name Llama Guard 3's S1 to S13, so a verdict is
|
||||
// recorded as "Violent Crimes" rather than "S1".
|
||||
var llamaGuard3Categories = []string{"Violent Crimes", "Non-Violent Crimes", "Sex-Related Crimes",
|
||||
"Child Sexual Exploitation", "Defamation", "Specialized Advice", "Privacy", "Intellectual Property",
|
||||
"Indiscriminate Weapons", "Hate", "Suicide & Self-Harm", "Sexual Content", "Elections"}
|
||||
|
||||
// guardModel is the judge's catalog entry: CPU only, so it takes no card from
|
||||
// the models people use, on whichever node has room.
|
||||
func guardModel(source string) catalog.Model {
|
||||
@@ -457,7 +453,7 @@ func guardModel(source string) catalog.Model {
|
||||
Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.",
|
||||
Source: source,
|
||||
Role: catalog.RoleGuard,
|
||||
Guard: &catalog.GuardModel{Format: "llamaguard3", Categories: llamaGuard3Categories},
|
||||
Guard: &catalog.GuardModel{Format: "qwen3guard"},
|
||||
// Two: every judged request waits on it, and with fail: closed a
|
||||
// judge that is not there refuses everything.
|
||||
Scaling: catalog.Scaling{Min: 2, Max: 2},
|
||||
|
||||
@@ -7,12 +7,14 @@ import (
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
)
|
||||
|
||||
func TestTheInitAgentUnitIsTheShippedUnit(t *testing.T) {
|
||||
@@ -150,9 +152,17 @@ func TestTheInstalledConfigurationIsHardenedAndLoads(t *testing.T) {
|
||||
if (p.Judge != nil) != judge {
|
||||
t.Errorf("judge=%v: the policy's judge is %+v", judge, p.Judge)
|
||||
}
|
||||
if judge && (p.Judge.Model != guardModelName || p.Judge.Input != "block") {
|
||||
if judge && (!slices.Equal(p.Judge.Judges(), []string{guardModelName}) || p.Judge.Input != "block") {
|
||||
t.Errorf("judge %+v", p.Judge)
|
||||
}
|
||||
// A category the judge never names would never count.
|
||||
if judge {
|
||||
for _, c := range p.Judge.Categories {
|
||||
if !slices.Contains(guard.DefaultCategories("qwen3guard"), c) {
|
||||
t.Errorf("the baseline counts %q, which Qwen3Guard does not name", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
if cfg.Auth.MaxKeyTTL != 90*24*time.Hour || cfg.Auth.SignupOpen || cfg.DebugPprof || cfg.AdminRemote {
|
||||
t.Errorf("not hardened: ttl %v signup %v pprof %v remote %v",
|
||||
cfg.Auth.MaxKeyTTL, cfg.Auth.SignupOpen, cfg.DebugPprof, cfg.AdminRemote)
|
||||
@@ -161,13 +171,13 @@ func TestTheInstalledConfigurationIsHardenedAndLoads(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTheGuardModelIsAValidJudge(t *testing.T) {
|
||||
m := guardModel("http://10.0.0.5:8081/llama-guard-3-1b/guard/g.gguf")
|
||||
m := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf")
|
||||
c, err := catalog.New([]catalog.Model{m})
|
||||
if err != nil {
|
||||
t.Fatalf("not a valid catalog entry: %v", err)
|
||||
}
|
||||
got, _ := c.Resolve(guardModelName)
|
||||
if !got.IsGuard() || got.Guard.Format != "llamaguard3" || len(got.Guard.Categories) != 13 || got.Scaling.Min != 2 {
|
||||
if !got.IsGuard() || got.Guard.Format != "qwen3guard" || got.Scaling.Min != 2 {
|
||||
t.Errorf("%+v", got)
|
||||
}
|
||||
for _, p := range got.Profiles {
|
||||
|
||||
Reference in new issue
Block a user