The guardrails' judge has its own command, and its trouble is shown
llmgw ctl guard judge status lists each judge model the policies ask: in the catalog, as a judge, replicas ready, its last failure to answer, and what is wrong in words. enable serves the missing one through model serve - Llama Guard 3 1B on CPUs, role guard, under the name the policies give it - with --replace for a model of that name that is not a judge; the console has it under Do. GET /api/v1/guard/judges, and the summary's judges, put the problem on ctl status and the console's Overview. A Llama Guard file served from the mirror page is added as a judge, under the name the policies ask, rather than as a model clients use the policies never find. ctl says plainly when the API answers with a page. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
2a942409ea
commit
a8bb417618
24 files changed
+667
-13
No files matched your search
@@ -324,10 +324,14 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// A judge's weights are served as a judge, under the name the
|
||||
// policies ask: served as a model clients use, the policies
|
||||
// never find it and refuse every request they judge.
|
||||
judge, renamed := p.asJudge(&model)
|
||||
// The CPU while GPUs sit free is a choice to make on purpose:
|
||||
// made by default, it was a model running at a tenth of the
|
||||
// speed with nothing saying why.
|
||||
if model.Profiles[0].CPUOnly() && r.FormValue("cpu_ok") == "" {
|
||||
// speed with nothing saying why. A judge is meant for the CPU.
|
||||
if !judge && model.Profiles[0].CPUOnly() && r.FormValue("cpu_ok") == "" {
|
||||
if n := freeGPUs(p.fleet); n > 0 {
|
||||
return "", errors.New(l.N("serve.err.cpu_with_free_gpus", int64(n), model.Name))
|
||||
}
|
||||
@@ -346,8 +350,15 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
|
||||
}
|
||||
// Placed on the next pass, not now: saying it is serving would
|
||||
// be wrong for however long the weights take to reach a node.
|
||||
return l.T("serve.msg.added", model.Name,
|
||||
l.N("serve.replicas_n", int64(model.Scaling.Min)), describeServe(l, model)), nil
|
||||
msg := l.T("serve.msg.added", model.Name,
|
||||
l.N("serve.replicas_n", int64(model.Scaling.Min)), describeServe(l, model))
|
||||
switch {
|
||||
case renamed != "":
|
||||
msg += " " + l.T("serve.msg.judge_renamed", renamed, model.Name)
|
||||
case judge:
|
||||
msg += " " + l.T("serve.msg.judge", model.Name)
|
||||
}
|
||||
return msg, nil
|
||||
}))
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user