Serve a judge model in one command: ctl model serve --guard-format

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 15:27:35 +02:00
1 parent 62038484d6
commit c69e6f151c
2 files changed
+16

No files matched your search

+2
View File
@@ -73,6 +73,8 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable")
f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models")
f.IntVar(&req.Replicas, "replicas", 1, "how many replicas to run")
f.StringVar(&req.GuardFormat, "guard-format", "",
"serve it as the guardrails' judge (role: guard), read as llamaguard3, shieldgemma or json; hidden from clients")
f.BoolVar(&onDemand, "on-demand", false, "run none until first asked for")
f.IntVar(&req.CtxPool, "ctx-pool", 16384, "llama.cpp's --ctx-size, shared by its parallel slots")
f.IntVar(&req.CPUCores, "cores", 0, "CPU cores a replica computes on; 0 takes half the smallest node it fits")
+14
View File
@@ -122,6 +122,9 @@ type Request struct {
Description string `json:"description,omitempty"`
// Replicas to run; 0 makes it on demand, loaded when first asked for.
Replicas int `json:"replicas"`
// GuardFormat makes it a judge model for the guardrails (role: guard):
// llamaguard3, shieldgemma or json. Empty serves clients.
GuardFormat string `json:"guard_format,omitempty"`
// CtxPool is llama.cpp's --ctx-size; ContextLength what one request may
// use. They default to 16384 and 4096, what four parallel slots divide.
CtxPool int `json:"ctx_pool,omitempty"`
@@ -224,6 +227,14 @@ func (s *Server) Start(req Request) (Job, error) {
default:
return Job{}, fmt.Errorf("the format is gguf or safetensors, not %q", req.Format)
}
switch req.GuardFormat {
case "", "llamaguard3", "shieldgemma", "json":
default:
return Job{}, fmt.Errorf("the guard format is llamaguard3, shieldgemma or json, not %q", req.GuardFormat)
}
if req.GuardFormat != "" && req.Replicas == 0 {
return Job{}, errors.New("a judge model must always run: give it --replicas 1 or more, not --on-demand")
}
if req.CtxPool <= 0 {
req.CtxPool = 16384
}
@@ -556,6 +567,9 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
Capabilities: catalog.Capabilities{Vision: req.Projector != ""},
Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)},
}
if req.GuardFormat != "" {
m.Role, m.Guard, m.RoutingProfile = catalog.RoleGuard, &catalog.GuardModel{Format: req.GuardFormat}, ""
}
s.step(j, "catalog", Running, "adding "+name)
if err := s.deps.Catalog.Add(ctx, m, req.By); err != nil {
s.fail(j, "catalog", err)