diff --git a/internal/cli/model_serve.go b/internal/cli/model_serve.go index a8e2105a..3ccaa7fc 100644 --- a/internal/cli/model_serve.go +++ b/internal/cli/model_serve.go @@ -73,6 +73,8 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command { f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable") f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models") f.IntVar(&req.Replicas, "replicas", 1, "how many replicas to run") + f.StringVar(&req.GuardFormat, "guard-format", "", + "serve it as the guardrails' judge (role: guard), read as llamaguard3, shieldgemma or json; hidden from clients") f.BoolVar(&onDemand, "on-demand", false, "run none until first asked for") f.IntVar(&req.CtxPool, "ctx-pool", 16384, "llama.cpp's --ctx-size, shared by its parallel slots") f.IntVar(&req.CPUCores, "cores", 0, "CPU cores a replica computes on; 0 takes half the smallest node it fits") diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index 49188e66..b00c9f91 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -122,6 +122,9 @@ type Request struct { Description string `json:"description,omitempty"` // Replicas to run; 0 makes it on demand, loaded when first asked for. Replicas int `json:"replicas"` + // GuardFormat makes it a judge model for the guardrails (role: guard): + // llamaguard3, shieldgemma or json. Empty serves clients. + GuardFormat string `json:"guard_format,omitempty"` // CtxPool is llama.cpp's --ctx-size; ContextLength what one request may // use. They default to 16384 and 4096, what four parallel slots divide. CtxPool int `json:"ctx_pool,omitempty"` @@ -224,6 +227,14 @@ func (s *Server) Start(req Request) (Job, error) { default: return Job{}, fmt.Errorf("the format is gguf or safetensors, not %q", req.Format) } + switch req.GuardFormat { + case "", "llamaguard3", "shieldgemma", "json": + default: + return Job{}, fmt.Errorf("the guard format is llamaguard3, shieldgemma or json, not %q", req.GuardFormat) + } + if req.GuardFormat != "" && req.Replicas == 0 { + return Job{}, errors.New("a judge model must always run: give it --replicas 1 or more, not --on-demand") + } if req.CtxPool <= 0 { req.CtxPool = 16384 } @@ -556,6 +567,9 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F Capabilities: catalog.Capabilities{Vision: req.Projector != ""}, Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)}, } + if req.GuardFormat != "" { + m.Role, m.Guard, m.RoutingProfile = catalog.RoleGuard, &catalog.GuardModel{Format: req.GuardFormat}, "" + } s.step(j, "catalog", Running, "adding "+name) if err := s.deps.Catalog.Add(ctx, m, req.By); err != nil { s.fail(j, "catalog", err)