Serve a judge model in one command: ctl model serve --guard-format
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
62038484d6
commit
c69e6f151c
2 files changed
+16
No files matched your search
@@ -73,6 +73,8 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
||||
f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable")
|
||||
f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models")
|
||||
f.IntVar(&req.Replicas, "replicas", 1, "how many replicas to run")
|
||||
f.StringVar(&req.GuardFormat, "guard-format", "",
|
||||
"serve it as the guardrails' judge (role: guard), read as llamaguard3, shieldgemma or json; hidden from clients")
|
||||
f.BoolVar(&onDemand, "on-demand", false, "run none until first asked for")
|
||||
f.IntVar(&req.CtxPool, "ctx-pool", 16384, "llama.cpp's --ctx-size, shared by its parallel slots")
|
||||
f.IntVar(&req.CPUCores, "cores", 0, "CPU cores a replica computes on; 0 takes half the smallest node it fits")
|
||||
|
||||
@@ -122,6 +122,9 @@ type Request struct {
|
||||
Description string `json:"description,omitempty"`
|
||||
// Replicas to run; 0 makes it on demand, loaded when first asked for.
|
||||
Replicas int `json:"replicas"`
|
||||
// GuardFormat makes it a judge model for the guardrails (role: guard):
|
||||
// llamaguard3, shieldgemma or json. Empty serves clients.
|
||||
GuardFormat string `json:"guard_format,omitempty"`
|
||||
// CtxPool is llama.cpp's --ctx-size; ContextLength what one request may
|
||||
// use. They default to 16384 and 4096, what four parallel slots divide.
|
||||
CtxPool int `json:"ctx_pool,omitempty"`
|
||||
@@ -224,6 +227,14 @@ func (s *Server) Start(req Request) (Job, error) {
|
||||
default:
|
||||
return Job{}, fmt.Errorf("the format is gguf or safetensors, not %q", req.Format)
|
||||
}
|
||||
switch req.GuardFormat {
|
||||
case "", "llamaguard3", "shieldgemma", "json":
|
||||
default:
|
||||
return Job{}, fmt.Errorf("the guard format is llamaguard3, shieldgemma or json, not %q", req.GuardFormat)
|
||||
}
|
||||
if req.GuardFormat != "" && req.Replicas == 0 {
|
||||
return Job{}, errors.New("a judge model must always run: give it --replicas 1 or more, not --on-demand")
|
||||
}
|
||||
if req.CtxPool <= 0 {
|
||||
req.CtxPool = 16384
|
||||
}
|
||||
@@ -556,6 +567,9 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
|
||||
Capabilities: catalog.Capabilities{Vision: req.Projector != ""},
|
||||
Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)},
|
||||
}
|
||||
if req.GuardFormat != "" {
|
||||
m.Role, m.Guard, m.RoutingProfile = catalog.RoleGuard, &catalog.GuardModel{Format: req.GuardFormat}, ""
|
||||
}
|
||||
s.step(j, "catalog", Running, "adding "+name)
|
||||
if err := s.deps.Catalog.Add(ctx, m, req.By); err != nil {
|
||||
s.fail(j, "catalog", err)
|
||||
|
||||
Reference in new issue
Block a user