Serve a judge model in one command: ctl model serve --guard-format
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
62038484d6
commit
c69e6f151c
2 files changed
+16
No files matched your search
@@ -122,6 +122,9 @@ type Request struct {
|
||||
Description string `json:"description,omitempty"`
|
||||
// Replicas to run; 0 makes it on demand, loaded when first asked for.
|
||||
Replicas int `json:"replicas"`
|
||||
// GuardFormat makes it a judge model for the guardrails (role: guard):
|
||||
// llamaguard3, shieldgemma or json. Empty serves clients.
|
||||
GuardFormat string `json:"guard_format,omitempty"`
|
||||
// CtxPool is llama.cpp's --ctx-size; ContextLength what one request may
|
||||
// use. They default to 16384 and 4096, what four parallel slots divide.
|
||||
CtxPool int `json:"ctx_pool,omitempty"`
|
||||
@@ -224,6 +227,14 @@ func (s *Server) Start(req Request) (Job, error) {
|
||||
default:
|
||||
return Job{}, fmt.Errorf("the format is gguf or safetensors, not %q", req.Format)
|
||||
}
|
||||
switch req.GuardFormat {
|
||||
case "", "llamaguard3", "shieldgemma", "json":
|
||||
default:
|
||||
return Job{}, fmt.Errorf("the guard format is llamaguard3, shieldgemma or json, not %q", req.GuardFormat)
|
||||
}
|
||||
if req.GuardFormat != "" && req.Replicas == 0 {
|
||||
return Job{}, errors.New("a judge model must always run: give it --replicas 1 or more, not --on-demand")
|
||||
}
|
||||
if req.CtxPool <= 0 {
|
||||
req.CtxPool = 16384
|
||||
}
|
||||
@@ -556,6 +567,9 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
|
||||
Capabilities: catalog.Capabilities{Vision: req.Projector != ""},
|
||||
Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)},
|
||||
}
|
||||
if req.GuardFormat != "" {
|
||||
m.Role, m.Guard, m.RoutingProfile = catalog.RoleGuard, &catalog.GuardModel{Format: req.GuardFormat}, ""
|
||||
}
|
||||
s.step(j, "catalog", Running, "adding "+name)
|
||||
if err := s.deps.Catalog.Add(ctx, m, req.By); err != nil {
|
||||
s.fail(j, "catalog", err)
|
||||
|
||||
Reference in new issue
Block a user