Ilker ManapandClaude Opus 5.5
d29f694aaf
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
102 lines
3.2 KiB
Go
102 lines
3.2 KiB
Go
package router
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
|
)
|
|
|
|
// /v1/moderations, answered by a judge model on the cluster: the same model
|
|
// the guardrails ask. OpenAI's shape — flagged, and the categories — with the
|
|
// judge's own categories named as it names them.
|
|
|
|
func (r *Router) handleModerations(w http.ResponseWriter, req *http.Request) {
|
|
var in struct {
|
|
Input json.RawMessage `json:"input"`
|
|
Model string `json:"model"`
|
|
}
|
|
if err := json.NewDecoder(io.LimitReader(req.Body, r.cfg.MaxBodyBytes)).Decode(&in); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", "the body is not JSON: "+err.Error())
|
|
return
|
|
}
|
|
var texts []string
|
|
var one string
|
|
if json.Unmarshal(in.Input, &one) == nil {
|
|
texts = []string{one}
|
|
} else if err := json.Unmarshal(in.Input, &texts); err != nil || len(texts) == 0 {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", "input is a string or a list of strings")
|
|
return
|
|
}
|
|
if len(texts) > 32 {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", "at most 32 inputs at once")
|
|
return
|
|
}
|
|
model := r.moderationModel(in.Model)
|
|
if model == "" {
|
|
writeErrorCode(w, http.StatusNotFound, "invalid_request_error", "model_not_found",
|
|
"no moderation model: the catalog has no model with role: guard")
|
|
return
|
|
}
|
|
format, names, _ := r.GuardFormat(model)
|
|
if len(names) == 0 {
|
|
names = guard.DefaultCategories(format)
|
|
}
|
|
results := make([]map[string]any, 0, len(texts))
|
|
for _, t := range texts {
|
|
unsafe, cats, err := guard.Moderate(req.Context(), r.Judge, model, format, names, t)
|
|
if err != nil {
|
|
writeError(w, http.StatusServiceUnavailable, "server_error", "the moderation model gave no verdict: "+err.Error())
|
|
return
|
|
}
|
|
categories, scores := map[string]bool{}, map[string]float64{}
|
|
for _, n := range names {
|
|
categories[moderationKey(n)], scores[moderationKey(n)] = false, 0
|
|
}
|
|
for _, c := range cats {
|
|
// The judge says both a code and the name it stands for; the
|
|
// name is what is shown, when there is one.
|
|
var n int
|
|
if _, err := fmt.Sscanf(c, "S%d", &n); err == nil && n >= 1 && n <= len(names) {
|
|
continue
|
|
}
|
|
categories[moderationKey(c)], scores[moderationKey(c)] = true, 1
|
|
}
|
|
results = append(results, map[string]any{"flagged": unsafe, "categories": categories, "category_scores": scores})
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{"id": newID("modr-"), "model": model, "results": results})
|
|
}
|
|
|
|
// moderationModel is the judge asked: the one named, if it is one, or the
|
|
// first in the catalog.
|
|
func (r *Router) moderationModel(asked string) string {
|
|
if asked != "" {
|
|
if canonical, e := r.resolve(asked); e.IsGuard() {
|
|
return canonical
|
|
}
|
|
}
|
|
var first string
|
|
for _, m := range r.modelMap() {
|
|
if m.IsGuard() && (first == "" || m.Name < first) {
|
|
first = m.Name
|
|
}
|
|
}
|
|
return first
|
|
}
|
|
|
|
// moderationKey is a category's name as a JSON key, as OpenAI writes them.
|
|
func moderationKey(name string) string {
|
|
return strings.Map(func(c rune) rune {
|
|
switch {
|
|
case c >= 'A' && c <= 'Z':
|
|
return c + 'a' - 'A'
|
|
case c >= 'a' && c <= 'z', c >= '0' && c <= '9', c == '/':
|
|
return c
|
|
}
|
|
return '-'
|
|
}, fmt.Sprint(name))
|
|
}
|