Show guardrail refusals and stopped answers on the chat page, and the verdict in ctl request
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
47e847423d
commit
7e18b8ab53
3 files changed
+41
-2
No files matched your search
@@ -13,7 +13,7 @@ import (
|
||||
func newRequestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "request ID",
|
||||
Short: "What one request went through: replica, queue, first token, tokens",
|
||||
Short: "What one request went through: replica, queue, first token, tokens, guardrails",
|
||||
Long: "Every response carries its id in X-Request-Id, and a client may send its\n" +
|
||||
"own. Given one, this says which replica answered, how long admission held\n" +
|
||||
"it, how long the first token took and what it cost — the answer to \"why\n" +
|
||||
@@ -39,6 +39,12 @@ func newRequestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
fmt.Fprintf(out, "time %d ms total: %d queued, %d to the first token\n", v.LatencyMs, v.QueueMs, v.TTFTMs)
|
||||
fmt.Fprintf(out, "tokens %d in, %d out\n", v.PromptTokens, v.CompletionTokens)
|
||||
fmt.Fprintf(out, "who key %s, project %s%s\n", v.KeyID, v.ProjectID, prefixed(", user ", v.UserID))
|
||||
if v.GuardAction != "" {
|
||||
fmt.Fprintf(out, "guard %s by policy %s (%s), %d ms\n", v.GuardAction, v.GuardPolicy,
|
||||
or(v.GuardCategory, "-"), v.GuardMs)
|
||||
} else if v.GuardMs > 0 {
|
||||
fmt.Fprintf(out, "guard allowed, %d ms\n", v.GuardMs)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1393,6 +1393,11 @@ type RequestView struct {
|
||||
PromptTokens int64 `json:"prompt_tokens"`
|
||||
CompletionTokens int64 `json:"completion_tokens"`
|
||||
CostMicros int64 `json:"cost_micros"`
|
||||
// What the guardrails did, by name.
|
||||
GuardAction string `json:"guard_action,omitempty"`
|
||||
GuardPolicy string `json:"guard_policy,omitempty"`
|
||||
GuardCategory string `json:"guard_category,omitempty"`
|
||||
GuardMs int64 `json:"guard_ms,omitempty"`
|
||||
}
|
||||
|
||||
func (a *API) request(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -1415,6 +1420,7 @@ func (a *API) request(w http.ResponseWriter, r *http.Request) {
|
||||
Model: u.Model, ServedModel: u.ServedModel, Replica: u.Replica, Status: u.Status,
|
||||
LatencyMs: u.LatencyMs, QueueMs: u.QueueMs, TTFTMs: u.TTFTMs,
|
||||
PromptTokens: u.PromptTokens, CompletionTokens: u.CompletionTokens, CostMicros: u.CostMicros,
|
||||
GuardAction: u.GuardAction, GuardPolicy: u.GuardPolicy, GuardCategory: u.GuardCategory, GuardMs: u.GuardMs,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -75,6 +75,11 @@ access to it.</p>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// The policy a guardrail acted by, when the response says.
|
||||
function policySuffix(resp) {
|
||||
const p = resp.headers.get("x-llmgw-guard-policy");
|
||||
return p ? " " + p : "";
|
||||
}
|
||||
// Vanilla, inline, no build step — the same reason the charts are hand-drawn
|
||||
// SVG. It streams because a chat that hangs for ten seconds and then dumps a
|
||||
// paragraph feels broken, and on CPU inference ten seconds is normal.
|
||||
@@ -365,7 +370,19 @@ access to it.</p>
|
||||
resp = await post();
|
||||
}
|
||||
if (!resp.ok) {
|
||||
out.textContent = "(" + resp.status + ") " + (await resp.text()).slice(0, 400);
|
||||
const text = await resp.text();
|
||||
let shown = "(" + resp.status + ") " + text.slice(0, 400);
|
||||
// Refused by a guardrail: said as what it is, not as a failure.
|
||||
try {
|
||||
const e = JSON.parse(text).error;
|
||||
if (e && e.code === "content_policy_violation") {
|
||||
const policy = resp.headers.get("x-llmgw-guard-policy");
|
||||
shown = "Refused by the gateway's policy" + (policy ? " " + policy : "") + ": " + e.message;
|
||||
} else if (e && e.code === "no_store") {
|
||||
shown = e.message;
|
||||
}
|
||||
} catch (_) { /* not JSON: shown as it came */ }
|
||||
out.textContent = shown;
|
||||
out.classList.add("failed");
|
||||
messages.pop();
|
||||
return;
|
||||
@@ -424,6 +441,12 @@ access to it.</p>
|
||||
dl.onclick = () => downloadCode(reply, dl);
|
||||
out.after(dl);
|
||||
}
|
||||
if (finish === "content_filter") {
|
||||
const note = document.createElement("p");
|
||||
note.className = "note truncated";
|
||||
note.textContent = "Stopped by the gateway's policy" + policySuffix(resp) + ".";
|
||||
out.after(note);
|
||||
}
|
||||
if (finish === "length") {
|
||||
// Appended as a sibling, never into the reply: out holds model text
|
||||
// placed with textContent, and this is ours.
|
||||
@@ -438,6 +461,10 @@ access to it.</p>
|
||||
await renderReply(out, reply);
|
||||
renderMath(out);
|
||||
}
|
||||
else if (finish === "content_filter") {
|
||||
out.textContent = "(stopped by the gateway's policy" + policySuffix(resp) + " before anything was shown)";
|
||||
out.classList.add("failed"); messages.pop();
|
||||
}
|
||||
else if (stopped) { out.textContent = "(stopped before anything came back)"; out.classList.add("failed"); messages.pop(); }
|
||||
else { out.textContent = "(no content came back)"; out.classList.add("failed"); }
|
||||
} catch (err) {
|
||||
|
||||
Reference in new issue
Block a user