Compare commits

...
Author SHA1 Message Date
Ilker ManapandClaude Opus 5.5 abaf086fd5 A standby's portal is the leader's; the changelog as the release notes read it
CI / test (push) Successful in 2m42s
Release / notes (push) Successful in 3s
Release / build (push) Successful in 3m20s
A proxy left pointing at a standby showed an empty cluster: the portal and
the admin pages read the standby's own memory. A standby now passes its
client port to the leader's, all but /health, /ready and /leader, which
must describe the machine a load balancer is choosing; and its admin port
passes the dashboard as well as the API. The leader trusts the control
planes' X-Forwarded-For, so clients keep their own address.

The changelog's candidate section is "Unreleased" again: release notes
for an rc read that section, and v0.11.3-rc.1 got none.

The e2e wait for two endpoints is 30s: under the race detector on a busy
runner, 15s was missed once.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-05 06:34:57 +02:00
Ilker ManapandClaude Opus 5.5 c5b34978f5 Guardrails in the console
CI / test (push) Successful in 2m52s
The console never mentioned them. A Guardrails view shows each policy with
its mode, fail behaviour and checks, whether its judge models have replicas
ready, where policies are attached (attach and detach from it), and what
they did in a window. The admin API gains GET /api/v1/guard/events, and GET
/api/v1/guard says what each policy does.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-05 03:36:17 +02:00
Ilker ManapandClaude Opus 5.5 77fb48ba32 The probe reads the floating address from its file each time
CI / test (push) Successful in 2m49s
The change to read it each time left the constructor storing the address
itself, so the file read each time was one named 10.0.0.161.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-05 02:56:23 +02:00
17 changed files with 812 additions and 12 deletions

No files matched your search

+12 -1
View File
@@ -21,7 +21,7 @@ is only released once its **exit criterion** is met, not when its code lands.
Until v1.0.0 the minor version is the breaking one: configuration formats and Go
APIs change between minors without a deprecation period.
## v0.11.3-rc.1 — 2026-10-05
## Unreleased
Two control planes, run in anger on the reference cluster.
@@ -42,6 +42,16 @@ Two control planes, run in anger on the reference cluster.
come back stops it. The other control plane is upgraded first, over ssh.
- `server add` says when the mirror's `--api-allow` would refuse the new
control plane.
- **Guardrails in the console**: a view of the policies and what each checks,
whether their judge models are serving, where they are attached (attach
and detach from it), and what they did in a window. `GET
/api/v1/guard/events` answers the same, and `GET /api/v1/guard` now says
what each policy does.
- **A standby's portal is the leader's.** Its client port passes everything
but `/health`, `/ready` and `/leader` to the leader, and its admin port the
dashboard as well as the API; a proxy left pointing at a standby showed an
empty cluster. The leader trusts the control planes' `X-Forwarded-For`.
### Fixed
@@ -53,6 +63,7 @@ Two control planes, run in anger on the reference cluster.
- An agent's reconnect backoff carried over from one outage to the next, so
each failover made the next one slower; it starts again after a session that
worked.
- The `CONTROL` line named no floating address.
## v0.11.2 — 2026-10-05
+153
View File
@@ -854,6 +854,43 @@
}
}
},
"/api/v1/guard/events": {
"get": {
"operationId": "getGuardEvents",
"summary": "What the guardrails did: totals and the newest requests",
"tags": [
"guard"
],
"parameters": [
{
"name": "since",
"in": "query",
"description": "How far back: 90m, 24h, 7d; 24h when absent.",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/GuardEventsView"
}
}
}
},
"401": {
"$ref": "#/components/responses/Unauthorized"
},
"default": {
"$ref": "#/components/responses/Error"
}
}
}
},
"/api/v1/jobs": {
"get": {
"operationId": "getJobs",
@@ -3849,6 +3886,115 @@
}
}
},
"GuardCountView": {
"type": "object",
"description": "GuardCountView is how many requests one policy did one thing to.",
"properties": {
"project": {
"type": "string"
},
"action": {
"type": "string"
},
"policy": {
"type": "string"
},
"category": {
"type": "string"
},
"count": {
"type": "integer",
"format": "int64"
}
}
},
"GuardEventView": {
"type": "object",
"description": "GuardEventView is one request the guardrails did something to.",
"properties": {
"at": {
"type": "string",
"format": "date-time"
},
"request_id": {
"type": "string"
},
"project": {
"type": "string"
},
"model": {
"type": "string"
},
"status": {
"type": "integer"
},
"action": {
"type": "string"
},
"policy": {
"type": "string"
},
"category": {
"type": "string"
},
"guard_ms": {
"type": "integer",
"format": "int64"
}
}
},
"GuardEventsView": {
"type": "object",
"description": "GuardEventsView is what the guardrails did since a moment: totals, and the newest requests.",
"properties": {
"since": {
"type": "string",
"format": "date-time"
},
"counts": {
"type": "array",
"items": {
"$ref": "#/components/schemas/GuardCountView"
}
},
"events": {
"type": "array",
"items": {
"$ref": "#/components/schemas/GuardEventView"
}
}
}
},
"GuardPolicyView": {
"type": "object",
"description": "GuardPolicyView is what one policy does, in brief.",
"properties": {
"name": {
"type": "string"
},
"mode": {
"type": "string",
"description": "Mode is enforce, shadow or off; Fail is open or closed."
},
"fail": {
"type": "string"
},
"no_store": {
"type": "boolean"
},
"judge": {
"type": "string",
"description": "Judge is the judge model, when one reads the text."
},
"checks": {
"type": "array",
"items": {
"type": "string"
},
"description": "Checks are what it looks for, one line each: \"input: pii email → redact\"."
}
}
},
"GuardView": {
"type": "object",
"description": "GuardView is the library and where its policies are attached.",
@@ -3870,6 +4016,13 @@
"items": {
"$ref": "#/components/schemas/GuardAttachmentView"
}
},
"details": {
"type": "array",
"items": {
"$ref": "#/components/schemas/GuardPolicyView"
},
"description": "Details says what each policy does, in the order of Policies."
}
}
},
+11
View File
@@ -209,3 +209,14 @@ Each request's usage row has the action, policy, category and the time the
checks took. The access log has the action, stage and policy. Neither has
the text, a matched value, or the placeholder map: the map exists only in
the gateway's memory while the request is served.
## In the console
`sudo llmgw` has a **Guardrails** view: each policy with its mode, what it
does when it cannot judge, and what it checks; whether each judge model it
leans on has replicas ready — a policy that fails closed refuses everything
while its judge has none; where policies are attached, with A to attach one
and Enter on a row to detach it; and what they did in the last day (W for an
hour or a week), totals first and the newest requests after, each with the
request id `llmgw ctl request` reads. The same is at `GET /api/v1/guard` and
`GET /api/v1/guard/events?since=24h`, and on the portal's Guardrails page.
+6 -2
View File
@@ -1940,8 +1940,12 @@ Without it, put a load balancer in front that checks `/ready` — not
`CONTROL` line: which leads, which stand by, one that does not answer, and
whether the floating address is on the leader. A standby with nothing to
take over from is the thing to see before it is needed.
- **Either machine's console.** A standby's admin API passes everything under
`/api/` to the leader, over the port control planes already reach each
- **Either machine, the leader's cluster.** A standby passes its client port
— the portal, its admin pages, the API — to the leader's, keeping only
`/health`, `/ready` and `/leader` to describe itself, so a proxy pointed at
the wrong machine still shows the cluster; the leader trusts a control
plane's `X-Forwarded-For`, so clients keep their own address. Its admin
API passes everything but `/debug/` and `/metrics` to the leader, over the port control planes already reach each
other on and with the cluster's certificates both ways, so `sudo llmgw` and
`llmgw ctl` show the same cluster on both — the leader's — and a change made
on the standby is made by the leader. A node's certificate is refused
+1 -1
View File
@@ -46,7 +46,7 @@ func newControlPlaneProbe(st *store.DB, listen string) *controlPlaneProbe {
self, _ := os.Hostname()
return &controlPlaneProbe{st: st, self: self,
leaderURL: func(host string) string { return "http://" + net.JoinHostPort(host, port) + "/leader" },
floating: floatingAddress(floatingPath),
floating: floatingPath,
client: &http.Client{Timeout: time.Second}}
}
+8
View File
@@ -132,3 +132,11 @@ func TestTheFloatingAddressIsReadFromWhatServerAddWrote(t *testing.T) {
t.Errorf("no file read as %q", got)
}
}
func TestTheProbeReadsTheFloatingAddressFromItsFile(t *testing.T) {
// The file, not what it held when the process started: server add
// writes it after restarting the control plane it adds.
if p := newControlPlaneProbe(nil, ":8080"); p.floating != floatingPath {
t.Errorf("the probe reads %q, not %s", p.floating, floatingPath)
}
}
+91 -2
View File
@@ -7,6 +7,7 @@ import (
"errors"
"fmt"
"log/slog"
"net"
"net/http"
"net/http/httputil"
"net/url"
@@ -16,6 +17,7 @@ import (
"golang.org/x/net/http2"
"gitea.mynodes.xyz/ilker/llmgw/internal/clientip"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
@@ -53,7 +55,7 @@ type leaderForward struct {
// process is not it.
func (f *leaderForward) standby(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !strings.HasPrefix(r.URL.Path, "/api/") || f.lead.IsLeader() {
if adminStaysHere(r.URL.Path) || f.lead.IsLeader() {
next.ServeHTTP(w, r)
return
}
@@ -90,6 +92,89 @@ func (f *leaderForward) standby(next http.Handler) http.Handler {
})
}
// adminStaysHere are the admin paths about this process rather than the
// cluster: its profile, its own metrics, its own health. Never forwarded, and
// never answered for another — pprof dumps the heap, keys and all.
func adminStaysHere(path string) bool {
return strings.HasPrefix(path, "/debug/") || path == "/metrics" || path == "/health"
}
// publicStaysHere are the client port's paths that must describe this
// process: a load balancer and keepalived ask them which machine to send
// traffic to, and a standby saying the leader is ready would draw it.
func publicStaysHere(path string) bool {
return path == "/health" || path == "/ready" || path == "/leader" || path == "/metrics"
}
// public wraps the client-facing handler: on a standby, everything but its
// own health goes to the leader's client port — the portal, its admin pages
// and the API, which on a standby would show an empty cluster and serve
// nothing. Plain HTTP, as the port is; the leader trusts a control plane's
// X-Forwarded-For, so a client is still known by its own address.
func (f *leaderForward) public(next http.Handler, port string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Once only: a request another control plane forwarded is answered
// here, leading or not, rather than sent on while the lease moves.
if publicStaysHere(r.URL.Path) || f.lead.IsLeader() || r.Header.Get(hdrForwardedBy) != "" {
next.ServeHTTP(w, r)
return
}
target, _, err := f.leader(r.Context())
if err != nil || target.Hostname() == "" {
next.ServeHTTP(w, r)
return
}
host := net.JoinHostPort(target.Hostname(), port)
p := &httputil.ReverseProxy{
Rewrite: func(pr *httputil.ProxyRequest) {
pr.Out.URL.Scheme, pr.Out.URL.Host = "http", host
// The name the client used, for the portal's links and cookies.
pr.Out.Host = pr.In.Host
peer, _, _ := net.SplitHostPort(pr.In.RemoteAddr)
xff := pr.In.Header.Get("X-Forwarded-For")
if xff != "" {
xff += ", "
}
pr.Out.Header.Set("X-Forwarded-For", xff+peer)
for _, h := range []string{"X-Forwarded-Proto", "X-Forwarded-Host"} {
if v := pr.In.Header.Get(h); v != "" {
pr.Out.Header.Set(h, v)
}
}
pr.Out.Header.Set(hdrForwardedBy, f.self)
},
FlushInterval: -1,
ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) {
writeForwardError(w, http.StatusBadGateway, "this control plane is standing by, and the leader did not answer: "+err.Error())
},
}
p.ServeHTTP(w, r)
})
}
// trustControlPlanes keeps the client port trusting the other control
// planes' X-Forwarded-For, besides router.trusted_proxies: a standby passes
// requests on, and the client is the address it was given, not the standby.
// Read again every half minute, for one added since.
func (f *leaderForward) trustControlPlanes(ctx context.Context, configured []string) {
for {
list := slices.Clone(configured)
if cps, err := f.st.ControlPlanes(ctx); err == nil {
for _, cp := range cps {
if u, err := url.Parse(cp.URL); err == nil && u.Hostname() != "" {
list = append(list, u.Hostname())
}
}
}
clientip.SetTrusted(list)
select {
case <-ctx.Done():
return
case <-time.After(30 * time.Second):
}
}
}
// leader is where the lease's holder is reached, and its name.
func (f *leaderForward) leader(ctx context.Context) (*url.URL, string, error) {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
@@ -99,6 +184,10 @@ func (f *leaderForward) leader(ctx context.Context) (*url.URL, string, error) {
return nil, "", errors.New("no control plane holds the lease right now")
}
name := control.HolderName(lease.Owner)
if name == f.self {
// This machine's previous process, whose lease has not run out.
return nil, name, errors.New("the lease is still held by this machine's previous run; it runs out within seconds")
}
cps, err := f.st.ControlPlanes(ctx)
if err != nil {
return nil, name, err
@@ -117,7 +206,7 @@ func (f *leaderForward) leader(ctx context.Context) (*url.URL, string, error) {
func (f *leaderForward) leaderSide(admin http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
path := strings.TrimPrefix(r.URL.Path, forwardPrefix)
if !strings.HasPrefix(path, "/api/") {
if adminStaysHere(path) {
http.NotFound(w, r)
return
}
+59 -3
View File
@@ -4,6 +4,7 @@ import (
"encoding/json"
"io"
"log/slog"
"net"
"net/http"
"net/http/httptest"
"path/filepath"
@@ -140,9 +141,15 @@ func TestAStandbysAdminAPIIsTheLeaders(t *testing.T) {
if got["actor"] != "root@ctl-b" {
t.Errorf("who asked was lost: %q", got["actor"])
}
// What is not the API stays on the machine: the dashboard, pprof.
if _, got := c.ask(t, "/debug/pprof/"); got["path"] != "local" {
t.Errorf("a non-API path was forwarded: %v", got)
// The dashboard's pages are the leader's too.
if _, got := c.ask(t, "/nodes/cuda01"); got["path"] != "/nodes/cuda01" {
t.Errorf("a dashboard page was answered here: %v", got)
}
// What is about this process stays on it: pprof, its metrics.
for _, p := range []string{"/debug/pprof/", "/metrics"} {
if _, got := c.ask(t, p); got["path"] != "local" {
t.Errorf("%s was forwarded: %v", p, got)
}
}
}
@@ -186,3 +193,52 @@ func TestAStandbyWithNoLeaderSaysSo(t *testing.T) {
t.Errorf("%d %v", resp.StatusCode, got)
}
}
func TestAStandbysPortalIsTheLeaders(t *testing.T) {
c := newForwardCluster(t)
// The leader's client port, on the address the store lists for it.
var got http.Header
var gotHost string
leader := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
got, gotHost = r.Header.Clone(), r.Host
_, _ = w.Write([]byte("leader " + r.URL.Path))
}))
defer leader.Close()
_, port, _ := net.SplitHostPort(strings.TrimPrefix(leader.URL, "http://"))
local := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("standby " + r.URL.Path)) })
srv := httptest.NewServer(c.standby.public(local, port))
defer srv.Close()
get := func(path string, hdr map[string]string) string {
req, _ := http.NewRequestWithContext(t.Context(), http.MethodGet, srv.URL+path, nil)
req.Host = "llmgw.example"
for k, v := range hdr {
req.Header.Set(k, v)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
b, _ := io.ReadAll(resp.Body)
return string(b)
}
if body := get("/admin/replicas", map[string]string{"X-Forwarded-For": "203.0.113.7", "X-Forwarded-Proto": "https"}); body != "leader /admin/replicas" {
t.Fatalf("the portal was answered by %q", body)
}
if gotHost != "llmgw.example" || got.Get("X-Forwarded-Proto") != "https" ||
!strings.HasPrefix(got.Get("X-Forwarded-For"), "203.0.113.7, ") || got.Get(hdrForwardedBy) != "ctl-b" {
t.Errorf("host %q, headers %v", gotHost, got)
}
// What a load balancer asks is about this machine.
for _, p := range []string{"/ready", "/health", "/leader"} {
if body := get(p, nil); body != "standby "+p {
t.Errorf("%s answered by %q", p, body)
}
}
// Forwarded once only.
if body := get("/v1/models", map[string]string{hdrForwardedBy: "ctl-c"}); body != "standby /v1/models" {
t.Errorf("a forwarded request was forwarded again: %q", body)
}
}
+3
View File
@@ -513,6 +513,9 @@ func newServerCmd() *cobra.Command {
served := adminHandler
if forward != nil {
served = forward.standby(adminHandler)
_, port, _ := net.SplitHostPort(cfg.Listen)
apiSrv.Handler = forward.public(apiSrv.Handler, port)
go forward.trustControlPlanes(ctx, cfg.Router.TrustedProxies)
}
adminSrv := withLimits(&http.Server{
Handler: router.AdminGuard(router.AdminAuth(adminToken, cfg.DebugPprof, served)),
+24 -1
View File
@@ -43,6 +43,7 @@ const (
cReplicas
cUsage
cQuotas
cGuard
cFederation
cMirror
cJobs
@@ -64,7 +65,7 @@ const (
cCount
)
var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Federation", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Audit", "Manual",
var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Guardrails", "Federation", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Audit", "Manual",
"server.yaml", "Add a node", "Add a control plane", "Replicate the store", "Serve a model", "Upgrade the cluster",
"Restart all replicas", "Back up now", "Leave"}
@@ -169,6 +170,7 @@ type consoleModel struct {
quotas []adminapi.QuotaView
quotasErr error
guard guardState
st *storeHolder
people peopleData
@@ -292,6 +294,9 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
if m.menu == cQuotas {
cmds = append(cmds, m.fetchQuotas())
}
if m.menu == cGuard {
cmds = append(cmds, m.fetchGuard())
}
if m.menu == cMirror {
cmds = append(cmds, m.fetchMirror())
}
@@ -350,6 +355,9 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
m.people = peopleData(msg)
case consoleQuotasMsg:
m.quotas, m.quotasErr = msg.rows, msg.err
case consoleGuardMsg:
window := m.guard.window
m.guard = guardState{view: msg.view, events: msg.events, err: msg.err, window: window}
case evalSuitesMsg:
return m.onEvalSuites(msg)
case evalPollMsg:
@@ -501,6 +509,8 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
m.menuD = m.replicaMenu(m.data.replicas[m.row])
case cQuotas:
m.menuD = m.quotaMenu(m.quotas[m.row])
case cGuard:
m.menuD = m.guardAttachmentMenu(m.guard.view.Attachments[m.row])
case cMirror:
m.menuD = m.mirrorMenu(m.mirrorGroups()[m.row])
case cJobs:
@@ -556,6 +566,8 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
switch m.menu {
case cQuotas:
m.quotaForm(adminapi.QuotaView{})
case cGuard:
m.guardAttachForm()
case cModels:
m.addModelForm()
case cMirror:
@@ -567,6 +579,11 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
case cOrgs:
m.createOrgForm()
}
case "w":
if m.menu == cGuard {
m.guard.window = (m.guard.window + 1) % len(guardWindows)
return m, m.fetchGuard()
}
case "i":
if m.menu == cMirror {
m.mirrorImportOllamaForm()
@@ -604,6 +621,8 @@ func (m consoleModel) onEnter() tea.Cmd {
return m.fetchUsage()
case cQuotas:
return m.fetchQuotas()
case cGuard:
return m.fetchGuard()
case cMirror:
return m.fetchMirror()
case cMonitor:
@@ -939,6 +958,8 @@ func (m consoleModel) rowCount() int {
return len(m.usage)
case cQuotas:
return len(m.quotas)
case cGuard:
return len(m.guard.view.Attachments)
case cMirror:
return len(m.mirrorGroups())
case cJobs:
@@ -1166,6 +1187,8 @@ func (m consoleModel) panel() (title, body, keys string) {
return "Usage", head + table([]string{"WHO", "REQUESTS", "TOKENS", "ERRORS", "SPENT"}, rows, m.row, pw, m.inPanel), keys
case cQuotas:
return m.quotasPanel(pw)
case cGuard:
return m.guardPanel(pw)
case cAccounts, cKeys, cOrgs:
return m.peoplePanel(pw)
case cManual:
+189
View File
@@ -0,0 +1,189 @@
package cli
import (
"context"
"fmt"
"net/http"
"strings"
"time"
tea "github.com/charmbracelet/bubbletea"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)
// The guardrails, as the console shows them: the policies server.yaml
// defines and what each checks, where they are attached, whether the judge
// models they lean on are serving, and what they did in a window.
// guardWindows are the windows W steps through.
var guardWindows = []string{"24h", "7d", "1h"}
type guardState struct {
view adminapi.GuardView
events adminapi.GuardEventsView
err error
// window is an index into guardWindows.
window int
}
type consoleGuardMsg struct {
view adminapi.GuardView
events adminapi.GuardEventsView
err error
}
func (m consoleModel) fetchGuard() tea.Cmd {
c, since := m.client, guardWindows[m.guard.window]
return func() tea.Msg {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
var msg consoleGuardMsg
if msg.err = c.get(ctx, "/api/v1/guard", &msg.view); msg.err != nil {
return msg
}
// What they did is a second question; a store without the record
// still shows the policies.
_ = c.get(ctx, "/api/v1/guard/events?since="+since, &msg.events)
return msg
}
}
func (m consoleModel) guardPanel(pw int) (string, string, string) {
keys := "→ or Enter open · ↑↓ choose · Q leave"
if m.inPanel {
keys = "↑↓ attachment · Enter actions · A attach · W window · ← back · Q leave"
}
g := m.guard
var b strings.Builder
b.WriteString("\n " + sDim.Render("What requests and answers are checked for, where, and what that did. A to attach a policy.") + "\n\n")
if g.err != nil {
if strings.Contains(g.err.Error(), "501") || strings.Contains(g.err.Error(), "no guard policies") {
b.WriteString(" " + sDim.Render("No policies: define them under guard: in server.yaml (Settings), then restart."))
} else {
b.WriteString(" " + sBad.Render(g.err.Error()))
}
return "Guardrails", b.String(), keys
}
if len(g.view.Details) == 0 && len(g.view.Policies) == 0 {
b.WriteString(" " + sDim.Render("asking the control plane…"))
return "Guardrails", b.String(), keys
}
def := "none"
if len(g.view.Default) > 0 {
def = strings.Join(g.view.Default, ", ")
}
b.WriteString(" " + sTitle.Render("POLICIES") + " " + sDim.Render("on every request: "+def) + "\n")
for _, p := range g.view.Details {
line := fmt.Sprintf(" %-14s %s · fail %s", p.Name, p.Mode, p.Fail)
if p.NoStore {
line += " · keeps nothing"
}
if p.Judge != "" {
line += " · judge " + p.Judge + " " + m.judgeState(p.Judge)
}
b.WriteString(line + "\n")
for _, c := range p.Checks {
b.WriteString(" " + strings.Repeat(" ", 15) + sDim.Render(c) + "\n")
}
}
b.WriteString("\n " + sTitle.Render("ATTACHED") + " " + sDim.Render("besides the default, to part of the tenancy tree, a person or a key") + "\n")
if len(g.view.Attachments) == 0 {
b.WriteString(" " + sDim.Render("Nowhere. A to attach one.") + "\n")
} else {
var rows [][]string
for _, a := range g.view.Attachments {
who := a.Name
if who == "" {
who = a.ScopeID
}
rows = append(rows, []string{a.Scope, who, a.Policy})
}
b.WriteString(table([]string{"SCOPE", "WHO", "POLICY"}, rows, m.row, pw, m.inPanel) + "\n")
}
window := guardWindows[g.window]
b.WriteString("\n " + sTitle.Render("IN THE LAST "+strings.ToUpper(window)) + " " + sDim.Render("W for another window") + "\n")
if len(g.events.Counts) == 0 {
b.WriteString(" " + sDim.Render("Nothing flagged, redacted or blocked.") + "\n")
} else {
var rows [][]string
for i, c := range g.events.Counts {
if i == 10 {
break
}
rows = append(rows, []string{c.Action, c.Policy, dash(c.Category), dash(c.Project), fmt.Sprint(c.Count)})
}
b.WriteString(table([]string{"ACTION", "POLICY", "CATEGORY", "PROJECT", "REQUESTS"}, rows, -1, pw, false) + "\n")
}
if len(g.events.Events) > 0 {
b.WriteString("\n " + sTitle.Render("NEWEST") + "\n")
var rows [][]string
for i, e := range g.events.Events {
if i == 10 {
break
}
rows = append(rows, []string{e.At.Local().Format("01-02 15:04:05"), e.Action, e.Policy, dash(e.Category),
e.Model, fmt.Sprint(e.Status), fmt.Sprintf("%dms", e.GuardMs), e.RequestID})
}
b.WriteString(table([]string{"AT", "ACTION", "POLICY", "CATEGORY", "MODEL", "STATUS", "GUARD", "REQUEST"}, rows, -1, pw, false))
}
return "Guardrails", b.String(), keys
}
// judgeState is how many of a judge model's replicas are ready: a policy
// that fails closed refuses everything while it has none.
func (m consoleModel) judgeState(model string) string {
for _, md := range m.data.models {
if md.Name == model {
s := fmt.Sprintf("(%d/%d ready)", md.Ready, md.Desired)
if md.Ready == 0 {
return sBad.Render(s)
}
return sOK.Render(s)
}
}
return sBad.Render("(not in the catalog)")
}
// guardAttachForm attaches a policy.
func (m *consoleModel) guardAttachForm() {
if len(m.guard.view.Policies) == 0 {
return
}
m.openForm(&formDialog{title: "Attach a guardrail policy",
note: "Requests from there are checked by it as well as by the default.",
fields: []formField{
choiceF("to", []string{"project", "team", "org", "user", "key"}, "project"),
textF("which", "", "ORG/TEAM/PROJECT, ORG/TEAM, ORG, or the ID"),
choiceF("policy", m.guard.view.Policies, m.guard.view.Policies[0]),
},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
if strings.TrimSpace(v[1]) == "" {
return nil, fmt.Errorf("which %s?", v[0])
}
req := adminapi.GuardAttachmentRequest{Target: v[0] + ":" + strings.TrimSpace(v[1]), Policy: v[2]}
return m.call("Attach "+v[2], http.MethodPost, "/api/v1/guard/attachments", req, nil), nil
}})
}
// guardAttachmentMenu is what can be done to one attachment.
func (m consoleModel) guardAttachmentMenu(a adminapi.GuardAttachmentView) *menuDialog {
who := a.Name
if who == "" {
who = a.ScopeID
}
return &menuDialog{title: a.Policy + " on " + a.Scope + " " + who, items: []menuItem{
{"Detach…", func(m *consoleModel) tea.Cmd {
m.confirmD = &confirmDialog{title: "Detach " + a.Policy,
text: "Requests from " + who + " are then checked by the default policies alone.",
yes: func(m *consoleModel) tea.Cmd {
req := adminapi.GuardAttachmentRequest{Target: a.Scope + ":" + a.ScopeID, Policy: a.Policy}
return m.call("Detach "+a.Policy, http.MethodDelete, "/api/v1/guard/attachments", req, nil)
}}
return nil
}},
}}
}
+26
View File
@@ -218,3 +218,29 @@ func TestTheConsoleOnAStandbySaysWhoseClusterItShows(t *testing.T) {
t.Error("the leader's console says it is standing by")
}
}
func TestTheConsoleShowsTheGuardrails(t *testing.T) {
m := consoleModel{menu: cGuard, loaded: true, width: 140, height: 60}
m.data.models = []adminapi.ModelView{{Name: "llama-guard-3-1b", Ready: 2, Desired: 2}}
m.guard.view = adminapi.GuardView{
Policies: []string{"baseline"}, Default: []string{"baseline"},
Details: []adminapi.GuardPolicyView{{Name: "baseline", Mode: "enforce", Fail: "closed", Judge: "llama-guard-3-1b",
Checks: []string{"input: pii email → redact"}}},
Attachments: []adminapi.GuardAttachmentView{{Name: "acme/ml"}},
}
m.guard.view.Attachments[0].Scope, m.guard.view.Attachments[0].Policy = "team", "baseline"
m.guard.events = adminapi.GuardEventsView{Counts: []adminapi.GuardCountView{{Action: "block", Policy: "baseline", Category: "S1", Count: 7}}}
v := m.View()
for _, want := range []string{"Guardrails", "baseline", "fail closed", "judge llama-guard-3-1b (2/2 ready)",
"input: pii email → redact", "acme/ml", "IN THE LAST 24H", "S1"} {
if !strings.Contains(v, want) {
t.Errorf("%q missing:\n%s", want, v)
}
}
// W moves the window, and asks again.
m.inPanel = true
mm, cmd := m.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("w")})
if c := mm.(consoleModel); guardWindows[c.guard.window] != "7d" || cmd == nil {
t.Errorf("window %q", guardWindows[c.guard.window])
}
}
+171
View File
@@ -4,11 +4,14 @@ import (
"context"
"encoding/json"
"fmt"
"maps"
"net/http"
"slices"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// GuardStore is where policy attachments are kept.
@@ -34,6 +37,114 @@ type GuardView struct {
Policies []string `json:"policies"`
Default []string `json:"default"`
Attachments []GuardAttachmentView `json:"attachments"`
// Details says what each policy does, in the order of Policies.
Details []GuardPolicyView `json:"details"`
}
// GuardPolicyView is what one policy does, in brief.
type GuardPolicyView struct {
Name string `json:"name"`
// Mode is enforce, shadow or off; Fail is open or closed.
Mode string `json:"mode"`
Fail string `json:"fail"`
NoStore bool `json:"no_store,omitempty"`
// Judge is the judge model, when one reads the text.
Judge string `json:"judge,omitempty"`
// Checks are what it looks for, one line each: "input: pii email → redact".
Checks []string `json:"checks"`
}
// GuardHistory is what the guardrails did, from the usage record; the store
// has it, and a store that does not leaves the events endpoint unanswered.
type GuardHistory interface {
GuardEvents(ctx context.Context, since time.Time, limit int) ([]store.GuardEvent, error)
GuardCounts(ctx context.Context, since time.Time) ([]store.GuardCount, error)
}
// GuardEventsView is what the guardrails did since a moment: totals, and the
// newest requests.
type GuardEventsView struct {
Since time.Time `json:"since"`
Counts []GuardCountView `json:"counts"`
Events []GuardEventView `json:"events"`
}
// GuardCountView is how many requests one policy did one thing to.
type GuardCountView struct {
Project string `json:"project,omitempty"`
Action string `json:"action"`
Policy string `json:"policy"`
Category string `json:"category,omitempty"`
Count int64 `json:"count"`
}
// GuardEventView is one request the guardrails did something to.
type GuardEventView struct {
At time.Time `json:"at"`
RequestID string `json:"request_id"`
Project string `json:"project,omitempty"`
Model string `json:"model"`
Status int `json:"status"`
Action string `json:"action"`
Policy string `json:"policy"`
Category string `json:"category,omitempty"`
GuardMs int64 `json:"guard_ms"`
}
// policyView sums one policy up.
func policyView(name string, p guard.Policy) GuardPolicyView {
v := GuardPolicyView{Name: name, Mode: or(p.Mode, "enforce"), Fail: or(p.Fail, "open"), NoStore: p.NoStore, Checks: []string{}}
rules := func(dir string, r guard.Rules) {
for _, kind := range slices.Sorted(maps.Keys(r.PII)) {
v.Checks = append(v.Checks, fmt.Sprintf("%s: pii %s → %s", dir, kind, r.PII[kind]))
}
if r.Injection != "" {
v.Checks = append(v.Checks, fmt.Sprintf("%s: prompt injection → %s", dir, r.Injection))
}
for _, d := range r.Deny {
v.Checks = append(v.Checks, fmt.Sprintf("%s: pattern %s → %s", dir, d.Name, or(d.Action, "block")))
}
if r.MaxChars > 0 {
v.Checks = append(v.Checks, fmt.Sprintf("%s: at most %d characters", dir, r.MaxChars))
}
}
rules("input", p.Input)
rules("output", p.Output)
if j := p.Judge; j != nil {
v.Judge = j.Model
var dirs []string
if j.Input != "" {
dirs = append(dirs, "input → "+j.Input)
}
if j.Output != "" {
dirs = append(dirs, "output → "+j.Output)
}
line := "judge " + j.Model + ": " + strings.Join(dirs, ", ")
if len(j.Categories) > 0 {
line += " (" + strings.Join(j.Categories, " ") + ")"
}
v.Checks = append(v.Checks, line)
}
if len(p.Tools.Allow) > 0 {
v.Checks = append(v.Checks, "tools: only "+strings.Join(p.Tools.Allow, ", "))
}
if len(p.Tools.Deny) > 0 {
v.Checks = append(v.Checks, "tools: never "+strings.Join(p.Tools.Deny, ", "))
}
if p.MaxOutputTokens > 0 {
v.Checks = append(v.Checks, fmt.Sprintf("answers of at most %d tokens", p.MaxOutputTokens))
}
if p.SystemPrompt != "" {
v.Checks = append(v.Checks, "a system prompt of its own")
}
return v
}
func or(s, dflt string) string {
if s == "" {
return dflt
}
return s
}
// GuardAttachmentView is one attachment, with the name people know its
@@ -43,6 +154,15 @@ type GuardAttachmentView struct {
Name string `json:"name,omitempty"`
}
// guardHistory is where what the guardrails did is read, when there is one.
func (a *API) guardHistory() (GuardHistory, bool) {
if a.guard == nil {
return nil, false
}
h, ok := a.guard.Store.(GuardHistory)
return h, ok
}
// guardTarget reads where a policy is to be attached.
func (a *API) guardTarget(ctx context.Context, target string) (guard.Attachment, error) {
kind, what, ok := strings.Cut(strings.TrimSpace(target), ":")
@@ -99,6 +219,57 @@ func (a *API) registerGuard(mux Mux) {
v.Policies = append(v.Policies, name)
}
slices.Sort(v.Policies)
for _, name := range v.Policies {
v.Details = append(v.Details, policyView(name, a.guard.Library.Policies[name]))
}
writeJSON(w, http.StatusOK, v)
})
mux.HandleFunc("GET /api/v1/guard/events", func(w http.ResponseWriter, r *http.Request) {
h, ok := a.guardHistory()
if !ok {
writeError(w, http.StatusNotImplemented, "no guard policies, or no store to record what they did")
return
}
since := 24 * time.Hour
if q := r.URL.Query().Get("since"); q != "" {
d, err := parseSince(q)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
since = d
}
from := time.Now().Add(-since)
counts, err := h.GuardCounts(r.Context(), from)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
events, err := h.GuardEvents(r.Context(), from, 100)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
var names map[string]string
if a.quotas != nil {
names, _ = a.scopeNames(r.Context())
}
project := func(id string) string {
if n := names["project:"+id]; n != "" {
return n
}
return id
}
v := GuardEventsView{Since: from, Counts: []GuardCountView{}, Events: []GuardEventView{}}
for _, c := range counts {
v.Counts = append(v.Counts, GuardCountView{Project: project(c.ProjectID), Action: c.Action,
Policy: c.Policy, Category: c.Category, Count: c.Count})
}
slices.SortStableFunc(v.Counts, func(a, b GuardCountView) int { return int(b.Count - a.Count) })
for _, e := range events {
v.Events = append(v.Events, GuardEventView{At: e.At, RequestID: e.RequestID, Project: project(e.ProjectID),
Model: e.Model, Status: e.Status, Action: e.Action, Policy: e.Policy, Category: e.Category, GuardMs: e.GuardMs})
}
writeJSON(w, http.StatusOK, v)
})
change := func(attach bool) http.HandlerFunc {
+51
View File
@@ -116,3 +116,54 @@ func TestUsageIsExportedAsCSVThatASpreadsheetCannotRun(t *testing.T) {
t.Errorf("%d %s %+v", rec.Code, body, m)
}
}
// memGuardHistory is a guard store that also remembers what was done.
type memGuardHistory struct {
memGuard
since time.Time
}
func (m *memGuardHistory) GuardEvents(_ context.Context, since time.Time, _ int) ([]store.GuardEvent, error) {
m.since = since
return []store.GuardEvent{{RequestID: "r1", Model: "m", Status: 400, Action: "block", Policy: "base", Category: "S1", GuardMs: 410}}, nil
}
func (m *memGuardHistory) GuardCounts(context.Context, time.Time) ([]store.GuardCount, error) {
return []store.GuardCount{
{Action: "flag", Policy: "base", Category: "email", Count: 2},
{Action: "block", Policy: "base", Category: "S1", Count: 7},
}, nil
}
func TestTheGuardrailsSayWhatEachPolicyDoesAndWhatTheyDid(t *testing.T) {
a, _ := setup(t, testModels())
h := &memGuardHistory{}
a.SetGuard(&GuardAdmin{Store: h, Library: guard.Config{Policies: map[string]guard.Policy{
"base": {Fail: "closed", Input: guard.Rules{PII: map[string]string{"email": "redact"}, Injection: "block"},
Judge: &guard.JudgePolicy{Model: "llama-guard-3-1b", Input: "block", Categories: []string{"S1", "S9"}}},
}}})
var v GuardView
if code := call(t, a, "GET", "/api/v1/guard", "", &v); code != 200 || len(v.Details) != 1 {
t.Fatalf("%d %+v", code, v)
}
d := v.Details[0]
want := []string{"input: pii email → redact", "input: prompt injection → block", "judge llama-guard-3-1b: input → block (S1 S9)"}
if d.Mode != "enforce" || d.Fail != "closed" || d.Judge != "llama-guard-3-1b" || !slices.Equal(d.Checks, want) {
t.Errorf("%+v", d)
}
var ev GuardEventsView
if code := call(t, a, "GET", "/api/v1/guard/events?since=1h", "", &ev); code != 200 {
t.Fatalf("events: %d", code)
}
if time.Since(h.since) > time.Hour+time.Minute || time.Since(h.since) < time.Hour-time.Minute {
t.Errorf("asked since %s", h.since)
}
if len(ev.Counts) != 2 || ev.Counts[0].Count != 7 || len(ev.Events) != 1 || ev.Events[0].GuardMs != 410 {
t.Errorf("most first, and the events: %+v", ev)
}
if code := call(t, a, "GET", "/api/v1/guard/events?since=soon", "", nil); code != 400 {
t.Errorf("a bad window: %d", code)
}
}
+5
View File
@@ -249,6 +249,11 @@ var Routes = []Route{
Response: Removed{}},
{Method: "GET", Path: "/api/v1/guard", Summary: "The guardrail policies and where they are attached",
Response: GuardView{}},
{Method: "GET", Path: "/api/v1/guard/events", Summary: "What the guardrails did: totals and the newest requests",
Query: []Param{
{Name: "since", Description: "How far back: 90m, 24h, 7d; 24h when absent."},
},
Response: GuardEventsView{}},
{Method: "POST", Path: "/api/v1/guard/attachments", Summary: "Attach a guardrail policy",
Request: GuardAttachmentRequest{}, Response: guard.Attachment{}},
{Method: "DELETE", Path: "/api/v1/guard/attachments", Summary: "Detach a guardrail policy",
+2 -2
View File
@@ -261,7 +261,7 @@ func TestControlPlaneFeedsRouter(t *testing.T) {
}()
defer close(stop)
h.waitForEndpoints(t, 2, 15*time.Second)
h.waitForEndpoints(t, 2, 30*time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
@@ -304,7 +304,7 @@ func TestRouterKeepsServingWhileNodeIsLost(t *testing.T) {
}()
defer close(stop)
h.waitForEndpoints(t, 2, 15*time.Second)
h.waitForEndpoints(t, 2, 30*time.Second)
// Continuous traffic for the whole experiment.
ctx, cancel := context.WithCancel(context.Background())