A hardened install by default: baseline guardrails with Llama Guard 3 1B judging on a CPU, keys at most 90 days

server init (and the setup screens) write guard.default [baseline] —
secrets refused, cards masked, injection recorded, the judge refusing
S1-S4, S7, S9-S12 in prompts and answers, fail open after 5s — fetch the
judge into the mirror and run it on a node's CPU. --no-guard-model leaves
it out. The closing text names what init cannot do: TLS on the client
port and backup recipients.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 17:45:31 +02:00
1 parent 6abd510628
commit ddb0472f9b
9 files changed
+284 -37

No files matched your search

+87 -1
View File
@@ -81,6 +81,9 @@ type serverInit struct {
// make it serve something: a mirror, this machine as a node, and a small
// model on it.
noMirror, noNode, noModel bool
// noGuardModel leaves out the guardrails' judge model; the pattern
// checks in the baseline policy stay.
noGuardModel bool
// noFirewall leaves this machine's firewall, and the store members',
// as they are.
noFirewall bool
@@ -164,6 +167,8 @@ func newServerInitCmd() *cobra.Command {
"use the mirror on another machine, set up there by 'llmgw mirror init': its management API, http://host:8082")
f.BoolVar(&in.noNode, "no-node", false, "do not make this machine a node (and serve no model on it)")
f.BoolVar(&in.noModel, "no-model", false, "do not fetch and serve the starter model ("+starterName+")")
f.BoolVar(&in.noGuardModel, "no-guard-model", false,
"do not fetch and serve the guardrails' judge model ("+guardModelName+"); the pattern checks stay")
f.BoolVar(&in.noFirewall, "no-firewall", false,
"leave the firewall alone, instead of keeping llmgw's internal ports to the cluster's networks")
f.BoolVar(&in.postgres, "postgres", false,
@@ -226,6 +231,13 @@ func (in *serverInit) run(ctx context.Context) error {
return fmt.Errorf("starter model: %w", err)
}
}
if in.judgeWanted() && in.configJudgesWith(guardModelName) {
// Not fatal: without its judge the baseline policy still checks for
// secrets and injection, and a judged request is served flagged.
if err := in.ensureGuardModel(ctx); err != nil {
fmt.Fprintf(in.out, " !! guard model: %v; requests are not judged until it runs\n", err)
}
}
if hasSystemd && !in.noFirewall {
if err := in.ensureFirewall(ctx); err != nil {
// The cluster works without it; said, not fatal.
@@ -236,6 +248,70 @@ func (in *serverInit) run(ctx context.Context) error {
return nil
}
// judgeWanted is whether this install fetches the judge model, and so
// whether the baseline policy asks it.
func (in *serverInit) judgeWanted() bool {
if in.noGuardModel || (in.noMirror && in.mirrorURL == "") || in.unitPath == "" {
return false
}
_, err := exec.LookPath("systemctl")
return err == nil
}
// configJudgesWith is whether server.yaml has a policy asking that judge: a
// configuration written before init installed one is never changed, so the
// model is not fetched for nothing.
func (in *serverInit) configJudgesWith(model string) bool {
cfg, err := config.Load(in.configPath())
if err != nil {
return false
}
for _, p := range cfg.Guard.Policies {
if p.Judge != nil && p.Judge.Model == model {
return true
}
}
return false
}
// guardSection is server.yaml's guard: the baseline every request passes,
// with the judge model when there is one.
func guardSection(judge bool) string {
j := ` # A judge model reads what patterns cannot: violence, weapons, abuse,
# self-harm. Installed by init; --no-guard-model leaves it out.
# judge: {model: ` + guardModelName + `, input: block, output: block, ...}
`
if judge {
j = ` # A judge model reads what patterns cannot: violence, weapons, abuse,
# self-harm. It runs on a CPU, about half a second for a short text;
# an answer is read every 600 characters as it streams. Slower than
# latency_budget, the request is served and recorded as flagged.
# Defamation (S5), specialized advice (S6), intellectual property (S8)
# and elections (S13) are left out: they refuse too much ordinary work.
judge:
model: ` + guardModelName + `
input: block
output: block
categories: [S1, S2, S3, S4, S7, S9, S10, S11, S12]
latency_budget: 5s
`
}
return `# Guardrails (docs/guardrails.md). The baseline applies to every request;
# attach stricter policies to parts of the tree with 'llmgw ctl guard attach'.
# mode: shadow records what would be done without doing it.
guard:
default: [baseline]
policies:
baseline:
fail: open
input:
pii: {secret: block, card: redact}
injection: flag
output:
pii: {secret: block, card: redact}
` + j
}
func (in *serverInit) done(format string, a ...any) {
fmt.Fprintf(in.out, " ok "+format+"\n", a...)
in.changed = true
@@ -448,7 +524,7 @@ func (in *serverInit) ensureConfig(context.Context) error {
contact = "platform@example.com"
}
body := fmt.Sprintf(serverConfigTemplate, in.etcDir, in.etcDir, in.etcDir,
in.storeAddress(), contact)
in.storeAddress(), contact, guardSection(in.judgeWanted()))
if err := os.WriteFile(path, []byte(body), 0o640); err != nil {
return err
}
@@ -503,10 +579,14 @@ auth:
default_price:
prompt_per_1k: 1000
completion_per_1k: 1000
# The longest any API key lives; a key made without an expiry gets this.
# Shorter per organization, team or project with 'llmgw keys ttl'.
max_key_ttl: 2160h # 90 days
landing:
contact: "%s"
%s
# The engines every node installs for itself. Omitted, each gets the llama.cpp
# build this release was checked against.
# engines:
@@ -700,5 +780,11 @@ The control plane is up. Next:
llmgw ctl model add -f model.yaml
The web UI is on port 8080; /admin once you are signed in as the administrator.
Before people outside this network use it:
# port 8080 is plain HTTP: put a proxy with a certificate in front
# (deploy/proxy/ in the release; Caddy gets the certificate itself)
# backups carry every secret: name who may open them, backup_recipients
# in server.yaml, and copy them off this machine
`)
}
+77 -28
View File
@@ -41,6 +41,13 @@ const (
starterSource = "hf:Qwen/Qwen2.5-0.5B-Instruct-GGUF/qwen2.5-0.5b-instruct-q4_k_m.gguf"
starterVersion = "starter"
// The guardrails' judge: Meta's Llama Guard 3 1B, which reads a prompt
// or an answer and says whether it is unsafe and why. Q4_K_M is about
// 900 MB and judges a short text in about half a second on a CPU.
guardModelName = "llama-guard-3-1b"
guardModelSource = "hf:QuantFactory/Llama-Guard-3-1B-GGUF/Llama-Guard-3-1B.Q4_K_M.gguf"
guardModelVersion = "guard"
// localLabel marks the control plane's own node, so the starter model is
// served where it was asked for and does not wander to the first GPU node
// that joins later.
@@ -436,46 +443,91 @@ func starterModel(source string, gpuClasses []string, pinned bool) catalog.Model
}
}
// llamaGuard3Categories name Llama Guard 3's S1 to S13, so a verdict is
// recorded as "Violent Crimes" rather than "S1".
var llamaGuard3Categories = []string{"Violent Crimes", "Non-Violent Crimes", "Sex-Related Crimes",
"Child Sexual Exploitation", "Defamation", "Specialized Advice", "Privacy", "Intellectual Property",
"Indiscriminate Weapons", "Hate", "Suicide & Self-Harm", "Sexual Content", "Elections"}
// guardModel is the judge's catalog entry: CPU only, so it takes no card from
// the models people use, on whichever node has room.
func guardModel(source string) catalog.Model {
return catalog.Model{
Name: guardModelName,
Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.",
Source: source,
Role: catalog.RoleGuard,
Guard: &catalog.GuardModel{Format: "llamaguard3", Categories: llamaGuard3Categories},
Scaling: catalog.Scaling{Min: 1, Max: 1},
Profiles: []catalog.Profile{{
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: 2,
Args: []string{"--ctx-size", "8192"},
}},
}
}
// ensureStarter fetches the starter model into the mirror and serves it here.
func (in *serverInit) ensureStarter(ctx context.Context) error {
// Served here when this machine is a node, pinned so it does not wander
// to the first GPU node that joins; otherwise on whichever node joins.
pinned := !in.noNode
return in.ensureServed(ctx, starterName, starterSource, starterVersion, pinned,
func(url string, classes []string) catalog.Model { return starterModel(url, classes, pinned) })
}
// ensureGuardModel fetches the guardrails' judge model into the mirror and
// adds it to the catalog, to run on any node with room.
func (in *serverInit) ensureGuardModel(ctx context.Context) error {
return in.ensureServed(ctx, guardModelName, guardModelSource, guardModelVersion, false,
func(url string, _ []string) catalog.Model { return guardModel(url) })
}
// ensureServed fetches a model into the mirror, adds it to the catalog, and
// when it runs here, waits for it to answer.
func (in *serverInit) ensureServed(ctx context.Context, name, source, version string, here bool,
entry func(url string, gpuClasses []string) catalog.Model,
) error {
client := in.adminAPI()
var models []adminapi.ModelView
if err := client.get(ctx, "/api/v1/models", &models); err != nil {
return fmt.Errorf("reading the catalog: %w", err)
}
for _, m := range models {
if m.Name == starterName {
in.kept("%s in the catalog", starterName)
return in.waitForStarter(ctx)
if m.Name == name {
in.kept("%s in the catalog", name)
if !here {
return nil
}
return in.waitServing(ctx, name)
}
}
upstream, ok := sourcecheck.HTTPSource(starterSource)
upstream, ok := sourcecheck.HTTPSource(source)
if !ok {
return fmt.Errorf("cannot turn %s into a download", starterSource)
return fmt.Errorf("cannot turn %s into a download", source)
}
m, err := mirror.NewClient(in.mirrorAPIURL(), tlsconf.Config{}, 15*time.Second)
if err != nil {
return err
}
st, err := m.Status(ctx, starterName, starterVersion)
st, err := m.Status(ctx, name, version)
var apiErr *mirror.APIError
switch {
case err == nil:
case errors.As(err, &apiErr) && apiErr.Status == 404:
if st, err = m.Fetch(ctx, mirror.FetchRequest{
Source: upstream, Model: starterName, Version: starterVersion,
Source: upstream, Model: name, Version: version,
}); err != nil {
return fmt.Errorf("asking the mirror for %s: %w", starterName, err)
return fmt.Errorf("asking the mirror for %s: %w", name, err)
}
fmt.Fprintf(in.out, " .. downloading %s into the mirror\n", starterName)
fmt.Fprintf(in.out, " .. downloading %s into the mirror\n", name)
default:
return err
}
lastPct := -10
err = waitFor(ctx, 30*time.Minute, starterName+" to download", func() (bool, error) {
cur, err := m.Status(ctx, starterName, starterVersion)
err = waitFor(ctx, 30*time.Minute, name+" to download", func() (bool, error) {
cur, err := m.Status(ctx, name, version)
if err != nil {
return false, nil
}
@@ -484,7 +536,7 @@ func (in *serverInit) ensureStarter(ctx context.Context) error {
case mirror.Present:
return true, nil
case mirror.Failed:
return false, fmt.Errorf("the mirror could not download %s: %s", starterName, cur.Status.Err)
return false, fmt.Errorf("the mirror could not download %s: %s", name, cur.Status.Err)
}
if pct, ok := cur.Status.Percent(); ok && pct >= lastPct+10 {
fmt.Fprintf(in.out, " .. %d%%\n", pct)
@@ -495,45 +547,42 @@ func (in *serverInit) ensureStarter(ctx context.Context) error {
if err != nil {
return err
}
in.done("%s in the mirror", starterName)
in.done("%s in the mirror", name)
// Served here when this machine is a node, pinned so it does not wander
// to the first GPU node that joins; otherwise on whichever node joins.
pinned := !in.noNode
var classes []string
if pinned {
if here {
classes, _ = gpuClasses(ctx)
}
body, err := json.Marshal(starterModel(st.URL, classes, pinned))
body, err := json.Marshal(entry(st.URL, classes))
if err != nil {
return err
}
if err := client.post(ctx, "/api/v1/models", body, &map[string]string{}); err != nil {
return fmt.Errorf("adding %s to the catalog: %w", starterName, err)
return fmt.Errorf("adding %s to the catalog: %w", name, err)
}
if !pinned {
in.done("%s in the catalog; it starts on the first node that joins", starterName)
if !here {
in.done("%s in the catalog; it runs on a node with room for it", name)
return nil
}
where := "on CPU"
if len(classes) > 0 {
where = "on " + strings.Join(classes, ", ") + ", or CPU if the card is busy"
}
in.done("%s in the catalog, served here %s", starterName, where)
return in.waitForStarter(ctx)
in.done("%s in the catalog, served here %s", name, where)
return in.waitServing(ctx, name)
}
func (in *serverInit) waitForStarter(ctx context.Context) error {
func (in *serverInit) waitServing(ctx context.Context, name string) error {
client := in.adminAPI()
last := ""
return waitFor(ctx, 10*time.Minute, starterName+" to start", func() (bool, error) {
return waitFor(ctx, 10*time.Minute, name+" to start", func() (bool, error) {
var reps []adminapi.ReplicaView
if err := client.get(ctx, "/api/v1/replicas?model="+starterName, &reps); err != nil {
if err := client.get(ctx, "/api/v1/replicas?model="+name, &reps); err != nil {
return false, nil
}
for _, r := range reps {
if r.State == "ready" {
in.kept("%s is serving (profile %s)", starterName, r.Profile)
in.kept("%s is serving (profile %s)", name, r.Profile)
return true, nil
}
}
@@ -543,7 +592,7 @@ func (in *serverInit) waitForStarter(ctx context.Context) error {
line += ": " + r.Err
}
if line != last {
fmt.Fprintf(in.out, " .. %s %s\n", starterName, line)
fmt.Fprintf(in.out, " .. %s %s\n", name, line)
last = line
}
}
+51
View File
@@ -1,12 +1,14 @@
package cli
import (
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
@@ -121,3 +123,52 @@ func TestTheMirrorAPIOnlyAnswersItsControlPlane(t *testing.T) {
t.Error("nonsense was accepted")
}
}
func TestTheInstalledConfigurationIsHardenedAndLoads(t *testing.T) {
for _, judge := range []bool{false, true} {
body := fmt.Sprintf(serverConfigTemplate, "/etc/llmgw", "/etc/llmgw", "/etc/llmgw",
"/var/lib/llmgw/llmgw.db", "ops@example.com", guardSection(judge))
path := filepath.Join(t.TempDir(), "server.yaml")
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
cfg, err := config.Load(path)
if err != nil {
t.Fatalf("judge=%v: %v", judge, err)
}
if err := cfg.Validate(); err != nil {
t.Fatalf("judge=%v: %v", judge, err)
}
if len(cfg.Guard.Default) != 1 || cfg.Guard.Default[0] != "baseline" {
t.Errorf("judge=%v: guard.default %v", judge, cfg.Guard.Default)
}
p := cfg.Guard.Policies["baseline"]
if (p.Judge != nil) != judge {
t.Errorf("judge=%v: the policy's judge is %+v", judge, p.Judge)
}
if judge && (p.Judge.Model != guardModelName || p.Judge.Input != "block") {
t.Errorf("judge %+v", p.Judge)
}
if cfg.Auth.MaxKeyTTL != 90*24*time.Hour || cfg.Auth.SignupOpen || cfg.DebugPprof || cfg.AdminRemote {
t.Errorf("not hardened: ttl %v signup %v pprof %v remote %v",
cfg.Auth.MaxKeyTTL, cfg.Auth.SignupOpen, cfg.DebugPprof, cfg.AdminRemote)
}
}
}
func TestTheGuardModelIsAValidJudge(t *testing.T) {
m := guardModel("http://10.0.0.5:8081/llama-guard-3-1b/guard/g.gguf")
c, err := catalog.New([]catalog.Model{m})
if err != nil {
t.Fatalf("not a valid catalog entry: %v", err)
}
got, _ := c.Resolve(guardModelName)
if !got.IsGuard() || got.Guard.Format != "llamaguard3" || len(got.Guard.Categories) != 13 {
t.Errorf("%+v", got)
}
for _, p := range got.Profiles {
if p.GPUClass != "" {
t.Errorf("the judge takes a card: %+v", p)
}
}
}
+5 -1
View File
@@ -54,6 +54,7 @@ const (
fMirror
fNode
fModel
fGuard
fStart
fCount
)
@@ -104,6 +105,7 @@ func newInitModel(in *serverInit, found []string, askForm bool) initModel {
m.toggles[fMirror] = !in.noMirror
m.toggles[fNode] = !in.noNode
m.toggles[fModel] = !in.noModel
m.toggles[fGuard] = !in.noGuardModel
if askForm {
m.admin.Focus()
}
@@ -216,6 +218,7 @@ func (m initModel) formKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
m.in.noMirror = !m.toggles[fMirror]
m.in.noNode = !m.toggles[fNode]
m.in.noModel = !m.toggles[fModel]
m.in.noGuardModel = !m.toggles[fGuard]
m.form = false
return m, tea.Batch(m.spin.Tick, func() tea.Msg { return startInitMsg{} })
}
@@ -228,7 +231,7 @@ func (m initModel) formKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
var cmd tea.Cmd
m.hosts, cmd = m.hosts.Update(k)
return m, cmd
case fPostgres, fMirror, fNode, fModel:
case fPostgres, fMirror, fNode, fModel, fGuard:
if k.String() == " " || k.String() == "x" {
m.toggles[m.focus] = !m.toggles[m.focus]
}
@@ -371,6 +374,7 @@ func (m initModel) formView() string {
}
b.WriteString(check(fNode, "This machine as a node", "serves models here too, on its GPUs or CPU"))
b.WriteString(check(fModel, "Starter model", "a small Qwen, so there is something to ask at once"))
b.WriteString(check(fGuard, "Guardrails' judge", "Llama Guard 3 1B on a CPU, refusing violent, criminal and abusive requests"))
b.WriteString("\n" + label(fStart, lipgloss.NewStyle().Bold(true).Render("Set up")) + "\n")
if m.formError != "" {
b.WriteString("\n" + sBad.Render(m.formError) + "\n")
+2 -1
View File
@@ -23,12 +23,13 @@ func TestTheInitFormBecomesInitsSettings(t *testing.T) {
m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(" ")}) // PostgreSQL on
m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab},
tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(" ")}) // starter model off
m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(" ")}) // judge off
m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyEnter})
if in.admin != "ops@example.com" || strings.Join(in.hosts, "|") != "llm.example.com|203.0.113.10" {
t.Errorf("admin %q hosts %q", in.admin, in.hosts)
}
if !in.postgres || in.noMirror || in.noNode || !in.noModel {
if !in.postgres || in.noMirror || in.noNode || !in.noModel || !in.noGuardModel {
t.Errorf("toggles: postgres %v mirror %v node %v model %v", in.postgres, !in.noMirror, !in.noNode, !in.noModel)
}
if m.(initModel).form {