A hardened install by default: baseline guardrails with Llama Guard 3 1B judging on a CPU, keys at most 90 days

server init (and the setup screens) write guard.default [baseline] —
secrets refused, cards masked, injection recorded, the judge refusing
S1-S4, S7, S9-S12 in prompts and answers, fail open after 5s — fetch the
judge into the mirror and run it on a node's CPU. --no-guard-model leaves
it out. The closing text names what init cannot do: TLS on the client
port and backup recipients.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 17:45:31 +02:00
1 parent 6abd510628
commit ddb0472f9b
9 files changed
+284 -37

No files matched your search

+77 -28
View File
@@ -41,6 +41,13 @@ const (
starterSource = "hf:Qwen/Qwen2.5-0.5B-Instruct-GGUF/qwen2.5-0.5b-instruct-q4_k_m.gguf"
starterVersion = "starter"
// The guardrails' judge: Meta's Llama Guard 3 1B, which reads a prompt
// or an answer and says whether it is unsafe and why. Q4_K_M is about
// 900 MB and judges a short text in about half a second on a CPU.
guardModelName = "llama-guard-3-1b"
guardModelSource = "hf:QuantFactory/Llama-Guard-3-1B-GGUF/Llama-Guard-3-1B.Q4_K_M.gguf"
guardModelVersion = "guard"
// localLabel marks the control plane's own node, so the starter model is
// served where it was asked for and does not wander to the first GPU node
// that joins later.
@@ -436,46 +443,91 @@ func starterModel(source string, gpuClasses []string, pinned bool) catalog.Model
}
}
// llamaGuard3Categories name Llama Guard 3's S1 to S13, so a verdict is
// recorded as "Violent Crimes" rather than "S1".
var llamaGuard3Categories = []string{"Violent Crimes", "Non-Violent Crimes", "Sex-Related Crimes",
"Child Sexual Exploitation", "Defamation", "Specialized Advice", "Privacy", "Intellectual Property",
"Indiscriminate Weapons", "Hate", "Suicide & Self-Harm", "Sexual Content", "Elections"}
// guardModel is the judge's catalog entry: CPU only, so it takes no card from
// the models people use, on whichever node has room.
func guardModel(source string) catalog.Model {
return catalog.Model{
Name: guardModelName,
Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.",
Source: source,
Role: catalog.RoleGuard,
Guard: &catalog.GuardModel{Format: "llamaguard3", Categories: llamaGuard3Categories},
Scaling: catalog.Scaling{Min: 1, Max: 1},
Profiles: []catalog.Profile{{
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: 2,
Args: []string{"--ctx-size", "8192"},
}},
}
}
// ensureStarter fetches the starter model into the mirror and serves it here.
func (in *serverInit) ensureStarter(ctx context.Context) error {
// Served here when this machine is a node, pinned so it does not wander
// to the first GPU node that joins; otherwise on whichever node joins.
pinned := !in.noNode
return in.ensureServed(ctx, starterName, starterSource, starterVersion, pinned,
func(url string, classes []string) catalog.Model { return starterModel(url, classes, pinned) })
}
// ensureGuardModel fetches the guardrails' judge model into the mirror and
// adds it to the catalog, to run on any node with room.
func (in *serverInit) ensureGuardModel(ctx context.Context) error {
return in.ensureServed(ctx, guardModelName, guardModelSource, guardModelVersion, false,
func(url string, _ []string) catalog.Model { return guardModel(url) })
}
// ensureServed fetches a model into the mirror, adds it to the catalog, and
// when it runs here, waits for it to answer.
func (in *serverInit) ensureServed(ctx context.Context, name, source, version string, here bool,
entry func(url string, gpuClasses []string) catalog.Model,
) error {
client := in.adminAPI()
var models []adminapi.ModelView
if err := client.get(ctx, "/api/v1/models", &models); err != nil {
return fmt.Errorf("reading the catalog: %w", err)
}
for _, m := range models {
if m.Name == starterName {
in.kept("%s in the catalog", starterName)
return in.waitForStarter(ctx)
if m.Name == name {
in.kept("%s in the catalog", name)
if !here {
return nil
}
return in.waitServing(ctx, name)
}
}
upstream, ok := sourcecheck.HTTPSource(starterSource)
upstream, ok := sourcecheck.HTTPSource(source)
if !ok {
return fmt.Errorf("cannot turn %s into a download", starterSource)
return fmt.Errorf("cannot turn %s into a download", source)
}
m, err := mirror.NewClient(in.mirrorAPIURL(), tlsconf.Config{}, 15*time.Second)
if err != nil {
return err
}
st, err := m.Status(ctx, starterName, starterVersion)
st, err := m.Status(ctx, name, version)
var apiErr *mirror.APIError
switch {
case err == nil:
case errors.As(err, &apiErr) && apiErr.Status == 404:
if st, err = m.Fetch(ctx, mirror.FetchRequest{
Source: upstream, Model: starterName, Version: starterVersion,
Source: upstream, Model: name, Version: version,
}); err != nil {
return fmt.Errorf("asking the mirror for %s: %w", starterName, err)
return fmt.Errorf("asking the mirror for %s: %w", name, err)
}
fmt.Fprintf(in.out, " .. downloading %s into the mirror\n", starterName)
fmt.Fprintf(in.out, " .. downloading %s into the mirror\n", name)
default:
return err
}
lastPct := -10
err = waitFor(ctx, 30*time.Minute, starterName+" to download", func() (bool, error) {
cur, err := m.Status(ctx, starterName, starterVersion)
err = waitFor(ctx, 30*time.Minute, name+" to download", func() (bool, error) {
cur, err := m.Status(ctx, name, version)
if err != nil {
return false, nil
}
@@ -484,7 +536,7 @@ func (in *serverInit) ensureStarter(ctx context.Context) error {
case mirror.Present:
return true, nil
case mirror.Failed:
return false, fmt.Errorf("the mirror could not download %s: %s", starterName, cur.Status.Err)
return false, fmt.Errorf("the mirror could not download %s: %s", name, cur.Status.Err)
}
if pct, ok := cur.Status.Percent(); ok && pct >= lastPct+10 {
fmt.Fprintf(in.out, " .. %d%%\n", pct)
@@ -495,45 +547,42 @@ func (in *serverInit) ensureStarter(ctx context.Context) error {
if err != nil {
return err
}
in.done("%s in the mirror", starterName)
in.done("%s in the mirror", name)
// Served here when this machine is a node, pinned so it does not wander
// to the first GPU node that joins; otherwise on whichever node joins.
pinned := !in.noNode
var classes []string
if pinned {
if here {
classes, _ = gpuClasses(ctx)
}
body, err := json.Marshal(starterModel(st.URL, classes, pinned))
body, err := json.Marshal(entry(st.URL, classes))
if err != nil {
return err
}
if err := client.post(ctx, "/api/v1/models", body, &map[string]string{}); err != nil {
return fmt.Errorf("adding %s to the catalog: %w", starterName, err)
return fmt.Errorf("adding %s to the catalog: %w", name, err)
}
if !pinned {
in.done("%s in the catalog; it starts on the first node that joins", starterName)
if !here {
in.done("%s in the catalog; it runs on a node with room for it", name)
return nil
}
where := "on CPU"
if len(classes) > 0 {
where = "on " + strings.Join(classes, ", ") + ", or CPU if the card is busy"
}
in.done("%s in the catalog, served here %s", starterName, where)
return in.waitForStarter(ctx)
in.done("%s in the catalog, served here %s", name, where)
return in.waitServing(ctx, name)
}
func (in *serverInit) waitForStarter(ctx context.Context) error {
func (in *serverInit) waitServing(ctx context.Context, name string) error {
client := in.adminAPI()
last := ""
return waitFor(ctx, 10*time.Minute, starterName+" to start", func() (bool, error) {
return waitFor(ctx, 10*time.Minute, name+" to start", func() (bool, error) {
var reps []adminapi.ReplicaView
if err := client.get(ctx, "/api/v1/replicas?model="+starterName, &reps); err != nil {
if err := client.get(ctx, "/api/v1/replicas?model="+name, &reps); err != nil {
return false, nil
}
for _, r := range reps {
if r.State == "ready" {
in.kept("%s is serving (profile %s)", starterName, r.Profile)
in.kept("%s is serving (profile %s)", name, r.Profile)
return true, nil
}
}
@@ -543,7 +592,7 @@ func (in *serverInit) waitForStarter(ctx context.Context) error {
line += ": " + r.Err
}
if line != last {
fmt.Fprintf(in.out, " .. %s %s\n", starterName, line)
fmt.Fprintf(in.out, " .. %s %s\n", name, line)
last = line
}
}