A hardened install by default: baseline guardrails with Llama Guard 3 1B judging on a CPU, keys at most 90 days
server init (and the setup screens) write guard.default [baseline] — secrets refused, cards masked, injection recorded, the judge refusing S1-S4, S7, S9-S12 in prompts and answers, fail open after 5s — fetch the judge into the mirror and run it on a node's CPU. --no-guard-model leaves it out. The closing text names what init cannot do: TLS on the client port and backup recipients. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
6abd510628
commit
ddb0472f9b
9 files changed
+284
-37
No files matched your search
@@ -41,6 +41,13 @@ const (
|
||||
starterSource = "hf:Qwen/Qwen2.5-0.5B-Instruct-GGUF/qwen2.5-0.5b-instruct-q4_k_m.gguf"
|
||||
starterVersion = "starter"
|
||||
|
||||
// The guardrails' judge: Meta's Llama Guard 3 1B, which reads a prompt
|
||||
// or an answer and says whether it is unsafe and why. Q4_K_M is about
|
||||
// 900 MB and judges a short text in about half a second on a CPU.
|
||||
guardModelName = "llama-guard-3-1b"
|
||||
guardModelSource = "hf:QuantFactory/Llama-Guard-3-1B-GGUF/Llama-Guard-3-1B.Q4_K_M.gguf"
|
||||
guardModelVersion = "guard"
|
||||
|
||||
// localLabel marks the control plane's own node, so the starter model is
|
||||
// served where it was asked for and does not wander to the first GPU node
|
||||
// that joins later.
|
||||
@@ -436,46 +443,91 @@ func starterModel(source string, gpuClasses []string, pinned bool) catalog.Model
|
||||
}
|
||||
}
|
||||
|
||||
// llamaGuard3Categories name Llama Guard 3's S1 to S13, so a verdict is
|
||||
// recorded as "Violent Crimes" rather than "S1".
|
||||
var llamaGuard3Categories = []string{"Violent Crimes", "Non-Violent Crimes", "Sex-Related Crimes",
|
||||
"Child Sexual Exploitation", "Defamation", "Specialized Advice", "Privacy", "Intellectual Property",
|
||||
"Indiscriminate Weapons", "Hate", "Suicide & Self-Harm", "Sexual Content", "Elections"}
|
||||
|
||||
// guardModel is the judge's catalog entry: CPU only, so it takes no card from
|
||||
// the models people use, on whichever node has room.
|
||||
func guardModel(source string) catalog.Model {
|
||||
return catalog.Model{
|
||||
Name: guardModelName,
|
||||
Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.",
|
||||
Source: source,
|
||||
Role: catalog.RoleGuard,
|
||||
Guard: &catalog.GuardModel{Format: "llamaguard3", Categories: llamaGuard3Categories},
|
||||
Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{
|
||||
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: 2,
|
||||
Args: []string{"--ctx-size", "8192"},
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// ensureStarter fetches the starter model into the mirror and serves it here.
|
||||
func (in *serverInit) ensureStarter(ctx context.Context) error {
|
||||
// Served here when this machine is a node, pinned so it does not wander
|
||||
// to the first GPU node that joins; otherwise on whichever node joins.
|
||||
pinned := !in.noNode
|
||||
return in.ensureServed(ctx, starterName, starterSource, starterVersion, pinned,
|
||||
func(url string, classes []string) catalog.Model { return starterModel(url, classes, pinned) })
|
||||
}
|
||||
|
||||
// ensureGuardModel fetches the guardrails' judge model into the mirror and
|
||||
// adds it to the catalog, to run on any node with room.
|
||||
func (in *serverInit) ensureGuardModel(ctx context.Context) error {
|
||||
return in.ensureServed(ctx, guardModelName, guardModelSource, guardModelVersion, false,
|
||||
func(url string, _ []string) catalog.Model { return guardModel(url) })
|
||||
}
|
||||
|
||||
// ensureServed fetches a model into the mirror, adds it to the catalog, and
|
||||
// when it runs here, waits for it to answer.
|
||||
func (in *serverInit) ensureServed(ctx context.Context, name, source, version string, here bool,
|
||||
entry func(url string, gpuClasses []string) catalog.Model,
|
||||
) error {
|
||||
client := in.adminAPI()
|
||||
var models []adminapi.ModelView
|
||||
if err := client.get(ctx, "/api/v1/models", &models); err != nil {
|
||||
return fmt.Errorf("reading the catalog: %w", err)
|
||||
}
|
||||
for _, m := range models {
|
||||
if m.Name == starterName {
|
||||
in.kept("%s in the catalog", starterName)
|
||||
return in.waitForStarter(ctx)
|
||||
if m.Name == name {
|
||||
in.kept("%s in the catalog", name)
|
||||
if !here {
|
||||
return nil
|
||||
}
|
||||
return in.waitServing(ctx, name)
|
||||
}
|
||||
}
|
||||
|
||||
upstream, ok := sourcecheck.HTTPSource(starterSource)
|
||||
upstream, ok := sourcecheck.HTTPSource(source)
|
||||
if !ok {
|
||||
return fmt.Errorf("cannot turn %s into a download", starterSource)
|
||||
return fmt.Errorf("cannot turn %s into a download", source)
|
||||
}
|
||||
m, err := mirror.NewClient(in.mirrorAPIURL(), tlsconf.Config{}, 15*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
st, err := m.Status(ctx, starterName, starterVersion)
|
||||
st, err := m.Status(ctx, name, version)
|
||||
var apiErr *mirror.APIError
|
||||
switch {
|
||||
case err == nil:
|
||||
case errors.As(err, &apiErr) && apiErr.Status == 404:
|
||||
if st, err = m.Fetch(ctx, mirror.FetchRequest{
|
||||
Source: upstream, Model: starterName, Version: starterVersion,
|
||||
Source: upstream, Model: name, Version: version,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asking the mirror for %s: %w", starterName, err)
|
||||
return fmt.Errorf("asking the mirror for %s: %w", name, err)
|
||||
}
|
||||
fmt.Fprintf(in.out, " .. downloading %s into the mirror\n", starterName)
|
||||
fmt.Fprintf(in.out, " .. downloading %s into the mirror\n", name)
|
||||
default:
|
||||
return err
|
||||
}
|
||||
|
||||
lastPct := -10
|
||||
err = waitFor(ctx, 30*time.Minute, starterName+" to download", func() (bool, error) {
|
||||
cur, err := m.Status(ctx, starterName, starterVersion)
|
||||
err = waitFor(ctx, 30*time.Minute, name+" to download", func() (bool, error) {
|
||||
cur, err := m.Status(ctx, name, version)
|
||||
if err != nil {
|
||||
return false, nil
|
||||
}
|
||||
@@ -484,7 +536,7 @@ func (in *serverInit) ensureStarter(ctx context.Context) error {
|
||||
case mirror.Present:
|
||||
return true, nil
|
||||
case mirror.Failed:
|
||||
return false, fmt.Errorf("the mirror could not download %s: %s", starterName, cur.Status.Err)
|
||||
return false, fmt.Errorf("the mirror could not download %s: %s", name, cur.Status.Err)
|
||||
}
|
||||
if pct, ok := cur.Status.Percent(); ok && pct >= lastPct+10 {
|
||||
fmt.Fprintf(in.out, " .. %d%%\n", pct)
|
||||
@@ -495,45 +547,42 @@ func (in *serverInit) ensureStarter(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
in.done("%s in the mirror", starterName)
|
||||
in.done("%s in the mirror", name)
|
||||
|
||||
// Served here when this machine is a node, pinned so it does not wander
|
||||
// to the first GPU node that joins; otherwise on whichever node joins.
|
||||
pinned := !in.noNode
|
||||
var classes []string
|
||||
if pinned {
|
||||
if here {
|
||||
classes, _ = gpuClasses(ctx)
|
||||
}
|
||||
body, err := json.Marshal(starterModel(st.URL, classes, pinned))
|
||||
body, err := json.Marshal(entry(st.URL, classes))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := client.post(ctx, "/api/v1/models", body, &map[string]string{}); err != nil {
|
||||
return fmt.Errorf("adding %s to the catalog: %w", starterName, err)
|
||||
return fmt.Errorf("adding %s to the catalog: %w", name, err)
|
||||
}
|
||||
if !pinned {
|
||||
in.done("%s in the catalog; it starts on the first node that joins", starterName)
|
||||
if !here {
|
||||
in.done("%s in the catalog; it runs on a node with room for it", name)
|
||||
return nil
|
||||
}
|
||||
where := "on CPU"
|
||||
if len(classes) > 0 {
|
||||
where = "on " + strings.Join(classes, ", ") + ", or CPU if the card is busy"
|
||||
}
|
||||
in.done("%s in the catalog, served here %s", starterName, where)
|
||||
return in.waitForStarter(ctx)
|
||||
in.done("%s in the catalog, served here %s", name, where)
|
||||
return in.waitServing(ctx, name)
|
||||
}
|
||||
|
||||
func (in *serverInit) waitForStarter(ctx context.Context) error {
|
||||
func (in *serverInit) waitServing(ctx context.Context, name string) error {
|
||||
client := in.adminAPI()
|
||||
last := ""
|
||||
return waitFor(ctx, 10*time.Minute, starterName+" to start", func() (bool, error) {
|
||||
return waitFor(ctx, 10*time.Minute, name+" to start", func() (bool, error) {
|
||||
var reps []adminapi.ReplicaView
|
||||
if err := client.get(ctx, "/api/v1/replicas?model="+starterName, &reps); err != nil {
|
||||
if err := client.get(ctx, "/api/v1/replicas?model="+name, &reps); err != nil {
|
||||
return false, nil
|
||||
}
|
||||
for _, r := range reps {
|
||||
if r.State == "ready" {
|
||||
in.kept("%s is serving (profile %s)", starterName, r.Profile)
|
||||
in.kept("%s is serving (profile %s)", name, r.Profile)
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
@@ -543,7 +592,7 @@ func (in *serverInit) waitForStarter(ctx context.Context) error {
|
||||
line += ": " + r.Err
|
||||
}
|
||||
if line != last {
|
||||
fmt.Fprintf(in.out, " .. %s %s\n", starterName, line)
|
||||
fmt.Fprintf(in.out, " .. %s %s\n", name, line)
|
||||
last = line
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user