server init: the firewall and the judge before the starter model, and a model that fits nowhere said within two minutes

A starter model with no room failed init after ten minutes and left the
firewall and the judge uninstalled.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 19:11:12 +02:00
1 parent ddb0472f9b
commit 07af03aa93
4 files changed
+79 -16

No files matched your search

+8 -5
View File
@@ -128,14 +128,17 @@ That is the whole step. In order, it:
8. makes this machine a node of its own control plane — on its GPUs when
`nvidia-smi` finds any, on CPU otherwise — and waits for it to install its
engine
9. fetches Qwen2.5 0.5B into the mirror and serves it on this node, so the
first thing anyone opens `/chat` to find is a model that answers
9. closes llmgw's internal ports to everything but the cluster's networks
10. fetches Llama Guard 3 1B, the guardrails' judge, and runs it on the CPU
of a node with room — this one, or the first that joins
11. closes llmgw's internal ports to everything but the cluster's networks
11. fetches Qwen2.5 0.5B into the mirror and serves it on this node, so the
first thing anyone opens `/chat` to find is a model that answers. A
machine without room for it is told so within two minutes, and why.
`--no-mirror`, `--no-node` and `--no-model` leave out 7 to 9,
`--no-guard-model` 10, and `--no-firewall` 11.
`--no-mirror` and `--no-node` leave out 7 and 8, `--no-firewall` 9,
`--no-guard-model` 10 and `--no-model` 11. An NVIDIA card with no working
driver is reported rather than silently ignored; install the driver and run
init again to serve on it.
**Secure by default.** What init writes is the hardened configuration, not a
demonstration one:
+13 -11
View File
@@ -222,13 +222,12 @@ func (in *serverInit) run(ctx context.Context) error {
return fmt.Errorf("node: %w", err)
}
}
switch {
case !hasSystemd || in.noModel:
case in.noMirror && in.mirrorURL == "":
fmt.Fprintln(in.out, " skip no starter model without the mirror")
default:
if err := in.ensureStarter(ctx); err != nil {
return fmt.Errorf("starter model: %w", err)
// The protections first: a starter model that fits nowhere must not
// leave the machine without its firewall or its judge.
if hasSystemd && !in.noFirewall {
if err := in.ensureFirewall(ctx); err != nil {
// The cluster works without it; said, not fatal.
fmt.Fprintf(in.out, " !! firewall: %v; llmgw's ports answer every network this machine is on\n", err)
}
}
if in.judgeWanted() && in.configJudgesWith(guardModelName) {
@@ -238,10 +237,13 @@ func (in *serverInit) run(ctx context.Context) error {
fmt.Fprintf(in.out, " !! guard model: %v; requests are not judged until it runs\n", err)
}
}
if hasSystemd && !in.noFirewall {
if err := in.ensureFirewall(ctx); err != nil {
// The cluster works without it; said, not fatal.
fmt.Fprintf(in.out, " !! firewall: %v; llmgw's ports answer every network this machine is on\n", err)
switch {
case !hasSystemd || in.noModel:
case in.noMirror && in.mirrorURL == "":
fmt.Fprintln(in.out, " skip no starter model without the mirror")
default:
if err := in.ensureStarter(ctx); err != nil {
return fmt.Errorf("starter model: %w", err)
}
}
in.nextSteps()
+26
View File
@@ -572,14 +572,40 @@ func (in *serverInit) ensureServed(ctx context.Context, name, source, version st
return in.waitServing(ctx, name)
}
// unplacedGrace is how long init lets placement say a model fits nowhere
// before believing it: long enough for a node still installing its engine.
var unplacedGrace = 90 * time.Second
func (in *serverInit) waitServing(ctx context.Context, name string) error {
client := in.adminAPI()
last := ""
var unplaced time.Time
return waitFor(ctx, 10*time.Minute, name+" to start", func() (bool, error) {
var reps []adminapi.ReplicaView
if err := client.get(ctx, "/api/v1/replicas?model="+name, &reps); err != nil {
return false, nil
}
// Nowhere to put it is not waited out for ten minutes: once
// placement has said why for unplacedGrace, that is the answer.
if len(reps) == 0 {
var ex adminapi.ExplainView
if err := client.get(ctx, "/api/v1/explain-placement?model="+name, &ex); err == nil &&
!ex.Placed && len(ex.Rejections) > 0 {
if unplaced.IsZero() {
unplaced = time.Now()
}
if time.Since(unplaced) > unplacedGrace {
var why []string
for _, r := range ex.Rejections {
why = append(why, r.Node+": "+r.Reason)
}
return false, fmt.Errorf("%s has nowhere to run (%s); add a node with room, "+
"or free some ('llmgw ctl explain-placement %s')", name, strings.Join(why, "; "), name)
}
return false, nil
}
}
unplaced = time.Time{}
for _, r := range reps {
if r.State == "ready" {
in.kept("%s is serving (profile %s)", name, r.Profile)
+32
View File
@@ -2,6 +2,7 @@ package cli
import (
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
@@ -172,3 +173,34 @@ func TestTheGuardModelIsAValidJudge(t *testing.T) {
}
}
}
func TestInitSaysAtOnceWhenAModelFitsNowhere(t *testing.T) {
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/v1/replicas":
w.Write([]byte(`[]`))
case "/api/v1/explain-placement":
w.Write([]byte(`{"model":"qwen","rejections":[{"profile":"cpu","node":"n1","reason":"has 3 GB of memory with 1 free, needs 2"}]}`))
default:
http.NotFound(w, r)
}
}))
defer api.Close()
dir := t.TempDir()
token := filepath.Join(dir, "token")
os.WriteFile(token, []byte(strings.Repeat("t", 40)), 0o600)
os.WriteFile(filepath.Join(dir, "server.yaml"),
[]byte("admin_listen: "+strings.TrimPrefix(api.URL, "http://")+"\nadmin_token_file: "+token+"\n"), 0o600)
defer func(g time.Duration) { unplacedGrace = g }(unplacedGrace)
unplacedGrace = 0
in := &serverInit{etcDir: dir, out: io.Discard}
start := time.Now()
err := in.waitServing(t.Context(), "qwen")
if err == nil || !strings.Contains(err.Error(), "needs 2") || !strings.Contains(err.Error(), "explain-placement") {
t.Fatalf("%v", err)
}
if time.Since(start) > 10*time.Second {
t.Errorf("took %s to say so", time.Since(start))
}
}