From 07af03aa931a214370c761375780d75d15d85a36 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 4 Oct 2026 19:11:12 +0200 Subject: [PATCH] server init: the firewall and the judge before the starter model, and a model that fits nowhere said within two minutes A starter model with no room failed init after ten minutes and left the firewall and the judge uninstalled. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/install.md | 13 ++++++---- internal/cli/serverinit.go | 24 ++++++++++--------- internal/cli/serverinit_cluster.go | 26 ++++++++++++++++++++ internal/cli/serverinit_cluster_test.go | 32 +++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 16 deletions(-) diff --git a/docs/install.md b/docs/install.md index 5c72973f..41efc5a4 100644 --- a/docs/install.md +++ b/docs/install.md @@ -128,14 +128,17 @@ That is the whole step. In order, it: 8. makes this machine a node of its own control plane — on its GPUs when `nvidia-smi` finds any, on CPU otherwise — and waits for it to install its engine -9. fetches Qwen2.5 0.5B into the mirror and serves it on this node, so the - first thing anyone opens `/chat` to find is a model that answers +9. closes llmgw's internal ports to everything but the cluster's networks 10. fetches Llama Guard 3 1B, the guardrails' judge, and runs it on the CPU of a node with room — this one, or the first that joins -11. closes llmgw's internal ports to everything but the cluster's networks +11. fetches Qwen2.5 0.5B into the mirror and serves it on this node, so the + first thing anyone opens `/chat` to find is a model that answers. A + machine without room for it is told so within two minutes, and why. -`--no-mirror`, `--no-node` and `--no-model` leave out 7 to 9, -`--no-guard-model` 10, and `--no-firewall` 11. +`--no-mirror` and `--no-node` leave out 7 and 8, `--no-firewall` 9, +`--no-guard-model` 10 and `--no-model` 11. An NVIDIA card with no working +driver is reported rather than silently ignored; install the driver and run +init again to serve on it. **Secure by default.** What init writes is the hardened configuration, not a demonstration one: diff --git a/internal/cli/serverinit.go b/internal/cli/serverinit.go index 63914b0b..6ddc6671 100644 --- a/internal/cli/serverinit.go +++ b/internal/cli/serverinit.go @@ -222,13 +222,12 @@ func (in *serverInit) run(ctx context.Context) error { return fmt.Errorf("node: %w", err) } } - switch { - case !hasSystemd || in.noModel: - case in.noMirror && in.mirrorURL == "": - fmt.Fprintln(in.out, " skip no starter model without the mirror") - default: - if err := in.ensureStarter(ctx); err != nil { - return fmt.Errorf("starter model: %w", err) + // The protections first: a starter model that fits nowhere must not + // leave the machine without its firewall or its judge. + if hasSystemd && !in.noFirewall { + if err := in.ensureFirewall(ctx); err != nil { + // The cluster works without it; said, not fatal. + fmt.Fprintf(in.out, " !! firewall: %v; llmgw's ports answer every network this machine is on\n", err) } } if in.judgeWanted() && in.configJudgesWith(guardModelName) { @@ -238,10 +237,13 @@ func (in *serverInit) run(ctx context.Context) error { fmt.Fprintf(in.out, " !! guard model: %v; requests are not judged until it runs\n", err) } } - if hasSystemd && !in.noFirewall { - if err := in.ensureFirewall(ctx); err != nil { - // The cluster works without it; said, not fatal. - fmt.Fprintf(in.out, " !! firewall: %v; llmgw's ports answer every network this machine is on\n", err) + switch { + case !hasSystemd || in.noModel: + case in.noMirror && in.mirrorURL == "": + fmt.Fprintln(in.out, " skip no starter model without the mirror") + default: + if err := in.ensureStarter(ctx); err != nil { + return fmt.Errorf("starter model: %w", err) } } in.nextSteps() diff --git a/internal/cli/serverinit_cluster.go b/internal/cli/serverinit_cluster.go index c752d2a2..058038f8 100644 --- a/internal/cli/serverinit_cluster.go +++ b/internal/cli/serverinit_cluster.go @@ -572,14 +572,40 @@ func (in *serverInit) ensureServed(ctx context.Context, name, source, version st return in.waitServing(ctx, name) } +// unplacedGrace is how long init lets placement say a model fits nowhere +// before believing it: long enough for a node still installing its engine. +var unplacedGrace = 90 * time.Second + func (in *serverInit) waitServing(ctx context.Context, name string) error { client := in.adminAPI() last := "" + var unplaced time.Time return waitFor(ctx, 10*time.Minute, name+" to start", func() (bool, error) { var reps []adminapi.ReplicaView if err := client.get(ctx, "/api/v1/replicas?model="+name, &reps); err != nil { return false, nil } + // Nowhere to put it is not waited out for ten minutes: once + // placement has said why for unplacedGrace, that is the answer. + if len(reps) == 0 { + var ex adminapi.ExplainView + if err := client.get(ctx, "/api/v1/explain-placement?model="+name, &ex); err == nil && + !ex.Placed && len(ex.Rejections) > 0 { + if unplaced.IsZero() { + unplaced = time.Now() + } + if time.Since(unplaced) > unplacedGrace { + var why []string + for _, r := range ex.Rejections { + why = append(why, r.Node+": "+r.Reason) + } + return false, fmt.Errorf("%s has nowhere to run (%s); add a node with room, "+ + "or free some ('llmgw ctl explain-placement %s')", name, strings.Join(why, "; "), name) + } + return false, nil + } + } + unplaced = time.Time{} for _, r := range reps { if r.State == "ready" { in.kept("%s is serving (profile %s)", name, r.Profile) diff --git a/internal/cli/serverinit_cluster_test.go b/internal/cli/serverinit_cluster_test.go index 6878ea1f..f04a522d 100644 --- a/internal/cli/serverinit_cluster_test.go +++ b/internal/cli/serverinit_cluster_test.go @@ -2,6 +2,7 @@ package cli import ( "fmt" + "io" "net/http" "net/http/httptest" "os" @@ -172,3 +173,34 @@ func TestTheGuardModelIsAValidJudge(t *testing.T) { } } } + +func TestInitSaysAtOnceWhenAModelFitsNowhere(t *testing.T) { + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/replicas": + w.Write([]byte(`[]`)) + case "/api/v1/explain-placement": + w.Write([]byte(`{"model":"qwen","rejections":[{"profile":"cpu","node":"n1","reason":"has 3 GB of memory with 1 free, needs 2"}]}`)) + default: + http.NotFound(w, r) + } + })) + defer api.Close() + dir := t.TempDir() + token := filepath.Join(dir, "token") + os.WriteFile(token, []byte(strings.Repeat("t", 40)), 0o600) + os.WriteFile(filepath.Join(dir, "server.yaml"), + []byte("admin_listen: "+strings.TrimPrefix(api.URL, "http://")+"\nadmin_token_file: "+token+"\n"), 0o600) + + defer func(g time.Duration) { unplacedGrace = g }(unplacedGrace) + unplacedGrace = 0 + in := &serverInit{etcDir: dir, out: io.Discard} + start := time.Now() + err := in.waitServing(t.Context(), "qwen") + if err == nil || !strings.Contains(err.Error(), "needs 2") || !strings.Contains(err.Error(), "explain-placement") { + t.Fatalf("%v", err) + } + if time.Since(start) > 10*time.Second { + t.Errorf("took %s to say so", time.Since(start)) + } +}