server init: the firewall and the judge before the starter model, and a model that fits nowhere said within two minutes
A starter model with no room failed init after ten minutes and left the firewall and the judge uninstalled. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
ddb0472f9b
commit
07af03aa93
4 files changed
+79
-16
No files matched your search
+8
-5
@@ -128,14 +128,17 @@ That is the whole step. In order, it:
|
||||
8. makes this machine a node of its own control plane — on its GPUs when
|
||||
`nvidia-smi` finds any, on CPU otherwise — and waits for it to install its
|
||||
engine
|
||||
9. fetches Qwen2.5 0.5B into the mirror and serves it on this node, so the
|
||||
first thing anyone opens `/chat` to find is a model that answers
|
||||
9. closes llmgw's internal ports to everything but the cluster's networks
|
||||
10. fetches Llama Guard 3 1B, the guardrails' judge, and runs it on the CPU
|
||||
of a node with room — this one, or the first that joins
|
||||
11. closes llmgw's internal ports to everything but the cluster's networks
|
||||
11. fetches Qwen2.5 0.5B into the mirror and serves it on this node, so the
|
||||
first thing anyone opens `/chat` to find is a model that answers. A
|
||||
machine without room for it is told so within two minutes, and why.
|
||||
|
||||
`--no-mirror`, `--no-node` and `--no-model` leave out 7 to 9,
|
||||
`--no-guard-model` 10, and `--no-firewall` 11.
|
||||
`--no-mirror` and `--no-node` leave out 7 and 8, `--no-firewall` 9,
|
||||
`--no-guard-model` 10 and `--no-model` 11. An NVIDIA card with no working
|
||||
driver is reported rather than silently ignored; install the driver and run
|
||||
init again to serve on it.
|
||||
|
||||
**Secure by default.** What init writes is the hardened configuration, not a
|
||||
demonstration one:
|
||||
|
||||
+13
-11
@@ -222,13 +222,12 @@ func (in *serverInit) run(ctx context.Context) error {
|
||||
return fmt.Errorf("node: %w", err)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case !hasSystemd || in.noModel:
|
||||
case in.noMirror && in.mirrorURL == "":
|
||||
fmt.Fprintln(in.out, " skip no starter model without the mirror")
|
||||
default:
|
||||
if err := in.ensureStarter(ctx); err != nil {
|
||||
return fmt.Errorf("starter model: %w", err)
|
||||
// The protections first: a starter model that fits nowhere must not
|
||||
// leave the machine without its firewall or its judge.
|
||||
if hasSystemd && !in.noFirewall {
|
||||
if err := in.ensureFirewall(ctx); err != nil {
|
||||
// The cluster works without it; said, not fatal.
|
||||
fmt.Fprintf(in.out, " !! firewall: %v; llmgw's ports answer every network this machine is on\n", err)
|
||||
}
|
||||
}
|
||||
if in.judgeWanted() && in.configJudgesWith(guardModelName) {
|
||||
@@ -238,10 +237,13 @@ func (in *serverInit) run(ctx context.Context) error {
|
||||
fmt.Fprintf(in.out, " !! guard model: %v; requests are not judged until it runs\n", err)
|
||||
}
|
||||
}
|
||||
if hasSystemd && !in.noFirewall {
|
||||
if err := in.ensureFirewall(ctx); err != nil {
|
||||
// The cluster works without it; said, not fatal.
|
||||
fmt.Fprintf(in.out, " !! firewall: %v; llmgw's ports answer every network this machine is on\n", err)
|
||||
switch {
|
||||
case !hasSystemd || in.noModel:
|
||||
case in.noMirror && in.mirrorURL == "":
|
||||
fmt.Fprintln(in.out, " skip no starter model without the mirror")
|
||||
default:
|
||||
if err := in.ensureStarter(ctx); err != nil {
|
||||
return fmt.Errorf("starter model: %w", err)
|
||||
}
|
||||
}
|
||||
in.nextSteps()
|
||||
|
||||
@@ -572,14 +572,40 @@ func (in *serverInit) ensureServed(ctx context.Context, name, source, version st
|
||||
return in.waitServing(ctx, name)
|
||||
}
|
||||
|
||||
// unplacedGrace is how long init lets placement say a model fits nowhere
|
||||
// before believing it: long enough for a node still installing its engine.
|
||||
var unplacedGrace = 90 * time.Second
|
||||
|
||||
func (in *serverInit) waitServing(ctx context.Context, name string) error {
|
||||
client := in.adminAPI()
|
||||
last := ""
|
||||
var unplaced time.Time
|
||||
return waitFor(ctx, 10*time.Minute, name+" to start", func() (bool, error) {
|
||||
var reps []adminapi.ReplicaView
|
||||
if err := client.get(ctx, "/api/v1/replicas?model="+name, &reps); err != nil {
|
||||
return false, nil
|
||||
}
|
||||
// Nowhere to put it is not waited out for ten minutes: once
|
||||
// placement has said why for unplacedGrace, that is the answer.
|
||||
if len(reps) == 0 {
|
||||
var ex adminapi.ExplainView
|
||||
if err := client.get(ctx, "/api/v1/explain-placement?model="+name, &ex); err == nil &&
|
||||
!ex.Placed && len(ex.Rejections) > 0 {
|
||||
if unplaced.IsZero() {
|
||||
unplaced = time.Now()
|
||||
}
|
||||
if time.Since(unplaced) > unplacedGrace {
|
||||
var why []string
|
||||
for _, r := range ex.Rejections {
|
||||
why = append(why, r.Node+": "+r.Reason)
|
||||
}
|
||||
return false, fmt.Errorf("%s has nowhere to run (%s); add a node with room, "+
|
||||
"or free some ('llmgw ctl explain-placement %s')", name, strings.Join(why, "; "), name)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
unplaced = time.Time{}
|
||||
for _, r := range reps {
|
||||
if r.State == "ready" {
|
||||
in.kept("%s is serving (profile %s)", name, r.Profile)
|
||||
|
||||
@@ -2,6 +2,7 @@ package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
@@ -172,3 +173,34 @@ func TestTheGuardModelIsAValidJudge(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitSaysAtOnceWhenAModelFitsNowhere(t *testing.T) {
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api/v1/replicas":
|
||||
w.Write([]byte(`[]`))
|
||||
case "/api/v1/explain-placement":
|
||||
w.Write([]byte(`{"model":"qwen","rejections":[{"profile":"cpu","node":"n1","reason":"has 3 GB of memory with 1 free, needs 2"}]}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer api.Close()
|
||||
dir := t.TempDir()
|
||||
token := filepath.Join(dir, "token")
|
||||
os.WriteFile(token, []byte(strings.Repeat("t", 40)), 0o600)
|
||||
os.WriteFile(filepath.Join(dir, "server.yaml"),
|
||||
[]byte("admin_listen: "+strings.TrimPrefix(api.URL, "http://")+"\nadmin_token_file: "+token+"\n"), 0o600)
|
||||
|
||||
defer func(g time.Duration) { unplacedGrace = g }(unplacedGrace)
|
||||
unplacedGrace = 0
|
||||
in := &serverInit{etcDir: dir, out: io.Discard}
|
||||
start := time.Now()
|
||||
err := in.waitServing(t.Context(), "qwen")
|
||||
if err == nil || !strings.Contains(err.Error(), "needs 2") || !strings.Contains(err.Error(), "explain-placement") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
if time.Since(start) > 10*time.Second {
|
||||
t.Errorf("took %s to say so", time.Since(start))
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user