diff --git a/deploy/bootstrap/onboard-node.sh b/deploy/bootstrap/onboard-node.sh index 35dc38c0..767cadf3 100755 --- a/deploy/bootstrap/onboard-node.sh +++ b/deploy/bootstrap/onboard-node.sh @@ -31,6 +31,9 @@ usage: $0 --node ID --addr HOST --server URL [options] a multi-node one. --mgmt-addr ADDR address engines are reachable on (default: --addr) --mgmt-if NAME interface NCCL bootstraps over (default eth0) + --llamacpp-bin P path to llama-server + --vllm-bin P path to the vllm entry point + --sglang-python P python interpreter of the SGLang virtualenv --nvidia-smi PATH where nvidia-smi lives, when it is not on PATH. Containerized driver installs routinely put it somewhere else. --ib-root DIR the sysfs InfiniBand directory (default /sys/class/infiniband) @@ -43,6 +46,7 @@ USAGE NODE= ADDR= SERVER= MGMT= MGMT_IF=eth0 NVIDIA_SMI= IB_ROOT=/sys/class/infiniband +LLAMACPP_BIN= VLLM_BIN= SGLANG_PYTHON= PKI=/etc/llmgw/pki BINARY= SSH_USER=ilker FABRIC_NAME= FABRIC_HCAS="mlx5_0,mlx5_1" @@ -54,6 +58,9 @@ while [ $# -gt 0 ]; do --fabric) FABRIC_NAME="${2%%:*}"; [ "${2#*:}" != "$2" ] && FABRIC_HCAS="${2#*:}"; shift 2;; --mgmt-addr) MGMT="$2"; shift 2;; --mgmt-if) MGMT_IF="$2"; shift 2;; + --llamacpp-bin) LLAMACPP_BIN="$2"; shift 2;; + --vllm-bin) VLLM_BIN="$2"; shift 2;; + --sglang-python) SGLANG_PYTHON="$2"; shift 2;; --nvidia-smi) NVIDIA_SMI="$2"; shift 2;; --ib-root) IB_ROOT="$2"; shift 2;; --pki) PKI="$2"; shift 2;; @@ -140,6 +147,19 @@ discovery: prefer_nvml: false skip_gpus: false +# Where the engines live. Empty means "on PATH", which is rarely true: a node +# that cannot find its engine registers healthy and then fails every shard +# placed on it with "executable file not found". +engines: + llamacpp: + bin: "$LLAMACPP_BIN" + parallel: 4 + vllm: + bin: "$VLLM_BIN" + launch_mode: ray + sglang: + python: "$SGLANG_PYTHON" + supervisor: work_dir: /var/lib/llmgw-agent log_lines: 512 @@ -179,6 +199,10 @@ $SSH 'set -e sudo systemctl restart llmgw-agent' echo +if [ -z "$LLAMACPP_BIN$VLLM_BIN$SGLANG_PYTHON" ]; then + say "no engine paths given; this node can only run the 'sim' engine until" + say "one is installed and the configuration points at it" +fi echo "$NODE onboarded. It should appear in 'llmgw ctl nodes' within a few seconds." echo "If it does not, the control plane's log says why — a certificate whose" echo "common name does not match the claimed node id is refused by design." diff --git a/docs/node-onboarding.md b/docs/node-onboarding.md index 333539d2..21279767 100644 --- a/docs/node-onboarding.md +++ b/docs/node-onboarding.md @@ -49,6 +49,27 @@ what makes a node's identity something it proves rather than something it picks. `authorized_keys` into the new machine, so a node built by the script above already satisfies it; an older machine may need the key added. +## What onboarding does not do + +It gives a machine an identity and an agent. It does **not** install inference +engines or model weights, and a node missing either looks healthy in +`llmgw ctl nodes` right up until a replica is placed on it and the shard dies +with "executable file not found". + +So the node is not finished until: + +- the engines its catalog profiles name are installed (`vllm`, `sglang`, + `llama-server`), at the paths the agent configuration gives +- the weights are on disk, at the `source` each model declares + +Both are deliberately outside this script. Engines are a platform decision with +versions and drivers attached, and distributing weights is its own problem — +hundreds of gigabytes that want a shared filesystem or a cache, not an scp from +a laptop. llmgw has no answer for it yet; that is a known gap, not an oversight. + +The `sim` engine needs neither, which is what makes it useful for exercising +everything else. + ## Fabrics `--fabric` is how a node says which InfiniBand fabric it is attached to, and diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index eaf3507f..8d6d0bc1 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -511,6 +511,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac ID: id, Model: m.Name, Profile: prof.Name, Fabric: pl.Fabric, Pool: pl.Pool, State: Placing, Weight: prof.Weight, CreatedAt: r.now(), Attempt: attempt, + WantShards: len(pl.Nodes), } // The leader's address and port are fixed before any shard starts: every @@ -773,6 +774,9 @@ func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []Ad Fabric: first.Fabric, Pool: first.Pool, Weight: prof.Weight, CreatedAt: r.now(), Endpoint: first.Endpoint, + // From the profile, not from what was reported: the point is to + // notice when what was reported is short. + WantShards: max(prof.Nodes, 1), } for _, s := range group { rep.Shards = append(rep.Shards, Shard{ diff --git a/internal/control/reconciler/replica.go b/internal/control/reconciler/replica.go index b58d3fb1..1031312b 100644 --- a/internal/control/reconciler/replica.go +++ b/internal/control/reconciler/replica.go @@ -84,6 +84,13 @@ type Replica struct { Endpoint string // Weight is the profile's capacity weight, passed to the router. Weight float64 + // WantShards is how many shards this replica should have, from its profile. + // + // Without it a replica cannot tell "every shard I have is ready" from + // "every shard I have left is ready". A control plane that adopted half of + // a two-node replica reported it Ready with one shard and no endpoint, and + // the router was handed a replica nothing could route to. + WantShards int CreatedAt time.Time ReadyAt time.Time @@ -163,12 +170,22 @@ func (r *Replica) deriveState() { } } - if ready == len(r.Shards) { + // Both conditions. A replica missing a shard is not ready however healthy + // the shards it still has look — and rank 0's absence in particular means + // there is no endpoint to route to at all. + if ready == len(r.Shards) && (r.WantShards == 0 || len(r.Shards) == r.WantShards) { if r.State != Ready { r.ReadyAt = time.Now() } r.State = Ready return } + if r.WantShards > 0 && len(r.Shards) < r.WantShards { + r.State = Failed + if r.Err == "" { + r.Err = fmt.Sprintf("replica has %d of %d shards", len(r.Shards), r.WantShards) + } + return + } r.State = Starting } diff --git a/internal/control/reconciler/stuck_test.go b/internal/control/reconciler/stuck_test.go index 0e71f8d1..d5898f27 100644 --- a/internal/control/reconciler/stuck_test.go +++ b/internal/control/reconciler/stuck_test.go @@ -340,3 +340,69 @@ func TestAnotherNodesShardsAreNotForgotten(t *testing.T) { t.Error("node 1's registration forgot node 2's shards") } } + +// TestAReplicaMissingAShardIsNotReady is what a control plane restart exposed: +// half of a two-node replica was adopted, every shard it still had was healthy, +// and it was reported Ready with no endpoint — so the router was handed a +// replica nothing could route to. +func TestAReplicaMissingAShardIsNotReady(t *testing.T) { + rep := &Replica{ + ID: "r1", WantShards: 2, + Shards: []Shard{{ID: "r1-r1", Rank: 1, State: supervisor.Ready}}, + } + rep.deriveState() + + if rep.State == Ready { + t.Fatal("a replica with one of two shards is Ready") + } + if !strings.Contains(rep.Err, "1 of 2") { + t.Errorf("the error does not say what is missing: %q", rep.Err) + } +} + +func TestACompleteReplicaIsReady(t *testing.T) { + rep := &Replica{ + ID: "r1", WantShards: 2, + Shards: []Shard{ + {ID: "r1-r0", Rank: 0, State: supervisor.Ready}, + {ID: "r1-r1", Rank: 1, State: supervisor.Ready}, + }, + } + rep.deriveState() + if rep.State != Ready { + t.Errorf("state = %s, want ready", rep.State) + } +} + +func TestAReplicaStillComingUpIsNotFailed(t *testing.T) { + // Every shard present, not all ready yet: that is Starting, not a shortage. + rep := &Replica{ + ID: "r1", WantShards: 2, + Shards: []Shard{ + {ID: "r1-r0", Rank: 0, State: supervisor.Ready}, + {ID: "r1-r1", Rank: 1, State: supervisor.Loading}, + }, + } + rep.deriveState() + if rep.State != Starting { + t.Errorf("state = %s, want starting", rep.State) + } +} + +func TestAMultiNodeReplicaKnowsHowManyShardsItWants(t *testing.T) { + // End to end through placement, because the count has to reach the replica + // from the profile rather than from whatever turned up. + m := testModel(1, catalog.Profile{ + Name: "tp8", GPUClass: "h100-80g", TP: 8, Nodes: 2, GPUsPerNode: 4, + }) + h := newHarness(t, m, gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a")) + h.r.Reconcile(context.Background()) + + reps := h.r.Replicas() + if len(reps) != 1 { + t.Fatalf("replicas: %+v", reps) + } + if reps[0].WantShards != 2 { + t.Errorf("WantShards = %d, want 2", reps[0].WantShards) + } +}