A replica missing a shard is not ready
Restarting the control plane over a running two-node replica adopted half of it. Every shard it still had was healthy, so it was reported Ready — with one shard of two and no endpoint at all, because the missing one was rank 0. The router was handed a replica nothing could route to. The cause is that a replica had no idea how many shards it should have. It could tell "every shard I have is ready" and nothing else, which is the same sentence whether or not half of it is gone. It now carries the count from its profile, set at placement and again at adoption, and a replica short of shards fails rather than pretending. Onboarding also wrote no engine paths, so a node onboarded by the script could not find llama-server and failed every shard placed on it with "executable file not found". The script now takes them, and says so when none are given — a node with no engine paths can run the sim engine and nothing else. The cluster now runs both at once: two-node simulated replicas on each fabric, and real llama.cpp inference on the CPU, which is a better test bed than either alone. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
c97b671ef9
commit
7b4acd9284
5 files changed
+133
-1
No files matched your search
@@ -31,6 +31,9 @@ usage: $0 --node ID --addr HOST --server URL [options]
|
||||
a multi-node one.
|
||||
--mgmt-addr ADDR address engines are reachable on (default: --addr)
|
||||
--mgmt-if NAME interface NCCL bootstraps over (default eth0)
|
||||
--llamacpp-bin P path to llama-server
|
||||
--vllm-bin P path to the vllm entry point
|
||||
--sglang-python P python interpreter of the SGLang virtualenv
|
||||
--nvidia-smi PATH where nvidia-smi lives, when it is not on PATH. Containerized
|
||||
driver installs routinely put it somewhere else.
|
||||
--ib-root DIR the sysfs InfiniBand directory (default /sys/class/infiniband)
|
||||
@@ -43,6 +46,7 @@ USAGE
|
||||
|
||||
NODE= ADDR= SERVER= MGMT= MGMT_IF=eth0
|
||||
NVIDIA_SMI= IB_ROOT=/sys/class/infiniband
|
||||
LLAMACPP_BIN= VLLM_BIN= SGLANG_PYTHON=
|
||||
PKI=/etc/llmgw/pki BINARY= SSH_USER=ilker
|
||||
FABRIC_NAME= FABRIC_HCAS="mlx5_0,mlx5_1"
|
||||
|
||||
@@ -54,6 +58,9 @@ while [ $# -gt 0 ]; do
|
||||
--fabric) FABRIC_NAME="${2%%:*}"; [ "${2#*:}" != "$2" ] && FABRIC_HCAS="${2#*:}"; shift 2;;
|
||||
--mgmt-addr) MGMT="$2"; shift 2;;
|
||||
--mgmt-if) MGMT_IF="$2"; shift 2;;
|
||||
--llamacpp-bin) LLAMACPP_BIN="$2"; shift 2;;
|
||||
--vllm-bin) VLLM_BIN="$2"; shift 2;;
|
||||
--sglang-python) SGLANG_PYTHON="$2"; shift 2;;
|
||||
--nvidia-smi) NVIDIA_SMI="$2"; shift 2;;
|
||||
--ib-root) IB_ROOT="$2"; shift 2;;
|
||||
--pki) PKI="$2"; shift 2;;
|
||||
@@ -140,6 +147,19 @@ discovery:
|
||||
prefer_nvml: false
|
||||
skip_gpus: false
|
||||
|
||||
# Where the engines live. Empty means "on PATH", which is rarely true: a node
|
||||
# that cannot find its engine registers healthy and then fails every shard
|
||||
# placed on it with "executable file not found".
|
||||
engines:
|
||||
llamacpp:
|
||||
bin: "$LLAMACPP_BIN"
|
||||
parallel: 4
|
||||
vllm:
|
||||
bin: "$VLLM_BIN"
|
||||
launch_mode: ray
|
||||
sglang:
|
||||
python: "$SGLANG_PYTHON"
|
||||
|
||||
supervisor:
|
||||
work_dir: /var/lib/llmgw-agent
|
||||
log_lines: 512
|
||||
@@ -179,6 +199,10 @@ $SSH 'set -e
|
||||
sudo systemctl restart llmgw-agent'
|
||||
|
||||
echo
|
||||
if [ -z "$LLAMACPP_BIN$VLLM_BIN$SGLANG_PYTHON" ]; then
|
||||
say "no engine paths given; this node can only run the 'sim' engine until"
|
||||
say "one is installed and the configuration points at it"
|
||||
fi
|
||||
echo "$NODE onboarded. It should appear in 'llmgw ctl nodes' within a few seconds."
|
||||
echo "If it does not, the control plane's log says why — a certificate whose"
|
||||
echo "common name does not match the claimed node id is refused by design."
|
||||
@@ -49,6 +49,27 @@ what makes a node's identity something it proves rather than something it picks.
|
||||
`authorized_keys` into the new machine, so a node built by the script above
|
||||
already satisfies it; an older machine may need the key added.
|
||||
|
||||
## What onboarding does not do
|
||||
|
||||
It gives a machine an identity and an agent. It does **not** install inference
|
||||
engines or model weights, and a node missing either looks healthy in
|
||||
`llmgw ctl nodes` right up until a replica is placed on it and the shard dies
|
||||
with "executable file not found".
|
||||
|
||||
So the node is not finished until:
|
||||
|
||||
- the engines its catalog profiles name are installed (`vllm`, `sglang`,
|
||||
`llama-server`), at the paths the agent configuration gives
|
||||
- the weights are on disk, at the `source` each model declares
|
||||
|
||||
Both are deliberately outside this script. Engines are a platform decision with
|
||||
versions and drivers attached, and distributing weights is its own problem —
|
||||
hundreds of gigabytes that want a shared filesystem or a cache, not an scp from
|
||||
a laptop. llmgw has no answer for it yet; that is a known gap, not an oversight.
|
||||
|
||||
The `sim` engine needs neither, which is what makes it useful for exercising
|
||||
everything else.
|
||||
|
||||
## Fabrics
|
||||
|
||||
`--fabric` is how a node says which InfiniBand fabric it is attached to, and
|
||||
|
||||
@@ -511,6 +511,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
|
||||
ID: id, Model: m.Name, Profile: prof.Name,
|
||||
Fabric: pl.Fabric, Pool: pl.Pool, State: Placing,
|
||||
Weight: prof.Weight, CreatedAt: r.now(), Attempt: attempt,
|
||||
WantShards: len(pl.Nodes),
|
||||
}
|
||||
|
||||
// The leader's address and port are fixed before any shard starts: every
|
||||
@@ -773,6 +774,9 @@ func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []Ad
|
||||
Fabric: first.Fabric, Pool: first.Pool,
|
||||
Weight: prof.Weight, CreatedAt: r.now(),
|
||||
Endpoint: first.Endpoint,
|
||||
// From the profile, not from what was reported: the point is to
|
||||
// notice when what was reported is short.
|
||||
WantShards: max(prof.Nodes, 1),
|
||||
}
|
||||
for _, s := range group {
|
||||
rep.Shards = append(rep.Shards, Shard{
|
||||
|
||||
@@ -84,6 +84,13 @@ type Replica struct {
|
||||
Endpoint string
|
||||
// Weight is the profile's capacity weight, passed to the router.
|
||||
Weight float64
|
||||
// WantShards is how many shards this replica should have, from its profile.
|
||||
//
|
||||
// Without it a replica cannot tell "every shard I have is ready" from
|
||||
// "every shard I have left is ready". A control plane that adopted half of
|
||||
// a two-node replica reported it Ready with one shard and no endpoint, and
|
||||
// the router was handed a replica nothing could route to.
|
||||
WantShards int
|
||||
|
||||
CreatedAt time.Time
|
||||
ReadyAt time.Time
|
||||
@@ -163,12 +170,22 @@ func (r *Replica) deriveState() {
|
||||
}
|
||||
}
|
||||
|
||||
if ready == len(r.Shards) {
|
||||
// Both conditions. A replica missing a shard is not ready however healthy
|
||||
// the shards it still has look — and rank 0's absence in particular means
|
||||
// there is no endpoint to route to at all.
|
||||
if ready == len(r.Shards) && (r.WantShards == 0 || len(r.Shards) == r.WantShards) {
|
||||
if r.State != Ready {
|
||||
r.ReadyAt = time.Now()
|
||||
}
|
||||
r.State = Ready
|
||||
return
|
||||
}
|
||||
if r.WantShards > 0 && len(r.Shards) < r.WantShards {
|
||||
r.State = Failed
|
||||
if r.Err == "" {
|
||||
r.Err = fmt.Sprintf("replica has %d of %d shards", len(r.Shards), r.WantShards)
|
||||
}
|
||||
return
|
||||
}
|
||||
r.State = Starting
|
||||
}
|
||||
@@ -340,3 +340,69 @@ func TestAnotherNodesShardsAreNotForgotten(t *testing.T) {
|
||||
t.Error("node 1's registration forgot node 2's shards")
|
||||
}
|
||||
}
|
||||
|
||||
// TestAReplicaMissingAShardIsNotReady is what a control plane restart exposed:
|
||||
// half of a two-node replica was adopted, every shard it still had was healthy,
|
||||
// and it was reported Ready with no endpoint — so the router was handed a
|
||||
// replica nothing could route to.
|
||||
func TestAReplicaMissingAShardIsNotReady(t *testing.T) {
|
||||
rep := &Replica{
|
||||
ID: "r1", WantShards: 2,
|
||||
Shards: []Shard{{ID: "r1-r1", Rank: 1, State: supervisor.Ready}},
|
||||
}
|
||||
rep.deriveState()
|
||||
|
||||
if rep.State == Ready {
|
||||
t.Fatal("a replica with one of two shards is Ready")
|
||||
}
|
||||
if !strings.Contains(rep.Err, "1 of 2") {
|
||||
t.Errorf("the error does not say what is missing: %q", rep.Err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACompleteReplicaIsReady(t *testing.T) {
|
||||
rep := &Replica{
|
||||
ID: "r1", WantShards: 2,
|
||||
Shards: []Shard{
|
||||
{ID: "r1-r0", Rank: 0, State: supervisor.Ready},
|
||||
{ID: "r1-r1", Rank: 1, State: supervisor.Ready},
|
||||
},
|
||||
}
|
||||
rep.deriveState()
|
||||
if rep.State != Ready {
|
||||
t.Errorf("state = %s, want ready", rep.State)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReplicaStillComingUpIsNotFailed(t *testing.T) {
|
||||
// Every shard present, not all ready yet: that is Starting, not a shortage.
|
||||
rep := &Replica{
|
||||
ID: "r1", WantShards: 2,
|
||||
Shards: []Shard{
|
||||
{ID: "r1-r0", Rank: 0, State: supervisor.Ready},
|
||||
{ID: "r1-r1", Rank: 1, State: supervisor.Loading},
|
||||
},
|
||||
}
|
||||
rep.deriveState()
|
||||
if rep.State != Starting {
|
||||
t.Errorf("state = %s, want starting", rep.State)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMultiNodeReplicaKnowsHowManyShardsItWants(t *testing.T) {
|
||||
// End to end through placement, because the count has to reach the replica
|
||||
// from the profile rather than from whatever turned up.
|
||||
m := testModel(1, catalog.Profile{
|
||||
Name: "tp8", GPUClass: "h100-80g", TP: 8, Nodes: 2, GPUsPerNode: 4,
|
||||
})
|
||||
h := newHarness(t, m, gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
||||
h.r.Reconcile(context.Background())
|
||||
|
||||
reps := h.r.Replicas()
|
||||
if len(reps) != 1 {
|
||||
t.Fatalf("replicas: %+v", reps)
|
||||
}
|
||||
if reps[0].WantShards != 2 {
|
||||
t.Errorf("WantShards = %d, want 2", reps[0].WantShards)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user