Onboard a node with a script, and find two examples that never worked
New nodes had been joined by hand, four times, by me. There was no procedure, so there was nothing to be wrong — and nothing to be right either. Two scripts now: provision-node.sh makes the VM on Proxmox, onboard-node.sh gives it an identity. They are separate because they are run by different people with different access, and the second must run where the certificate authority is, which is the one part that cannot be delegated. Running them turned up why phase 4 was never verified. FabricAttachment carried no koanf tags, so the `name:` key the shipped example documents bound to nothing and `rate_gbps` was never read at all. An agent configured exactly as instructed refused to start with "fabrics[0]: name is required" — naming the key the operator had just written. Every fabric in this project had been configured in a Go test, where the tags do not matter. The server example did not load either: its models had no source, so it failed validation the moment anything tried to parse it. Both examples are now parsed by a test. A published example that has never been loaded is a guess, and these two had been guesses since they were written. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
a075ff4120
commit
c1ac2f022f
11 files changed
+687
-9
No files matched your search
@@ -142,6 +142,7 @@ func NewRegistry(cfg Config) *Registry {
|
||||
r.Add(&VLLM{cfg: cfg.VLLM})
|
||||
r.Add(&SGLang{cfg: cfg.SGLang})
|
||||
r.Add(&LlamaCpp{cfg: cfg.LlamaCpp})
|
||||
r.Add(&Sim{cfg: cfg.Sim})
|
||||
return r
|
||||
}
|
||||
|
||||
@@ -172,6 +173,7 @@ type Config struct {
|
||||
VLLM VLLMConfig `koanf:"vllm"`
|
||||
SGLang SGLangConfig `koanf:"sglang"`
|
||||
LlamaCpp LlamaCppConfig `koanf:"llamacpp"`
|
||||
Sim SimConfig `koanf:"sim"`
|
||||
}
|
||||
|
||||
// --- shared helpers ---
|
||||
|
||||
@@ -397,14 +397,14 @@ func TestValidateRequiresModelPath(t *testing.T) {
|
||||
|
||||
func TestRegistry(t *testing.T) {
|
||||
r := NewRegistry(Config{})
|
||||
if got := r.Names(); !slices.Equal(got, []string{"llamacpp", "sglang", "vllm"}) {
|
||||
if got := r.Names(); !slices.Equal(got, []string{"llamacpp", "sglang", "sim", "vllm"}) {
|
||||
t.Errorf("registry contents: %v", got)
|
||||
}
|
||||
if _, err := r.Get("vllm"); err != nil {
|
||||
t.Errorf("vllm missing: %v", err)
|
||||
}
|
||||
err := func() error { _, err := r.Get("nope"); return err }()
|
||||
if err == nil || !strings.Contains(err.Error(), "llamacpp, sglang, vllm") {
|
||||
if err == nil || !strings.Contains(err.Error(), "llamacpp, sglang, sim, vllm") {
|
||||
t.Errorf("an unknown engine should list what is available: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package engines
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// SimConfig locates the simulated engine.
|
||||
type SimConfig struct {
|
||||
// Bin is the llmgw binary itself; the engine is one of its subcommands.
|
||||
// Empty means whatever started this agent, which is right almost always.
|
||||
Bin string `koanf:"bin"`
|
||||
}
|
||||
|
||||
// Sim launches llmgw's own simulated engine.
|
||||
//
|
||||
// It is in the production registry on purpose. A cluster can be brought up and
|
||||
// exercised end to end — placement, fabrics, multi-node sequencing, failure and
|
||||
// rebuild — on machines that have no GPUs, which is the only way most of that
|
||||
// path gets tested at all. The multi-node launch sequence went four phases
|
||||
// without ever running, because the only things exercising it were stub
|
||||
// processes that exercised the command line and nothing else.
|
||||
//
|
||||
// Nothing about it pretends to be real: the model name it serves is whatever it
|
||||
// is told, the tokens are invented, and it says so in its own logs.
|
||||
type Sim struct{ cfg SimConfig }
|
||||
|
||||
// Name identifies the engine in the catalog.
|
||||
func (*Sim) Name() string { return "sim" }
|
||||
|
||||
// HealthPath is the simulated engine's readiness endpoint.
|
||||
func (*Sim) HealthPath() string { return "/health" }
|
||||
|
||||
// MetricsPath is its Prometheus endpoint, in vLLM's vocabulary.
|
||||
func (*Sim) MetricsPath() string { return "/metrics" }
|
||||
|
||||
// ServesHTTP reports whether this rank answers the API.
|
||||
//
|
||||
// Only the leader, exactly as vLLM and SGLang behave — which is the behaviour
|
||||
// this engine exists to let us test.
|
||||
func (*Sim) ServesHTTP(s ShardSpec) bool { return s.Leader() }
|
||||
|
||||
// Validate accepts anything the arithmetic allows.
|
||||
//
|
||||
// No compute-capability checks and no quantization rules: there is no hardware
|
||||
// to be wrong about, and inventing refusals would make this a worse stand-in
|
||||
// for the engines it stands in for.
|
||||
func (*Sim) Validate(p Profile, gpus []GPUInfo) error {
|
||||
return validateCommon(p, gpus)
|
||||
}
|
||||
|
||||
// Commands returns the single process that makes up one shard.
|
||||
func (s *Sim) Commands(spec ShardSpec) ([]Command, error) {
|
||||
bin := s.cfg.Bin
|
||||
if bin == "" {
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("locating the llmgw binary for the sim engine: %w", err)
|
||||
}
|
||||
bin = exe
|
||||
}
|
||||
p := spec.Profile
|
||||
|
||||
args := []string{"sim-engine",
|
||||
"--listen", "0.0.0.0:" + strconv.Itoa(spec.HTTPPort),
|
||||
"--model", p.ServedModelName,
|
||||
}
|
||||
if spec.NNodes > 1 {
|
||||
args = append(args,
|
||||
"--nnodes", strconv.Itoa(spec.NNodes),
|
||||
"--node-rank", strconv.Itoa(spec.Rank),
|
||||
"--dist-init-addr", spec.LeaderAddr+":"+strconv.Itoa(spec.LeaderPort),
|
||||
)
|
||||
}
|
||||
args = append(args, p.Args...)
|
||||
|
||||
return []Command{{
|
||||
Name: "sim-engine", Path: bin, Args: args,
|
||||
Env: materializeEnv(baseEnv(spec)),
|
||||
// Workers dial the leader, so the leader has to be listening first.
|
||||
// Both ranks express the same wait because both know the same address.
|
||||
WaitForLeaderPort: spec.NNodes > 1 && !spec.Leader(),
|
||||
}}, nil
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package engines
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func simSpec(rank, nnodes int) ShardSpec {
|
||||
return ShardSpec{
|
||||
ReplicaID: "rep-1", ShardID: "sh-1", Rank: rank, NNodes: nnodes,
|
||||
LeaderAddr: "10.0.0.5", LeaderPort: 20001,
|
||||
GPUIndices: []int{0, 1}, HTTPPort: 20000, MgmtInterface: "eth0",
|
||||
HCAs: []string{"mlx5_0", "mlx5_1"},
|
||||
Profile: Profile{
|
||||
Name: "tp4", Engine: "sim", TP: 4, GPUsPerNode: 2, Nodes: nnodes,
|
||||
ModelPath: "/models/x", ServedModelName: "big-model",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyTheLeaderServes(t *testing.T) {
|
||||
// The behaviour that made replicas fail while they were coming up perfectly
|
||||
// well: health checking a worker's port, which never opens.
|
||||
if !(&Sim{}).ServesHTTP(simSpec(0, 2)) {
|
||||
t.Error("the leader does not serve")
|
||||
}
|
||||
if (&Sim{}).ServesHTTP(simSpec(1, 2)) {
|
||||
t.Error("a worker claims to serve")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMultiNodeShardCarriesItsRendezvous(t *testing.T) {
|
||||
cmds, err := (&Sim{cfg: SimConfig{Bin: "/usr/local/bin/llmgw"}}).Commands(simSpec(1, 2))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cmds) != 1 {
|
||||
t.Fatalf("%d commands", len(cmds))
|
||||
}
|
||||
c := cmds[0]
|
||||
|
||||
if c.Args[0] != "sim-engine" {
|
||||
t.Errorf("the engine is a subcommand of llmgw: %v", c.Args)
|
||||
}
|
||||
for flag, want := range map[string]string{
|
||||
"--nnodes": "2", "--node-rank": "1", "--dist-init-addr": "10.0.0.5:20001",
|
||||
} {
|
||||
if got, _ := flagValue(c.Args, flag); got != want {
|
||||
t.Errorf("%s = %q, want %q", flag, got, want)
|
||||
}
|
||||
}
|
||||
// A worker must wait for the leader's port, or the launch is a race.
|
||||
if !c.WaitForLeaderPort {
|
||||
t.Error("a worker does not wait for the leader")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheLeaderDoesNotWaitForItself(t *testing.T) {
|
||||
cmds, _ := (&Sim{cfg: SimConfig{Bin: "x"}}).Commands(simSpec(0, 2))
|
||||
if cmds[0].WaitForLeaderPort {
|
||||
t.Error("the leader waits for a port it is about to open")
|
||||
}
|
||||
}
|
||||
|
||||
func TestASingleNodeShardCarriesNoRendezvous(t *testing.T) {
|
||||
cmds, _ := (&Sim{cfg: SimConfig{Bin: "x"}}).Commands(simSpec(0, 1))
|
||||
for _, flag := range []string{"--nnodes", "--node-rank", "--dist-init-addr"} {
|
||||
if slices.Contains(cmds[0].Args, flag) {
|
||||
t.Errorf("a single-node shard was given %s", flag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheShardIsStillPinnedToItsFabric(t *testing.T) {
|
||||
// The simulated engine does not use NCCL, but the environment is generated
|
||||
// by the same code that generates it for the ones that do — so a cluster
|
||||
// exercised with this engine is exercising that too.
|
||||
cmds, _ := (&Sim{cfg: SimConfig{Bin: "x"}}).Commands(simSpec(1, 2))
|
||||
env := cmds[0].Env
|
||||
for _, want := range []string{
|
||||
"NCCL_IB_HCA=mlx5_0,mlx5_1", "NCCL_SOCKET_IFNAME=eth0",
|
||||
"CUDA_VISIBLE_DEVICES=0,1", "NCCL_IB_DISABLE=0",
|
||||
} {
|
||||
if !slices.Contains(env, want) {
|
||||
t.Errorf("the environment is missing %s", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user