package inventory import ( "slices" "sync" "testing" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" ) func gpus(n int) []discovery.GPU { out := make([]discovery.GPU, n) for i := range out { out[i] = discovery.GPU{Index: i, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559} } return out } func TestUpsertPreservesCordon(t *testing.T) { // An agent reconnecting must not undo an operator's decision: a cordon is // usually in place because something is wrong with the node. s := NewStore() s.Upsert(Node{ID: "n1", GPUs: gpus(8)}) s.SetState("n1", Cordoned) s.Upsert(Node{ID: "n1", GPUs: gpus(8)}) // re-registration n, _ := s.Get("n1") if n.State != Cordoned { t.Errorf("state=%s; re-registration cleared the cordon", n.State) } } func TestUpsertUpdatesHardware(t *testing.T) { s := NewStore() s.Upsert(Node{ID: "n1", GPUs: gpus(4)}) s.Upsert(Node{ID: "n1", GPUs: gpus(8)}) n, _ := s.Get("n1") if len(n.GPUs) != 8 { t.Errorf("hardware was not updated: %d GPUs", len(n.GPUs)) } } func TestTouchRevivesUnreachableNode(t *testing.T) { s := NewStore() s.Upsert(Node{ID: "n1"}) s.SetState("n1", Unreachable) s.Touch("n1") if n, _ := s.Get("n1"); n.State != Ready { t.Errorf("a heartbeat did not revive the node: %s", n.State) } } func TestTouchDoesNotClearCordon(t *testing.T) { s := NewStore() s.Upsert(Node{ID: "n1"}) s.SetState("n1", Cordoned) s.Touch("n1") if n, _ := s.Get("n1"); n.State != Cordoned { t.Errorf("a heartbeat cleared the cordon: %s", n.State) } } func TestFreeGPUs(t *testing.T) { s := NewStore() s.Upsert(Node{ID: "n1", GPUs: gpus(8)}) if got := s.FreeGPUs("n1"); len(got) != 8 { t.Fatalf("expected 8 free, got %d", len(got)) } s.Allocate("n1", Allocation{ShardID: "s1", ReplicaID: "r1", GPUIndices: []int{0, 1}}) got := s.FreeGPUs("n1") if !slices.Equal(got, []int{2, 3, 4, 5, 6, 7}) { t.Errorf("free=%v", got) } } func TestUnhealthyGPUsAreNotFree(t *testing.T) { s := NewStore() n := Node{ID: "n1", GPUs: gpus(4)} n.GPUs[2].Health.Problems = []string{"ECC"} s.Upsert(n) got := s.FreeGPUs("n1") if slices.Contains(got, 2) { t.Errorf("an unhealthy GPU was reported free: %v", got) } if len(got) != 3 { t.Errorf("free=%v", got) } } func TestReleaseShard(t *testing.T) { s := NewStore() s.Upsert(Node{ID: "n1", GPUs: gpus(8)}) s.Allocate("n1", Allocation{ShardID: "s1", ReplicaID: "r1", GPUIndices: []int{0, 1}}) s.Allocate("n1", Allocation{ShardID: "s2", ReplicaID: "r2", GPUIndices: []int{2, 3}}) s.Release("s1") if got := s.FreeGPUs("n1"); slices.Contains(got, 2) || !slices.Contains(got, 0) { t.Errorf("wrong shard released: free=%v", got) } } func TestReleaseReplicaFreesEveryShard(t *testing.T) { // Replicas fail whole, so they are freed whole: one dead shard makes the // others useless and the control plane rebuilds the replica (plan ยง5.4). s := NewStore() s.Upsert(Node{ID: "n1", GPUs: gpus(8)}) s.Upsert(Node{ID: "n2", GPUs: gpus(8)}) s.Allocate("n1", Allocation{ShardID: "r1-0", ReplicaID: "r1", GPUIndices: []int{0, 1, 2, 3}}) s.Allocate("n2", Allocation{ShardID: "r1-1", ReplicaID: "r1", GPUIndices: []int{0, 1, 2, 3}}) s.ReleaseReplica("r1") for _, id := range []string{"n1", "n2"} { if got := len(s.FreeGPUs(id)); got != 8 { t.Errorf("%s has %d free GPUs; the whole replica should be released", id, got) } if got := s.Allocations(id); len(got) != 0 { t.Errorf("%s still has allocations: %v", id, got) } } } func TestHCAsForSelectsOneFabric(t *testing.T) { // A node on two fabrics must hand out only the requested fabric's // adapters. Mixing them lets NCCL pick one whose peer is unreachable. n := Node{ID: "n1", Fabrics: []FabricAttachment{ {Fabric: "ib-a", HCAs: []string{"mlx5_0", "mlx5_1"}}, {Fabric: "ib-b", HCAs: []string{"mlx5_2", "mlx5_3"}}, }} if got := n.HCAsFor("ib-a"); !slices.Equal(got, []string{"mlx5_0", "mlx5_1"}) { t.Errorf("ib-a HCAs=%v", got) } if got := n.HCAsFor("ib-b"); !slices.Equal(got, []string{"mlx5_2", "mlx5_3"}) { t.Errorf("ib-b HCAs=%v", got) } if got := n.HCAsFor("ib-c"); got != nil { t.Errorf("an unattached fabric returned %v", got) } } func TestHCAsForReturnsACopy(t *testing.T) { // The caller passes these into a shard spec; mutating them must not corrupt // the inventory. n := Node{ID: "n1", Fabrics: []FabricAttachment{{Fabric: "ib-a", HCAs: []string{"mlx5_0"}}}} got := n.HCAsFor("ib-a") got[0] = "tampered" if n.Fabrics[0].HCAs[0] != "mlx5_0" { t.Error("HCAsFor exposed the inventory's own slice") } } func TestFabrics(t *testing.T) { s := NewStore() s.Upsert(Node{ID: "n1", Fabrics: []FabricAttachment{{Fabric: "ib-b"}}}) s.Upsert(Node{ID: "n2", Fabrics: []FabricAttachment{{Fabric: "ib-a"}, {Fabric: "ib-b"}}}) s.Upsert(Node{ID: "n3"}) // no fabric if got := s.Fabrics(); !slices.Equal(got, []string{"ib-a", "ib-b"}) { t.Errorf("fabrics=%v", got) } } func TestNodesSorted(t *testing.T) { s := NewStore() for _, id := range []string{"n3", "n1", "n2"} { s.Upsert(Node{ID: id}) } got := s.Nodes() if len(got) != 3 || got[0].ID != "n1" || got[2].ID != "n3" { t.Errorf("not sorted: %v", got) } } func TestGetReturnsACopy(t *testing.T) { s := NewStore() s.Upsert(Node{ID: "n1", GPUs: gpus(2), Labels: map[string]string{"rack": "r1"}}) n, _ := s.Get("n1") n.GPUs[0].Index = 99 again, _ := s.Get("n1") if again.GPUs[0].Index == 99 { // The GPU slice is shared by the copy; document the boundary rather // than pretending it is deep. t.Log("note: Get returns a shallow copy; slices are shared") } if again.ID != "n1" { t.Error("the node was corrupted") } } func TestRemove(t *testing.T) { s := NewStore() s.Upsert(Node{ID: "n1", GPUs: gpus(4)}) s.Allocate("n1", Allocation{ShardID: "s1", GPUIndices: []int{0}}) s.Remove("n1") if _, ok := s.Get("n1"); ok { t.Error("the node survived removal") } if got := s.Allocations("n1"); len(got) != 0 { t.Error("allocations outlived the node") } } func TestStateStrings(t *testing.T) { for st, want := range map[NodeState]string{ Ready: "ready", Cordoned: "cordoned", Draining: "draining", Unreachable: "unreachable", Lost: "lost", } { if got := st.String(); got != want { t.Errorf("%d -> %q, want %q", int(st), got, want) } } if !Ready.Schedulable() || Cordoned.Schedulable() || Lost.Schedulable() { t.Error("Schedulable is wrong") } } func TestConcurrentAccess(t *testing.T) { s := NewStore() s.Upsert(Node{ID: "n1", GPUs: gpus(8)}) var wg sync.WaitGroup for i := range 100 { wg.Add(1) go func() { defer wg.Done() switch i % 5 { case 0: s.Upsert(Node{ID: "n1", GPUs: gpus(8)}) case 1: s.Allocate("n1", Allocation{ShardID: "s", ReplicaID: "r", GPUIndices: []int{i % 8}}) case 2: s.Release("s") case 3: _ = s.FreeGPUs("n1") default: _ = s.Nodes() } }() } wg.Wait() } func TestSharedFilesystemsAreNotThisNodesSpaceToSpend(t *testing.T) { // Twenty nodes mounting one filer have 500 GB between them, not ten // terabytes. Counting a shared mount per node is how a scheduler places // twenty models into space for one. n := Node{Storage: []discovery.Storage{ {Path: "/mnt/filer", FreeBytes: 500 << 30, TotalBytes: 900 << 30, Shared: true}, {Path: "/var/lib/llmgw-models", FreeBytes: 20 << 30, TotalBytes: 60 << 30}, }} free, total := n.LocalDisk() if free != 20<<30 || total != 60<<30 { t.Errorf("shared space was counted as local: free=%d total=%d", free, total) } } func TestANodeThatReportedNoStorageIsUnknownNotFull(t *testing.T) { // An agent too old to report storage, and a node with no model directories, // both send nothing. Reading that as a full disk would take every such node // out of placement for a reason that is not true. free, total := Node{}.LocalDisk() if free != 0 || total != 0 { t.Errorf("a node with no storage reported %d/%d", free, total) } // The contract callers rely on: total of zero is the signal for "unknown", // which is why nothing may read free alone. if total != 0 { t.Error("callers cannot distinguish unknown from full") } } func TestLocalDiskSumsSeveralDisks(t *testing.T) { n := Node{Storage: []discovery.Storage{ {Path: "/a", FreeBytes: 10, TotalBytes: 100}, {Path: "/b", FreeBytes: 5, TotalBytes: 50}, }} if free, total := n.LocalDisk(); free != 15 || total != 150 { t.Errorf("free=%d total=%d, want 15/150", free, total) } }