package adminapi import ( "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" ) func TestSummaryCountsCapacityByClass(t *testing.T) { // The question a per-node table answers badly at two hundred nodes: how // full am I. a, _ := setup(t, testModels(), gpuNode("n1", 8), gpuNode("n2", 8)) s := a.Summary() if s.Nodes.Total != 2 || s.Nodes.Ready != 2 { t.Errorf("nodes = %+v", s.Nodes) } if len(s.GPUs) != 1 { t.Fatalf("expected one GPU class, got %+v", s.GPUs) } if g := s.GPUs[0]; g.Total != 16 || g.Free() != 16 { t.Errorf("class totals wrong: %+v", g) } } func TestUnhealthyGPUsAreNotCountedAsFree(t *testing.T) { // Counting them free is how a cluster looks like it has room it does not, // which is worse than showing less capacity than exists. n := gpuNode("n1", 4) n.GPUs[0].Health.Problems = []string{"3 uncorrected ECC errors"} a, _ := setup(t, testModels(), n) g := a.Summary().GPUs[0] if g.Unhealthy != 1 { t.Errorf("unhealthy = %d", g.Unhealthy) } if g.Free() != 3 { t.Errorf("free = %d; a faulty card was counted as available", g.Free()) } } func TestFabricFreeBlocksCountWholeNodesOnly(t *testing.T) { // A multi-node replica may never span fabrics, and a node with one card in // use cannot contribute a full-width shard. Free cards scattered across // partly-used nodes are not capacity for the model that needs them. free := gpuNode("free", 8, "ib-a") partial := gpuNode("partial", 8, "ib-a") a, ctl := setup(t, testModels(), free, partial) // Take one GPU on the second node. ctl.Inventory().Allocate("partial", inventory.Allocation{ ShardID: "s", ReplicaID: "r", GPUIndices: []int{0}, }) s := a.Summary() if len(s.Fabrics) != 1 { t.Fatalf("fabrics = %+v", s.Fabrics) } if got := s.Fabrics[0].LargestFreeBlock; got != 1 { t.Errorf("fully free nodes = %d, want 1: a partly used node is not one", got) } } func TestANodeWithNoGPUsDoesNotInventAClass(t *testing.T) { // The CPU-only node this fleet acquired. It has no class, and a summary // that invents one for it would report capacity nobody has. a, _ := setup(t, testModels(), inventory.Node{ID: "cpu1", State: inventory.Ready}) s := a.Summary() for _, g := range s.GPUs { if g.Class == "" { t.Errorf("an empty class was reported: %+v", g) } } if s.Nodes.Total != 1 { t.Errorf("the node was not counted: %+v", s.Nodes) } } func TestIdleReplicasNeedRealIdleness(t *testing.T) { // A replica the router has said nothing about carries -1, and must not be // read as idle since the epoch — otherwise a router restart nominates the // whole fleet for eviction at once. for _, tc := range []struct { name string idle int64 want bool }{ {"never reported", -1, false}, {"just used", 5, false}, {"idle for an hour", int64(time.Hour.Seconds()), true}, } { t.Run(tc.name, func(t *testing.T) { got := tc.idle >= int64(idleThreshold.Seconds()) if got != tc.want { t.Errorf("idle=%d counted as idle=%v, want %v", tc.idle, got, tc.want) } }) } } func TestSummaryNamesModelsThatCannotBePlaced(t *testing.T) { // The line that turns "something is wrong" into a place to look. a, _ := setup(t, testModels()) // no nodes at all s := a.Summary() if len(s.Unplaceable) == 0 { t.Fatal("a model with no hardware to run on was not named") } if s.Models.Short == 0 { t.Error("no model was counted short") } } func TestWeightsAreCountedPerFabric(t *testing.T) { // Fabrics are not connected, so weights in one are not weights in another. // A model seeded into a single fabric leaves the rest of the cluster unable // to serve it without first moving tens of gigabytes over the management // network, and nothing said so. a1 := gpuNode("a1", 4, "ib-a") a1.Weights = []discovery.LocalWeights{{Path: "/models/llama.gguf"}} a2 := gpuNode("a2", 4, "ib-a") a2.Weights = []discovery.LocalWeights{{Path: "/models/llama.gguf"}} b1 := gpuNode("b1", 4, "ib-b") // the same fabric capacity, none of the weights api, _ := setup(t, testModels(), a1, a2, b1) s := api.Summary() byName := map[string]FabricUse{} for _, f := range s.Fabrics { byName[f.Fabric] = f } if got := byName["ib-a"]; got.Models != 1 || got.NodesWithWeights != 2 { t.Errorf("ib-a = %+v, want one model on two nodes", got) } if got := byName["ib-b"]; got.Models != 0 || got.NodesWithWeights != 0 { t.Errorf("ib-b = %+v; it holds nothing and should say so", got) } } func TestOneSharedCopyCountsAsMuchAsTwenty(t *testing.T) { // A filer mounted on every node and a local copy on every node are the same // answer to "can this fabric serve it", which is the question being asked. var nodes []inventory.Node for _, id := range []string{"n1", "n2", "n3"} { n := gpuNode(id, 4, "ib-a") n.Weights = []discovery.LocalWeights{{Path: "/mnt/shared/llama.gguf", Shared: true}} nodes = append(nodes, n) } api, _ := setup(t, testModels(), nodes...) f := api.Summary().Fabrics[0] if f.Models != 1 { t.Errorf("counted %d models; three nodes seeing one file is one model", f.Models) } }