diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index a58a3e9f..ece7595b 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -147,6 +147,17 @@ type Allocation struct { ShardID string ReplicaID string GPUIndices []int + // MemoryGB is the host RAM this shard's profile asked for. + // + // Recorded for the same reason the GPU indices are: it was claimed, and + // placement has to stop handing it out twice. Without it a node's memory + // was compared against its total on every decision, so two models each + // declaring most of a machine were both placed on it — and the second one + // is what the kernel kills. + // + // Zero for a profile that declared nothing, which constrains nothing and + // is counted as nothing. + MemoryGB int } // Store holds the cluster inventory. It is safe for concurrent use. diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index 34b14242..60af93b8 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -413,9 +413,18 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje // older than that field would otherwise have its node quietly emptied // by a control plane upgrade, which is a worse failure than the one // this prevents. + // + // What is left, not what the machine has. Comparing against the total + // meant a node with 31 GB accepted two models that each declared 15, + // because each was asked about on its own — and the second one is what + // the kernel kills. GPUs were never handed out twice this way; memory + // was, because nothing recorded that it had been claimed. if p.MinMemoryGB > 0 && n.MemoryTotalMB > 0 { - if have := n.MemoryTotalMB / 1024; have < p.MinMemoryGB { - reject(fmt.Sprintf("has %d GB of memory, needs %d", have, p.MinMemoryGB)) + total := int(n.MemoryTotalMB / 1024) + free := total - s.claimedMemory(n.ID) + if free < p.MinMemoryGB { + reject(fmt.Sprintf("has %d GB of memory with %d free, needs %d", + total, free, p.MinMemoryGB)) continue } } @@ -502,3 +511,18 @@ func (s *Scheduler) pickGPUs(n inventory.Node, free []discovery.GPU, want int) [ slices.Sort(out) return out } + +// claimedMemory is what the shards already on a node asked for. +// +// Declared, not measured. A profile that says it needs 15 GB is treated as +// holding 15 whether the process has touched that much yet or not, which is the +// only figure available when the decision is made — the alternative is placing +// against what a model has grown into so far and being wrong every time one is +// still loading. +func (s *Scheduler) claimedMemory(nodeID string) int { + total := 0 + for _, a := range s.inv.Allocations(nodeID) { + total += a.MemoryGB + } + return total +} diff --git a/internal/control/placement/placement_test.go b/internal/control/placement/placement_test.go index 91dfd2f8..13f7f0e2 100644 --- a/internal/control/placement/placement_test.go +++ b/internal/control/placement/placement_test.go @@ -738,3 +738,62 @@ func TestANodeThatReportsNoEnginesIsStillEligible(t *testing.T) { t.Errorf("an agent that says nothing must not be excluded:\n%s", res.Explain()) } } + +func TestASecondLargeModelDoesNotLandOnAFullNode(t *testing.T) { + // 31 GB with 20 already claimed. Compared against the total it was placed + // anyway: each model was asked about on its own and the machine was asked + // nothing at all, so the second one is what the kernel kills. + big := inventory.Node{ID: "big", State: inventory.Ready, MemoryTotalMB: 31 * 1024} + st := store(big) + st.Allocate("big", inventory.Allocation{ + ShardID: "first-r0", ReplicaID: "first", MemoryGB: 20, + }) + + s := New(st) + res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 15, Nodes: 1}}, nil) + if res.Placement != nil { + t.Fatalf("placed on %s with 11 GB left of 31 and 15 needed", + res.Placement.Nodes[0].NodeID) + } + // Both numbers, or "needs 15" beside a 31 GB machine reads as a bug in the + // scheduler rather than a full node. + if !strings.Contains(res.Explain(), "31 GB of memory with 11 free") { + t.Errorf("the rejection should show what is left:\n%s", res.Explain()) + } +} + +func TestOneLargeModelStillFits(t *testing.T) { + big := inventory.Node{ID: "big", State: inventory.Ready, MemoryTotalMB: 31 * 1024} + s := New(store(big)) + res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 15, Nodes: 1}}, nil) + if res.Placement == nil { + t.Errorf("an empty node should take the first one:\n%s", res.Explain()) + } +} + +func TestTwoSmallModelsStillShareANode(t *testing.T) { + // Accounting for what is claimed must not turn into one model per machine. + big := inventory.Node{ID: "big", State: inventory.Ready, MemoryTotalMB: 31 * 1024} + st := store(big) + st.Allocate("big", inventory.Allocation{ShardID: "a-r0", ReplicaID: "a", MemoryGB: 3}) + + s := New(st) + res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 3, Nodes: 1}}, nil) + if res.Placement == nil { + t.Errorf("3 GB beside 3 GB on a 31 GB node should fit:\n%s", res.Explain()) + } +} + +func TestAProfileThatDeclaresNothingClaimsNothing(t *testing.T) { + // Zero is "nobody said", and counting it as a claim would let one + // undeclared model make a node look full. + big := inventory.Node{ID: "big", State: inventory.Ready, MemoryTotalMB: 31 * 1024} + st := store(big) + st.Allocate("big", inventory.Allocation{ShardID: "a-r0", ReplicaID: "a"}) + + s := New(st) + res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 30, Nodes: 1}}, nil) + if res.Placement == nil { + t.Errorf("an undeclared neighbour should not fill the node:\n%s", res.Explain()) + } +} diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index 3da64c1b..f13859b3 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -837,6 +837,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac }) r.inv.Allocate(a.NodeID, inventory.Allocation{ ShardID: shardID, ReplicaID: id, GPUIndices: a.GPUIndices, + MemoryGB: prof.MinMemoryGB, }) } rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort) @@ -1123,9 +1124,12 @@ func (r *Reconciler) mergeAdopted(rep *Replica, group []AdoptedShard) { GPUIndices: s.GPUIndices, HCAs: s.HCAs, HTTPPort: s.HTTPPort, State: s.State, }) - // Reclaim the GPUs, or the next placement will hand them out twice. + // Reclaim the GPUs and the memory, or the next placement hands them + // out twice. The memory comes from the catalog rather than the agent: + // what a profile asked for is not something a running process reports. r.inv.Allocate(s.NodeID, inventory.Allocation{ ShardID: s.ShardID, ReplicaID: rep.ID, GPUIndices: s.GPUIndices, + MemoryGB: r.profileMemory(rep.Model, rep.Profile), }) // The endpoint comes from the leader, which may be in any of the // groups — the rank 0 node is not necessarily the first to register. @@ -1349,3 +1353,22 @@ func addrOf(n inventory.Node) string { } return n.ID } + +// profileMemory is what a model's profile declared it needs, or zero. +// +// Looked up when a replica is adopted after a restart, because the allocation +// has to carry the same claim it did before — a control plane that forgot every +// claim on restart would place a second large model onto a node already full of +// the first. +func (r *Reconciler) profileMemory(model, profile string) int { + m, ok := r.cat.Load().Resolve(model) + if !ok { + return 0 + } + for _, p := range m.Profiles { + if p.Name == profile { + return p.MinMemoryGB + } + } + return 0 +}