Stop handing the same memory to two models
CI / test (push) Successful in 1m46s

Placement compared a profile's memory requirement against the node's total, so
a 31 GB machine accepted two models that each declared 15: each was asked about
on its own, and nothing recorded that the first had claimed anything. GPUs were
never handed out twice this way — they are tracked as claimed indices — and
memory was, because the allocation carried no figure for it.

An allocation records what its profile asked for, and eligibility subtracts
what a node already owes before comparing. The rejection names both numbers,
because "needs 15" beside a 31 GB machine reads as a bug in the scheduler
rather than a full node.

Declared, not measured. A profile that says 15 is treated as holding 15 whether
the process has touched that much yet or not — the alternative is deciding
against what a model has grown into so far, which is wrong every time one is
still loading.

Zero is nobody said, and claims nothing. Counting it would let one undeclared
model make a whole node look full.

Adoption after a restart re-reads the figure from the catalog rather than from
the agent: what a profile asked for is not something a running process reports,
and a control plane that forgot every claim on restart would put a second large
model onto a node already full of the first.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-27 17:44:26 +02:00
1 parent d998ea7ef5
commit 8fd4c1392d
4 files changed
+120 -3

No files matched your search

+11
View File
@@ -147,6 +147,17 @@ type Allocation struct {
ShardID string
ReplicaID string
GPUIndices []int
// MemoryGB is the host RAM this shard's profile asked for.
//
// Recorded for the same reason the GPU indices are: it was claimed, and
// placement has to stop handing it out twice. Without it a node's memory
// was compared against its total on every decision, so two models each
// declaring most of a machine were both placed on it — and the second one
// is what the kernel kills.
//
// Zero for a profile that declared nothing, which constrains nothing and
// is counted as nothing.
MemoryGB int
}
// Store holds the cluster inventory. It is safe for concurrent use.
+26 -2
View File
@@ -413,9 +413,18 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
// older than that field would otherwise have its node quietly emptied
// by a control plane upgrade, which is a worse failure than the one
// this prevents.
//
// What is left, not what the machine has. Comparing against the total
// meant a node with 31 GB accepted two models that each declared 15,
// because each was asked about on its own — and the second one is what
// the kernel kills. GPUs were never handed out twice this way; memory
// was, because nothing recorded that it had been claimed.
if p.MinMemoryGB > 0 && n.MemoryTotalMB > 0 {
if have := n.MemoryTotalMB / 1024; have < p.MinMemoryGB {
reject(fmt.Sprintf("has %d GB of memory, needs %d", have, p.MinMemoryGB))
total := int(n.MemoryTotalMB / 1024)
free := total - s.claimedMemory(n.ID)
if free < p.MinMemoryGB {
reject(fmt.Sprintf("has %d GB of memory with %d free, needs %d",
total, free, p.MinMemoryGB))
continue
}
}
@@ -502,3 +511,18 @@ func (s *Scheduler) pickGPUs(n inventory.Node, free []discovery.GPU, want int) [
slices.Sort(out)
return out
}
// claimedMemory is what the shards already on a node asked for.
//
// Declared, not measured. A profile that says it needs 15 GB is treated as
// holding 15 whether the process has touched that much yet or not, which is the
// only figure available when the decision is made — the alternative is placing
// against what a model has grown into so far and being wrong every time one is
// still loading.
func (s *Scheduler) claimedMemory(nodeID string) int {
total := 0
for _, a := range s.inv.Allocations(nodeID) {
total += a.MemoryGB
}
return total
}
@@ -738,3 +738,62 @@ func TestANodeThatReportsNoEnginesIsStillEligible(t *testing.T) {
t.Errorf("an agent that says nothing must not be excluded:\n%s", res.Explain())
}
}
func TestASecondLargeModelDoesNotLandOnAFullNode(t *testing.T) {
// 31 GB with 20 already claimed. Compared against the total it was placed
// anyway: each model was asked about on its own and the machine was asked
// nothing at all, so the second one is what the kernel kills.
big := inventory.Node{ID: "big", State: inventory.Ready, MemoryTotalMB: 31 * 1024}
st := store(big)
st.Allocate("big", inventory.Allocation{
ShardID: "first-r0", ReplicaID: "first", MemoryGB: 20,
})
s := New(st)
res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 15, Nodes: 1}}, nil)
if res.Placement != nil {
t.Fatalf("placed on %s with 11 GB left of 31 and 15 needed",
res.Placement.Nodes[0].NodeID)
}
// Both numbers, or "needs 15" beside a 31 GB machine reads as a bug in the
// scheduler rather than a full node.
if !strings.Contains(res.Explain(), "31 GB of memory with 11 free") {
t.Errorf("the rejection should show what is left:\n%s", res.Explain())
}
}
func TestOneLargeModelStillFits(t *testing.T) {
big := inventory.Node{ID: "big", State: inventory.Ready, MemoryTotalMB: 31 * 1024}
s := New(store(big))
res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 15, Nodes: 1}}, nil)
if res.Placement == nil {
t.Errorf("an empty node should take the first one:\n%s", res.Explain())
}
}
func TestTwoSmallModelsStillShareANode(t *testing.T) {
// Accounting for what is claimed must not turn into one model per machine.
big := inventory.Node{ID: "big", State: inventory.Ready, MemoryTotalMB: 31 * 1024}
st := store(big)
st.Allocate("big", inventory.Allocation{ShardID: "a-r0", ReplicaID: "a", MemoryGB: 3})
s := New(st)
res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 3, Nodes: 1}}, nil)
if res.Placement == nil {
t.Errorf("3 GB beside 3 GB on a 31 GB node should fit:\n%s", res.Explain())
}
}
func TestAProfileThatDeclaresNothingClaimsNothing(t *testing.T) {
// Zero is "nobody said", and counting it as a claim would let one
// undeclared model make a node look full.
big := inventory.Node{ID: "big", State: inventory.Ready, MemoryTotalMB: 31 * 1024}
st := store(big)
st.Allocate("big", inventory.Allocation{ShardID: "a-r0", ReplicaID: "a"})
s := New(st)
res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 30, Nodes: 1}}, nil)
if res.Placement == nil {
t.Errorf("an undeclared neighbour should not fill the node:\n%s", res.Explain())
}
}
+24 -1
View File
@@ -837,6 +837,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
})
r.inv.Allocate(a.NodeID, inventory.Allocation{
ShardID: shardID, ReplicaID: id, GPUIndices: a.GPUIndices,
MemoryGB: prof.MinMemoryGB,
})
}
rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort)
@@ -1123,9 +1124,12 @@ func (r *Reconciler) mergeAdopted(rep *Replica, group []AdoptedShard) {
GPUIndices: s.GPUIndices, HCAs: s.HCAs,
HTTPPort: s.HTTPPort, State: s.State,
})
// Reclaim the GPUs, or the next placement will hand them out twice.
// Reclaim the GPUs and the memory, or the next placement hands them
// out twice. The memory comes from the catalog rather than the agent:
// what a profile asked for is not something a running process reports.
r.inv.Allocate(s.NodeID, inventory.Allocation{
ShardID: s.ShardID, ReplicaID: rep.ID, GPUIndices: s.GPUIndices,
MemoryGB: r.profileMemory(rep.Model, rep.Profile),
})
// The endpoint comes from the leader, which may be in any of the
// groups — the rank 0 node is not necessarily the first to register.
@@ -1349,3 +1353,22 @@ func addrOf(n inventory.Node) string {
}
return n.ID
}
// profileMemory is what a model's profile declared it needs, or zero.
//
// Looked up when a replica is adopted after a restart, because the allocation
// has to carry the same claim it did before — a control plane that forgot every
// claim on restart would place a second large model onto a node already full of
// the first.
func (r *Reconciler) profileMemory(model, profile string) int {
m, ok := r.cat.Load().Resolve(model)
if !ok {
return 0
}
for _, p := range m.Profiles {
if p.Name == profile {
return p.MinMemoryGB
}
}
return 0
}