Stop handing the same memory to two models
Placement compared a profile's memory requirement against the node's total, so a 31 GB machine accepted two models that each declared 15: each was asked about on its own, and nothing recorded that the first had claimed anything. GPUs were never handed out twice this way — they are tracked as claimed indices — and memory was, because the allocation carried no figure for it. An allocation records what its profile asked for, and eligibility subtracts what a node already owes before comparing. The rejection names both numbers, because "needs 15" beside a 31 GB machine reads as a bug in the scheduler rather than a full node. Declared, not measured. A profile that says 15 is treated as holding 15 whether the process has touched that much yet or not — the alternative is deciding against what a model has grown into so far, which is wrong every time one is still loading. Zero is nobody said, and claims nothing. Counting it would let one undeclared model make a whole node look full. Adoption after a restart re-reads the figure from the catalog rather than from the agent: what a profile asked for is not something a running process reports, and a control plane that forgot every claim on restart would put a second large model onto a node already full of the first. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d998ea7ef5
commit
8fd4c1392d
4 files changed
+120
-3
No files matched your search
@@ -147,6 +147,17 @@ type Allocation struct {
|
||||
ShardID string
|
||||
ReplicaID string
|
||||
GPUIndices []int
|
||||
// MemoryGB is the host RAM this shard's profile asked for.
|
||||
//
|
||||
// Recorded for the same reason the GPU indices are: it was claimed, and
|
||||
// placement has to stop handing it out twice. Without it a node's memory
|
||||
// was compared against its total on every decision, so two models each
|
||||
// declaring most of a machine were both placed on it — and the second one
|
||||
// is what the kernel kills.
|
||||
//
|
||||
// Zero for a profile that declared nothing, which constrains nothing and
|
||||
// is counted as nothing.
|
||||
MemoryGB int
|
||||
}
|
||||
|
||||
// Store holds the cluster inventory. It is safe for concurrent use.
|
||||
|
||||
@@ -413,9 +413,18 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
|
||||
// older than that field would otherwise have its node quietly emptied
|
||||
// by a control plane upgrade, which is a worse failure than the one
|
||||
// this prevents.
|
||||
//
|
||||
// What is left, not what the machine has. Comparing against the total
|
||||
// meant a node with 31 GB accepted two models that each declared 15,
|
||||
// because each was asked about on its own — and the second one is what
|
||||
// the kernel kills. GPUs were never handed out twice this way; memory
|
||||
// was, because nothing recorded that it had been claimed.
|
||||
if p.MinMemoryGB > 0 && n.MemoryTotalMB > 0 {
|
||||
if have := n.MemoryTotalMB / 1024; have < p.MinMemoryGB {
|
||||
reject(fmt.Sprintf("has %d GB of memory, needs %d", have, p.MinMemoryGB))
|
||||
total := int(n.MemoryTotalMB / 1024)
|
||||
free := total - s.claimedMemory(n.ID)
|
||||
if free < p.MinMemoryGB {
|
||||
reject(fmt.Sprintf("has %d GB of memory with %d free, needs %d",
|
||||
total, free, p.MinMemoryGB))
|
||||
continue
|
||||
}
|
||||
}
|
||||
@@ -502,3 +511,18 @@ func (s *Scheduler) pickGPUs(n inventory.Node, free []discovery.GPU, want int) [
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// claimedMemory is what the shards already on a node asked for.
|
||||
//
|
||||
// Declared, not measured. A profile that says it needs 15 GB is treated as
|
||||
// holding 15 whether the process has touched that much yet or not, which is the
|
||||
// only figure available when the decision is made — the alternative is placing
|
||||
// against what a model has grown into so far and being wrong every time one is
|
||||
// still loading.
|
||||
func (s *Scheduler) claimedMemory(nodeID string) int {
|
||||
total := 0
|
||||
for _, a := range s.inv.Allocations(nodeID) {
|
||||
total += a.MemoryGB
|
||||
}
|
||||
return total
|
||||
}
|
||||
@@ -738,3 +738,62 @@ func TestANodeThatReportsNoEnginesIsStillEligible(t *testing.T) {
|
||||
t.Errorf("an agent that says nothing must not be excluded:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestASecondLargeModelDoesNotLandOnAFullNode(t *testing.T) {
|
||||
// 31 GB with 20 already claimed. Compared against the total it was placed
|
||||
// anyway: each model was asked about on its own and the machine was asked
|
||||
// nothing at all, so the second one is what the kernel kills.
|
||||
big := inventory.Node{ID: "big", State: inventory.Ready, MemoryTotalMB: 31 * 1024}
|
||||
st := store(big)
|
||||
st.Allocate("big", inventory.Allocation{
|
||||
ShardID: "first-r0", ReplicaID: "first", MemoryGB: 20,
|
||||
})
|
||||
|
||||
s := New(st)
|
||||
res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 15, Nodes: 1}}, nil)
|
||||
if res.Placement != nil {
|
||||
t.Fatalf("placed on %s with 11 GB left of 31 and 15 needed",
|
||||
res.Placement.Nodes[0].NodeID)
|
||||
}
|
||||
// Both numbers, or "needs 15" beside a 31 GB machine reads as a bug in the
|
||||
// scheduler rather than a full node.
|
||||
if !strings.Contains(res.Explain(), "31 GB of memory with 11 free") {
|
||||
t.Errorf("the rejection should show what is left:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneLargeModelStillFits(t *testing.T) {
|
||||
big := inventory.Node{ID: "big", State: inventory.Ready, MemoryTotalMB: 31 * 1024}
|
||||
s := New(store(big))
|
||||
res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 15, Nodes: 1}}, nil)
|
||||
if res.Placement == nil {
|
||||
t.Errorf("an empty node should take the first one:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoSmallModelsStillShareANode(t *testing.T) {
|
||||
// Accounting for what is claimed must not turn into one model per machine.
|
||||
big := inventory.Node{ID: "big", State: inventory.Ready, MemoryTotalMB: 31 * 1024}
|
||||
st := store(big)
|
||||
st.Allocate("big", inventory.Allocation{ShardID: "a-r0", ReplicaID: "a", MemoryGB: 3})
|
||||
|
||||
s := New(st)
|
||||
res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 3, Nodes: 1}}, nil)
|
||||
if res.Placement == nil {
|
||||
t.Errorf("3 GB beside 3 GB on a 31 GB node should fit:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProfileThatDeclaresNothingClaimsNothing(t *testing.T) {
|
||||
// Zero is "nobody said", and counting it as a claim would let one
|
||||
// undeclared model make a node look full.
|
||||
big := inventory.Node{ID: "big", State: inventory.Ready, MemoryTotalMB: 31 * 1024}
|
||||
st := store(big)
|
||||
st.Allocate("big", inventory.Allocation{ShardID: "a-r0", ReplicaID: "a"})
|
||||
|
||||
s := New(st)
|
||||
res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 30, Nodes: 1}}, nil)
|
||||
if res.Placement == nil {
|
||||
t.Errorf("an undeclared neighbour should not fill the node:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
@@ -837,6 +837,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
|
||||
})
|
||||
r.inv.Allocate(a.NodeID, inventory.Allocation{
|
||||
ShardID: shardID, ReplicaID: id, GPUIndices: a.GPUIndices,
|
||||
MemoryGB: prof.MinMemoryGB,
|
||||
})
|
||||
}
|
||||
rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort)
|
||||
@@ -1123,9 +1124,12 @@ func (r *Reconciler) mergeAdopted(rep *Replica, group []AdoptedShard) {
|
||||
GPUIndices: s.GPUIndices, HCAs: s.HCAs,
|
||||
HTTPPort: s.HTTPPort, State: s.State,
|
||||
})
|
||||
// Reclaim the GPUs, or the next placement will hand them out twice.
|
||||
// Reclaim the GPUs and the memory, or the next placement hands them
|
||||
// out twice. The memory comes from the catalog rather than the agent:
|
||||
// what a profile asked for is not something a running process reports.
|
||||
r.inv.Allocate(s.NodeID, inventory.Allocation{
|
||||
ShardID: s.ShardID, ReplicaID: rep.ID, GPUIndices: s.GPUIndices,
|
||||
MemoryGB: r.profileMemory(rep.Model, rep.Profile),
|
||||
})
|
||||
// The endpoint comes from the leader, which may be in any of the
|
||||
// groups — the rank 0 node is not necessarily the first to register.
|
||||
@@ -1349,3 +1353,22 @@ func addrOf(n inventory.Node) string {
|
||||
}
|
||||
return n.ID
|
||||
}
|
||||
|
||||
// profileMemory is what a model's profile declared it needs, or zero.
|
||||
//
|
||||
// Looked up when a replica is adopted after a restart, because the allocation
|
||||
// has to carry the same claim it did before — a control plane that forgot every
|
||||
// claim on restart would place a second large model onto a node already full of
|
||||
// the first.
|
||||
func (r *Reconciler) profileMemory(model, profile string) int {
|
||||
m, ok := r.cat.Load().Resolve(model)
|
||||
if !ok {
|
||||
return 0
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
if p.Name == profile {
|
||||
return p.MinMemoryGB
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
Reference in new issue
Block a user