Place on nodes that can actually hold the model
CI / test (push) Successful in 1m23s

Placement had no idea how much memory a node has or a model needs. A CPU
profile has no card to be free, so eligibleNodes accepted every ready node:
Qwen2.5-Coder 14B, whose working set is about 24 GB, would have been placed
just as happily on one of the 8 GB nodes, which reports itself perfectly
healthy right up until the OOM killer arrives.

node_selector was yesterday's stopgap and it is the operator restating by
hand something the system can know. Agents now report MemTotal, profiles
declare min_memory_gb, and placement filters on it, naming what was short —
"has 8 GB of memory, needs 24" is actionable where "no candidates were
considered" is not. The label selector stays, because "this node is for
coding models" is a real thing to want to say that memory does not express.

Two deliberate non-exclusions. A profile with no requirement fits anywhere,
which is every profile that exists today. And a node that reports no memory
is still usable: an agent older than the field says nothing, and letting a
control plane upgrade quietly empty half a fleet is a worse failure than the
one this prevents.

ctl nodes grows a MEM column showing "-" rather than 0 for a node that did
not say, because an old agent is not a small machine.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-16 23:12:44 +02:00
1 parent e9fe3ad379
commit 3081c6f69b
13 files changed
+233 -8

No files matched your search

+19 -2
View File
@@ -336,6 +336,14 @@ type Register struct {
// — so the new engines fail to allocate, with an error naming the wrong
// process (plan §6.5).
RunningShards []*RunningShard `protobuf:"bytes,9,rep,name=running_shards,json=runningShards,proto3" json:"running_shards,omitempty"`
// memory_total_mb is the machine's RAM, and 0 means the agent did not say —
// which is what every agent older than this field reports, and is treated as
// no constraint rather than as a node with no memory.
//
// Placement needs it because a CPU profile has no card to be free, so every
// ready node looks like a candidate: a 14B model whose working set is 24 GB
// would be placed just as happily on a node with 8 GB.
MemoryTotalMb int64 `protobuf:"varint,10,opt,name=memory_total_mb,json=memoryTotalMb,proto3" json:"memory_total_mb,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -433,6 +441,13 @@ func (x *Register) GetRunningShards() []*RunningShard {
return nil
}
func (x *Register) GetMemoryTotalMb() int64 {
if x != nil {
return x.MemoryTotalMb
}
return 0
}
// RunningShard describes an engine already running on a node.
//
// It carries enough for the control plane to rebuild the replica it belongs to,
@@ -1406,7 +1421,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" +
"\x05drain\x18\x03 \x01(\v2\x14.llmgw.v1.DrainShardH\x00R\x05drain\x12!\n" +
"\x03ack\x18\x04 \x01(\v2\r.llmgw.v1.AckH\x00R\x03ackB\x05\n" +
"\x03msg\"\xa0\x04\n" +
"\x03msg\"\xc8\x04\n" +
"\bRegister\x12\x17\n" +
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" +
"\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" +
@@ -1416,7 +1431,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\afabrics\x18\x06 \x03(\v2\x1a.llmgw.v1.FabricAttachmentR\afabrics\x12#\n" +
"\ragent_version\x18\a \x01(\tR\fagentVersion\x12O\n" +
"\x0fengine_versions\x18\b \x03(\v2&.llmgw.v1.Register.EngineVersionsEntryR\x0eengineVersions\x12=\n" +
"\x0erunning_shards\x18\t \x03(\v2\x16.llmgw.v1.RunningShardR\rrunningShards\x1a9\n" +
"\x0erunning_shards\x18\t \x03(\v2\x16.llmgw.v1.RunningShardR\rrunningShards\x12&\n" +
"\x0fmemory_total_mb\x18\n" +
" \x01(\x03R\rmemoryTotalMb\x1a9\n" +
"\vLabelsEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" +
+9
View File
@@ -67,6 +67,15 @@ message Register {
// — so the new engines fail to allocate, with an error naming the wrong
// process (plan §6.5).
repeated RunningShard running_shards = 9;
// memory_total_mb is the machine's RAM, and 0 means the agent did not say —
// which is what every agent older than this field reports, and is treated as
// no constraint rather than as a node with no memory.
//
// Placement needs it because a CPU profile has no card to be free, so every
// ready node looks like a candidate: a 14B model whose working set is 24 GB
// would be placed just as happily on a node with 8 GB.
int64 memory_total_mb = 10;
}
// RunningShard describes an engine already running on a node.
+3
View File
@@ -65,6 +65,9 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
return inv, fmt.Errorf("infiniband discovery: %w", err)
}
inv.HCAs = hcas
// Not fatal when it cannot be read: a node that does not report its memory
// constrains nothing, which is how every node behaved before this existed.
inv.MemoryTotalMB = MemoryTotalMB()
return inv, nil
}
+3
View File
@@ -100,6 +100,9 @@ type Inventory struct {
// Source names the backend that produced the GPU list, so a consumer can
// tell "no GPUs" from "discovery did not work".
Source string `json:"source"`
// MemoryTotalMB is the machine's RAM. Zero means unknown, which is what an
// agent older than this field reports, and is treated as no constraint.
MemoryTotalMB int `json:"memory_total_mb,omitempty"`
}
// Classes returns the distinct GPU classes present, sorted.
+51
View File
@@ -0,0 +1,51 @@
package discovery
import (
"bufio"
"os"
"strconv"
"strings"
)
// MemoryTotalMB reports the machine's total RAM.
//
// It exists because placement had no idea how much a node could hold. A CPU
// profile has no card to be free, so every ready node looked like a candidate:
// a 14B model whose working set is 24 GB would have been placed just as
// happily on a node with 8 GB, which reports itself perfectly healthy right up
// until the OOM killer arrives. Labels were the stopgap; this is the fact they
// were standing in for.
//
// Read from /proc/meminfo, which is Linux-only — every node is. On anything
// else it returns 0, which callers treat as "unknown" and therefore no
// constraint, exactly as an older agent that does not report it at all.
func MemoryTotalMB() int { return memTotalMBFrom("/proc/meminfo") }
func memTotalMBFrom(path string) int {
f, err := os.Open(path)
if err != nil {
return 0
}
defer f.Close()
sc := bufio.NewScanner(f)
for sc.Scan() {
rest, ok := strings.CutPrefix(sc.Text(), "MemTotal:")
if !ok {
continue
}
fields := strings.Fields(rest)
if len(fields) == 0 {
return 0
}
kb, err := strconv.Atoi(fields[0])
if err != nil {
return 0
}
// MemTotal is what the kernel can hand out, already short of the
// installed total by the firmware's reservations. That is the right
// number for deciding what fits.
return kb / 1024
}
return 0
}
+39
View File
@@ -0,0 +1,39 @@
package discovery
import (
"os"
"path/filepath"
"testing"
)
func TestMemTotalIsReadFromMeminfo(t *testing.T) {
dir := t.TempDir()
p := filepath.Join(dir, "meminfo")
// The real shape, including the fields before and after, so the parser is
// not tested against something tidier than it will see.
if err := os.WriteFile(p, []byte(
"MemTotal: 32876244 kB\nMemFree: 1234 kB\nMemAvailable: 5678 kB\n"), 0o600); err != nil {
t.Fatal(err)
}
if got, want := memTotalMBFrom(p), 32876244/1024; got != want {
t.Errorf("memTotalMBFrom = %d, want %d", got, want)
}
}
func TestUnreadableMeminfoReportsUnknownRatherThanZeroMemory(t *testing.T) {
// Zero means "did not say" everywhere downstream, and placement treats it
// as no constraint. A node with genuinely no memory is not a case.
if got := memTotalMBFrom(filepath.Join(t.TempDir(), "absent")); got != 0 {
t.Errorf("got %d for a missing file", got)
}
dir := t.TempDir()
p := filepath.Join(dir, "meminfo")
for _, junk := range []string{"", "MemTotal:\n", "MemTotal: not-a-number kB\n", "Other: 1 kB\n"} {
if err := os.WriteFile(p, []byte(junk), 0o600); err != nil {
t.Fatal(err)
}
if got := memTotalMBFrom(p); got != 0 {
t.Errorf("got %d for %q", got, junk)
}
}
}
+12 -3
View File
@@ -133,10 +133,10 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
}
w := tab()
fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN")
fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN")
for _, n := range nodes {
fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%d\t%s\n",
n.ID, n.State, n.GPUFree, n.GPUTotal,
fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%d\t%s\n",
n.ID, n.State, n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB),
join(n.Classes), join(n.Fabrics), n.Replicas, n.LastSeenAgo)
}
_ = w.Flush()
@@ -156,6 +156,15 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
}
}
// memColumn shows a node's memory, distinguishing "did not say" from a number.
// An older agent reports nothing, and that is not the same as a small machine.
func memColumn(gb int) string {
if gb <= 0 {
return "-"
}
return fmt.Sprintf("%dG", gb)
}
func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
return &cobra.Command{
Use: "models",
+8 -3
View File
@@ -44,8 +44,12 @@ type NodeView struct {
State string `json:"state"`
Labels map[string]string `json:"labels,omitempty"`
GPUTotal int `json:"gpu_total"`
GPUFree int `json:"gpu_free"`
GPUTotal int `json:"gpu_total"`
GPUFree int `json:"gpu_free"`
// MemoryGB is the machine's RAM, 0 when the agent did not report it. It is
// what decides whether a CPU profile can be placed here, so it belongs
// where someone looks when a model will not schedule.
MemoryGB int `json:"memory_gb,omitempty"`
Classes []string `json:"gpu_classes,omitempty"`
Fabrics []string `json:"fabrics,omitempty"`
@@ -140,7 +144,8 @@ func (a *API) Nodes() []NodeView {
ID: n.ID, Hostname: n.Hostname, MgmtAddr: n.MgmtAddr,
State: n.State.String(), Labels: n.Labels,
GPUTotal: len(n.GPUs), GPUFree: len(inv.FreeGPUs(n.ID)),
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion,
MemoryGB: n.MemoryTotalMB / 1024,
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion,
Replicas: replicasOn[n.ID],
}
seen := map[string]struct{}{}
+2
View File
@@ -23,6 +23,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
State: inventory.Ready,
AgentVersion: r.GetAgentVersion(),
EngineVersions: r.GetEngineVersions(),
MemoryTotalMB: int(r.GetMemoryTotalMb()),
}
for _, g := range r.GetGpus() {
n.GPUs = append(n.GPUs, discovery.GPU{
@@ -61,6 +62,7 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
r := &llmgwv1.Register{
NodeId: nodeID, Hostname: hostname, MgmtAddr: mgmtAddr,
Labels: labels, AgentVersion: agentVersion,
MemoryTotalMb: int64(inv.MemoryTotalMB),
}
for _, g := range inv.GPUs {
r.Gpus = append(r.Gpus, &llmgwv1.GPU{
+15
View File
@@ -41,6 +41,20 @@ type Profile struct {
Args []string `koanf:"args"`
Env map[string]string `koanf:"env"`
// MinMemoryGB is the host RAM this profile needs to run.
//
// The number that matters is the model's working set: the weights, plus the
// KV cache for the whole context pool, plus the engine's own overhead. For
// a GPU profile that lives in the cards and this is small; for a CPU
// profile it is the whole thing, and getting it wrong means an OOM on a
// node that reported itself healthy.
//
// Zero means unconstrained, which is every profile that existed before
// this. A node that does not report its memory is not excluded either —
// an older agent must not have its nodes quietly emptied by an upgrade of
// the control plane.
MinMemoryGB int `koanf:"min_memory_gb"`
// NodeSelector restricts this profile to nodes carrying every one of these
// labels, with these values.
//
@@ -307,6 +321,7 @@ func (m Model) PlacementProfiles() []placement.Profile {
Nodes: p.Nodes,
Priority: p.Priority,
NodeSelector: p.NodeSelector,
MinMemoryGB: p.MinMemoryGB,
})
}
return out
+5
View File
@@ -75,6 +75,11 @@ type Node struct {
MgmtAddr string `json:"mgmt_addr"`
Labels map[string]string `json:"labels,omitempty"`
// MemoryTotalMB is the machine's RAM. Zero means the agent did not report
// it, which is treated as no constraint — the same as every node before
// this field existed.
MemoryTotalMB int `json:"memory_total_mb,omitempty"`
GPUs []discovery.GPU `json:"gpus"`
// Fabrics is empty on a node without IB. Such a node is its own
// single-node island and can never host a multi-node replica.
+12
View File
@@ -34,6 +34,8 @@ type Profile struct {
Priority int
// NodeSelector restricts this profile to nodes carrying these labels.
NodeSelector map[string]string
// MinMemoryGB is the host RAM this profile needs. Zero is unconstrained.
MinMemoryGB int
}
// CPUOnly reports whether this profile needs no GPUs. The catalog only lets a
@@ -340,6 +342,16 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
reject("does not carry " + miss)
continue
}
// A node that does not report its memory is not excluded: an agent
// older than that field would otherwise have its node quietly emptied
// by a control plane upgrade, which is a worse failure than the one
// this prevents.
if p.MinMemoryGB > 0 && n.MemoryTotalMB > 0 {
if have := n.MemoryTotalMB / 1024; have < p.MinMemoryGB {
reject(fmt.Sprintf("has %d GB of memory, needs %d", have, p.MinMemoryGB))
continue
}
}
if p.CPUOnly() {
// Every ready node can run it; there is no card to be free.
@@ -527,3 +527,58 @@ func TestAnEmptySelectorConstrainsNothing(t *testing.T) {
t.Fatalf("a profile with no selector was rejected:\n%s", res.Explain())
}
}
func TestAProfileIsNotPlacedOnANodeTooSmallForIt(t *testing.T) {
// The failure this prevents: a CPU profile has no card to be free, so every
// ready node is a candidate. Qwen2.5-Coder 14B has a working set around
// 24 GB and the small nodes here have 8, and the node reports itself
// perfectly healthy right up until the OOM killer arrives.
small := node("small", "", 0)
small.MemoryTotalMB = 8 * 1024
big := node("big", "", 0)
big.MemoryTotalMB = 32 * 1024
s := New(store(small, big))
res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 24}}, nil)
if res.Placement == nil {
t.Fatalf("nothing was placed:\n%s", res.Explain())
}
if got := res.Placement.Nodes[0].NodeID; got != "big" {
t.Fatalf("placed on %q, which cannot hold the model", got)
}
}
func TestTheRejectionSaysHowMuchMemoryIsMissing(t *testing.T) {
small := node("small", "", 0)
small.MemoryTotalMB = 8 * 1024
s := New(store(small))
res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 24}}, nil)
if res.Placement != nil {
t.Fatal("placed on a node with too little memory")
}
if e := res.Explain(); !strings.Contains(e, "8 GB") || !strings.Contains(e, "24") {
t.Errorf("the rejection does not say what was short:\n%s", e)
}
}
func TestANodeThatDoesNotReportMemoryIsStillUsable(t *testing.T) {
// An agent older than the field reports nothing. Excluding those would let
// a control plane upgrade quietly empty half a fleet, which is worse than
// the failure the check prevents.
old := node("old", "", 0) // MemoryTotalMB left at zero
s := New(store(old))
if res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 24}}, nil); res.Placement == nil {
t.Fatalf("a node that reports no memory was excluded:\n%s", res.Explain())
}
}
func TestAProfileWithNoMemoryRequirementFitsAnywhere(t *testing.T) {
// Every profile that exists today.
small := node("small", "", 0)
small.MemoryTotalMB = 1024
s := New(store(small))
if res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp"}}, nil); res.Placement == nil {
t.Fatalf("an unconstrained profile was rejected:\n%s", res.Explain())
}
}