From 3081c6f69b2e622adf2c00b8f046e3defbb61092 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 16 Sep 2026 23:12:44 +0200 Subject: [PATCH] Place on nodes that can actually hold the model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Placement had no idea how much memory a node has or a model needs. A CPU profile has no card to be free, so eligibleNodes accepted every ready node: Qwen2.5-Coder 14B, whose working set is about 24 GB, would have been placed just as happily on one of the 8 GB nodes, which reports itself perfectly healthy right up until the OOM killer arrives. node_selector was yesterday's stopgap and it is the operator restating by hand something the system can know. Agents now report MemTotal, profiles declare min_memory_gb, and placement filters on it, naming what was short — "has 8 GB of memory, needs 24" is actionable where "no candidates were considered" is not. The label selector stays, because "this node is for coding models" is a real thing to want to say that memory does not express. Two deliberate non-exclusions. A profile with no requirement fits anywhere, which is every profile that exists today. And a node that reports no memory is still usable: an agent older than the field says nothing, and letting a control plane upgrade quietly empty half a fleet is a worse failure than the one this prevents. ctl nodes grows a MEM column showing "-" rather than 0 for a node that did not say, because an old agent is not a small machine. Co-Authored-By: Claude Opus 5 (1M context) --- api/gen/llmgw/v1/agent.pb.go | 21 +++++++- api/proto/llmgw/v1/agent.proto | 9 ++++ internal/agent/discovery/discover.go | 3 ++ internal/agent/discovery/gpu.go | 3 ++ internal/agent/discovery/memory.go | 51 ++++++++++++++++++ internal/agent/discovery/memory_test.go | 39 ++++++++++++++ internal/cli/ctl.go | 15 ++++-- internal/control/adminapi/api.go | 11 ++-- internal/control/agentsvc/convert.go | 2 + internal/control/catalog/catalog.go | 15 ++++++ internal/control/inventory/inventory.go | 5 ++ internal/control/placement/placement.go | 12 +++++ internal/control/placement/placement_test.go | 55 ++++++++++++++++++++ 13 files changed, 233 insertions(+), 8 deletions(-) create mode 100644 internal/agent/discovery/memory.go create mode 100644 internal/agent/discovery/memory_test.go diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index a92eaad8..45ebe9d4 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -336,6 +336,14 @@ type Register struct { // — so the new engines fail to allocate, with an error naming the wrong // process (plan §6.5). RunningShards []*RunningShard `protobuf:"bytes,9,rep,name=running_shards,json=runningShards,proto3" json:"running_shards,omitempty"` + // memory_total_mb is the machine's RAM, and 0 means the agent did not say — + // which is what every agent older than this field reports, and is treated as + // no constraint rather than as a node with no memory. + // + // Placement needs it because a CPU profile has no card to be free, so every + // ready node looks like a candidate: a 14B model whose working set is 24 GB + // would be placed just as happily on a node with 8 GB. + MemoryTotalMb int64 `protobuf:"varint,10,opt,name=memory_total_mb,json=memoryTotalMb,proto3" json:"memory_total_mb,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -433,6 +441,13 @@ func (x *Register) GetRunningShards() []*RunningShard { return nil } +func (x *Register) GetMemoryTotalMb() int64 { + if x != nil { + return x.MemoryTotalMb + } + return 0 +} + // RunningShard describes an engine already running on a node. // // It carries enough for the control plane to rebuild the replica it belongs to, @@ -1406,7 +1421,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" + "\x05drain\x18\x03 \x01(\v2\x14.llmgw.v1.DrainShardH\x00R\x05drain\x12!\n" + "\x03ack\x18\x04 \x01(\v2\r.llmgw.v1.AckH\x00R\x03ackB\x05\n" + - "\x03msg\"\xa0\x04\n" + + "\x03msg\"\xc8\x04\n" + "\bRegister\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" + "\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" + @@ -1416,7 +1431,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\afabrics\x18\x06 \x03(\v2\x1a.llmgw.v1.FabricAttachmentR\afabrics\x12#\n" + "\ragent_version\x18\a \x01(\tR\fagentVersion\x12O\n" + "\x0fengine_versions\x18\b \x03(\v2&.llmgw.v1.Register.EngineVersionsEntryR\x0eengineVersions\x12=\n" + - "\x0erunning_shards\x18\t \x03(\v2\x16.llmgw.v1.RunningShardR\rrunningShards\x1a9\n" + + "\x0erunning_shards\x18\t \x03(\v2\x16.llmgw.v1.RunningShardR\rrunningShards\x12&\n" + + "\x0fmemory_total_mb\x18\n" + + " \x01(\x03R\rmemoryTotalMb\x1a9\n" + "\vLabelsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 214e3da7..2e20b610 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -67,6 +67,15 @@ message Register { // — so the new engines fail to allocate, with an error naming the wrong // process (plan §6.5). repeated RunningShard running_shards = 9; + + // memory_total_mb is the machine's RAM, and 0 means the agent did not say — + // which is what every agent older than this field reports, and is treated as + // no constraint rather than as a node with no memory. + // + // Placement needs it because a CPU profile has no card to be free, so every + // ready node looks like a candidate: a 14B model whose working set is 24 GB + // would be placed just as happily on a node with 8 GB. + int64 memory_total_mb = 10; } // RunningShard describes an engine already running on a node. diff --git a/internal/agent/discovery/discover.go b/internal/agent/discovery/discover.go index 06585c88..6ed80269 100644 --- a/internal/agent/discovery/discover.go +++ b/internal/agent/discovery/discover.go @@ -65,6 +65,9 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e return inv, fmt.Errorf("infiniband discovery: %w", err) } inv.HCAs = hcas + // Not fatal when it cannot be read: a node that does not report its memory + // constrains nothing, which is how every node behaved before this existed. + inv.MemoryTotalMB = MemoryTotalMB() return inv, nil } diff --git a/internal/agent/discovery/gpu.go b/internal/agent/discovery/gpu.go index c8b37423..6b4c71f5 100644 --- a/internal/agent/discovery/gpu.go +++ b/internal/agent/discovery/gpu.go @@ -100,6 +100,9 @@ type Inventory struct { // Source names the backend that produced the GPU list, so a consumer can // tell "no GPUs" from "discovery did not work". Source string `json:"source"` + // MemoryTotalMB is the machine's RAM. Zero means unknown, which is what an + // agent older than this field reports, and is treated as no constraint. + MemoryTotalMB int `json:"memory_total_mb,omitempty"` } // Classes returns the distinct GPU classes present, sorted. diff --git a/internal/agent/discovery/memory.go b/internal/agent/discovery/memory.go new file mode 100644 index 00000000..c0d0838d --- /dev/null +++ b/internal/agent/discovery/memory.go @@ -0,0 +1,51 @@ +package discovery + +import ( + "bufio" + "os" + "strconv" + "strings" +) + +// MemoryTotalMB reports the machine's total RAM. +// +// It exists because placement had no idea how much a node could hold. A CPU +// profile has no card to be free, so every ready node looked like a candidate: +// a 14B model whose working set is 24 GB would have been placed just as +// happily on a node with 8 GB, which reports itself perfectly healthy right up +// until the OOM killer arrives. Labels were the stopgap; this is the fact they +// were standing in for. +// +// Read from /proc/meminfo, which is Linux-only — every node is. On anything +// else it returns 0, which callers treat as "unknown" and therefore no +// constraint, exactly as an older agent that does not report it at all. +func MemoryTotalMB() int { return memTotalMBFrom("/proc/meminfo") } + +func memTotalMBFrom(path string) int { + f, err := os.Open(path) + if err != nil { + return 0 + } + defer f.Close() + + sc := bufio.NewScanner(f) + for sc.Scan() { + rest, ok := strings.CutPrefix(sc.Text(), "MemTotal:") + if !ok { + continue + } + fields := strings.Fields(rest) + if len(fields) == 0 { + return 0 + } + kb, err := strconv.Atoi(fields[0]) + if err != nil { + return 0 + } + // MemTotal is what the kernel can hand out, already short of the + // installed total by the firmware's reservations. That is the right + // number for deciding what fits. + return kb / 1024 + } + return 0 +} diff --git a/internal/agent/discovery/memory_test.go b/internal/agent/discovery/memory_test.go new file mode 100644 index 00000000..46e4f125 --- /dev/null +++ b/internal/agent/discovery/memory_test.go @@ -0,0 +1,39 @@ +package discovery + +import ( + "os" + "path/filepath" + "testing" +) + +func TestMemTotalIsReadFromMeminfo(t *testing.T) { + dir := t.TempDir() + p := filepath.Join(dir, "meminfo") + // The real shape, including the fields before and after, so the parser is + // not tested against something tidier than it will see. + if err := os.WriteFile(p, []byte( + "MemTotal: 32876244 kB\nMemFree: 1234 kB\nMemAvailable: 5678 kB\n"), 0o600); err != nil { + t.Fatal(err) + } + if got, want := memTotalMBFrom(p), 32876244/1024; got != want { + t.Errorf("memTotalMBFrom = %d, want %d", got, want) + } +} + +func TestUnreadableMeminfoReportsUnknownRatherThanZeroMemory(t *testing.T) { + // Zero means "did not say" everywhere downstream, and placement treats it + // as no constraint. A node with genuinely no memory is not a case. + if got := memTotalMBFrom(filepath.Join(t.TempDir(), "absent")); got != 0 { + t.Errorf("got %d for a missing file", got) + } + dir := t.TempDir() + p := filepath.Join(dir, "meminfo") + for _, junk := range []string{"", "MemTotal:\n", "MemTotal: not-a-number kB\n", "Other: 1 kB\n"} { + if err := os.WriteFile(p, []byte(junk), 0o600); err != nil { + t.Fatal(err) + } + if got := memTotalMBFrom(p); got != 0 { + t.Errorf("got %d for %q", got, junk) + } + } +} diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index d3c68443..9d3d1363 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -133,10 +133,10 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { } w := tab() - fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN") + fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN") for _, n := range nodes { - fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%d\t%s\n", - n.ID, n.State, n.GPUFree, n.GPUTotal, + fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%d\t%s\n", + n.ID, n.State, n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB), join(n.Classes), join(n.Fabrics), n.Replicas, n.LastSeenAgo) } _ = w.Flush() @@ -156,6 +156,15 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { } } +// memColumn shows a node's memory, distinguishing "did not say" from a number. +// An older agent reports nothing, and that is not the same as a small machine. +func memColumn(gb int) string { + if gb <= 0 { + return "-" + } + return fmt.Sprintf("%dG", gb) +} + func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { return &cobra.Command{ Use: "models", diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 88766e8e..86ead6e9 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -44,8 +44,12 @@ type NodeView struct { State string `json:"state"` Labels map[string]string `json:"labels,omitempty"` - GPUTotal int `json:"gpu_total"` - GPUFree int `json:"gpu_free"` + GPUTotal int `json:"gpu_total"` + GPUFree int `json:"gpu_free"` + // MemoryGB is the machine's RAM, 0 when the agent did not report it. It is + // what decides whether a CPU profile can be placed here, so it belongs + // where someone looks when a model will not schedule. + MemoryGB int `json:"memory_gb,omitempty"` Classes []string `json:"gpu_classes,omitempty"` Fabrics []string `json:"fabrics,omitempty"` @@ -140,7 +144,8 @@ func (a *API) Nodes() []NodeView { ID: n.ID, Hostname: n.Hostname, MgmtAddr: n.MgmtAddr, State: n.State.String(), Labels: n.Labels, GPUTotal: len(n.GPUs), GPUFree: len(inv.FreeGPUs(n.ID)), - Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, + MemoryGB: n.MemoryTotalMB / 1024, + Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Replicas: replicasOn[n.ID], } seen := map[string]struct{}{} diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go index e7d9d688..f124d4a5 100644 --- a/internal/control/agentsvc/convert.go +++ b/internal/control/agentsvc/convert.go @@ -23,6 +23,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node { State: inventory.Ready, AgentVersion: r.GetAgentVersion(), EngineVersions: r.GetEngineVersions(), + MemoryTotalMB: int(r.GetMemoryTotalMb()), } for _, g := range r.GetGpus() { n.GPUs = append(n.GPUs, discovery.GPU{ @@ -61,6 +62,7 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string, r := &llmgwv1.Register{ NodeId: nodeID, Hostname: hostname, MgmtAddr: mgmtAddr, Labels: labels, AgentVersion: agentVersion, + MemoryTotalMb: int64(inv.MemoryTotalMB), } for _, g := range inv.GPUs { r.Gpus = append(r.Gpus, &llmgwv1.GPU{ diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index 74452ce2..d3a55a6a 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -41,6 +41,20 @@ type Profile struct { Args []string `koanf:"args"` Env map[string]string `koanf:"env"` + // MinMemoryGB is the host RAM this profile needs to run. + // + // The number that matters is the model's working set: the weights, plus the + // KV cache for the whole context pool, plus the engine's own overhead. For + // a GPU profile that lives in the cards and this is small; for a CPU + // profile it is the whole thing, and getting it wrong means an OOM on a + // node that reported itself healthy. + // + // Zero means unconstrained, which is every profile that existed before + // this. A node that does not report its memory is not excluded either — + // an older agent must not have its nodes quietly emptied by an upgrade of + // the control plane. + MinMemoryGB int `koanf:"min_memory_gb"` + // NodeSelector restricts this profile to nodes carrying every one of these // labels, with these values. // @@ -307,6 +321,7 @@ func (m Model) PlacementProfiles() []placement.Profile { Nodes: p.Nodes, Priority: p.Priority, NodeSelector: p.NodeSelector, + MinMemoryGB: p.MinMemoryGB, }) } return out diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index 7c5eceed..03bad142 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -75,6 +75,11 @@ type Node struct { MgmtAddr string `json:"mgmt_addr"` Labels map[string]string `json:"labels,omitempty"` + // MemoryTotalMB is the machine's RAM. Zero means the agent did not report + // it, which is treated as no constraint — the same as every node before + // this field existed. + MemoryTotalMB int `json:"memory_total_mb,omitempty"` + GPUs []discovery.GPU `json:"gpus"` // Fabrics is empty on a node without IB. Such a node is its own // single-node island and can never host a multi-node replica. diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index 234667b6..6a52f1ba 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -34,6 +34,8 @@ type Profile struct { Priority int // NodeSelector restricts this profile to nodes carrying these labels. NodeSelector map[string]string + // MinMemoryGB is the host RAM this profile needs. Zero is unconstrained. + MinMemoryGB int } // CPUOnly reports whether this profile needs no GPUs. The catalog only lets a @@ -340,6 +342,16 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje reject("does not carry " + miss) continue } + // A node that does not report its memory is not excluded: an agent + // older than that field would otherwise have its node quietly emptied + // by a control plane upgrade, which is a worse failure than the one + // this prevents. + if p.MinMemoryGB > 0 && n.MemoryTotalMB > 0 { + if have := n.MemoryTotalMB / 1024; have < p.MinMemoryGB { + reject(fmt.Sprintf("has %d GB of memory, needs %d", have, p.MinMemoryGB)) + continue + } + } if p.CPUOnly() { // Every ready node can run it; there is no card to be free. diff --git a/internal/control/placement/placement_test.go b/internal/control/placement/placement_test.go index 35a2e5d6..3b0d9ff5 100644 --- a/internal/control/placement/placement_test.go +++ b/internal/control/placement/placement_test.go @@ -527,3 +527,58 @@ func TestAnEmptySelectorConstrainsNothing(t *testing.T) { t.Fatalf("a profile with no selector was rejected:\n%s", res.Explain()) } } + +func TestAProfileIsNotPlacedOnANodeTooSmallForIt(t *testing.T) { + // The failure this prevents: a CPU profile has no card to be free, so every + // ready node is a candidate. Qwen2.5-Coder 14B has a working set around + // 24 GB and the small nodes here have 8, and the node reports itself + // perfectly healthy right up until the OOM killer arrives. + small := node("small", "", 0) + small.MemoryTotalMB = 8 * 1024 + big := node("big", "", 0) + big.MemoryTotalMB = 32 * 1024 + + s := New(store(small, big)) + res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 24}}, nil) + if res.Placement == nil { + t.Fatalf("nothing was placed:\n%s", res.Explain()) + } + if got := res.Placement.Nodes[0].NodeID; got != "big" { + t.Fatalf("placed on %q, which cannot hold the model", got) + } +} + +func TestTheRejectionSaysHowMuchMemoryIsMissing(t *testing.T) { + small := node("small", "", 0) + small.MemoryTotalMB = 8 * 1024 + + s := New(store(small)) + res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 24}}, nil) + if res.Placement != nil { + t.Fatal("placed on a node with too little memory") + } + if e := res.Explain(); !strings.Contains(e, "8 GB") || !strings.Contains(e, "24") { + t.Errorf("the rejection does not say what was short:\n%s", e) + } +} + +func TestANodeThatDoesNotReportMemoryIsStillUsable(t *testing.T) { + // An agent older than the field reports nothing. Excluding those would let + // a control plane upgrade quietly empty half a fleet, which is worse than + // the failure the check prevents. + old := node("old", "", 0) // MemoryTotalMB left at zero + s := New(store(old)) + if res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 24}}, nil); res.Placement == nil { + t.Fatalf("a node that reports no memory was excluded:\n%s", res.Explain()) + } +} + +func TestAProfileWithNoMemoryRequirementFitsAnywhere(t *testing.T) { + // Every profile that exists today. + small := node("small", "", 0) + small.MemoryTotalMB = 1024 + s := New(store(small)) + if res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp"}}, nil); res.Placement == nil { + t.Fatalf("an unconstrained profile was rejected:\n%s", res.Explain()) + } +}