Placement had no idea how much memory a node has or a model needs. A CPU profile has no card to be free, so eligibleNodes accepted every ready node: Qwen2.5-Coder 14B, whose working set is about 24 GB, would have been placed just as happily on one of the 8 GB nodes, which reports itself perfectly healthy right up until the OOM killer arrives. node_selector was yesterday's stopgap and it is the operator restating by hand something the system can know. Agents now report MemTotal, profiles declare min_memory_gb, and placement filters on it, naming what was short — "has 8 GB of memory, needs 24" is actionable where "no candidates were considered" is not. The label selector stays, because "this node is for coding models" is a real thing to want to say that memory does not express. Two deliberate non-exclusions. A profile with no requirement fits anywhere, which is every profile that exists today. And a node that reports no memory is still usable: an agent older than the field says nothing, and letting a control plane upgrade quietly empty half a fleet is a worse failure than the one this prevents. ctl nodes grows a MEM column showing "-" rather than 0 for a node that did not say, because an old agent is not a small machine. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
e9fe3ad379
commit
3081c6f69b
13 files changed
+233
-8
No files matched your search
@@ -336,6 +336,14 @@ type Register struct {
|
||||
// — so the new engines fail to allocate, with an error naming the wrong
|
||||
// process (plan §6.5).
|
||||
RunningShards []*RunningShard `protobuf:"bytes,9,rep,name=running_shards,json=runningShards,proto3" json:"running_shards,omitempty"`
|
||||
// memory_total_mb is the machine's RAM, and 0 means the agent did not say —
|
||||
// which is what every agent older than this field reports, and is treated as
|
||||
// no constraint rather than as a node with no memory.
|
||||
//
|
||||
// Placement needs it because a CPU profile has no card to be free, so every
|
||||
// ready node looks like a candidate: a 14B model whose working set is 24 GB
|
||||
// would be placed just as happily on a node with 8 GB.
|
||||
MemoryTotalMb int64 `protobuf:"varint,10,opt,name=memory_total_mb,json=memoryTotalMb,proto3" json:"memory_total_mb,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -433,6 +441,13 @@ func (x *Register) GetRunningShards() []*RunningShard {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *Register) GetMemoryTotalMb() int64 {
|
||||
if x != nil {
|
||||
return x.MemoryTotalMb
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// RunningShard describes an engine already running on a node.
|
||||
//
|
||||
// It carries enough for the control plane to rebuild the replica it belongs to,
|
||||
@@ -1406,7 +1421,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" +
|
||||
"\x05drain\x18\x03 \x01(\v2\x14.llmgw.v1.DrainShardH\x00R\x05drain\x12!\n" +
|
||||
"\x03ack\x18\x04 \x01(\v2\r.llmgw.v1.AckH\x00R\x03ackB\x05\n" +
|
||||
"\x03msg\"\xa0\x04\n" +
|
||||
"\x03msg\"\xc8\x04\n" +
|
||||
"\bRegister\x12\x17\n" +
|
||||
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" +
|
||||
"\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" +
|
||||
@@ -1416,7 +1431,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\afabrics\x18\x06 \x03(\v2\x1a.llmgw.v1.FabricAttachmentR\afabrics\x12#\n" +
|
||||
"\ragent_version\x18\a \x01(\tR\fagentVersion\x12O\n" +
|
||||
"\x0fengine_versions\x18\b \x03(\v2&.llmgw.v1.Register.EngineVersionsEntryR\x0eengineVersions\x12=\n" +
|
||||
"\x0erunning_shards\x18\t \x03(\v2\x16.llmgw.v1.RunningShardR\rrunningShards\x1a9\n" +
|
||||
"\x0erunning_shards\x18\t \x03(\v2\x16.llmgw.v1.RunningShardR\rrunningShards\x12&\n" +
|
||||
"\x0fmemory_total_mb\x18\n" +
|
||||
" \x01(\x03R\rmemoryTotalMb\x1a9\n" +
|
||||
"\vLabelsEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" +
|
||||
|
||||
@@ -67,6 +67,15 @@ message Register {
|
||||
// — so the new engines fail to allocate, with an error naming the wrong
|
||||
// process (plan §6.5).
|
||||
repeated RunningShard running_shards = 9;
|
||||
|
||||
// memory_total_mb is the machine's RAM, and 0 means the agent did not say —
|
||||
// which is what every agent older than this field reports, and is treated as
|
||||
// no constraint rather than as a node with no memory.
|
||||
//
|
||||
// Placement needs it because a CPU profile has no card to be free, so every
|
||||
// ready node looks like a candidate: a 14B model whose working set is 24 GB
|
||||
// would be placed just as happily on a node with 8 GB.
|
||||
int64 memory_total_mb = 10;
|
||||
}
|
||||
|
||||
// RunningShard describes an engine already running on a node.
|
||||
|
||||
@@ -65,6 +65,9 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
|
||||
return inv, fmt.Errorf("infiniband discovery: %w", err)
|
||||
}
|
||||
inv.HCAs = hcas
|
||||
// Not fatal when it cannot be read: a node that does not report its memory
|
||||
// constrains nothing, which is how every node behaved before this existed.
|
||||
inv.MemoryTotalMB = MemoryTotalMB()
|
||||
return inv, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -100,6 +100,9 @@ type Inventory struct {
|
||||
// Source names the backend that produced the GPU list, so a consumer can
|
||||
// tell "no GPUs" from "discovery did not work".
|
||||
Source string `json:"source"`
|
||||
// MemoryTotalMB is the machine's RAM. Zero means unknown, which is what an
|
||||
// agent older than this field reports, and is treated as no constraint.
|
||||
MemoryTotalMB int `json:"memory_total_mb,omitempty"`
|
||||
}
|
||||
|
||||
// Classes returns the distinct GPU classes present, sorted.
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// MemoryTotalMB reports the machine's total RAM.
|
||||
//
|
||||
// It exists because placement had no idea how much a node could hold. A CPU
|
||||
// profile has no card to be free, so every ready node looked like a candidate:
|
||||
// a 14B model whose working set is 24 GB would have been placed just as
|
||||
// happily on a node with 8 GB, which reports itself perfectly healthy right up
|
||||
// until the OOM killer arrives. Labels were the stopgap; this is the fact they
|
||||
// were standing in for.
|
||||
//
|
||||
// Read from /proc/meminfo, which is Linux-only — every node is. On anything
|
||||
// else it returns 0, which callers treat as "unknown" and therefore no
|
||||
// constraint, exactly as an older agent that does not report it at all.
|
||||
func MemoryTotalMB() int { return memTotalMBFrom("/proc/meminfo") }
|
||||
|
||||
func memTotalMBFrom(path string) int {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
sc := bufio.NewScanner(f)
|
||||
for sc.Scan() {
|
||||
rest, ok := strings.CutPrefix(sc.Text(), "MemTotal:")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(rest)
|
||||
if len(fields) == 0 {
|
||||
return 0
|
||||
}
|
||||
kb, err := strconv.Atoi(fields[0])
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
// MemTotal is what the kernel can hand out, already short of the
|
||||
// installed total by the firmware's reservations. That is the right
|
||||
// number for deciding what fits.
|
||||
return kb / 1024
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMemTotalIsReadFromMeminfo(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p := filepath.Join(dir, "meminfo")
|
||||
// The real shape, including the fields before and after, so the parser is
|
||||
// not tested against something tidier than it will see.
|
||||
if err := os.WriteFile(p, []byte(
|
||||
"MemTotal: 32876244 kB\nMemFree: 1234 kB\nMemAvailable: 5678 kB\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := memTotalMBFrom(p), 32876244/1024; got != want {
|
||||
t.Errorf("memTotalMBFrom = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnreadableMeminfoReportsUnknownRatherThanZeroMemory(t *testing.T) {
|
||||
// Zero means "did not say" everywhere downstream, and placement treats it
|
||||
// as no constraint. A node with genuinely no memory is not a case.
|
||||
if got := memTotalMBFrom(filepath.Join(t.TempDir(), "absent")); got != 0 {
|
||||
t.Errorf("got %d for a missing file", got)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
p := filepath.Join(dir, "meminfo")
|
||||
for _, junk := range []string{"", "MemTotal:\n", "MemTotal: not-a-number kB\n", "Other: 1 kB\n"} {
|
||||
if err := os.WriteFile(p, []byte(junk), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := memTotalMBFrom(p); got != 0 {
|
||||
t.Errorf("got %d for %q", got, junk)
|
||||
}
|
||||
}
|
||||
}
|
||||
+12
-3
@@ -133,10 +133,10 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
}
|
||||
|
||||
w := tab()
|
||||
fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN")
|
||||
fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN")
|
||||
for _, n := range nodes {
|
||||
fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%d\t%s\n",
|
||||
n.ID, n.State, n.GPUFree, n.GPUTotal,
|
||||
fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%d\t%s\n",
|
||||
n.ID, n.State, n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB),
|
||||
join(n.Classes), join(n.Fabrics), n.Replicas, n.LastSeenAgo)
|
||||
}
|
||||
_ = w.Flush()
|
||||
@@ -156,6 +156,15 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
}
|
||||
}
|
||||
|
||||
// memColumn shows a node's memory, distinguishing "did not say" from a number.
|
||||
// An older agent reports nothing, and that is not the same as a small machine.
|
||||
func memColumn(gb int) string {
|
||||
if gb <= 0 {
|
||||
return "-"
|
||||
}
|
||||
return fmt.Sprintf("%dG", gb)
|
||||
}
|
||||
|
||||
func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "models",
|
||||
|
||||
@@ -44,8 +44,12 @@ type NodeView struct {
|
||||
State string `json:"state"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
|
||||
GPUTotal int `json:"gpu_total"`
|
||||
GPUFree int `json:"gpu_free"`
|
||||
GPUTotal int `json:"gpu_total"`
|
||||
GPUFree int `json:"gpu_free"`
|
||||
// MemoryGB is the machine's RAM, 0 when the agent did not report it. It is
|
||||
// what decides whether a CPU profile can be placed here, so it belongs
|
||||
// where someone looks when a model will not schedule.
|
||||
MemoryGB int `json:"memory_gb,omitempty"`
|
||||
Classes []string `json:"gpu_classes,omitempty"`
|
||||
Fabrics []string `json:"fabrics,omitempty"`
|
||||
|
||||
@@ -140,7 +144,8 @@ func (a *API) Nodes() []NodeView {
|
||||
ID: n.ID, Hostname: n.Hostname, MgmtAddr: n.MgmtAddr,
|
||||
State: n.State.String(), Labels: n.Labels,
|
||||
GPUTotal: len(n.GPUs), GPUFree: len(inv.FreeGPUs(n.ID)),
|
||||
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion,
|
||||
MemoryGB: n.MemoryTotalMB / 1024,
|
||||
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion,
|
||||
Replicas: replicasOn[n.ID],
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
|
||||
@@ -23,6 +23,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
|
||||
State: inventory.Ready,
|
||||
AgentVersion: r.GetAgentVersion(),
|
||||
EngineVersions: r.GetEngineVersions(),
|
||||
MemoryTotalMB: int(r.GetMemoryTotalMb()),
|
||||
}
|
||||
for _, g := range r.GetGpus() {
|
||||
n.GPUs = append(n.GPUs, discovery.GPU{
|
||||
@@ -61,6 +62,7 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
|
||||
r := &llmgwv1.Register{
|
||||
NodeId: nodeID, Hostname: hostname, MgmtAddr: mgmtAddr,
|
||||
Labels: labels, AgentVersion: agentVersion,
|
||||
MemoryTotalMb: int64(inv.MemoryTotalMB),
|
||||
}
|
||||
for _, g := range inv.GPUs {
|
||||
r.Gpus = append(r.Gpus, &llmgwv1.GPU{
|
||||
|
||||
@@ -41,6 +41,20 @@ type Profile struct {
|
||||
Args []string `koanf:"args"`
|
||||
Env map[string]string `koanf:"env"`
|
||||
|
||||
// MinMemoryGB is the host RAM this profile needs to run.
|
||||
//
|
||||
// The number that matters is the model's working set: the weights, plus the
|
||||
// KV cache for the whole context pool, plus the engine's own overhead. For
|
||||
// a GPU profile that lives in the cards and this is small; for a CPU
|
||||
// profile it is the whole thing, and getting it wrong means an OOM on a
|
||||
// node that reported itself healthy.
|
||||
//
|
||||
// Zero means unconstrained, which is every profile that existed before
|
||||
// this. A node that does not report its memory is not excluded either —
|
||||
// an older agent must not have its nodes quietly emptied by an upgrade of
|
||||
// the control plane.
|
||||
MinMemoryGB int `koanf:"min_memory_gb"`
|
||||
|
||||
// NodeSelector restricts this profile to nodes carrying every one of these
|
||||
// labels, with these values.
|
||||
//
|
||||
@@ -307,6 +321,7 @@ func (m Model) PlacementProfiles() []placement.Profile {
|
||||
Nodes: p.Nodes,
|
||||
Priority: p.Priority,
|
||||
NodeSelector: p.NodeSelector,
|
||||
MinMemoryGB: p.MinMemoryGB,
|
||||
})
|
||||
}
|
||||
return out
|
||||
|
||||
@@ -75,6 +75,11 @@ type Node struct {
|
||||
MgmtAddr string `json:"mgmt_addr"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
|
||||
// MemoryTotalMB is the machine's RAM. Zero means the agent did not report
|
||||
// it, which is treated as no constraint — the same as every node before
|
||||
// this field existed.
|
||||
MemoryTotalMB int `json:"memory_total_mb,omitempty"`
|
||||
|
||||
GPUs []discovery.GPU `json:"gpus"`
|
||||
// Fabrics is empty on a node without IB. Such a node is its own
|
||||
// single-node island and can never host a multi-node replica.
|
||||
|
||||
@@ -34,6 +34,8 @@ type Profile struct {
|
||||
Priority int
|
||||
// NodeSelector restricts this profile to nodes carrying these labels.
|
||||
NodeSelector map[string]string
|
||||
// MinMemoryGB is the host RAM this profile needs. Zero is unconstrained.
|
||||
MinMemoryGB int
|
||||
}
|
||||
|
||||
// CPUOnly reports whether this profile needs no GPUs. The catalog only lets a
|
||||
@@ -340,6 +342,16 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
|
||||
reject("does not carry " + miss)
|
||||
continue
|
||||
}
|
||||
// A node that does not report its memory is not excluded: an agent
|
||||
// older than that field would otherwise have its node quietly emptied
|
||||
// by a control plane upgrade, which is a worse failure than the one
|
||||
// this prevents.
|
||||
if p.MinMemoryGB > 0 && n.MemoryTotalMB > 0 {
|
||||
if have := n.MemoryTotalMB / 1024; have < p.MinMemoryGB {
|
||||
reject(fmt.Sprintf("has %d GB of memory, needs %d", have, p.MinMemoryGB))
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if p.CPUOnly() {
|
||||
// Every ready node can run it; there is no card to be free.
|
||||
|
||||
@@ -527,3 +527,58 @@ func TestAnEmptySelectorConstrainsNothing(t *testing.T) {
|
||||
t.Fatalf("a profile with no selector was rejected:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProfileIsNotPlacedOnANodeTooSmallForIt(t *testing.T) {
|
||||
// The failure this prevents: a CPU profile has no card to be free, so every
|
||||
// ready node is a candidate. Qwen2.5-Coder 14B has a working set around
|
||||
// 24 GB and the small nodes here have 8, and the node reports itself
|
||||
// perfectly healthy right up until the OOM killer arrives.
|
||||
small := node("small", "", 0)
|
||||
small.MemoryTotalMB = 8 * 1024
|
||||
big := node("big", "", 0)
|
||||
big.MemoryTotalMB = 32 * 1024
|
||||
|
||||
s := New(store(small, big))
|
||||
res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 24}}, nil)
|
||||
if res.Placement == nil {
|
||||
t.Fatalf("nothing was placed:\n%s", res.Explain())
|
||||
}
|
||||
if got := res.Placement.Nodes[0].NodeID; got != "big" {
|
||||
t.Fatalf("placed on %q, which cannot hold the model", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheRejectionSaysHowMuchMemoryIsMissing(t *testing.T) {
|
||||
small := node("small", "", 0)
|
||||
small.MemoryTotalMB = 8 * 1024
|
||||
|
||||
s := New(store(small))
|
||||
res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 24}}, nil)
|
||||
if res.Placement != nil {
|
||||
t.Fatal("placed on a node with too little memory")
|
||||
}
|
||||
if e := res.Explain(); !strings.Contains(e, "8 GB") || !strings.Contains(e, "24") {
|
||||
t.Errorf("the rejection does not say what was short:\n%s", e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeThatDoesNotReportMemoryIsStillUsable(t *testing.T) {
|
||||
// An agent older than the field reports nothing. Excluding those would let
|
||||
// a control plane upgrade quietly empty half a fleet, which is worse than
|
||||
// the failure the check prevents.
|
||||
old := node("old", "", 0) // MemoryTotalMB left at zero
|
||||
s := New(store(old))
|
||||
if res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 24}}, nil); res.Placement == nil {
|
||||
t.Fatalf("a node that reports no memory was excluded:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProfileWithNoMemoryRequirementFitsAnywhere(t *testing.T) {
|
||||
// Every profile that exists today.
|
||||
small := node("small", "", 0)
|
||||
small.MemoryTotalMB = 1024
|
||||
s := New(store(small))
|
||||
if res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp"}}, nil); res.Placement == nil {
|
||||
t.Fatalf("an unconstrained profile was rejected:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user