Let a profile say which nodes it is for
Agents have reported labels since the beginning and nothing read them. That
was survivable while every CPU profile was a 0.5B model that fits anywhere,
and stops being survivable with a 14B one: a CPU profile has no card to be
free, so eligibleNodes accepts every ready node, and placing Qwen2.5-Coder
14B on one of the 8 GB nodes would OOM a machine that reported itself
perfectly healthy.
Nothing in the inventory says how much memory a node has, and inferring it
would be guessing at the wrong layer — the amount that matters is the
model's working set, which the scheduler does not know either. So the
operator says which nodes a profile is for, with the labels that already
exist:
node_selector:
ram: 32g
A rejection names the label it did not find, because "no candidates were
considered" is not something anyone can act on and "does not carry ram=32g"
is. An absent selector constrains nothing, which is every profile that
exists today, and there is a test that says so.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
aa4da3264d
commit
8a894af1ca
3 files changed
+84
-6
No files matched your search
@@ -41,6 +41,16 @@ type Profile struct {
|
||||
Args []string `koanf:"args"`
|
||||
Env map[string]string `koanf:"env"`
|
||||
|
||||
// NodeSelector restricts this profile to nodes carrying every one of these
|
||||
// labels, with these values.
|
||||
//
|
||||
// A CPU profile otherwise fits everywhere, because there is no card to be
|
||||
// free — which is right for a 0.5B model and wrong for a 14B one. Nothing
|
||||
// in the inventory says how much memory a node has, so the operator says
|
||||
// which nodes are meant for a model instead of the scheduler inferring it.
|
||||
// Agents have reported labels since the beginning; this is what reads them.
|
||||
NodeSelector map[string]string `koanf:"node_selector"`
|
||||
|
||||
// Weight is the router's capacity weight. It should come from ctl bench
|
||||
// rather than a guess (plan §6.2.1).
|
||||
Weight float64 `koanf:"weight"`
|
||||
@@ -257,12 +267,13 @@ func (m Model) PlacementProfiles() []placement.Profile {
|
||||
out := make([]placement.Profile, 0, len(m.Profiles))
|
||||
for _, p := range m.Profiles {
|
||||
out = append(out, placement.Profile{
|
||||
Name: p.Name,
|
||||
Engine: p.Engine,
|
||||
GPUClass: p.GPUClass,
|
||||
GPUsPerNode: p.GPUsPerNode,
|
||||
Nodes: p.Nodes,
|
||||
Priority: p.Priority,
|
||||
Name: p.Name,
|
||||
Engine: p.Engine,
|
||||
GPUClass: p.GPUClass,
|
||||
GPUsPerNode: p.GPUsPerNode,
|
||||
Nodes: p.Nodes,
|
||||
Priority: p.Priority,
|
||||
NodeSelector: p.NodeSelector,
|
||||
})
|
||||
}
|
||||
return out
|
||||
|
||||
@@ -32,6 +32,8 @@ type Profile struct {
|
||||
Nodes int
|
||||
// Priority orders profiles; the highest that fits is used.
|
||||
Priority int
|
||||
// NodeSelector restricts this profile to nodes carrying these labels.
|
||||
NodeSelector map[string]string
|
||||
}
|
||||
|
||||
// CPUOnly reports whether this profile needs no GPUs. The catalog only lets a
|
||||
@@ -287,6 +289,20 @@ type candidate struct {
|
||||
//
|
||||
// fabric is empty for single-node placement, in which case attachment is not
|
||||
// checked at all.
|
||||
// unmatchedLabel reports the first selector entry a node does not satisfy.
|
||||
//
|
||||
// It returns the label rather than a bare false so the rejection can name it:
|
||||
// "does not carry ram=32g" is a placement failure someone can act on, and
|
||||
// "no candidates were considered" is not.
|
||||
func unmatchedLabel(selector, labels map[string]string) (string, bool) {
|
||||
for k, want := range selector {
|
||||
if got, ok := labels[k]; !ok || got != want {
|
||||
return k + "=" + want, false
|
||||
}
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
|
||||
func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, rejections *[]Rejection) []candidate {
|
||||
pool := "none"
|
||||
if fabric != "" {
|
||||
@@ -320,6 +336,11 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
|
||||
}
|
||||
}
|
||||
|
||||
if miss, ok := unmatchedLabel(p.NodeSelector, n.Labels); !ok {
|
||||
reject("does not carry " + miss)
|
||||
continue
|
||||
}
|
||||
|
||||
if p.CPUOnly() {
|
||||
// Every ready node can run it; there is no card to be free.
|
||||
out = append(out, candidate{node: n})
|
||||
|
||||
@@ -481,3 +481,49 @@ func checkInvariants(t *testing.T, st *inventory.Store, p Profile, pl *Placement
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeSelectorKeepsAProfileOffTheWrongNodes(t *testing.T) {
|
||||
// A CPU profile fits everywhere, because there is no card to be free. That
|
||||
// is right for a 0.5B model and wrong for a 14B one: the small nodes have
|
||||
// 8 GB of RAM and the model needs 24, and nothing in the inventory says so.
|
||||
small := node("small", "", 0)
|
||||
small.Labels = map[string]string{"ram": "8g"}
|
||||
big := node("big", "", 0)
|
||||
big.Labels = map[string]string{"ram": "32g"}
|
||||
|
||||
s := New(store(small, big))
|
||||
p := Profile{Name: "cpu", Engine: "llamacpp", NodeSelector: map[string]string{"ram": "32g"}}
|
||||
|
||||
res := s.Place([]Profile{p}, nil)
|
||||
if res.Placement == nil {
|
||||
t.Fatalf("nothing was placed:\n%s", res.Explain())
|
||||
}
|
||||
if got := res.Placement.Nodes[0].NodeID; got != "big" {
|
||||
t.Fatalf("placed on %q; only the labelled node can hold this model", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnsatisfiableSelectorSaysWhichLabelIsMissing(t *testing.T) {
|
||||
// "no candidates were considered" is not something an operator can act on.
|
||||
small := node("small", "", 0)
|
||||
small.Labels = map[string]string{"ram": "8g"}
|
||||
|
||||
s := New(store(small))
|
||||
res := s.Place([]Profile{{
|
||||
Name: "cpu", Engine: "llamacpp", NodeSelector: map[string]string{"ram": "32g"},
|
||||
}}, nil)
|
||||
if res.Placement != nil {
|
||||
t.Fatal("placed on a node that does not carry the label")
|
||||
}
|
||||
if !strings.Contains(res.Explain(), "ram=32g") {
|
||||
t.Errorf("the rejection does not name the missing label:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEmptySelectorConstrainsNothing(t *testing.T) {
|
||||
// Every existing profile has no selector, and must keep fitting everywhere.
|
||||
s := New(store(node("plain", "", 0)))
|
||||
if res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp"}}, nil); res.Placement == nil {
|
||||
t.Fatalf("a profile with no selector was rejected:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user