diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index c53e161..35a4552 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -41,6 +41,16 @@ type Profile struct { Args []string `koanf:"args"` Env map[string]string `koanf:"env"` + // NodeSelector restricts this profile to nodes carrying every one of these + // labels, with these values. + // + // A CPU profile otherwise fits everywhere, because there is no card to be + // free — which is right for a 0.5B model and wrong for a 14B one. Nothing + // in the inventory says how much memory a node has, so the operator says + // which nodes are meant for a model instead of the scheduler inferring it. + // Agents have reported labels since the beginning; this is what reads them. + NodeSelector map[string]string `koanf:"node_selector"` + // Weight is the router's capacity weight. It should come from ctl bench // rather than a guess (plan §6.2.1). Weight float64 `koanf:"weight"` @@ -257,12 +267,13 @@ func (m Model) PlacementProfiles() []placement.Profile { out := make([]placement.Profile, 0, len(m.Profiles)) for _, p := range m.Profiles { out = append(out, placement.Profile{ - Name: p.Name, - Engine: p.Engine, - GPUClass: p.GPUClass, - GPUsPerNode: p.GPUsPerNode, - Nodes: p.Nodes, - Priority: p.Priority, + Name: p.Name, + Engine: p.Engine, + GPUClass: p.GPUClass, + GPUsPerNode: p.GPUsPerNode, + Nodes: p.Nodes, + Priority: p.Priority, + NodeSelector: p.NodeSelector, }) } return out diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index 724dcc1..234667b 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -32,6 +32,8 @@ type Profile struct { Nodes int // Priority orders profiles; the highest that fits is used. Priority int + // NodeSelector restricts this profile to nodes carrying these labels. + NodeSelector map[string]string } // CPUOnly reports whether this profile needs no GPUs. The catalog only lets a @@ -287,6 +289,20 @@ type candidate struct { // // fabric is empty for single-node placement, in which case attachment is not // checked at all. +// unmatchedLabel reports the first selector entry a node does not satisfy. +// +// It returns the label rather than a bare false so the rejection can name it: +// "does not carry ram=32g" is a placement failure someone can act on, and +// "no candidates were considered" is not. +func unmatchedLabel(selector, labels map[string]string) (string, bool) { + for k, want := range selector { + if got, ok := labels[k]; !ok || got != want { + return k + "=" + want, false + } + } + return "", true +} + func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, rejections *[]Rejection) []candidate { pool := "none" if fabric != "" { @@ -320,6 +336,11 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje } } + if miss, ok := unmatchedLabel(p.NodeSelector, n.Labels); !ok { + reject("does not carry " + miss) + continue + } + if p.CPUOnly() { // Every ready node can run it; there is no card to be free. out = append(out, candidate{node: n}) diff --git a/internal/control/placement/placement_test.go b/internal/control/placement/placement_test.go index 56a061a..35a2e5d 100644 --- a/internal/control/placement/placement_test.go +++ b/internal/control/placement/placement_test.go @@ -481,3 +481,49 @@ func checkInvariants(t *testing.T, st *inventory.Store, p Profile, pl *Placement } } } + +func TestNodeSelectorKeepsAProfileOffTheWrongNodes(t *testing.T) { + // A CPU profile fits everywhere, because there is no card to be free. That + // is right for a 0.5B model and wrong for a 14B one: the small nodes have + // 8 GB of RAM and the model needs 24, and nothing in the inventory says so. + small := node("small", "", 0) + small.Labels = map[string]string{"ram": "8g"} + big := node("big", "", 0) + big.Labels = map[string]string{"ram": "32g"} + + s := New(store(small, big)) + p := Profile{Name: "cpu", Engine: "llamacpp", NodeSelector: map[string]string{"ram": "32g"}} + + res := s.Place([]Profile{p}, nil) + if res.Placement == nil { + t.Fatalf("nothing was placed:\n%s", res.Explain()) + } + if got := res.Placement.Nodes[0].NodeID; got != "big" { + t.Fatalf("placed on %q; only the labelled node can hold this model", got) + } +} + +func TestAnUnsatisfiableSelectorSaysWhichLabelIsMissing(t *testing.T) { + // "no candidates were considered" is not something an operator can act on. + small := node("small", "", 0) + small.Labels = map[string]string{"ram": "8g"} + + s := New(store(small)) + res := s.Place([]Profile{{ + Name: "cpu", Engine: "llamacpp", NodeSelector: map[string]string{"ram": "32g"}, + }}, nil) + if res.Placement != nil { + t.Fatal("placed on a node that does not carry the label") + } + if !strings.Contains(res.Explain(), "ram=32g") { + t.Errorf("the rejection does not name the missing label:\n%s", res.Explain()) + } +} + +func TestAnEmptySelectorConstrainsNothing(t *testing.T) { + // Every existing profile has no selector, and must keep fitting everywhere. + s := New(store(node("plain", "", 0))) + if res := s.Place([]Profile{{Name: "cpu", Engine: "llamacpp"}}, nil); res.Placement == nil { + t.Fatalf("a profile with no selector was rejected:\n%s", res.Explain()) + } +}