diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index 88f11a9..8efa8ba 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -5031,6 +5031,19 @@ }, "description": "Engines are the inference engines installed here, as the agent found them. Absent means the agent is older than the probe and said nothing, which is not the same as having none — placement treats it that way too, or a control plane upgrade would empty a fleet." }, + "engine_versions": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "EngineVersions are the installed engines with their versions: llamacpp → b10991. ComputeCapabilities are its GPUs' CUDA compute capabilities, 7.0 for a V100, so an engine build's support can be read against them." + }, + "compute_capabilities": { + "type": "array", + "items": { + "type": "string" + } + }, "provision": { "type": "array", "items": { @@ -6434,6 +6447,10 @@ "engine": { "type": "string" }, + "engine_version": { + "type": "string", + "description": "EngineVersion restricts the profile to nodes whose installed engine's version starts with it: \"0.11\" takes vLLM 0.11.0 and 0.11.1, not 0.10.2. Empty takes whatever version the node has." + }, "gpu_class": { "type": "string" }, diff --git a/docs/reference.md b/docs/reference.md index ed125c9..c8f13b5 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -203,6 +203,7 @@ rather than repeated. | `files` | `[]OriginFile` | [PreviousOrigin](#previousorigin) | | `name` | `string` | [Profile](#profile) | | `engine` | `string` | [Profile](#profile) | +| `engine_version` | `string` | [Profile](#profile) | | `gpu_class` | `string` | [Profile](#profile) | | `tp` | `int` | [Profile](#profile) | | `pp` | `int` | [Profile](#profile) | @@ -1756,6 +1757,14 @@ before affinity is abandoned. `string` +### `engine_version` + +`string` + +EngineVersion restricts the profile to nodes whose installed engine's +version starts with it: "0.11" takes vLLM 0.11.0 and 0.11.1, not +0.10.2. Empty takes whatever version the node has. + ### `gpu_class` `string` diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index 3e43202..d1e0297 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -232,7 +232,7 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { // What the node can run, which is the first thing to look // at when a model will not place and the hardware looks // fine. - join(n.Engines), + enginesColumn(n), classesColumn(n), join(n.Fabrics), n.Replicas, n.LastSeenAgo) } _ = w.Flush() @@ -596,7 +596,32 @@ func emitJSON(v any) error { // more than one or MIG instances among them: a card cut into two 3g.40gb // instances beside a whole card is "a100-80g-mig-3g.40gb×2,a100-80g×1", where // the GPUS column's 3 alone would read as three of something. +// enginesColumn is each engine with its version: llamacpp b10991. +func enginesColumn(n adminapi.NodeView) string { + if len(n.EngineVersions) == 0 { + return join(n.Engines) + } + parts := make([]string, 0, len(n.Engines)) + for _, e := range n.Engines { + if v := n.EngineVersions[e]; v != "" && v != "unknown" { + e += " " + v + } + parts = append(parts, e) + } + return strings.Join(parts, ",") +} + +// classesColumn is the node's GPU classes, with their compute capability +// when they report one: 6000-24g (7.5). func classesColumn(n adminapi.NodeView) string { + s := classesOnly(n) + if len(n.ComputeCapabilities) > 0 && s != "-" { + s += " (" + strings.Join(n.ComputeCapabilities, ",") + ")" + } + return s +} + +func classesOnly(n adminapi.NodeView) string { counted := len(n.ClassGPUs) > 1 for _, c := range n.ClassGPUs { counted = counted || c.MIG diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 63b79b8..52551e7 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -12,6 +12,7 @@ import ( "errors" "fmt" "io" + "maps" "net/http" "net/url" "slices" @@ -228,6 +229,12 @@ type NodeView struct { // which is not the same as having none — placement treats it that way too, // or a control plane upgrade would empty a fleet. Engines []string `json:"engines,omitempty"` + // EngineVersions are the installed engines with their versions: + // llamacpp → b10991. ComputeCapabilities are its GPUs' CUDA compute + // capabilities, 7.0 for a V100, so an engine build's support can be read + // against them. + EngineVersions map[string]string `json:"engine_versions,omitempty"` + ComputeCapabilities []string `json:"compute_capabilities,omitempty"` // Provision is where installing each wanted engine stands. An engine is in // Engines only once its installation here is ready. Provision []inventory.EngineInstall `json:"provision,omitempty"` @@ -647,6 +654,15 @@ func (a *API) Nodes() []NodeView { v.Engines = append(v.Engines, name) } slices.Sort(v.Engines) + if len(n.EngineVersions) > 0 { + v.EngineVersions = maps.Clone(n.EngineVersions) + } + for _, g := range n.GPUs { + if g.ComputeCapability != "" && !slices.Contains(v.ComputeCapabilities, g.ComputeCapability) { + v.ComputeCapabilities = append(v.ComputeCapabilities, g.ComputeCapability) + } + } + slices.Sort(v.ComputeCapabilities) v.Provision = n.Provision if ch := a.ctl.EngineChoices(); ch != nil && len(n.GPUs) > 0 { chosen := ch.For(n.ID) diff --git a/internal/control/adminapi/judges_test.go b/internal/control/adminapi/judges_test.go index fa4fc27..6d8fa09 100644 --- a/internal/control/adminapi/judges_test.go +++ b/internal/control/adminapi/judges_test.go @@ -89,3 +89,18 @@ func TestAJudgeIsAskedAndItsWordsShown(t *testing.T) { t.Errorf("no text: %d", code) } } + +func TestANodeSaysItsEnginesVersionsAndCapabilities(t *testing.T) { + n := gpuNode("volta", 2) + n.EngineVersions = map[string]string{"llamacpp": "b10991", "vllm": "0.10.2"} + for i := range n.GPUs { + n.GPUs[i].ComputeCapability = "7.0" + } + a, _ := setup(t, testModels(), n) + var nodes []NodeView + call(t, a, "GET", "/api/v1/nodes", "", &nodes) + if len(nodes) != 1 || nodes[0].EngineVersions["vllm"] != "0.10.2" || + len(nodes[0].ComputeCapabilities) != 1 || nodes[0].ComputeCapabilities[0] != "7.0" { + t.Errorf("%+v", nodes) + } +} diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index aab717c..e2dab0f 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -164,9 +164,13 @@ type SLO struct { // Profile is how a model runs on one GPU class. type Profile struct { - Name string `json:"name,omitempty" koanf:"name"` - Engine string `json:"engine,omitempty" koanf:"engine"` - GPUClass string `json:"gpu_class,omitempty" koanf:"gpu_class"` + Name string `json:"name,omitempty" koanf:"name"` + Engine string `json:"engine,omitempty" koanf:"engine"` + // EngineVersion restricts the profile to nodes whose installed engine's + // version starts with it: "0.11" takes vLLM 0.11.0 and 0.11.1, not + // 0.10.2. Empty takes whatever version the node has. + EngineVersion string `json:"engine_version,omitempty" koanf:"engine_version"` + GPUClass string `json:"gpu_class,omitempty" koanf:"gpu_class"` TP int `json:"tp,omitempty" koanf:"tp"` PP int `json:"pp,omitempty" koanf:"pp"` @@ -885,18 +889,19 @@ func (m Model) PlacementProfiles() []placement.Profile { out := make([]placement.Profile, 0, len(m.Profiles)) for _, p := range m.Profiles { out = append(out, placement.Profile{ - Name: p.Name, - Engine: p.Engine, - GPUClass: p.GPUClass, - GPUsPerNode: p.GPUsPerNode, - Nodes: p.Nodes, - Priority: p.Priority, - NodeSelector: p.NodeSelector, - MinMemoryGB: p.MinMemoryGB, - MinDiskGB: p.MinDiskGB, - CPUCores: p.CPUCores, - PowerWatts: p.PowerWatts, - KeepWithin: facility.Level(strings.ToLower(strings.TrimSpace(p.KeepWithin))), + Name: p.Name, + Engine: p.Engine, + EngineVersion: p.EngineVersion, + GPUClass: p.GPUClass, + GPUsPerNode: p.GPUsPerNode, + Nodes: p.Nodes, + Priority: p.Priority, + NodeSelector: p.NodeSelector, + MinMemoryGB: p.MinMemoryGB, + MinDiskGB: p.MinDiskGB, + CPUCores: p.CPUCores, + PowerWatts: p.PowerWatts, + KeepWithin: facility.Level(strings.ToLower(strings.TrimSpace(p.KeepWithin))), // The model's source, so placement can prefer nodes that already // hold the weights rather than one that must first move them. Source: m.Source, diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index 945b8e6..06eeb38 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -25,9 +25,12 @@ import ( // Profile is the subset of a catalog profile placement needs. type Profile struct { - Name string - Engine string - GPUClass string + Name string + Engine string + // EngineVersion is a prefix the node's installed engine version must + // have; empty takes any. + EngineVersion string + GPUClass string // GPUsPerNode is how many GPUs each node contributes. GPUsPerNode int // Nodes is how many nodes the replica spans. Above 1, a fabric is required. @@ -614,8 +617,13 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje // and either way not a place to start an engine. sim is built into // the agent and never installed. if p.Engine != "" && p.Engine != "sim" && (n.EnginesProbed || len(n.EngineVersions) > 0) { - if _, ok := n.EngineVersions[p.Engine]; !ok { - reject("does not have " + p.Engine + " installed" + installing(n, p.Engine)) + v, ok := n.EngineVersions[p.Engine] + if !ok { + reject("does not have " + p.Engine + " installed" + installing(n, p.Engine) + hasEngines(n)) + continue + } + if p.EngineVersion != "" && !strings.HasPrefix(v, p.EngineVersion) { + reject(fmt.Sprintf("has %s %s, not the %s the profile asks for", p.Engine, v, p.EngineVersion)) continue } } @@ -828,6 +836,19 @@ func (s *Scheduler) claimedCores(nodeID string) int { } // installing says why an engine is missing, when the node has said. +// hasEngines is what a node does have, for a rejection over what it does not. +func hasEngines(n inventory.Node) string { + if len(n.EngineVersions) == 0 { + return "" + } + var parts []string + for e, v := range n.EngineVersions { + parts = append(parts, e+" "+v) + } + slices.Sort(parts) + return "; it has " + strings.Join(parts, ", ") +} + func installing(n inventory.Node, engine string) string { for _, p := range n.Provision { if p.Engine == engine && p.Detail != "" { diff --git a/internal/control/placement/placement_test.go b/internal/control/placement/placement_test.go index 87e5274..d888cbc 100644 --- a/internal/control/placement/placement_test.go +++ b/internal/control/placement/placement_test.go @@ -1071,3 +1071,31 @@ func TestANodeWithoutIMEXIsNotCountedInItsDomain(t *testing.T) { t.Errorf("the explanation should say why the domain was passed over:\n%s", res.Explain()) } } + +func TestAProfileCanAskForAnEngineVersion(t *testing.T) { + old := withEngines("volta", 8, "vllm", "llamacpp") + old.EngineVersions["vllm"] = "0.10.2" + newer := withEngines("hopper", 8, "vllm") + newer.EngineVersions["vllm"] = "0.11.1" + s := New(store(old, newer)) + + res := s.Place([]Profile{{Name: "tp1", Engine: "vllm", EngineVersion: "0.11", GPUClass: "h100-80g", + GPUsPerNode: 1, Nodes: 1}}, nil) + if res.Placement == nil || res.Placement.Nodes[0].NodeID != "hopper" { + t.Fatalf("placed where the version is not:\n%s", res.Explain()) + } + + only := New(store(old)) + res = only.Place([]Profile{{Name: "tp1", Engine: "vllm", EngineVersion: "0.11", GPUClass: "h100-80g", + GPUsPerNode: 1, Nodes: 1}}, nil) + if res.Placement != nil || !strings.Contains(res.Explain(), "has vllm 0.10.2, not the 0.11") { + t.Errorf("the rejection should say which version is there:\n%s", res.Explain()) + } + + // A node without the engine says what it has instead. + res = New(store(withEngines("cpu-only", 8, "llamacpp"))).Place([]Profile{{Name: "tp1", Engine: "vllm", + GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1}}, nil) + if !strings.Contains(res.Explain(), "it has llamacpp") { + t.Errorf("the rejection should say what the node has:\n%s", res.Explain()) + } +} diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index e9c24e9..d836ee9 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -261,6 +261,7 @@ var Reference = []Block{ {Package: "catalog", Name: "Profile", Title: "Profile", Doc: "", Fields: []Field{ {Key: "name", Type: "string", Doc: "", Nested: ""}, {Key: "engine", Type: "string", Doc: "", Nested: ""}, + {Key: "engine_version", Type: "string", Doc: "EngineVersion restricts the profile to nodes whose installed engine's\nversion starts with it: \"0.11\" takes vLLM 0.11.0 and 0.11.1, not\n0.10.2. Empty takes whatever version the node has.", Nested: ""}, {Key: "gpu_class", Type: "string", Doc: "", Nested: ""}, {Key: "tp", Type: "int", Doc: "", Nested: ""}, {Key: "pp", Type: "int", Doc: "", Nested: ""},