diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index 88f11a96..8efa8bae 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -5031,6 +5031,19 @@ }, "description": "Engines are the inference engines installed here, as the agent found them. Absent means the agent is older than the probe and said nothing, which is not the same as having none — placement treats it that way too, or a control plane upgrade would empty a fleet." }, + "engine_versions": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "EngineVersions are the installed engines with their versions: llamacpp → b10991. ComputeCapabilities are its GPUs' CUDA compute capabilities, 7.0 for a V100, so an engine build's support can be read against them." + }, + "compute_capabilities": { + "type": "array", + "items": { + "type": "string" + } + }, "provision": { "type": "array", "items": { @@ -6434,6 +6447,10 @@ "engine": { "type": "string" }, + "engine_version": { + "type": "string", + "description": "EngineVersion restricts the profile to nodes whose installed engine's version starts with it: \"0.11\" takes vLLM 0.11.0 and 0.11.1, not 0.10.2. Empty takes whatever version the node has." + }, "gpu_class": { "type": "string" }, diff --git a/docs/reference.md b/docs/reference.md index ed125c92..c8f13b59 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -203,6 +203,7 @@ rather than repeated. | `files` | `[]OriginFile` | [PreviousOrigin](#previousorigin) | | `name` | `string` | [Profile](#profile) | | `engine` | `string` | [Profile](#profile) | +| `engine_version` | `string` | [Profile](#profile) | | `gpu_class` | `string` | [Profile](#profile) | | `tp` | `int` | [Profile](#profile) | | `pp` | `int` | [Profile](#profile) | @@ -1756,6 +1757,14 @@ before affinity is abandoned. `string` +### `engine_version` + +`string` + +EngineVersion restricts the profile to nodes whose installed engine's +version starts with it: "0.11" takes vLLM 0.11.0 and 0.11.1, not +0.10.2. Empty takes whatever version the node has. + ### `gpu_class` `string` diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index 3e432024..d1e02975 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -232,7 +232,7 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { // What the node can run, which is the first thing to look // at when a model will not place and the hardware looks // fine. - join(n.Engines), + enginesColumn(n), classesColumn(n), join(n.Fabrics), n.Replicas, n.LastSeenAgo) } _ = w.Flush() @@ -596,7 +596,32 @@ func emitJSON(v any) error { // more than one or MIG instances among them: a card cut into two 3g.40gb // instances beside a whole card is "a100-80g-mig-3g.40gb×2,a100-80g×1", where // the GPUS column's 3 alone would read as three of something. +// enginesColumn is each engine with its version: llamacpp b10991. +func enginesColumn(n adminapi.NodeView) string { + if len(n.EngineVersions) == 0 { + return join(n.Engines) + } + parts := make([]string, 0, len(n.Engines)) + for _, e := range n.Engines { + if v := n.EngineVersions[e]; v != "" && v != "unknown" { + e += " " + v + } + parts = append(parts, e) + } + return strings.Join(parts, ",") +} + +// classesColumn is the node's GPU classes, with their compute capability +// when they report one: 6000-24g (7.5). func classesColumn(n adminapi.NodeView) string { + s := classesOnly(n) + if len(n.ComputeCapabilities) > 0 && s != "-" { + s += " (" + strings.Join(n.ComputeCapabilities, ",") + ")" + } + return s +} + +func classesOnly(n adminapi.NodeView) string { counted := len(n.ClassGPUs) > 1 for _, c := range n.ClassGPUs { counted = counted || c.MIG diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 63b79b8f..52551e71 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -12,6 +12,7 @@ import ( "errors" "fmt" "io" + "maps" "net/http" "net/url" "slices" @@ -228,6 +229,12 @@ type NodeView struct { // which is not the same as having none — placement treats it that way too, // or a control plane upgrade would empty a fleet. Engines []string `json:"engines,omitempty"` + // EngineVersions are the installed engines with their versions: + // llamacpp → b10991. ComputeCapabilities are its GPUs' CUDA compute + // capabilities, 7.0 for a V100, so an engine build's support can be read + // against them. + EngineVersions map[string]string `json:"engine_versions,omitempty"` + ComputeCapabilities []string `json:"compute_capabilities,omitempty"` // Provision is where installing each wanted engine stands. An engine is in // Engines only once its installation here is ready. Provision []inventory.EngineInstall `json:"provision,omitempty"` @@ -647,6 +654,15 @@ func (a *API) Nodes() []NodeView { v.Engines = append(v.Engines, name) } slices.Sort(v.Engines) + if len(n.EngineVersions) > 0 { + v.EngineVersions = maps.Clone(n.EngineVersions) + } + for _, g := range n.GPUs { + if g.ComputeCapability != "" && !slices.Contains(v.ComputeCapabilities, g.ComputeCapability) { + v.ComputeCapabilities = append(v.ComputeCapabilities, g.ComputeCapability) + } + } + slices.Sort(v.ComputeCapabilities) v.Provision = n.Provision if ch := a.ctl.EngineChoices(); ch != nil && len(n.GPUs) > 0 { chosen := ch.For(n.ID) diff --git a/internal/control/adminapi/judges_test.go b/internal/control/adminapi/judges_test.go index fa4fc279..6d8fa097 100644 --- a/internal/control/adminapi/judges_test.go +++ b/internal/control/adminapi/judges_test.go @@ -89,3 +89,18 @@ func TestAJudgeIsAskedAndItsWordsShown(t *testing.T) { t.Errorf("no text: %d", code) } } + +func TestANodeSaysItsEnginesVersionsAndCapabilities(t *testing.T) { + n := gpuNode("volta", 2) + n.EngineVersions = map[string]string{"llamacpp": "b10991", "vllm": "0.10.2"} + for i := range n.GPUs { + n.GPUs[i].ComputeCapability = "7.0" + } + a, _ := setup(t, testModels(), n) + var nodes []NodeView + call(t, a, "GET", "/api/v1/nodes", "", &nodes) + if len(nodes) != 1 || nodes[0].EngineVersions["vllm"] != "0.10.2" || + len(nodes[0].ComputeCapabilities) != 1 || nodes[0].ComputeCapabilities[0] != "7.0" { + t.Errorf("%+v", nodes) + } +} diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index aab717c2..e2dab0fd 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -164,9 +164,13 @@ type SLO struct { // Profile is how a model runs on one GPU class. type Profile struct { - Name string `json:"name,omitempty" koanf:"name"` - Engine string `json:"engine,omitempty" koanf:"engine"` - GPUClass string `json:"gpu_class,omitempty" koanf:"gpu_class"` + Name string `json:"name,omitempty" koanf:"name"` + Engine string `json:"engine,omitempty" koanf:"engine"` + // EngineVersion restricts the profile to nodes whose installed engine's + // version starts with it: "0.11" takes vLLM 0.11.0 and 0.11.1, not + // 0.10.2. Empty takes whatever version the node has. + EngineVersion string `json:"engine_version,omitempty" koanf:"engine_version"` + GPUClass string `json:"gpu_class,omitempty" koanf:"gpu_class"` TP int `json:"tp,omitempty" koanf:"tp"` PP int `json:"pp,omitempty" koanf:"pp"` @@ -885,18 +889,19 @@ func (m Model) PlacementProfiles() []placement.Profile { out := make([]placement.Profile, 0, len(m.Profiles)) for _, p := range m.Profiles { out = append(out, placement.Profile{ - Name: p.Name, - Engine: p.Engine, - GPUClass: p.GPUClass, - GPUsPerNode: p.GPUsPerNode, - Nodes: p.Nodes, - Priority: p.Priority, - NodeSelector: p.NodeSelector, - MinMemoryGB: p.MinMemoryGB, - MinDiskGB: p.MinDiskGB, - CPUCores: p.CPUCores, - PowerWatts: p.PowerWatts, - KeepWithin: facility.Level(strings.ToLower(strings.TrimSpace(p.KeepWithin))), + Name: p.Name, + Engine: p.Engine, + EngineVersion: p.EngineVersion, + GPUClass: p.GPUClass, + GPUsPerNode: p.GPUsPerNode, + Nodes: p.Nodes, + Priority: p.Priority, + NodeSelector: p.NodeSelector, + MinMemoryGB: p.MinMemoryGB, + MinDiskGB: p.MinDiskGB, + CPUCores: p.CPUCores, + PowerWatts: p.PowerWatts, + KeepWithin: facility.Level(strings.ToLower(strings.TrimSpace(p.KeepWithin))), // The model's source, so placement can prefer nodes that already // hold the weights rather than one that must first move them. Source: m.Source, diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index 945b8e65..06eeb389 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -25,9 +25,12 @@ import ( // Profile is the subset of a catalog profile placement needs. type Profile struct { - Name string - Engine string - GPUClass string + Name string + Engine string + // EngineVersion is a prefix the node's installed engine version must + // have; empty takes any. + EngineVersion string + GPUClass string // GPUsPerNode is how many GPUs each node contributes. GPUsPerNode int // Nodes is how many nodes the replica spans. Above 1, a fabric is required. @@ -614,8 +617,13 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje // and either way not a place to start an engine. sim is built into // the agent and never installed. if p.Engine != "" && p.Engine != "sim" && (n.EnginesProbed || len(n.EngineVersions) > 0) { - if _, ok := n.EngineVersions[p.Engine]; !ok { - reject("does not have " + p.Engine + " installed" + installing(n, p.Engine)) + v, ok := n.EngineVersions[p.Engine] + if !ok { + reject("does not have " + p.Engine + " installed" + installing(n, p.Engine) + hasEngines(n)) + continue + } + if p.EngineVersion != "" && !strings.HasPrefix(v, p.EngineVersion) { + reject(fmt.Sprintf("has %s %s, not the %s the profile asks for", p.Engine, v, p.EngineVersion)) continue } } @@ -828,6 +836,19 @@ func (s *Scheduler) claimedCores(nodeID string) int { } // installing says why an engine is missing, when the node has said. +// hasEngines is what a node does have, for a rejection over what it does not. +func hasEngines(n inventory.Node) string { + if len(n.EngineVersions) == 0 { + return "" + } + var parts []string + for e, v := range n.EngineVersions { + parts = append(parts, e+" "+v) + } + slices.Sort(parts) + return "; it has " + strings.Join(parts, ", ") +} + func installing(n inventory.Node, engine string) string { for _, p := range n.Provision { if p.Engine == engine && p.Detail != "" { diff --git a/internal/control/placement/placement_test.go b/internal/control/placement/placement_test.go index 87e52741..d888cbcf 100644 --- a/internal/control/placement/placement_test.go +++ b/internal/control/placement/placement_test.go @@ -1071,3 +1071,31 @@ func TestANodeWithoutIMEXIsNotCountedInItsDomain(t *testing.T) { t.Errorf("the explanation should say why the domain was passed over:\n%s", res.Explain()) } } + +func TestAProfileCanAskForAnEngineVersion(t *testing.T) { + old := withEngines("volta", 8, "vllm", "llamacpp") + old.EngineVersions["vllm"] = "0.10.2" + newer := withEngines("hopper", 8, "vllm") + newer.EngineVersions["vllm"] = "0.11.1" + s := New(store(old, newer)) + + res := s.Place([]Profile{{Name: "tp1", Engine: "vllm", EngineVersion: "0.11", GPUClass: "h100-80g", + GPUsPerNode: 1, Nodes: 1}}, nil) + if res.Placement == nil || res.Placement.Nodes[0].NodeID != "hopper" { + t.Fatalf("placed where the version is not:\n%s", res.Explain()) + } + + only := New(store(old)) + res = only.Place([]Profile{{Name: "tp1", Engine: "vllm", EngineVersion: "0.11", GPUClass: "h100-80g", + GPUsPerNode: 1, Nodes: 1}}, nil) + if res.Placement != nil || !strings.Contains(res.Explain(), "has vllm 0.10.2, not the 0.11") { + t.Errorf("the rejection should say which version is there:\n%s", res.Explain()) + } + + // A node without the engine says what it has instead. + res = New(store(withEngines("cpu-only", 8, "llamacpp"))).Place([]Profile{{Name: "tp1", Engine: "vllm", + GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1}}, nil) + if !strings.Contains(res.Explain(), "it has llamacpp") { + t.Errorf("the rejection should say what the node has:\n%s", res.Explain()) + } +} diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index e9c24e91..d836ee91 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -261,6 +261,7 @@ var Reference = []Block{ {Package: "catalog", Name: "Profile", Title: "Profile", Doc: "", Fields: []Field{ {Key: "name", Type: "string", Doc: "", Nested: ""}, {Key: "engine", Type: "string", Doc: "", Nested: ""}, + {Key: "engine_version", Type: "string", Doc: "EngineVersion restricts the profile to nodes whose installed engine's\nversion starts with it: \"0.11\" takes vLLM 0.11.0 and 0.11.1, not\n0.10.2. Empty takes whatever version the node has.", Nested: ""}, {Key: "gpu_class", Type: "string", Doc: "", Nested: ""}, {Key: "tp", Type: "int", Doc: "", Nested: ""}, {Key: "pp", Type: "int", Doc: "", Nested: ""},