Nodes show their engines' versions and GPUs' capability; a profile can ask for an engine version

NodeView carries engine_versions and compute_capabilities; ctl nodes shows
llamacpp b10991 and 6000-24g (7.5). A profile's engine_version restricts it
to nodes whose engine version starts with it, and explain-placement says
which version a node has, or which engines it has when it lacks the one
asked for.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

docs/admin-api.openapi.json

@@ -5031,6 +5031,19 @@
             },
             "description": "Engines are the inference engines installed here, as the agent found them. Absent means the agent is older than the probe and said nothing, which is not the same as having none — placement treats it that way too, or a control plane upgrade would empty a fleet."
           },
+          "engine_versions": {
+            "type": "object",
+            "additionalProperties": {
+              "type": "string"
+            },
+            "description": "EngineVersions are the installed engines with their versions: llamacpp → b10991. ComputeCapabilities are its GPUs' CUDA compute capabilities, 7.0 for a V100, so an engine build's support can be read against them."
+          },
+          "compute_capabilities": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
           "provision": {
             "type": "array",
             "items": {
@@ -6434,6 +6447,10 @@
           "engine": {
             "type": "string"
           },
+          "engine_version": {
+            "type": "string",
+            "description": "EngineVersion restricts the profile to nodes whose installed engine's version starts with it: \"0.11\" takes vLLM 0.11.0 and 0.11.1, not 0.10.2. Empty takes whatever version the node has."
+          },
           "gpu_class": {
             "type": "string"
           },

docs/reference.md

@@ -203,6 +203,7 @@ rather than repeated.
 | `files` | `[]OriginFile` | [PreviousOrigin](#previousorigin) |
 | `name` | `string` | [Profile](#profile) |
 | `engine` | `string` | [Profile](#profile) |
+| `engine_version` | `string` | [Profile](#profile) |
 | `gpu_class` | `string` | [Profile](#profile) |
 | `tp` | `int` | [Profile](#profile) |
 | `pp` | `int` | [Profile](#profile) |
@@ -1756,6 +1757,14 @@ before affinity is abandoned.
 
 `string`
 
+### `engine_version`
+
+`string`
+
+EngineVersion restricts the profile to nodes whose installed engine's
+version starts with it: "0.11" takes vLLM 0.11.0 and 0.11.1, not
+0.10.2. Empty takes whatever version the node has.
+
 ### `gpu_class`
 
 `string`

internal/cli/ctl.go

@@ -232,7 +232,7 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
 					// What the node can run, which is the first thing to look
 					// at when a model will not place and the hardware looks
 					// fine.
-					join(n.Engines),
+					enginesColumn(n),
 					classesColumn(n), join(n.Fabrics), n.Replicas, n.LastSeenAgo)
 			}
 			_ = w.Flush()
@@ -596,7 +596,32 @@ func emitJSON(v any) error {
 // more than one or MIG instances among them: a card cut into two 3g.40gb
 // instances beside a whole card is "a100-80g-mig-3g.40gb×2,a100-80g×1", where
 // the GPUS column's 3 alone would read as three of something.
+// enginesColumn is each engine with its version: llamacpp b10991.
+func enginesColumn(n adminapi.NodeView) string {
+	if len(n.EngineVersions) == 0 {
+		return join(n.Engines)
+	}
+	parts := make([]string, 0, len(n.Engines))
+	for _, e := range n.Engines {
+		if v := n.EngineVersions[e]; v != "" && v != "unknown" {
+			e += " " + v
+		}
+		parts = append(parts, e)
+	}
+	return strings.Join(parts, ",")
+}
+
+// classesColumn is the node's GPU classes, with their compute capability
+// when they report one: 6000-24g (7.5).
 func classesColumn(n adminapi.NodeView) string {
+	s := classesOnly(n)
+	if len(n.ComputeCapabilities) > 0 && s != "-" {
+		s += " (" + strings.Join(n.ComputeCapabilities, ",") + ")"
+	}
+	return s
+}
+
+func classesOnly(n adminapi.NodeView) string {
 	counted := len(n.ClassGPUs) > 1
 	for _, c := range n.ClassGPUs {
 		counted = counted || c.MIG

internal/control/adminapi/api.go

@@ -12,6 +12,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"maps"
 	"net/http"
 	"net/url"
 	"slices"
@@ -228,6 +229,12 @@ type NodeView struct {
 	// which is not the same as having none — placement treats it that way too,
 	// or a control plane upgrade would empty a fleet.
 	Engines []string `json:"engines,omitempty"`
+	// EngineVersions are the installed engines with their versions:
+	// llamacpp → b10991. ComputeCapabilities are its GPUs' CUDA compute
+	// capabilities, 7.0 for a V100, so an engine build's support can be read
+	// against them.
+	EngineVersions      map[string]string `json:"engine_versions,omitempty"`
+	ComputeCapabilities []string          `json:"compute_capabilities,omitempty"`
 	// Provision is where installing each wanted engine stands. An engine is in
 	// Engines only once its installation here is ready.
 	Provision []inventory.EngineInstall `json:"provision,omitempty"`
@@ -647,6 +654,15 @@ func (a *API) Nodes() []NodeView {
 			v.Engines = append(v.Engines, name)
 		}
 		slices.Sort(v.Engines)
+		if len(n.EngineVersions) > 0 {
+			v.EngineVersions = maps.Clone(n.EngineVersions)
+		}
+		for _, g := range n.GPUs {
+			if g.ComputeCapability != "" && !slices.Contains(v.ComputeCapabilities, g.ComputeCapability) {
+				v.ComputeCapabilities = append(v.ComputeCapabilities, g.ComputeCapability)
+			}
+		}
+		slices.Sort(v.ComputeCapabilities)
 		v.Provision = n.Provision
 		if ch := a.ctl.EngineChoices(); ch != nil && len(n.GPUs) > 0 {
 			chosen := ch.For(n.ID)

internal/control/adminapi/judges_test.go

@@ -89,3 +89,18 @@ func TestAJudgeIsAskedAndItsWordsShown(t *testing.T) {
 		t.Errorf("no text: %d", code)
 	}
 }
+
+func TestANodeSaysItsEnginesVersionsAndCapabilities(t *testing.T) {
+	n := gpuNode("volta", 2)
+	n.EngineVersions = map[string]string{"llamacpp": "b10991", "vllm": "0.10.2"}
+	for i := range n.GPUs {
+		n.GPUs[i].ComputeCapability = "7.0"
+	}
+	a, _ := setup(t, testModels(), n)
+	var nodes []NodeView
+	call(t, a, "GET", "/api/v1/nodes", "", &nodes)
+	if len(nodes) != 1 || nodes[0].EngineVersions["vllm"] != "0.10.2" ||
+		len(nodes[0].ComputeCapabilities) != 1 || nodes[0].ComputeCapabilities[0] != "7.0" {
+		t.Errorf("%+v", nodes)
+	}
+}

internal/control/catalog/catalog.go

@@ -164,9 +164,13 @@ type SLO struct {
 
 // Profile is how a model runs on one GPU class.
 type Profile struct {
-	Name     string `json:"name,omitempty" koanf:"name"`
-	Engine   string `json:"engine,omitempty" koanf:"engine"`
-	GPUClass string `json:"gpu_class,omitempty" koanf:"gpu_class"`
+	Name   string `json:"name,omitempty" koanf:"name"`
+	Engine string `json:"engine,omitempty" koanf:"engine"`
+	// EngineVersion restricts the profile to nodes whose installed engine's
+	// version starts with it: "0.11" takes vLLM 0.11.0 and 0.11.1, not
+	// 0.10.2. Empty takes whatever version the node has.
+	EngineVersion string `json:"engine_version,omitempty" koanf:"engine_version"`
+	GPUClass      string `json:"gpu_class,omitempty" koanf:"gpu_class"`
 
 	TP int `json:"tp,omitempty" koanf:"tp"`
 	PP int `json:"pp,omitempty" koanf:"pp"`
@@ -885,18 +889,19 @@ func (m Model) PlacementProfiles() []placement.Profile {
 	out := make([]placement.Profile, 0, len(m.Profiles))
 	for _, p := range m.Profiles {
 		out = append(out, placement.Profile{
-			Name:         p.Name,
-			Engine:       p.Engine,
-			GPUClass:     p.GPUClass,
-			GPUsPerNode:  p.GPUsPerNode,
-			Nodes:        p.Nodes,
-			Priority:     p.Priority,
-			NodeSelector: p.NodeSelector,
-			MinMemoryGB:  p.MinMemoryGB,
-			MinDiskGB:    p.MinDiskGB,
-			CPUCores:     p.CPUCores,
-			PowerWatts:   p.PowerWatts,
-			KeepWithin:   facility.Level(strings.ToLower(strings.TrimSpace(p.KeepWithin))),
+			Name:          p.Name,
+			Engine:        p.Engine,
+			EngineVersion: p.EngineVersion,
+			GPUClass:      p.GPUClass,
+			GPUsPerNode:   p.GPUsPerNode,
+			Nodes:         p.Nodes,
+			Priority:      p.Priority,
+			NodeSelector:  p.NodeSelector,
+			MinMemoryGB:   p.MinMemoryGB,
+			MinDiskGB:     p.MinDiskGB,
+			CPUCores:      p.CPUCores,
+			PowerWatts:    p.PowerWatts,
+			KeepWithin:    facility.Level(strings.ToLower(strings.TrimSpace(p.KeepWithin))),
 			// The model's source, so placement can prefer nodes that already
 			// hold the weights rather than one that must first move them.
 			Source: m.Source,

internal/control/placement/placement.go

@@ -25,9 +25,12 @@ import (
 
 // Profile is the subset of a catalog profile placement needs.
 type Profile struct {
-	Name     string
-	Engine   string
-	GPUClass string
+	Name   string
+	Engine string
+	// EngineVersion is a prefix the node's installed engine version must
+	// have; empty takes any.
+	EngineVersion string
+	GPUClass      string
 	// GPUsPerNode is how many GPUs each node contributes.
 	GPUsPerNode int
 	// Nodes is how many nodes the replica spans. Above 1, a fabric is required.
@@ -614,8 +617,13 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
 		// and either way not a place to start an engine. sim is built into
 		// the agent and never installed.
 		if p.Engine != "" && p.Engine != "sim" && (n.EnginesProbed || len(n.EngineVersions) > 0) {
-			if _, ok := n.EngineVersions[p.Engine]; !ok {
-				reject("does not have " + p.Engine + " installed" + installing(n, p.Engine))
+			v, ok := n.EngineVersions[p.Engine]
+			if !ok {
+				reject("does not have " + p.Engine + " installed" + installing(n, p.Engine) + hasEngines(n))
+				continue
+			}
+			if p.EngineVersion != "" && !strings.HasPrefix(v, p.EngineVersion) {
+				reject(fmt.Sprintf("has %s %s, not the %s the profile asks for", p.Engine, v, p.EngineVersion))
 				continue
 			}
 		}
@@ -828,6 +836,19 @@ func (s *Scheduler) claimedCores(nodeID string) int {
 }
 
 // installing says why an engine is missing, when the node has said.
+// hasEngines is what a node does have, for a rejection over what it does not.
+func hasEngines(n inventory.Node) string {
+	if len(n.EngineVersions) == 0 {
+		return ""
+	}
+	var parts []string
+	for e, v := range n.EngineVersions {
+		parts = append(parts, e+" "+v)
+	}
+	slices.Sort(parts)
+	return "; it has " + strings.Join(parts, ", ")
+}
+
 func installing(n inventory.Node, engine string) string {
 	for _, p := range n.Provision {
 		if p.Engine == engine && p.Detail != "" {

internal/control/placement/placement_test.go

@@ -1071,3 +1071,31 @@ func TestANodeWithoutIMEXIsNotCountedInItsDomain(t *testing.T) {
 		t.Errorf("the explanation should say why the domain was passed over:\n%s", res.Explain())
 	}
 }
+
+func TestAProfileCanAskForAnEngineVersion(t *testing.T) {
+	old := withEngines("volta", 8, "vllm", "llamacpp")
+	old.EngineVersions["vllm"] = "0.10.2"
+	newer := withEngines("hopper", 8, "vllm")
+	newer.EngineVersions["vllm"] = "0.11.1"
+	s := New(store(old, newer))
+
+	res := s.Place([]Profile{{Name: "tp1", Engine: "vllm", EngineVersion: "0.11", GPUClass: "h100-80g",
+		GPUsPerNode: 1, Nodes: 1}}, nil)
+	if res.Placement == nil || res.Placement.Nodes[0].NodeID != "hopper" {
+		t.Fatalf("placed where the version is not:\n%s", res.Explain())
+	}
+
+	only := New(store(old))
+	res = only.Place([]Profile{{Name: "tp1", Engine: "vllm", EngineVersion: "0.11", GPUClass: "h100-80g",
+		GPUsPerNode: 1, Nodes: 1}}, nil)
+	if res.Placement != nil || !strings.Contains(res.Explain(), "has vllm 0.10.2, not the 0.11") {
+		t.Errorf("the rejection should say which version is there:\n%s", res.Explain())
+	}
+
+	// A node without the engine says what it has instead.
+	res = New(store(withEngines("cpu-only", 8, "llamacpp"))).Place([]Profile{{Name: "tp1", Engine: "vllm",
+		GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1}}, nil)
+	if !strings.Contains(res.Explain(), "it has llamacpp") {
+		t.Errorf("the rejection should say what the node has:\n%s", res.Explain())
+	}
+}

internal/docgen/reference_gen.go

@@ -261,6 +261,7 @@ var Reference = []Block{
 	{Package: "catalog", Name: "Profile", Title: "Profile", Doc: "", Fields: []Field{
 		{Key: "name", Type: "string", Doc: "", Nested: ""},
 		{Key: "engine", Type: "string", Doc: "", Nested: ""},
+		{Key: "engine_version", Type: "string", Doc: "EngineVersion restricts the profile to nodes whose installed engine's\nversion starts with it: \"0.11\" takes vLLM 0.11.0 and 0.11.1, not\n0.10.2. Empty takes whatever version the node has.", Nested: ""},
 		{Key: "gpu_class", Type: "string", Doc: "", Nested: ""},
 		{Key: "tp", Type: "int", Doc: "", Nested: ""},
 		{Key: "pp", Type: "int", Doc: "", Nested: ""},