Give each GPU node a vLLM build it can run, chosen on the cluster page
Release / build (push) Successful in 1m12s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m55s

A node that cannot run the CUDA 13 build installs the same version's
CUDA 12.9 build from vLLM's GitHub release, with PyTorch's cu129 torch;
the install computes on the GPU before it counts. The build can be
chosen per node — auto, cu13, cu129, none — on the cluster page, with
ctl engine-build, or over the admin API; the control plane keeps the
choice and tells the node at once.

Refs #2.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-01 12:39:14 +02:00
1 parent 2735c5a318
commit 4f280a3546
32 files changed
+1004 -71

No files matched your search

+57
View File
@@ -9,6 +9,7 @@ package adminapi
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -111,6 +112,7 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("GET /api/v1/explain-placement", a.explainPlacement)
mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState)
mux.HandleFunc("POST /api/v1/nodes/{id}/upgrade", a.upgradeNode)
mux.HandleFunc("POST /api/v1/nodes/{id}/engines", a.setNodeEngine)
mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode)
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
@@ -156,6 +158,10 @@ type NodeView struct {
// Provision is where installing each wanted engine stands. An engine is in
// Engines only once its installation here is ready.
Provision []inventory.EngineInstall `json:"provision,omitempty"`
// EngineBuilds are the builds chosen for this node, by engine: "auto"
// (its hardware decides), a build ("cu129"), or "none". Only on a node
// with GPUs, the only kind with a choice to make.
EngineBuilds map[string]string `json:"engine_builds,omitempty"`
// OS is the operating system and architecture, as the agent read them.
OS string `json:"os,omitempty"`
// Warnings are facts about the machine that make it slow or unusable
@@ -473,6 +479,13 @@ func (a *API) Nodes() []NodeView {
}
slices.Sort(v.Engines)
v.Provision = n.Provision
if ch := a.ctl.EngineChoices(); ch != nil && len(n.GPUs) > 0 {
chosen := ch.For(n.ID)
v.EngineBuilds = map[string]string{"vllm": "auto"}
for e, b := range chosen {
v.EngineBuilds[e] = b
}
}
if h := n.Host; h.OSID != "" {
v.OS = strings.TrimSpace(h.OSID + " " + h.OSVersion + " " + h.Arch)
if h.Arch == "amd64" && !h.HasCPUFlag("avx2") {
@@ -1532,3 +1545,47 @@ func (a *API) setAdapters(w http.ResponseWriter, r *http.Request) {
}
writeJSON(w, http.StatusOK, AdaptersRequest{Adapters: out.Adapters})
}
// NodeEngineRequest chooses one engine's build for a node.
type NodeEngineRequest struct {
Engine string `json:"engine"`
// Build is "auto", a build such as "cu129", or "none".
Build string `json:"build"`
}
// setNodeEngine chooses an engine's build for a node, and has the node
// install it.
func (a *API) setNodeEngine(w http.ResponseWriter, r *http.Request) {
ch := a.ctl.EngineChoices()
if ch == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no store to keep the choice in")
return
}
id := r.PathValue("id")
if _, ok := a.ctl.Inventory().Get(id); !ok {
writeError(w, http.StatusNotFound, "no node "+id)
return
}
var req NodeEngineRequest
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
}
if err := ch.Set(r.Context(), id, req.Engine, req.Build, actorOf(r)); err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, req)
}
// SetNodeEngine chooses an engine's build for a node, for the cluster page.
func (a *API) SetNodeEngine(ctx context.Context, node, engine, build, by string) error {
ch := a.ctl.EngineChoices()
if ch == nil {
return errors.New("this control plane has no store to keep the choice in")
}
if _, ok := a.ctl.Inventory().Get(node); !ok {
return errors.New("no node " + node)
}
return ch.Set(ctx, node, engine, build, by)
}
@@ -0,0 +1,63 @@
package adminapi
import (
"context"
"log/slog"
"net/http"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// choiceStore keeps engine choices in memory.
type choiceStore struct{ m map[string]map[string]string }
func (c *choiceStore) NodeEngines(context.Context) (map[string]map[string]string, error) {
return c.m, nil
}
func (c *choiceStore) SetNodeEngine(_ context.Context, node, engine, variant, _ string) error {
if c.m[node] == nil {
c.m[node] = map[string]string{}
}
c.m[node][engine] = variant
return nil
}
func TestAnEngineBuildIsChosenForANodeAndTheNodeIsTold(t *testing.T) {
a, ctl := setup(t, testModels(), gpuNode("gpu07", 8), inventory.Node{ID: "cpu01", State: inventory.Ready})
ch := control.NewEngineChoices(t.Context(), &choiceStore{m: map[string]map[string]string{}}, slog.New(slog.DiscardHandler))
var told []string
ch.OnChange(func(node string) error { told = append(told, node); return nil })
ctl.SetEngineChoices(ch)
if code := call(t, a, http.MethodPost, "/api/v1/nodes/gpu07/engines", `{"engine":"vllm","build":"cu129"}`, nil); code != http.StatusOK {
t.Fatalf("status %d", code)
}
if len(told) != 1 || told[0] != "gpu07" {
t.Errorf("the node was not told: %v", told)
}
var nodes []NodeView
call(t, a, http.MethodGet, "/api/v1/nodes", "", &nodes)
for _, n := range nodes {
switch n.ID {
case "gpu07":
if n.EngineBuilds["vllm"] != "cu129" {
t.Errorf("gpu07 shows %v", n.EngineBuilds)
}
case "cpu01":
if n.EngineBuilds != nil {
t.Errorf("a node with no GPU has no build to choose: %v", n.EngineBuilds)
}
}
}
for _, bad := range []string{`{"engine":"vllm","build":"cu200"}`, `{"engine":"llamacpp","build":"cu129"}`} {
if code := call(t, a, http.MethodPost, "/api/v1/nodes/gpu07/engines", bad, nil); code != http.StatusBadRequest {
t.Errorf("%s: status %d", bad, code)
}
}
if code := call(t, a, http.MethodPost, "/api/v1/nodes/gpu07/engines", `{"engine":"vllm","build":"auto"}`, nil); code != http.StatusOK ||
ch.For("gpu07")["vllm"] != "" {
t.Errorf("auto should leave it to the node again: %d %v", code, ch.For("gpu07"))
}
}
+29 -3
View File
@@ -43,14 +43,40 @@ func TestEachNodeIsToldTheEnginesDecidedForIt(t *testing.T) {
}
return out
}
if got := want(s.provisionFor(inventory.Node{ID: "volta"}, reg)); got["vllm"] != "0.25.1" {
if got := want(s.provisionFor(inventory.Node{ID: "volta"}, installable(reg))); got["vllm"] != "0.25.1" {
t.Errorf("the Volta node: %v", got)
}
if got := want(s.provisionFor(inventory.Node{ID: "hopper"}, reg)); got["vllm"] != "0.30.0" {
if got := want(s.provisionFor(inventory.Node{ID: "hopper"}, installable(reg))); got["vllm"] != "0.30.0" {
t.Errorf("the Hopper node: %v", got)
}
// A node that disabled vLLM does not offer it, and is not asked for it.
if got := want(s.provisionFor(inventory.Node{ID: "hopper"}, &llmgwv1.Register{InstallableEngines: []string{"llamacpp"}})); got["vllm"] != "" {
if got := want(s.provisionFor(inventory.Node{ID: "hopper"}, []string{"llamacpp"})); got["vllm"] != "" {
t.Errorf("asked a node for an engine it does not install: %v", got)
}
}
func TestABuildChosenForANodeIsWhatItIsTold(t *testing.T) {
s := &Service{}
s.SetEngines(map[string]string{"llamacpp": "b1", "vllm": "0.30.0"})
s.SetEngineChoices(func(node string) map[string]string {
return map[string]map[string]string{"gpu07": {"vllm": "cu129"}, "gpu08": {"vllm": "none"}}[node]
})
can := []string{"llamacpp", "vllm"}
variant := func(node string) (string, bool) {
for _, e := range s.provisionFor(inventory.Node{ID: node}, can).GetEngines() {
if e.GetEngine() == "vllm" {
return e.GetVariant(), true
}
}
return "", false
}
if v, ok := variant("gpu07"); !ok || v != "cu129" {
t.Errorf("gpu07: %q, %v", v, ok)
}
if _, ok := variant("gpu08"); ok {
t.Error("gpu08 was asked for a vLLM it was chosen to run none of")
}
if v, ok := variant("gpu09"); !ok || v != "" {
t.Errorf("gpu09, left to its hardware: %q, %v", v, ok)
}
}
+42 -5
View File
@@ -56,6 +56,10 @@ type Service struct {
// enginesFor is what a node is told to install, set once at start: the
// cluster's engines, with any override that picks the node laid over.
enginesFor func(node inventory.Node) map[string]string
// choices are the engine builds chosen per node, and installableBy what
// each connected node said it can install, for telling it again.
choices func(nodeID string) map[string]string
installableBy map[string][]string
// logWaits are the log requests awaiting a reply, by request id.
logMu sync.Mutex
@@ -78,6 +82,29 @@ func (s *Service) SetEngines(e map[string]string) {
// of two GPU generations needs two vLLM builds, or vLLM on only one.
func (s *Service) SetEnginesFor(f func(node inventory.Node) map[string]string) { s.enginesFor = f }
// SetEngineChoices sets where the builds chosen for each node come from: by
// engine, a build ("cu129") or "none". An engine not named is left to the
// node's hardware.
func (s *Service) SetEngineChoices(f func(nodeID string) map[string]string) { s.choices = f }
// Reprovision tells a connected node again what to install, after a choice
// for it changed; one not connected is told when it next connects.
func (s *Service) Reprovision(nodeID string) error {
if s.enginesFor == nil {
return nil
}
s.mu.RLock()
can, ok := s.installableBy[nodeID]
s.mu.RUnlock()
node, known := s.inv.Get(nodeID)
if !ok || !known {
return nil
}
return s.send(nodeID, &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Provision{Provision: s.provisionFor(node, can)},
})
}
// conn is one agent's open stream.
type conn struct {
nodeID string
@@ -188,7 +215,14 @@ func (s *Service) Connect(
// node rebuilt from nothing is told the same as one that has been here a
// month, and a changed version reaches every node on its next connect.
if s.enginesFor != nil {
p := s.provisionFor(node, reg)
can := installable(reg)
s.mu.Lock()
if s.installableBy == nil {
s.installableBy = map[string][]string{}
}
s.installableBy[nodeID] = can
s.mu.Unlock()
p := s.provisionFor(node, can)
if err := stream.Send(&llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Provision{Provision: p},
}); err != nil {
@@ -856,15 +890,18 @@ func rate(before, now uint64, secs float64) float64 {
// provisionFor is what a node is told to install: the engines decided for
// it, those it can install.
func (s *Service) provisionFor(node inventory.Node, reg *llmgwv1.Register) *llmgwv1.Provision {
func (s *Service) provisionFor(node inventory.Node, can []string) *llmgwv1.Provision {
p := &llmgwv1.Provision{}
can := installable(reg)
wants := s.enginesFor(node)
var chosen map[string]string
if s.choices != nil {
chosen = s.choices(node.ID)
}
for _, e := range slices.Sorted(maps.Keys(wants)) {
if !slices.Contains(can, e) {
if !slices.Contains(can, e) || chosen[e] == "none" {
continue
}
p.Engines = append(p.Engines, &llmgwv1.EngineWant{Engine: e, Version: wants[e]})
p.Engines = append(p.Engines, &llmgwv1.EngineWant{Engine: e, Version: wants[e], Variant: chosen[e]})
}
return p
}
+18 -9
View File
@@ -75,15 +75,18 @@ func DefaultConfig() Config {
// Controller owns the control plane's state.
type Controller struct {
cfg Config
log *slog.Logger
inv *inventory.Store
cat *catalog.Live
rec *reconciler.Reconciler
lead *Leadership
disp reconciler.Dispatcher
sink EndpointSink
usage UsageSource
// engineChoices are the engine builds chosen per node; nil without a
// store to keep them in.
engineChoices *EngineChoices
cfg Config
log *slog.Logger
inv *inventory.Store
cat *catalog.Live
rec *reconciler.Reconciler
lead *Leadership
disp reconciler.Dispatcher
sink EndpointSink
usage UsageSource
mu sync.Mutex
last string // fingerprint of the last published list
@@ -408,3 +411,9 @@ func EngineKeys(secret []byte) func(replicaID string) string {
// QuotaUse is every capacity quota beside what its scope holds.
func (c *Controller) QuotaUse() []reconciler.QuotaUse { return c.rec.QuotaUse() }
// SetEngineChoices gives the controller the engine builds chosen per node.
func (c *Controller) SetEngineChoices(e *EngineChoices) { c.engineChoices = e }
// EngineChoices are the engine builds chosen per node, or nil.
func (c *Controller) EngineChoices() *EngineChoices { return c.engineChoices }
+108
View File
@@ -0,0 +1,108 @@
package control
import (
"context"
"fmt"
"log/slog"
"maps"
"slices"
"strings"
"sync"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
)
// EngineChoiceStore is where the engine builds chosen per node are kept.
type EngineChoiceStore interface {
NodeEngines(ctx context.Context) (map[string]map[string]string, error)
SetNodeEngine(ctx context.Context, node, engine, variant, by string) error
}
// EngineChoices are the engine builds chosen for nodes, rather than left to
// their hardware: vLLM's CUDA 12.9 build on a node its CUDA 13 build would
// also install on, or no vLLM at all. Kept in the store, held here for the
// agent service to ask on every registration, and pushed to a node the
// moment its choice changes.
type EngineChoices struct {
st EngineChoiceStore
log *slog.Logger
mu sync.RWMutex
byNode map[string]map[string]string
notify func(node string) error
}
// NewEngineChoices reads what was chosen before.
func NewEngineChoices(ctx context.Context, st EngineChoiceStore, log *slog.Logger) *EngineChoices {
c := &EngineChoices{st: st, log: log, byNode: map[string]map[string]string{}}
if got, err := st.NodeEngines(ctx); err != nil {
log.Error("could not read the engine builds chosen per node; every node installs what its hardware runs", "err", err)
} else {
c.byNode = got
}
return c
}
// OnChange is called with a node whose choice changed, to tell it.
func (c *EngineChoices) OnChange(f func(node string) error) {
c.mu.Lock()
c.notify = f
c.mu.Unlock()
}
// For is what was chosen for one node, by engine: a build, or "none".
func (c *EngineChoices) For(node string) map[string]string {
c.mu.RLock()
defer c.mu.RUnlock()
return maps.Clone(c.byNode[node])
}
// EngineBuilds are what can be chosen for each engine that has a choice:
// "auto" is the node's hardware deciding, "none" no engine at all.
func EngineBuilds(engine string) []string {
switch engine {
case "vllm":
return append(append([]string{"auto"}, provision.VLLMVariants...), "none")
case "sglang":
return []string{"auto", "none"}
}
return nil
}
// Set chooses an engine's build for a node, keeps it, and tells the node.
func (c *EngineChoices) Set(ctx context.Context, node, engine, build, by string) error {
builds := EngineBuilds(engine)
if builds == nil {
return fmt.Errorf("%s has no builds to choose between", engine)
}
if !slices.Contains(builds, build) {
return fmt.Errorf("%s has no build %q; choose %s", engine, build, strings.Join(builds, ", "))
}
variant := build
if build == "auto" {
variant = ""
}
if err := c.st.SetNodeEngine(ctx, node, engine, variant, by); err != nil {
return err
}
c.mu.Lock()
if variant == "" {
delete(c.byNode[node], engine)
} else {
if c.byNode[node] == nil {
c.byNode[node] = map[string]string{}
}
c.byNode[node][engine] = variant
}
notify := c.notify
c.mu.Unlock()
c.log.Info("engine build chosen for a node", "node", node, "engine", engine, "build", build, "by", by)
if notify != nil {
if err := notify(node); err != nil {
// Kept and applied on its next connection; said, since a node that
// is not connected is the usual reason.
c.log.Warn("the node was not told yet; it is when it next connects", "node", node, "err", err)
}
}
return nil
}