Give each GPU node a vLLM build it can run, chosen on the cluster page
A node that cannot run the CUDA 13 build installs the same version's CUDA 12.9 build from vLLM's GitHub release, with PyTorch's cu129 torch; the install computes on the GPU before it counts. The build can be chosen per node — auto, cu13, cu129, none — on the cluster page, with ctl engine-build, or over the admin API; the control plane keeps the choice and tells the node at once. Refs #2. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
2735c5a318
commit
4f280a3546
32 files changed
+1004
-71
No files matched your search
@@ -9,6 +9,7 @@ package adminapi
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -111,6 +112,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /api/v1/explain-placement", a.explainPlacement)
|
||||
mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState)
|
||||
mux.HandleFunc("POST /api/v1/nodes/{id}/upgrade", a.upgradeNode)
|
||||
mux.HandleFunc("POST /api/v1/nodes/{id}/engines", a.setNodeEngine)
|
||||
mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode)
|
||||
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
|
||||
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
|
||||
@@ -156,6 +158,10 @@ type NodeView struct {
|
||||
// Provision is where installing each wanted engine stands. An engine is in
|
||||
// Engines only once its installation here is ready.
|
||||
Provision []inventory.EngineInstall `json:"provision,omitempty"`
|
||||
// EngineBuilds are the builds chosen for this node, by engine: "auto"
|
||||
// (its hardware decides), a build ("cu129"), or "none". Only on a node
|
||||
// with GPUs, the only kind with a choice to make.
|
||||
EngineBuilds map[string]string `json:"engine_builds,omitempty"`
|
||||
// OS is the operating system and architecture, as the agent read them.
|
||||
OS string `json:"os,omitempty"`
|
||||
// Warnings are facts about the machine that make it slow or unusable
|
||||
@@ -473,6 +479,13 @@ func (a *API) Nodes() []NodeView {
|
||||
}
|
||||
slices.Sort(v.Engines)
|
||||
v.Provision = n.Provision
|
||||
if ch := a.ctl.EngineChoices(); ch != nil && len(n.GPUs) > 0 {
|
||||
chosen := ch.For(n.ID)
|
||||
v.EngineBuilds = map[string]string{"vllm": "auto"}
|
||||
for e, b := range chosen {
|
||||
v.EngineBuilds[e] = b
|
||||
}
|
||||
}
|
||||
if h := n.Host; h.OSID != "" {
|
||||
v.OS = strings.TrimSpace(h.OSID + " " + h.OSVersion + " " + h.Arch)
|
||||
if h.Arch == "amd64" && !h.HasCPUFlag("avx2") {
|
||||
@@ -1532,3 +1545,47 @@ func (a *API) setAdapters(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
writeJSON(w, http.StatusOK, AdaptersRequest{Adapters: out.Adapters})
|
||||
}
|
||||
|
||||
// NodeEngineRequest chooses one engine's build for a node.
|
||||
type NodeEngineRequest struct {
|
||||
Engine string `json:"engine"`
|
||||
// Build is "auto", a build such as "cu129", or "none".
|
||||
Build string `json:"build"`
|
||||
}
|
||||
|
||||
// setNodeEngine chooses an engine's build for a node, and has the node
|
||||
// install it.
|
||||
func (a *API) setNodeEngine(w http.ResponseWriter, r *http.Request) {
|
||||
ch := a.ctl.EngineChoices()
|
||||
if ch == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store to keep the choice in")
|
||||
return
|
||||
}
|
||||
id := r.PathValue("id")
|
||||
if _, ok := a.ctl.Inventory().Get(id); !ok {
|
||||
writeError(w, http.StatusNotFound, "no node "+id)
|
||||
return
|
||||
}
|
||||
var req NodeEngineRequest
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
if err := ch.Set(r.Context(), id, req.Engine, req.Build, actorOf(r)); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, req)
|
||||
}
|
||||
|
||||
// SetNodeEngine chooses an engine's build for a node, for the cluster page.
|
||||
func (a *API) SetNodeEngine(ctx context.Context, node, engine, build, by string) error {
|
||||
ch := a.ctl.EngineChoices()
|
||||
if ch == nil {
|
||||
return errors.New("this control plane has no store to keep the choice in")
|
||||
}
|
||||
if _, ok := a.ctl.Inventory().Get(node); !ok {
|
||||
return errors.New("no node " + node)
|
||||
}
|
||||
return ch.Set(ctx, node, engine, build, by)
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
// choiceStore keeps engine choices in memory.
|
||||
type choiceStore struct{ m map[string]map[string]string }
|
||||
|
||||
func (c *choiceStore) NodeEngines(context.Context) (map[string]map[string]string, error) {
|
||||
return c.m, nil
|
||||
}
|
||||
func (c *choiceStore) SetNodeEngine(_ context.Context, node, engine, variant, _ string) error {
|
||||
if c.m[node] == nil {
|
||||
c.m[node] = map[string]string{}
|
||||
}
|
||||
c.m[node][engine] = variant
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestAnEngineBuildIsChosenForANodeAndTheNodeIsTold(t *testing.T) {
|
||||
a, ctl := setup(t, testModels(), gpuNode("gpu07", 8), inventory.Node{ID: "cpu01", State: inventory.Ready})
|
||||
ch := control.NewEngineChoices(t.Context(), &choiceStore{m: map[string]map[string]string{}}, slog.New(slog.DiscardHandler))
|
||||
var told []string
|
||||
ch.OnChange(func(node string) error { told = append(told, node); return nil })
|
||||
ctl.SetEngineChoices(ch)
|
||||
|
||||
if code := call(t, a, http.MethodPost, "/api/v1/nodes/gpu07/engines", `{"engine":"vllm","build":"cu129"}`, nil); code != http.StatusOK {
|
||||
t.Fatalf("status %d", code)
|
||||
}
|
||||
if len(told) != 1 || told[0] != "gpu07" {
|
||||
t.Errorf("the node was not told: %v", told)
|
||||
}
|
||||
var nodes []NodeView
|
||||
call(t, a, http.MethodGet, "/api/v1/nodes", "", &nodes)
|
||||
for _, n := range nodes {
|
||||
switch n.ID {
|
||||
case "gpu07":
|
||||
if n.EngineBuilds["vllm"] != "cu129" {
|
||||
t.Errorf("gpu07 shows %v", n.EngineBuilds)
|
||||
}
|
||||
case "cpu01":
|
||||
if n.EngineBuilds != nil {
|
||||
t.Errorf("a node with no GPU has no build to choose: %v", n.EngineBuilds)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{`{"engine":"vllm","build":"cu200"}`, `{"engine":"llamacpp","build":"cu129"}`} {
|
||||
if code := call(t, a, http.MethodPost, "/api/v1/nodes/gpu07/engines", bad, nil); code != http.StatusBadRequest {
|
||||
t.Errorf("%s: status %d", bad, code)
|
||||
}
|
||||
}
|
||||
if code := call(t, a, http.MethodPost, "/api/v1/nodes/gpu07/engines", `{"engine":"vllm","build":"auto"}`, nil); code != http.StatusOK ||
|
||||
ch.For("gpu07")["vllm"] != "" {
|
||||
t.Errorf("auto should leave it to the node again: %d %v", code, ch.For("gpu07"))
|
||||
}
|
||||
}
|
||||
@@ -43,14 +43,40 @@ func TestEachNodeIsToldTheEnginesDecidedForIt(t *testing.T) {
|
||||
}
|
||||
return out
|
||||
}
|
||||
if got := want(s.provisionFor(inventory.Node{ID: "volta"}, reg)); got["vllm"] != "0.25.1" {
|
||||
if got := want(s.provisionFor(inventory.Node{ID: "volta"}, installable(reg))); got["vllm"] != "0.25.1" {
|
||||
t.Errorf("the Volta node: %v", got)
|
||||
}
|
||||
if got := want(s.provisionFor(inventory.Node{ID: "hopper"}, reg)); got["vllm"] != "0.30.0" {
|
||||
if got := want(s.provisionFor(inventory.Node{ID: "hopper"}, installable(reg))); got["vllm"] != "0.30.0" {
|
||||
t.Errorf("the Hopper node: %v", got)
|
||||
}
|
||||
// A node that disabled vLLM does not offer it, and is not asked for it.
|
||||
if got := want(s.provisionFor(inventory.Node{ID: "hopper"}, &llmgwv1.Register{InstallableEngines: []string{"llamacpp"}})); got["vllm"] != "" {
|
||||
if got := want(s.provisionFor(inventory.Node{ID: "hopper"}, []string{"llamacpp"})); got["vllm"] != "" {
|
||||
t.Errorf("asked a node for an engine it does not install: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuildChosenForANodeIsWhatItIsTold(t *testing.T) {
|
||||
s := &Service{}
|
||||
s.SetEngines(map[string]string{"llamacpp": "b1", "vllm": "0.30.0"})
|
||||
s.SetEngineChoices(func(node string) map[string]string {
|
||||
return map[string]map[string]string{"gpu07": {"vllm": "cu129"}, "gpu08": {"vllm": "none"}}[node]
|
||||
})
|
||||
can := []string{"llamacpp", "vllm"}
|
||||
variant := func(node string) (string, bool) {
|
||||
for _, e := range s.provisionFor(inventory.Node{ID: node}, can).GetEngines() {
|
||||
if e.GetEngine() == "vllm" {
|
||||
return e.GetVariant(), true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
if v, ok := variant("gpu07"); !ok || v != "cu129" {
|
||||
t.Errorf("gpu07: %q, %v", v, ok)
|
||||
}
|
||||
if _, ok := variant("gpu08"); ok {
|
||||
t.Error("gpu08 was asked for a vLLM it was chosen to run none of")
|
||||
}
|
||||
if v, ok := variant("gpu09"); !ok || v != "" {
|
||||
t.Errorf("gpu09, left to its hardware: %q, %v", v, ok)
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,10 @@ type Service struct {
|
||||
// enginesFor is what a node is told to install, set once at start: the
|
||||
// cluster's engines, with any override that picks the node laid over.
|
||||
enginesFor func(node inventory.Node) map[string]string
|
||||
// choices are the engine builds chosen per node, and installableBy what
|
||||
// each connected node said it can install, for telling it again.
|
||||
choices func(nodeID string) map[string]string
|
||||
installableBy map[string][]string
|
||||
|
||||
// logWaits are the log requests awaiting a reply, by request id.
|
||||
logMu sync.Mutex
|
||||
@@ -78,6 +82,29 @@ func (s *Service) SetEngines(e map[string]string) {
|
||||
// of two GPU generations needs two vLLM builds, or vLLM on only one.
|
||||
func (s *Service) SetEnginesFor(f func(node inventory.Node) map[string]string) { s.enginesFor = f }
|
||||
|
||||
// SetEngineChoices sets where the builds chosen for each node come from: by
|
||||
// engine, a build ("cu129") or "none". An engine not named is left to the
|
||||
// node's hardware.
|
||||
func (s *Service) SetEngineChoices(f func(nodeID string) map[string]string) { s.choices = f }
|
||||
|
||||
// Reprovision tells a connected node again what to install, after a choice
|
||||
// for it changed; one not connected is told when it next connects.
|
||||
func (s *Service) Reprovision(nodeID string) error {
|
||||
if s.enginesFor == nil {
|
||||
return nil
|
||||
}
|
||||
s.mu.RLock()
|
||||
can, ok := s.installableBy[nodeID]
|
||||
s.mu.RUnlock()
|
||||
node, known := s.inv.Get(nodeID)
|
||||
if !ok || !known {
|
||||
return nil
|
||||
}
|
||||
return s.send(nodeID, &llmgwv1.ControlMessage{
|
||||
Msg: &llmgwv1.ControlMessage_Provision{Provision: s.provisionFor(node, can)},
|
||||
})
|
||||
}
|
||||
|
||||
// conn is one agent's open stream.
|
||||
type conn struct {
|
||||
nodeID string
|
||||
@@ -188,7 +215,14 @@ func (s *Service) Connect(
|
||||
// node rebuilt from nothing is told the same as one that has been here a
|
||||
// month, and a changed version reaches every node on its next connect.
|
||||
if s.enginesFor != nil {
|
||||
p := s.provisionFor(node, reg)
|
||||
can := installable(reg)
|
||||
s.mu.Lock()
|
||||
if s.installableBy == nil {
|
||||
s.installableBy = map[string][]string{}
|
||||
}
|
||||
s.installableBy[nodeID] = can
|
||||
s.mu.Unlock()
|
||||
p := s.provisionFor(node, can)
|
||||
if err := stream.Send(&llmgwv1.ControlMessage{
|
||||
Msg: &llmgwv1.ControlMessage_Provision{Provision: p},
|
||||
}); err != nil {
|
||||
@@ -856,15 +890,18 @@ func rate(before, now uint64, secs float64) float64 {
|
||||
|
||||
// provisionFor is what a node is told to install: the engines decided for
|
||||
// it, those it can install.
|
||||
func (s *Service) provisionFor(node inventory.Node, reg *llmgwv1.Register) *llmgwv1.Provision {
|
||||
func (s *Service) provisionFor(node inventory.Node, can []string) *llmgwv1.Provision {
|
||||
p := &llmgwv1.Provision{}
|
||||
can := installable(reg)
|
||||
wants := s.enginesFor(node)
|
||||
var chosen map[string]string
|
||||
if s.choices != nil {
|
||||
chosen = s.choices(node.ID)
|
||||
}
|
||||
for _, e := range slices.Sorted(maps.Keys(wants)) {
|
||||
if !slices.Contains(can, e) {
|
||||
if !slices.Contains(can, e) || chosen[e] == "none" {
|
||||
continue
|
||||
}
|
||||
p.Engines = append(p.Engines, &llmgwv1.EngineWant{Engine: e, Version: wants[e]})
|
||||
p.Engines = append(p.Engines, &llmgwv1.EngineWant{Engine: e, Version: wants[e], Variant: chosen[e]})
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
@@ -75,15 +75,18 @@ func DefaultConfig() Config {
|
||||
|
||||
// Controller owns the control plane's state.
|
||||
type Controller struct {
|
||||
cfg Config
|
||||
log *slog.Logger
|
||||
inv *inventory.Store
|
||||
cat *catalog.Live
|
||||
rec *reconciler.Reconciler
|
||||
lead *Leadership
|
||||
disp reconciler.Dispatcher
|
||||
sink EndpointSink
|
||||
usage UsageSource
|
||||
// engineChoices are the engine builds chosen per node; nil without a
|
||||
// store to keep them in.
|
||||
engineChoices *EngineChoices
|
||||
cfg Config
|
||||
log *slog.Logger
|
||||
inv *inventory.Store
|
||||
cat *catalog.Live
|
||||
rec *reconciler.Reconciler
|
||||
lead *Leadership
|
||||
disp reconciler.Dispatcher
|
||||
sink EndpointSink
|
||||
usage UsageSource
|
||||
|
||||
mu sync.Mutex
|
||||
last string // fingerprint of the last published list
|
||||
@@ -408,3 +411,9 @@ func EngineKeys(secret []byte) func(replicaID string) string {
|
||||
|
||||
// QuotaUse is every capacity quota beside what its scope holds.
|
||||
func (c *Controller) QuotaUse() []reconciler.QuotaUse { return c.rec.QuotaUse() }
|
||||
|
||||
// SetEngineChoices gives the controller the engine builds chosen per node.
|
||||
func (c *Controller) SetEngineChoices(e *EngineChoices) { c.engineChoices = e }
|
||||
|
||||
// EngineChoices are the engine builds chosen per node, or nil.
|
||||
func (c *Controller) EngineChoices() *EngineChoices { return c.engineChoices }
|
||||
@@ -0,0 +1,108 @@
|
||||
package control
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
|
||||
)
|
||||
|
||||
// EngineChoiceStore is where the engine builds chosen per node are kept.
|
||||
type EngineChoiceStore interface {
|
||||
NodeEngines(ctx context.Context) (map[string]map[string]string, error)
|
||||
SetNodeEngine(ctx context.Context, node, engine, variant, by string) error
|
||||
}
|
||||
|
||||
// EngineChoices are the engine builds chosen for nodes, rather than left to
|
||||
// their hardware: vLLM's CUDA 12.9 build on a node its CUDA 13 build would
|
||||
// also install on, or no vLLM at all. Kept in the store, held here for the
|
||||
// agent service to ask on every registration, and pushed to a node the
|
||||
// moment its choice changes.
|
||||
type EngineChoices struct {
|
||||
st EngineChoiceStore
|
||||
log *slog.Logger
|
||||
|
||||
mu sync.RWMutex
|
||||
byNode map[string]map[string]string
|
||||
notify func(node string) error
|
||||
}
|
||||
|
||||
// NewEngineChoices reads what was chosen before.
|
||||
func NewEngineChoices(ctx context.Context, st EngineChoiceStore, log *slog.Logger) *EngineChoices {
|
||||
c := &EngineChoices{st: st, log: log, byNode: map[string]map[string]string{}}
|
||||
if got, err := st.NodeEngines(ctx); err != nil {
|
||||
log.Error("could not read the engine builds chosen per node; every node installs what its hardware runs", "err", err)
|
||||
} else {
|
||||
c.byNode = got
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// OnChange is called with a node whose choice changed, to tell it.
|
||||
func (c *EngineChoices) OnChange(f func(node string) error) {
|
||||
c.mu.Lock()
|
||||
c.notify = f
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// For is what was chosen for one node, by engine: a build, or "none".
|
||||
func (c *EngineChoices) For(node string) map[string]string {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return maps.Clone(c.byNode[node])
|
||||
}
|
||||
|
||||
// EngineBuilds are what can be chosen for each engine that has a choice:
|
||||
// "auto" is the node's hardware deciding, "none" no engine at all.
|
||||
func EngineBuilds(engine string) []string {
|
||||
switch engine {
|
||||
case "vllm":
|
||||
return append(append([]string{"auto"}, provision.VLLMVariants...), "none")
|
||||
case "sglang":
|
||||
return []string{"auto", "none"}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Set chooses an engine's build for a node, keeps it, and tells the node.
|
||||
func (c *EngineChoices) Set(ctx context.Context, node, engine, build, by string) error {
|
||||
builds := EngineBuilds(engine)
|
||||
if builds == nil {
|
||||
return fmt.Errorf("%s has no builds to choose between", engine)
|
||||
}
|
||||
if !slices.Contains(builds, build) {
|
||||
return fmt.Errorf("%s has no build %q; choose %s", engine, build, strings.Join(builds, ", "))
|
||||
}
|
||||
variant := build
|
||||
if build == "auto" {
|
||||
variant = ""
|
||||
}
|
||||
if err := c.st.SetNodeEngine(ctx, node, engine, variant, by); err != nil {
|
||||
return err
|
||||
}
|
||||
c.mu.Lock()
|
||||
if variant == "" {
|
||||
delete(c.byNode[node], engine)
|
||||
} else {
|
||||
if c.byNode[node] == nil {
|
||||
c.byNode[node] = map[string]string{}
|
||||
}
|
||||
c.byNode[node][engine] = variant
|
||||
}
|
||||
notify := c.notify
|
||||
c.mu.Unlock()
|
||||
c.log.Info("engine build chosen for a node", "node", node, "engine", engine, "build", build, "by", by)
|
||||
if notify != nil {
|
||||
if err := notify(node); err != nil {
|
||||
// Kept and applied on its next connection; said, since a node that
|
||||
// is not connected is the usual reason.
|
||||
c.log.Warn("the node was not told yet; it is when it next connects", "node", node, "err", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user