Give each GPU node a vLLM build it can run, chosen on the cluster page
A node that cannot run the CUDA 13 build installs the same version's CUDA 12.9 build from vLLM's GitHub release, with PyTorch's cu129 torch; the install computes on the GPU before it counts. The build can be chosen per node — auto, cu13, cu129, none — on the cluster page, with ctl engine-build, or over the admin API; the control plane keeps the choice and tells the node at once. Refs #2. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
2735c5a318
commit
4f280a3546
32 files changed
+1004
-71
No files matched your search
@@ -0,0 +1,63 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
// choiceStore keeps engine choices in memory.
|
||||
type choiceStore struct{ m map[string]map[string]string }
|
||||
|
||||
func (c *choiceStore) NodeEngines(context.Context) (map[string]map[string]string, error) {
|
||||
return c.m, nil
|
||||
}
|
||||
func (c *choiceStore) SetNodeEngine(_ context.Context, node, engine, variant, _ string) error {
|
||||
if c.m[node] == nil {
|
||||
c.m[node] = map[string]string{}
|
||||
}
|
||||
c.m[node][engine] = variant
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestAnEngineBuildIsChosenForANodeAndTheNodeIsTold(t *testing.T) {
|
||||
a, ctl := setup(t, testModels(), gpuNode("gpu07", 8), inventory.Node{ID: "cpu01", State: inventory.Ready})
|
||||
ch := control.NewEngineChoices(t.Context(), &choiceStore{m: map[string]map[string]string{}}, slog.New(slog.DiscardHandler))
|
||||
var told []string
|
||||
ch.OnChange(func(node string) error { told = append(told, node); return nil })
|
||||
ctl.SetEngineChoices(ch)
|
||||
|
||||
if code := call(t, a, http.MethodPost, "/api/v1/nodes/gpu07/engines", `{"engine":"vllm","build":"cu129"}`, nil); code != http.StatusOK {
|
||||
t.Fatalf("status %d", code)
|
||||
}
|
||||
if len(told) != 1 || told[0] != "gpu07" {
|
||||
t.Errorf("the node was not told: %v", told)
|
||||
}
|
||||
var nodes []NodeView
|
||||
call(t, a, http.MethodGet, "/api/v1/nodes", "", &nodes)
|
||||
for _, n := range nodes {
|
||||
switch n.ID {
|
||||
case "gpu07":
|
||||
if n.EngineBuilds["vllm"] != "cu129" {
|
||||
t.Errorf("gpu07 shows %v", n.EngineBuilds)
|
||||
}
|
||||
case "cpu01":
|
||||
if n.EngineBuilds != nil {
|
||||
t.Errorf("a node with no GPU has no build to choose: %v", n.EngineBuilds)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{`{"engine":"vllm","build":"cu200"}`, `{"engine":"llamacpp","build":"cu129"}`} {
|
||||
if code := call(t, a, http.MethodPost, "/api/v1/nodes/gpu07/engines", bad, nil); code != http.StatusBadRequest {
|
||||
t.Errorf("%s: status %d", bad, code)
|
||||
}
|
||||
}
|
||||
if code := call(t, a, http.MethodPost, "/api/v1/nodes/gpu07/engines", `{"engine":"vllm","build":"auto"}`, nil); code != http.StatusOK ||
|
||||
ch.For("gpu07")["vllm"] != "" {
|
||||
t.Errorf("auto should leave it to the node again: %d %v", code, ch.For("gpu07"))
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user