Give each GPU node a vLLM build it can run, chosen on the cluster page
A node that cannot run the CUDA 13 build installs the same version's CUDA 12.9 build from vLLM's GitHub release, with PyTorch's cu129 torch; the install computes on the GPU before it counts. The build can be chosen per node — auto, cu13, cu129, none — on the cluster page, with ctl engine-build, or over the admin API; the control plane keeps the choice and tells the node at once. Refs #2. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
2735c5a318
commit
4f280a3546
32 files changed
+1004
-71
No files matched your search
@@ -0,0 +1,49 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// NodeEngineAdmin chooses an engine's build for a node: "auto", a build
|
||||
// such as "cu129", or "none".
|
||||
type NodeEngineAdmin interface {
|
||||
SetNodeEngine(ctx context.Context, node, engine, build, by string) error
|
||||
}
|
||||
|
||||
// RegisterNodeEngines mounts the choice of engine builds on the cluster
|
||||
// page: for an administrator, with the session's CSRF token, and answered by
|
||||
// going back to the page with what happened.
|
||||
func (p *AccountPages) RegisterNodeEngines(mux *http.ServeMux, a NodeEngineAdmin) {
|
||||
mux.HandleFunc("POST /admin/nodes/{id}/engines", p.adminOnly(func(w http.ResponseWriter, r *http.Request, admin store.User) {
|
||||
sess, _, _ := p.session(r)
|
||||
if !p.accounts.CheckCSRF(sess, r.FormValue("csrf")) {
|
||||
http.Error(w, "bad request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
node, engine, build := r.PathValue("id"), r.FormValue("engine"), r.FormValue("build")
|
||||
q := url.Values{}
|
||||
if err := a.SetNodeEngine(r.Context(), node, engine, build, admin.Email); err != nil {
|
||||
p.log.Warn("an engine build was not chosen", "admin", admin.Email, "node", node, "err", err)
|
||||
q.Set("problem", node+": "+err.Error())
|
||||
} else {
|
||||
p.log.Info("an engine build was chosen", "admin", admin.Email, "node", node, "engine", engine, "build", build)
|
||||
q.Set("notice", node+" installs "+engine+" "+buildWords(build)+
|
||||
"; the node is told now, and installs it in the background")
|
||||
}
|
||||
http.Redirect(w, r, "/admin?"+q.Encode(), http.StatusSeeOther)
|
||||
}))
|
||||
}
|
||||
|
||||
func buildWords(b string) string {
|
||||
switch b {
|
||||
case "auto":
|
||||
return "in the build its hardware runs"
|
||||
case "none":
|
||||
return "no longer"
|
||||
}
|
||||
return "built for " + map[string]string{"cu13": "CUDA 13", "cu129": "CUDA 12.9"}[b]
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// buildFleet is one GPU node whose vLLM build was chosen, and one CPU node.
|
||||
type buildFleet struct{ fakeFleet }
|
||||
|
||||
func (buildFleet) Nodes() []adminapi.NodeView {
|
||||
return []adminapi.NodeView{
|
||||
{ID: "gpu07", State: "ready", GPUTotal: 8, GPUFree: 8,
|
||||
EngineBuilds: map[string]string{"vllm": "cu129"},
|
||||
Provision: []inventory.EngineInstall{{Engine: "vllm", Version: "0.30.0", Variant: "cu129", State: "installing"}}},
|
||||
{ID: "cpu01", State: "ready"},
|
||||
}
|
||||
}
|
||||
|
||||
type recordedChoice struct{ node, engine, build, by string }
|
||||
|
||||
type fakeEngineAdmin struct{ got []recordedChoice }
|
||||
|
||||
func (f *fakeEngineAdmin) SetNodeEngine(_ context.Context, node, engine, build, by string) error {
|
||||
f.got = append(f.got, recordedChoice{node, engine, build, by})
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestTheVLLMBuildIsChosenOnTheClusterPage(t *testing.T) {
|
||||
a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}}
|
||||
p := NewAccountPages(a, nil, nil, "v0", slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /admin", p.AdminPage(Dashboard(buildFleet{}, nil, "v1", time.Second, false, NavExtras{}, nil)))
|
||||
fe := &fakeEngineAdmin{}
|
||||
p.RegisterNodeEngines(mux, fe)
|
||||
|
||||
body := mirrorGet(t, mux, "/admin").Body.String()
|
||||
if !strings.Contains(body, `action="/admin/nodes/gpu07/engines"`) ||
|
||||
!strings.Contains(body, `<option value="cu129" selected>CUDA 12.9</option>`) {
|
||||
t.Fatalf("no choice of build for the GPU node, with its current one selected: %s", body)
|
||||
}
|
||||
if strings.Contains(body, `action="/admin/nodes/cpu01/engines"`) {
|
||||
t.Error("a node with no GPU was offered a vLLM build")
|
||||
}
|
||||
if !strings.Contains(body, "CUDA 12.9 installing") {
|
||||
t.Error("what is installed is not shown beside the choice")
|
||||
}
|
||||
|
||||
// The read-only page on the admin listener has no session, so no form.
|
||||
_, ro := get(t, Dashboard(buildFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
|
||||
if strings.Contains(ro, "/engines\"") {
|
||||
t.Error("the admin listener's page offers a change it cannot sign")
|
||||
}
|
||||
|
||||
w := mirrorPost(t, mux, "/admin/nodes/gpu07/engines", url.Values{"csrf": {"tok"}, "engine": {"vllm"}, "build": {"none"}})
|
||||
if w.Code != http.StatusSeeOther || len(fe.got) != 1 || fe.got[0] != (recordedChoice{"gpu07", "vllm", "none", "a@example.com"}) {
|
||||
t.Fatalf("status %d, chose %+v", w.Code, fe.got)
|
||||
}
|
||||
if loc := w.Header().Get("Location"); !strings.Contains(loc, "notice=") {
|
||||
t.Errorf("redirected to %s, saying nothing", loc)
|
||||
}
|
||||
if w := mirrorPost(t, mux, "/admin/nodes/gpu07/engines", url.Values{"csrf": {"bad"}, "engine": {"vllm"}, "build": {"none"}}); w.Code != http.StatusBadRequest || len(fe.got) != 1 {
|
||||
t.Errorf("a forged form was accepted: %d", w.Code)
|
||||
}
|
||||
}
|
||||
@@ -19,13 +19,16 @@ type DashboardData struct {
|
||||
// of a GPU table that would only say there are none.
|
||||
CPU cpuCapacity
|
||||
// Menu is the shared top menu; Nav below is what the cluster page adds.
|
||||
Menu Nav
|
||||
Nodes []adminapi.NodeView
|
||||
Models []adminapi.ModelView
|
||||
Replicas []adminapi.ReplicaView
|
||||
Unplaced []adminapi.ExplainView
|
||||
Usage []store.KeyUsage
|
||||
Summary adminapi.Summary
|
||||
Menu Nav
|
||||
// Notice and Problem are what the last change made from this page did,
|
||||
// carried over the redirect back to it.
|
||||
Notice, Problem string
|
||||
Nodes []adminapi.NodeView
|
||||
Models []adminapi.ModelView
|
||||
Replicas []adminapi.ReplicaView
|
||||
Unplaced []adminapi.ExplainView
|
||||
Usage []store.KeyUsage
|
||||
Summary adminapi.Summary
|
||||
// Fabrics is the physical fabrics and what disagrees about them, from a
|
||||
// source that can say.
|
||||
Fabrics *fabrics.Report
|
||||
@@ -170,6 +173,9 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat
|
||||
Now: time.Now().UTC().Format(time.RFC3339),
|
||||
AuthEnabled: usage != nil,
|
||||
}
|
||||
if d.Menu.Admin {
|
||||
d.Notice, d.Problem = r.URL.Query().Get("notice"), r.URL.Query().Get("problem")
|
||||
}
|
||||
if src != nil {
|
||||
d.Nodes = src.Nodes()
|
||||
d.CPU = cpuCapacityOf(d.Nodes)
|
||||
|
||||
@@ -50,8 +50,9 @@
|
||||
{{else}}<p class="empty">No nodes have registered.</p>{{end}}
|
||||
|
||||
<h2>Nodes</h2>
|
||||
{{with .Notice}}<p class="notice">{{.}}</p>{{end}}{{with .Problem}}<p class="error">{{.}}</p>{{end}}
|
||||
{{if .Nodes}}
|
||||
<table class="nodes rows"><tr><th>Node</th><th>State</th><th>CPU</th><th>RAM</th>{{if .AnyGPUs}}<th class="num">GPUs</th>{{end}}<th class="num">Disk</th><th>Engines</th>{{if .AnyClasses}}<th>Classes</th>{{end}}{{if .AnyFabrics}}<th>Fabrics</th>{{end}}<th class="num">Models</th><th class="num">Replicas</th><th>Seen</th></tr>
|
||||
<table class="nodes rows"><tr><th>Node</th><th>State</th><th>CPU</th><th>RAM</th>{{if .AnyGPUs}}<th class="num">GPUs</th>{{end}}<th class="num">Disk</th><th>Engines</th>{{if .AnyGPUs}}<th>vLLM build</th>{{end}}{{if .AnyClasses}}<th>Classes</th>{{end}}{{if .AnyFabrics}}<th>Fabrics</th>{{end}}<th class="num">Models</th><th class="num">Replicas</th><th>Seen</th></tr>
|
||||
{{range .Nodes}}<tr>
|
||||
<td><a href="{{if $.OnAdminListener}}/nodes/{{.ID}}{{else}}/admin/nodes/{{.ID}}{{end}}" title="{{.ID}}"><code>{{shortNode .ID $.NodePrefix}}</code></a>{{if .MgmtAddr}} <span class="note">{{.MgmtAddr}}</span>{{end}}
|
||||
<a class="note" href="{{if $.OnAdminListener}}/nodes/{{.ID}}/logs{{else}}/admin/nodes/{{.ID}}/logs{{end}}">logs</a></td>
|
||||
@@ -66,6 +67,23 @@
|
||||
was never asked about its disk must not read as a full one. */}}
|
||||
<td class="num">{{if .DiskTotalGB}}{{.DiskFreeGB}}G/{{.DiskTotalGB}}G{{else}}—{{end}}</td>
|
||||
<td>{{if .Engines}}{{join .Engines}}{{else}}<span class="note">none yet</span>{{end}}</td>
|
||||
{{/* Which vLLM build the node installs: left to its hardware, or chosen.
|
||||
What is installed beside it, so a choice and its outcome are read
|
||||
together. Only a GPU node has one; changing it is for an
|
||||
administrator, on the page reached through a session. */}}
|
||||
{{if $.AnyGPUs}}<td class="clip">{{if .GPUTotal}}{{$node := .}}
|
||||
{{range .Provision}}{{if eq .Engine "vllm"}}<span class="note" title="{{.Detail}}">{{with .Variant}}{{buildLabel .}} {{end}}{{.State}}</span>{{end}}{{end}}
|
||||
{{if and $.Menu.CSRF .EngineBuilds}}
|
||||
<form method="post" action="/admin/nodes/{{.ID}}/engines" class="inline build-choice">
|
||||
<input type="hidden" name="csrf" value="{{$.Menu.CSRF}}">
|
||||
<input type="hidden" name="engine" value="vllm">
|
||||
<select name="build" title="the vLLM build this node installs">
|
||||
{{range engineBuilds "vllm"}}<option value="{{.}}"{{if eq . (index $node.EngineBuilds "vllm")}} selected{{end}}>{{buildLabel .}}</option>{{end}}
|
||||
</select>
|
||||
<noscript><button type="submit">Save</button></noscript>
|
||||
</form>
|
||||
{{end}}
|
||||
{{else}}—{{end}}</td>{{end}}
|
||||
{{if $.AnyClasses}}<td>{{join .Classes}}</td>{{end}}
|
||||
{{if $.AnyFabrics}}<td>{{join .Fabrics}}</td>{{end}}
|
||||
<td class="num">{{if .Models}}{{len .Models}}{{else}}—{{end}}</td>
|
||||
@@ -75,6 +93,13 @@
|
||||
{{range .Provision}}{{if and (ne .State "ready") (ne .State "skipped")}}<tr><td colspan="12" class="note{{if or (eq .State "failed") (eq .State "unsupported")}} bad{{end}}">{{.Engine}} {{.Version}}{{if .Variant}} ({{.Variant}}){{end}} {{.State}}: {{.Detail}}</td></tr>{{end}}{{end}}
|
||||
{{range .Warnings}}<tr><td colspan="12" class="note bad">{{.}}</td></tr>{{end}}
|
||||
{{end}}</table>
|
||||
<script>
|
||||
// A build chosen is saved as it is chosen, with the page reloaded on the
|
||||
// answer: one select, no separate button to forget.
|
||||
document.querySelectorAll("form.build-choice select").forEach(function (s) {
|
||||
s.addEventListener("change", function () { s.form.submit(); });
|
||||
});
|
||||
</script>
|
||||
{{else}}<p class="empty">No agents have registered.</p>{{end}}
|
||||
|
||||
<h2>Models</h2>
|
||||
|
||||
+20
-1
@@ -26,6 +26,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
@@ -118,7 +119,25 @@ var funcs = template.FuncMap{
|
||||
},
|
||||
// shortNode is a node's name without what every node's name starts with.
|
||||
"shortNode": strings.TrimPrefix,
|
||||
"age": age,
|
||||
// engineBuilds are the builds an engine can be given on a node, and
|
||||
// buildLabel says one in words.
|
||||
"engineBuilds": control.EngineBuilds,
|
||||
"buildLabel": func(b string) string {
|
||||
switch b {
|
||||
case "auto":
|
||||
return "auto: what the hardware runs"
|
||||
case "cu13":
|
||||
return "CUDA 13"
|
||||
case "cu129":
|
||||
return "CUDA 12.9"
|
||||
case "cu12":
|
||||
return "CUDA 12"
|
||||
case "none":
|
||||
return "none"
|
||||
}
|
||||
return b
|
||||
},
|
||||
"age": age,
|
||||
// pctbar is a percentage as the minibar template takes it: whole, and
|
||||
// held to 0..100 so a measurement a little over does not draw past its box.
|
||||
"pctbar": func(v float64) struct{ P int } {
|
||||
|
||||
Reference in new issue
Block a user