From d7f1c1d54542a4b45d892b68750ca62bb23598cf Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 30 Sep 2026 10:19:31 +0200 Subject: [PATCH] Open a held model to its replicas, nodes and room for more The Held tab's model name opens a card: each replica with its node, state, age, CPU and memory, the room for more found by dry-running the scheduler on a copy of the inventory, and a form for the always-running and under-load replica counts. An always-running count placement cannot meet is refused with the reason. Headroom is also on the admin API. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 13 ++ internal/control/adminapi/api.go | 40 ++++ internal/control/inventory/inventory.go | 17 ++ internal/control/reconciler/headroom.go | 86 ++++++++ internal/control/reconciler/headroom_test.go | 59 ++++++ internal/web/admin_mirror.go | 22 +- internal/web/mirror_detail.go | 207 +++++++++++++++++++ internal/web/mirror_detail_test.go | 145 +++++++++++++ internal/web/static/base.css | 20 +- internal/web/templates/admin_mirror.html | 77 ++++++- internal/web/web.go | 1 + 11 files changed, 683 insertions(+), 4 deletions(-) create mode 100644 internal/control/reconciler/headroom.go create mode 100644 internal/control/reconciler/headroom_test.go create mode 100644 internal/web/mirror_detail.go create mode 100644 internal/web/mirror_detail_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e2a36b42..f1cf68c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,19 @@ Held reloads while something downloads. A download started from the explorer links to Held, where its progress is; a fetch the mirror refused comes back on the form it was sent from. +### A held model opens to where it runs, and how many + +Clicking a model under Held opens a card beneath it: each replica with its +node, state, age and the CPU and memory its engine uses, and how many more +replicas the cluster has room for now, on which nodes, and why not more. The +replicas it always runs and the most it grows to under load are changed there. +Room is found by trying the scheduler itself on a copy of the inventory — the +same decisions the next passes would make, not an estimate beside them — and a +number of always-running replicas it cannot place is refused with the reason, +rather than accepted and left short. A ceiling above the room is taken, and +said. The same answer is at `GET /api/v1/models/{name}/headroom` on the admin +API. + ## v0.10.4 — 2026-09-30 Published first as release candidates v0.10.4-rc.1 to rc.9. diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 3730ac07..cc817ff9 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -15,6 +15,7 @@ import ( "io" "net/http" "slices" + "strconv" "strings" "time" "unicode" @@ -91,6 +92,15 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("GET /api/v1/jobs/{id}", a.job) mux.HandleFunc("GET /api/v1/requests/{id}", a.request) mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel) + mux.HandleFunc("GET /api/v1/models/{name}/headroom", func(w http.ResponseWriter, r *http.Request) { + limit, _ := strconv.Atoi(r.URL.Query().Get("limit")) + v, ok := a.Headroom(r.PathValue("name"), limit) + if !ok { + http.Error(w, "no such model", http.StatusNotFound) + return + } + writeJSON(w, http.StatusOK, v) + }) } // --- responses --- @@ -336,6 +346,22 @@ type ExplainView struct { Rejections []RejectionView `json:"rejections,omitempty"` } +// HeadroomView is how many more replicas of a model fit now, and where. +type HeadroomView struct { + Model string `json:"model"` + // Current is the replicas holding capacity: starting or ready. + Current int `json:"current"` + // More is how many more placement would find room for now. + More int `json:"more"` + // Nodes is where each of those would go. + Nodes [][]string `json:"nodes,omitempty"` + // Limited is set when More stopped at the limit asked rather than at the + // cluster's edge: there may be room for more. + Limited bool `json:"limited,omitempty"` + // Blocked says why one more does not fit, a line per node considered. + Blocked []string `json:"blocked,omitempty"` +} + // RejectionView is one rejected candidate. type RejectionView struct { Profile string `json:"profile"` @@ -836,6 +862,20 @@ func (a *API) explainPlacement(w http.ResponseWriter, r *http.Request) { // It does not check that the model exists; the handler above does that so it // can answer 404, while the dashboard only ever asks about models it just read // out of the catalog. +// Headroom tries placing up to limit more replicas of a model, on a copy of +// the inventory, and says how many fit. A limit of zero or less takes 16. +func (a *API) Headroom(model string, limit int) (HeadroomView, bool) { + if limit <= 0 || limit > 64 { + limit = 16 + } + h, ok := a.ctl.Reconciler().Headroom(model, limit) + if !ok { + return HeadroomView{}, false + } + return HeadroomView{Model: model, Current: h.Current, More: h.More, Nodes: h.Nodes, + Limited: h.Limited, Blocked: h.Blocked}, true +} + func (a *API) Explain(model string) ExplainView { v := ExplainView{Model: model} for _, rep := range a.ctl.Reconciler().Replicas() { diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index 7cef569f..8bfcce57 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -392,6 +392,23 @@ func NewStore() *Store { return &Store{nodes: map[string]*Node{}, allocs: map[string][]Allocation{}} } +// Clone is a copy to try decisions on without making them: allocations made +// on it are its own. Nodes are copied by value; the slices inside them are +// shared, and a clone must not write through them. +func (s *Store) Clone() *Store { + s.mu.RLock() + defer s.mu.RUnlock() + c := NewStore() + for id, n := range s.nodes { + cp := *n + c.nodes[id] = &cp + } + for id, list := range s.allocs { + c.allocs[id] = slices.Clone(list) + } + return c +} + // Upsert records a node's registration, preserving operator-set state such as // a cordon: an agent reconnecting must not silently undo an operator's decision. // RecordSMChange keeps a port's change of master subnet manager, the last diff --git a/internal/control/reconciler/headroom.go b/internal/control/reconciler/headroom.go new file mode 100644 index 00000000..fae46a58 --- /dev/null +++ b/internal/control/reconciler/headroom.go @@ -0,0 +1,86 @@ +package reconciler + +import ( + "slices" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/placement" +) + +// Headroom is how many more replicas of a model the cluster has room for now. +type Headroom struct { + // Current is the replicas the model has that hold capacity: starting or + // ready, not those on their way out. + Current int + // More is how many more could be placed now, at most the limit asked. + More int + // Nodes is where each of those would go, in order, a replica's nodes + // joined when it spans several. + Nodes [][]string + // Limited is set when More reached the limit asked, so there may be room + // for more than that. A CPU profile that declares no memory is always + // limited: nothing it asks for runs out. + Limited bool + // Blocked is why one more than More does not fit, a line per node that + // was considered. Empty when Limited. + Blocked []string +} + +// Headroom tries placing up to limit more replicas of a model on a copy of +// the inventory, each one's allocation made on the copy before the next is +// tried, exactly as the reconciler would place them one pass at a time. +// +// The scheduler's own decisions, not an estimate beside them: what the page +// offers is then what the next passes will do, and a count that the +// arithmetic of a second implementation allowed but placement refuses would +// leave a model short with nobody told why. +func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) { + m, ok := r.cat.Load().Resolve(model) + if !ok { + return Headroom{}, false + } + var h Headroom + var avoid []string + r.mu.Lock() + for _, rep := range r.replicas { + if rep.Model == m.Name && rep.State.Counts() && rep.RetiringAt.IsZero() { + h.Current++ + avoid = append(avoid, rep.NodeIDs()...) + } + } + r.mu.Unlock() + + inv := r.inv.Clone() + sched := placement.New(inv) + for h.More < limit { + res := sched.Place(m.PlacementProfiles(), avoid) + if res.Placement == nil && len(avoid) > 0 { + // As scaleUp does: spreading is a preference, not a rule. + res = sched.Place(m.PlacementProfiles(), nil) + } + if res.Placement == nil { + for _, rej := range res.Rejections { + h.Blocked = append(h.Blocked, rej.String()) + } + slices.Sort(h.Blocked) + h.Blocked = slices.Compact(h.Blocked) + return h, true + } + mem := 0 + if prof, ok := m.ProfileByName(res.Placement.Profile.Name); ok { + mem = prof.MinMemoryGB + } + var nodes []string + for _, a := range res.Placement.Nodes { + inv.Allocate(a.NodeID, inventory.Allocation{ + ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem, + }) + nodes = append(nodes, a.NodeID) + } + avoid = append(avoid, nodes...) + h.Nodes = append(h.Nodes, nodes) + h.More++ + } + h.Limited = true + return h, true +} diff --git a/internal/control/reconciler/headroom_test.go b/internal/control/reconciler/headroom_test.go new file mode 100644 index 00000000..0bf3c1d7 --- /dev/null +++ b/internal/control/reconciler/headroom_test.go @@ -0,0 +1,59 @@ +package reconciler + +import ( + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +func TestHeadroomIsWhatPlacementWouldDoNext(t *testing.T) { + // Two 8-GPU nodes and a 4-GPU profile: four replicas in all, one placed. + h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8)) + h.r.Reconcile(t.Context()) + h.markReady() + + got, ok := h.r.Headroom("llama", 10) + if !ok { + t.Fatal("the model was not found") + } + if got.Current != 1 || got.More != 3 || got.Limited { + t.Fatalf("headroom %+v, expected 1 placed and room for 3 more", got) + } + // Spread first: the next one goes to the node without one. + if first := got.Nodes[0][0]; first == h.r.Replicas()[0].NodeIDs()[0] { + t.Errorf("the next replica would stack on %s beside the first", first) + } + if len(got.Blocked) == 0 || !strings.Contains(strings.Join(got.Blocked, "\n"), "GPU") { + t.Errorf("no reason given for the fifth: %v", got.Blocked) + } + // Tried on a copy: nothing was taken from the real inventory. + if free := len(h.inv.FreeGPUs("1")) + len(h.inv.FreeGPUs("2")); free != 12 { + t.Errorf("%d GPUs free after asking, expected the 12 the one replica leaves", free) + } + + capped, _ := h.r.Headroom("llama", 2) + if capped.More != 2 || !capped.Limited || capped.Blocked != nil { + t.Errorf("asked for at most 2: %+v", capped) + } + if _, ok := h.r.Headroom("nope", 5); ok { + t.Error("an unknown model has headroom") + } +} + +func TestHeadroomCountsMemoryAlreadyClaimed(t *testing.T) { + cpu := func(id string) inventory.Node { + return inventory.Node{ID: id, MgmtAddr: "10.0.0." + id, State: inventory.Ready, MemoryTotalMB: 16 * 1024} + } + m := catalog.Model{Name: "small", Source: "/m.gguf", Scaling: catalog.Scaling{Min: 1, Max: 1}, + Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 6}}} + h := newHarness(t, m, cpu("a"), cpu("b")) + h.r.Reconcile(t.Context()) + + // 16 GB each at 6 GB a replica: two per node, one of them taken. + got, _ := h.r.Headroom("small", 10) + if got.Current != 1 || got.More != 3 { + t.Errorf("headroom %+v, expected room for 3 more", got) + } +} diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go index eecc1126..a9f6d1c3 100644 --- a/internal/web/admin_mirror.go +++ b/internal/web/admin_mirror.go @@ -67,6 +67,8 @@ type modelUse struct { Name string Mode string // always, ondemand or stopped Replicas int + // Min and Max are its scaling as the catalog holds it. + Min, Max int // Declared models are read from server.yaml on every start, so a mode // changed here would revert; the page says where to change it instead. Declared bool @@ -83,6 +85,11 @@ type mirrorRow struct { // operator clicked. Nil on a row that is still arriving: a catalog entry // pointing at a partial file is the thing versioned paths exist to prevent. Serve *serveForm + // Detail is the card opened beneath the row: what runs it, on which + // nodes, and how many replicas it runs. + Detail *heldDetail + // Open is the row's ?open= value. + Open string // HasPercent is false when the upstream never sent a length. The bar is // then not drawn at all: a progress bar that has to invent its own total is @@ -274,6 +281,8 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) { return name + " now runs all the time", nil })) + mux.HandleFunc("POST /admin/mirror/replicas", p.mirrorAction(m, p.setReplicas)) + mux.HandleFunc("POST /admin/mirror/serve", p.mirrorAction(m, func(ctx context.Context, admin store.User, r *http.Request) (string, error) { model, err := buildModel(r.FormValue) @@ -428,11 +437,22 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request, for _, m := range sources[d.Rows[i].URL] { d.Rows[i].UsedBy = append(d.Rows[i].UsedBy, modelUse{ Name: m.Name, Mode: modeOf(m.Scaling), Replicas: max(m.Scaling.Max, 1), + Min: m.Scaling.Min, Max: m.Scaling.Max, Declared: p.catalog.Declared(m.Name), }) } } + // The one row whose card is open. From the form too, so a change made in + // the card comes back with the card still open to show it. + open := r.FormValue("open") + for i := range d.Rows { + d.Rows[i].Open = openKey(d.Rows[i].Model, d.Rows[i].Version) + if open != "" && d.Rows[i].Open == open && d.Rows[i].State == mirror.Present { + d.Rows[i].Detail = p.heldDetailFor(d.Rows[i]) + } + } + // The one row whose serve form is open, and only when the catalog can be // changed at all. if p.catalog != nil { @@ -476,7 +496,7 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request, // Only on the list: a reload anywhere else takes with it a search, an // open repository or a URL half typed. - if d.Fetching > 0 && d.Tab == tabHeld { + if d.Fetching > 0 && d.Tab == tabHeld && open == "" { d.RefreshSec = 5 } d.Tabs = mirrorTabs(d) diff --git a/internal/web/mirror_detail.go b/internal/web/mirror_detail.go new file mode 100644 index 00000000..c32d8da3 --- /dev/null +++ b/internal/web/mirror_detail.go @@ -0,0 +1,207 @@ +package web + +import ( + "context" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// HeadroomSource is a fleet that can say how many more replicas of a model +// fit now. Optional: without it the card shows what runs and offers no change +// it cannot check. +type HeadroomSource interface { + Headroom(model string, limit int) (adminapi.HeadroomView, bool) +} + +// maxReplicas bounds what the form accepts. Not a cluster limit, a typing one: +// 300 where 3 was meant would otherwise be taken as a ceiling. +const maxReplicas = 64 + +// heldDetail is the card opened beneath a held version: what runs it, where, +// and room to change it. +type heldDetail struct { + Models []heldModel +} + +// heldModel is one catalog model running a held version. +type heldModel struct { + modelUse + View adminapi.ModelView + Replicas []heldReplica + Headroom *adminapi.HeadroomView + // MostNow is how many replicas could always run, Current plus the room + // found; zero when the room was not measured or had no end in sight. + MostNow int +} + +// heldReplica is one replica, by the nodes it runs on. +type heldReplica struct { + ID string + State string + Restart string + Age int64 + Endpoint string + Nodes []heldShard +} + +type heldShard struct { + Node string + MemoryGB int // the node's RAM, 0 when unknown + CPU float64 + MemBytes int64 +} + +// openKey names a row the way ?open= does. +func openKey(model, version string) string { return model + ":" + version } + +// heldDetailFor reads, for each model using a row, how it runs and how much +// more room it has. +func (p *AccountPages) heldDetailFor(row mirrorRow) *heldDetail { + d := &heldDetail{} + if p.fleet == nil { + for _, u := range row.UsedBy { + d.Models = append(d.Models, heldModel{modelUse: u}) + } + return d + } + views := map[string]adminapi.ModelView{} + for _, v := range p.fleet.Models() { + views[v.Name] = v + } + memory := map[string]int{} + for _, n := range p.fleet.Nodes() { + memory[n.ID] = n.MemoryGB + } + hs, _ := p.fleet.(HeadroomSource) + for _, u := range row.UsedBy { + hm := heldModel{modelUse: u, View: views[u.Name]} + for _, rep := range p.fleet.Replicas(u.Name) { + // Built field by field: a ReplicaView carries the engine's key, + // which is for the admin listener and never for a page. + hr := heldReplica{ID: rep.ID, State: rep.State, Restart: rep.Restart, + Age: rep.AgeSec, Endpoint: rep.Endpoint} + for _, sh := range rep.Shards { + hr.Nodes = append(hr.Nodes, heldShard{Node: sh.Node, MemoryGB: memory[sh.Node], + CPU: sh.CPUCores, MemBytes: sh.MemoryBytes}) + } + hm.Replicas = append(hm.Replicas, hr) + } + if hs != nil { + if h, ok := hs.Headroom(u.Name, 16); ok { + hm.Headroom = &h + if !h.Limited { + hm.MostNow = h.Current + h.More + } + } + } + d.Models = append(d.Models, hm) + } + return d +} + +// setReplicas changes how many replicas a model always runs and how far it +// may grow, refusing an always-on count the cluster has no room for. +// +// Refused rather than taken and left short: a minimum placement cannot meet +// is a model reported as unplaceable on every pass from then on, which is the +// same answer given later and less plainly. A maximum above the room is +// accepted and said: it is reached only under load, by which time something +// may have made room, and refusing it would make the ceiling depend on what +// happened to be running when it was set. +func (p *AccountPages) setReplicas(ctx context.Context, admin store.User, r *http.Request) (string, error) { + name := r.FormValue("model") + lo, err1 := strconv.Atoi(strings.TrimSpace(r.FormValue("min"))) + hi, err2 := strconv.Atoi(strings.TrimSpace(r.FormValue("max"))) + switch { + case name == "": + return "", errors.New("which model?") + case err1 != nil || err2 != nil: + return "", errors.New("replicas are whole numbers") + case lo < 0 || hi < lo: + return "", fmt.Errorf("always running (%d) cannot be more than the most under load (%d)", lo, hi) + case hi > maxReplicas: + return "", fmt.Errorf("at most %d replicas", maxReplicas) + } + current, err := p.currentModel(ctx, name) + if err != nil { + return "", err + } + if p.catalog.Declared(name) { + return "", fmt.Errorf("%s is set in server.yaml; change it there", name) + } + + note := "" + if hs, ok := p.fleet.(HeadroomSource); ok && hi > 0 { + h, ok := hs.Headroom(name, hi) + if !ok { + return "", fmt.Errorf("no model %q", name) + } + fits := h.Current + h.More + if lo > fits && !h.Limited { + why := "nothing more fits" + if len(h.Blocked) > 0 { + why = strings.Join(h.Blocked, "; ") + } + return "", fmt.Errorf("%s can always run at most %d now (%d running, room for %d more): %s", + name, fits, h.Current, h.More, why) + } + if hi > fits && !h.Limited { + note = fmt.Sprintf("; there is room for %d now, so it grows past that only if room is made", fits) + } + } + + sc := catalog.Scaling{Min: lo, Max: hi, IdleTimeout: current.Scaling.IdleTimeout} + if sc.OnDemand() && sc.IdleTimeout <= 0 { + sc.IdleTimeout = defaultIdle + } + if _, err := p.catalog.Scale(ctx, name, sc, admin.Email); err != nil { + return "", err + } + switch { + case hi == 0: + return name + " is stopped; its replicas are being taken down", nil + case lo == 0: + return fmt.Sprintf("%s loads when asked for, up to %d under load%s", name, hi, note), nil + case lo == hi: + return fmt.Sprintf("%s runs %s", name, plural(lo, "replica", "replicas")), nil + } + return fmt.Sprintf("%s runs %d, and up to %d under load%s", name, lo, hi, note), nil +} + +// currentModel is a model as the catalog holds it now. +func (p *AccountPages) currentModel(ctx context.Context, name string) (catalog.Model, error) { + sources, err := p.catalog.Sources(ctx) + if err != nil { + return catalog.Model{}, err + } + for _, ms := range sources { + for _, m := range ms { + if m.Name == name { + return m, nil + } + } + } + return catalog.Model{}, fmt.Errorf("no model %q runs from the mirror", name) +} + +// age renders seconds the way the replicas table does. +func age(sec int64) string { + d := time.Duration(sec) * time.Second + switch { + case d < time.Minute: + return fmt.Sprintf("%ds", sec) + case d < time.Hour: + return fmt.Sprintf("%dm", int(d.Minutes())) + case d < 48*time.Hour: + return fmt.Sprintf("%dh%02dm", int(d.Hours()), int(d.Minutes())%60) + } + return fmt.Sprintf("%dd", int(d.Hours()/24)) +} diff --git a/internal/web/mirror_detail_test.go b/internal/web/mirror_detail_test.go new file mode 100644 index 00000000..d9c434f2 --- /dev/null +++ b/internal/web/mirror_detail_test.go @@ -0,0 +1,145 @@ +package web + +import ( + "net/url" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" +) + +// roomyFleet runs one replica of "coder" on n2 and has room for `more`. +type roomyFleet struct { + fakeFleet + more int + blocked []string + asked int +} + +func (f *roomyFleet) Nodes() []adminapi.NodeView { + return []adminapi.NodeView{{ID: "n1", MemoryGB: 16}, {ID: "n2", MemoryGB: 32}} +} +func (f *roomyFleet) Models() []adminapi.ModelView { + return []adminapi.ModelView{{Name: "coder", Min: 1, Max: 1, Ready: 1, Total: 1, Desired: 1}} +} +func (f *roomyFleet) Replicas(string) []adminapi.ReplicaView { + return []adminapi.ReplicaView{{ID: "coder-abc", Model: "coder", State: "ready", AgeSec: 3700, + EngineKey: "secret-engine-key", + Shards: []adminapi.ShardView{{Node: "n2", CPUCores: 3.5, MemoryBytes: 6 << 30}}}} +} +func (f *roomyFleet) Headroom(model string, limit int) (adminapi.HeadroomView, bool) { + f.asked = limit + more := min(f.more, limit) + h := adminapi.HeadroomView{Model: model, Current: 1, More: more} + for range more { + h.Nodes = append(h.Nodes, []string{"n1"}) + } + if more < limit { + h.Blocked = f.blocked + } else { + h.Limited = true + } + return h, true +} + +func detailPage(t *testing.T, fleet *roomyFleet, cat *fakeCatalog) func(string) string { + t.Helper() + m := &fakeMirror{models: []mirror.Status{ + {Model: "coder", Version: "v1", File: "c.gguf", State: mirror.Present, BytesDone: 5 << 30}, + }} + mux := mirrorPageDeps(t, MirrorDeps{Mirror: m, Catalog: cat, Fleet: fleet}, true) + return func(path string) string { return mirrorGet(t, mux, path).Body.String() } +} + +func coderCatalog() *fakeCatalog { + return &fakeCatalog{sources: map[string][]catalog.Model{"http://mirror:8081/coder/v1/c.gguf": { + {Name: "coder", Scaling: catalog.Scaling{Min: 1, Max: 1}}}}} +} + +func TestAHeldModelOpensToWhereItRunsAndHowMuchRoomIsLeft(t *testing.T) { + fleet := &roomyFleet{more: 2, blocked: []string{"n2: 2 GB free of 32, profile needs 6"}} + get := detailPage(t, fleet, coderCatalog()) + + closed := get("/admin/mirror") + if strings.Contains(closed, "held-card") { + t.Error("a card is open before anything was clicked") + } + if !strings.Contains(closed, `href="/admin/mirror?tab=held&open=coder%3av1"`) { + t.Error("the model's name does not open its card") + } + + body := get("/admin/mirror?tab=held&open=coder:v1") + for _, want := range []string{ + "held-card", `n2`, "32 GB", "coder-abc", "1h01m", "3.5 cores", "6.0 GiB", + "Room for 2 more now, on n1, n1.", "n2: 2 GB free of 32, profile needs 6", + `name="min" value="1" min="0" max="3"`, `action="/admin/mirror/replicas"`, + `name="open" value="coder:v1"`, + } { + if !strings.Contains(body, want) { + t.Errorf("the card is missing %q", want) + } + } + if strings.Contains(body, "secret-engine-key") { + t.Error("the engine's key reached the page") + } + if strings.Contains(body, `http-equiv="refresh"`) { + t.Error("a page with an open form reloads under the operator") + } +} + +func TestAReplicaCountTheClusterHasNoRoomForIsRefused(t *testing.T) { + fleet := &roomyFleet{more: 1, blocked: []string{"n1: 1 GB free of 16, profile needs 6"}} + cat := coderCatalog() + mux := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{models: []mirror.Status{ + {Model: "coder", Version: "v1", File: "c.gguf", State: mirror.Present, BytesDone: 1}, + }}, Catalog: cat, Fleet: fleet}, true) + post := func(lo, hi string) string { + return mirrorPost(t, mux, "/admin/mirror/replicas", url.Values{"csrf": {"tok"}, + "model": {"coder"}, "min": {lo}, "max": {hi}, "open": {"coder:v1"}}).Body.String() + } + + body := post("3", "3") + if len(cat.scaled) != 0 { + t.Fatalf("three always-on replicas were set with room for two: %+v", cat.lastScale) + } + if !strings.Contains(body, "can always run at most 2 now") || !strings.Contains(body, "1 GB free of 16") { + t.Errorf("the refusal should say how many fit and why: %s", body) + } + if !strings.Contains(body, "held-card") { + t.Error("the card closed on the refusal") + } + + body = post("2", "4") + if len(cat.scaled) != 1 || cat.lastScale.Min != 2 || cat.lastScale.Max != 4 { + t.Fatalf("what fits was not set: %+v", cat.lastScale) + } + if !strings.Contains(body, "coder runs 2, and up to 4 under load; there is room for 2 now") { + t.Errorf("a ceiling above the room should be taken and said: %s", body) + } + + post("0", "2") + if cat.lastScale.Min != 0 || cat.lastScale.Max != 2 || cat.lastScale.IdleTimeout == 0 { + t.Errorf("on demand was set as %+v", cat.lastScale) + } + for _, bad := range [][2]string{{"3", "2"}, {"-1", "1"}, {"x", "1"}, {"1", "100"}} { + n := len(cat.scaled) + post(bad[0], bad[1]) + if len(cat.scaled) != n { + t.Errorf("min %s max %s was accepted", bad[0], bad[1]) + } + } +} + +func TestAModelSetInConfigurationIsShownButNotChanged(t *testing.T) { + cat := coderCatalog() + cat.declared = "coder" + body := detailPage(t, &roomyFleet{more: 20}, cat)("/admin/mirror?open=coder:v1") + if strings.Contains(body, `action="/admin/mirror/replicas"`) || !strings.Contains(body, "change its replicas there") { + t.Error("a declared model offered a change that would revert") + } + if !strings.Contains(body, "Room for at least 16 more") { + t.Error("room that reached the limit asked should read as at least") + } +} diff --git a/internal/web/static/base.css b/internal/web/static/base.css index 74780395..cfb0404c 100644 --- a/internal/web/static/base.css +++ b/internal/web/static/base.css @@ -60,7 +60,7 @@ li { margin: .25rem 0; } a { color: var(--accent); } .wrap.narrow { max-width: 30rem; } label { display: block; margin: .9rem 0; font-size: .9rem; font-weight: 600; } -input[type=text], input[type=email], input[type=password], input[type=search] { +input[type=text], input[type=email], input[type=password], input[type=search], input[type=number] { display: block; width: 100%; margin-top: .3rem; font: inherit; font-weight: 400; padding: .5rem .6rem; border: 1px solid var(--line); border-radius: 6px; background: var(--card); color: var(--fg); @@ -813,3 +813,21 @@ details.chat-settings output { font-variant-numeric: tabular-nums; font-weight: .copyable button { margin-right: .75rem; font-size: .8rem; } .connect-form .field-row label { flex: 1 1 12rem; } table.connect-fields th { white-space: nowrap; } + +/* The card opened beneath a held version on the mirror page. */ +tr.held-detail > td { background: var(--bg); padding: .75rem 1rem 1rem; } +.held-card .held-model { margin-top: .75rem; padding-top: .75rem; border-top: 1px solid var(--line); } +.held-card .held-model:first-of-type { border-top: 0; padding-top: 0; } +.held-card h3 { font-size: 1rem; margin: 0 0 .5rem; display: flex; gap: .6rem; align-items: baseline; flex-wrap: wrap; } +.held-card table.replicas { margin: .25rem 0 .5rem; } +.held-card .headroom { margin: .5rem 0 .25rem; font-weight: 600; } +.held-card details.blocked { font-size: .85rem; color: var(--muted); margin-bottom: .5rem; } +.held-card details.blocked ul { margin: .35rem 0 0 1.1rem; padding: 0; } +.held-card .replicas-form .field-row { align-items: flex-end; } +.held-card .replicas-form .field-row label { flex: 0 0 auto; } +.held-card .replicas-form input[type=number] { width: 8rem; margin: .25rem 0 0; } +.held-card .replicas-form button { margin: 0; } +a.open-held { text-decoration: none; color: inherit; } +a.open-held:hover strong { text-decoration: underline; } +a.open-held[aria-expanded=true]::before { content: "▾ "; color: var(--muted); } +a.open-held[aria-expanded=false]::before { content: "▸ "; color: var(--muted); } diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html index 54887ed6..364534bb 100644 --- a/internal/web/templates/admin_mirror.html +++ b/internal/web/templates/admin_mirror.html @@ -44,8 +44,10 @@ reachable from this machine.

{{if .Failed}}failed{{end}} {{if .InUse}}in use{{end}} {{end}} -{{range .Rows}} - {{.Model}} {{.Version}} +{{range .Rows}}{{$row0 := .}} + {{if eq (printf "%s" .State) "present"}}{{.Model}} + {{else}}{{.Model}}{{end}} {{.Version}} {{if gt .Files 1}}{{.Files}} files{{else}}{{.File}}{{end}} @@ -112,6 +114,7 @@ reachable from this machine.

{{if .Err}}{{.Err}}{{end}} +{{with .Detail}}{{template "mirror_held_detail" dict "D" . "Row" $row0 "CSRF" $.CSRF}}{{end}} {{with .Serve}} @@ -429,3 +432,73 @@ computes once it has it.

No weight files in {{.Repo.ID}}.

{{end}} {{end}} + +{{/* The card beneath a held version: what runs it, where, and how many. */}} +{{define "mirror_held_detail"}}{{$row := .Row}}{{$csrf := .CSRF}} +
+

+ {{if gt $row.Files 1}}{{$row.Files}} files{{else}}{{$row.File}}{{end}} + {{bytes $row.BytesDone}} + {{with $row.SHA256}}sha256 {{slice . 0 12}}{{end}} + {{$row.URL}} +

+{{if not .D.Models}} +

Not in the catalog: nothing runs this version. +{{if $row.Serve}}{{else}}Use serve on its row to put it into service.{{end}}

+{{end}} +{{range .D.Models}} +
+

{{.Name}} + {{with .View}}{{.Ready}} of {{.Desired}} ready{{if ne .Total .Ready}} · {{.Total}} in all{{end}} + {{if .Parked}}parked{{else if .Unplaceable}}does not fit{{end}}{{end}} +

+{{with .View.Reason}}

{{.}}

{{end}} + +{{if .Replicas}} + + +{{range .Replicas}}{{$rep := .}}{{range $i, $s := .Nodes}} + + + + + + +{{end}}{{end}} +
ReplicaNodeStateAgeCPUMemory
{{if eq $i 0}}{{$rep.ID}}{{end}}{{$s.Node}}{{if $s.MemoryGB}} {{$s.MemoryGB}} GB{{end}}{{if eq $i 0}}{{$rep.State}} + {{with $rep.Restart}}restart {{.}}{{end}}{{end}}{{if eq $i 0}}{{age $rep.Age}}{{end}}{{if $s.CPU}}{{printf "%.1f" $s.CPU}} cores{{else}}—{{end}}{{if $s.MemBytes}}{{bytes $s.MemBytes}}{{else}}—{{end}}
+{{else}}

No replica runs now{{if eq .Mode "ondemand"}}: it starts when somebody asks for it{{end}}.

{{end}} + +{{with .Headroom}} +

+{{if .Limited}}Room for at least {{.More}} more replicas. +{{else if .More}}Room for {{.More}} more now{{if .Nodes}}, on {{range $i, $n := .Nodes}}{{if $i}}, {{end}}{{join $n}}{{end}}{{end}}. +{{else}}No room for another replica now.{{end}} +

+{{if and (not .Limited) .Blocked}}
Why not {{if .More}}more{{else}}another{{end}} +
    {{range .Blocked}}
  • {{.}}
  • {{end}}
{{end}} +{{end}} + +{{if .Declared}} +

Set in server.yaml, which is read again on every start: change its replicas there.

+{{else}} +
+ + + +
+ + + +
+

{{if .MostNow}}At most {{.MostNow}} can always run with what is free now. {{end}}0 always running with a + higher limit loads it when asked for; 0 and 0 stops it. A replica is added under load when requests wait, and + taken away after ten quiet minutes.

+
+{{end}} +
+{{end}} +
+{{end}} diff --git a/internal/web/web.go b/internal/web/web.go index 52ed4d68..21ca6051 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -109,6 +109,7 @@ var funcs = template.FuncMap{ return p }, "ago": ago, + "age": age, // pctbar is a percentage as the minibar template takes it: whole, and // held to 0..100 so a measurement a little over does not draw past its box. "pctbar": func(v float64) struct{ P int } {