Compare commits

...
1 Commits
Author SHA1 Message Date
Ilker ManapandClaude Opus 5.5 f288f3c423 Set capacity quotas, class, owner and reservation from the admin pages
Release / build (push) Successful in 1m9s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m49s
The quotas page gains a capacity section over the same API methods the CLI
uses; a model's card sets its class, owner, memory and cores, applying only
what changed. Headroom counts the replicas a model could get by preempting
what its class outranks, and the card and ctl model target use it.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 22:40:45 +02:00
20 changed files with 709 additions and 44 deletions

No files matched your search

+22
View File
@@ -20,6 +20,28 @@ is only released once its **exit criterion** is met, not when its code lands.
Until v1.0.0 the minor version is the breaking one: configuration formats and Go
APIs change between minors without a deprecation period.
## Unreleased
Changes since v0.10.5, published as release candidates (v0.10.6-rc.N) until
there are enough for v0.10.6.
### Capacity from the admin pages
What v0.10.5 made settable from the command line is now on the pages too:
- **Quotas** has a Capacity section: every scope's guaranteed and most GPUs,
cores and memory beside what it holds, with a form to set, change or remove
one. It goes through the same checks as `llmgw ctl quota`.
- **A model's card** on the mirror page sets its class, its owner (when there
are quotas to count against), and the memory and cores a replica reserves,
beside its replica counts. Only what changed is applied, and each change is
said.
- **Room counts what a class may take.** The card, `llmgw ctl model target` and
the headroom API count the replicas a model could get by stopping those its
class outranks, separately from free room, and name whose they would be. A
critical model's card no longer refuses a replica that preemption would make
room for.
## v0.10.5 — 2026-09-30
Published first as release candidates v0.10.5-rc.1 to rc.14; rc.5 failed its
+4
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"net/url"
"strconv"
"strings"
"github.com/spf13/cobra"
@@ -192,6 +193,9 @@ func newTargetCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
switch {
case v.More == 0:
fmt.Fprintln(out, "It already has them.")
case v.Fits && len(v.Preempts) > 0:
fmt.Fprintf(out, "%d more; there is room once replicas of %s are stopped, which its class may do.\n",
v.More, strings.Join(v.Preempts, ", "))
case v.Fits:
fmt.Fprintf(out, "%d more, and there is room for them.\n", v.More)
default:
+6 -1
View File
@@ -408,6 +408,10 @@ type HeadroomView struct {
Blocked []string `json:"blocked,omitempty"`
// TokPerSec is what one more replica was measured to serve; 0 unmeasured.
TokPerSec float64 `json:"tok_per_sec,omitempty"`
// ByPreempting is how many more would fit by stopping replicas this
// model's class may take; Preempts the models those belong to.
ByPreempting int `json:"by_preempting,omitempty"`
Preempts []string `json:"preempts,omitempty"`
}
// RejectionView is one rejected candidate.
@@ -967,7 +971,8 @@ func (a *API) Headroom(model string, limit int) (HeadroomView, bool) {
return HeadroomView{}, false
}
return HeadroomView{Model: model, Current: h.Current, More: h.More, Nodes: h.Nodes,
Limited: h.Limited, Blocked: h.Blocked, TokPerSec: h.TokPerSec}, true
Limited: h.Limited, Blocked: h.Blocked, TokPerSec: h.TokPerSec,
ByPreempting: h.ByPreempting, Preempts: h.Preempts}, true
}
func (a *API) Explain(model string) ExplainView {
+66 -18
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"io"
"net/http"
"sort"
"strings"
"time"
@@ -72,38 +73,24 @@ func (a *API) registerQuotas(mux *http.ServeMux) {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
}
scope, id, _, err := a.resolveScope(r.Context(), req.Scope)
key, err := a.SetQuota(r.Context(), req, actorOf(r))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
if !req.Guarantee.Any() && !req.Limit.Any() {
writeError(w, http.StatusBadRequest, "a quota needs a guarantee or a limit; to remove one, DELETE it")
return
}
q := store.CapacityQuota{Scope: scope, ScopeID: id, Guarantee: req.Guarantee, Limit: req.Limit,
UpdatedBy: actorOf(r)}
if err := a.quotas.PutCapacityQuota(r.Context(), q); err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"scope": quota.Key(scope, id)})
writeJSON(w, http.StatusOK, map[string]string{"scope": key})
})
mux.HandleFunc("DELETE /api/v1/quotas/{scope}", func(w http.ResponseWriter, r *http.Request) {
if a.quotas == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no store")
return
}
scope, id, _, err := a.resolveScope(r.Context(), r.PathValue("scope"))
key, err := a.ClearQuota(r.Context(), r.PathValue("scope"))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
if err := a.quotas.DeleteCapacityQuota(r.Context(), scope, id); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"removed": quota.Key(scope, id)})
writeJSON(w, http.StatusOK, map[string]string{"removed": key})
})
mux.HandleFunc("POST /api/v1/models/{name}/owner", func(w http.ResponseWriter, r *http.Request) {
if a.models == nil || a.quotas == nil {
@@ -137,6 +124,9 @@ func (a *API) registerQuotas(mux *http.ServeMux) {
// QuotaViews lists every quota with what its scope holds now.
func (a *API) QuotaViews(ctx context.Context) ([]QuotaView, error) {
if a.quotas == nil {
return nil, nil
}
qs, err := a.quotas.CapacityQuotas(ctx)
if err != nil {
return nil, err
@@ -235,3 +225,61 @@ func (a *API) resolveScope(ctx context.Context, s string) (store.Scope, string,
}
return "", "", "", fmt.Errorf("%q could be any of %d %ss; give its ID", what, len(found), kind)
}
// SetQuota sets a scope's capacity quota, named by path or ID, and returns
// the scope as org:ID.
func (a *API) SetQuota(ctx context.Context, req QuotaRequest, by string) (string, error) {
if a.quotas == nil {
return "", fmt.Errorf("this control plane has no store")
}
scope, id, _, err := a.resolveScope(ctx, req.Scope)
if err != nil {
return "", err
}
if !req.Guarantee.Any() && !req.Limit.Any() {
return "", fmt.Errorf("a quota needs a guarantee or a limit; to remove one, clear it")
}
q := store.CapacityQuota{Scope: scope, ScopeID: id, Guarantee: req.Guarantee, Limit: req.Limit, UpdatedBy: by}
if err := a.quotas.PutCapacityQuota(ctx, q); err != nil {
return "", err
}
return quota.Key(scope, id), nil
}
// ClearQuota removes a scope's capacity quota.
func (a *API) ClearQuota(ctx context.Context, scope string) (string, error) {
if a.quotas == nil {
return "", fmt.Errorf("this control plane has no store")
}
s, id, _, err := a.resolveScope(ctx, scope)
if err != nil {
return "", err
}
if err := a.quotas.DeleteCapacityQuota(ctx, s, id); err != nil {
return "", err
}
return quota.Key(s, id), nil
}
// ScopeName is a part of the tenancy tree: org:ID and its path.
type ScopeName struct {
Key string `json:"scope"`
Name string `json:"name"`
}
// Scopes lists every organization, team and project, by path.
func (a *API) Scopes(ctx context.Context) ([]ScopeName, error) {
if a.quotas == nil {
return nil, nil
}
names, err := a.scopeNames(ctx)
if err != nil {
return nil, err
}
out := make([]ScopeName, 0, len(names))
for k, n := range names {
out = append(out, ScopeName{Key: k, Name: n})
}
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
return out, nil
}
+8 -2
View File
@@ -24,6 +24,9 @@ type TargetView struct {
Room int `json:"room"`
Fits bool `json:"fits"`
Blocked []string `json:"blocked,omitempty"`
// Preempts is whose replicas would be stopped to make the room, when
// free room alone is not enough; Room counts that room too.
Preempts []string `json:"preempts,omitempty"`
}
// Target sizes a model for a throughput.
@@ -40,8 +43,11 @@ func (a *API) Target(model string, tokPerSec float64) (TargetView, error) {
}
v.Current = h.Current
v.More = max(v.Needed-h.Current, 0)
v.Room = h.More
v.Fits = v.More <= h.More || h.Limited
v.Room = h.More + h.ByPreempting
v.Fits = v.More <= v.Room || h.Limited
if v.More > h.More {
v.Preempts = h.Preempts
}
if !v.Fits {
v.Blocked = h.Blocked
}
+31 -3
View File
@@ -28,6 +28,11 @@ type Headroom struct {
// TokPerSec is what one more replica was measured to serve, on the
// profile placement would give it; zero when it was never measured.
TokPerSec float64
// ByPreempting is how many more, beyond More, would fit by stopping
// replicas the model's class may take, and Preempts the models those
// replicas belong to: what "room" means for a model that outranks others.
ByPreempting int
Preempts []string
}
// Headroom tries placing up to limit more replicas of a model on a copy of
@@ -61,12 +66,29 @@ func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) {
if pol != nil {
usage = pol.Usage(r.Holdings())
}
for h.More < limit {
place := func() placement.Result {
res := sched.Place(r.orderedProfiles(m), avoid)
if res.Placement == nil && len(avoid) > 0 {
// As scaleUp does: spreading is a preference, not a rule.
res = sched.Place(r.orderedProfiles(m), nil)
}
return res
}
// What the model's class may take, in the order preemption would, each
// released on the copy only when free room has run out.
victims, nextVictim, preempting := r.victims(m), 0, false
for h.More+h.ByPreempting < limit {
res := place()
for res.Placement == nil && nextVictim < len(victims) {
v := victims[nextVictim]
nextVictim++
inv.ReleaseReplica(v.ID)
if !slices.Contains(h.Preempts, v.Model) {
h.Preempts = append(h.Preempts, v.Model)
}
preempting = true
res = place()
}
if res.Placement == nil {
for _, rej := range res.Rejections {
h.Blocked = append(h.Blocked, rej.String())
@@ -103,8 +125,14 @@ func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) {
nodes = append(nodes, a.NodeID)
}
avoid = append(avoid, nodes...)
h.Nodes = append(h.Nodes, nodes)
h.More++
// Once anything was stopped, what fits after is counted as fitting
// by preempting: it may be sitting in the room that was freed.
if preempting {
h.ByPreempting++
} else {
h.Nodes = append(h.Nodes, nodes)
h.More++
}
}
h.Limited = true
return h, true
@@ -312,3 +312,22 @@ func TestOfTwoThatWouldEachMakeRoomTheSmallerIsStopped(t *testing.T) {
}
}
}
func TestHeadroomCountsWhatAHigherClassCouldTake(t *testing.T) {
h := fillThenWant(t, gpuModel("vital", catalog.ClassCritical, 1), gpuModel("chat", "", 1), gpuNode("1", 8))
got, _ := h.r.Headroom("vital", 3)
if got.More != 0 || got.ByPreempting != 1 || len(got.Preempts) != 1 || got.Preempts[0] != "chat" {
t.Errorf("a critical model should see one more by stopping chat: %+v", got)
}
// Tried on a copy: nothing was stopped.
if replicasOf(h, "chat") != 1 {
t.Fatal("asking about room stopped a replica")
}
// A model of chat's own class may not take its minimum, so has no such room.
peer := newFleetHarness(t, []catalog.Model{gpuModel("chat", "", 1), withMin(gpuModel("peer", "", 1), 0)}, gpuNode("1", 8))
peer.settle(t, 3)
peer.advance(peer.r.cfg.MinLifetime + time.Minute)
if got, _ := peer.r.Headroom("peer", 3); got.ByPreempting != 0 {
t.Errorf("a standard model counted room in another's minimum: %+v", got)
}
}
+5
View File
@@ -185,6 +185,11 @@ func (r *Router) Handler() http.Handler {
pages.RegisterAdminUsers(mux, r.store)
// Quotas are written on the same tree the accounts page builds, so
// they are mounted together.
// Capacity quotas are the control plane's, reached through the
// same admin API the CLI uses.
if c, ok := r.clusterView.(web.CapacityAdmin); ok {
pages.SetCapacity(c)
}
pages.RegisterAdminQuotas(mux, r.store, r.acct)
// And the report of what that tree actually used.
pages.RegisterAdminUsage(mux, r.store, r.acct)
+2
View File
@@ -61,6 +61,8 @@ type AccountPages struct {
hasMirror bool
// hasQuotas is set when /admin/quotas is registered.
hasQuotas bool
// capacity sets capacity quotas, from the quotas page; nil leaves them off.
capacity CapacityAdmin
// personalOff refuses personal keys; see SetPersonalOff.
personalOff bool
// personalDefaults is what personal use falls under by default.
+138
View File
@@ -0,0 +1,138 @@
package web
import (
"context"
"fmt"
"net/http"
"strconv"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// CapacityAdmin sets what parts of the tenancy tree are guaranteed of the
// cluster and may hold. The admin API implements it; the page and the CLI go
// through the same checks.
type CapacityAdmin interface {
QuotaViews(ctx context.Context) ([]adminapi.QuotaView, error)
SetQuota(ctx context.Context, req adminapi.QuotaRequest, by string) (string, error)
ClearQuota(ctx context.Context, scope string) (string, error)
Scopes(ctx context.Context) ([]adminapi.ScopeName, error)
}
// SetCapacity gives the quotas page capacity quotas to show and set. Without
// it the section is left off.
func (p *AccountPages) SetCapacity(c CapacityAdmin) { p.capacity = c }
// capacityRow is one scope's capacity quota as the page shows it.
type capacityRow struct {
adminapi.QuotaView
GPUs, Cores, Memory string
}
// capacityForm is what the form starts with: a scope's quota to change, or
// nothing for a new one.
type capacityForm struct {
Scope string
GPUs, Cores, Memory string
LGPUs, LCores, LMem string
}
func (p *AccountPages) capacitySection(ctx context.Context, d *adminQuotasData, editing string) {
if p.capacity == nil {
return
}
d.HasCapacity = true
views, err := p.capacity.QuotaViews(ctx)
if err != nil {
d.Error = err.Error()
return
}
for _, v := range views {
d.Capacity = append(d.Capacity, capacityRow{QuotaView: v,
GPUs: capCell(v.Holds.GPUs, v.Guarantee.GPUs, v.Limit.GPUs),
Cores: capCell(v.Holds.CPUCores, v.Guarantee.CPUCores, v.Limit.CPUCores),
Memory: capCell(v.Holds.MemoryGB, v.Guarantee.MemoryGB, v.Limit.MemoryGB)})
if v.Scope == editing {
d.CapForm = capacityForm{Scope: v.Scope,
GPUs: intStr(v.Guarantee.GPUs), Cores: intStr(v.Guarantee.CPUCores), Memory: intStr(v.Guarantee.MemoryGB),
LGPUs: intStr(v.Limit.GPUs), LCores: intStr(v.Limit.CPUCores), LMem: intStr(v.Limit.MemoryGB)}
}
}
if d.Scopes, err = p.capacity.Scopes(ctx); err != nil {
d.Error = err.Error()
}
}
// capCell is "6 of 8 (at most 16)": held, guaranteed, limit.
func capCell(holds int, g, l *int) string {
if g == nil && l == nil {
return ""
}
out := strconv.Itoa(holds)
if g != nil {
out += " of " + strconv.Itoa(*g)
}
if l != nil {
out += fmt.Sprintf(" (at most %d)", *l)
}
return out
}
func intStr(v *int) string {
if v == nil {
return ""
}
return strconv.Itoa(*v)
}
// applyCapacity sets or clears one scope's capacity quota from the form.
func (p *AccountPages) applyCapacity(r *http.Request, admin store.User) (string, error) {
scope := strings.TrimSpace(r.FormValue("scope"))
if scope == "" {
return "", fmt.Errorf("choose where the quota applies")
}
if r.FormValue("clear") != "" {
key, err := p.capacity.ClearQuota(r.Context(), scope)
if err != nil {
return "", err
}
return "capacity quota removed from " + key, nil
}
field := func(name, what string) (*int, error) {
v := strings.TrimSpace(r.FormValue(name))
if v == "" {
return nil, nil
}
n, err := strconv.Atoi(v)
if err != nil || n < 0 {
return nil, fmt.Errorf("%s must be a whole number, or blank for none", what)
}
return &n, nil
}
var req adminapi.QuotaRequest
req.Scope = scope
var err error
for _, f := range []struct {
dst **int
name string
what string
}{
{&req.Guarantee.GPUs, "gpus", "guaranteed GPUs"},
{&req.Guarantee.CPUCores, "cores", "guaranteed cores"},
{&req.Guarantee.MemoryGB, "memory", "guaranteed memory"},
{&req.Limit.GPUs, "limit_gpus", "the GPU limit"},
{&req.Limit.CPUCores, "limit_cores", "the core limit"},
{&req.Limit.MemoryGB, "limit_memory", "the memory limit"},
} {
if *f.dst, err = field(f.name, f.what); err != nil {
return "", err
}
}
key, err := p.capacity.SetQuota(r.Context(), req, admin.Email)
if err != nil {
return "", err
}
return "capacity quota set on " + key + "; it takes effect within half a minute", nil
}
+88
View File
@@ -0,0 +1,88 @@
package web
import (
"context"
"errors"
"log/slog"
"net/http"
"net/url"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
type fakeCapacity struct {
views []adminapi.QuotaView
set []adminapi.QuotaRequest
cleared []string
}
func (f *fakeCapacity) QuotaViews(context.Context) ([]adminapi.QuotaView, error) { return f.views, nil }
func (f *fakeCapacity) SetQuota(_ context.Context, r adminapi.QuotaRequest, _ string) (string, error) {
if r.Scope == "team:nope" {
return "", errors.New(`there is no team "nope"`)
}
f.set = append(f.set, r)
return r.Scope, nil
}
func (f *fakeCapacity) ClearQuota(_ context.Context, s string) (string, error) {
f.cleared = append(f.cleared, s)
return s, nil
}
func (f *fakeCapacity) Scopes(context.Context) ([]adminapi.ScopeName, error) {
return []adminapi.ScopeName{{Key: "org:o1", Name: "Acme"}, {Key: "team:t1", Name: "Acme / ml"}}, nil
}
func TestCapacityQuotasAreShownAndSetFromTheQuotasPage(t *testing.T) {
eight, sixteen := 8, 16
fc := &fakeCapacity{views: []adminapi.QuotaView{{Scope: "team:t1", Name: "Acme / ml",
Guarantee: store.Resources{GPUs: &eight}, Limit: store.Resources{GPUs: &sixteen},
Holds: quota.Amount{GPUs: 6}, UpdatedBy: "ops@acme"}}}
a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}}
p := NewAccountPages(a, nil, nil, "v0", slog.New(slog.DiscardHandler))
p.SetCapacity(fc)
mux := http.NewServeMux()
p.RegisterAdminQuotas(mux, &fakeQuotas{}, pricedYes{})
body := mirrorGet(t, mux, "/admin/quotas").Body.String()
for _, want := range []string{`id="capacity"`, "Acme / ml", "6 of 8 (at most 16)", "ops@acme",
`<option value="team:t1">Acme / ml</option>`} {
if !strings.Contains(body, want) {
t.Errorf("the page lacks %q", want)
}
}
// "change" fills the form with the scope's figures.
edit := mirrorGet(t, mux, "/admin/quotas?cap=team:t1").Body.String()
if !strings.Contains(edit, `<option value="team:t1" selected>`) || !strings.Contains(edit, `name="limit_gpus" value="16"`) {
t.Error("change did not open the form on the scope's quota")
}
res := mirrorPost(t, mux, "/admin/quotas/capacity", url.Values{"csrf": {"tok"}, "scope": {"team:t1"},
"gpus": {"8"}, "cores": {""}, "limit_memory": {"64"}}).Body.String()
if len(fc.set) != 1 || *fc.set[0].Guarantee.GPUs != 8 || fc.set[0].Guarantee.CPUCores != nil || *fc.set[0].Limit.MemoryGB != 64 {
t.Errorf("sent %+v", fc.set)
}
if !strings.Contains(res, "capacity quota set on team:t1") {
t.Errorf("no word of it: %s", res)
}
for _, bad := range []url.Values{{"scope": {"team:t1"}, "gpus": {"eight"}}, {"scope": {"team:nope"}, "gpus": {"1"}}, {"gpus": {"1"}}} {
bad.Set("csrf", "tok")
n := len(fc.set)
out := mirrorPost(t, mux, "/admin/quotas/capacity", bad).Body.String()
if len(fc.set) != n || !strings.Contains(out, `class="error"`) {
t.Errorf("%v was not refused with a reason", bad)
}
}
mirrorPost(t, mux, "/admin/quotas/capacity", url.Values{"csrf": {"tok"}, "scope": {"team:t1"}, "clear": {"1"}})
if len(fc.cleared) != 1 || fc.cleared[0] != "team:t1" {
t.Errorf("cleared %v", fc.cleared)
}
// Without a CSRF token nothing happens.
mirrorPost(t, mux, "/admin/quotas/capacity", url.Values{"scope": {"team:t1"}, "clear": {"1"}})
if len(fc.cleared) != 1 {
t.Error("a change was made without the CSRF token")
}
}
+11 -2
View File
@@ -71,6 +71,11 @@ type modelUse struct {
Replicas int
// Min and Max are its scaling as the catalog holds it.
Min, Max int
// Class is what it set, empty for its default; EffClass what that comes
// to. Owner is org:ID or empty for shared. MemoryGB and CPUCores are what
// a replica of its first profile reserves.
Class, EffClass, Owner string
MemoryGB, CPUCores int
// Ready, Wanted and Nodes are how it runs now, for the row itself:
// replicas ready of those it should have, and the nodes they are on.
// Fleet is false where nothing could be asked, so the row says nothing
@@ -293,7 +298,7 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
return name + " now runs all the time", nil
}))
mux.HandleFunc("POST /admin/mirror/replicas", p.mirrorAction(m, p.setReplicas))
mux.HandleFunc("POST /admin/mirror/replicas", p.mirrorAction(m, p.setModel))
mux.HandleFunc("POST /admin/mirror/serve", p.mirrorAction(m,
func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
@@ -463,8 +468,12 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
u := modelUse{
Name: m.Name, Mode: modeOf(m.Scaling), Replicas: max(m.Scaling.Max, 1),
Min: m.Scaling.Min, Max: m.Scaling.Max,
Class: m.Class, EffClass: m.EffectiveClass(), Owner: m.Owner,
Declared: p.catalog.Declared(m.Name),
}
if len(m.Profiles) > 0 {
u.MemoryGB, u.CPUCores = m.Profiles[0].MinMemoryGB, m.Profiles[0].CPUCores
}
if v, ok := views[m.Name]; ok {
u.Fleet, u.Ready, u.Wanted = true, v.Ready, v.Desired
u.Parked, u.Unplaceable = v.Parked, v.Unplaceable
@@ -484,7 +493,7 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
for i := range d.Rows {
d.Rows[i].Open = openKey(d.Rows[i].Model, d.Rows[i].Version)
if open != "" && d.Rows[i].Open == open && d.Rows[i].State == mirror.Present {
d.Rows[i].Detail = p.heldDetailFor(d.Rows[i])
d.Rows[i].Detail = p.heldDetailFor(r.Context(), d.Rows[i])
}
}
+31 -6
View File
@@ -9,6 +9,7 @@ import (
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
@@ -129,12 +130,17 @@ type adminQuotasData struct {
// Editing is the "scope:id" whose form is open, so the template can keep
// the link on every other row.
Editing string
Since string
Message string
Error string
CSRF string
Version string
Now string
// The capacity section: quotas on GPUs, cores and memory.
HasCapacity bool
Capacity []capacityRow
Scopes []adminapi.ScopeName
CapForm capacityForm
Since string
Message string
Error string
CSRF string
Version string
Now string
}
// RegisterAdminQuotas mounts quota administration under /admin/quotas.
@@ -176,6 +182,24 @@ func (p *AccountPages) RegisterAdminQuotas(mux *http.ServeMux, s QuotaAdmin, pri
}
p.renderQuotas(w, r, s, msg, "")
}))
mux.HandleFunc("POST /admin/quotas/capacity", p.adminOnly(
func(w http.ResponseWriter, r *http.Request, admin store.User) {
sess, _, _ := p.session(r)
if p.capacity == nil || !p.accounts.CheckCSRF(sess, r.FormValue("csrf")) {
http.Error(w, "bad request", http.StatusBadRequest)
return
}
msg, err := p.applyCapacity(r, admin)
if err != nil {
p.log.Warn("a capacity quota change was refused",
"admin", admin.Email, "scope", r.FormValue("scope"), "err", err)
p.renderQuotas(w, r, s, "", err.Error())
return
}
p.log.Info("capacity quota changed", "admin", admin.Email, "what", msg)
p.renderQuotas(w, r, s, msg, "")
}))
}
// applyQuota writes one scope's limits, and says what it did.
@@ -259,6 +283,7 @@ func (p *AccountPages) renderQuotas(w http.ResponseWriter, r *http.Request,
d.BudgetsBind = p.priced != nil && p.priced.AnythingPriced()
d.Editing = r.URL.Query().Get("edit")
p.capacitySection(r.Context(), &d, r.FormValue("cap"))
limits, err := s.AllLimits(r.Context())
if err != nil {
+5
View File
@@ -22,6 +22,11 @@ type CatalogAdmin interface {
// Scale changes how a model added at runtime runs: always, on demand, or
// not at all. One declared in configuration is refused.
Scale(ctx context.Context, name string, sc catalog.Scaling, by string) (catalog.Model, error)
// SetClass, SetOwner and Resources change what a model's capacity is
// worth, whose quota it counts against, and what a replica reserves.
SetClass(ctx context.Context, name, class, by string) (catalog.Model, error)
SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error)
Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error)
}
// serveForm is the offer to put one mirrored version into service.
+19
View File
@@ -3,6 +3,7 @@ package web
import (
"context"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
@@ -21,6 +22,9 @@ type fakeCatalog struct {
scaled []string
lastScale catalog.Scaling
err error
classes []string
owners []string
resources []string
}
func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error {
@@ -35,6 +39,21 @@ func (f *fakeCatalog) Sources(context.Context) (map[string][]catalog.Model, erro
return f.sources, nil
}
func (f *fakeCatalog) SetClass(_ context.Context, name, class, _ string) (catalog.Model, error) {
f.classes = append(f.classes, name+"="+class)
return catalog.Model{Name: name, Class: class}, f.err
}
func (f *fakeCatalog) SetOwner(_ context.Context, name, owner, _ string) (catalog.Model, error) {
f.owners = append(f.owners, name+"="+owner)
return catalog.Model{Name: name, Owner: owner}, f.err
}
func (f *fakeCatalog) Resources(_ context.Context, name, _ string, mem, cores int, _ string) (catalog.Model, error) {
f.resources = append(f.resources, fmt.Sprintf("%s=%d/%d", name, mem, cores))
return catalog.Model{Name: name}, f.err
}
func (f *fakeCatalog) Scale(_ context.Context, name string, sc catalog.Scaling, _ string) (catalog.Model, error) {
f.scaled = append(f.scaled, name)
f.lastScale = sc
+81 -4
View File
@@ -29,6 +29,9 @@ const maxReplicas = 64
// and room to change it.
type heldDetail struct {
Models []heldModel
// Scopes are the parts of the tenancy tree a model can belong to; empty
// when capacity quotas are not set up, and the owner choice is left off.
Scopes []adminapi.ScopeName
}
// heldModel is one catalog model running a held version.
@@ -64,8 +67,11 @@ func openKey(model, version string) string { return model + ":" + version }
// heldDetailFor reads, for each model using a row, how it runs and how much
// more room it has.
func (p *AccountPages) heldDetailFor(row mirrorRow) *heldDetail {
func (p *AccountPages) heldDetailFor(ctx context.Context, row mirrorRow) *heldDetail {
d := &heldDetail{}
if p.capacity != nil {
d.Scopes, _ = p.capacity.Scopes(ctx)
}
if p.fleet == nil {
for _, u := range row.UsedBy {
d.Models = append(d.Models, heldModel{modelUse: u})
@@ -98,7 +104,7 @@ func (p *AccountPages) heldDetailFor(row mirrorRow) *heldDetail {
if h, ok := hs.Headroom(u.Name, 16); ok {
hm.Headroom = &h
if !h.Limited {
hm.MostNow = h.Current + h.More
hm.MostNow = h.Current + h.More + h.ByPreempting
}
}
}
@@ -144,7 +150,7 @@ func (p *AccountPages) setReplicas(ctx context.Context, admin store.User, r *htt
if !ok {
return "", fmt.Errorf("no model %q", name)
}
fits := h.Current + h.More
fits := h.Current + h.More + h.ByPreempting
if lo > fits && !h.Limited {
why := "nothing more fits"
if len(h.Blocked) > 0 {
@@ -156,6 +162,11 @@ func (p *AccountPages) setReplicas(ctx context.Context, admin store.User, r *htt
if hi > fits && !h.Limited {
note = fmt.Sprintf("; there is room for %d now, so it grows past that only if room is made", fits)
}
// Room its class would make by stopping others: said, since it is
// somebody else's replica that goes.
if lo > h.Current+h.More && len(h.Preempts) > 0 {
note += "; to make room, replicas of " + strings.Join(h.Preempts, ", ") + " will be stopped"
}
}
sc := catalog.Scaling{Min: lo, Max: hi, IdleTimeout: current.Scaling.IdleTimeout}
@@ -171,7 +182,7 @@ func (p *AccountPages) setReplicas(ctx context.Context, admin store.User, r *htt
case lo == 0:
return fmt.Sprintf("%s loads when asked for, up to %d under load%s", name, hi, note), nil
case lo == hi:
return fmt.Sprintf("%s runs %s", name, plural(lo, "replica", "replicas")), nil
return fmt.Sprintf("%s runs %s%s", name, plural(lo, "replica", "replicas"), note), nil
}
return fmt.Sprintf("%s runs %d, and up to %d under load%s", name, lo, hi, note), nil
}
@@ -205,3 +216,69 @@ func age(sec int64) string {
}
return fmt.Sprintf("%dd", int(d.Hours()/24))
}
// setModel applies the card's settings form: what a replica reserves, the
// model's class and owner, then how many replicas it runs — in that order, so
// the room checked for the replicas is the room its new class and reservation
// would get. Only what changed is applied, and each change is said.
func (p *AccountPages) setModel(ctx context.Context, admin store.User, r *http.Request) (string, error) {
name := r.FormValue("model")
current, err := p.currentModel(ctx, name)
if err != nil {
return "", err
}
if p.catalog.Declared(name) {
return "", fmt.Errorf("%s is set in server.yaml; change it there", name)
}
var done []string
if r.Form.Has("memory") && len(current.Profiles) > 0 {
mem, err1 := strconv.Atoi(strings.TrimSpace(r.FormValue("memory")))
cores, err2 := strconv.Atoi(strings.TrimSpace(r.FormValue("cores")))
if err1 != nil || err2 != nil || mem < 0 || cores < 0 {
return "", errors.New("memory and cores are whole numbers")
}
if cur := current.Profiles[0]; mem != cur.MinMemoryGB || cores != cur.CPUCores {
if _, err := p.catalog.Resources(ctx, name, "", mem, cores, admin.Email); err != nil {
return "", err
}
done = append(done, fmt.Sprintf("a replica reserves %d GB and %d cores (running ones keep theirs until restarted)", mem, cores))
}
}
if r.Form.Has("class") {
if c := r.FormValue("class"); c != current.Class {
m, err := p.catalog.SetClass(ctx, name, c, admin.Email)
if err != nil {
return "", err
}
done = append(done, "is "+m.EffectiveClass())
}
}
if r.Form.Has("owner") && p.capacity != nil {
if o := r.FormValue("owner"); o != current.Owner {
if _, err := p.catalog.SetOwner(ctx, name, o, admin.Email); err != nil {
return "", err
}
if o == "" {
done = append(done, "is shared")
} else {
done = append(done, "counts against "+o)
}
}
}
lo, _ := strconv.Atoi(strings.TrimSpace(r.FormValue("min")))
hi, _ := strconv.Atoi(strings.TrimSpace(r.FormValue("max")))
if r.Form.Has("min") && (lo != current.Scaling.Min || hi != current.Scaling.Max) {
msg, err := p.setReplicas(ctx, admin, r)
if err != nil {
if len(done) > 0 {
return "", fmt.Errorf("%s %s, but not the replicas: %w", name, strings.Join(done, ", "), err)
}
return "", err
}
done = append(done, strings.TrimPrefix(msg, name+" "))
}
if len(done) == 0 {
return name + ": nothing changed", nil
}
return name + " " + strings.Join(done, "; "), nil
}
+91 -3
View File
@@ -1,6 +1,8 @@
package web
import (
"log/slog"
"net/http"
"net/url"
"strings"
"testing"
@@ -8,14 +10,17 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// roomyFleet runs one replica of "coder" on n2 and has room for `more`.
type roomyFleet struct {
fakeFleet
more int
blocked []string
asked int
more int
blocked []string
asked int
preempt int
preempts []string
}
func (f *roomyFleet) Nodes() []adminapi.NodeView {
@@ -41,6 +46,7 @@ func (f *roomyFleet) Headroom(model string, limit int) (adminapi.HeadroomView, b
} else {
h.Limited = true
}
h.ByPreempting, h.Preempts = f.preempt, f.preempts
return h, true
}
@@ -181,3 +187,85 @@ func TestNodeNamesLoseOnlyWhatEveryNodeShares(t *testing.T) {
}
}
}
func settingsPage(t *testing.T, cat *fakeCatalog, withQuotas bool) *http.ServeMux {
t.Helper()
a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}}
p := NewAccountPages(a, nil, nil, "v0", slog.New(slog.DiscardHandler))
if withQuotas {
p.SetCapacity(&fakeCapacity{})
}
mux := http.NewServeMux()
p.RegisterAdminMirror(mux, MirrorDeps{Mirror: &fakeMirror{models: []mirror.Status{
{Model: "coder", Version: "v1", File: "c.gguf", State: mirror.Present, BytesDone: 1},
}}, Catalog: cat, Fleet: &roomyFleet{more: 3}})
return mux
}
func TestACardSetsClassOwnerAndReservationAndOnlyWhatChanged(t *testing.T) {
cat := coderCatalog()
cat.sources["http://mirror:8081/coder/v1/c.gguf"][0].Profiles = []catalog.Profile{{Name: "cpu", MinMemoryGB: 12, CPUCores: 4}}
mux := settingsPage(t, cat, true)
card := mirrorGet(t, mux, "/admin/mirror?open=coder:v1").Body.String()
for _, want := range []string{`name="class"`, `<option value="" selected>default (standard)</option>`,
`name="owner"`, `<option value="team:t1">Acme / ml</option>`, `name="memory" value="12"`, `name="cores" value="4"`} {
if !strings.Contains(card, want) {
t.Errorf("the card lacks %q", want)
}
}
post := func(v url.Values) string {
v.Set("csrf", "tok")
v.Set("model", "coder")
v.Set("open", "coder:v1")
return mirrorPost(t, mux, "/admin/mirror/replicas", v).Body.String()
}
same := url.Values{"min": {"1"}, "max": {"1"}, "class": {""}, "owner": {""}, "memory": {"12"}, "cores": {"4"}}
if body := post(same); !strings.Contains(body, "coder: nothing changed") ||
len(cat.classes)+len(cat.owners)+len(cat.resources)+len(cat.scaled) != 0 {
t.Errorf("an unchanged form changed something: %v %v %v %v", cat.classes, cat.owners, cat.resources, cat.scaled)
}
changed := url.Values{"min": {"1"}, "max": {"1"}, "class": {"critical"}, "owner": {"team:t1"}, "memory": {"14"}, "cores": {"4"}}
body := post(changed)
if strings.Join(cat.resources, ",") != "coder=14/4" || strings.Join(cat.classes, ",") != "coder=critical" ||
strings.Join(cat.owners, ",") != "coder=team:t1" || len(cat.scaled) != 0 {
t.Errorf("applied %v %v %v %v", cat.resources, cat.classes, cat.owners, cat.scaled)
}
if !strings.Contains(body, "a replica reserves 14 GB and 4 cores") || !strings.Contains(body, "counts against team:t1") {
t.Errorf("the change was not said: %s", body)
}
// Without capacity quotas there is no owner to choose, and none is sent.
plain := mirrorGet(t, settingsPage(t, coderCatalog(), false), "/admin/mirror?open=coder:v1").Body.String()
if strings.Contains(plain, `name="owner"`) {
t.Error("an owner choice with no quotas to count against")
}
}
func TestRoomAHigherClassWouldMakeIsOfferedAndSaid(t *testing.T) {
fleet := &roomyFleet{more: 0, preempt: 1, preempts: []string{"chat"}, blocked: []string{"n1: full"}}
cat := coderCatalog()
mux := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{models: []mirror.Status{
{Model: "coder", Version: "v1", File: "c.gguf", State: mirror.Present, BytesDone: 1},
}}, Catalog: cat, Fleet: fleet}, true)
card := mirrorGet(t, mux, "/admin/mirror?open=coder:v1").Body.String()
if !strings.Contains(card, "No free room for another replica now.") ||
!strings.Contains(card, "1 by stopping replicas its class outranks: chat.") ||
!strings.Contains(card, `name="min" value="1" min="0" max="2"`) {
t.Errorf("the card should offer the room preemption makes, and say whose")
}
body := mirrorPost(t, mux, "/admin/mirror/replicas", url.Values{"csrf": {"tok"},
"model": {"coder"}, "min": {"2"}, "max": {"2"}, "open": {"coder:v1"}}).Body.String()
if len(cat.scaled) != 1 || cat.lastScale.Min != 2 {
t.Fatalf("two were refused though preemption makes room: %s", body)
}
if !strings.Contains(body, "replicas of chat will be stopped") {
t.Errorf("stopping somebody else's replica should be said: %s", body)
}
body = mirrorPost(t, mux, "/admin/mirror/replicas", url.Values{"csrf": {"tok"},
"model": {"coder"}, "min": {"3"}, "max": {"3"}, "open": {"coder:v1"}}).Body.String()
if len(cat.scaled) != 1 || !strings.Contains(body, "at most 2") {
t.Errorf("three is past even what preemption makes: %s", body)
}
}
+3
View File
@@ -855,3 +855,6 @@ table.explore .from { margin-left: .5rem; font-size: .78rem; color: var(--muted)
.clip-inline { display: inline-block; max-width: 24rem; overflow: hidden; text-overflow: ellipsis;
vertical-align: bottom; font-size: .8rem; color: var(--muted); margin-left: .4rem; }
table.rows td.warn { color: var(--warn); cursor: help; }
.held-card .replicas-form select, .capacity-form select { display: block; margin-top: .25rem; }
.held-card .replicas-form .field-row { flex-wrap: wrap; gap: .75rem 1rem; }
.quota-tree td.q-act form.inline { margin-left: .75rem; }
+25 -5
View File
@@ -443,7 +443,7 @@ computes once it has it.</p>
{{end}}
{{/* The card beneath a held version: what runs it, where, and how many. */}}
{{define "mirror_held_detail"}}{{$row := .Row}}{{$csrf := .CSRF}}
{{define "mirror_held_detail"}}{{$row := .Row}}{{$csrf := .CSRF}}{{$scopes := .D.Scopes}}
<div class="held-card">
<p class="repo-meta">
<span>{{if gt $row.Files 1}}{{$row.Files}} files{{else}}<code>{{$row.File}}</code>{{end}}</span>
@@ -482,7 +482,8 @@ computes once it has it.</p>
<p class="headroom">
{{if .Limited}}Room for at least {{.More}} more replicas.
{{else if .More}}Room for {{.More}} more now{{if .Nodes}}, on {{range $i, $n := .Nodes}}{{if $i}}, {{end}}{{join $n}}{{end}}{{end}}.
{{else}}No room for another replica now.{{end}}
{{else}}No free room for another replica now.{{end}}
{{if .ByPreempting}}<br><span class="note">{{if .More}}And {{end}}{{.ByPreempting}} {{if .More}}more {{end}}by stopping replicas its class outranks: {{join .Preempts}}.</span>{{end}}
</p>
{{if and (not .Limited) .Blocked}}<details class="blocked"><summary>Why not {{if .More}}more{{else}}another{{end}}</summary>
<ul>{{range .Blocked}}<li>{{.}}</li>{{end}}</ul></details>{{end}}
@@ -500,11 +501,30 @@ computes once it has it.</p>
<input type="number" name="min" value="{{.Min}}" min="0"{{if .MostNow}} max="{{.MostNow}}"{{end}} required></label>
<label>Up to, under load
<input type="number" name="max" value="{{.Max}}" min="0" max="64" required></label>
<button type="submit">Update</button>
<label>Class
<select name="class">
<option value=""{{if eq .Class ""}} selected{{end}}>default ({{if eq .Mode "ondemand"}}opportunistic{{else}}standard{{end}})</option>
<option value="critical"{{if eq .Class "critical"}} selected{{end}}>critical</option>
<option value="standard"{{if eq .Class "standard"}} selected{{end}}>standard</option>
<option value="opportunistic"{{if eq .Class "opportunistic"}} selected{{end}}>opportunistic</option>
</select></label>
{{if $scopes}}{{$owner := .Owner}}<label>Owner
<select name="owner">
<option value=""{{if eq $owner ""}} selected{{end}}>shared</option>
{{range $scopes}}<option value="{{.Key}}"{{if eq .Key $owner}} selected{{end}}>{{.Name}}</option>{{end}}
</select></label>{{end}}
</div>
<p class="note">{{if .MostNow}}At most {{.MostNow}} can always run with what is free now. {{end}}0 always running with a
<div class="field-row">
<label>Memory GB <span class="opt">per replica</span>
<input type="number" name="memory" value="{{.MemoryGB}}" min="0"></label>
<label>Cores <span class="opt">per replica, its threads</span>
<input type="number" name="cores" value="{{.CPUCores}}" min="0"></label>
<button type="submit">Save</button>
</div>
<p class="note">{{if .MostNow}}At most {{.MostNow}} can always run now{{if .Headroom}}{{if .Headroom.ByPreempting}}, counting what its class may stop to make room{{end}}{{end}}. {{end}}0 always running with a
higher limit loads it when asked for; 0 and 0 stops it. A replica is added under load when requests wait, and
taken away after ten quiet minutes.</p>
taken away after ten quiet minutes. A higher class may stop replicas of lower ones to make room; memory and
cores apply to replicas started after the change.</p>
</form>
{{end}}
</section>
+54
View File
@@ -147,6 +147,60 @@ keys, projects and the chat page. {{.Personal}}</p>
</table>
{{else}}<p class="empty">Nobody has been approved yet.</p>{{end}}
{{if .HasCapacity}}
<h2 id="capacity">Capacity</h2>
<p class="note">What a part of the tree is guaranteed of the cluster, and the most
it may hold: GPUs, CPU cores and memory, counted from the replicas of the models
it owns and of those beneath it. What it holds within its guarantee is never taken
for someone else's model; beyond it, it borrows free room and gives it back when
asked. A model's owner is set on its card on the mirror page.</p>
{{if .Capacity}}
<table class="rows quota-tree">
<tr><th>Where</th><th class="num">GPUs</th><th class="num">Cores</th><th class="num">Memory GB</th><th>Set by</th><th></th></tr>
{{range .Capacity}}<tr>
<td class="q-where"><span class="q-name">{{or .Name .Scope}}</span></td>
<td class="num">{{or .GPUs "—"}}</td>
<td class="num">{{or .Cores "—"}}</td>
<td class="num">{{or .Memory "—"}}</td>
<td class="note">{{.UpdatedBy}}</td>
<td class="q-act"><a href="/admin/quotas?cap={{.Scope}}#capacity">change</a>
<form method="post" action="/admin/quotas/capacity" class="inline">
<input type="hidden" name="csrf" value="{{$.CSRF}}">
<input type="hidden" name="scope" value="{{.Scope}}">
<button type="submit" name="clear" value="1" class="link danger">remove</button>
</form></td>
</tr>{{end}}
</table>
<p class="q-legend">Each figure is held of guaranteed (at most the limit); — is not governed.</p>
{{else}}<p class="empty">No capacity quotas: every model shares the cluster by its class alone.</p>{{end}}
{{if .Scopes}}
<form method="post" action="/admin/quotas/capacity" class="q-form capacity-form">
<input type="hidden" name="csrf" value="{{.CSRF}}">
<p class="q-form-title"><strong>{{if .CapForm.Scope}}Change{{else}}Set{{end}} a capacity quota.</strong>
A blank field is not governed; 0 is none at all.</p>
<label>Where
<select name="scope" required>
<option value="">choose…</option>
{{range .Scopes}}<option value="{{.Key}}"{{if eq .Key $.CapForm.Scope}} selected{{end}}>{{.Name}}</option>{{end}}
</select></label>
<div class="field-row">
<label>Guaranteed GPUs <input type="text" name="gpus" value="{{.CapForm.GPUs}}" placeholder="—"></label>
<label>Cores <input type="text" name="cores" value="{{.CapForm.Cores}}" placeholder="—"></label>
<label>Memory GB <input type="text" name="memory" value="{{.CapForm.Memory}}" placeholder="—"></label>
</div>
<div class="field-row">
<label>At most GPUs <input type="text" name="limit_gpus" value="{{.CapForm.LGPUs}}" placeholder="—"></label>
<label>Cores <input type="text" name="limit_cores" value="{{.CapForm.LCores}}" placeholder="—"></label>
<label>Memory GB <input type="text" name="limit_memory" value="{{.CapForm.LMem}}" placeholder="—"></label>
</div>
<div class="q-form-actions"><button type="submit">Save</button>
{{if .CapForm.Scope}}<a href="/admin/quotas#capacity">Cancel</a>{{end}}</div>
</form>
{{else}}<p class="empty">Nothing to give a capacity quota to yet. Make an organization on the
<a href="/admin/users">people page</a> first.</p>{{end}}
{{end}}
<details class="explain">
<summary>How limits combine</summary>
<p>Along a branch the most restrictive value wins, so a team cannot raise what