Compare commits
1
Commits
v0.10.5
...
v0.10.6-rc.1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f288f3c423 |
No files matched your search
@@ -20,6 +20,28 @@ is only released once its **exit criterion** is met, not when its code lands.
|
||||
Until v1.0.0 the minor version is the breaking one: configuration formats and Go
|
||||
APIs change between minors without a deprecation period.
|
||||
|
||||
## Unreleased
|
||||
|
||||
Changes since v0.10.5, published as release candidates (v0.10.6-rc.N) until
|
||||
there are enough for v0.10.6.
|
||||
|
||||
### Capacity from the admin pages
|
||||
|
||||
What v0.10.5 made settable from the command line is now on the pages too:
|
||||
|
||||
- **Quotas** has a Capacity section: every scope's guaranteed and most GPUs,
|
||||
cores and memory beside what it holds, with a form to set, change or remove
|
||||
one. It goes through the same checks as `llmgw ctl quota`.
|
||||
- **A model's card** on the mirror page sets its class, its owner (when there
|
||||
are quotas to count against), and the memory and cores a replica reserves,
|
||||
beside its replica counts. Only what changed is applied, and each change is
|
||||
said.
|
||||
- **Room counts what a class may take.** The card, `llmgw ctl model target` and
|
||||
the headroom API count the replicas a model could get by stopping those its
|
||||
class outranks, separately from free room, and name whose they would be. A
|
||||
critical model's card no longer refuses a replica that preemption would make
|
||||
room for.
|
||||
|
||||
## v0.10.5 — 2026-09-30
|
||||
|
||||
Published first as release candidates v0.10.5-rc.1 to rc.14; rc.5 failed its
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
@@ -192,6 +193,9 @@ func newTargetCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
switch {
|
||||
case v.More == 0:
|
||||
fmt.Fprintln(out, "It already has them.")
|
||||
case v.Fits && len(v.Preempts) > 0:
|
||||
fmt.Fprintf(out, "%d more; there is room once replicas of %s are stopped, which its class may do.\n",
|
||||
v.More, strings.Join(v.Preempts, ", "))
|
||||
case v.Fits:
|
||||
fmt.Fprintf(out, "%d more, and there is room for them.\n", v.More)
|
||||
default:
|
||||
|
||||
@@ -408,6 +408,10 @@ type HeadroomView struct {
|
||||
Blocked []string `json:"blocked,omitempty"`
|
||||
// TokPerSec is what one more replica was measured to serve; 0 unmeasured.
|
||||
TokPerSec float64 `json:"tok_per_sec,omitempty"`
|
||||
// ByPreempting is how many more would fit by stopping replicas this
|
||||
// model's class may take; Preempts the models those belong to.
|
||||
ByPreempting int `json:"by_preempting,omitempty"`
|
||||
Preempts []string `json:"preempts,omitempty"`
|
||||
}
|
||||
|
||||
// RejectionView is one rejected candidate.
|
||||
@@ -967,7 +971,8 @@ func (a *API) Headroom(model string, limit int) (HeadroomView, bool) {
|
||||
return HeadroomView{}, false
|
||||
}
|
||||
return HeadroomView{Model: model, Current: h.Current, More: h.More, Nodes: h.Nodes,
|
||||
Limited: h.Limited, Blocked: h.Blocked, TokPerSec: h.TokPerSec}, true
|
||||
Limited: h.Limited, Blocked: h.Blocked, TokPerSec: h.TokPerSec,
|
||||
ByPreempting: h.ByPreempting, Preempts: h.Preempts}, true
|
||||
}
|
||||
|
||||
func (a *API) Explain(model string) ExplainView {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -72,38 +73,24 @@ func (a *API) registerQuotas(mux *http.ServeMux) {
|
||||
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
scope, id, _, err := a.resolveScope(r.Context(), req.Scope)
|
||||
key, err := a.SetQuota(r.Context(), req, actorOf(r))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
if !req.Guarantee.Any() && !req.Limit.Any() {
|
||||
writeError(w, http.StatusBadRequest, "a quota needs a guarantee or a limit; to remove one, DELETE it")
|
||||
return
|
||||
}
|
||||
q := store.CapacityQuota{Scope: scope, ScopeID: id, Guarantee: req.Guarantee, Limit: req.Limit,
|
||||
UpdatedBy: actorOf(r)}
|
||||
if err := a.quotas.PutCapacityQuota(r.Context(), q); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"scope": quota.Key(scope, id)})
|
||||
writeJSON(w, http.StatusOK, map[string]string{"scope": key})
|
||||
})
|
||||
mux.HandleFunc("DELETE /api/v1/quotas/{scope}", func(w http.ResponseWriter, r *http.Request) {
|
||||
if a.quotas == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||
return
|
||||
}
|
||||
scope, id, _, err := a.resolveScope(r.Context(), r.PathValue("scope"))
|
||||
key, err := a.ClearQuota(r.Context(), r.PathValue("scope"))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
if err := a.quotas.DeleteCapacityQuota(r.Context(), scope, id); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"removed": quota.Key(scope, id)})
|
||||
writeJSON(w, http.StatusOK, map[string]string{"removed": key})
|
||||
})
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/owner", func(w http.ResponseWriter, r *http.Request) {
|
||||
if a.models == nil || a.quotas == nil {
|
||||
@@ -137,6 +124,9 @@ func (a *API) registerQuotas(mux *http.ServeMux) {
|
||||
|
||||
// QuotaViews lists every quota with what its scope holds now.
|
||||
func (a *API) QuotaViews(ctx context.Context) ([]QuotaView, error) {
|
||||
if a.quotas == nil {
|
||||
return nil, nil
|
||||
}
|
||||
qs, err := a.quotas.CapacityQuotas(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -235,3 +225,61 @@ func (a *API) resolveScope(ctx context.Context, s string) (store.Scope, string,
|
||||
}
|
||||
return "", "", "", fmt.Errorf("%q could be any of %d %ss; give its ID", what, len(found), kind)
|
||||
}
|
||||
|
||||
// SetQuota sets a scope's capacity quota, named by path or ID, and returns
|
||||
// the scope as org:ID.
|
||||
func (a *API) SetQuota(ctx context.Context, req QuotaRequest, by string) (string, error) {
|
||||
if a.quotas == nil {
|
||||
return "", fmt.Errorf("this control plane has no store")
|
||||
}
|
||||
scope, id, _, err := a.resolveScope(ctx, req.Scope)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !req.Guarantee.Any() && !req.Limit.Any() {
|
||||
return "", fmt.Errorf("a quota needs a guarantee or a limit; to remove one, clear it")
|
||||
}
|
||||
q := store.CapacityQuota{Scope: scope, ScopeID: id, Guarantee: req.Guarantee, Limit: req.Limit, UpdatedBy: by}
|
||||
if err := a.quotas.PutCapacityQuota(ctx, q); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return quota.Key(scope, id), nil
|
||||
}
|
||||
|
||||
// ClearQuota removes a scope's capacity quota.
|
||||
func (a *API) ClearQuota(ctx context.Context, scope string) (string, error) {
|
||||
if a.quotas == nil {
|
||||
return "", fmt.Errorf("this control plane has no store")
|
||||
}
|
||||
s, id, _, err := a.resolveScope(ctx, scope)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := a.quotas.DeleteCapacityQuota(ctx, s, id); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return quota.Key(s, id), nil
|
||||
}
|
||||
|
||||
// ScopeName is a part of the tenancy tree: org:ID and its path.
|
||||
type ScopeName struct {
|
||||
Key string `json:"scope"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// Scopes lists every organization, team and project, by path.
|
||||
func (a *API) Scopes(ctx context.Context) ([]ScopeName, error) {
|
||||
if a.quotas == nil {
|
||||
return nil, nil
|
||||
}
|
||||
names, err := a.scopeNames(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]ScopeName, 0, len(names))
|
||||
for k, n := range names {
|
||||
out = append(out, ScopeName{Key: k, Name: n})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
|
||||
return out, nil
|
||||
}
|
||||
@@ -24,6 +24,9 @@ type TargetView struct {
|
||||
Room int `json:"room"`
|
||||
Fits bool `json:"fits"`
|
||||
Blocked []string `json:"blocked,omitempty"`
|
||||
// Preempts is whose replicas would be stopped to make the room, when
|
||||
// free room alone is not enough; Room counts that room too.
|
||||
Preempts []string `json:"preempts,omitempty"`
|
||||
}
|
||||
|
||||
// Target sizes a model for a throughput.
|
||||
@@ -40,8 +43,11 @@ func (a *API) Target(model string, tokPerSec float64) (TargetView, error) {
|
||||
}
|
||||
v.Current = h.Current
|
||||
v.More = max(v.Needed-h.Current, 0)
|
||||
v.Room = h.More
|
||||
v.Fits = v.More <= h.More || h.Limited
|
||||
v.Room = h.More + h.ByPreempting
|
||||
v.Fits = v.More <= v.Room || h.Limited
|
||||
if v.More > h.More {
|
||||
v.Preempts = h.Preempts
|
||||
}
|
||||
if !v.Fits {
|
||||
v.Blocked = h.Blocked
|
||||
}
|
||||
|
||||
@@ -28,6 +28,11 @@ type Headroom struct {
|
||||
// TokPerSec is what one more replica was measured to serve, on the
|
||||
// profile placement would give it; zero when it was never measured.
|
||||
TokPerSec float64
|
||||
// ByPreempting is how many more, beyond More, would fit by stopping
|
||||
// replicas the model's class may take, and Preempts the models those
|
||||
// replicas belong to: what "room" means for a model that outranks others.
|
||||
ByPreempting int
|
||||
Preempts []string
|
||||
}
|
||||
|
||||
// Headroom tries placing up to limit more replicas of a model on a copy of
|
||||
@@ -61,12 +66,29 @@ func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) {
|
||||
if pol != nil {
|
||||
usage = pol.Usage(r.Holdings())
|
||||
}
|
||||
for h.More < limit {
|
||||
place := func() placement.Result {
|
||||
res := sched.Place(r.orderedProfiles(m), avoid)
|
||||
if res.Placement == nil && len(avoid) > 0 {
|
||||
// As scaleUp does: spreading is a preference, not a rule.
|
||||
res = sched.Place(r.orderedProfiles(m), nil)
|
||||
}
|
||||
return res
|
||||
}
|
||||
// What the model's class may take, in the order preemption would, each
|
||||
// released on the copy only when free room has run out.
|
||||
victims, nextVictim, preempting := r.victims(m), 0, false
|
||||
for h.More+h.ByPreempting < limit {
|
||||
res := place()
|
||||
for res.Placement == nil && nextVictim < len(victims) {
|
||||
v := victims[nextVictim]
|
||||
nextVictim++
|
||||
inv.ReleaseReplica(v.ID)
|
||||
if !slices.Contains(h.Preempts, v.Model) {
|
||||
h.Preempts = append(h.Preempts, v.Model)
|
||||
}
|
||||
preempting = true
|
||||
res = place()
|
||||
}
|
||||
if res.Placement == nil {
|
||||
for _, rej := range res.Rejections {
|
||||
h.Blocked = append(h.Blocked, rej.String())
|
||||
@@ -103,8 +125,14 @@ func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) {
|
||||
nodes = append(nodes, a.NodeID)
|
||||
}
|
||||
avoid = append(avoid, nodes...)
|
||||
h.Nodes = append(h.Nodes, nodes)
|
||||
h.More++
|
||||
// Once anything was stopped, what fits after is counted as fitting
|
||||
// by preempting: it may be sitting in the room that was freed.
|
||||
if preempting {
|
||||
h.ByPreempting++
|
||||
} else {
|
||||
h.Nodes = append(h.Nodes, nodes)
|
||||
h.More++
|
||||
}
|
||||
}
|
||||
h.Limited = true
|
||||
return h, true
|
||||
|
||||
@@ -312,3 +312,22 @@ func TestOfTwoThatWouldEachMakeRoomTheSmallerIsStopped(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeadroomCountsWhatAHigherClassCouldTake(t *testing.T) {
|
||||
h := fillThenWant(t, gpuModel("vital", catalog.ClassCritical, 1), gpuModel("chat", "", 1), gpuNode("1", 8))
|
||||
got, _ := h.r.Headroom("vital", 3)
|
||||
if got.More != 0 || got.ByPreempting != 1 || len(got.Preempts) != 1 || got.Preempts[0] != "chat" {
|
||||
t.Errorf("a critical model should see one more by stopping chat: %+v", got)
|
||||
}
|
||||
// Tried on a copy: nothing was stopped.
|
||||
if replicasOf(h, "chat") != 1 {
|
||||
t.Fatal("asking about room stopped a replica")
|
||||
}
|
||||
// A model of chat's own class may not take its minimum, so has no such room.
|
||||
peer := newFleetHarness(t, []catalog.Model{gpuModel("chat", "", 1), withMin(gpuModel("peer", "", 1), 0)}, gpuNode("1", 8))
|
||||
peer.settle(t, 3)
|
||||
peer.advance(peer.r.cfg.MinLifetime + time.Minute)
|
||||
if got, _ := peer.r.Headroom("peer", 3); got.ByPreempting != 0 {
|
||||
t.Errorf("a standard model counted room in another's minimum: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -185,6 +185,11 @@ func (r *Router) Handler() http.Handler {
|
||||
pages.RegisterAdminUsers(mux, r.store)
|
||||
// Quotas are written on the same tree the accounts page builds, so
|
||||
// they are mounted together.
|
||||
// Capacity quotas are the control plane's, reached through the
|
||||
// same admin API the CLI uses.
|
||||
if c, ok := r.clusterView.(web.CapacityAdmin); ok {
|
||||
pages.SetCapacity(c)
|
||||
}
|
||||
pages.RegisterAdminQuotas(mux, r.store, r.acct)
|
||||
// And the report of what that tree actually used.
|
||||
pages.RegisterAdminUsage(mux, r.store, r.acct)
|
||||
|
||||
@@ -61,6 +61,8 @@ type AccountPages struct {
|
||||
hasMirror bool
|
||||
// hasQuotas is set when /admin/quotas is registered.
|
||||
hasQuotas bool
|
||||
// capacity sets capacity quotas, from the quotas page; nil leaves them off.
|
||||
capacity CapacityAdmin
|
||||
// personalOff refuses personal keys; see SetPersonalOff.
|
||||
personalOff bool
|
||||
// personalDefaults is what personal use falls under by default.
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// CapacityAdmin sets what parts of the tenancy tree are guaranteed of the
|
||||
// cluster and may hold. The admin API implements it; the page and the CLI go
|
||||
// through the same checks.
|
||||
type CapacityAdmin interface {
|
||||
QuotaViews(ctx context.Context) ([]adminapi.QuotaView, error)
|
||||
SetQuota(ctx context.Context, req adminapi.QuotaRequest, by string) (string, error)
|
||||
ClearQuota(ctx context.Context, scope string) (string, error)
|
||||
Scopes(ctx context.Context) ([]adminapi.ScopeName, error)
|
||||
}
|
||||
|
||||
// SetCapacity gives the quotas page capacity quotas to show and set. Without
|
||||
// it the section is left off.
|
||||
func (p *AccountPages) SetCapacity(c CapacityAdmin) { p.capacity = c }
|
||||
|
||||
// capacityRow is one scope's capacity quota as the page shows it.
|
||||
type capacityRow struct {
|
||||
adminapi.QuotaView
|
||||
GPUs, Cores, Memory string
|
||||
}
|
||||
|
||||
// capacityForm is what the form starts with: a scope's quota to change, or
|
||||
// nothing for a new one.
|
||||
type capacityForm struct {
|
||||
Scope string
|
||||
GPUs, Cores, Memory string
|
||||
LGPUs, LCores, LMem string
|
||||
}
|
||||
|
||||
func (p *AccountPages) capacitySection(ctx context.Context, d *adminQuotasData, editing string) {
|
||||
if p.capacity == nil {
|
||||
return
|
||||
}
|
||||
d.HasCapacity = true
|
||||
views, err := p.capacity.QuotaViews(ctx)
|
||||
if err != nil {
|
||||
d.Error = err.Error()
|
||||
return
|
||||
}
|
||||
for _, v := range views {
|
||||
d.Capacity = append(d.Capacity, capacityRow{QuotaView: v,
|
||||
GPUs: capCell(v.Holds.GPUs, v.Guarantee.GPUs, v.Limit.GPUs),
|
||||
Cores: capCell(v.Holds.CPUCores, v.Guarantee.CPUCores, v.Limit.CPUCores),
|
||||
Memory: capCell(v.Holds.MemoryGB, v.Guarantee.MemoryGB, v.Limit.MemoryGB)})
|
||||
if v.Scope == editing {
|
||||
d.CapForm = capacityForm{Scope: v.Scope,
|
||||
GPUs: intStr(v.Guarantee.GPUs), Cores: intStr(v.Guarantee.CPUCores), Memory: intStr(v.Guarantee.MemoryGB),
|
||||
LGPUs: intStr(v.Limit.GPUs), LCores: intStr(v.Limit.CPUCores), LMem: intStr(v.Limit.MemoryGB)}
|
||||
}
|
||||
}
|
||||
if d.Scopes, err = p.capacity.Scopes(ctx); err != nil {
|
||||
d.Error = err.Error()
|
||||
}
|
||||
}
|
||||
|
||||
// capCell is "6 of 8 (at most 16)": held, guaranteed, limit.
|
||||
func capCell(holds int, g, l *int) string {
|
||||
if g == nil && l == nil {
|
||||
return ""
|
||||
}
|
||||
out := strconv.Itoa(holds)
|
||||
if g != nil {
|
||||
out += " of " + strconv.Itoa(*g)
|
||||
}
|
||||
if l != nil {
|
||||
out += fmt.Sprintf(" (at most %d)", *l)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func intStr(v *int) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
return strconv.Itoa(*v)
|
||||
}
|
||||
|
||||
// applyCapacity sets or clears one scope's capacity quota from the form.
|
||||
func (p *AccountPages) applyCapacity(r *http.Request, admin store.User) (string, error) {
|
||||
scope := strings.TrimSpace(r.FormValue("scope"))
|
||||
if scope == "" {
|
||||
return "", fmt.Errorf("choose where the quota applies")
|
||||
}
|
||||
if r.FormValue("clear") != "" {
|
||||
key, err := p.capacity.ClearQuota(r.Context(), scope)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "capacity quota removed from " + key, nil
|
||||
}
|
||||
field := func(name, what string) (*int, error) {
|
||||
v := strings.TrimSpace(r.FormValue(name))
|
||||
if v == "" {
|
||||
return nil, nil
|
||||
}
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 0 {
|
||||
return nil, fmt.Errorf("%s must be a whole number, or blank for none", what)
|
||||
}
|
||||
return &n, nil
|
||||
}
|
||||
var req adminapi.QuotaRequest
|
||||
req.Scope = scope
|
||||
var err error
|
||||
for _, f := range []struct {
|
||||
dst **int
|
||||
name string
|
||||
what string
|
||||
}{
|
||||
{&req.Guarantee.GPUs, "gpus", "guaranteed GPUs"},
|
||||
{&req.Guarantee.CPUCores, "cores", "guaranteed cores"},
|
||||
{&req.Guarantee.MemoryGB, "memory", "guaranteed memory"},
|
||||
{&req.Limit.GPUs, "limit_gpus", "the GPU limit"},
|
||||
{&req.Limit.CPUCores, "limit_cores", "the core limit"},
|
||||
{&req.Limit.MemoryGB, "limit_memory", "the memory limit"},
|
||||
} {
|
||||
if *f.dst, err = field(f.name, f.what); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
key, err := p.capacity.SetQuota(r.Context(), req, admin.Email)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "capacity quota set on " + key + "; it takes effect within half a minute", nil
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
type fakeCapacity struct {
|
||||
views []adminapi.QuotaView
|
||||
set []adminapi.QuotaRequest
|
||||
cleared []string
|
||||
}
|
||||
|
||||
func (f *fakeCapacity) QuotaViews(context.Context) ([]adminapi.QuotaView, error) { return f.views, nil }
|
||||
func (f *fakeCapacity) SetQuota(_ context.Context, r adminapi.QuotaRequest, _ string) (string, error) {
|
||||
if r.Scope == "team:nope" {
|
||||
return "", errors.New(`there is no team "nope"`)
|
||||
}
|
||||
f.set = append(f.set, r)
|
||||
return r.Scope, nil
|
||||
}
|
||||
func (f *fakeCapacity) ClearQuota(_ context.Context, s string) (string, error) {
|
||||
f.cleared = append(f.cleared, s)
|
||||
return s, nil
|
||||
}
|
||||
func (f *fakeCapacity) Scopes(context.Context) ([]adminapi.ScopeName, error) {
|
||||
return []adminapi.ScopeName{{Key: "org:o1", Name: "Acme"}, {Key: "team:t1", Name: "Acme / ml"}}, nil
|
||||
}
|
||||
|
||||
func TestCapacityQuotasAreShownAndSetFromTheQuotasPage(t *testing.T) {
|
||||
eight, sixteen := 8, 16
|
||||
fc := &fakeCapacity{views: []adminapi.QuotaView{{Scope: "team:t1", Name: "Acme / ml",
|
||||
Guarantee: store.Resources{GPUs: &eight}, Limit: store.Resources{GPUs: &sixteen},
|
||||
Holds: quota.Amount{GPUs: 6}, UpdatedBy: "ops@acme"}}}
|
||||
a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}}
|
||||
p := NewAccountPages(a, nil, nil, "v0", slog.New(slog.DiscardHandler))
|
||||
p.SetCapacity(fc)
|
||||
mux := http.NewServeMux()
|
||||
p.RegisterAdminQuotas(mux, &fakeQuotas{}, pricedYes{})
|
||||
|
||||
body := mirrorGet(t, mux, "/admin/quotas").Body.String()
|
||||
for _, want := range []string{`id="capacity"`, "Acme / ml", "6 of 8 (at most 16)", "ops@acme",
|
||||
`<option value="team:t1">Acme / ml</option>`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("the page lacks %q", want)
|
||||
}
|
||||
}
|
||||
// "change" fills the form with the scope's figures.
|
||||
edit := mirrorGet(t, mux, "/admin/quotas?cap=team:t1").Body.String()
|
||||
if !strings.Contains(edit, `<option value="team:t1" selected>`) || !strings.Contains(edit, `name="limit_gpus" value="16"`) {
|
||||
t.Error("change did not open the form on the scope's quota")
|
||||
}
|
||||
|
||||
res := mirrorPost(t, mux, "/admin/quotas/capacity", url.Values{"csrf": {"tok"}, "scope": {"team:t1"},
|
||||
"gpus": {"8"}, "cores": {""}, "limit_memory": {"64"}}).Body.String()
|
||||
if len(fc.set) != 1 || *fc.set[0].Guarantee.GPUs != 8 || fc.set[0].Guarantee.CPUCores != nil || *fc.set[0].Limit.MemoryGB != 64 {
|
||||
t.Errorf("sent %+v", fc.set)
|
||||
}
|
||||
if !strings.Contains(res, "capacity quota set on team:t1") {
|
||||
t.Errorf("no word of it: %s", res)
|
||||
}
|
||||
for _, bad := range []url.Values{{"scope": {"team:t1"}, "gpus": {"eight"}}, {"scope": {"team:nope"}, "gpus": {"1"}}, {"gpus": {"1"}}} {
|
||||
bad.Set("csrf", "tok")
|
||||
n := len(fc.set)
|
||||
out := mirrorPost(t, mux, "/admin/quotas/capacity", bad).Body.String()
|
||||
if len(fc.set) != n || !strings.Contains(out, `class="error"`) {
|
||||
t.Errorf("%v was not refused with a reason", bad)
|
||||
}
|
||||
}
|
||||
mirrorPost(t, mux, "/admin/quotas/capacity", url.Values{"csrf": {"tok"}, "scope": {"team:t1"}, "clear": {"1"}})
|
||||
if len(fc.cleared) != 1 || fc.cleared[0] != "team:t1" {
|
||||
t.Errorf("cleared %v", fc.cleared)
|
||||
}
|
||||
// Without a CSRF token nothing happens.
|
||||
mirrorPost(t, mux, "/admin/quotas/capacity", url.Values{"scope": {"team:t1"}, "clear": {"1"}})
|
||||
if len(fc.cleared) != 1 {
|
||||
t.Error("a change was made without the CSRF token")
|
||||
}
|
||||
}
|
||||
@@ -71,6 +71,11 @@ type modelUse struct {
|
||||
Replicas int
|
||||
// Min and Max are its scaling as the catalog holds it.
|
||||
Min, Max int
|
||||
// Class is what it set, empty for its default; EffClass what that comes
|
||||
// to. Owner is org:ID or empty for shared. MemoryGB and CPUCores are what
|
||||
// a replica of its first profile reserves.
|
||||
Class, EffClass, Owner string
|
||||
MemoryGB, CPUCores int
|
||||
// Ready, Wanted and Nodes are how it runs now, for the row itself:
|
||||
// replicas ready of those it should have, and the nodes they are on.
|
||||
// Fleet is false where nothing could be asked, so the row says nothing
|
||||
@@ -293,7 +298,7 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
|
||||
return name + " now runs all the time", nil
|
||||
}))
|
||||
|
||||
mux.HandleFunc("POST /admin/mirror/replicas", p.mirrorAction(m, p.setReplicas))
|
||||
mux.HandleFunc("POST /admin/mirror/replicas", p.mirrorAction(m, p.setModel))
|
||||
|
||||
mux.HandleFunc("POST /admin/mirror/serve", p.mirrorAction(m,
|
||||
func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
|
||||
@@ -463,8 +468,12 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
|
||||
u := modelUse{
|
||||
Name: m.Name, Mode: modeOf(m.Scaling), Replicas: max(m.Scaling.Max, 1),
|
||||
Min: m.Scaling.Min, Max: m.Scaling.Max,
|
||||
Class: m.Class, EffClass: m.EffectiveClass(), Owner: m.Owner,
|
||||
Declared: p.catalog.Declared(m.Name),
|
||||
}
|
||||
if len(m.Profiles) > 0 {
|
||||
u.MemoryGB, u.CPUCores = m.Profiles[0].MinMemoryGB, m.Profiles[0].CPUCores
|
||||
}
|
||||
if v, ok := views[m.Name]; ok {
|
||||
u.Fleet, u.Ready, u.Wanted = true, v.Ready, v.Desired
|
||||
u.Parked, u.Unplaceable = v.Parked, v.Unplaceable
|
||||
@@ -484,7 +493,7 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
|
||||
for i := range d.Rows {
|
||||
d.Rows[i].Open = openKey(d.Rows[i].Model, d.Rows[i].Version)
|
||||
if open != "" && d.Rows[i].Open == open && d.Rows[i].State == mirror.Present {
|
||||
d.Rows[i].Detail = p.heldDetailFor(d.Rows[i])
|
||||
d.Rows[i].Detail = p.heldDetailFor(r.Context(), d.Rows[i])
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
@@ -129,12 +130,17 @@ type adminQuotasData struct {
|
||||
// Editing is the "scope:id" whose form is open, so the template can keep
|
||||
// the link on every other row.
|
||||
Editing string
|
||||
Since string
|
||||
Message string
|
||||
Error string
|
||||
CSRF string
|
||||
Version string
|
||||
Now string
|
||||
// The capacity section: quotas on GPUs, cores and memory.
|
||||
HasCapacity bool
|
||||
Capacity []capacityRow
|
||||
Scopes []adminapi.ScopeName
|
||||
CapForm capacityForm
|
||||
Since string
|
||||
Message string
|
||||
Error string
|
||||
CSRF string
|
||||
Version string
|
||||
Now string
|
||||
}
|
||||
|
||||
// RegisterAdminQuotas mounts quota administration under /admin/quotas.
|
||||
@@ -176,6 +182,24 @@ func (p *AccountPages) RegisterAdminQuotas(mux *http.ServeMux, s QuotaAdmin, pri
|
||||
}
|
||||
p.renderQuotas(w, r, s, msg, "")
|
||||
}))
|
||||
|
||||
mux.HandleFunc("POST /admin/quotas/capacity", p.adminOnly(
|
||||
func(w http.ResponseWriter, r *http.Request, admin store.User) {
|
||||
sess, _, _ := p.session(r)
|
||||
if p.capacity == nil || !p.accounts.CheckCSRF(sess, r.FormValue("csrf")) {
|
||||
http.Error(w, "bad request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
msg, err := p.applyCapacity(r, admin)
|
||||
if err != nil {
|
||||
p.log.Warn("a capacity quota change was refused",
|
||||
"admin", admin.Email, "scope", r.FormValue("scope"), "err", err)
|
||||
p.renderQuotas(w, r, s, "", err.Error())
|
||||
return
|
||||
}
|
||||
p.log.Info("capacity quota changed", "admin", admin.Email, "what", msg)
|
||||
p.renderQuotas(w, r, s, msg, "")
|
||||
}))
|
||||
}
|
||||
|
||||
// applyQuota writes one scope's limits, and says what it did.
|
||||
@@ -259,6 +283,7 @@ func (p *AccountPages) renderQuotas(w http.ResponseWriter, r *http.Request,
|
||||
|
||||
d.BudgetsBind = p.priced != nil && p.priced.AnythingPriced()
|
||||
d.Editing = r.URL.Query().Get("edit")
|
||||
p.capacitySection(r.Context(), &d, r.FormValue("cap"))
|
||||
|
||||
limits, err := s.AllLimits(r.Context())
|
||||
if err != nil {
|
||||
|
||||
@@ -22,6 +22,11 @@ type CatalogAdmin interface {
|
||||
// Scale changes how a model added at runtime runs: always, on demand, or
|
||||
// not at all. One declared in configuration is refused.
|
||||
Scale(ctx context.Context, name string, sc catalog.Scaling, by string) (catalog.Model, error)
|
||||
// SetClass, SetOwner and Resources change what a model's capacity is
|
||||
// worth, whose quota it counts against, and what a replica reserves.
|
||||
SetClass(ctx context.Context, name, class, by string) (catalog.Model, error)
|
||||
SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error)
|
||||
Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error)
|
||||
}
|
||||
|
||||
// serveForm is the offer to put one mirrored version into service.
|
||||
|
||||
@@ -3,6 +3,7 @@ package web
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
@@ -21,6 +22,9 @@ type fakeCatalog struct {
|
||||
scaled []string
|
||||
lastScale catalog.Scaling
|
||||
err error
|
||||
classes []string
|
||||
owners []string
|
||||
resources []string
|
||||
}
|
||||
|
||||
func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error {
|
||||
@@ -35,6 +39,21 @@ func (f *fakeCatalog) Sources(context.Context) (map[string][]catalog.Model, erro
|
||||
return f.sources, nil
|
||||
}
|
||||
|
||||
func (f *fakeCatalog) SetClass(_ context.Context, name, class, _ string) (catalog.Model, error) {
|
||||
f.classes = append(f.classes, name+"="+class)
|
||||
return catalog.Model{Name: name, Class: class}, f.err
|
||||
}
|
||||
|
||||
func (f *fakeCatalog) SetOwner(_ context.Context, name, owner, _ string) (catalog.Model, error) {
|
||||
f.owners = append(f.owners, name+"="+owner)
|
||||
return catalog.Model{Name: name, Owner: owner}, f.err
|
||||
}
|
||||
|
||||
func (f *fakeCatalog) Resources(_ context.Context, name, _ string, mem, cores int, _ string) (catalog.Model, error) {
|
||||
f.resources = append(f.resources, fmt.Sprintf("%s=%d/%d", name, mem, cores))
|
||||
return catalog.Model{Name: name}, f.err
|
||||
}
|
||||
|
||||
func (f *fakeCatalog) Scale(_ context.Context, name string, sc catalog.Scaling, _ string) (catalog.Model, error) {
|
||||
f.scaled = append(f.scaled, name)
|
||||
f.lastScale = sc
|
||||
|
||||
@@ -29,6 +29,9 @@ const maxReplicas = 64
|
||||
// and room to change it.
|
||||
type heldDetail struct {
|
||||
Models []heldModel
|
||||
// Scopes are the parts of the tenancy tree a model can belong to; empty
|
||||
// when capacity quotas are not set up, and the owner choice is left off.
|
||||
Scopes []adminapi.ScopeName
|
||||
}
|
||||
|
||||
// heldModel is one catalog model running a held version.
|
||||
@@ -64,8 +67,11 @@ func openKey(model, version string) string { return model + ":" + version }
|
||||
|
||||
// heldDetailFor reads, for each model using a row, how it runs and how much
|
||||
// more room it has.
|
||||
func (p *AccountPages) heldDetailFor(row mirrorRow) *heldDetail {
|
||||
func (p *AccountPages) heldDetailFor(ctx context.Context, row mirrorRow) *heldDetail {
|
||||
d := &heldDetail{}
|
||||
if p.capacity != nil {
|
||||
d.Scopes, _ = p.capacity.Scopes(ctx)
|
||||
}
|
||||
if p.fleet == nil {
|
||||
for _, u := range row.UsedBy {
|
||||
d.Models = append(d.Models, heldModel{modelUse: u})
|
||||
@@ -98,7 +104,7 @@ func (p *AccountPages) heldDetailFor(row mirrorRow) *heldDetail {
|
||||
if h, ok := hs.Headroom(u.Name, 16); ok {
|
||||
hm.Headroom = &h
|
||||
if !h.Limited {
|
||||
hm.MostNow = h.Current + h.More
|
||||
hm.MostNow = h.Current + h.More + h.ByPreempting
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -144,7 +150,7 @@ func (p *AccountPages) setReplicas(ctx context.Context, admin store.User, r *htt
|
||||
if !ok {
|
||||
return "", fmt.Errorf("no model %q", name)
|
||||
}
|
||||
fits := h.Current + h.More
|
||||
fits := h.Current + h.More + h.ByPreempting
|
||||
if lo > fits && !h.Limited {
|
||||
why := "nothing more fits"
|
||||
if len(h.Blocked) > 0 {
|
||||
@@ -156,6 +162,11 @@ func (p *AccountPages) setReplicas(ctx context.Context, admin store.User, r *htt
|
||||
if hi > fits && !h.Limited {
|
||||
note = fmt.Sprintf("; there is room for %d now, so it grows past that only if room is made", fits)
|
||||
}
|
||||
// Room its class would make by stopping others: said, since it is
|
||||
// somebody else's replica that goes.
|
||||
if lo > h.Current+h.More && len(h.Preempts) > 0 {
|
||||
note += "; to make room, replicas of " + strings.Join(h.Preempts, ", ") + " will be stopped"
|
||||
}
|
||||
}
|
||||
|
||||
sc := catalog.Scaling{Min: lo, Max: hi, IdleTimeout: current.Scaling.IdleTimeout}
|
||||
@@ -171,7 +182,7 @@ func (p *AccountPages) setReplicas(ctx context.Context, admin store.User, r *htt
|
||||
case lo == 0:
|
||||
return fmt.Sprintf("%s loads when asked for, up to %d under load%s", name, hi, note), nil
|
||||
case lo == hi:
|
||||
return fmt.Sprintf("%s runs %s", name, plural(lo, "replica", "replicas")), nil
|
||||
return fmt.Sprintf("%s runs %s%s", name, plural(lo, "replica", "replicas"), note), nil
|
||||
}
|
||||
return fmt.Sprintf("%s runs %d, and up to %d under load%s", name, lo, hi, note), nil
|
||||
}
|
||||
@@ -205,3 +216,69 @@ func age(sec int64) string {
|
||||
}
|
||||
return fmt.Sprintf("%dd", int(d.Hours()/24))
|
||||
}
|
||||
|
||||
// setModel applies the card's settings form: what a replica reserves, the
|
||||
// model's class and owner, then how many replicas it runs — in that order, so
|
||||
// the room checked for the replicas is the room its new class and reservation
|
||||
// would get. Only what changed is applied, and each change is said.
|
||||
func (p *AccountPages) setModel(ctx context.Context, admin store.User, r *http.Request) (string, error) {
|
||||
name := r.FormValue("model")
|
||||
current, err := p.currentModel(ctx, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if p.catalog.Declared(name) {
|
||||
return "", fmt.Errorf("%s is set in server.yaml; change it there", name)
|
||||
}
|
||||
var done []string
|
||||
if r.Form.Has("memory") && len(current.Profiles) > 0 {
|
||||
mem, err1 := strconv.Atoi(strings.TrimSpace(r.FormValue("memory")))
|
||||
cores, err2 := strconv.Atoi(strings.TrimSpace(r.FormValue("cores")))
|
||||
if err1 != nil || err2 != nil || mem < 0 || cores < 0 {
|
||||
return "", errors.New("memory and cores are whole numbers")
|
||||
}
|
||||
if cur := current.Profiles[0]; mem != cur.MinMemoryGB || cores != cur.CPUCores {
|
||||
if _, err := p.catalog.Resources(ctx, name, "", mem, cores, admin.Email); err != nil {
|
||||
return "", err
|
||||
}
|
||||
done = append(done, fmt.Sprintf("a replica reserves %d GB and %d cores (running ones keep theirs until restarted)", mem, cores))
|
||||
}
|
||||
}
|
||||
if r.Form.Has("class") {
|
||||
if c := r.FormValue("class"); c != current.Class {
|
||||
m, err := p.catalog.SetClass(ctx, name, c, admin.Email)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
done = append(done, "is "+m.EffectiveClass())
|
||||
}
|
||||
}
|
||||
if r.Form.Has("owner") && p.capacity != nil {
|
||||
if o := r.FormValue("owner"); o != current.Owner {
|
||||
if _, err := p.catalog.SetOwner(ctx, name, o, admin.Email); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if o == "" {
|
||||
done = append(done, "is shared")
|
||||
} else {
|
||||
done = append(done, "counts against "+o)
|
||||
}
|
||||
}
|
||||
}
|
||||
lo, _ := strconv.Atoi(strings.TrimSpace(r.FormValue("min")))
|
||||
hi, _ := strconv.Atoi(strings.TrimSpace(r.FormValue("max")))
|
||||
if r.Form.Has("min") && (lo != current.Scaling.Min || hi != current.Scaling.Max) {
|
||||
msg, err := p.setReplicas(ctx, admin, r)
|
||||
if err != nil {
|
||||
if len(done) > 0 {
|
||||
return "", fmt.Errorf("%s %s, but not the replicas: %w", name, strings.Join(done, ", "), err)
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
done = append(done, strings.TrimPrefix(msg, name+" "))
|
||||
}
|
||||
if len(done) == 0 {
|
||||
return name + ": nothing changed", nil
|
||||
}
|
||||
return name + " " + strings.Join(done, "; "), nil
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -8,14 +10,17 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// roomyFleet runs one replica of "coder" on n2 and has room for `more`.
|
||||
type roomyFleet struct {
|
||||
fakeFleet
|
||||
more int
|
||||
blocked []string
|
||||
asked int
|
||||
more int
|
||||
blocked []string
|
||||
asked int
|
||||
preempt int
|
||||
preempts []string
|
||||
}
|
||||
|
||||
func (f *roomyFleet) Nodes() []adminapi.NodeView {
|
||||
@@ -41,6 +46,7 @@ func (f *roomyFleet) Headroom(model string, limit int) (adminapi.HeadroomView, b
|
||||
} else {
|
||||
h.Limited = true
|
||||
}
|
||||
h.ByPreempting, h.Preempts = f.preempt, f.preempts
|
||||
return h, true
|
||||
}
|
||||
|
||||
@@ -181,3 +187,85 @@ func TestNodeNamesLoseOnlyWhatEveryNodeShares(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func settingsPage(t *testing.T, cat *fakeCatalog, withQuotas bool) *http.ServeMux {
|
||||
t.Helper()
|
||||
a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}}
|
||||
p := NewAccountPages(a, nil, nil, "v0", slog.New(slog.DiscardHandler))
|
||||
if withQuotas {
|
||||
p.SetCapacity(&fakeCapacity{})
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
p.RegisterAdminMirror(mux, MirrorDeps{Mirror: &fakeMirror{models: []mirror.Status{
|
||||
{Model: "coder", Version: "v1", File: "c.gguf", State: mirror.Present, BytesDone: 1},
|
||||
}}, Catalog: cat, Fleet: &roomyFleet{more: 3}})
|
||||
return mux
|
||||
}
|
||||
|
||||
func TestACardSetsClassOwnerAndReservationAndOnlyWhatChanged(t *testing.T) {
|
||||
cat := coderCatalog()
|
||||
cat.sources["http://mirror:8081/coder/v1/c.gguf"][0].Profiles = []catalog.Profile{{Name: "cpu", MinMemoryGB: 12, CPUCores: 4}}
|
||||
mux := settingsPage(t, cat, true)
|
||||
|
||||
card := mirrorGet(t, mux, "/admin/mirror?open=coder:v1").Body.String()
|
||||
for _, want := range []string{`name="class"`, `<option value="" selected>default (standard)</option>`,
|
||||
`name="owner"`, `<option value="team:t1">Acme / ml</option>`, `name="memory" value="12"`, `name="cores" value="4"`} {
|
||||
if !strings.Contains(card, want) {
|
||||
t.Errorf("the card lacks %q", want)
|
||||
}
|
||||
}
|
||||
post := func(v url.Values) string {
|
||||
v.Set("csrf", "tok")
|
||||
v.Set("model", "coder")
|
||||
v.Set("open", "coder:v1")
|
||||
return mirrorPost(t, mux, "/admin/mirror/replicas", v).Body.String()
|
||||
}
|
||||
same := url.Values{"min": {"1"}, "max": {"1"}, "class": {""}, "owner": {""}, "memory": {"12"}, "cores": {"4"}}
|
||||
if body := post(same); !strings.Contains(body, "coder: nothing changed") ||
|
||||
len(cat.classes)+len(cat.owners)+len(cat.resources)+len(cat.scaled) != 0 {
|
||||
t.Errorf("an unchanged form changed something: %v %v %v %v", cat.classes, cat.owners, cat.resources, cat.scaled)
|
||||
}
|
||||
changed := url.Values{"min": {"1"}, "max": {"1"}, "class": {"critical"}, "owner": {"team:t1"}, "memory": {"14"}, "cores": {"4"}}
|
||||
body := post(changed)
|
||||
if strings.Join(cat.resources, ",") != "coder=14/4" || strings.Join(cat.classes, ",") != "coder=critical" ||
|
||||
strings.Join(cat.owners, ",") != "coder=team:t1" || len(cat.scaled) != 0 {
|
||||
t.Errorf("applied %v %v %v %v", cat.resources, cat.classes, cat.owners, cat.scaled)
|
||||
}
|
||||
if !strings.Contains(body, "a replica reserves 14 GB and 4 cores") || !strings.Contains(body, "counts against team:t1") {
|
||||
t.Errorf("the change was not said: %s", body)
|
||||
}
|
||||
|
||||
// Without capacity quotas there is no owner to choose, and none is sent.
|
||||
plain := mirrorGet(t, settingsPage(t, coderCatalog(), false), "/admin/mirror?open=coder:v1").Body.String()
|
||||
if strings.Contains(plain, `name="owner"`) {
|
||||
t.Error("an owner choice with no quotas to count against")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoomAHigherClassWouldMakeIsOfferedAndSaid(t *testing.T) {
|
||||
fleet := &roomyFleet{more: 0, preempt: 1, preempts: []string{"chat"}, blocked: []string{"n1: full"}}
|
||||
cat := coderCatalog()
|
||||
mux := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{models: []mirror.Status{
|
||||
{Model: "coder", Version: "v1", File: "c.gguf", State: mirror.Present, BytesDone: 1},
|
||||
}}, Catalog: cat, Fleet: fleet}, true)
|
||||
|
||||
card := mirrorGet(t, mux, "/admin/mirror?open=coder:v1").Body.String()
|
||||
if !strings.Contains(card, "No free room for another replica now.") ||
|
||||
!strings.Contains(card, "1 by stopping replicas its class outranks: chat.") ||
|
||||
!strings.Contains(card, `name="min" value="1" min="0" max="2"`) {
|
||||
t.Errorf("the card should offer the room preemption makes, and say whose")
|
||||
}
|
||||
body := mirrorPost(t, mux, "/admin/mirror/replicas", url.Values{"csrf": {"tok"},
|
||||
"model": {"coder"}, "min": {"2"}, "max": {"2"}, "open": {"coder:v1"}}).Body.String()
|
||||
if len(cat.scaled) != 1 || cat.lastScale.Min != 2 {
|
||||
t.Fatalf("two were refused though preemption makes room: %s", body)
|
||||
}
|
||||
if !strings.Contains(body, "replicas of chat will be stopped") {
|
||||
t.Errorf("stopping somebody else's replica should be said: %s", body)
|
||||
}
|
||||
body = mirrorPost(t, mux, "/admin/mirror/replicas", url.Values{"csrf": {"tok"},
|
||||
"model": {"coder"}, "min": {"3"}, "max": {"3"}, "open": {"coder:v1"}}).Body.String()
|
||||
if len(cat.scaled) != 1 || !strings.Contains(body, "at most 2") {
|
||||
t.Errorf("three is past even what preemption makes: %s", body)
|
||||
}
|
||||
}
|
||||
@@ -855,3 +855,6 @@ table.explore .from { margin-left: .5rem; font-size: .78rem; color: var(--muted)
|
||||
.clip-inline { display: inline-block; max-width: 24rem; overflow: hidden; text-overflow: ellipsis;
|
||||
vertical-align: bottom; font-size: .8rem; color: var(--muted); margin-left: .4rem; }
|
||||
table.rows td.warn { color: var(--warn); cursor: help; }
|
||||
.held-card .replicas-form select, .capacity-form select { display: block; margin-top: .25rem; }
|
||||
.held-card .replicas-form .field-row { flex-wrap: wrap; gap: .75rem 1rem; }
|
||||
.quota-tree td.q-act form.inline { margin-left: .75rem; }
|
||||
@@ -443,7 +443,7 @@ computes once it has it.</p>
|
||||
{{end}}
|
||||
|
||||
{{/* The card beneath a held version: what runs it, where, and how many. */}}
|
||||
{{define "mirror_held_detail"}}{{$row := .Row}}{{$csrf := .CSRF}}
|
||||
{{define "mirror_held_detail"}}{{$row := .Row}}{{$csrf := .CSRF}}{{$scopes := .D.Scopes}}
|
||||
<div class="held-card">
|
||||
<p class="repo-meta">
|
||||
<span>{{if gt $row.Files 1}}{{$row.Files}} files{{else}}<code>{{$row.File}}</code>{{end}}</span>
|
||||
@@ -482,7 +482,8 @@ computes once it has it.</p>
|
||||
<p class="headroom">
|
||||
{{if .Limited}}Room for at least {{.More}} more replicas.
|
||||
{{else if .More}}Room for {{.More}} more now{{if .Nodes}}, on {{range $i, $n := .Nodes}}{{if $i}}, {{end}}{{join $n}}{{end}}{{end}}.
|
||||
{{else}}No room for another replica now.{{end}}
|
||||
{{else}}No free room for another replica now.{{end}}
|
||||
{{if .ByPreempting}}<br><span class="note">{{if .More}}And {{end}}{{.ByPreempting}} {{if .More}}more {{end}}by stopping replicas its class outranks: {{join .Preempts}}.</span>{{end}}
|
||||
</p>
|
||||
{{if and (not .Limited) .Blocked}}<details class="blocked"><summary>Why not {{if .More}}more{{else}}another{{end}}</summary>
|
||||
<ul>{{range .Blocked}}<li>{{.}}</li>{{end}}</ul></details>{{end}}
|
||||
@@ -500,11 +501,30 @@ computes once it has it.</p>
|
||||
<input type="number" name="min" value="{{.Min}}" min="0"{{if .MostNow}} max="{{.MostNow}}"{{end}} required></label>
|
||||
<label>Up to, under load
|
||||
<input type="number" name="max" value="{{.Max}}" min="0" max="64" required></label>
|
||||
<button type="submit">Update</button>
|
||||
<label>Class
|
||||
<select name="class">
|
||||
<option value=""{{if eq .Class ""}} selected{{end}}>default ({{if eq .Mode "ondemand"}}opportunistic{{else}}standard{{end}})</option>
|
||||
<option value="critical"{{if eq .Class "critical"}} selected{{end}}>critical</option>
|
||||
<option value="standard"{{if eq .Class "standard"}} selected{{end}}>standard</option>
|
||||
<option value="opportunistic"{{if eq .Class "opportunistic"}} selected{{end}}>opportunistic</option>
|
||||
</select></label>
|
||||
{{if $scopes}}{{$owner := .Owner}}<label>Owner
|
||||
<select name="owner">
|
||||
<option value=""{{if eq $owner ""}} selected{{end}}>shared</option>
|
||||
{{range $scopes}}<option value="{{.Key}}"{{if eq .Key $owner}} selected{{end}}>{{.Name}}</option>{{end}}
|
||||
</select></label>{{end}}
|
||||
</div>
|
||||
<p class="note">{{if .MostNow}}At most {{.MostNow}} can always run with what is free now. {{end}}0 always running with a
|
||||
<div class="field-row">
|
||||
<label>Memory GB <span class="opt">per replica</span>
|
||||
<input type="number" name="memory" value="{{.MemoryGB}}" min="0"></label>
|
||||
<label>Cores <span class="opt">per replica, its threads</span>
|
||||
<input type="number" name="cores" value="{{.CPUCores}}" min="0"></label>
|
||||
<button type="submit">Save</button>
|
||||
</div>
|
||||
<p class="note">{{if .MostNow}}At most {{.MostNow}} can always run now{{if .Headroom}}{{if .Headroom.ByPreempting}}, counting what its class may stop to make room{{end}}{{end}}. {{end}}0 always running with a
|
||||
higher limit loads it when asked for; 0 and 0 stops it. A replica is added under load when requests wait, and
|
||||
taken away after ten quiet minutes.</p>
|
||||
taken away after ten quiet minutes. A higher class may stop replicas of lower ones to make room; memory and
|
||||
cores apply to replicas started after the change.</p>
|
||||
</form>
|
||||
{{end}}
|
||||
</section>
|
||||
|
||||
@@ -147,6 +147,60 @@ keys, projects and the chat page. {{.Personal}}</p>
|
||||
</table>
|
||||
{{else}}<p class="empty">Nobody has been approved yet.</p>{{end}}
|
||||
|
||||
{{if .HasCapacity}}
|
||||
<h2 id="capacity">Capacity</h2>
|
||||
<p class="note">What a part of the tree is guaranteed of the cluster, and the most
|
||||
it may hold: GPUs, CPU cores and memory, counted from the replicas of the models
|
||||
it owns and of those beneath it. What it holds within its guarantee is never taken
|
||||
for someone else's model; beyond it, it borrows free room and gives it back when
|
||||
asked. A model's owner is set on its card on the mirror page.</p>
|
||||
{{if .Capacity}}
|
||||
<table class="rows quota-tree">
|
||||
<tr><th>Where</th><th class="num">GPUs</th><th class="num">Cores</th><th class="num">Memory GB</th><th>Set by</th><th></th></tr>
|
||||
{{range .Capacity}}<tr>
|
||||
<td class="q-where"><span class="q-name">{{or .Name .Scope}}</span></td>
|
||||
<td class="num">{{or .GPUs "—"}}</td>
|
||||
<td class="num">{{or .Cores "—"}}</td>
|
||||
<td class="num">{{or .Memory "—"}}</td>
|
||||
<td class="note">{{.UpdatedBy}}</td>
|
||||
<td class="q-act"><a href="/admin/quotas?cap={{.Scope}}#capacity">change</a>
|
||||
<form method="post" action="/admin/quotas/capacity" class="inline">
|
||||
<input type="hidden" name="csrf" value="{{$.CSRF}}">
|
||||
<input type="hidden" name="scope" value="{{.Scope}}">
|
||||
<button type="submit" name="clear" value="1" class="link danger">remove</button>
|
||||
</form></td>
|
||||
</tr>{{end}}
|
||||
</table>
|
||||
<p class="q-legend">Each figure is held of guaranteed (at most the limit); — is not governed.</p>
|
||||
{{else}}<p class="empty">No capacity quotas: every model shares the cluster by its class alone.</p>{{end}}
|
||||
|
||||
{{if .Scopes}}
|
||||
<form method="post" action="/admin/quotas/capacity" class="q-form capacity-form">
|
||||
<input type="hidden" name="csrf" value="{{.CSRF}}">
|
||||
<p class="q-form-title"><strong>{{if .CapForm.Scope}}Change{{else}}Set{{end}} a capacity quota.</strong>
|
||||
A blank field is not governed; 0 is none at all.</p>
|
||||
<label>Where
|
||||
<select name="scope" required>
|
||||
<option value="">choose…</option>
|
||||
{{range .Scopes}}<option value="{{.Key}}"{{if eq .Key $.CapForm.Scope}} selected{{end}}>{{.Name}}</option>{{end}}
|
||||
</select></label>
|
||||
<div class="field-row">
|
||||
<label>Guaranteed GPUs <input type="text" name="gpus" value="{{.CapForm.GPUs}}" placeholder="—"></label>
|
||||
<label>Cores <input type="text" name="cores" value="{{.CapForm.Cores}}" placeholder="—"></label>
|
||||
<label>Memory GB <input type="text" name="memory" value="{{.CapForm.Memory}}" placeholder="—"></label>
|
||||
</div>
|
||||
<div class="field-row">
|
||||
<label>At most GPUs <input type="text" name="limit_gpus" value="{{.CapForm.LGPUs}}" placeholder="—"></label>
|
||||
<label>Cores <input type="text" name="limit_cores" value="{{.CapForm.LCores}}" placeholder="—"></label>
|
||||
<label>Memory GB <input type="text" name="limit_memory" value="{{.CapForm.LMem}}" placeholder="—"></label>
|
||||
</div>
|
||||
<div class="q-form-actions"><button type="submit">Save</button>
|
||||
{{if .CapForm.Scope}}<a href="/admin/quotas#capacity">Cancel</a>{{end}}</div>
|
||||
</form>
|
||||
{{else}}<p class="empty">Nothing to give a capacity quota to yet. Make an organization on the
|
||||
<a href="/admin/users">people page</a> first.</p>{{end}}
|
||||
{{end}}
|
||||
|
||||
<details class="explain">
|
||||
<summary>How limits combine</summary>
|
||||
<p>Along a branch the most restrictive value wins, so a team cannot raise what
|
||||
|
||||
Reference in new issue
Block a user