Guarantee and limit capacity per part of the tenancy tree
Release / build (push) Successful in 1m8s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m51s

A model has an owner (org, team or project). Quotas guarantee a scope
GPUs, cores and memory and may limit it; what a scope holds is summed from
its models' replicas and those beneath it. A replica within its owner's
guarantee is not taken for another scope; what is held beyond a guarantee
is borrowed and taken back for a scope within its own. A limit stops a
model before it starts, and headroom stops there too. ctl quota and ctl
model owner set them; metrics and a CapacityAtLimit alert report them.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 18:15:38 +02:00
1 parent ac24e204a6
commit 9dc1b62fdc
28 files changed
+1475 -16

No files matched your search

+5 -1
View File
@@ -42,6 +42,7 @@ type API struct {
models Models
capacity Capacity
usage Usage
quotas Quotas
ctl *control.Controller
}
@@ -58,6 +59,7 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("DELETE /api/v1/models/{name}", a.removeModel)
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
a.registerQuotas(mux)
mux.HandleFunc("POST /api/v1/models/{name}/class", func(w http.ResponseWriter, r *http.Request) {
if a.models == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no store")
@@ -304,6 +306,7 @@ type ModelView struct {
// Class is its capacity class; Preempted the replicas it lost to a higher
// class; HeldFor the model it is giving room to now, if any.
Class string `json:"class"`
Owner string `json:"owner,omitempty"`
Preempted int `json:"preempted,omitempty"`
HeldFor string `json:"held_for,omitempty"`
}
@@ -552,7 +555,7 @@ func (a *API) Models() []ModelView {
Fallbacks: m.Fallbacks,
Desired: h.Desired, Failures: h.Failures,
Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable,
Class: m.EffectiveClass(), Preempted: h.Preempted, HeldFor: h.HeldFor,
Class: m.EffectiveClass(), Owner: m.Owner, Preempted: h.Preempted, HeldFor: h.HeldFor,
}
for _, p := range m.Profiles {
v.Profiles = append(v.Profiles, p.Name)
@@ -634,6 +637,7 @@ type Models interface {
Scale(ctx context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error)
Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error)
SetClass(ctx context.Context, name, class, by string) (catalog.Model, error)
SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error)
}
// ResourcesRequest sets what a model's profiles reserve of a node.
+4
View File
@@ -258,6 +258,10 @@ type fakeModels struct {
err error
}
func (f *fakeModels) SetOwner(_ context.Context, name, owner, _ string) (catalog.Model, error) {
return catalog.Model{Name: name, Owner: owner}, f.err
}
func (f *fakeModels) SetClass(_ context.Context, name, class, _ string) (catalog.Model, error) {
return catalog.Model{Name: name, Class: class}, f.err
}
+237
View File
@@ -0,0 +1,237 @@
package adminapi
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// Quotas is where capacity quotas and the tenancy tree they name are kept.
// The store implements it.
type Quotas interface {
CapacityQuotas(ctx context.Context) ([]store.CapacityQuota, error)
PutCapacityQuota(ctx context.Context, q store.CapacityQuota) error
DeleteCapacityQuota(ctx context.Context, scope store.Scope, id string) error
ScopeParents(ctx context.Context) (map[string][]string, error)
AllOrgs(ctx context.Context) ([]store.Org, error)
AllTeams(ctx context.Context) ([]store.TeamPath, error)
AllProjects(ctx context.Context) ([]store.ProjectPath, error)
}
// SetQuotas supplies the quota store, which mounts the quota endpoints'
// behaviour; without it they answer 501.
func (a *API) SetQuotas(q Quotas) { a.quotas = q }
// QuotaView is one scope's quota with what it holds now.
type QuotaView struct {
// Scope is team:ID; Name is how people know it, "Acme / ml".
Scope string `json:"scope"`
Name string `json:"name"`
Guarantee store.Resources `json:"guarantee"`
Limit store.Resources `json:"limit"`
Holds quota.Amount `json:"holds"`
UpdatedBy string `json:"updated_by,omitempty"`
UpdatedAt time.Time `json:"updated_at"`
}
// QuotaRequest sets a scope's quota. Scope may be written with an ID or a
// name: org:Acme, team:Acme/ml, project:Acme/ml/training.
type QuotaRequest struct {
Scope string `json:"scope"`
Guarantee store.Resources `json:"guarantee"`
Limit store.Resources `json:"limit"`
}
func (a *API) registerQuotas(mux *http.ServeMux) {
mux.HandleFunc("GET /api/v1/quotas", func(w http.ResponseWriter, r *http.Request) {
if a.quotas == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no store")
return
}
views, err := a.QuotaViews(r.Context())
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, views)
})
mux.HandleFunc("PUT /api/v1/quotas", func(w http.ResponseWriter, r *http.Request) {
if a.quotas == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no store")
return
}
var req QuotaRequest
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<14)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
}
scope, id, _, err := a.resolveScope(r.Context(), req.Scope)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
if !req.Guarantee.Any() && !req.Limit.Any() {
writeError(w, http.StatusBadRequest, "a quota needs a guarantee or a limit; to remove one, DELETE it")
return
}
q := store.CapacityQuota{Scope: scope, ScopeID: id, Guarantee: req.Guarantee, Limit: req.Limit,
UpdatedBy: actorOf(r)}
if err := a.quotas.PutCapacityQuota(r.Context(), q); err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"scope": quota.Key(scope, id)})
})
mux.HandleFunc("DELETE /api/v1/quotas/{scope}", func(w http.ResponseWriter, r *http.Request) {
if a.quotas == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no store")
return
}
scope, id, _, err := a.resolveScope(r.Context(), r.PathValue("scope"))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
if err := a.quotas.DeleteCapacityQuota(r.Context(), scope, id); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"removed": quota.Key(scope, id)})
})
mux.HandleFunc("POST /api/v1/models/{name}/owner", func(w http.ResponseWriter, r *http.Request) {
if a.models == nil || a.quotas == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no store")
return
}
var req struct {
Owner string `json:"owner"`
}
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
}
owner := ""
if req.Owner != "" {
scope, id, _, err := a.resolveScope(r.Context(), req.Owner)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
owner = quota.Key(scope, id)
}
m, err := a.models.SetOwner(r.Context(), r.PathValue("name"), owner, actorOf(r))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"model": m.Name, "owner": m.Owner})
})
}
// QuotaViews lists every quota with what its scope holds now.
func (a *API) QuotaViews(ctx context.Context) ([]QuotaView, error) {
qs, err := a.quotas.CapacityQuotas(ctx)
if err != nil {
return nil, err
}
parents, err := a.quotas.ScopeParents(ctx)
if err != nil {
return nil, err
}
names, err := a.scopeNames(ctx)
if err != nil {
return nil, err
}
usage := quota.New(qs, parents).Usage(a.ctl.Reconciler().Holdings())
out := make([]QuotaView, 0, len(qs))
for _, q := range qs {
key := quota.Key(q.Scope, q.ScopeID)
out = append(out, QuotaView{Scope: key, Name: names[key], Guarantee: q.Guarantee, Limit: q.Limit,
Holds: usage[key], UpdatedBy: q.UpdatedBy, UpdatedAt: q.UpdatedAt})
}
return out, nil
}
// scopeNames maps org:ID, team:ID and project:ID to their paths.
func (a *API) scopeNames(ctx context.Context) (map[string]string, error) {
out := map[string]string{}
orgs, err := a.quotas.AllOrgs(ctx)
if err != nil {
return nil, err
}
for _, o := range orgs {
out["org:"+o.ID] = o.Name
}
teams, err := a.quotas.AllTeams(ctx)
if err != nil {
return nil, err
}
for _, t := range teams {
out["team:"+t.ID] = t.OrgName + " / " + t.Name
}
projects, err := a.quotas.AllProjects(ctx)
if err != nil {
return nil, err
}
for _, p := range projects {
out["project:"+p.ID] = p.Path
}
return out, nil
}
// resolveScope reads org:X, team:X or project:X, where X is an ID or the
// scope's path — Acme, Acme/ml, Acme/ml/training — and returns its ID and
// path. A path that could mean two scopes is refused rather than guessed.
func (a *API) resolveScope(ctx context.Context, s string) (store.Scope, string, string, error) {
kind, what, ok := strings.Cut(strings.TrimSpace(s), ":")
scope := store.Scope(kind)
if !ok || what == "" || (scope != store.ScopeOrg && scope != store.ScopeTeam && scope != store.ScopeProject) {
return "", "", "", fmt.Errorf("%q is not org:NAME, team:ORG/TEAM or project:ORG/TEAM/PROJECT", s)
}
names, err := a.scopeNames(ctx)
if err != nil {
return "", "", "", err
}
norm := func(p string) string {
parts := strings.Split(p, "/")
for i := range parts {
parts[i] = strings.ToLower(strings.TrimSpace(parts[i]))
}
return strings.Join(parts, "/")
}
if name, ok := names[kind+":"+what]; ok {
return scope, what, name, nil
}
match := func(same func(name string) bool) []string {
var found []string
for key, name := range names {
if strings.HasPrefix(key, kind+":") && same(name) {
found = append(found, key)
}
}
return found
}
found := match(func(name string) bool { return norm(name) == norm(what) })
if len(found) == 0 {
// Its own name alone — ml rather than Acme/ml — when that is unique.
found = match(func(name string) bool {
parts := strings.Split(norm(name), "/")
return parts[len(parts)-1] == norm(what)
})
}
switch len(found) {
case 0:
return "", "", "", fmt.Errorf("there is no %s %q", kind, what)
case 1:
_, id, _ := strings.Cut(found[0], ":")
return scope, id, names[found[0]], nil
}
return "", "", "", fmt.Errorf("%q could be any of %d %ss; give its ID", what, len(found), kind)
}
+107
View File
@@ -0,0 +1,107 @@
package adminapi
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// fakeQuotaStore is Acme with teams ml and web, and Scania with another ml.
type fakeQuotaStore struct {
quotas []store.CapacityQuota
}
func (f *fakeQuotaStore) CapacityQuotas(context.Context) ([]store.CapacityQuota, error) {
return f.quotas, nil
}
func (f *fakeQuotaStore) PutCapacityQuota(_ context.Context, q store.CapacityQuota) error {
for i := range f.quotas {
if f.quotas[i].Scope == q.Scope && f.quotas[i].ScopeID == q.ScopeID {
f.quotas[i] = q
return nil
}
}
f.quotas = append(f.quotas, q)
return nil
}
func (f *fakeQuotaStore) DeleteCapacityQuota(_ context.Context, s store.Scope, id string) error {
for i, q := range f.quotas {
if q.Scope == s && q.ScopeID == id {
f.quotas = append(f.quotas[:i], f.quotas[i+1:]...)
}
}
return nil
}
func (f *fakeQuotaStore) ScopeParents(context.Context) (map[string][]string, error) {
return map[string][]string{"team:t-ml": {"org:o-acme"}, "team:t-web": {"org:o-acme"},
"team:t-sml": {"org:o-scania"}}, nil
}
func (f *fakeQuotaStore) AllOrgs(context.Context) ([]store.Org, error) {
return []store.Org{{ID: "o-acme", Name: "Acme"}, {ID: "o-scania", Name: "Scania"}}, nil
}
func (f *fakeQuotaStore) AllTeams(context.Context) ([]store.TeamPath, error) {
return []store.TeamPath{{ID: "t-ml", Name: "ml", OrgID: "o-acme", OrgName: "Acme"},
{ID: "t-web", Name: "web", OrgID: "o-acme", OrgName: "Acme"},
{ID: "t-sml", Name: "ml", OrgID: "o-scania", OrgName: "Scania"}}, nil
}
func (f *fakeQuotaStore) AllProjects(context.Context) ([]store.ProjectPath, error) { return nil, nil }
func TestAScopeIsNamedByPathOrIDAndNeverGuessed(t *testing.T) {
a, _ := setup(t, testModels())
a.SetQuotas(&fakeQuotaStore{})
for in, want := range map[string]string{
"org:Acme": "o-acme", "org:o-acme": "o-acme", "team:Acme/ml": "t-ml",
"team: acme / ML ": "t-ml", "team:Scania/ml": "t-sml", "team:web": "t-web",
} {
_, id, _, err := a.resolveScope(t.Context(), in)
if err != nil || id != want {
t.Errorf("%q: %q %v, expected %q", in, id, err, want)
}
}
for _, bad := range []string{"team:ml", "team:Acme/nope", "group:Acme", "Acme", "org:"} {
if _, _, _, err := a.resolveScope(t.Context(), bad); err == nil {
t.Errorf("%q was resolved", bad)
}
}
if _, _, _, err := a.resolveScope(t.Context(), "team:ml"); err == nil || !strings.Contains(err.Error(), "could be any of 2") {
t.Errorf("an ambiguous name should say so: %v", err)
}
}
func TestAQuotaIsSetListedAndCleared(t *testing.T) {
a, _ := setup(t, testModels())
qs := &fakeQuotaStore{}
a.SetQuotas(qs)
mux := http.NewServeMux()
a.Register(mux)
call := func(method, path, body string) *httptest.ResponseRecorder {
req := httptest.NewRequest(method, path, strings.NewReader(body))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-Llmgw-Actor", "ops")
rec := httptest.NewRecorder()
mux.ServeHTTP(rec, req)
return rec
}
if rec := call("PUT", "/api/v1/quotas", `{"scope":"team:Acme/ml","guarantee":{"gpus":8},"limit":{"gpus":16}}`); rec.Code != 200 {
t.Fatalf("set: %d %s", rec.Code, rec.Body)
}
if len(qs.quotas) != 1 || qs.quotas[0].ScopeID != "t-ml" || *qs.quotas[0].Guarantee.GPUs != 8 || qs.quotas[0].UpdatedBy != "ops" {
t.Errorf("stored %+v", qs.quotas)
}
rec := call("GET", "/api/v1/quotas", "")
var views []QuotaView
if err := json.Unmarshal(rec.Body.Bytes(), &views); err != nil || len(views) != 1 || views[0].Name != "Acme / ml" {
t.Errorf("list %d %s", rec.Code, rec.Body)
}
if rec := call("PUT", "/api/v1/quotas", `{"scope":"team:Acme/ml"}`); rec.Code != 400 {
t.Errorf("an empty quota was accepted: %d", rec.Code)
}
if rec := call("DELETE", "/api/v1/quotas/team:Acme%2Fml", ""); rec.Code != 200 || len(qs.quotas) != 0 {
t.Errorf("clear: %d %s %+v", rec.Code, rec.Body, qs.quotas)
}
}