Guarantee and limit capacity per part of the tenancy tree
A model has an owner (org, team or project). Quotas guarantee a scope GPUs, cores and memory and may limit it; what a scope holds is summed from its models' replicas and those beneath it. A replica within its owner's guarantee is not taken for another scope; what is held beyond a guarantee is borrowed and taken back for a scope within its own. A limit stops a model before it starts, and headroom stops there too. ctl quota and ctl model owner set them; metrics and a CapacityAtLimit alert report them. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
ac24e204a6
commit
9dc1b62fdc
28 files changed
+1475
-16
No files matched your search
@@ -42,6 +42,7 @@ type API struct {
|
||||
models Models
|
||||
capacity Capacity
|
||||
usage Usage
|
||||
quotas Quotas
|
||||
ctl *control.Controller
|
||||
}
|
||||
|
||||
@@ -58,6 +59,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("DELETE /api/v1/models/{name}", a.removeModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
|
||||
a.registerQuotas(mux)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/class", func(w http.ResponseWriter, r *http.Request) {
|
||||
if a.models == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||
@@ -304,6 +306,7 @@ type ModelView struct {
|
||||
// Class is its capacity class; Preempted the replicas it lost to a higher
|
||||
// class; HeldFor the model it is giving room to now, if any.
|
||||
Class string `json:"class"`
|
||||
Owner string `json:"owner,omitempty"`
|
||||
Preempted int `json:"preempted,omitempty"`
|
||||
HeldFor string `json:"held_for,omitempty"`
|
||||
}
|
||||
@@ -552,7 +555,7 @@ func (a *API) Models() []ModelView {
|
||||
Fallbacks: m.Fallbacks,
|
||||
Desired: h.Desired, Failures: h.Failures,
|
||||
Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable,
|
||||
Class: m.EffectiveClass(), Preempted: h.Preempted, HeldFor: h.HeldFor,
|
||||
Class: m.EffectiveClass(), Owner: m.Owner, Preempted: h.Preempted, HeldFor: h.HeldFor,
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
v.Profiles = append(v.Profiles, p.Name)
|
||||
@@ -634,6 +637,7 @@ type Models interface {
|
||||
Scale(ctx context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error)
|
||||
Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error)
|
||||
SetClass(ctx context.Context, name, class, by string) (catalog.Model, error)
|
||||
SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error)
|
||||
}
|
||||
|
||||
// ResourcesRequest sets what a model's profiles reserve of a node.
|
||||
|
||||
@@ -258,6 +258,10 @@ type fakeModels struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeModels) SetOwner(_ context.Context, name, owner, _ string) (catalog.Model, error) {
|
||||
return catalog.Model{Name: name, Owner: owner}, f.err
|
||||
}
|
||||
|
||||
func (f *fakeModels) SetClass(_ context.Context, name, class, _ string) (catalog.Model, error) {
|
||||
return catalog.Model{Name: name, Class: class}, f.err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// Quotas is where capacity quotas and the tenancy tree they name are kept.
|
||||
// The store implements it.
|
||||
type Quotas interface {
|
||||
CapacityQuotas(ctx context.Context) ([]store.CapacityQuota, error)
|
||||
PutCapacityQuota(ctx context.Context, q store.CapacityQuota) error
|
||||
DeleteCapacityQuota(ctx context.Context, scope store.Scope, id string) error
|
||||
ScopeParents(ctx context.Context) (map[string][]string, error)
|
||||
AllOrgs(ctx context.Context) ([]store.Org, error)
|
||||
AllTeams(ctx context.Context) ([]store.TeamPath, error)
|
||||
AllProjects(ctx context.Context) ([]store.ProjectPath, error)
|
||||
}
|
||||
|
||||
// SetQuotas supplies the quota store, which mounts the quota endpoints'
|
||||
// behaviour; without it they answer 501.
|
||||
func (a *API) SetQuotas(q Quotas) { a.quotas = q }
|
||||
|
||||
// QuotaView is one scope's quota with what it holds now.
|
||||
type QuotaView struct {
|
||||
// Scope is team:ID; Name is how people know it, "Acme / ml".
|
||||
Scope string `json:"scope"`
|
||||
Name string `json:"name"`
|
||||
Guarantee store.Resources `json:"guarantee"`
|
||||
Limit store.Resources `json:"limit"`
|
||||
Holds quota.Amount `json:"holds"`
|
||||
UpdatedBy string `json:"updated_by,omitempty"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// QuotaRequest sets a scope's quota. Scope may be written with an ID or a
|
||||
// name: org:Acme, team:Acme/ml, project:Acme/ml/training.
|
||||
type QuotaRequest struct {
|
||||
Scope string `json:"scope"`
|
||||
Guarantee store.Resources `json:"guarantee"`
|
||||
Limit store.Resources `json:"limit"`
|
||||
}
|
||||
|
||||
func (a *API) registerQuotas(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /api/v1/quotas", func(w http.ResponseWriter, r *http.Request) {
|
||||
if a.quotas == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||
return
|
||||
}
|
||||
views, err := a.QuotaViews(r.Context())
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, views)
|
||||
})
|
||||
mux.HandleFunc("PUT /api/v1/quotas", func(w http.ResponseWriter, r *http.Request) {
|
||||
if a.quotas == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||
return
|
||||
}
|
||||
var req QuotaRequest
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<14)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
scope, id, _, err := a.resolveScope(r.Context(), req.Scope)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
if !req.Guarantee.Any() && !req.Limit.Any() {
|
||||
writeError(w, http.StatusBadRequest, "a quota needs a guarantee or a limit; to remove one, DELETE it")
|
||||
return
|
||||
}
|
||||
q := store.CapacityQuota{Scope: scope, ScopeID: id, Guarantee: req.Guarantee, Limit: req.Limit,
|
||||
UpdatedBy: actorOf(r)}
|
||||
if err := a.quotas.PutCapacityQuota(r.Context(), q); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"scope": quota.Key(scope, id)})
|
||||
})
|
||||
mux.HandleFunc("DELETE /api/v1/quotas/{scope}", func(w http.ResponseWriter, r *http.Request) {
|
||||
if a.quotas == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||
return
|
||||
}
|
||||
scope, id, _, err := a.resolveScope(r.Context(), r.PathValue("scope"))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
if err := a.quotas.DeleteCapacityQuota(r.Context(), scope, id); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"removed": quota.Key(scope, id)})
|
||||
})
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/owner", func(w http.ResponseWriter, r *http.Request) {
|
||||
if a.models == nil || a.quotas == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Owner string `json:"owner"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
owner := ""
|
||||
if req.Owner != "" {
|
||||
scope, id, _, err := a.resolveScope(r.Context(), req.Owner)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
owner = quota.Key(scope, id)
|
||||
}
|
||||
m, err := a.models.SetOwner(r.Context(), r.PathValue("name"), owner, actorOf(r))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"model": m.Name, "owner": m.Owner})
|
||||
})
|
||||
}
|
||||
|
||||
// QuotaViews lists every quota with what its scope holds now.
|
||||
func (a *API) QuotaViews(ctx context.Context) ([]QuotaView, error) {
|
||||
qs, err := a.quotas.CapacityQuotas(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
parents, err := a.quotas.ScopeParents(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names, err := a.scopeNames(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
usage := quota.New(qs, parents).Usage(a.ctl.Reconciler().Holdings())
|
||||
out := make([]QuotaView, 0, len(qs))
|
||||
for _, q := range qs {
|
||||
key := quota.Key(q.Scope, q.ScopeID)
|
||||
out = append(out, QuotaView{Scope: key, Name: names[key], Guarantee: q.Guarantee, Limit: q.Limit,
|
||||
Holds: usage[key], UpdatedBy: q.UpdatedBy, UpdatedAt: q.UpdatedAt})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// scopeNames maps org:ID, team:ID and project:ID to their paths.
|
||||
func (a *API) scopeNames(ctx context.Context) (map[string]string, error) {
|
||||
out := map[string]string{}
|
||||
orgs, err := a.quotas.AllOrgs(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, o := range orgs {
|
||||
out["org:"+o.ID] = o.Name
|
||||
}
|
||||
teams, err := a.quotas.AllTeams(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, t := range teams {
|
||||
out["team:"+t.ID] = t.OrgName + " / " + t.Name
|
||||
}
|
||||
projects, err := a.quotas.AllProjects(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, p := range projects {
|
||||
out["project:"+p.ID] = p.Path
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// resolveScope reads org:X, team:X or project:X, where X is an ID or the
|
||||
// scope's path — Acme, Acme/ml, Acme/ml/training — and returns its ID and
|
||||
// path. A path that could mean two scopes is refused rather than guessed.
|
||||
func (a *API) resolveScope(ctx context.Context, s string) (store.Scope, string, string, error) {
|
||||
kind, what, ok := strings.Cut(strings.TrimSpace(s), ":")
|
||||
scope := store.Scope(kind)
|
||||
if !ok || what == "" || (scope != store.ScopeOrg && scope != store.ScopeTeam && scope != store.ScopeProject) {
|
||||
return "", "", "", fmt.Errorf("%q is not org:NAME, team:ORG/TEAM or project:ORG/TEAM/PROJECT", s)
|
||||
}
|
||||
names, err := a.scopeNames(ctx)
|
||||
if err != nil {
|
||||
return "", "", "", err
|
||||
}
|
||||
norm := func(p string) string {
|
||||
parts := strings.Split(p, "/")
|
||||
for i := range parts {
|
||||
parts[i] = strings.ToLower(strings.TrimSpace(parts[i]))
|
||||
}
|
||||
return strings.Join(parts, "/")
|
||||
}
|
||||
if name, ok := names[kind+":"+what]; ok {
|
||||
return scope, what, name, nil
|
||||
}
|
||||
match := func(same func(name string) bool) []string {
|
||||
var found []string
|
||||
for key, name := range names {
|
||||
if strings.HasPrefix(key, kind+":") && same(name) {
|
||||
found = append(found, key)
|
||||
}
|
||||
}
|
||||
return found
|
||||
}
|
||||
found := match(func(name string) bool { return norm(name) == norm(what) })
|
||||
if len(found) == 0 {
|
||||
// Its own name alone — ml rather than Acme/ml — when that is unique.
|
||||
found = match(func(name string) bool {
|
||||
parts := strings.Split(norm(name), "/")
|
||||
return parts[len(parts)-1] == norm(what)
|
||||
})
|
||||
}
|
||||
switch len(found) {
|
||||
case 0:
|
||||
return "", "", "", fmt.Errorf("there is no %s %q", kind, what)
|
||||
case 1:
|
||||
_, id, _ := strings.Cut(found[0], ":")
|
||||
return scope, id, names[found[0]], nil
|
||||
}
|
||||
return "", "", "", fmt.Errorf("%q could be any of %d %ss; give its ID", what, len(found), kind)
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// fakeQuotaStore is Acme with teams ml and web, and Scania with another ml.
|
||||
type fakeQuotaStore struct {
|
||||
quotas []store.CapacityQuota
|
||||
}
|
||||
|
||||
func (f *fakeQuotaStore) CapacityQuotas(context.Context) ([]store.CapacityQuota, error) {
|
||||
return f.quotas, nil
|
||||
}
|
||||
func (f *fakeQuotaStore) PutCapacityQuota(_ context.Context, q store.CapacityQuota) error {
|
||||
for i := range f.quotas {
|
||||
if f.quotas[i].Scope == q.Scope && f.quotas[i].ScopeID == q.ScopeID {
|
||||
f.quotas[i] = q
|
||||
return nil
|
||||
}
|
||||
}
|
||||
f.quotas = append(f.quotas, q)
|
||||
return nil
|
||||
}
|
||||
func (f *fakeQuotaStore) DeleteCapacityQuota(_ context.Context, s store.Scope, id string) error {
|
||||
for i, q := range f.quotas {
|
||||
if q.Scope == s && q.ScopeID == id {
|
||||
f.quotas = append(f.quotas[:i], f.quotas[i+1:]...)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (f *fakeQuotaStore) ScopeParents(context.Context) (map[string][]string, error) {
|
||||
return map[string][]string{"team:t-ml": {"org:o-acme"}, "team:t-web": {"org:o-acme"},
|
||||
"team:t-sml": {"org:o-scania"}}, nil
|
||||
}
|
||||
func (f *fakeQuotaStore) AllOrgs(context.Context) ([]store.Org, error) {
|
||||
return []store.Org{{ID: "o-acme", Name: "Acme"}, {ID: "o-scania", Name: "Scania"}}, nil
|
||||
}
|
||||
func (f *fakeQuotaStore) AllTeams(context.Context) ([]store.TeamPath, error) {
|
||||
return []store.TeamPath{{ID: "t-ml", Name: "ml", OrgID: "o-acme", OrgName: "Acme"},
|
||||
{ID: "t-web", Name: "web", OrgID: "o-acme", OrgName: "Acme"},
|
||||
{ID: "t-sml", Name: "ml", OrgID: "o-scania", OrgName: "Scania"}}, nil
|
||||
}
|
||||
func (f *fakeQuotaStore) AllProjects(context.Context) ([]store.ProjectPath, error) { return nil, nil }
|
||||
|
||||
func TestAScopeIsNamedByPathOrIDAndNeverGuessed(t *testing.T) {
|
||||
a, _ := setup(t, testModels())
|
||||
a.SetQuotas(&fakeQuotaStore{})
|
||||
for in, want := range map[string]string{
|
||||
"org:Acme": "o-acme", "org:o-acme": "o-acme", "team:Acme/ml": "t-ml",
|
||||
"team: acme / ML ": "t-ml", "team:Scania/ml": "t-sml", "team:web": "t-web",
|
||||
} {
|
||||
_, id, _, err := a.resolveScope(t.Context(), in)
|
||||
if err != nil || id != want {
|
||||
t.Errorf("%q: %q %v, expected %q", in, id, err, want)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"team:ml", "team:Acme/nope", "group:Acme", "Acme", "org:"} {
|
||||
if _, _, _, err := a.resolveScope(t.Context(), bad); err == nil {
|
||||
t.Errorf("%q was resolved", bad)
|
||||
}
|
||||
}
|
||||
if _, _, _, err := a.resolveScope(t.Context(), "team:ml"); err == nil || !strings.Contains(err.Error(), "could be any of 2") {
|
||||
t.Errorf("an ambiguous name should say so: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAQuotaIsSetListedAndCleared(t *testing.T) {
|
||||
a, _ := setup(t, testModels())
|
||||
qs := &fakeQuotaStore{}
|
||||
a.SetQuotas(qs)
|
||||
mux := http.NewServeMux()
|
||||
a.Register(mux)
|
||||
call := func(method, path, body string) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest(method, path, strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("X-Llmgw-Actor", "ops")
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
if rec := call("PUT", "/api/v1/quotas", `{"scope":"team:Acme/ml","guarantee":{"gpus":8},"limit":{"gpus":16}}`); rec.Code != 200 {
|
||||
t.Fatalf("set: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
if len(qs.quotas) != 1 || qs.quotas[0].ScopeID != "t-ml" || *qs.quotas[0].Guarantee.GPUs != 8 || qs.quotas[0].UpdatedBy != "ops" {
|
||||
t.Errorf("stored %+v", qs.quotas)
|
||||
}
|
||||
rec := call("GET", "/api/v1/quotas", "")
|
||||
var views []QuotaView
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &views); err != nil || len(views) != 1 || views[0].Name != "Acme / ml" {
|
||||
t.Errorf("list %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
if rec := call("PUT", "/api/v1/quotas", `{"scope":"team:Acme/ml"}`); rec.Code != 400 {
|
||||
t.Errorf("an empty quota was accepted: %d", rec.Code)
|
||||
}
|
||||
if rec := call("DELETE", "/api/v1/quotas/team:Acme%2Fml", ""); rec.Code != 200 || len(qs.quotas) != 0 {
|
||||
t.Errorf("clear: %d %s %+v", rec.Code, rec.Body, qs.quotas)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user