From 9dc1b62fdc55756d6169a37c7b238ecb67649304 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 30 Sep 2026 18:15:38 +0200 Subject: [PATCH] Guarantee and limit capacity per part of the tenancy tree A model has an owner (org, team or project). Quotas guarantee a scope GPUs, cores and memory and may limit it; what a scope holds is summed from its models' replicas and those beneath it. A replica within its owner's guarantee is not taken for another scope; what is held beyond a guarantee is borrowed and taken back for a scope within its own. A limit stops a model before it starts, and headroom stops there too. ctl quota and ctl model owner set them; metrics and a CapacityAtLimit alert report them. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 25 ++ deploy/monitoring/llmgw-rules.yml | 11 + docs/reference.md | 9 + internal/cli/ctl.go | 1 + internal/cli/model.go | 2 +- internal/cli/quota.go | 164 ++++++++++++ internal/cli/server.go | 5 + internal/control/adminapi/api.go | 6 +- internal/control/adminapi/api_test.go | 4 + internal/control/adminapi/quotas.go | 237 ++++++++++++++++++ internal/control/adminapi/quotas_test.go | 107 ++++++++ internal/control/catalog/catalog.go | 18 +- internal/control/control.go | 3 + internal/control/fleetmetrics/fleetmetrics.go | 63 ++++- internal/control/modelsvc/modelsvc.go | 13 + internal/control/quota/quota.go | 192 ++++++++++++++ internal/control/quota/quota_test.go | 83 ++++++ internal/control/quotas.go | 52 ++++ internal/control/reconciler/headroom.go | 17 ++ internal/control/reconciler/preempt.go | 49 +++- internal/control/reconciler/quotas.go | 105 ++++++++ internal/control/reconciler/quotas_test.go | 81 ++++++ internal/control/reconciler/reconciler.go | 13 + internal/store/copy.go | 3 + .../postgres/0013_capacity_quotas.sql | 19 ++ .../sqlite/0013_capacity_quotas.sql | 19 ++ internal/store/quotas.go | 127 ++++++++++ internal/store/quotas_test.go | 63 +++++ 28 files changed, 1475 insertions(+), 16 deletions(-) create mode 100644 internal/cli/quota.go create mode 100644 internal/control/adminapi/quotas.go create mode 100644 internal/control/adminapi/quotas_test.go create mode 100644 internal/control/quota/quota.go create mode 100644 internal/control/quota/quota_test.go create mode 100644 internal/control/quotas.go create mode 100644 internal/control/reconciler/quotas.go create mode 100644 internal/control/reconciler/quotas_test.go create mode 100644 internal/store/migrations/postgres/0013_capacity_quotas.sql create mode 100644 internal/store/migrations/sqlite/0013_capacity_quotas.sql create mode 100644 internal/store/quotas.go create mode 100644 internal/store/quotas_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 9792204..c176a51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,31 @@ it gave way to has its replicas, so it does not take the room straight back. cluster page shows the class and how many replicas were lost; the metric is `llmgw_model_preempted_total{model,class}`, and every preemption is logged. +### Capacity quotas for the tenancy tree + +An organization, team or project can be guaranteed GPUs, CPU cores and +memory, and limited to at most some amount of each. What it holds is what the +replicas of the models it owns hold — and those of the teams and projects +beneath it. `llmgw ctl model owner MODEL team:Acme/ml` says whose a model is; +one with no owner is shared and counts against nobody. + +- **A guarantee is kept.** A replica within its owner's guarantee is never + taken for another scope's model, whatever the classes: a guarantee a higher + class elsewhere could take would not be one. +- **Beyond it is borrowing.** Free room is used; when a scope within its own + guarantee needs room, what others hold beyond theirs is taken back first, + whatever their class — except a critical model's minimum. +- **A limit is a ceiling.** A model whose owner would go past a limit anywhere + up the tree gets no further replica, and says which limit; headroom and the + mirror card stop there too. + +`llmgw ctl quota set team:Acme/ml --gpus 8 --limit-gpus 16` (also `--cores`, +`--memory` and their limits), `llmgw ctl quota list` with what each scope holds, +`llmgw ctl quota clear`. Scopes are named by path or ID; a name that could mean +two is refused. Changes take effect within half a minute. Metrics: +`llmgw_capacity_holds`, `llmgw_capacity_guarantee` and `llmgw_capacity_limit` +by scope and resource; alert: CapacityAtLimit. + ### The mirror page in tabs /admin/mirror was one long page: what the mirror holds, the Hugging Face diff --git a/deploy/monitoring/llmgw-rules.yml b/deploy/monitoring/llmgw-rules.yml index e28dc53..4393b0e 100644 --- a/deploy/monitoring/llmgw-rules.yml +++ b/deploy/monitoring/llmgw-rules.yml @@ -95,6 +95,17 @@ groups: the cluster is too small for what is asked of it at these classes. `llmgw ctl models` shows who it gave way to. + - alert: CapacityAtLimit + expr: llmgw_capacity_holds >= on(scope, resource) llmgw_capacity_limit + for: 15m + labels: { severity: info } + annotations: + summary: "{{ $labels.scope }} holds all the {{ $labels.resource }} it may" + description: >- + Its models get no further replicas, under load or not, until + something of theirs stops or the limit is raised: `llmgw ctl quota + list`, `llmgw ctl quota set`. + - alert: EngineInstallFailed expr: llmgw_node_engine_install{state="failed"} == 1 for: 15m diff --git a/docs/reference.md b/docs/reference.md index 43b93d6..bcef9c7 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -93,6 +93,7 @@ rather than repeated. | `profiles` | `[]Profile` | [Model](#model) | | `scaling` | `Scaling` | [Model](#model) | | `class` | `string` | [Model](#model) | +| `owner` | `string` | [Model](#model) | | `fallbacks` | `[]string` | [Model](#model) | | `routing_profile` | `string` | [Model](#model) | | `slo` | `SLO` | [Model](#model) | @@ -746,6 +747,14 @@ when there is not enough for both: critical, standard or opportunistic. Empty is standard, or opportunistic for a model loaded on demand. See EffectiveClass. +### `owner` + +`string` + +Owner is the part of the tenancy tree whose capacity quota this model +counts against, written org:ID, team:ID or project:ID. Empty is shared: +held by the cluster, counted against nobody. + ### `fallbacks` `[]string` diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index cec53ce..abea7d7 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -146,6 +146,7 @@ func newCtlCmd() *cobra.Command { newToolCheckCmd(), newWeightsCmd(client, &asJSON), newCapacityCmd(client, &asJSON), + newQuotaCmd(client, &asJSON), newUsageCmd(client, &asJSON), newCertCmd(), ) diff --git a/internal/cli/model.go b/internal/cli/model.go index 4abcff4..75a39f6 100644 --- a/internal/cli/model.go +++ b/internal/cli/model.go @@ -30,7 +30,7 @@ func newModelCmd(client func() *ctlClient) *cobra.Command { Short: "Add or remove models while the cluster is running", } c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client), - newResourcesCmd(client), newClassCmd(client), + newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client), newModelServeCmd(client)) return c } diff --git a/internal/cli/quota.go b/internal/cli/quota.go new file mode 100644 index 0000000..530ba0f --- /dev/null +++ b/internal/cli/quota.go @@ -0,0 +1,164 @@ +package cli + +import ( + "encoding/json" + "fmt" + "net/url" + "strconv" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +func newQuotaCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { + c := &cobra.Command{ + Use: "quota", + Short: "What parts of the tenancy tree are guaranteed of the cluster, and may hold", + Long: "A capacity quota belongs to an organization, team or project, and counts the\n" + + "GPUs, CPU cores and memory held by replicas of the models it owns (see\n" + + "`llmgw ctl model owner`), and of those its teams and projects own.\n\n" + + "The guarantee is kept for it: when it holds less and a model of its needs\n" + + "room, replicas other scopes hold beyond their own guarantees are stopped to\n" + + "make it — whatever their class. What it holds within its guarantee is\n" + + "never taken by another scope's model. Beyond it, it borrows: free room is\n" + + "used, and given back when asked for.\n\n" + + "The limit is the most it may hold, borrowing included; a model that would\n" + + "take it past is not given another replica.\n\n" + + "Scopes are written org:Acme, team:Acme/ml or project:Acme/ml/training, or\n" + + "with IDs. Changes take effect within half a minute.", + } + c.AddCommand(newQuotaListCmd(client, asJSON), newQuotaSetCmd(client), newQuotaClearCmd(client)) + return c +} + +func newQuotaListCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { + return &cobra.Command{ + Use: "list", + Short: "Every quota, with what its scope holds now", + RunE: func(cmd *cobra.Command, _ []string) error { + var qs []adminapi.QuotaView + if err := client().get(cmd.Context(), "/api/v1/quotas", &qs); err != nil { + return err + } + if *asJSON { + return json.NewEncoder(cmd.OutOrStdout()).Encode(qs) + } + if len(qs) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No capacity quotas: everything shares the cluster by class alone.") + return nil + } + w := tab() + fmt.Fprintln(w, "SCOPE\tNAME\tGPUS\tCORES\tMEMORY GB") + for _, q := range qs { + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", q.Scope, q.Name, + quotaCell(q.Holds.GPUs, q.Guarantee.GPUs, q.Limit.GPUs), + quotaCell(q.Holds.CPUCores, q.Guarantee.CPUCores, q.Limit.CPUCores), + quotaCell(q.Holds.MemoryGB, q.Guarantee.MemoryGB, q.Limit.MemoryGB)) + } + if err := w.Flush(); err != nil { + return err + } + fmt.Fprintln(cmd.OutOrStdout(), "\nEach cell: holds / guaranteed (at most limit). - is not governed.") + return nil + }, + } +} + +// quotaCell is "6 / 8 (≤ 16)": what is held, what is guaranteed, the limit. +func quotaCell(holds int, guarantee, limit *int) string { + if guarantee == nil && limit == nil { + return "-" + } + out := strconv.Itoa(holds) + " / " + if guarantee != nil { + out += strconv.Itoa(*guarantee) + } else { + out += "-" + } + if limit != nil { + out += fmt.Sprintf(" (≤ %d)", *limit) + } + return out +} + +func newQuotaSetCmd(client func() *ctlClient) *cobra.Command { + var gpus, cores, mem, lgpus, lcores, lmem int + c := &cobra.Command{ + Use: "set SCOPE [--gpus N] [--cores N] [--memory GB] [--limit-gpus N] ...", + Short: "Set a scope's guarantee and limit, replacing what it had", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + f := cmd.Flags() + pick := func(name string, v int) *int { + if !f.Changed(name) { + return nil + } + return &v + } + req := adminapi.QuotaRequest{Scope: args[0], + Guarantee: store.Resources{GPUs: pick("gpus", gpus), CPUCores: pick("cores", cores), MemoryGB: pick("memory", mem)}, + Limit: store.Resources{GPUs: pick("limit-gpus", lgpus), CPUCores: pick("limit-cores", lcores), MemoryGB: pick("limit-memory", lmem)}, + } + body, _ := json.Marshal(req) + var res map[string]string + if err := client().put(cmd.Context(), "/api/v1/quotas", body, &res); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "%s: set; it takes effect within half a minute\n", res["scope"]) + return nil + }, + } + f := c.Flags() + f.IntVar(&gpus, "gpus", 0, "GPUs guaranteed") + f.IntVar(&cores, "cores", 0, "CPU cores guaranteed") + f.IntVar(&mem, "memory", 0, "memory guaranteed, in GB") + f.IntVar(&lgpus, "limit-gpus", 0, "the most GPUs it may hold") + f.IntVar(&lcores, "limit-cores", 0, "the most CPU cores it may hold") + f.IntVar(&lmem, "limit-memory", 0, "the most memory it may hold, in GB") + return c +} + +func newQuotaClearCmd(client func() *ctlClient) *cobra.Command { + return &cobra.Command{ + Use: "clear SCOPE", + Short: "Remove a scope's quota", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + var res map[string]string + if err := client().delete(cmd.Context(), "/api/v1/quotas/"+url.PathEscape(args[0]), &res); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "%s: quota removed\n", res["removed"]) + return nil + }, + } +} + +func newOwnerCmd(client func() *ctlClient) *cobra.Command { + return &cobra.Command{ + Use: "owner SCOPE|shared", + Short: "Set whose capacity quota a model counts against", + Long: "SCOPE is org:Acme, team:Acme/ml or project:Acme/ml/training. A shared\n" + + "model belongs to the cluster and counts against nobody's quota.", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + owner := args[1] + if owner == "shared" { + owner = "" + } + body, _ := json.Marshal(map[string]string{"owner": owner}) + var res map[string]string + if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/owner", body, &res); err != nil { + return err + } + if res["owner"] == "" { + fmt.Fprintf(cmd.OutOrStdout(), "%s is shared\n", res["model"]) + } else { + fmt.Fprintf(cmd.OutOrStdout(), "%s counts against %s\n", res["model"], res["owner"]) + } + return nil + }, + } +} diff --git a/internal/cli/server.go b/internal/cli/server.go index 3a90420..016d454 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -226,6 +226,8 @@ func newServerCmd() *cobra.Command { // And what each group of the tenancy tree spent, which is // the other half of a quota being reviewable. api.SetUsage(st) + // Capacity quotas, set and read over this API. + api.SetQuotas(st) } api.Register(adminMux) @@ -483,6 +485,9 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live if st := h.Store(); st != nil { lead = control.NewLeadership(st, cfg.Control.LeaseTTL, log) ctl.SetLeadership(lead) + // The tenancy tree's capacity quotas: what each part is + // guaranteed and may hold. + ctl.Reconciler().SetQuotas(control.QuotaPolicy(st, log)) log.Info("competing to be the control plane that places work", "owner", lead.Owner()) } diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 4ad938f..53a5d6d 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -42,6 +42,7 @@ type API struct { models Models capacity Capacity usage Usage + quotas Quotas ctl *control.Controller } @@ -58,6 +59,7 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("DELETE /api/v1/models/{name}", a.removeModel) mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel) mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources) + a.registerQuotas(mux) mux.HandleFunc("POST /api/v1/models/{name}/class", func(w http.ResponseWriter, r *http.Request) { if a.models == nil { writeError(w, http.StatusNotImplemented, "this control plane has no store") @@ -304,6 +306,7 @@ type ModelView struct { // Class is its capacity class; Preempted the replicas it lost to a higher // class; HeldFor the model it is giving room to now, if any. Class string `json:"class"` + Owner string `json:"owner,omitempty"` Preempted int `json:"preempted,omitempty"` HeldFor string `json:"held_for,omitempty"` } @@ -552,7 +555,7 @@ func (a *API) Models() []ModelView { Fallbacks: m.Fallbacks, Desired: h.Desired, Failures: h.Failures, Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable, - Class: m.EffectiveClass(), Preempted: h.Preempted, HeldFor: h.HeldFor, + Class: m.EffectiveClass(), Owner: m.Owner, Preempted: h.Preempted, HeldFor: h.HeldFor, } for _, p := range m.Profiles { v.Profiles = append(v.Profiles, p.Name) @@ -634,6 +637,7 @@ type Models interface { Scale(ctx context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error) Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error) SetClass(ctx context.Context, name, class, by string) (catalog.Model, error) + SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error) } // ResourcesRequest sets what a model's profiles reserve of a node. diff --git a/internal/control/adminapi/api_test.go b/internal/control/adminapi/api_test.go index 949be67..3d465d8 100644 --- a/internal/control/adminapi/api_test.go +++ b/internal/control/adminapi/api_test.go @@ -258,6 +258,10 @@ type fakeModels struct { err error } +func (f *fakeModels) SetOwner(_ context.Context, name, owner, _ string) (catalog.Model, error) { + return catalog.Model{Name: name, Owner: owner}, f.err +} + func (f *fakeModels) SetClass(_ context.Context, name, class, _ string) (catalog.Model, error) { return catalog.Model{Name: name, Class: class}, f.err } diff --git a/internal/control/adminapi/quotas.go b/internal/control/adminapi/quotas.go new file mode 100644 index 0000000..af5bf44 --- /dev/null +++ b/internal/control/adminapi/quotas.go @@ -0,0 +1,237 @@ +package adminapi + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/quota" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// Quotas is where capacity quotas and the tenancy tree they name are kept. +// The store implements it. +type Quotas interface { + CapacityQuotas(ctx context.Context) ([]store.CapacityQuota, error) + PutCapacityQuota(ctx context.Context, q store.CapacityQuota) error + DeleteCapacityQuota(ctx context.Context, scope store.Scope, id string) error + ScopeParents(ctx context.Context) (map[string][]string, error) + AllOrgs(ctx context.Context) ([]store.Org, error) + AllTeams(ctx context.Context) ([]store.TeamPath, error) + AllProjects(ctx context.Context) ([]store.ProjectPath, error) +} + +// SetQuotas supplies the quota store, which mounts the quota endpoints' +// behaviour; without it they answer 501. +func (a *API) SetQuotas(q Quotas) { a.quotas = q } + +// QuotaView is one scope's quota with what it holds now. +type QuotaView struct { + // Scope is team:ID; Name is how people know it, "Acme / ml". + Scope string `json:"scope"` + Name string `json:"name"` + Guarantee store.Resources `json:"guarantee"` + Limit store.Resources `json:"limit"` + Holds quota.Amount `json:"holds"` + UpdatedBy string `json:"updated_by,omitempty"` + UpdatedAt time.Time `json:"updated_at"` +} + +// QuotaRequest sets a scope's quota. Scope may be written with an ID or a +// name: org:Acme, team:Acme/ml, project:Acme/ml/training. +type QuotaRequest struct { + Scope string `json:"scope"` + Guarantee store.Resources `json:"guarantee"` + Limit store.Resources `json:"limit"` +} + +func (a *API) registerQuotas(mux *http.ServeMux) { + mux.HandleFunc("GET /api/v1/quotas", func(w http.ResponseWriter, r *http.Request) { + if a.quotas == nil { + writeError(w, http.StatusNotImplemented, "this control plane has no store") + return + } + views, err := a.QuotaViews(r.Context()) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, http.StatusOK, views) + }) + mux.HandleFunc("PUT /api/v1/quotas", func(w http.ResponseWriter, r *http.Request) { + if a.quotas == nil { + writeError(w, http.StatusNotImplemented, "this control plane has no store") + return + } + var req QuotaRequest + if err := json.NewDecoder(io.LimitReader(r.Body, 1<<14)).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error()) + return + } + scope, id, _, err := a.resolveScope(r.Context(), req.Scope) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + if !req.Guarantee.Any() && !req.Limit.Any() { + writeError(w, http.StatusBadRequest, "a quota needs a guarantee or a limit; to remove one, DELETE it") + return + } + q := store.CapacityQuota{Scope: scope, ScopeID: id, Guarantee: req.Guarantee, Limit: req.Limit, + UpdatedBy: actorOf(r)} + if err := a.quotas.PutCapacityQuota(r.Context(), q); err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusOK, map[string]string{"scope": quota.Key(scope, id)}) + }) + mux.HandleFunc("DELETE /api/v1/quotas/{scope}", func(w http.ResponseWriter, r *http.Request) { + if a.quotas == nil { + writeError(w, http.StatusNotImplemented, "this control plane has no store") + return + } + scope, id, _, err := a.resolveScope(r.Context(), r.PathValue("scope")) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + if err := a.quotas.DeleteCapacityQuota(r.Context(), scope, id); err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, http.StatusOK, map[string]string{"removed": quota.Key(scope, id)}) + }) + mux.HandleFunc("POST /api/v1/models/{name}/owner", func(w http.ResponseWriter, r *http.Request) { + if a.models == nil || a.quotas == nil { + writeError(w, http.StatusNotImplemented, "this control plane has no store") + return + } + var req struct { + Owner string `json:"owner"` + } + if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error()) + return + } + owner := "" + if req.Owner != "" { + scope, id, _, err := a.resolveScope(r.Context(), req.Owner) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + owner = quota.Key(scope, id) + } + m, err := a.models.SetOwner(r.Context(), r.PathValue("name"), owner, actorOf(r)) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusOK, map[string]string{"model": m.Name, "owner": m.Owner}) + }) +} + +// QuotaViews lists every quota with what its scope holds now. +func (a *API) QuotaViews(ctx context.Context) ([]QuotaView, error) { + qs, err := a.quotas.CapacityQuotas(ctx) + if err != nil { + return nil, err + } + parents, err := a.quotas.ScopeParents(ctx) + if err != nil { + return nil, err + } + names, err := a.scopeNames(ctx) + if err != nil { + return nil, err + } + usage := quota.New(qs, parents).Usage(a.ctl.Reconciler().Holdings()) + out := make([]QuotaView, 0, len(qs)) + for _, q := range qs { + key := quota.Key(q.Scope, q.ScopeID) + out = append(out, QuotaView{Scope: key, Name: names[key], Guarantee: q.Guarantee, Limit: q.Limit, + Holds: usage[key], UpdatedBy: q.UpdatedBy, UpdatedAt: q.UpdatedAt}) + } + return out, nil +} + +// scopeNames maps org:ID, team:ID and project:ID to their paths. +func (a *API) scopeNames(ctx context.Context) (map[string]string, error) { + out := map[string]string{} + orgs, err := a.quotas.AllOrgs(ctx) + if err != nil { + return nil, err + } + for _, o := range orgs { + out["org:"+o.ID] = o.Name + } + teams, err := a.quotas.AllTeams(ctx) + if err != nil { + return nil, err + } + for _, t := range teams { + out["team:"+t.ID] = t.OrgName + " / " + t.Name + } + projects, err := a.quotas.AllProjects(ctx) + if err != nil { + return nil, err + } + for _, p := range projects { + out["project:"+p.ID] = p.Path + } + return out, nil +} + +// resolveScope reads org:X, team:X or project:X, where X is an ID or the +// scope's path — Acme, Acme/ml, Acme/ml/training — and returns its ID and +// path. A path that could mean two scopes is refused rather than guessed. +func (a *API) resolveScope(ctx context.Context, s string) (store.Scope, string, string, error) { + kind, what, ok := strings.Cut(strings.TrimSpace(s), ":") + scope := store.Scope(kind) + if !ok || what == "" || (scope != store.ScopeOrg && scope != store.ScopeTeam && scope != store.ScopeProject) { + return "", "", "", fmt.Errorf("%q is not org:NAME, team:ORG/TEAM or project:ORG/TEAM/PROJECT", s) + } + names, err := a.scopeNames(ctx) + if err != nil { + return "", "", "", err + } + norm := func(p string) string { + parts := strings.Split(p, "/") + for i := range parts { + parts[i] = strings.ToLower(strings.TrimSpace(parts[i])) + } + return strings.Join(parts, "/") + } + if name, ok := names[kind+":"+what]; ok { + return scope, what, name, nil + } + match := func(same func(name string) bool) []string { + var found []string + for key, name := range names { + if strings.HasPrefix(key, kind+":") && same(name) { + found = append(found, key) + } + } + return found + } + found := match(func(name string) bool { return norm(name) == norm(what) }) + if len(found) == 0 { + // Its own name alone — ml rather than Acme/ml — when that is unique. + found = match(func(name string) bool { + parts := strings.Split(norm(name), "/") + return parts[len(parts)-1] == norm(what) + }) + } + switch len(found) { + case 0: + return "", "", "", fmt.Errorf("there is no %s %q", kind, what) + case 1: + _, id, _ := strings.Cut(found[0], ":") + return scope, id, names[found[0]], nil + } + return "", "", "", fmt.Errorf("%q could be any of %d %ss; give its ID", what, len(found), kind) +} diff --git a/internal/control/adminapi/quotas_test.go b/internal/control/adminapi/quotas_test.go new file mode 100644 index 0000000..640f5d7 --- /dev/null +++ b/internal/control/adminapi/quotas_test.go @@ -0,0 +1,107 @@ +package adminapi + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// fakeQuotaStore is Acme with teams ml and web, and Scania with another ml. +type fakeQuotaStore struct { + quotas []store.CapacityQuota +} + +func (f *fakeQuotaStore) CapacityQuotas(context.Context) ([]store.CapacityQuota, error) { + return f.quotas, nil +} +func (f *fakeQuotaStore) PutCapacityQuota(_ context.Context, q store.CapacityQuota) error { + for i := range f.quotas { + if f.quotas[i].Scope == q.Scope && f.quotas[i].ScopeID == q.ScopeID { + f.quotas[i] = q + return nil + } + } + f.quotas = append(f.quotas, q) + return nil +} +func (f *fakeQuotaStore) DeleteCapacityQuota(_ context.Context, s store.Scope, id string) error { + for i, q := range f.quotas { + if q.Scope == s && q.ScopeID == id { + f.quotas = append(f.quotas[:i], f.quotas[i+1:]...) + } + } + return nil +} +func (f *fakeQuotaStore) ScopeParents(context.Context) (map[string][]string, error) { + return map[string][]string{"team:t-ml": {"org:o-acme"}, "team:t-web": {"org:o-acme"}, + "team:t-sml": {"org:o-scania"}}, nil +} +func (f *fakeQuotaStore) AllOrgs(context.Context) ([]store.Org, error) { + return []store.Org{{ID: "o-acme", Name: "Acme"}, {ID: "o-scania", Name: "Scania"}}, nil +} +func (f *fakeQuotaStore) AllTeams(context.Context) ([]store.TeamPath, error) { + return []store.TeamPath{{ID: "t-ml", Name: "ml", OrgID: "o-acme", OrgName: "Acme"}, + {ID: "t-web", Name: "web", OrgID: "o-acme", OrgName: "Acme"}, + {ID: "t-sml", Name: "ml", OrgID: "o-scania", OrgName: "Scania"}}, nil +} +func (f *fakeQuotaStore) AllProjects(context.Context) ([]store.ProjectPath, error) { return nil, nil } + +func TestAScopeIsNamedByPathOrIDAndNeverGuessed(t *testing.T) { + a, _ := setup(t, testModels()) + a.SetQuotas(&fakeQuotaStore{}) + for in, want := range map[string]string{ + "org:Acme": "o-acme", "org:o-acme": "o-acme", "team:Acme/ml": "t-ml", + "team: acme / ML ": "t-ml", "team:Scania/ml": "t-sml", "team:web": "t-web", + } { + _, id, _, err := a.resolveScope(t.Context(), in) + if err != nil || id != want { + t.Errorf("%q: %q %v, expected %q", in, id, err, want) + } + } + for _, bad := range []string{"team:ml", "team:Acme/nope", "group:Acme", "Acme", "org:"} { + if _, _, _, err := a.resolveScope(t.Context(), bad); err == nil { + t.Errorf("%q was resolved", bad) + } + } + if _, _, _, err := a.resolveScope(t.Context(), "team:ml"); err == nil || !strings.Contains(err.Error(), "could be any of 2") { + t.Errorf("an ambiguous name should say so: %v", err) + } +} + +func TestAQuotaIsSetListedAndCleared(t *testing.T) { + a, _ := setup(t, testModels()) + qs := &fakeQuotaStore{} + a.SetQuotas(qs) + mux := http.NewServeMux() + a.Register(mux) + call := func(method, path, body string) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, path, strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Llmgw-Actor", "ops") + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + return rec + } + if rec := call("PUT", "/api/v1/quotas", `{"scope":"team:Acme/ml","guarantee":{"gpus":8},"limit":{"gpus":16}}`); rec.Code != 200 { + t.Fatalf("set: %d %s", rec.Code, rec.Body) + } + if len(qs.quotas) != 1 || qs.quotas[0].ScopeID != "t-ml" || *qs.quotas[0].Guarantee.GPUs != 8 || qs.quotas[0].UpdatedBy != "ops" { + t.Errorf("stored %+v", qs.quotas) + } + rec := call("GET", "/api/v1/quotas", "") + var views []QuotaView + if err := json.Unmarshal(rec.Body.Bytes(), &views); err != nil || len(views) != 1 || views[0].Name != "Acme / ml" { + t.Errorf("list %d %s", rec.Code, rec.Body) + } + if rec := call("PUT", "/api/v1/quotas", `{"scope":"team:Acme/ml"}`); rec.Code != 400 { + t.Errorf("an empty quota was accepted: %d", rec.Code) + } + if rec := call("DELETE", "/api/v1/quotas/team:Acme%2Fml", ""); rec.Code != 200 || len(qs.quotas) != 0 { + t.Errorf("clear: %d %s %+v", rec.Code, rec.Body, qs.quotas) + } +} diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index a6de869..6c34948 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -156,7 +156,11 @@ type Model struct { // when there is not enough for both: critical, standard or opportunistic. // Empty is standard, or opportunistic for a model loaded on demand. See // EffectiveClass. - Class string `json:"class,omitempty" koanf:"class"` + Class string `json:"class,omitempty" koanf:"class"` + // Owner is the part of the tenancy tree whose capacity quota this model + // counts against, written org:ID, team:ID or project:ID. Empty is shared: + // held by the cluster, counted against nobody. + Owner string `json:"owner,omitempty" koanf:"owner"` Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"` // RoutingProfile selects how the router scores endpoints for this model. RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"` @@ -304,6 +308,9 @@ func (m Model) validate() error { if len(m.Profiles) == 0 { return fmt.Errorf("model %q: at least one profile is required", m.Name) } + if m.Owner != "" && !validOwner(m.Owner) { + return fmt.Errorf("model %q: owner %q is not org:ID, team:ID or project:ID", m.Name, m.Owner) + } if !validClass(m.Class) { return fmt.Errorf("model %q: class %q is not one of critical, standard, opportunistic", m.Name, m.Class) } @@ -548,3 +555,12 @@ func ClassRank(c string) int { return 1 } } + +func validOwner(o string) bool { + for _, p := range []string{"org:", "team:", "project:"} { + if strings.HasPrefix(o, p) && len(o) > len(p) { + return true + } + } + return false +} diff --git a/internal/control/control.go b/internal/control/control.go index 2ebf98a..06066c1 100644 --- a/internal/control/control.go +++ b/internal/control/control.go @@ -375,3 +375,6 @@ func EngineKeys(secret []byte) func(replicaID string) string { return "llmgw-" + hex.EncodeToString(m.Sum(nil))[:40] } } + +// QuotaUse is every capacity quota beside what its scope holds. +func (c *Controller) QuotaUse() []reconciler.QuotaUse { return c.rec.QuotaUse() } diff --git a/internal/control/fleetmetrics/fleetmetrics.go b/internal/control/fleetmetrics/fleetmetrics.go index 602f9c9..37bd848 100644 --- a/internal/control/fleetmetrics/fleetmetrics.go +++ b/internal/control/fleetmetrics/fleetmetrics.go @@ -51,13 +51,16 @@ type HealthSource interface { type Collector struct { src Source - modelDesired *prometheus.Desc - modelFailures *prometheus.Desc - modelParked *prometheus.Desc - modelUnplaceable *prometheus.Desc - modelPreempted *prometheus.Desc - engineInstall *prometheus.Desc - leader *prometheus.Desc + modelDesired *prometheus.Desc + modelFailures *prometheus.Desc + modelParked *prometheus.Desc + modelUnplaceable *prometheus.Desc + modelPreempted *prometheus.Desc + capacityHolds *prometheus.Desc + capacityGuarantee *prometheus.Desc + capacityLimit *prometheus.Desc + engineInstall *prometheus.Desc + leader *prometheus.Desc nodeUp *prometheus.Desc nodeGPUs *prometheus.Desc @@ -169,6 +172,15 @@ func New(src Source) *Collector { modelUnplaceable: prometheus.NewDesc("llmgw_model_unplaceable", "1 when a model is short of replicas and the last attempt found no node to place one on.", []string{"model"}, nil), + capacityHolds: prometheus.NewDesc("llmgw_capacity_holds", + "What a part of the tenancy tree holds of a resource, its own models' replicas and those beneath it.", + []string{"scope", "resource"}, nil), + capacityGuarantee: prometheus.NewDesc("llmgw_capacity_guarantee", + "What a part of the tenancy tree is guaranteed of a resource.", + []string{"scope", "resource"}, nil), + capacityLimit: prometheus.NewDesc("llmgw_capacity_limit", + "The most of a resource a part of the tenancy tree may hold.", + []string{"scope", "resource"}, nil), modelPreempted: prometheus.NewDesc("llmgw_model_preempted_total", "Replicas of a model stopped to make room for a model of a higher capacity class.", []string{"model", "class"}, nil), @@ -241,6 +253,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) { c.gpuNVLink, c.gpuECC, c.replicaState, c.replicaReady, c.replicaIdle, c.modelDesired, c.modelFailures, c.modelParked, c.modelUnplaceable, c.modelPreempted, + c.capacityHolds, c.capacityGuarantee, c.capacityLimit, c.engineInstall, c.leader, } { ch <- d @@ -250,6 +263,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) { // Collect implements prometheus.Collector. func (c *Collector) Collect(ch chan<- prometheus.Metric) { now := time.Now() + c.collectQuotas(ch) loads := map[string]*inventory.Load{} for _, n := range c.src.Nodes() { @@ -469,3 +483,38 @@ func b2f(b bool) float64 { } return 0 } + +// QuotaSource adds the tenancy tree's capacity quotas. Asked for by type +// assertion, like HealthSource. +type QuotaSource interface { + QuotaUse() []reconciler.QuotaUse +} + +func (c *Collector) collectQuotas(ch chan<- prometheus.Metric) { + qs, ok := c.src.(QuotaSource) + if !ok { + return + } + for _, q := range qs.QuotaUse() { + for _, r := range []struct { + name string + holds int + gua, lim *int + }{ + {"gpus", q.Holds.GPUs, q.Guarantee.GPUs, q.Limit.GPUs}, + {"cpu_cores", q.Holds.CPUCores, q.Guarantee.CPUCores, q.Limit.CPUCores}, + {"memory_gb", q.Holds.MemoryGB, q.Guarantee.MemoryGB, q.Limit.MemoryGB}, + } { + if r.gua == nil && r.lim == nil { + continue + } + ch <- prometheus.MustNewConstMetric(c.capacityHolds, prometheus.GaugeValue, float64(r.holds), q.Scope, r.name) + if r.gua != nil { + ch <- prometheus.MustNewConstMetric(c.capacityGuarantee, prometheus.GaugeValue, float64(*r.gua), q.Scope, r.name) + } + if r.lim != nil { + ch <- prometheus.MustNewConstMetric(c.capacityLimit, prometheus.GaugeValue, float64(*r.lim), q.Scope, r.name) + } + } + } +} diff --git a/internal/control/modelsvc/modelsvc.go b/internal/control/modelsvc/modelsvc.go index 2778610..6ffb3e5 100644 --- a/internal/control/modelsvc/modelsvc.go +++ b/internal/control/modelsvc/modelsvc.go @@ -255,6 +255,19 @@ func (s *Service) SetClass(ctx context.Context, name, class, by string) (catalog return final, err } +// SetOwner sets the part of the tenancy tree a model's capacity counts +// against, org:ID, team:ID or project:ID; empty makes it shared. +func (s *Service) SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error) { + final, err := s.update(ctx, name, by, "change its owner", func(m *catalog.Model) error { + m.Owner = owner + return nil + }) + if err == nil { + s.log.Info("model owner set", "model", name, "owner", owner, "by", by) + } + return final, err +} + // update changes one model added at runtime and keeps the change. // // Built before anything is written, like Add: catalog.New is what applies the diff --git a/internal/control/quota/quota.go b/internal/control/quota/quota.go new file mode 100644 index 0000000..ef6f62b --- /dev/null +++ b/internal/control/quota/quota.go @@ -0,0 +1,192 @@ +// Package quota decides what the tenancy tree is guaranteed of the cluster +// and how much of it each part may hold. +// +// A scope — an organization, team or project — may be guaranteed GPUs, CPU +// cores and memory, and limited to at most some amount of each. What a scope +// holds is what the replicas of the models it owns, and of the models its +// teams and projects own, have reserved. Holding more than its guarantee is +// borrowing: allowed while the room is free, and taken back when another +// scope within its own guarantee needs it. +package quota + +import ( + "fmt" + "sort" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// Amount is how much of each resource something holds or needs. +type Amount struct { + GPUs, CPUCores, MemoryGB int +} + +// Plus adds two amounts. +func (a Amount) Plus(b Amount) Amount { + return Amount{a.GPUs + b.GPUs, a.CPUCores + b.CPUCores, a.MemoryGB + b.MemoryGB} +} + +// Over names the first resource in which a is above the governed amounts of +// l, with both figures; empty when it is above none. +func Over(a Amount, l store.Resources) string { + switch { + case l.GPUs != nil && a.GPUs > *l.GPUs: + return fmt.Sprintf("%d GPUs of %d", a.GPUs, *l.GPUs) + case l.CPUCores != nil && a.CPUCores > *l.CPUCores: + return fmt.Sprintf("%d cores of %d", a.CPUCores, *l.CPUCores) + case l.MemoryGB != nil && a.MemoryGB > *l.MemoryGB: + return fmt.Sprintf("%d GB of memory of %d", a.MemoryGB, *l.MemoryGB) + } + return "" +} + +// Holding is one replica's hold on the cluster, for working out who holds +// what. +type Holding struct { + Replica string + Owner string + Amount Amount + Since time.Time +} + +// Policy is the quotas and the tree they sit in, read at one moment. +type Policy struct { + quotas map[string]store.CapacityQuota + parents map[string][]string +} + +// New builds a policy from stored quotas and each scope's parents. +func New(quotas []store.CapacityQuota, parents map[string][]string) *Policy { + p := &Policy{quotas: map[string]store.CapacityQuota{}, parents: parents} + for _, q := range quotas { + p.quotas[Key(q.Scope, q.ScopeID)] = q + } + return p +} + +// Key writes a scope the way owners are written: team:ID. +func Key(s store.Scope, id string) string { return string(s) + ":" + id } + +// Chain is an owner and every scope above it, nearest first. +func (p *Policy) Chain(owner string) []string { + if owner == "" { + return nil + } + return append([]string{owner}, p.parents[owner]...) +} + +// Governing is the quotas along an owner's chain, nearest first. +func (p *Policy) Governing(owner string) []store.CapacityQuota { + if p == nil { + return nil + } + var out []store.CapacityQuota + for _, s := range p.Chain(owner) { + if q, ok := p.quotas[s]; ok { + out = append(out, q) + } + } + return out +} + +// Quotas lists every quota. +func (p *Policy) Quotas() []store.CapacityQuota { + if p == nil { + return nil + } + out := make([]store.CapacityQuota, 0, len(p.quotas)) + for _, q := range p.quotas { + out = append(out, q) + } + sort.Slice(out, func(i, j int) bool { + return Key(out[i].Scope, out[i].ScopeID) < Key(out[j].Scope, out[j].ScopeID) + }) + return out +} + +// Usage is what every scope holds, its own models' replicas and those of +// everything beneath it. +func (p *Policy) Usage(hs []Holding) map[string]Amount { + out := map[string]Amount{} + if p == nil { + return out + } + for _, h := range hs { + for _, s := range p.Chain(h.Owner) { + out[s] = out[s].Plus(h.Amount) + } + } + return out +} + +// Borrowed is the replicas held beyond a guarantee: of each governed scope's +// replicas, oldest first, the ones that take it past what it is guaranteed. +// A replica beyond any guarantee along its chain is borrowed. +func (p *Policy) Borrowed(hs []Holding) map[string]bool { + out := map[string]bool{} + if p == nil || len(p.quotas) == 0 { + return out + } + sorted := append([]Holding(nil), hs...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Since.Before(sorted[j].Since) }) + for key, q := range p.quotas { + if !q.Guarantee.Any() { + continue + } + var sum Amount + for _, h := range sorted { + if !contains(p.Chain(h.Owner), key) { + continue + } + sum = sum.Plus(h.Amount) + if Over(sum, q.Guarantee) != "" { + out[h.Replica] = true + } + } + } + return out +} + +// Guaranteed reports whether an owner, holding what it holds, is still within +// every guarantee along its chain with need added — so it may take back what +// others borrowed. An owner with no guarantee anywhere is never guaranteed. +func (p *Policy) Guaranteed(owner string, usage map[string]Amount, need Amount) bool { + gov := p.Governing(owner) + any := false + for _, q := range gov { + if !q.Guarantee.Any() { + continue + } + any = true + if Over(usage[Key(q.Scope, q.ScopeID)].Plus(need), q.Guarantee) != "" { + return false + } + } + return any +} + +// AtLimit says which limit along an owner's chain need would take it past, +// empty when none. +func (p *Policy) AtLimit(owner string, usage map[string]Amount, need Amount) string { + for _, q := range p.Governing(owner) { + if !q.Limit.Any() { + continue + } + key := Key(q.Scope, q.ScopeID) + if over := Over(usage[key].Plus(need), q.Limit); over != "" { + return fmt.Sprintf("%s is at its capacity limit (%s)", key, strings.TrimSpace(over)) + } + } + return "" +} + +func contains(list []string, v string) bool { + for _, x := range list { + if x == v { + return true + } + } + return false +} diff --git a/internal/control/quota/quota_test.go b/internal/control/quota/quota_test.go new file mode 100644 index 0000000..03218c3 --- /dev/null +++ b/internal/control/quota/quota_test.go @@ -0,0 +1,83 @@ +package quota + +import ( + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +func n(v int) *int { return &v } + +// acme has two teams; ml is guaranteed 8 GPUs, the org as a whole 12 and +// no more than 16. +func policy() *Policy { + return New([]store.CapacityQuota{ + {Scope: store.ScopeTeam, ScopeID: "ml", Guarantee: store.Resources{GPUs: n(8)}}, + {Scope: store.ScopeOrg, ScopeID: "acme", Guarantee: store.Resources{GPUs: n(12)}, Limit: store.Resources{GPUs: n(16)}}, + }, map[string][]string{ + "team:ml": {"org:acme"}, "team:web": {"org:acme"}, + "project:train": {"team:ml", "org:acme"}, + }) +} + +func at(min int) time.Time { return time.Date(2026, 1, 1, 0, min, 0, 0, time.UTC) } + +func TestAScopeHoldsWhatEverythingBeneathItHolds(t *testing.T) { + p := policy() + u := p.Usage([]Holding{ + {Replica: "a", Owner: "project:train", Amount: Amount{GPUs: 4}}, + {Replica: "b", Owner: "team:web", Amount: Amount{GPUs: 2, CPUCores: 3}}, + {Replica: "c", Owner: "", Amount: Amount{GPUs: 8}}, // shared: nobody's + }) + if u["team:ml"].GPUs != 4 || u["org:acme"].GPUs != 6 || u["org:acme"].CPUCores != 3 || u["project:train"].GPUs != 4 { + t.Errorf("usage %+v", u) + } + if _, ok := u[""]; ok { + t.Error("a shared model was counted against somebody") + } +} + +func TestWhatGoesPastAGuaranteeIsBorrowedNewestFirst(t *testing.T) { + p := policy() + b := p.Borrowed([]Holding{ + {Replica: "old", Owner: "team:ml", Amount: Amount{GPUs: 8}, Since: at(1)}, + {Replica: "new", Owner: "team:ml", Amount: Amount{GPUs: 4}, Since: at(2)}, + {Replica: "web", Owner: "team:web", Amount: Amount{GPUs: 4}, Since: at(3)}, + }) + // ml: 8 within its 8, then 4 past it. acme: 8 + 4 = 12 within, web's 4 + // takes the org to 16, past its 12. + if b["old"] || !b["new"] || !b["web"] { + t.Errorf("borrowed %v", b) + } +} + +func TestAGuaranteeIsOnlyForWhoHasOne(t *testing.T) { + p := policy() + u := map[string]Amount{"team:ml": {GPUs: 4}, "org:acme": {GPUs: 4}} + if !p.Guaranteed("project:train", u, Amount{GPUs: 4}) { + t.Error("8 of ml's 8 and 8 of acme's 12 is within both") + } + if p.Guaranteed("project:train", u, Amount{GPUs: 5}) { + t.Error("9 GPUs is past ml's 8") + } + if p.Guaranteed("", u, Amount{GPUs: 1}) || New(nil, nil).Guaranteed("team:ml", u, Amount{}) { + t.Error("an owner with no guarantee is not guaranteed anything") + } +} + +func TestALimitStopsGrowthWhereverItIsSet(t *testing.T) { + p := policy() + u := map[string]Amount{"org:acme": {GPUs: 14}} + if why := p.AtLimit("team:web", u, Amount{GPUs: 2}); why != "" { + t.Errorf("16 of 16 is at the limit, not past it: %s", why) + } + why := p.AtLimit("team:web", u, Amount{GPUs: 4}) + if !strings.Contains(why, "org:acme is at its capacity limit (18 GPUs of 16)") { + t.Errorf("the refusal should name the scope and the figures: %q", why) + } + if p.AtLimit("", u, Amount{GPUs: 99}) != "" { + t.Error("a shared model was limited") + } +} diff --git a/internal/control/quotas.go b/internal/control/quotas.go new file mode 100644 index 0000000..3ed0f44 --- /dev/null +++ b/internal/control/quotas.go @@ -0,0 +1,52 @@ +package control + +import ( + "context" + "log/slog" + "sync" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/quota" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// QuotaStore is where capacity quotas and the tenancy tree are kept. +type QuotaStore interface { + CapacityQuotas(ctx context.Context) ([]store.CapacityQuota, error) + ScopeParents(ctx context.Context) (map[string][]string, error) +} + +// quotaRefresh is how stale the reconciler's view of quotas may get. A quota +// changed by an operator takes effect within this; the reconciler asks many +// times a pass and the store is not asked each time. +const quotaRefresh = 30 * time.Second + +// QuotaPolicy returns what the reconciler asks for its quotas: the last +// policy read from the store, read again once it is older than quotaRefresh. +// A store that fails keeps the last good policy, so a database hiccup does +// not drop every guarantee and limit at once. +func QuotaPolicy(st QuotaStore, log *slog.Logger) func() *quota.Policy { + var mu sync.Mutex + var cur *quota.Policy + var at time.Time + return func() *quota.Policy { + mu.Lock() + defer mu.Unlock() + if cur != nil && time.Since(at) < quotaRefresh { + return cur + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + qs, err := st.CapacityQuotas(ctx) + if err == nil { + var parents map[string][]string + if parents, err = st.ScopeParents(ctx); err == nil { + cur, at = quota.New(qs, parents), time.Now() + return cur + } + } + log.Warn("could not read capacity quotas; keeping the last ones", "err", err) + at = time.Now() // not again on every call while the store is down + return cur + } +} diff --git a/internal/control/reconciler/headroom.go b/internal/control/reconciler/headroom.go index 6080b03..d078661 100644 --- a/internal/control/reconciler/headroom.go +++ b/internal/control/reconciler/headroom.go @@ -5,6 +5,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/control/placement" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/quota" ) // Headroom is how many more replicas of a model the cluster has room for now. @@ -52,6 +53,11 @@ func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) { inv := r.inv.Clone() sched := placement.New(inv) + pol := r.policy() + var usage map[string]quota.Amount + if pol != nil { + usage = pol.Usage(r.Holdings()) + } for h.More < limit { res := sched.Place(m.PlacementProfiles(), avoid) if res.Placement == nil && len(avoid) > 0 { @@ -66,6 +72,17 @@ func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) { h.Blocked = slices.Compact(h.Blocked) return h, true } + // A quota limit is as much an edge as a full node. + if pol != nil && m.Owner != "" { + need := placementNeed(m, res.Placement) + if why := pol.AtLimit(m.Owner, usage, need); why != "" { + h.Blocked = []string{why} + return h, true + } + for _, s := range pol.Chain(m.Owner) { + usage[s] = usage[s].Plus(need) + } + } mem, cores := 0, 0 if prof, ok := m.ProfileByName(res.Placement.Profile.Name); ok { mem, cores = prof.MinMemoryGB, cpuCores(prof) diff --git a/internal/control/reconciler/preempt.go b/internal/control/reconciler/preempt.go index 883855a..75cdec6 100644 --- a/internal/control/reconciler/preempt.go +++ b/internal/control/reconciler/preempt.go @@ -8,6 +8,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/placement" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/quota" ) // hold keeps a model that gave up room from taking it back. @@ -162,15 +163,29 @@ func (r *Reconciler) liveCount(model string) int { } // victims lists the replicas that may be stopped for m, in the order they -// should be: idle on-demand ones first, then what lower classes grew into -// under load, then the same within m's class, then lower classes' minimums; -// within each, the longest unused first. +// should be: idle on-demand ones first, then what other tenants borrowed past +// their guarantees when m's owner is within its own, then what lower classes +// grew into under load, then the same within m's class, then lower classes' +// minimums; within each, the longest unused first. +// +// A replica within its owner's guarantee is taken only for a model of the +// same owner: a guarantee that another tenant's higher class could take is +// not one. func (r *Reconciler) victims(m catalog.Model) []victim { cat := r.cat.Load() now := r.now() myRank := catalog.ClassRank(m.EffectiveClass()) belowMin := r.liveCount(m.Name) < m.Scaling.Min + pol := r.policy() + var borrowed map[string]bool + reclaim := false + if pol != nil { + hs := r.Holdings() + borrowed = pol.Borrowed(hs) + reclaim = pol.Guaranteed(m.Owner, pol.Usage(hs), profileNeed(m)) + } + r.mu.Lock() defer r.mu.Unlock() @@ -201,11 +216,23 @@ func (r *Reconciler) victims(m catalog.Model) []victim { case rank < myRank && other.Scaling.OnDemand() && idleFor(rep, now) >= r.cfg.EvictionGrace: tier = 0 case rank < myRank && aboveMin: - tier = 1 - case rank == myRank && aboveMin && belowMin: tier = 2 - case rank < myRank && class != catalog.ClassCritical: + case rank == myRank && aboveMin && belowMin: tier = 3 + case rank < myRank && class != catalog.ClassCritical: + tier = 4 + } + if pol != nil && other.Owner != m.Owner { + switch { + case borrowed[rep.ID] && reclaim && (class != catalog.ClassCritical || aboveMin): + // Past its owner's guarantee, wanted by an owner within + // its own: taken back whatever the classes. + if tier != 0 { + tier = 1 + } + case !borrowed[rep.ID] && guaranteed(pol, other.Owner): + continue // held within its owner's guarantee + } } if tier < 0 { continue @@ -247,3 +274,13 @@ func nodesOf(rep *Replica) map[string]struct{} { } return out } + +// guaranteed reports whether an owner has a guarantee anywhere up its tree. +func guaranteed(pol *quota.Policy, owner string) bool { + for _, q := range pol.Governing(owner) { + if q.Guarantee.Any() { + return true + } + } + return false +} diff --git a/internal/control/reconciler/quotas.go b/internal/control/reconciler/quotas.go new file mode 100644 index 0000000..32a3a06 --- /dev/null +++ b/internal/control/reconciler/quotas.go @@ -0,0 +1,105 @@ +package reconciler + +import ( + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/placement" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/quota" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// SetQuotas supplies the tenancy tree's capacity quotas, read afresh each +// time it is called. Without it nothing is guaranteed or limited, and +// preemption goes by class alone. +func (r *Reconciler) SetQuotas(f func() *quota.Policy) { + r.mu.Lock() + defer r.mu.Unlock() + r.quotas = f +} + +func (r *Reconciler) policy() *quota.Policy { + r.mu.Lock() + f := r.quotas + r.mu.Unlock() + if f == nil { + return nil + } + return f() +} + +// Holdings is what every replica holds, and for whom. +func (r *Reconciler) Holdings() []quota.Holding { + cat := r.cat.Load() + r.mu.Lock() + defer r.mu.Unlock() + out := make([]quota.Holding, 0, len(r.replicas)) + for _, rep := range r.replicas { + m, ok := cat.Resolve(rep.Model) + if !ok { + continue + } + prof, _ := m.ProfileByName(rep.Profile) + var a quota.Amount + for _, sh := range rep.Shards { + a = a.Plus(quota.Amount{GPUs: len(sh.GPUIndices), CPUCores: cpuCores(prof), MemoryGB: prof.MinMemoryGB}) + } + out = append(out, quota.Holding{Replica: rep.ID, Owner: m.Owner, Amount: a, Since: rep.CreatedAt}) + } + return out +} + +// placementNeed is what a placement would hold. +func placementNeed(m catalog.Model, pl *placement.Placement) quota.Amount { + prof, _ := m.ProfileByName(pl.Profile.Name) + var a quota.Amount + for _, n := range pl.Nodes { + a = a.Plus(quota.Amount{GPUs: len(n.GPUIndices), CPUCores: cpuCores(prof), MemoryGB: prof.MinMemoryGB}) + } + return a +} + +// profileNeed is what a replica of a model's first profile would hold, for +// deciding before there is a placement. +func profileNeed(m catalog.Model) quota.Amount { + if len(m.Profiles) == 0 { + return quota.Amount{} + } + p := m.Profiles[0] + nodes := max(p.Nodes, 1) + gpus := p.GPUsPerNode + if gpus == 0 && !p.CPUOnly() { + gpus = max(p.TP, 1) + } + return quota.Amount{GPUs: gpus * nodes, CPUCores: cpuCores(p) * nodes, MemoryGB: p.MinMemoryGB * nodes} +} + +// overLimit says why a placement would take its model's owner past a limit. +func (r *Reconciler) overLimit(m catalog.Model, pl *placement.Placement) string { + pol := r.policy() + if pol == nil || m.Owner == "" { + return "" + } + return pol.AtLimit(m.Owner, pol.Usage(r.Holdings()), placementNeed(m, pl)) +} + +// QuotaUse is one scope's quota beside what it holds. +type QuotaUse struct { + Scope string + Guarantee store.Resources + Limit store.Resources + Holds quota.Amount +} + +// QuotaUse lists every quota with what its scope holds now. +func (r *Reconciler) QuotaUse() []QuotaUse { + pol := r.policy() + if pol == nil { + return nil + } + usage := pol.Usage(r.Holdings()) + var out []QuotaUse + for _, q := range pol.Quotas() { + key := quota.Key(q.Scope, q.ScopeID) + out = append(out, QuotaUse{Scope: key, Guarantee: q.Guarantee, Limit: q.Limit, Holds: usage[key]}) + } + return out +} diff --git a/internal/control/reconciler/quotas_test.go b/internal/control/reconciler/quotas_test.go new file mode 100644 index 0000000..f6668b6 --- /dev/null +++ b/internal/control/reconciler/quotas_test.go @@ -0,0 +1,81 @@ +package reconciler + +import ( + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/quota" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +func owned(m catalog.Model, owner string) catalog.Model { m.Owner = owner; return m } + +func gpus(n int) store.Resources { return store.Resources{GPUs: &n} } + +// Two teams in one org; each 8-GPU node holds one tp8 replica. +func withQuotas(h *harness, qs ...store.CapacityQuota) { + pol := quota.New(qs, map[string][]string{"team:ml": {"org:acme"}, "team:web": {"org:acme"}}) + h.r.SetQuotas(func() *quota.Policy { return pol }) +} + +func TestATenantWithinItsGuaranteeTakesBackWhatAnotherBorrowed(t *testing.T) { + // web is guaranteed one node and borrowed the second; ml is guaranteed + // one node and wants it. Same class, and web's replica is at its minimum + // — by class alone nothing could be taken. + web := owned(gpuModel("web-chat", "", 2), "team:web") + ml := owned(gpuModel("ml-train", "", 1), "team:ml") + h := newFleetHarness(t, []catalog.Model{web, withMin(ml, 0)}, gpuNode("1", 8), gpuNode("2", 8)) + withQuotas(h, + store.CapacityQuota{Scope: store.ScopeTeam, ScopeID: "web", Guarantee: gpus(8)}, + store.CapacityQuota{Scope: store.ScopeTeam, ScopeID: "ml", Guarantee: gpus(8)}) + h.settle(t, 4) + if replicasOf(h, "web-chat") != 2 { + t.Fatalf("setup: web did not borrow the second node: %+v", h.r.Replicas()) + } + h.advance(h.r.cfg.MinLifetime + time.Minute) + cat, _ := catalog.New([]catalog.Model{web, ml}) + h.r.cat.Store(cat) + + h.settle(t, 5) + if replicasOf(h, "ml-train") != 1 || replicasOf(h, "web-chat") != 1 { + t.Errorf("the borrowed node was not taken back: web %d, ml %d", + replicasOf(h, "web-chat"), replicasOf(h, "ml-train")) + } +} + +func TestAReplicaWithinItsGuaranteeIsNotTakenByAHigherClassElsewhere(t *testing.T) { + web := owned(gpuModel("web-chat", "", 1), "team:web") + vital := owned(gpuModel("vital", catalog.ClassCritical, 1), "team:ml") + h := fillThenWant(t, vital, web, gpuNode("1", 8)) + withQuotas(h, store.CapacityQuota{Scope: store.ScopeTeam, ScopeID: "web", Guarantee: gpus(8)}) + h.settle(t, 3) + if replicasOf(h, "web-chat") != 1 || replicasOf(h, "vital") != 0 { + t.Errorf("a guaranteed replica was preempted by another tenant's class: %+v", h.r.Replicas()) + } + // Without the guarantee, critical takes it: the guarantee was the reason. + h2 := fillThenWant(t, vital, web, gpuNode("1", 8)) + h2.settle(t, 3) + if replicasOf(h2, "vital") != 1 { + t.Error("setup: without a guarantee the critical model should have taken the node") + } +} + +func TestALimitStopsAModelBeforeItStarts(t *testing.T) { + web := owned(gpuModel("web-chat", "", 2), "team:web") + h := newFleetHarness(t, []catalog.Model{web}, gpuNode("1", 8), gpuNode("2", 8)) + withQuotas(h, store.CapacityQuota{Scope: store.ScopeOrg, ScopeID: "acme", Limit: gpus(8)}) + h.settle(t, 4) + if replicasOf(h, "web-chat") != 1 { + t.Errorf("a second replica went past the org's 8-GPU limit: %+v", h.r.Replicas()) + } + hs := health(h, "web-chat") + if !hs.Unplaceable || !strings.Contains(hs.Placement, "org:acme is at its capacity limit (16 GPUs of 8)") { + t.Errorf("the refusal should name the limit: %+v", hs) + } + room, _ := h.r.Headroom("web-chat", 5) + if room.More != 0 || len(room.Blocked) != 1 || !strings.Contains(room.Blocked[0], "capacity limit") { + t.Errorf("headroom should stop at the limit, not the empty node: %+v", room) + } +} diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index 3cfb083..e87373d 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -15,6 +15,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/control/placement" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/quota" ) // ShardCommand is everything an agent needs to launch one shard. @@ -270,6 +271,8 @@ type Reconciler struct { preempting map[string][]string held map[string]hold preempted map[string]int + // quotas is the tenancy tree's capacity quotas as last read; nil is none. + quotas func() *quota.Policy wake chan struct{} stop chan struct{} @@ -811,6 +814,16 @@ func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []strin r.notePlacementFailure(m.Name, res.Explain()) return } + if why := r.overLimit(m, res.Placement); why != "" { + // Recorded as the placement's answer, so the model reads as not + // placeable, and says why, wherever that is shown. + r.mu.Lock() + r.lastExplain[m.Name] = placement.Result{Rejections: []placement.Rejection{ + {Profile: res.Placement.Profile.Name, Reason: why}}} + r.mu.Unlock() + r.notePlacementFailure(m.Name, why) + return + } if err := r.startReplica(ctx, m, res.Placement); err != nil { r.log.Error("starting replica failed", "model", m.Name, "err", err) } diff --git a/internal/store/copy.go b/internal/store/copy.go index c4dae48..11ef3e5 100644 --- a/internal/store/copy.go +++ b/internal/store/copy.go @@ -29,6 +29,9 @@ var copyOrder = []string{ "key_claims", "sessions", "limits", + // Beside limits, and like them keyed by scope with no foreign key: a + // quota on a scope that is later removed is inert, not an error. + "capacity_quotas", "usage", "models", // No foreign key to models, deliberately: a measurement outlives the diff --git a/internal/store/migrations/postgres/0013_capacity_quotas.sql b/internal/store/migrations/postgres/0013_capacity_quotas.sql new file mode 100644 index 0000000..c4d09b9 --- /dev/null +++ b/internal/store/migrations/postgres/0013_capacity_quotas.sql @@ -0,0 +1,19 @@ +-- Capacity a part of the tenancy tree is guaranteed, and the most it may hold. +-- +-- Beside limits rather than in it: limits are counted on the request path, per +-- key, in requests and tokens; these are counted by the reconciler, per model, +-- in the GPUs, cores and memory its replicas hold. A NULL is a resource this +-- row does not govern: guaranteed nothing, or limited by nothing. +CREATE TABLE capacity_quotas ( + scope_type TEXT NOT NULL, + scope_id TEXT NOT NULL, + gpus INTEGER, + cpu_cores INTEGER, + memory_gb INTEGER, + limit_gpus INTEGER, + limit_cpu_cores INTEGER, + limit_memory_gb INTEGER, + updated_at BIGINT NOT NULL, + updated_by TEXT NOT NULL, + PRIMARY KEY (scope_type, scope_id) +); diff --git a/internal/store/migrations/sqlite/0013_capacity_quotas.sql b/internal/store/migrations/sqlite/0013_capacity_quotas.sql new file mode 100644 index 0000000..c4d09b9 --- /dev/null +++ b/internal/store/migrations/sqlite/0013_capacity_quotas.sql @@ -0,0 +1,19 @@ +-- Capacity a part of the tenancy tree is guaranteed, and the most it may hold. +-- +-- Beside limits rather than in it: limits are counted on the request path, per +-- key, in requests and tokens; these are counted by the reconciler, per model, +-- in the GPUs, cores and memory its replicas hold. A NULL is a resource this +-- row does not govern: guaranteed nothing, or limited by nothing. +CREATE TABLE capacity_quotas ( + scope_type TEXT NOT NULL, + scope_id TEXT NOT NULL, + gpus INTEGER, + cpu_cores INTEGER, + memory_gb INTEGER, + limit_gpus INTEGER, + limit_cpu_cores INTEGER, + limit_memory_gb INTEGER, + updated_at BIGINT NOT NULL, + updated_by TEXT NOT NULL, + PRIMARY KEY (scope_type, scope_id) +); diff --git a/internal/store/quotas.go b/internal/store/quotas.go new file mode 100644 index 0000000..ad43554 --- /dev/null +++ b/internal/store/quotas.go @@ -0,0 +1,127 @@ +package store + +import ( + "context" + "database/sql" + "fmt" + "time" +) + +// Resources are amounts of the three things a replica holds of a cluster. +// Nil is not governed: nothing guaranteed, or nothing limited. +type Resources struct { + GPUs *int `json:"gpus,omitempty"` + CPUCores *int `json:"cpu_cores,omitempty"` + MemoryGB *int `json:"memory_gb,omitempty"` +} + +// Any reports whether any resource is governed. +func (r Resources) Any() bool { return r.GPUs != nil || r.CPUCores != nil || r.MemoryGB != nil } + +// CapacityQuota is what one part of the tenancy tree is guaranteed of the +// cluster, and the most it may hold. +type CapacityQuota struct { + Scope Scope `json:"scope"` + ScopeID string `json:"scope_id"` + // Guarantee is held for the scope: while it holds less, what others + // borrowed beyond their own guarantees is taken back for it. + Guarantee Resources `json:"guarantee"` + // Limit is the most it may hold, borrowing included. + Limit Resources `json:"limit"` + UpdatedAt time.Time `json:"updated_at"` + UpdatedBy string `json:"updated_by"` +} + +// PutCapacityQuota sets a scope's quota, replacing what was there. +func (s *DB) PutCapacityQuota(ctx context.Context, q CapacityQuota) error { + if !validMembershipScope(q.Scope) || q.ScopeID == "" { + return fmt.Errorf("a capacity quota belongs to an organization, team or project") + } + for _, v := range []*int{q.Guarantee.GPUs, q.Guarantee.CPUCores, q.Guarantee.MemoryGB, + q.Limit.GPUs, q.Limit.CPUCores, q.Limit.MemoryGB} { + if v != nil && *v < 0 { + return fmt.Errorf("a quota cannot be negative") + } + } + if q.UpdatedAt.IsZero() { + q.UpdatedAt = time.Now() + } + _, err := s.db.ExecContext(ctx, + `INSERT INTO capacity_quotas (scope_type, scope_id, gpus, cpu_cores, memory_gb, + limit_gpus, limit_cpu_cores, limit_memory_gb, updated_at, updated_by) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (scope_type, scope_id) DO UPDATE SET + gpus = excluded.gpus, cpu_cores = excluded.cpu_cores, memory_gb = excluded.memory_gb, + limit_gpus = excluded.limit_gpus, limit_cpu_cores = excluded.limit_cpu_cores, + limit_memory_gb = excluded.limit_memory_gb, + updated_at = excluded.updated_at, updated_by = excluded.updated_by`, + string(q.Scope), q.ScopeID, + nullInt(q.Guarantee.GPUs), nullInt(q.Guarantee.CPUCores), nullInt(q.Guarantee.MemoryGB), + nullInt(q.Limit.GPUs), nullInt(q.Limit.CPUCores), nullInt(q.Limit.MemoryGB), + q.UpdatedAt.UnixMilli(), q.UpdatedBy) + return err +} + +// CapacityQuotas lists every quota. +func (s *DB) CapacityQuotas(ctx context.Context) ([]CapacityQuota, error) { + return queryAll(ctx, s.db, + `SELECT scope_type, scope_id, gpus, cpu_cores, memory_gb, + limit_gpus, limit_cpu_cores, limit_memory_gb, updated_at, updated_by + FROM capacity_quotas ORDER BY scope_type, scope_id`, nil, + func(rows *sql.Rows) (CapacityQuota, error) { + var q CapacityQuota + var scope string + var g [6]sql.NullInt64 + var at int64 + err := rows.Scan(&scope, &q.ScopeID, &g[0], &g[1], &g[2], &g[3], &g[4], &g[5], &at, &q.UpdatedBy) + q.Scope = Scope(scope) + q.Guarantee = Resources{GPUs: intOf(g[0]), CPUCores: intOf(g[1]), MemoryGB: intOf(g[2])} + q.Limit = Resources{GPUs: intOf(g[3]), CPUCores: intOf(g[4]), MemoryGB: intOf(g[5])} + q.UpdatedAt = time.UnixMilli(at) + return q, err + }) +} + +// DeleteCapacityQuota removes a scope's quota. +func (s *DB) DeleteCapacityQuota(ctx context.Context, scope Scope, id string) error { + _, err := s.db.ExecContext(ctx, + `DELETE FROM capacity_quotas WHERE scope_type = ? AND scope_id = ?`, string(scope), id) + return err +} + +// ScopeParents maps every team and project to the scopes above it, nearest +// first: a project to its team and organization, a team to its organization. +// Keys and values are written scope:id. +func (s *DB) ScopeParents(ctx context.Context) (map[string][]string, error) { + out := map[string][]string{} + teams, err := s.AllTeams(ctx) + if err != nil { + return nil, err + } + orgOf := map[string]string{} + for _, t := range teams { + orgOf[t.ID] = t.OrgID + out["team:"+t.ID] = []string{"org:" + t.OrgID} + } + type pt struct{ project, team string } + rows, err := queryAll(ctx, s.db, `SELECT id, team_id FROM projects`, nil, + func(r *sql.Rows) (pt, error) { + var p pt + return p, r.Scan(&p.project, &p.team) + }) + if err != nil { + return nil, err + } + for _, p := range rows { + out["project:"+p.project] = []string{"team:" + p.team, "org:" + orgOf[p.team]} + } + return out, nil +} + +func intOf(n sql.NullInt64) *int { + if !n.Valid { + return nil + } + v := int(n.Int64) + return &v +} diff --git a/internal/store/quotas_test.go b/internal/store/quotas_test.go new file mode 100644 index 0000000..0411fe8 --- /dev/null +++ b/internal/store/quotas_test.go @@ -0,0 +1,63 @@ +package store + +import "testing" + +func ip(v int) *int { return &v } + +func TestAQuotaKeepsWhatItGovernsAndNothingElse(t *testing.T) { + s := tenancyStore(t) + org, _ := s.CreateOrg(t.Context(), "Acme") + q := CapacityQuota{Scope: ScopeOrg, ScopeID: org, UpdatedBy: "ops", + Guarantee: Resources{GPUs: ip(8)}, Limit: Resources{GPUs: ip(16), CPUCores: ip(0)}} + if err := s.PutCapacityQuota(t.Context(), q); err != nil { + t.Fatal(err) + } + all, err := s.CapacityQuotas(t.Context()) + if err != nil || len(all) != 1 { + t.Fatalf("%v %+v", err, all) + } + got := all[0] + if *got.Guarantee.GPUs != 8 || got.Guarantee.CPUCores != nil || got.Guarantee.MemoryGB != nil { + t.Errorf("guarantee %+v: only GPUs were set", got.Guarantee) + } + // Zero is a limit of nothing, not an absent one. + if *got.Limit.GPUs != 16 || got.Limit.CPUCores == nil || *got.Limit.CPUCores != 0 { + t.Errorf("limit %+v", got.Limit) + } + q.Guarantee = Resources{MemoryGB: ip(64)} + if err := s.PutCapacityQuota(t.Context(), q); err != nil { + t.Fatal(err) + } + all, _ = s.CapacityQuotas(t.Context()) + if len(all) != 1 || all[0].Guarantee.GPUs != nil || *all[0].Guarantee.MemoryGB != 64 { + t.Errorf("a second put should replace the first: %+v", all) + } + if err := s.DeleteCapacityQuota(t.Context(), ScopeOrg, org); err != nil { + t.Fatal(err) + } + if all, _ = s.CapacityQuotas(t.Context()); len(all) != 0 { + t.Error("the quota survived its deletion") + } + if err := s.PutCapacityQuota(t.Context(), CapacityQuota{Scope: ScopeOrg, ScopeID: org, + Guarantee: Resources{GPUs: ip(-1)}}); err == nil { + t.Error("a negative quota was stored") + } +} + +func TestEveryScopeKnowsWhatIsAboveIt(t *testing.T) { + s := tenancyStore(t) + org, _ := s.CreateOrg(t.Context(), "Acme") + team, _ := s.CreateTeam(t.Context(), org, "ml") + proj, _ := s.CreateProject(t.Context(), team, "training") + p, err := s.ScopeParents(t.Context()) + if err != nil { + t.Fatal(err) + } + want := []string{"team:" + team, "org:" + org} + if got := p["project:"+proj]; len(got) != 2 || got[0] != want[0] || got[1] != want[1] { + t.Errorf("project's parents %v, expected %v", got, want) + } + if got := p["team:"+team]; len(got) != 1 || got[0] != "org:"+org { + t.Errorf("team's parents %v", got) + } +}