Guarantee and limit capacity per part of the tenancy tree
A model has an owner (org, team or project). Quotas guarantee a scope GPUs, cores and memory and may limit it; what a scope holds is summed from its models' replicas and those beneath it. A replica within its owner's guarantee is not taken for another scope; what is held beyond a guarantee is borrowed and taken back for a scope within its own. A limit stops a model before it starts, and headroom stops there too. ctl quota and ctl model owner set them; metrics and a CapacityAtLimit alert report them. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
ac24e204a6
commit
9dc1b62fdc
28 files changed
+1475
-16
No files matched your search
@@ -42,6 +42,7 @@ type API struct {
|
||||
models Models
|
||||
capacity Capacity
|
||||
usage Usage
|
||||
quotas Quotas
|
||||
ctl *control.Controller
|
||||
}
|
||||
|
||||
@@ -58,6 +59,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("DELETE /api/v1/models/{name}", a.removeModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
|
||||
a.registerQuotas(mux)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/class", func(w http.ResponseWriter, r *http.Request) {
|
||||
if a.models == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||
@@ -304,6 +306,7 @@ type ModelView struct {
|
||||
// Class is its capacity class; Preempted the replicas it lost to a higher
|
||||
// class; HeldFor the model it is giving room to now, if any.
|
||||
Class string `json:"class"`
|
||||
Owner string `json:"owner,omitempty"`
|
||||
Preempted int `json:"preempted,omitempty"`
|
||||
HeldFor string `json:"held_for,omitempty"`
|
||||
}
|
||||
@@ -552,7 +555,7 @@ func (a *API) Models() []ModelView {
|
||||
Fallbacks: m.Fallbacks,
|
||||
Desired: h.Desired, Failures: h.Failures,
|
||||
Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable,
|
||||
Class: m.EffectiveClass(), Preempted: h.Preempted, HeldFor: h.HeldFor,
|
||||
Class: m.EffectiveClass(), Owner: m.Owner, Preempted: h.Preempted, HeldFor: h.HeldFor,
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
v.Profiles = append(v.Profiles, p.Name)
|
||||
@@ -634,6 +637,7 @@ type Models interface {
|
||||
Scale(ctx context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error)
|
||||
Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error)
|
||||
SetClass(ctx context.Context, name, class, by string) (catalog.Model, error)
|
||||
SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error)
|
||||
}
|
||||
|
||||
// ResourcesRequest sets what a model's profiles reserve of a node.
|
||||
|
||||
Reference in new issue
Block a user