A model has an owner (org, team or project). Quotas guarantee a scope GPUs, cores and memory and may limit it; what a scope holds is summed from its models' replicas and those beneath it. A replica within its owner's guarantee is not taken for another scope; what is held beyond a guarantee is borrowed and taken back for a scope within its own. A limit stops a model before it starts, and headroom stops there too. ctl quota and ctl model owner set them; metrics and a CapacityAtLimit alert report them. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
193 lines
5.3 KiB
Go
193 lines
5.3 KiB
Go
// Package quota decides what the tenancy tree is guaranteed of the cluster
|
|
// and how much of it each part may hold.
|
|
//
|
|
// A scope — an organization, team or project — may be guaranteed GPUs, CPU
|
|
// cores and memory, and limited to at most some amount of each. What a scope
|
|
// holds is what the replicas of the models it owns, and of the models its
|
|
// teams and projects own, have reserved. Holding more than its guarantee is
|
|
// borrowing: allowed while the room is free, and taken back when another
|
|
// scope within its own guarantee needs it.
|
|
package quota
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// Amount is how much of each resource something holds or needs.
|
|
type Amount struct {
|
|
GPUs, CPUCores, MemoryGB int
|
|
}
|
|
|
|
// Plus adds two amounts.
|
|
func (a Amount) Plus(b Amount) Amount {
|
|
return Amount{a.GPUs + b.GPUs, a.CPUCores + b.CPUCores, a.MemoryGB + b.MemoryGB}
|
|
}
|
|
|
|
// Over names the first resource in which a is above the governed amounts of
|
|
// l, with both figures; empty when it is above none.
|
|
func Over(a Amount, l store.Resources) string {
|
|
switch {
|
|
case l.GPUs != nil && a.GPUs > *l.GPUs:
|
|
return fmt.Sprintf("%d GPUs of %d", a.GPUs, *l.GPUs)
|
|
case l.CPUCores != nil && a.CPUCores > *l.CPUCores:
|
|
return fmt.Sprintf("%d cores of %d", a.CPUCores, *l.CPUCores)
|
|
case l.MemoryGB != nil && a.MemoryGB > *l.MemoryGB:
|
|
return fmt.Sprintf("%d GB of memory of %d", a.MemoryGB, *l.MemoryGB)
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// Holding is one replica's hold on the cluster, for working out who holds
|
|
// what.
|
|
type Holding struct {
|
|
Replica string
|
|
Owner string
|
|
Amount Amount
|
|
Since time.Time
|
|
}
|
|
|
|
// Policy is the quotas and the tree they sit in, read at one moment.
|
|
type Policy struct {
|
|
quotas map[string]store.CapacityQuota
|
|
parents map[string][]string
|
|
}
|
|
|
|
// New builds a policy from stored quotas and each scope's parents.
|
|
func New(quotas []store.CapacityQuota, parents map[string][]string) *Policy {
|
|
p := &Policy{quotas: map[string]store.CapacityQuota{}, parents: parents}
|
|
for _, q := range quotas {
|
|
p.quotas[Key(q.Scope, q.ScopeID)] = q
|
|
}
|
|
return p
|
|
}
|
|
|
|
// Key writes a scope the way owners are written: team:ID.
|
|
func Key(s store.Scope, id string) string { return string(s) + ":" + id }
|
|
|
|
// Chain is an owner and every scope above it, nearest first.
|
|
func (p *Policy) Chain(owner string) []string {
|
|
if owner == "" {
|
|
return nil
|
|
}
|
|
return append([]string{owner}, p.parents[owner]...)
|
|
}
|
|
|
|
// Governing is the quotas along an owner's chain, nearest first.
|
|
func (p *Policy) Governing(owner string) []store.CapacityQuota {
|
|
if p == nil {
|
|
return nil
|
|
}
|
|
var out []store.CapacityQuota
|
|
for _, s := range p.Chain(owner) {
|
|
if q, ok := p.quotas[s]; ok {
|
|
out = append(out, q)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Quotas lists every quota.
|
|
func (p *Policy) Quotas() []store.CapacityQuota {
|
|
if p == nil {
|
|
return nil
|
|
}
|
|
out := make([]store.CapacityQuota, 0, len(p.quotas))
|
|
for _, q := range p.quotas {
|
|
out = append(out, q)
|
|
}
|
|
sort.Slice(out, func(i, j int) bool {
|
|
return Key(out[i].Scope, out[i].ScopeID) < Key(out[j].Scope, out[j].ScopeID)
|
|
})
|
|
return out
|
|
}
|
|
|
|
// Usage is what every scope holds, its own models' replicas and those of
|
|
// everything beneath it.
|
|
func (p *Policy) Usage(hs []Holding) map[string]Amount {
|
|
out := map[string]Amount{}
|
|
if p == nil {
|
|
return out
|
|
}
|
|
for _, h := range hs {
|
|
for _, s := range p.Chain(h.Owner) {
|
|
out[s] = out[s].Plus(h.Amount)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Borrowed is the replicas held beyond a guarantee: of each governed scope's
|
|
// replicas, oldest first, the ones that take it past what it is guaranteed.
|
|
// A replica beyond any guarantee along its chain is borrowed.
|
|
func (p *Policy) Borrowed(hs []Holding) map[string]bool {
|
|
out := map[string]bool{}
|
|
if p == nil || len(p.quotas) == 0 {
|
|
return out
|
|
}
|
|
sorted := append([]Holding(nil), hs...)
|
|
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Since.Before(sorted[j].Since) })
|
|
for key, q := range p.quotas {
|
|
if !q.Guarantee.Any() {
|
|
continue
|
|
}
|
|
var sum Amount
|
|
for _, h := range sorted {
|
|
if !contains(p.Chain(h.Owner), key) {
|
|
continue
|
|
}
|
|
sum = sum.Plus(h.Amount)
|
|
if Over(sum, q.Guarantee) != "" {
|
|
out[h.Replica] = true
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Guaranteed reports whether an owner, holding what it holds, is still within
|
|
// every guarantee along its chain with need added — so it may take back what
|
|
// others borrowed. An owner with no guarantee anywhere is never guaranteed.
|
|
func (p *Policy) Guaranteed(owner string, usage map[string]Amount, need Amount) bool {
|
|
gov := p.Governing(owner)
|
|
any := false
|
|
for _, q := range gov {
|
|
if !q.Guarantee.Any() {
|
|
continue
|
|
}
|
|
any = true
|
|
if Over(usage[Key(q.Scope, q.ScopeID)].Plus(need), q.Guarantee) != "" {
|
|
return false
|
|
}
|
|
}
|
|
return any
|
|
}
|
|
|
|
// AtLimit says which limit along an owner's chain need would take it past,
|
|
// empty when none.
|
|
func (p *Policy) AtLimit(owner string, usage map[string]Amount, need Amount) string {
|
|
for _, q := range p.Governing(owner) {
|
|
if !q.Limit.Any() {
|
|
continue
|
|
}
|
|
key := Key(q.Scope, q.ScopeID)
|
|
if over := Over(usage[key].Plus(need), q.Limit); over != "" {
|
|
return fmt.Sprintf("%s is at its capacity limit (%s)", key, strings.TrimSpace(over))
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func contains(list []string, v string) bool {
|
|
for _, x := range list {
|
|
if x == v {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|