Files
llmgw/internal/control/quota/quota.go
T
Ilker ManapandClaude Opus 5.5 9dc1b62fdc
Release / build (push) Successful in 1m8s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m51s
Guarantee and limit capacity per part of the tenancy tree
A model has an owner (org, team or project). Quotas guarantee a scope
GPUs, cores and memory and may limit it; what a scope holds is summed from
its models' replicas and those beneath it. A replica within its owner's
guarantee is not taken for another scope; what is held beyond a guarantee
is borrowed and taken back for a scope within its own. A limit stops a
model before it starts, and headroom stops there too. ctl quota and ctl
model owner set them; metrics and a CapacityAtLimit alert report them.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 18:15:38 +02:00

193 lines
5.3 KiB
Go

// Package quota decides what the tenancy tree is guaranteed of the cluster
// and how much of it each part may hold.
//
// A scope — an organization, team or project — may be guaranteed GPUs, CPU
// cores and memory, and limited to at most some amount of each. What a scope
// holds is what the replicas of the models it owns, and of the models its
// teams and projects own, have reserved. Holding more than its guarantee is
// borrowing: allowed while the room is free, and taken back when another
// scope within its own guarantee needs it.
package quota
import (
"fmt"
"sort"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// Amount is how much of each resource something holds or needs.
type Amount struct {
GPUs, CPUCores, MemoryGB int
}
// Plus adds two amounts.
func (a Amount) Plus(b Amount) Amount {
return Amount{a.GPUs + b.GPUs, a.CPUCores + b.CPUCores, a.MemoryGB + b.MemoryGB}
}
// Over names the first resource in which a is above the governed amounts of
// l, with both figures; empty when it is above none.
func Over(a Amount, l store.Resources) string {
switch {
case l.GPUs != nil && a.GPUs > *l.GPUs:
return fmt.Sprintf("%d GPUs of %d", a.GPUs, *l.GPUs)
case l.CPUCores != nil && a.CPUCores > *l.CPUCores:
return fmt.Sprintf("%d cores of %d", a.CPUCores, *l.CPUCores)
case l.MemoryGB != nil && a.MemoryGB > *l.MemoryGB:
return fmt.Sprintf("%d GB of memory of %d", a.MemoryGB, *l.MemoryGB)
}
return ""
}
// Holding is one replica's hold on the cluster, for working out who holds
// what.
type Holding struct {
Replica string
Owner string
Amount Amount
Since time.Time
}
// Policy is the quotas and the tree they sit in, read at one moment.
type Policy struct {
quotas map[string]store.CapacityQuota
parents map[string][]string
}
// New builds a policy from stored quotas and each scope's parents.
func New(quotas []store.CapacityQuota, parents map[string][]string) *Policy {
p := &Policy{quotas: map[string]store.CapacityQuota{}, parents: parents}
for _, q := range quotas {
p.quotas[Key(q.Scope, q.ScopeID)] = q
}
return p
}
// Key writes a scope the way owners are written: team:ID.
func Key(s store.Scope, id string) string { return string(s) + ":" + id }
// Chain is an owner and every scope above it, nearest first.
func (p *Policy) Chain(owner string) []string {
if owner == "" {
return nil
}
return append([]string{owner}, p.parents[owner]...)
}
// Governing is the quotas along an owner's chain, nearest first.
func (p *Policy) Governing(owner string) []store.CapacityQuota {
if p == nil {
return nil
}
var out []store.CapacityQuota
for _, s := range p.Chain(owner) {
if q, ok := p.quotas[s]; ok {
out = append(out, q)
}
}
return out
}
// Quotas lists every quota.
func (p *Policy) Quotas() []store.CapacityQuota {
if p == nil {
return nil
}
out := make([]store.CapacityQuota, 0, len(p.quotas))
for _, q := range p.quotas {
out = append(out, q)
}
sort.Slice(out, func(i, j int) bool {
return Key(out[i].Scope, out[i].ScopeID) < Key(out[j].Scope, out[j].ScopeID)
})
return out
}
// Usage is what every scope holds, its own models' replicas and those of
// everything beneath it.
func (p *Policy) Usage(hs []Holding) map[string]Amount {
out := map[string]Amount{}
if p == nil {
return out
}
for _, h := range hs {
for _, s := range p.Chain(h.Owner) {
out[s] = out[s].Plus(h.Amount)
}
}
return out
}
// Borrowed is the replicas held beyond a guarantee: of each governed scope's
// replicas, oldest first, the ones that take it past what it is guaranteed.
// A replica beyond any guarantee along its chain is borrowed.
func (p *Policy) Borrowed(hs []Holding) map[string]bool {
out := map[string]bool{}
if p == nil || len(p.quotas) == 0 {
return out
}
sorted := append([]Holding(nil), hs...)
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Since.Before(sorted[j].Since) })
for key, q := range p.quotas {
if !q.Guarantee.Any() {
continue
}
var sum Amount
for _, h := range sorted {
if !contains(p.Chain(h.Owner), key) {
continue
}
sum = sum.Plus(h.Amount)
if Over(sum, q.Guarantee) != "" {
out[h.Replica] = true
}
}
}
return out
}
// Guaranteed reports whether an owner, holding what it holds, is still within
// every guarantee along its chain with need added — so it may take back what
// others borrowed. An owner with no guarantee anywhere is never guaranteed.
func (p *Policy) Guaranteed(owner string, usage map[string]Amount, need Amount) bool {
gov := p.Governing(owner)
any := false
for _, q := range gov {
if !q.Guarantee.Any() {
continue
}
any = true
if Over(usage[Key(q.Scope, q.ScopeID)].Plus(need), q.Guarantee) != "" {
return false
}
}
return any
}
// AtLimit says which limit along an owner's chain need would take it past,
// empty when none.
func (p *Policy) AtLimit(owner string, usage map[string]Amount, need Amount) string {
for _, q := range p.Governing(owner) {
if !q.Limit.Any() {
continue
}
key := Key(q.Scope, q.ScopeID)
if over := Over(usage[key].Plus(need), q.Limit); over != "" {
return fmt.Sprintf("%s is at its capacity limit (%s)", key, strings.TrimSpace(over))
}
}
return ""
}
func contains(list []string, v string) bool {
for _, x := range list {
if x == v {
return true
}
}
return false
}