Files
llmgw/internal/control/reconciler/quotas.go
T
Ilker ManapandClaude Opus 5.5 9dc1b62fdc
Release / build (push) Successful in 1m8s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m51s
Guarantee and limit capacity per part of the tenancy tree
A model has an owner (org, team or project). Quotas guarantee a scope
GPUs, cores and memory and may limit it; what a scope holds is summed from
its models' replicas and those beneath it. A replica within its owner's
guarantee is not taken for another scope; what is held beyond a guarantee
is borrowed and taken back for a scope within its own. A limit stops a
model before it starts, and headroom stops there too. ctl quota and ctl
model owner set them; metrics and a CapacityAtLimit alert report them.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 18:15:38 +02:00

106 lines
2.9 KiB
Go

package reconciler
import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// SetQuotas supplies the tenancy tree's capacity quotas, read afresh each
// time it is called. Without it nothing is guaranteed or limited, and
// preemption goes by class alone.
func (r *Reconciler) SetQuotas(f func() *quota.Policy) {
r.mu.Lock()
defer r.mu.Unlock()
r.quotas = f
}
func (r *Reconciler) policy() *quota.Policy {
r.mu.Lock()
f := r.quotas
r.mu.Unlock()
if f == nil {
return nil
}
return f()
}
// Holdings is what every replica holds, and for whom.
func (r *Reconciler) Holdings() []quota.Holding {
cat := r.cat.Load()
r.mu.Lock()
defer r.mu.Unlock()
out := make([]quota.Holding, 0, len(r.replicas))
for _, rep := range r.replicas {
m, ok := cat.Resolve(rep.Model)
if !ok {
continue
}
prof, _ := m.ProfileByName(rep.Profile)
var a quota.Amount
for _, sh := range rep.Shards {
a = a.Plus(quota.Amount{GPUs: len(sh.GPUIndices), CPUCores: cpuCores(prof), MemoryGB: prof.MinMemoryGB})
}
out = append(out, quota.Holding{Replica: rep.ID, Owner: m.Owner, Amount: a, Since: rep.CreatedAt})
}
return out
}
// placementNeed is what a placement would hold.
func placementNeed(m catalog.Model, pl *placement.Placement) quota.Amount {
prof, _ := m.ProfileByName(pl.Profile.Name)
var a quota.Amount
for _, n := range pl.Nodes {
a = a.Plus(quota.Amount{GPUs: len(n.GPUIndices), CPUCores: cpuCores(prof), MemoryGB: prof.MinMemoryGB})
}
return a
}
// profileNeed is what a replica of a model's first profile would hold, for
// deciding before there is a placement.
func profileNeed(m catalog.Model) quota.Amount {
if len(m.Profiles) == 0 {
return quota.Amount{}
}
p := m.Profiles[0]
nodes := max(p.Nodes, 1)
gpus := p.GPUsPerNode
if gpus == 0 && !p.CPUOnly() {
gpus = max(p.TP, 1)
}
return quota.Amount{GPUs: gpus * nodes, CPUCores: cpuCores(p) * nodes, MemoryGB: p.MinMemoryGB * nodes}
}
// overLimit says why a placement would take its model's owner past a limit.
func (r *Reconciler) overLimit(m catalog.Model, pl *placement.Placement) string {
pol := r.policy()
if pol == nil || m.Owner == "" {
return ""
}
return pol.AtLimit(m.Owner, pol.Usage(r.Holdings()), placementNeed(m, pl))
}
// QuotaUse is one scope's quota beside what it holds.
type QuotaUse struct {
Scope string
Guarantee store.Resources
Limit store.Resources
Holds quota.Amount
}
// QuotaUse lists every quota with what its scope holds now.
func (r *Reconciler) QuotaUse() []QuotaUse {
pol := r.policy()
if pol == nil {
return nil
}
usage := pol.Usage(r.Holdings())
var out []QuotaUse
for _, q := range pol.Quotas() {
key := quota.Key(q.Scope, q.ScopeID)
out = append(out, QuotaUse{Scope: key, Guarantee: q.Guarantee, Limit: q.Limit, Holds: usage[key]})
}
return out
}