A model has an owner (org, team or project). Quotas guarantee a scope GPUs, cores and memory and may limit it; what a scope holds is summed from its models' replicas and those beneath it. A replica within its owner's guarantee is not taken for another scope; what is held beyond a guarantee is borrowed and taken back for a scope within its own. A limit stops a model before it starts, and headroom stops there too. ctl quota and ctl model owner set them; metrics and a CapacityAtLimit alert report them. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
106 lines
2.9 KiB
Go
106 lines
2.9 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// SetQuotas supplies the tenancy tree's capacity quotas, read afresh each
|
|
// time it is called. Without it nothing is guaranteed or limited, and
|
|
// preemption goes by class alone.
|
|
func (r *Reconciler) SetQuotas(f func() *quota.Policy) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
r.quotas = f
|
|
}
|
|
|
|
func (r *Reconciler) policy() *quota.Policy {
|
|
r.mu.Lock()
|
|
f := r.quotas
|
|
r.mu.Unlock()
|
|
if f == nil {
|
|
return nil
|
|
}
|
|
return f()
|
|
}
|
|
|
|
// Holdings is what every replica holds, and for whom.
|
|
func (r *Reconciler) Holdings() []quota.Holding {
|
|
cat := r.cat.Load()
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
out := make([]quota.Holding, 0, len(r.replicas))
|
|
for _, rep := range r.replicas {
|
|
m, ok := cat.Resolve(rep.Model)
|
|
if !ok {
|
|
continue
|
|
}
|
|
prof, _ := m.ProfileByName(rep.Profile)
|
|
var a quota.Amount
|
|
for _, sh := range rep.Shards {
|
|
a = a.Plus(quota.Amount{GPUs: len(sh.GPUIndices), CPUCores: cpuCores(prof), MemoryGB: prof.MinMemoryGB})
|
|
}
|
|
out = append(out, quota.Holding{Replica: rep.ID, Owner: m.Owner, Amount: a, Since: rep.CreatedAt})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// placementNeed is what a placement would hold.
|
|
func placementNeed(m catalog.Model, pl *placement.Placement) quota.Amount {
|
|
prof, _ := m.ProfileByName(pl.Profile.Name)
|
|
var a quota.Amount
|
|
for _, n := range pl.Nodes {
|
|
a = a.Plus(quota.Amount{GPUs: len(n.GPUIndices), CPUCores: cpuCores(prof), MemoryGB: prof.MinMemoryGB})
|
|
}
|
|
return a
|
|
}
|
|
|
|
// profileNeed is what a replica of a model's first profile would hold, for
|
|
// deciding before there is a placement.
|
|
func profileNeed(m catalog.Model) quota.Amount {
|
|
if len(m.Profiles) == 0 {
|
|
return quota.Amount{}
|
|
}
|
|
p := m.Profiles[0]
|
|
nodes := max(p.Nodes, 1)
|
|
gpus := p.GPUsPerNode
|
|
if gpus == 0 && !p.CPUOnly() {
|
|
gpus = max(p.TP, 1)
|
|
}
|
|
return quota.Amount{GPUs: gpus * nodes, CPUCores: cpuCores(p) * nodes, MemoryGB: p.MinMemoryGB * nodes}
|
|
}
|
|
|
|
// overLimit says why a placement would take its model's owner past a limit.
|
|
func (r *Reconciler) overLimit(m catalog.Model, pl *placement.Placement) string {
|
|
pol := r.policy()
|
|
if pol == nil || m.Owner == "" {
|
|
return ""
|
|
}
|
|
return pol.AtLimit(m.Owner, pol.Usage(r.Holdings()), placementNeed(m, pl))
|
|
}
|
|
|
|
// QuotaUse is one scope's quota beside what it holds.
|
|
type QuotaUse struct {
|
|
Scope string
|
|
Guarantee store.Resources
|
|
Limit store.Resources
|
|
Holds quota.Amount
|
|
}
|
|
|
|
// QuotaUse lists every quota with what its scope holds now.
|
|
func (r *Reconciler) QuotaUse() []QuotaUse {
|
|
pol := r.policy()
|
|
if pol == nil {
|
|
return nil
|
|
}
|
|
usage := pol.Usage(r.Holdings())
|
|
var out []QuotaUse
|
|
for _, q := range pol.Quotas() {
|
|
key := quota.Key(q.Scope, q.ScopeID)
|
|
out = append(out, QuotaUse{Scope: key, Guarantee: q.Guarantee, Limit: q.Limit, Holds: usage[key]})
|
|
}
|
|
return out
|
|
}
|