Files
llmgw/internal/control/quotas.go
T
Ilker ManapandClaude Opus 5.5 9dc1b62fdc
Release / build (push) Successful in 1m8s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m51s
Guarantee and limit capacity per part of the tenancy tree
A model has an owner (org, team or project). Quotas guarantee a scope
GPUs, cores and memory and may limit it; what a scope holds is summed from
its models' replicas and those beneath it. A replica within its owner's
guarantee is not taken for another scope; what is held beyond a guarantee
is borrowed and taken back for a scope within its own. A limit stops a
model before it starts, and headroom stops there too. ctl quota and ctl
model owner set them; metrics and a CapacityAtLimit alert report them.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 18:15:38 +02:00

53 lines
1.6 KiB
Go

package control
import (
"context"
"log/slog"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// QuotaStore is where capacity quotas and the tenancy tree are kept.
type QuotaStore interface {
CapacityQuotas(ctx context.Context) ([]store.CapacityQuota, error)
ScopeParents(ctx context.Context) (map[string][]string, error)
}
// quotaRefresh is how stale the reconciler's view of quotas may get. A quota
// changed by an operator takes effect within this; the reconciler asks many
// times a pass and the store is not asked each time.
const quotaRefresh = 30 * time.Second
// QuotaPolicy returns what the reconciler asks for its quotas: the last
// policy read from the store, read again once it is older than quotaRefresh.
// A store that fails keeps the last good policy, so a database hiccup does
// not drop every guarantee and limit at once.
func QuotaPolicy(st QuotaStore, log *slog.Logger) func() *quota.Policy {
var mu sync.Mutex
var cur *quota.Policy
var at time.Time
return func() *quota.Policy {
mu.Lock()
defer mu.Unlock()
if cur != nil && time.Since(at) < quotaRefresh {
return cur
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
qs, err := st.CapacityQuotas(ctx)
if err == nil {
var parents map[string][]string
if parents, err = st.ScopeParents(ctx); err == nil {
cur, at = quota.New(qs, parents), time.Now()
return cur
}
}
log.Warn("could not read capacity quotas; keeping the last ones", "err", err)
at = time.Now() // not again on every call while the store is down
return cur
}
}