A model has an owner (org, team or project). Quotas guarantee a scope GPUs, cores and memory and may limit it; what a scope holds is summed from its models' replicas and those beneath it. A replica within its owner's guarantee is not taken for another scope; what is held beyond a guarantee is borrowed and taken back for a scope within its own. A limit stops a model before it starts, and headroom stops there too. ctl quota and ctl model owner set them; metrics and a CapacityAtLimit alert report them. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
53 lines
1.6 KiB
Go
53 lines
1.6 KiB
Go
package control
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"sync"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// QuotaStore is where capacity quotas and the tenancy tree are kept.
|
|
type QuotaStore interface {
|
|
CapacityQuotas(ctx context.Context) ([]store.CapacityQuota, error)
|
|
ScopeParents(ctx context.Context) (map[string][]string, error)
|
|
}
|
|
|
|
// quotaRefresh is how stale the reconciler's view of quotas may get. A quota
|
|
// changed by an operator takes effect within this; the reconciler asks many
|
|
// times a pass and the store is not asked each time.
|
|
const quotaRefresh = 30 * time.Second
|
|
|
|
// QuotaPolicy returns what the reconciler asks for its quotas: the last
|
|
// policy read from the store, read again once it is older than quotaRefresh.
|
|
// A store that fails keeps the last good policy, so a database hiccup does
|
|
// not drop every guarantee and limit at once.
|
|
func QuotaPolicy(st QuotaStore, log *slog.Logger) func() *quota.Policy {
|
|
var mu sync.Mutex
|
|
var cur *quota.Policy
|
|
var at time.Time
|
|
return func() *quota.Policy {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if cur != nil && time.Since(at) < quotaRefresh {
|
|
return cur
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
qs, err := st.CapacityQuotas(ctx)
|
|
if err == nil {
|
|
var parents map[string][]string
|
|
if parents, err = st.ScopeParents(ctx); err == nil {
|
|
cur, at = quota.New(qs, parents), time.Now()
|
|
return cur
|
|
}
|
|
}
|
|
log.Warn("could not read capacity quotas; keeping the last ones", "err", err)
|
|
at = time.Now() // not again on every call while the store is down
|
|
return cur
|
|
}
|
|
}
|