Count a group's quota against the group, not against each of its keys
The tenancy tree handed limits down and never pooled what was spent against them. A limit written on a team was inherited by every key beneath it, and then enforced with the key as the counter — so a project given a budget of 1000 and ten keys permitted ten thousand, and a team given 60 RPM got 60 per member. That is "each of you may spend this much", which is a useful thing to be able to say. It is not what somebody writing a budget on a team means, and nothing in the system said which one you were getting. Resolution now records which scope set each value that won, and the counter is keyed by that scope. The value and the counter come from the same place, so a project tightening its team's limit is enforced against the project's own pool rather than eating into a quota another project never agreed to. Each limit is owned separately, because they resolve separately: a team budget with a per-key rate limit — the group's money, the individual's throughput — is the ordinary arrangement and has to stay expressible. Cluster defaults are owned by the key. Read as a shared pool, one default would become a single cluster-wide rate limit that every key in every organization competes for: not a default, an outage. Restoring after a restart had to change with it. Per-key totals rebuilt a counter named after the key while the check read one named after the team, so the control plane came back believing the team had spent nothing — a budget that reset on every restart, for exactly the scopes where it matters most. SpendSinceByScope returns each key's place in the tree and the accountant folds it into every scope that might own a budget. Counters are namespaced by level throughout. A bare key id in one path and a "key:"-prefixed one in another are two counters for the same key, which is how the restore was silently doing nothing. Still missing, and worth naming: no usage report rolls up by project, team or org, so a shared quota can now be enforced but not yet shown. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
3aa250428a
commit
3febb6ef63
5 files changed
+503
-16
No files matched your search
@@ -70,19 +70,44 @@ func (a *Accountant) RestoreBudgets(ctx context.Context, s store.Store, period s
|
||||
if start.IsZero() {
|
||||
return nil
|
||||
}
|
||||
spend, err := s.SpendSince(ctx, start)
|
||||
rows, err := s.SpendSinceByScope(ctx, start)
|
||||
if err != nil {
|
||||
return fmt.Errorf("restoring budgets: %w", err)
|
||||
}
|
||||
// Every scope a key sits under, not just the key. A budget owned by a team
|
||||
// is one counter fed by every key beneath it, and restoring only per-key
|
||||
// totals would bring the control plane back up believing the team had spent
|
||||
// nothing — which is a budget that resets on every restart, for exactly the
|
||||
// scopes where it matters most.
|
||||
spend := make(map[string]int64, len(rows)*4)
|
||||
for _, r := range rows {
|
||||
for _, o := range []store.LimitOwner{
|
||||
{Scope: store.ScopeKey, ID: r.KeyID},
|
||||
{Scope: store.ScopeProject, ID: r.ProjectID},
|
||||
{Scope: store.ScopeTeam, ID: r.TeamID},
|
||||
{Scope: store.ScopeOrg, ID: r.OrgID},
|
||||
} {
|
||||
if k := o.CounterKey(); k != "" {
|
||||
spend[k] += r.Micros
|
||||
}
|
||||
}
|
||||
}
|
||||
a.budgets.Restore(period, spend)
|
||||
a.log.Info("restored budgets from the store", "keys", len(spend), "since", start)
|
||||
a.log.Info("restored budgets from the store",
|
||||
"keys", len(rows), "counters", len(spend), "since", start)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reservation is what Check returns: the estimate it charged, so Record can
|
||||
// settle the difference against the real figure.
|
||||
type Reservation struct {
|
||||
KeyID string
|
||||
KeyID string
|
||||
// BudgetCounter and TokenCounter are what the check counted against, carried
|
||||
// so the settlement lands on the same counter. Deriving them again here
|
||||
// would mean re-resolving the limits, and a budget checked against the team
|
||||
// and settled against the key is a budget that never fills up.
|
||||
BudgetCounter string
|
||||
TokenCounter string
|
||||
EstimatedTokens int64
|
||||
Price Price
|
||||
// Period is the key's budget period, carried from the check so the record
|
||||
@@ -99,14 +124,21 @@ type Reservation struct {
|
||||
// figure only exists once the request has run; Record settles the difference.
|
||||
func (a *Accountant) Check(p auth.Principal, model string, estimatedTokens int64) (Reservation, error) {
|
||||
price := a.pricing.For(model)
|
||||
lim := p.Limits
|
||||
// Counted against whichever scope set the limit. A budget written on a team
|
||||
// is one pool the team shares; counting it per key would hand the team's
|
||||
// number to every key under it and permit the total times the headcount.
|
||||
budgetCounter := counterFor(lim.BudgetOwner, p.KeyID)
|
||||
tokenCounter := counterFor(lim.TPMOwner, p.KeyID)
|
||||
|
||||
res := Reservation{
|
||||
KeyID: p.KeyID, EstimatedTokens: estimatedTokens,
|
||||
Price: price, Period: p.Limits.BudgetPeriod,
|
||||
BudgetCounter: budgetCounter, TokenCounter: tokenCounter,
|
||||
Price: price, Period: lim.BudgetPeriod,
|
||||
}
|
||||
if p.KeyID == "" {
|
||||
return res, nil
|
||||
}
|
||||
lim := p.Limits
|
||||
|
||||
// Budget first. It is the limit whose refusal cannot be retried away, so
|
||||
// telling the client about it first is more useful than reporting a rate
|
||||
@@ -115,14 +147,14 @@ func (a *Accountant) Check(p auth.Principal, model string, estimatedTokens int64
|
||||
// A model that costs nothing is exempt, which is what lets a key with no
|
||||
// budget left still be served a cheap fallback (plan §7.7).
|
||||
if lim.Budget != nil && !price.Zero() &&
|
||||
a.budgets.Exhausted(p.KeyID, lim.BudgetPeriod, *lim.Budget) {
|
||||
a.budgets.Exhausted(budgetCounter, lim.BudgetPeriod, *lim.Budget) {
|
||||
return res, &LimitError{Err: ErrBudgetSpent}
|
||||
}
|
||||
|
||||
if lim.RPM != nil && !a.requests.Allow(p.KeyID, *lim.RPM, 1) {
|
||||
if lim.RPM != nil && !a.requests.Allow(counterFor(lim.RPMOwner, p.KeyID), *lim.RPM, 1) {
|
||||
return res, &LimitError{Err: ErrRateLimited, RetryAfter: time.Second}
|
||||
}
|
||||
if lim.TPM != nil && !a.tokens.Allow(p.KeyID, *lim.TPM, estimatedTokens) {
|
||||
if lim.TPM != nil && !a.tokens.Allow(tokenCounter, *lim.TPM, estimatedTokens) {
|
||||
// The request was not charged against the request-rate limit in vain:
|
||||
// it was, and that is correct — it did arrive.
|
||||
return res, &LimitError{Err: ErrTokenLimited, RetryAfter: time.Second}
|
||||
@@ -141,10 +173,10 @@ func (a *Accountant) Record(res Reservation, u store.Usage) {
|
||||
// The estimate was charged up front; charge the difference now. A caller
|
||||
// whose requests are consistently larger than estimated is still bounded,
|
||||
// one window later than with a perfect estimate.
|
||||
a.tokens.Charge(res.KeyID, actual-res.EstimatedTokens)
|
||||
a.tokens.Charge(or(res.TokenCounter, keyCounter(res.KeyID)), actual-res.EstimatedTokens)
|
||||
|
||||
u.CostMicros = res.Price.Cost(u.PromptTokens, u.CompletionTokens)
|
||||
a.budgets.Add(res.KeyID, res.Period, u.CostMicros)
|
||||
a.budgets.Add(or(res.BudgetCounter, keyCounter(res.KeyID)), res.Period, u.CostMicros)
|
||||
|
||||
if a.recorder != nil {
|
||||
a.recorder.Record(u)
|
||||
@@ -156,6 +188,42 @@ func (a *Accountant) Status(p auth.Principal) (spent int64, rpm, tpm float64) {
|
||||
if p.KeyID == "" {
|
||||
return 0, 0, 0
|
||||
}
|
||||
return a.budgets.Spent(p.KeyID, p.Limits.BudgetPeriod),
|
||||
a.requests.Rate(p.KeyID), a.tokens.Rate(p.KeyID)
|
||||
// Reported against the same counters the check reads, so a header saying
|
||||
// how much is left agrees with the refusal the next request gets. A member
|
||||
// of a team sharing a budget sees the team's spend, which is the number
|
||||
// that will stop them.
|
||||
return a.budgets.Spent(counterFor(p.Limits.BudgetOwner, p.KeyID), p.Limits.BudgetPeriod),
|
||||
a.requests.Rate(counterFor(p.Limits.RPMOwner, p.KeyID)),
|
||||
a.tokens.Rate(counterFor(p.Limits.TPMOwner, p.KeyID))
|
||||
}
|
||||
|
||||
// counterFor is the counter a limit owned by this scope should use.
|
||||
//
|
||||
// Falls back to the key's own counter when no scope owns the limit, which
|
||||
// happens when nothing in the chain set one.
|
||||
//
|
||||
// Namespaced even then. A bare key id here and a "key:"-prefixed one in the
|
||||
// restore path would be two different counters for the same key, so a budget
|
||||
// would check against an empty one and survive every restart — which is how
|
||||
// this was wrong when it was first written.
|
||||
func counterFor(o store.LimitOwner, keyID string) string {
|
||||
if k := o.CounterKey(); k != "" {
|
||||
return k
|
||||
}
|
||||
return keyCounter(keyID)
|
||||
}
|
||||
|
||||
// keyCounter is a key's own counter, in the same namespace as every other.
|
||||
func or(v, fallback string) string {
|
||||
if v == "" {
|
||||
return fallback
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func keyCounter(keyID string) string {
|
||||
if keyID == "" {
|
||||
return ""
|
||||
}
|
||||
return store.LimitOwner{Scope: store.ScopeKey, ID: keyID}.CounterKey()
|
||||
}
|
||||
@@ -288,3 +288,174 @@ func must(t *testing.T, err error) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// teamMember is a key whose budget and rate limit were set on its team, which
|
||||
// is the arrangement an enterprise actually asks for: a group gets a quota and
|
||||
// the people in it share it.
|
||||
func teamMember(keyID, teamID string, tune func(*store.Effective)) auth.Principal {
|
||||
e := store.Effective{KeyID: keyID, ProjectID: "proj", TeamID: teamID}
|
||||
owner := store.LimitOwner{Scope: store.ScopeTeam, ID: teamID}
|
||||
e.BudgetOwner, e.RPMOwner, e.TPMOwner = owner, owner, owner
|
||||
if tune != nil {
|
||||
tune(&e)
|
||||
}
|
||||
return auth.Principal{KeyID: keyID, ProjectID: "proj", Limits: e}
|
||||
}
|
||||
|
||||
func TestATeamBudgetIsOnePoolRatherThanOnePerMember(t *testing.T) {
|
||||
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
||||
budget := func(e *store.Effective) {
|
||||
e.Budget = i64p(3000)
|
||||
e.BudgetPeriod = store.PeriodMonth
|
||||
}
|
||||
first := teamMember("k1", "t1", budget)
|
||||
second := teamMember("k2", "t1", budget)
|
||||
|
||||
// One member spends the lot.
|
||||
res, err := a.Check(first, "paid", 1000)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 1000, CompletionTokens: 1000})
|
||||
|
||||
// The second must now be refused. Counted per key it would have its own
|
||||
// untouched 3000 — a team budget of 3000 with ten keys would permit thirty
|
||||
// thousand, which is the bug this change exists for.
|
||||
if _, err := a.Check(second, "paid", 10); !errors.Is(err, ErrBudgetSpent) {
|
||||
t.Errorf("the team's budget should be shared; the second member got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestATeamRateLimitIsSharedToo(t *testing.T) {
|
||||
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
||||
rpm := func(e *store.Effective) { e.RPM = intp(1) }
|
||||
|
||||
if _, err := a.Check(teamMember("k1", "t1", rpm), "free", 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := a.Check(teamMember("k2", "t1", rpm), "free", 1); !errors.Is(err, ErrRateLimited) {
|
||||
t.Error("a rate limit set on the team should be shared by its keys")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoTeamsDoNotShareACounter(t *testing.T) {
|
||||
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
||||
rpm := func(e *store.Effective) { e.RPM = intp(1) }
|
||||
|
||||
if _, err := a.Check(teamMember("k1", "t1", rpm), "free", 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Pooling is the point; pooling the wrong things is the danger. One team
|
||||
// exhausting another's quota would be worse than the bug being fixed.
|
||||
if _, err := a.Check(teamMember("k2", "t2", rpm), "free", 1); err != nil {
|
||||
t.Errorf("a second team should have its own counter, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAKeyLimitStaysPerKey(t *testing.T) {
|
||||
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
||||
perKey := func(id string) auth.Principal {
|
||||
return principal(id, func(e *store.Effective) {
|
||||
e.RPM = intp(1)
|
||||
e.RPMOwner = store.LimitOwner{Scope: store.ScopeKey, ID: id}
|
||||
})
|
||||
}
|
||||
|
||||
if _, err := a.Check(perKey("k1"), "free", 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The tree still has to be able to say "each of you may do this much".
|
||||
// Sharing a group pool must not become the only thing expressible.
|
||||
if _, err := a.Check(perKey("k2"), "free", 1); err != nil {
|
||||
t.Errorf("a limit owned by the key should not be shared, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheDifferentLimitsCanBeOwnedByDifferentScopes(t *testing.T) {
|
||||
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
||||
// A team budget with a per-key rate limit is the ordinary arrangement: the
|
||||
// group's money, the individual's throughput.
|
||||
mk := func(id string) auth.Principal {
|
||||
e := store.Effective{KeyID: id, TeamID: "t1"}
|
||||
e.Budget, e.BudgetPeriod = i64p(3000), store.PeriodMonth
|
||||
e.BudgetOwner = store.LimitOwner{Scope: store.ScopeTeam, ID: "t1"}
|
||||
e.RPM = intp(1)
|
||||
e.RPMOwner = store.LimitOwner{Scope: store.ScopeKey, ID: id}
|
||||
return auth.Principal{KeyID: id, Limits: e}
|
||||
}
|
||||
|
||||
res, err := a.Check(mk("k1"), "paid", 1000)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 1000, CompletionTokens: 1000})
|
||||
|
||||
// k2's rate limit is its own and untouched; the budget is the team's and
|
||||
// gone. The budget is checked first, so that is what comes back.
|
||||
if _, err := a.Check(mk("k2"), "paid", 10); !errors.Is(err, ErrBudgetSpent) {
|
||||
t.Errorf("expected the shared budget to bite, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusReportsTheCounterThatWillRefuseYou(t *testing.T) {
|
||||
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
||||
budget := func(e *store.Effective) {
|
||||
e.Budget = i64p(9000)
|
||||
e.BudgetPeriod = store.PeriodMonth
|
||||
}
|
||||
res, err := a.Check(teamMember("k1", "t1", budget), "paid", 1000)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 1000, CompletionTokens: 1000})
|
||||
|
||||
// A member who has spent nothing themselves still sees the team's spend,
|
||||
// because that is the number that will stop them. A header disagreeing with
|
||||
// the refusal it precedes is worse than no header.
|
||||
spent, _, _ := a.Status(teamMember("k2", "t1", budget))
|
||||
if spent == 0 {
|
||||
t.Error("a member should see the pool's spend, not their own")
|
||||
}
|
||||
}
|
||||
|
||||
func TestATeamBudgetSurvivesARestart(t *testing.T) {
|
||||
// The per-key restore could not do this: it rebuilt a counter named after
|
||||
// the key, while the check reads one named after the team. The control
|
||||
// plane came back believing the team had spent nothing — a budget that
|
||||
// resets on every restart, for exactly the scope where it matters most.
|
||||
ctx := context.Background()
|
||||
s, err := store.OpenSQLite(ctx, filepath.Join(t.TempDir(), "team.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
must(t, s.PutOrg(ctx, store.Org{ID: "o", Name: "o"}))
|
||||
must(t, s.PutTeam(ctx, store.Team{ID: "t1", OrgID: "o", Name: "t"}))
|
||||
must(t, s.PutProject(ctx, store.Project{ID: "proj", TeamID: "t1", Name: "p"}))
|
||||
for _, id := range []string{"k1", "k2"} {
|
||||
_, k, _ := store.NewKey(id, "n", "proj")
|
||||
k.CreatedAt = time.Now()
|
||||
must(t, s.CreateKey(ctx, k))
|
||||
}
|
||||
|
||||
// Two members of one team, 2500 each: neither alone exceeds a 4000 budget,
|
||||
// together they do.
|
||||
must(t, s.WriteUsage(ctx, []store.Usage{
|
||||
{At: time.Now(), KeyID: "k1", ProjectID: "proj", Model: "paid",
|
||||
ServedModel: "paid", CostMicros: 2500, Status: 200},
|
||||
{At: time.Now(), KeyID: "k2", ProjectID: "proj", Model: "paid",
|
||||
ServedModel: "paid", CostMicros: 2500, Status: 200},
|
||||
}))
|
||||
|
||||
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
||||
must(t, a.RestoreBudgets(ctx, s, store.PeriodMonth))
|
||||
|
||||
p := teamMember("k1", "t1", func(e *store.Effective) {
|
||||
e.Budget = i64p(4000)
|
||||
e.BudgetPeriod = store.PeriodMonth
|
||||
})
|
||||
if _, err := a.Check(p, "paid", 10); !errors.Is(err, ErrBudgetSpent) {
|
||||
t.Errorf("the team's spend should have been restored and pooled: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func treeStore(t *testing.T) *SQLite {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
s, err := OpenSQLite(ctx, filepath.Join(t.TempDir(), "t.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
for _, err := range []error{
|
||||
s.PutOrg(ctx, Org{ID: "o", Name: "o"}),
|
||||
s.PutTeam(ctx, Team{ID: "t", OrgID: "o", Name: "t"}),
|
||||
s.PutProject(ctx, Project{ID: "p", TeamID: "t", Name: "p"}),
|
||||
} {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
_, k, _ := NewKey("k1", "n", "p")
|
||||
k.CreatedAt = time.Now()
|
||||
if err := s.CreateKey(ctx, k); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func TestAResolvedLimitRemembersWhichScopeSetIt(t *testing.T) {
|
||||
s := treeStore(t)
|
||||
rpm := 60
|
||||
if err := s.PutLimits(t.Context(), Limits{Scope: ScopeTeam, ScopeID: "t", RPM: &rpm}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
e, err := s.Resolve(t.Context(), "k1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Without this the tree hands down values and never pools spending: the
|
||||
// team's 60 becomes 60 for each key under it.
|
||||
if e.RPMOwner.Scope != ScopeTeam || e.RPMOwner.ID != "t" {
|
||||
t.Errorf("expected the team to own the rate limit, got %+v", e.RPMOwner)
|
||||
}
|
||||
if e.RPMOwner.CounterKey() != "team:t" {
|
||||
t.Errorf("counter key should name the level too; got %q", e.RPMOwner.CounterKey())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheTighterScopeTakesOwnershipWithTheValue(t *testing.T) {
|
||||
s := treeStore(t)
|
||||
loose, tight := 100, 10
|
||||
must := func(err error) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
must(s.PutLimits(t.Context(), Limits{Scope: ScopeTeam, ScopeID: "t", RPM: &loose}))
|
||||
must(s.PutLimits(t.Context(), Limits{Scope: ScopeProject, ScopeID: "p", RPM: &tight}))
|
||||
|
||||
e, err := s.Resolve(t.Context(), "k1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The value and the counter have to come from the same place. Enforcing
|
||||
// the project's 10 against the team's pool would let one project's traffic
|
||||
// exhaust a limit another project never agreed to.
|
||||
if *e.RPM != 10 || e.RPMOwner.ID != "p" {
|
||||
t.Errorf("expected the project's value and ownership, got %d from %+v", *e.RPM, e.RPMOwner)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALooserInnerScopeDoesNotTakeOwnership(t *testing.T) {
|
||||
s := treeStore(t)
|
||||
tight, loose := 10, 100
|
||||
if err := s.PutLimits(t.Context(), Limits{Scope: ScopeTeam, ScopeID: "t", RPM: &tight}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.PutLimits(t.Context(), Limits{Scope: ScopeProject, ScopeID: "p", RPM: &loose}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
e, err := s.Resolve(t.Context(), "k1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A project cannot raise what its team set, so the team still owns both the
|
||||
// number and the counter it is enforced against.
|
||||
if *e.RPM != 10 || e.RPMOwner.ID != "t" {
|
||||
t.Errorf("expected the team to keep ownership, got %d from %+v", *e.RPM, e.RPMOwner)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAClusterDefaultIsOwnedByTheKey(t *testing.T) {
|
||||
s := treeStore(t)
|
||||
rpm := 30
|
||||
s.SetDefaultLimits(&rpm, nil)
|
||||
|
||||
e, err := s.Resolve(t.Context(), "k1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A default is a fallback for each caller given no limit of its own. Owned
|
||||
// by a shared scope it would become one cluster-wide rate limit that every
|
||||
// key in every organization competes for — not a default, an outage.
|
||||
if e.RPMOwner.Scope != ScopeKey || e.RPMOwner.ID != "k1" {
|
||||
t.Errorf("a default should be counted per key, got %+v", e.RPMOwner)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpendComesBackWithThePlaceInTheTree(t *testing.T) {
|
||||
s := treeStore(t)
|
||||
if err := s.WriteUsage(t.Context(), []Usage{
|
||||
{At: time.Now(), KeyID: "k1", ProjectID: "p", Model: "m",
|
||||
ServedModel: "m", CostMicros: 700, Status: 200},
|
||||
{At: time.Now(), KeyID: "k1", ProjectID: "p", Model: "m",
|
||||
ServedModel: "m", CostMicros: 300, Status: 200},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rows, err := s.SpendSinceByScope(t.Context(), time.Now().Add(-time.Hour))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) != 1 {
|
||||
t.Fatalf("expected one key's total, got %+v", rows)
|
||||
}
|
||||
// The tree, not just the key: a budget owned by a team is rebuilt by
|
||||
// summing every key under it, and per-key totals alone cannot say which
|
||||
// keys those are.
|
||||
got := rows[0]
|
||||
if got.Micros != 1000 {
|
||||
t.Errorf("expected the two rows summed, got %d", got.Micros)
|
||||
}
|
||||
if got.ProjectID != "p" || got.TeamID != "t" || got.OrgID != "o" {
|
||||
t.Errorf("the whole chain should come back, got %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -336,26 +336,73 @@ func (s *SQLite) SetDefaultLimits(rpm, tpm *int) {
|
||||
s.defaultRPM, s.defaultTPM = rpm, tpm
|
||||
}
|
||||
|
||||
// applyDefaults fills in what the chain left unset.
|
||||
//
|
||||
// Owned by the key, not by any scope. A cluster-wide default is a fallback for
|
||||
// each caller that was given no limit of its own — read as a shared pool it
|
||||
// would become one cluster-wide rate limit that every key in every
|
||||
// organization competes for, which is not a default, it is an outage.
|
||||
func (s *SQLite) applyDefaults(e *Effective) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
own := LimitOwner{Scope: ScopeKey, ID: e.KeyID}
|
||||
if e.RPM == nil && s.defaultRPM != nil {
|
||||
v := *s.defaultRPM
|
||||
e.RPM = &v
|
||||
e.RPM, e.RPMOwner = &v, own
|
||||
}
|
||||
if e.TPM == nil && s.defaultTPM != nil {
|
||||
v := *s.defaultTPM
|
||||
e.TPM = &v
|
||||
e.TPM, e.TPMOwner = &v, own
|
||||
}
|
||||
}
|
||||
|
||||
// SpendSinceByScope totals cost per key, with the key's place in the tree.
|
||||
func (s *SQLite) SpendSinceByScope(ctx context.Context, since time.Time) ([]ScopeSpend, error) {
|
||||
rows, err := s.db.QueryContext(ctx,
|
||||
`SELECT k.id, p.id, t.id, o.id, SUM(u.cost_micros)
|
||||
FROM usage u
|
||||
JOIN api_keys k ON k.id = u.key_id
|
||||
JOIN projects p ON p.id = k.project_id
|
||||
JOIN teams t ON t.id = p.team_id
|
||||
JOIN organizations o ON o.id = t.org_id
|
||||
WHERE u.ts >= ?
|
||||
GROUP BY k.id, p.id, t.id, o.id`, millis(since))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []ScopeSpend
|
||||
for rows.Next() {
|
||||
var sp ScopeSpend
|
||||
var total sql.NullInt64
|
||||
if err := rows.Scan(&sp.KeyID, &sp.ProjectID, &sp.TeamID, &sp.OrgID, &total); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sp.Micros = total.Int64
|
||||
out = append(out, sp)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// tighten applies one scope's limits, keeping whichever is stricter.
|
||||
//
|
||||
// Each value that wins records the scope it came from. That is what lets
|
||||
// consumption be counted against the scope that set the limit: a budget written
|
||||
// on a team is one pool the team shares, not a copy of the number handed to
|
||||
// every key beneath it.
|
||||
func (e *Effective) tighten(l Limits) {
|
||||
e.RPM = minPtr(e.RPM, l.RPM)
|
||||
e.TPM = minPtr(e.TPM, l.TPM)
|
||||
owner := LimitOwner{Scope: l.Scope, ID: l.ScopeID}
|
||||
if next := minPtr(e.RPM, l.RPM); next != e.RPM {
|
||||
e.RPM, e.RPMOwner = next, owner
|
||||
}
|
||||
if next := minPtr(e.TPM, l.TPM); next != e.TPM {
|
||||
e.TPM, e.TPMOwner = next, owner
|
||||
}
|
||||
if l.Budget != nil && (e.Budget == nil || *l.Budget < *e.Budget) {
|
||||
e.Budget = l.Budget
|
||||
e.BudgetPeriod = l.BudgetPeriod
|
||||
e.BudgetOwner = owner
|
||||
}
|
||||
if l.AllowedModels != nil {
|
||||
if e.AllowedModels == nil {
|
||||
|
||||
@@ -128,6 +128,41 @@ type Effective struct {
|
||||
BudgetPeriod Period
|
||||
// AllowedModels is the intersection along the chain. nil means every model.
|
||||
AllowedModels []string
|
||||
|
||||
// RPMOwner, TPMOwner and BudgetOwner say which scope set the value that
|
||||
// won, so consumption can be counted against that scope rather than
|
||||
// against each key under it.
|
||||
//
|
||||
// Without this the tree hands down values but never pools spending: a
|
||||
// project given a budget of 1000 and ten keys permits ten thousand, because
|
||||
// each key gets its own counter holding the project's number. That is "each
|
||||
// member may spend 1000", which is a useful thing to be able to say and is
|
||||
// not what somebody writing a budget on a project means.
|
||||
//
|
||||
// Each limit is owned separately because they resolve separately: a budget
|
||||
// can come from the team while the rate limit comes from the key.
|
||||
RPMOwner LimitOwner
|
||||
TPMOwner LimitOwner
|
||||
BudgetOwner LimitOwner
|
||||
}
|
||||
|
||||
// LimitOwner is the scope a resolved limit came from.
|
||||
type LimitOwner struct {
|
||||
Scope Scope
|
||||
ID string
|
||||
}
|
||||
|
||||
// CounterKey is what a usage counter for this limit should be keyed by.
|
||||
//
|
||||
// Empty when no scope set the limit, which means nothing is being counted and
|
||||
// the caller should not be asking. Scope and id together, because ids are only
|
||||
// unique within a level and a project could otherwise share a counter with a
|
||||
// team that happened to be named the same.
|
||||
func (o LimitOwner) CounterKey() string {
|
||||
if o.ID == "" {
|
||||
return ""
|
||||
}
|
||||
return string(o.Scope) + ":" + o.ID
|
||||
}
|
||||
|
||||
// Allows reports whether a model may be served to this key.
|
||||
@@ -193,6 +228,11 @@ type Store interface {
|
||||
// counter can be rebuilt after a restart. A restart must not reset a
|
||||
// budget, which it would if the counter lived only in memory.
|
||||
SpendSince(ctx context.Context, since time.Time) (map[string]int64, error)
|
||||
// SpendSinceByScope is the same totals with each key's place in the tree,
|
||||
// so a budget owned by a team can be rebuilt by summing every key under it.
|
||||
// Per key alone cannot do that: the counter is one number for the whole
|
||||
// team, and the rows that made it up are spread across its projects.
|
||||
SpendSinceByScope(ctx context.Context, since time.Time) ([]ScopeSpend, error)
|
||||
// UsageSince summarizes activity per key. A reporting query: it scans and
|
||||
// groups, so it belongs behind a dashboard refresh rather than a request.
|
||||
UsageSince(ctx context.Context, since time.Time) ([]KeyUsage, error)
|
||||
@@ -228,3 +268,17 @@ type Bucket struct {
|
||||
// an SLO measurement.
|
||||
LatencyMs int64
|
||||
}
|
||||
|
||||
// ScopeSpend is one key's spend in a period, with where that key sits in the
|
||||
// tenancy tree.
|
||||
//
|
||||
// Returned rather than pre-aggregated per scope because the roll-up depends on
|
||||
// which scopes own budgets, which the store does not know and the accountant
|
||||
// does. One query, and the folding happens where it can be tested.
|
||||
type ScopeSpend struct {
|
||||
KeyID string
|
||||
ProjectID string
|
||||
TeamID string
|
||||
OrgID string
|
||||
Micros int64
|
||||
}
|
||||
Reference in new issue
Block a user