diff --git a/internal/accounting/accounting.go b/internal/accounting/accounting.go index b154035..8746279 100644 --- a/internal/accounting/accounting.go +++ b/internal/accounting/accounting.go @@ -70,19 +70,44 @@ func (a *Accountant) RestoreBudgets(ctx context.Context, s store.Store, period s if start.IsZero() { return nil } - spend, err := s.SpendSince(ctx, start) + rows, err := s.SpendSinceByScope(ctx, start) if err != nil { return fmt.Errorf("restoring budgets: %w", err) } + // Every scope a key sits under, not just the key. A budget owned by a team + // is one counter fed by every key beneath it, and restoring only per-key + // totals would bring the control plane back up believing the team had spent + // nothing — which is a budget that resets on every restart, for exactly the + // scopes where it matters most. + spend := make(map[string]int64, len(rows)*4) + for _, r := range rows { + for _, o := range []store.LimitOwner{ + {Scope: store.ScopeKey, ID: r.KeyID}, + {Scope: store.ScopeProject, ID: r.ProjectID}, + {Scope: store.ScopeTeam, ID: r.TeamID}, + {Scope: store.ScopeOrg, ID: r.OrgID}, + } { + if k := o.CounterKey(); k != "" { + spend[k] += r.Micros + } + } + } a.budgets.Restore(period, spend) - a.log.Info("restored budgets from the store", "keys", len(spend), "since", start) + a.log.Info("restored budgets from the store", + "keys", len(rows), "counters", len(spend), "since", start) return nil } // Reservation is what Check returns: the estimate it charged, so Record can // settle the difference against the real figure. type Reservation struct { - KeyID string + KeyID string + // BudgetCounter and TokenCounter are what the check counted against, carried + // so the settlement lands on the same counter. Deriving them again here + // would mean re-resolving the limits, and a budget checked against the team + // and settled against the key is a budget that never fills up. + BudgetCounter string + TokenCounter string EstimatedTokens int64 Price Price // Period is the key's budget period, carried from the check so the record @@ -99,14 +124,21 @@ type Reservation struct { // figure only exists once the request has run; Record settles the difference. func (a *Accountant) Check(p auth.Principal, model string, estimatedTokens int64) (Reservation, error) { price := a.pricing.For(model) + lim := p.Limits + // Counted against whichever scope set the limit. A budget written on a team + // is one pool the team shares; counting it per key would hand the team's + // number to every key under it and permit the total times the headcount. + budgetCounter := counterFor(lim.BudgetOwner, p.KeyID) + tokenCounter := counterFor(lim.TPMOwner, p.KeyID) + res := Reservation{ KeyID: p.KeyID, EstimatedTokens: estimatedTokens, - Price: price, Period: p.Limits.BudgetPeriod, + BudgetCounter: budgetCounter, TokenCounter: tokenCounter, + Price: price, Period: lim.BudgetPeriod, } if p.KeyID == "" { return res, nil } - lim := p.Limits // Budget first. It is the limit whose refusal cannot be retried away, so // telling the client about it first is more useful than reporting a rate @@ -115,14 +147,14 @@ func (a *Accountant) Check(p auth.Principal, model string, estimatedTokens int64 // A model that costs nothing is exempt, which is what lets a key with no // budget left still be served a cheap fallback (plan §7.7). if lim.Budget != nil && !price.Zero() && - a.budgets.Exhausted(p.KeyID, lim.BudgetPeriod, *lim.Budget) { + a.budgets.Exhausted(budgetCounter, lim.BudgetPeriod, *lim.Budget) { return res, &LimitError{Err: ErrBudgetSpent} } - if lim.RPM != nil && !a.requests.Allow(p.KeyID, *lim.RPM, 1) { + if lim.RPM != nil && !a.requests.Allow(counterFor(lim.RPMOwner, p.KeyID), *lim.RPM, 1) { return res, &LimitError{Err: ErrRateLimited, RetryAfter: time.Second} } - if lim.TPM != nil && !a.tokens.Allow(p.KeyID, *lim.TPM, estimatedTokens) { + if lim.TPM != nil && !a.tokens.Allow(tokenCounter, *lim.TPM, estimatedTokens) { // The request was not charged against the request-rate limit in vain: // it was, and that is correct — it did arrive. return res, &LimitError{Err: ErrTokenLimited, RetryAfter: time.Second} @@ -141,10 +173,10 @@ func (a *Accountant) Record(res Reservation, u store.Usage) { // The estimate was charged up front; charge the difference now. A caller // whose requests are consistently larger than estimated is still bounded, // one window later than with a perfect estimate. - a.tokens.Charge(res.KeyID, actual-res.EstimatedTokens) + a.tokens.Charge(or(res.TokenCounter, keyCounter(res.KeyID)), actual-res.EstimatedTokens) u.CostMicros = res.Price.Cost(u.PromptTokens, u.CompletionTokens) - a.budgets.Add(res.KeyID, res.Period, u.CostMicros) + a.budgets.Add(or(res.BudgetCounter, keyCounter(res.KeyID)), res.Period, u.CostMicros) if a.recorder != nil { a.recorder.Record(u) @@ -156,6 +188,42 @@ func (a *Accountant) Status(p auth.Principal) (spent int64, rpm, tpm float64) { if p.KeyID == "" { return 0, 0, 0 } - return a.budgets.Spent(p.KeyID, p.Limits.BudgetPeriod), - a.requests.Rate(p.KeyID), a.tokens.Rate(p.KeyID) + // Reported against the same counters the check reads, so a header saying + // how much is left agrees with the refusal the next request gets. A member + // of a team sharing a budget sees the team's spend, which is the number + // that will stop them. + return a.budgets.Spent(counterFor(p.Limits.BudgetOwner, p.KeyID), p.Limits.BudgetPeriod), + a.requests.Rate(counterFor(p.Limits.RPMOwner, p.KeyID)), + a.tokens.Rate(counterFor(p.Limits.TPMOwner, p.KeyID)) +} + +// counterFor is the counter a limit owned by this scope should use. +// +// Falls back to the key's own counter when no scope owns the limit, which +// happens when nothing in the chain set one. +// +// Namespaced even then. A bare key id here and a "key:"-prefixed one in the +// restore path would be two different counters for the same key, so a budget +// would check against an empty one and survive every restart — which is how +// this was wrong when it was first written. +func counterFor(o store.LimitOwner, keyID string) string { + if k := o.CounterKey(); k != "" { + return k + } + return keyCounter(keyID) +} + +// keyCounter is a key's own counter, in the same namespace as every other. +func or(v, fallback string) string { + if v == "" { + return fallback + } + return v +} + +func keyCounter(keyID string) string { + if keyID == "" { + return "" + } + return store.LimitOwner{Scope: store.ScopeKey, ID: keyID}.CounterKey() } diff --git a/internal/accounting/accounting_test.go b/internal/accounting/accounting_test.go index 327273e..492c304 100644 --- a/internal/accounting/accounting_test.go +++ b/internal/accounting/accounting_test.go @@ -288,3 +288,174 @@ func must(t *testing.T, err error) { t.Fatal(err) } } + +// teamMember is a key whose budget and rate limit were set on its team, which +// is the arrangement an enterprise actually asks for: a group gets a quota and +// the people in it share it. +func teamMember(keyID, teamID string, tune func(*store.Effective)) auth.Principal { + e := store.Effective{KeyID: keyID, ProjectID: "proj", TeamID: teamID} + owner := store.LimitOwner{Scope: store.ScopeTeam, ID: teamID} + e.BudgetOwner, e.RPMOwner, e.TPMOwner = owner, owner, owner + if tune != nil { + tune(&e) + } + return auth.Principal{KeyID: keyID, ProjectID: "proj", Limits: e} +} + +func TestATeamBudgetIsOnePoolRatherThanOnePerMember(t *testing.T) { + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + budget := func(e *store.Effective) { + e.Budget = i64p(3000) + e.BudgetPeriod = store.PeriodMonth + } + first := teamMember("k1", "t1", budget) + second := teamMember("k2", "t1", budget) + + // One member spends the lot. + res, err := a.Check(first, "paid", 1000) + if err != nil { + t.Fatal(err) + } + a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 1000, CompletionTokens: 1000}) + + // The second must now be refused. Counted per key it would have its own + // untouched 3000 — a team budget of 3000 with ten keys would permit thirty + // thousand, which is the bug this change exists for. + if _, err := a.Check(second, "paid", 10); !errors.Is(err, ErrBudgetSpent) { + t.Errorf("the team's budget should be shared; the second member got %v", err) + } +} + +func TestATeamRateLimitIsSharedToo(t *testing.T) { + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + rpm := func(e *store.Effective) { e.RPM = intp(1) } + + if _, err := a.Check(teamMember("k1", "t1", rpm), "free", 1); err != nil { + t.Fatal(err) + } + if _, err := a.Check(teamMember("k2", "t1", rpm), "free", 1); !errors.Is(err, ErrRateLimited) { + t.Error("a rate limit set on the team should be shared by its keys") + } +} + +func TestTwoTeamsDoNotShareACounter(t *testing.T) { + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + rpm := func(e *store.Effective) { e.RPM = intp(1) } + + if _, err := a.Check(teamMember("k1", "t1", rpm), "free", 1); err != nil { + t.Fatal(err) + } + // Pooling is the point; pooling the wrong things is the danger. One team + // exhausting another's quota would be worse than the bug being fixed. + if _, err := a.Check(teamMember("k2", "t2", rpm), "free", 1); err != nil { + t.Errorf("a second team should have its own counter, got %v", err) + } +} + +func TestAKeyLimitStaysPerKey(t *testing.T) { + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + perKey := func(id string) auth.Principal { + return principal(id, func(e *store.Effective) { + e.RPM = intp(1) + e.RPMOwner = store.LimitOwner{Scope: store.ScopeKey, ID: id} + }) + } + + if _, err := a.Check(perKey("k1"), "free", 1); err != nil { + t.Fatal(err) + } + // The tree still has to be able to say "each of you may do this much". + // Sharing a group pool must not become the only thing expressible. + if _, err := a.Check(perKey("k2"), "free", 1); err != nil { + t.Errorf("a limit owned by the key should not be shared, got %v", err) + } +} + +func TestTheDifferentLimitsCanBeOwnedByDifferentScopes(t *testing.T) { + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + // A team budget with a per-key rate limit is the ordinary arrangement: the + // group's money, the individual's throughput. + mk := func(id string) auth.Principal { + e := store.Effective{KeyID: id, TeamID: "t1"} + e.Budget, e.BudgetPeriod = i64p(3000), store.PeriodMonth + e.BudgetOwner = store.LimitOwner{Scope: store.ScopeTeam, ID: "t1"} + e.RPM = intp(1) + e.RPMOwner = store.LimitOwner{Scope: store.ScopeKey, ID: id} + return auth.Principal{KeyID: id, Limits: e} + } + + res, err := a.Check(mk("k1"), "paid", 1000) + if err != nil { + t.Fatal(err) + } + a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 1000, CompletionTokens: 1000}) + + // k2's rate limit is its own and untouched; the budget is the team's and + // gone. The budget is checked first, so that is what comes back. + if _, err := a.Check(mk("k2"), "paid", 10); !errors.Is(err, ErrBudgetSpent) { + t.Errorf("expected the shared budget to bite, got %v", err) + } +} + +func TestStatusReportsTheCounterThatWillRefuseYou(t *testing.T) { + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + budget := func(e *store.Effective) { + e.Budget = i64p(9000) + e.BudgetPeriod = store.PeriodMonth + } + res, err := a.Check(teamMember("k1", "t1", budget), "paid", 1000) + if err != nil { + t.Fatal(err) + } + a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 1000, CompletionTokens: 1000}) + + // A member who has spent nothing themselves still sees the team's spend, + // because that is the number that will stop them. A header disagreeing with + // the refusal it precedes is worse than no header. + spent, _, _ := a.Status(teamMember("k2", "t1", budget)) + if spent == 0 { + t.Error("a member should see the pool's spend, not their own") + } +} + +func TestATeamBudgetSurvivesARestart(t *testing.T) { + // The per-key restore could not do this: it rebuilt a counter named after + // the key, while the check reads one named after the team. The control + // plane came back believing the team had spent nothing — a budget that + // resets on every restart, for exactly the scope where it matters most. + ctx := context.Background() + s, err := store.OpenSQLite(ctx, filepath.Join(t.TempDir(), "team.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + + must(t, s.PutOrg(ctx, store.Org{ID: "o", Name: "o"})) + must(t, s.PutTeam(ctx, store.Team{ID: "t1", OrgID: "o", Name: "t"})) + must(t, s.PutProject(ctx, store.Project{ID: "proj", TeamID: "t1", Name: "p"})) + for _, id := range []string{"k1", "k2"} { + _, k, _ := store.NewKey(id, "n", "proj") + k.CreatedAt = time.Now() + must(t, s.CreateKey(ctx, k)) + } + + // Two members of one team, 2500 each: neither alone exceeds a 4000 budget, + // together they do. + must(t, s.WriteUsage(ctx, []store.Usage{ + {At: time.Now(), KeyID: "k1", ProjectID: "proj", Model: "paid", + ServedModel: "paid", CostMicros: 2500, Status: 200}, + {At: time.Now(), KeyID: "k2", ProjectID: "proj", Model: "paid", + ServedModel: "paid", CostMicros: 2500, Status: 200}, + })) + + a := New(pricing(t), nil, slog.New(slog.DiscardHandler)) + must(t, a.RestoreBudgets(ctx, s, store.PeriodMonth)) + + p := teamMember("k1", "t1", func(e *store.Effective) { + e.Budget = i64p(4000) + e.BudgetPeriod = store.PeriodMonth + }) + if _, err := a.Check(p, "paid", 10); !errors.Is(err, ErrBudgetSpent) { + t.Errorf("the team's spend should have been restored and pooled: %v", err) + } +} diff --git a/internal/store/limits_owner_test.go b/internal/store/limits_owner_test.go new file mode 100644 index 0000000..e14a711 --- /dev/null +++ b/internal/store/limits_owner_test.go @@ -0,0 +1,147 @@ +package store + +import ( + "context" + "path/filepath" + "testing" + "time" +) + +func treeStore(t *testing.T) *SQLite { + t.Helper() + ctx := context.Background() + s, err := OpenSQLite(ctx, filepath.Join(t.TempDir(), "t.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + + for _, err := range []error{ + s.PutOrg(ctx, Org{ID: "o", Name: "o"}), + s.PutTeam(ctx, Team{ID: "t", OrgID: "o", Name: "t"}), + s.PutProject(ctx, Project{ID: "p", TeamID: "t", Name: "p"}), + } { + if err != nil { + t.Fatal(err) + } + } + _, k, _ := NewKey("k1", "n", "p") + k.CreatedAt = time.Now() + if err := s.CreateKey(ctx, k); err != nil { + t.Fatal(err) + } + return s +} + +func TestAResolvedLimitRemembersWhichScopeSetIt(t *testing.T) { + s := treeStore(t) + rpm := 60 + if err := s.PutLimits(t.Context(), Limits{Scope: ScopeTeam, ScopeID: "t", RPM: &rpm}); err != nil { + t.Fatal(err) + } + + e, err := s.Resolve(t.Context(), "k1") + if err != nil { + t.Fatal(err) + } + // Without this the tree hands down values and never pools spending: the + // team's 60 becomes 60 for each key under it. + if e.RPMOwner.Scope != ScopeTeam || e.RPMOwner.ID != "t" { + t.Errorf("expected the team to own the rate limit, got %+v", e.RPMOwner) + } + if e.RPMOwner.CounterKey() != "team:t" { + t.Errorf("counter key should name the level too; got %q", e.RPMOwner.CounterKey()) + } +} + +func TestTheTighterScopeTakesOwnershipWithTheValue(t *testing.T) { + s := treeStore(t) + loose, tight := 100, 10 + must := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + must(s.PutLimits(t.Context(), Limits{Scope: ScopeTeam, ScopeID: "t", RPM: &loose})) + must(s.PutLimits(t.Context(), Limits{Scope: ScopeProject, ScopeID: "p", RPM: &tight})) + + e, err := s.Resolve(t.Context(), "k1") + if err != nil { + t.Fatal(err) + } + // The value and the counter have to come from the same place. Enforcing + // the project's 10 against the team's pool would let one project's traffic + // exhaust a limit another project never agreed to. + if *e.RPM != 10 || e.RPMOwner.ID != "p" { + t.Errorf("expected the project's value and ownership, got %d from %+v", *e.RPM, e.RPMOwner) + } +} + +func TestALooserInnerScopeDoesNotTakeOwnership(t *testing.T) { + s := treeStore(t) + tight, loose := 10, 100 + if err := s.PutLimits(t.Context(), Limits{Scope: ScopeTeam, ScopeID: "t", RPM: &tight}); err != nil { + t.Fatal(err) + } + if err := s.PutLimits(t.Context(), Limits{Scope: ScopeProject, ScopeID: "p", RPM: &loose}); err != nil { + t.Fatal(err) + } + + e, err := s.Resolve(t.Context(), "k1") + if err != nil { + t.Fatal(err) + } + // A project cannot raise what its team set, so the team still owns both the + // number and the counter it is enforced against. + if *e.RPM != 10 || e.RPMOwner.ID != "t" { + t.Errorf("expected the team to keep ownership, got %d from %+v", *e.RPM, e.RPMOwner) + } +} + +func TestAClusterDefaultIsOwnedByTheKey(t *testing.T) { + s := treeStore(t) + rpm := 30 + s.SetDefaultLimits(&rpm, nil) + + e, err := s.Resolve(t.Context(), "k1") + if err != nil { + t.Fatal(err) + } + // A default is a fallback for each caller given no limit of its own. Owned + // by a shared scope it would become one cluster-wide rate limit that every + // key in every organization competes for — not a default, an outage. + if e.RPMOwner.Scope != ScopeKey || e.RPMOwner.ID != "k1" { + t.Errorf("a default should be counted per key, got %+v", e.RPMOwner) + } +} + +func TestSpendComesBackWithThePlaceInTheTree(t *testing.T) { + s := treeStore(t) + if err := s.WriteUsage(t.Context(), []Usage{ + {At: time.Now(), KeyID: "k1", ProjectID: "p", Model: "m", + ServedModel: "m", CostMicros: 700, Status: 200}, + {At: time.Now(), KeyID: "k1", ProjectID: "p", Model: "m", + ServedModel: "m", CostMicros: 300, Status: 200}, + }); err != nil { + t.Fatal(err) + } + + rows, err := s.SpendSinceByScope(t.Context(), time.Now().Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 { + t.Fatalf("expected one key's total, got %+v", rows) + } + // The tree, not just the key: a budget owned by a team is rebuilt by + // summing every key under it, and per-key totals alone cannot say which + // keys those are. + got := rows[0] + if got.Micros != 1000 { + t.Errorf("expected the two rows summed, got %d", got.Micros) + } + if got.ProjectID != "p" || got.TeamID != "t" || got.OrgID != "o" { + t.Errorf("the whole chain should come back, got %+v", got) + } +} diff --git a/internal/store/sqlite.go b/internal/store/sqlite.go index 2ac85a8..ed2cf49 100644 --- a/internal/store/sqlite.go +++ b/internal/store/sqlite.go @@ -336,26 +336,73 @@ func (s *SQLite) SetDefaultLimits(rpm, tpm *int) { s.defaultRPM, s.defaultTPM = rpm, tpm } +// applyDefaults fills in what the chain left unset. +// +// Owned by the key, not by any scope. A cluster-wide default is a fallback for +// each caller that was given no limit of its own — read as a shared pool it +// would become one cluster-wide rate limit that every key in every +// organization competes for, which is not a default, it is an outage. func (s *SQLite) applyDefaults(e *Effective) { s.mu.RLock() defer s.mu.RUnlock() + own := LimitOwner{Scope: ScopeKey, ID: e.KeyID} if e.RPM == nil && s.defaultRPM != nil { v := *s.defaultRPM - e.RPM = &v + e.RPM, e.RPMOwner = &v, own } if e.TPM == nil && s.defaultTPM != nil { v := *s.defaultTPM - e.TPM = &v + e.TPM, e.TPMOwner = &v, own } } +// SpendSinceByScope totals cost per key, with the key's place in the tree. +func (s *SQLite) SpendSinceByScope(ctx context.Context, since time.Time) ([]ScopeSpend, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT k.id, p.id, t.id, o.id, SUM(u.cost_micros) + FROM usage u + JOIN api_keys k ON k.id = u.key_id + JOIN projects p ON p.id = k.project_id + JOIN teams t ON t.id = p.team_id + JOIN organizations o ON o.id = t.org_id + WHERE u.ts >= ? + GROUP BY k.id, p.id, t.id, o.id`, millis(since)) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []ScopeSpend + for rows.Next() { + var sp ScopeSpend + var total sql.NullInt64 + if err := rows.Scan(&sp.KeyID, &sp.ProjectID, &sp.TeamID, &sp.OrgID, &total); err != nil { + return nil, err + } + sp.Micros = total.Int64 + out = append(out, sp) + } + return out, rows.Err() +} + // tighten applies one scope's limits, keeping whichever is stricter. +// +// Each value that wins records the scope it came from. That is what lets +// consumption be counted against the scope that set the limit: a budget written +// on a team is one pool the team shares, not a copy of the number handed to +// every key beneath it. func (e *Effective) tighten(l Limits) { - e.RPM = minPtr(e.RPM, l.RPM) - e.TPM = minPtr(e.TPM, l.TPM) + owner := LimitOwner{Scope: l.Scope, ID: l.ScopeID} + if next := minPtr(e.RPM, l.RPM); next != e.RPM { + e.RPM, e.RPMOwner = next, owner + } + if next := minPtr(e.TPM, l.TPM); next != e.TPM { + e.TPM, e.TPMOwner = next, owner + } if l.Budget != nil && (e.Budget == nil || *l.Budget < *e.Budget) { e.Budget = l.Budget e.BudgetPeriod = l.BudgetPeriod + e.BudgetOwner = owner } if l.AllowedModels != nil { if e.AllowedModels == nil { diff --git a/internal/store/store.go b/internal/store/store.go index 559a0d7..57ef138 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -128,6 +128,41 @@ type Effective struct { BudgetPeriod Period // AllowedModels is the intersection along the chain. nil means every model. AllowedModels []string + + // RPMOwner, TPMOwner and BudgetOwner say which scope set the value that + // won, so consumption can be counted against that scope rather than + // against each key under it. + // + // Without this the tree hands down values but never pools spending: a + // project given a budget of 1000 and ten keys permits ten thousand, because + // each key gets its own counter holding the project's number. That is "each + // member may spend 1000", which is a useful thing to be able to say and is + // not what somebody writing a budget on a project means. + // + // Each limit is owned separately because they resolve separately: a budget + // can come from the team while the rate limit comes from the key. + RPMOwner LimitOwner + TPMOwner LimitOwner + BudgetOwner LimitOwner +} + +// LimitOwner is the scope a resolved limit came from. +type LimitOwner struct { + Scope Scope + ID string +} + +// CounterKey is what a usage counter for this limit should be keyed by. +// +// Empty when no scope set the limit, which means nothing is being counted and +// the caller should not be asking. Scope and id together, because ids are only +// unique within a level and a project could otherwise share a counter with a +// team that happened to be named the same. +func (o LimitOwner) CounterKey() string { + if o.ID == "" { + return "" + } + return string(o.Scope) + ":" + o.ID } // Allows reports whether a model may be served to this key. @@ -193,6 +228,11 @@ type Store interface { // counter can be rebuilt after a restart. A restart must not reset a // budget, which it would if the counter lived only in memory. SpendSince(ctx context.Context, since time.Time) (map[string]int64, error) + // SpendSinceByScope is the same totals with each key's place in the tree, + // so a budget owned by a team can be rebuilt by summing every key under it. + // Per key alone cannot do that: the counter is one number for the whole + // team, and the rows that made it up are spread across its projects. + SpendSinceByScope(ctx context.Context, since time.Time) ([]ScopeSpend, error) // UsageSince summarizes activity per key. A reporting query: it scans and // groups, so it belongs behind a dashboard refresh rather than a request. UsageSince(ctx context.Context, since time.Time) ([]KeyUsage, error) @@ -228,3 +268,17 @@ type Bucket struct { // an SLO measurement. LatencyMs int64 } + +// ScopeSpend is one key's spend in a period, with where that key sits in the +// tenancy tree. +// +// Returned rather than pre-aggregated per scope because the roll-up depends on +// which scopes own budgets, which the store does not know and the accountant +// does. One query, and the folding happens where it can be tested. +type ScopeSpend struct { + KeyID string + ProjectID string + TeamID string + OrgID string + Micros int64 +}