A cached prompt costs the cached rate
cached_per_1k in a price charges the prompt tokens the answer counts as cached_tokens at their own rate; zero keeps charging them as prompt tokens. The count reaches the cost and not a column: CostMicros holds what it changed. Cached beyond the prompt is clamped, an engine's miscount rather than a refund. BACKLOG L5. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
73e1bb6733
commit
ed0d1ec0ad
12 files changed
+85
-11
No files matched your search
@@ -323,7 +323,7 @@ func (a *Accountant) Record(res Reservation, u store.Usage) {
|
||||
if u.ServedModel != "" && u.ServedModel != u.Model && a.pricing != nil {
|
||||
price = a.pricing.For(u.ServedModel)
|
||||
}
|
||||
u.CostMicros = price.Cost(u.PromptTokens, u.CompletionTokens, u.Images, u.Meshes)
|
||||
u.CostMicros = price.Cost(u.PromptTokens, u.CachedTokens, u.CompletionTokens, u.Images, u.Meshes)
|
||||
for _, b := range res.Budgets {
|
||||
a.budgets.Add(b.Counter, b.Period, u.CostMicros)
|
||||
}
|
||||
|
||||
@@ -39,20 +39,37 @@ func TestCostRoundsUpSoSmallRequestsAreNotFree(t *testing.T) {
|
||||
// Rounding down would make a stream of tiny requests cost nothing, which is
|
||||
// exactly the traffic shape a quota exists to bound.
|
||||
p := Price{PromptPer1k: 1000, CompletionPer1k: 3000}
|
||||
if got := p.Cost(1, 0, 0, 0); got != 1 {
|
||||
if got := p.Cost(1, 0, 0, 0, 0); got != 1 {
|
||||
t.Errorf("one prompt token cost %d", got)
|
||||
}
|
||||
if got := p.Cost(1000, 1000, 0, 0); got != 4000 {
|
||||
if got := p.Cost(1000, 0, 1000, 0, 0); got != 4000 {
|
||||
t.Errorf("a thousand of each cost %d, want 4000", got)
|
||||
}
|
||||
if got := p.Cost(0, 0, 0, 0); got != 0 {
|
||||
if got := p.Cost(0, 0, 0, 0, 0); got != 0 {
|
||||
t.Errorf("an empty request cost %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACachedPromptCostsTheCachedRate(t *testing.T) {
|
||||
p := Price{PromptPer1k: 1000, CachedPer1k: 100, CompletionPer1k: 3000}
|
||||
// 10,000 prompt tokens of which 9,000 were read from the cache: 1,000 at
|
||||
// the prompt rate and 9,000 at the cached one.
|
||||
if got, want := p.Cost(10000, 9000, 0, 0, 0), int64(1000+900); got != want {
|
||||
t.Errorf("a 90%% cached prompt cost %d, want %d", got, want)
|
||||
}
|
||||
// More cached than prompt is an engine's miscount, not a refund.
|
||||
if got := p.Cost(100, 500, 0, 0, 0); got != 10 {
|
||||
t.Errorf("cached beyond the prompt cost %d, want 10", got)
|
||||
}
|
||||
// No cached rate: cached tokens cost what any prompt token does.
|
||||
if got := (Price{PromptPer1k: 1000}).Cost(10000, 9000, 0, 0, 0); got != 10000 {
|
||||
t.Errorf("without a cached rate a cached prompt cost %d, want 10000", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPictureCostsItsPriceOnTopOfItsPrompt(t *testing.T) {
|
||||
p := Price{PromptPer1k: 1000, PerImage: 50000}
|
||||
if got := p.Cost(20, 0, 3, 0); got != 20+150000 {
|
||||
if got := p.Cost(20, 0, 0, 3, 0); got != 20+150000 {
|
||||
t.Errorf("three pictures from a 20-token prompt cost %d", got)
|
||||
}
|
||||
if (Price{PerImage: 1}).Zero() {
|
||||
|
||||
@@ -18,6 +18,11 @@ import (
|
||||
type Price struct {
|
||||
PromptPer1k int64 `koanf:"prompt_per_1k"`
|
||||
CompletionPer1k int64 `koanf:"completion_per_1k"`
|
||||
// CachedPer1k is what a prompt token costs that the engine or provider
|
||||
// read from its cache instead of computing: the part of prompt_tokens
|
||||
// the answer counts as cached_tokens. Zero charges those at
|
||||
// PromptPer1k, as before there was a rate for them.
|
||||
CachedPer1k int64 `koanf:"cached_per_1k"`
|
||||
// PerImage is what one picture costs, for a model that makes them: a
|
||||
// picture's cost is the card's time, which its prompt's few tokens do
|
||||
// not measure.
|
||||
@@ -33,15 +38,24 @@ type Price struct {
|
||||
// whose budget is spent, which is what makes the plan's budget fallback work
|
||||
// (§7.7).
|
||||
func (p Price) Zero() bool {
|
||||
return p.PromptPer1k == 0 && p.CompletionPer1k == 0 && p.PerImage == 0 && p.PerMesh == 0
|
||||
return p.PromptPer1k == 0 && p.CompletionPer1k == 0 && p.CachedPer1k == 0 && p.PerImage == 0 && p.PerMesh == 0
|
||||
}
|
||||
|
||||
// Cost computes what a request spent, rounding up.
|
||||
//
|
||||
// Up rather than to nearest: rounding down would make a stream of tiny requests
|
||||
// free, which is exactly the shape of traffic a quota exists to bound.
|
||||
func (p Price) Cost(promptTokens, completionTokens, images, meshes int64) int64 {
|
||||
return divUp(promptTokens*p.PromptPer1k, 1000) + divUp(completionTokens*p.CompletionPer1k, 1000) +
|
||||
//
|
||||
// cachedTokens are counted within promptTokens, as OpenAI's usage counts
|
||||
// them, and are charged at the cached rate instead of the prompt rate.
|
||||
func (p Price) Cost(promptTokens, cachedTokens, completionTokens, images, meshes int64) int64 {
|
||||
cachedTokens = min(max(cachedTokens, 0), max(promptTokens, 0))
|
||||
cachedRate := p.CachedPer1k
|
||||
if cachedRate == 0 {
|
||||
cachedRate = p.PromptPer1k
|
||||
}
|
||||
return divUp((promptTokens-cachedTokens)*p.PromptPer1k+cachedTokens*cachedRate, 1000) +
|
||||
divUp(completionTokens*p.CompletionPer1k, 1000) +
|
||||
max(images, 0)*p.PerImage + max(meshes, 0)*p.PerMesh
|
||||
}
|
||||
|
||||
|
||||
@@ -503,6 +503,10 @@ type Personal struct {
|
||||
type Price struct {
|
||||
PromptPer1k int64 `koanf:"prompt_per_1k"`
|
||||
CompletionPer1k int64 `koanf:"completion_per_1k"`
|
||||
// CachedPer1k is the cost of a prompt token the engine or provider read
|
||||
// from its cache (the answer's cached_tokens, counted within the
|
||||
// prompt's). Zero charges them at prompt_per_1k.
|
||||
CachedPer1k int64 `koanf:"cached_per_1k"`
|
||||
// PerImage is the cost of one picture, for a model that makes them
|
||||
// (/v1/images/generations), on top of its prompt's tokens.
|
||||
PerImage int64 `koanf:"per_image"`
|
||||
|
||||
@@ -550,6 +550,7 @@ var Reference = []Block{
|
||||
{Package: "accounting", Name: "Price", Title: "accounting.Price", Doc: "", Fields: []Field{
|
||||
{Key: "prompt_per_1k", Type: "int64", Doc: "", Nested: ""},
|
||||
{Key: "completion_per_1k", Type: "int64", Doc: "", Nested: ""},
|
||||
{Key: "cached_per_1k", Type: "int64", Doc: "CachedPer1k is what a prompt token costs that the engine or provider\nread from its cache instead of computing: the part of prompt_tokens\nthe answer counts as cached_tokens. Zero charges those at\nPromptPer1k, as before there was a rate for them.", Nested: ""},
|
||||
{Key: "per_image", Type: "int64", Doc: "PerImage is what one picture costs, for a model that makes them: a\npicture's cost is the card's time, which its prompt's few tokens do\nnot measure.", Nested: ""},
|
||||
{Key: "per_mesh", Type: "int64", Doc: "PerMesh is what one 3D model costs: minutes of a card, which no\ntoken count measures.", Nested: ""},
|
||||
}},
|
||||
@@ -598,6 +599,7 @@ var Reference = []Block{
|
||||
{Package: "config", Name: "Price", Title: "config.Price", Doc: "", Fields: []Field{
|
||||
{Key: "prompt_per_1k", Type: "int64", Doc: "", Nested: ""},
|
||||
{Key: "completion_per_1k", Type: "int64", Doc: "", Nested: ""},
|
||||
{Key: "cached_per_1k", Type: "int64", Doc: "CachedPer1k is the cost of a prompt token the engine or provider read\nfrom its cache (the answer's cached_tokens, counted within the\nprompt's). Zero charges them at prompt_per_1k.", Nested: ""},
|
||||
{Key: "per_image", Type: "int64", Doc: "PerImage is the cost of one picture, for a model that makes them\n(/v1/images/generations), on top of its prompt's tokens.", Nested: ""},
|
||||
{Key: "per_mesh", Type: "int64", Doc: "PerMesh is the cost of one 3D model (/v1/3d/generations).", Nested: ""},
|
||||
}},
|
||||
|
||||
@@ -825,6 +825,7 @@ func (r *Router) account(req *parsedRequest, res dispatchResult, status int, too
|
||||
if usage := res.outcome.Usage; usage != nil {
|
||||
u.PromptTokens = int64(usage.PromptTokens)
|
||||
u.CompletionTokens = int64(usage.CompletionTokens)
|
||||
u.CachedTokens = int64(usage.CachedTokens)
|
||||
} else {
|
||||
// No usage came back — a failure before the engine answered, or an
|
||||
// engine that does not report it. Charging the estimate is closer to
|
||||
|
||||
@@ -400,7 +400,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) {
|
||||
prices := make(map[string]accounting.Price, len(cfg.Auth.Pricing))
|
||||
for name, p := range cfg.Auth.Pricing {
|
||||
prices[name] = accounting.Price{
|
||||
PromptPer1k: p.PromptPer1k, CompletionPer1k: p.CompletionPer1k, PerImage: p.PerImage, PerMesh: p.PerMesh,
|
||||
PromptPer1k: p.PromptPer1k, CompletionPer1k: p.CompletionPer1k, CachedPer1k: p.CachedPer1k, PerImage: p.PerImage, PerMesh: p.PerMesh,
|
||||
}
|
||||
}
|
||||
r.recorder = accounting.NewRecorder(st, r.auth, accounting.RecorderConfig{
|
||||
@@ -430,6 +430,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) {
|
||||
accounting.NewPricing(prices, accounting.Price{
|
||||
PromptPer1k: cfg.Auth.DefaultPrice.PromptPer1k,
|
||||
CompletionPer1k: cfg.Auth.DefaultPrice.CompletionPer1k,
|
||||
CachedPer1k: cfg.Auth.DefaultPrice.CachedPer1k,
|
||||
PerImage: cfg.Auth.DefaultPrice.PerImage,
|
||||
PerMesh: cfg.Auth.DefaultPrice.PerMesh,
|
||||
}),
|
||||
|
||||
@@ -277,6 +277,10 @@ type Usage struct {
|
||||
ServedModel string
|
||||
PromptTokens int64
|
||||
CompletionTokens int64
|
||||
// CachedTokens is how many of PromptTokens the engine read from its
|
||||
// cache, for the price of those. Not a column: what it changed is in
|
||||
// CostMicros.
|
||||
CachedTokens int64
|
||||
// Images is how many pictures were made, for a request for them.
|
||||
Images int64
|
||||
// Meshes is how many 3D models were made.
|
||||
|
||||
Reference in new issue
Block a user