From ed0d1ec0ad3bcdb24eeeeb0c829f669726a010a9 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 25 Oct 2026 17:15:48 +0100 Subject: [PATCH] A cached prompt costs the cached rate cached_per_1k in a price charges the prompt tokens the answer counts as cached_tokens at their own rate; zero keeps charging them as prompt tokens. The count reaches the cost and not a column: CostMicros holds what it changed. Cached beyond the prompt is clamped, an engine's miscount rather than a refund. BACKLOG L5. Co-Authored-By: Claude Opus 5.5 --- BACKLOG.md | 6 +++++- CHANGELOG.md | 4 ++++ docs/install.md | 6 +++++- docs/reference.md | 19 +++++++++++++++++++ internal/accounting/accounting.go | 2 +- internal/accounting/accounting_test.go | 25 +++++++++++++++++++++---- internal/accounting/cost.go | 20 +++++++++++++++++--- internal/config/config.go | 4 ++++ internal/docgen/reference_gen.go | 2 ++ internal/router/handlers.go | 1 + internal/router/router.go | 3 ++- internal/store/store.go | 4 ++++ 12 files changed, 85 insertions(+), 11 deletions(-) diff --git a/BACKLOG.md b/BACKLOG.md index 3a17b73..cd66be8 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -1741,7 +1741,11 @@ used, and in priority tiers. - Done when: with two replicas of one model, one on a slower card, `latency` sends most of the traffic to the faster and the slower still gets some. -### L5. Prices by kind and by cache (S–M) +### L5. Prices by kind and by cache (S–M) — done 2026-10-25 +`cached_per_1k`, charged on the answer's `cached_tokens`. A budget in money was already +there (`keys limits --budget`). Not done: cache-write and audio/video token prices — +no engine llmgw runs reports them, and no provider it reaches is billed by them yet. +The original item follows. `catalog` prices are per million input/output tokens and per picture/mesh. Kong prices cached reads and cache writes apart, and audio, image and video tokens apart, and limits by cost. diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cf3dab..58462a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,10 @@ APIs change between minors without a deprecation period. ### Added +- **A cached prompt costs the cached rate.** `cached_per_1k` in a price + charges the prompt tokens an engine or provider read from its cache (the + answer's `cached_tokens`) at their own rate; without it they cost what any + prompt token does, as before. - **Chat goes to the replica answering soonest.** The router keeps each replica's time to first token, smoothed over its last answers, and the `chat` profile scores by it beside the queue and the KV cache: of two diff --git a/docs/install.md b/docs/install.md index d996f55..b43eb6c 100644 --- a/docs/install.md +++ b/docs/install.md @@ -391,7 +391,11 @@ by itself. Three things are worth knowing before you edit it: - `default_rpm`, `default_tpm` and `default_price` are what make limits and budgets mean anything. Without a price every request costs zero and a budget never binds; without the defaults, a project created without limits is - metered by nothing at all. + metered by nothing at all. A price may set `cached_per_1k` beside + `prompt_per_1k`: the prompt tokens the engine or provider read from its + cache (the answer's `cached_tokens`) are charged at it, the rest at the + prompt rate. A provider that bills cached reads at a tenth is + `{prompt_per_1k: 2500, cached_per_1k: 250, completion_per_1k: 10000}`. - `control.enabled: true` is what makes this a control plane while its catalog is empty. Without it, a file with no models is refused as having nothing to serve. diff --git a/docs/reference.md b/docs/reference.md index f91ab53..359d7ca 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -424,6 +424,7 @@ rather than repeated. | `secret_file` | `string` | [Webhook](#webhook) | | `prompt_per_1k` | `int64` | [accounting.Price](#accountingprice) | | `completion_per_1k` | `int64` | [accounting.Price](#accountingprice) | +| `cached_per_1k` | `int64` | [accounting.Price](#accountingprice) | | `per_image` | `int64` | [accounting.Price](#accountingprice) | | `per_mesh` | `int64` | [accounting.Price](#accountingprice) | | `name` | `string` | [admission.Band](#admissionband) | @@ -462,6 +463,7 @@ rather than repeated. | `queue_ttl` | `time.Duration` | [config.Band](#configband) | | `prompt_per_1k` | `int64` | [config.Price](#configprice) | | `completion_per_1k` | `int64` | [config.Price](#configprice) | +| `cached_per_1k` | `int64` | [config.Price](#configprice) | | `per_image` | `int64` | [config.Price](#configprice) | | `per_mesh` | `int64` | [config.Price](#configprice) | | `disabled` | `[]string` | [engines.Config](#enginesconfig) | @@ -3600,6 +3602,15 @@ is sha256=. `int64` +### `cached_per_1k` + +`int64` + +CachedPer1k is what a prompt token costs that the engine or provider +read from its cache instead of computing: the part of prompt_tokens +the answer counts as cached_tokens. Zero charges those at +PromptPer1k, as before there was a rate for them. + ### `per_image` `int64` @@ -3874,6 +3885,14 @@ had rather wait behind people than be turned away. `int64` +### `cached_per_1k` + +`int64` + +CachedPer1k is the cost of a prompt token the engine or provider read +from its cache (the answer's cached_tokens, counted within the +prompt's). Zero charges them at prompt_per_1k. + ### `per_image` `int64` diff --git a/internal/accounting/accounting.go b/internal/accounting/accounting.go index c510852..2c26add 100644 --- a/internal/accounting/accounting.go +++ b/internal/accounting/accounting.go @@ -323,7 +323,7 @@ func (a *Accountant) Record(res Reservation, u store.Usage) { if u.ServedModel != "" && u.ServedModel != u.Model && a.pricing != nil { price = a.pricing.For(u.ServedModel) } - u.CostMicros = price.Cost(u.PromptTokens, u.CompletionTokens, u.Images, u.Meshes) + u.CostMicros = price.Cost(u.PromptTokens, u.CachedTokens, u.CompletionTokens, u.Images, u.Meshes) for _, b := range res.Budgets { a.budgets.Add(b.Counter, b.Period, u.CostMicros) } diff --git a/internal/accounting/accounting_test.go b/internal/accounting/accounting_test.go index e00b909..ceb96f5 100644 --- a/internal/accounting/accounting_test.go +++ b/internal/accounting/accounting_test.go @@ -39,20 +39,37 @@ func TestCostRoundsUpSoSmallRequestsAreNotFree(t *testing.T) { // Rounding down would make a stream of tiny requests cost nothing, which is // exactly the traffic shape a quota exists to bound. p := Price{PromptPer1k: 1000, CompletionPer1k: 3000} - if got := p.Cost(1, 0, 0, 0); got != 1 { + if got := p.Cost(1, 0, 0, 0, 0); got != 1 { t.Errorf("one prompt token cost %d", got) } - if got := p.Cost(1000, 1000, 0, 0); got != 4000 { + if got := p.Cost(1000, 0, 1000, 0, 0); got != 4000 { t.Errorf("a thousand of each cost %d, want 4000", got) } - if got := p.Cost(0, 0, 0, 0); got != 0 { + if got := p.Cost(0, 0, 0, 0, 0); got != 0 { t.Errorf("an empty request cost %d", got) } } +func TestACachedPromptCostsTheCachedRate(t *testing.T) { + p := Price{PromptPer1k: 1000, CachedPer1k: 100, CompletionPer1k: 3000} + // 10,000 prompt tokens of which 9,000 were read from the cache: 1,000 at + // the prompt rate and 9,000 at the cached one. + if got, want := p.Cost(10000, 9000, 0, 0, 0), int64(1000+900); got != want { + t.Errorf("a 90%% cached prompt cost %d, want %d", got, want) + } + // More cached than prompt is an engine's miscount, not a refund. + if got := p.Cost(100, 500, 0, 0, 0); got != 10 { + t.Errorf("cached beyond the prompt cost %d, want 10", got) + } + // No cached rate: cached tokens cost what any prompt token does. + if got := (Price{PromptPer1k: 1000}).Cost(10000, 9000, 0, 0, 0); got != 10000 { + t.Errorf("without a cached rate a cached prompt cost %d, want 10000", got) + } +} + func TestAPictureCostsItsPriceOnTopOfItsPrompt(t *testing.T) { p := Price{PromptPer1k: 1000, PerImage: 50000} - if got := p.Cost(20, 0, 3, 0); got != 20+150000 { + if got := p.Cost(20, 0, 0, 3, 0); got != 20+150000 { t.Errorf("three pictures from a 20-token prompt cost %d", got) } if (Price{PerImage: 1}).Zero() { diff --git a/internal/accounting/cost.go b/internal/accounting/cost.go index f7797fd..9ab047b 100644 --- a/internal/accounting/cost.go +++ b/internal/accounting/cost.go @@ -18,6 +18,11 @@ import ( type Price struct { PromptPer1k int64 `koanf:"prompt_per_1k"` CompletionPer1k int64 `koanf:"completion_per_1k"` + // CachedPer1k is what a prompt token costs that the engine or provider + // read from its cache instead of computing: the part of prompt_tokens + // the answer counts as cached_tokens. Zero charges those at + // PromptPer1k, as before there was a rate for them. + CachedPer1k int64 `koanf:"cached_per_1k"` // PerImage is what one picture costs, for a model that makes them: a // picture's cost is the card's time, which its prompt's few tokens do // not measure. @@ -33,15 +38,24 @@ type Price struct { // whose budget is spent, which is what makes the plan's budget fallback work // (§7.7). func (p Price) Zero() bool { - return p.PromptPer1k == 0 && p.CompletionPer1k == 0 && p.PerImage == 0 && p.PerMesh == 0 + return p.PromptPer1k == 0 && p.CompletionPer1k == 0 && p.CachedPer1k == 0 && p.PerImage == 0 && p.PerMesh == 0 } // Cost computes what a request spent, rounding up. // // Up rather than to nearest: rounding down would make a stream of tiny requests // free, which is exactly the shape of traffic a quota exists to bound. -func (p Price) Cost(promptTokens, completionTokens, images, meshes int64) int64 { - return divUp(promptTokens*p.PromptPer1k, 1000) + divUp(completionTokens*p.CompletionPer1k, 1000) + +// +// cachedTokens are counted within promptTokens, as OpenAI's usage counts +// them, and are charged at the cached rate instead of the prompt rate. +func (p Price) Cost(promptTokens, cachedTokens, completionTokens, images, meshes int64) int64 { + cachedTokens = min(max(cachedTokens, 0), max(promptTokens, 0)) + cachedRate := p.CachedPer1k + if cachedRate == 0 { + cachedRate = p.PromptPer1k + } + return divUp((promptTokens-cachedTokens)*p.PromptPer1k+cachedTokens*cachedRate, 1000) + + divUp(completionTokens*p.CompletionPer1k, 1000) + max(images, 0)*p.PerImage + max(meshes, 0)*p.PerMesh } diff --git a/internal/config/config.go b/internal/config/config.go index 9662cd7..b21ea99 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -503,6 +503,10 @@ type Personal struct { type Price struct { PromptPer1k int64 `koanf:"prompt_per_1k"` CompletionPer1k int64 `koanf:"completion_per_1k"` + // CachedPer1k is the cost of a prompt token the engine or provider read + // from its cache (the answer's cached_tokens, counted within the + // prompt's). Zero charges them at prompt_per_1k. + CachedPer1k int64 `koanf:"cached_per_1k"` // PerImage is the cost of one picture, for a model that makes them // (/v1/images/generations), on top of its prompt's tokens. PerImage int64 `koanf:"per_image"` diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index f438361..fc0f173 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -550,6 +550,7 @@ var Reference = []Block{ {Package: "accounting", Name: "Price", Title: "accounting.Price", Doc: "", Fields: []Field{ {Key: "prompt_per_1k", Type: "int64", Doc: "", Nested: ""}, {Key: "completion_per_1k", Type: "int64", Doc: "", Nested: ""}, + {Key: "cached_per_1k", Type: "int64", Doc: "CachedPer1k is what a prompt token costs that the engine or provider\nread from its cache instead of computing: the part of prompt_tokens\nthe answer counts as cached_tokens. Zero charges those at\nPromptPer1k, as before there was a rate for them.", Nested: ""}, {Key: "per_image", Type: "int64", Doc: "PerImage is what one picture costs, for a model that makes them: a\npicture's cost is the card's time, which its prompt's few tokens do\nnot measure.", Nested: ""}, {Key: "per_mesh", Type: "int64", Doc: "PerMesh is what one 3D model costs: minutes of a card, which no\ntoken count measures.", Nested: ""}, }}, @@ -598,6 +599,7 @@ var Reference = []Block{ {Package: "config", Name: "Price", Title: "config.Price", Doc: "", Fields: []Field{ {Key: "prompt_per_1k", Type: "int64", Doc: "", Nested: ""}, {Key: "completion_per_1k", Type: "int64", Doc: "", Nested: ""}, + {Key: "cached_per_1k", Type: "int64", Doc: "CachedPer1k is the cost of a prompt token the engine or provider read\nfrom its cache (the answer's cached_tokens, counted within the\nprompt's). Zero charges them at prompt_per_1k.", Nested: ""}, {Key: "per_image", Type: "int64", Doc: "PerImage is the cost of one picture, for a model that makes them\n(/v1/images/generations), on top of its prompt's tokens.", Nested: ""}, {Key: "per_mesh", Type: "int64", Doc: "PerMesh is the cost of one 3D model (/v1/3d/generations).", Nested: ""}, }}, diff --git a/internal/router/handlers.go b/internal/router/handlers.go index b874234..b63ff94 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -825,6 +825,7 @@ func (r *Router) account(req *parsedRequest, res dispatchResult, status int, too if usage := res.outcome.Usage; usage != nil { u.PromptTokens = int64(usage.PromptTokens) u.CompletionTokens = int64(usage.CompletionTokens) + u.CachedTokens = int64(usage.CachedTokens) } else { // No usage came back — a failure before the engine answered, or an // engine that does not report it. Charging the estimate is closer to diff --git a/internal/router/router.go b/internal/router/router.go index 1a8ad4b..2212703 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -400,7 +400,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) { prices := make(map[string]accounting.Price, len(cfg.Auth.Pricing)) for name, p := range cfg.Auth.Pricing { prices[name] = accounting.Price{ - PromptPer1k: p.PromptPer1k, CompletionPer1k: p.CompletionPer1k, PerImage: p.PerImage, PerMesh: p.PerMesh, + PromptPer1k: p.PromptPer1k, CompletionPer1k: p.CompletionPer1k, CachedPer1k: p.CachedPer1k, PerImage: p.PerImage, PerMesh: p.PerMesh, } } r.recorder = accounting.NewRecorder(st, r.auth, accounting.RecorderConfig{ @@ -430,6 +430,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) { accounting.NewPricing(prices, accounting.Price{ PromptPer1k: cfg.Auth.DefaultPrice.PromptPer1k, CompletionPer1k: cfg.Auth.DefaultPrice.CompletionPer1k, + CachedPer1k: cfg.Auth.DefaultPrice.CachedPer1k, PerImage: cfg.Auth.DefaultPrice.PerImage, PerMesh: cfg.Auth.DefaultPrice.PerMesh, }), diff --git a/internal/store/store.go b/internal/store/store.go index 5fac888..f64e8ad 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -277,6 +277,10 @@ type Usage struct { ServedModel string PromptTokens int64 CompletionTokens int64 + // CachedTokens is how many of PromptTokens the engine read from its + // cache, for the price of those. Not a column: what it changed is in + // CostMicros. + CachedTokens int64 // Images is how many pictures were made, for a request for them. Images int64 // Meshes is how many 3D models were made.