Files
llmgw/internal/accounting/suggest.go
T
Ilker ManapandClaude Opus 5.5 8547528fb7
CI / test (push) Successful in 9m51s
Prices from what the cluster costs: ctl prices suggest
auth.gpu_hour_cost (and core_hour_cost) is what an hour of a GPU (a core)
costs the operator. The leader keeps, by the hour, the GPU- and core-hours
each model's replicas hold (held_1h); llmgw ctl prices suggest spreads a
model's over the tokens it served, an output token counted as four input
ones and a cached prompt token as a tenth of one, and --apply writes the
prices into auth.pricing. Idle hours are in the cost, so a model held and
little used comes out dear; one held and billed nothing is shown with what
it held.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 06:54:55 +01:00

92 lines
3.0 KiB
Go

package accounting
import (
"math"
"slices"
)
// Prices from what the cluster costs: what each model held of it (its
// replicas' GPU- and core-hours) at what an hour of a GPU or a core costs
// the operator, spread over the tokens it served. Idle time is in it: a
// model held all month and used an hour costs what it held, not the hour.
//
// An output token is priced OutputPerInput input tokens, as producing one
// is that much dearer (one step a token, against a prompt read in one
// pass); a cached prompt token a tenth of an input one.
const (
OutputPerInput = 4
CachedPerInput = 0.1
)
// Basis is what an hour costs, in the units budgets are in.
type Basis struct {
GPUHour float64
CoreHour float64
}
// Held is what a model held, and Served what it answered with.
type Held struct {
GPUSeconds, CoreSeconds int64
}
// Served is a model's tokens.
type Served struct {
Prompt, Completion int64
}
// Suggestion is one model's price from its cost.
type Suggestion struct {
Model string `json:"model"`
GPUHours float64 `json:"gpu_hours"`
CoreHours float64 `json:"core_hours"`
// Cost is what it held cost, in budget units.
Cost float64 `json:"cost"`
Prompt int64 `json:"prompt_tokens"`
Completion int64 `json:"completion_tokens"`
// Price is the suggestion; zero with a Note saying why there is none.
Price Price `json:"price"`
Note string `json:"note,omitempty"`
}
// Suggest prices every model that held something or served something.
func Suggest(held map[string]Held, served map[string]Served, b Basis) []Suggestion {
var names []string
for m := range held {
names = append(names, m)
}
for m := range served {
if _, ok := held[m]; !ok {
names = append(names, m)
}
}
slices.Sort(names)
out := make([]Suggestion, 0, len(names))
for _, m := range names {
h, s := held[m], served[m]
g := Suggestion{Model: m, GPUHours: float64(h.GPUSeconds) / 3600, CoreHours: float64(h.CoreSeconds) / 3600,
Prompt: s.Prompt, Completion: s.Completion}
g.Cost = g.GPUHours*b.GPUHour + g.CoreHours*b.CoreHour
// Tokens counted as output ones: an input token is a fraction of one.
units := float64(s.Completion) + float64(s.Prompt)/OutputPerInput
switch {
case g.Cost == 0 && units > 0:
g.Note = "held nothing recorded here: an outside provider's, or served before the hours were kept"
case g.Cost > 0 && units == 0:
g.Note = "held the cluster and was billed no tokens (a guard's judge, or a model priced by the picture or 3D model): no one pays what it held"
case g.Cost > 0:
// Micro-units per thousand output tokens, rounded up so a
// cheap model is never free.
out1k := int64(math.Ceil(g.Cost / units * 1000 * 1e6))
g.Price = Price{
CompletionPer1k: out1k,
PromptPer1k: ceilDiv(out1k, OutputPerInput),
CachedPer1k: max(int64(math.Ceil(float64(ceilDiv(out1k, OutputPerInput))*CachedPerInput)), 1),
}
}
out = append(out, g)
}
return out
}
func ceilDiv(a, b int64) int64 { return (a + b - 1) / b }