CI / test (push) Successful in 9m51s
auth.gpu_hour_cost (and core_hour_cost) is what an hour of a GPU (a core) costs the operator. The leader keeps, by the hour, the GPU- and core-hours each model's replicas hold (held_1h); llmgw ctl prices suggest spreads a model's over the tokens it served, an output token counted as four input ones and a cached prompt token as a tenth of one, and --apply writes the prices into auth.pricing. Idle hours are in the cost, so a model held and little used comes out dear; one held and billed nothing is shown with what it held. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
92 lines
3.0 KiB
Go
92 lines
3.0 KiB
Go
package accounting
|
|
|
|
import (
|
|
"math"
|
|
"slices"
|
|
)
|
|
|
|
// Prices from what the cluster costs: what each model held of it (its
|
|
// replicas' GPU- and core-hours) at what an hour of a GPU or a core costs
|
|
// the operator, spread over the tokens it served. Idle time is in it: a
|
|
// model held all month and used an hour costs what it held, not the hour.
|
|
//
|
|
// An output token is priced OutputPerInput input tokens, as producing one
|
|
// is that much dearer (one step a token, against a prompt read in one
|
|
// pass); a cached prompt token a tenth of an input one.
|
|
const (
|
|
OutputPerInput = 4
|
|
CachedPerInput = 0.1
|
|
)
|
|
|
|
// Basis is what an hour costs, in the units budgets are in.
|
|
type Basis struct {
|
|
GPUHour float64
|
|
CoreHour float64
|
|
}
|
|
|
|
// Held is what a model held, and Served what it answered with.
|
|
type Held struct {
|
|
GPUSeconds, CoreSeconds int64
|
|
}
|
|
|
|
// Served is a model's tokens.
|
|
type Served struct {
|
|
Prompt, Completion int64
|
|
}
|
|
|
|
// Suggestion is one model's price from its cost.
|
|
type Suggestion struct {
|
|
Model string `json:"model"`
|
|
GPUHours float64 `json:"gpu_hours"`
|
|
CoreHours float64 `json:"core_hours"`
|
|
// Cost is what it held cost, in budget units.
|
|
Cost float64 `json:"cost"`
|
|
Prompt int64 `json:"prompt_tokens"`
|
|
Completion int64 `json:"completion_tokens"`
|
|
// Price is the suggestion; zero with a Note saying why there is none.
|
|
Price Price `json:"price"`
|
|
Note string `json:"note,omitempty"`
|
|
}
|
|
|
|
// Suggest prices every model that held something or served something.
|
|
func Suggest(held map[string]Held, served map[string]Served, b Basis) []Suggestion {
|
|
var names []string
|
|
for m := range held {
|
|
names = append(names, m)
|
|
}
|
|
for m := range served {
|
|
if _, ok := held[m]; !ok {
|
|
names = append(names, m)
|
|
}
|
|
}
|
|
slices.Sort(names)
|
|
out := make([]Suggestion, 0, len(names))
|
|
for _, m := range names {
|
|
h, s := held[m], served[m]
|
|
g := Suggestion{Model: m, GPUHours: float64(h.GPUSeconds) / 3600, CoreHours: float64(h.CoreSeconds) / 3600,
|
|
Prompt: s.Prompt, Completion: s.Completion}
|
|
g.Cost = g.GPUHours*b.GPUHour + g.CoreHours*b.CoreHour
|
|
// Tokens counted as output ones: an input token is a fraction of one.
|
|
units := float64(s.Completion) + float64(s.Prompt)/OutputPerInput
|
|
switch {
|
|
case g.Cost == 0 && units > 0:
|
|
g.Note = "held nothing recorded here: an outside provider's, or served before the hours were kept"
|
|
case g.Cost > 0 && units == 0:
|
|
g.Note = "held the cluster and was billed no tokens (a guard's judge, or a model priced by the picture or 3D model): no one pays what it held"
|
|
case g.Cost > 0:
|
|
// Micro-units per thousand output tokens, rounded up so a
|
|
// cheap model is never free.
|
|
out1k := int64(math.Ceil(g.Cost / units * 1000 * 1e6))
|
|
g.Price = Price{
|
|
CompletionPer1k: out1k,
|
|
PromptPer1k: ceilDiv(out1k, OutputPerInput),
|
|
CachedPer1k: max(int64(math.Ceil(float64(ceilDiv(out1k, OutputPerInput))*CachedPerInput)), 1),
|
|
}
|
|
}
|
|
out = append(out, g)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func ceilDiv(a, b int64) int64 { return (a + b - 1) / b }
|