CI / test (push) Successful in 9m51s
auth.gpu_hour_cost (and core_hour_cost) is what an hour of a GPU (a core) costs the operator. The leader keeps, by the hour, the GPU- and core-hours each model's replicas hold (held_1h); llmgw ctl prices suggest spreads a model's over the tokens it served, an output token counted as four input ones and a cached prompt token as a tenth of one, and --apply writes the prices into auth.pricing. Idle hours are in the cost, so a model held and little used comes out dear; one held and billed nothing is shown with what it held. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
46 lines
1.6 KiB
Go
46 lines
1.6 KiB
Go
package accounting
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestSuggestSpreadsWhatAModelHeldOverItsTokens(t *testing.T) {
|
|
// The worked example: two cards for 1000 hours at 1.35 an hour is 2700;
|
|
// 3.0 billion output tokens and 4.0 billion input ones, counted as 1.0
|
|
// billion output ones, are 4.0 billion: 0.675 a million output tokens.
|
|
held := map[string]Held{
|
|
"llama-70b": {GPUSeconds: 2 * 1000 * 3600},
|
|
"idle": {GPUSeconds: 3600},
|
|
"small-cpu": {CoreSeconds: 8 * 100 * 3600},
|
|
}
|
|
served := map[string]Served{
|
|
"llama-70b": {Prompt: 4_000_000_000, Completion: 3_000_000_000},
|
|
"small-cpu": {Prompt: 1_000_000, Completion: 1_000_000},
|
|
"outside": {Prompt: 10, Completion: 10},
|
|
}
|
|
got := Suggest(held, served, Basis{GPUHour: 1.35, CoreHour: 0.01})
|
|
by := map[string]Suggestion{}
|
|
for _, s := range got {
|
|
by[s.Model] = s
|
|
}
|
|
l := by["llama-70b"]
|
|
if l.Cost != 2700 || l.Price.CompletionPer1k != 675 || l.Price.PromptPer1k != 169 || l.Price.CachedPer1k != 17 {
|
|
t.Errorf("llama: %+v", l)
|
|
}
|
|
// What it costs back at that price is what it held, within the rounding.
|
|
back := float64(l.Price.Cost(l.Prompt, 0, l.Completion, 0, 0)) / 1e6
|
|
if back < l.Cost || back > l.Cost*1.01 {
|
|
t.Errorf("charged %.2f for a cost of %.2f", back, l.Cost)
|
|
}
|
|
if s := by["small-cpu"]; s.Cost != 8 || s.Price.CompletionPer1k == 0 {
|
|
t.Errorf("cpu: %+v", s)
|
|
}
|
|
if s := by["idle"]; !s.Price.Zero() || !strings.Contains(s.Note, "billed no tokens") {
|
|
t.Errorf("idle: %+v", s)
|
|
}
|
|
if s := by["outside"]; !s.Price.Zero() || !strings.Contains(s.Note, "held nothing") {
|
|
t.Errorf("outside: %+v", s)
|
|
}
|
|
}
|