package accounting import ( "strings" "testing" ) func TestSuggestSpreadsWhatAModelHeldOverItsTokens(t *testing.T) { // The worked example: two cards for 1000 hours at 1.35 an hour is 2700; // 3.0 billion output tokens and 4.0 billion input ones, counted as 1.0 // billion output ones, are 4.0 billion: 0.675 a million output tokens. held := map[string]Held{ "llama-70b": {GPUSeconds: 2 * 1000 * 3600}, "idle": {GPUSeconds: 3600}, "small-cpu": {CoreSeconds: 8 * 100 * 3600}, } served := map[string]Served{ "llama-70b": {Prompt: 4_000_000_000, Completion: 3_000_000_000}, "small-cpu": {Prompt: 1_000_000, Completion: 1_000_000}, "outside": {Prompt: 10, Completion: 10}, } got := Suggest(held, served, Basis{GPUHour: 1.35, CoreHour: 0.01}) by := map[string]Suggestion{} for _, s := range got { by[s.Model] = s } l := by["llama-70b"] if l.Cost != 2700 || l.Price.CompletionPer1k != 675 || l.Price.PromptPer1k != 169 || l.Price.CachedPer1k != 17 { t.Errorf("llama: %+v", l) } // What it costs back at that price is what it held, within the rounding. back := float64(l.Price.Cost(l.Prompt, 0, l.Completion, 0, 0)) / 1e6 if back < l.Cost || back > l.Cost*1.01 { t.Errorf("charged %.2f for a cost of %.2f", back, l.Cost) } if s := by["small-cpu"]; s.Cost != 8 || s.Price.CompletionPer1k == 0 { t.Errorf("cpu: %+v", s) } if s := by["idle"]; !s.Price.Zero() || !strings.Contains(s.Note, "billed no tokens") { t.Errorf("idle: %+v", s) } if s := by["outside"]; !s.Price.Zero() || !strings.Contains(s.Note, "held nothing") { t.Errorf("outside: %+v", s) } }