auth.gpu_hour_cost (and core_hour_cost) is what an hour of a GPU (a core) costs the operator. The leader keeps, by the hour, the GPU- and core-hours each model's replicas hold (held_1h); llmgw ctl prices suggest spreads a model's over the tokens it served, an output token counted as four input ones and a cached prompt token as a tenth of one, and --apply writes the prices into auth.pricing. Idle hours are in the cost, so a model held and little used comes out dear; one held and billed nothing is shown with what it held. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b8d3c084c5
commit
8547528fb7
22 files changed
+834
-1
No files matched your search
@@ -0,0 +1,40 @@
|
||||
package held
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
type inv map[string][]inventory.Allocation
|
||||
|
||||
func (i inv) Nodes() []inventory.Node {
|
||||
var out []inventory.Node
|
||||
for id := range i {
|
||||
out = append(out, inventory.Node{ID: id})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (i inv) Allocations(id string) []inventory.Allocation { return i[id] }
|
||||
|
||||
func TestSampleCountsCardsAndCores(t *testing.T) {
|
||||
in := inv{
|
||||
"gpu1": {{ReplicaID: "big-1", GPUIndices: []int{0, 1}}, {ReplicaID: "headroom", GPUIndices: []int{2}}},
|
||||
"gpu2": {{ReplicaID: "big-2", GPUIndices: []int{0, 1}}},
|
||||
"cpu1": {{ReplicaID: "small-1", CPUCores: 4}, {ReplicaID: "tiny-1"}},
|
||||
}
|
||||
models := map[string]string{"big-1": "big", "big-2": "big", "small-1": "small", "tiny-1": "tiny"}
|
||||
got := map[string]store.Held{}
|
||||
for _, h := range Sample(in, models, 5*time.Minute) {
|
||||
got[h.Model] = h
|
||||
}
|
||||
if got["big"].GPUSeconds != 4*300 || got["small"].CoreSeconds != 4*300 || got["tiny"].CoreSeconds != 300 {
|
||||
t.Errorf("held %+v", got)
|
||||
}
|
||||
if len(got) != 3 {
|
||||
t.Errorf("headroom counted: %+v", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user