auth.gpu_hour_cost (and core_hour_cost) is what an hour of a GPU (a core) costs the operator. The leader keeps, by the hour, the GPU- and core-hours each model's replicas hold (held_1h); llmgw ctl prices suggest spreads a model's over the tokens it served, an output token counted as four input ones and a cached prompt token as a tenth of one, and --apply writes the prices into auth.pricing. Idle hours are in the cost, so a model held and little used comes out dear; one held and billed nothing is shown with what it held. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b8d3c084c5
commit
8547528fb7
22 files changed
+834
-1
No files matched your search
@@ -0,0 +1,93 @@
|
||||
// Package held records, on the leader, what each model holds of the
|
||||
// cluster: its replicas' GPUs and, for one on CPUs, its cores. Kept by the
|
||||
// hour, it is the cost side of a token's price (llmgw ctl prices suggest).
|
||||
package held
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// Inventory is where the allocations are.
|
||||
type Inventory interface {
|
||||
Nodes() []inventory.Node
|
||||
Allocations(nodeID string) []inventory.Allocation
|
||||
}
|
||||
|
||||
// Store keeps what was held.
|
||||
type Store interface {
|
||||
AddHeld(ctx context.Context, at time.Time, held []store.Held) error
|
||||
}
|
||||
|
||||
// Recorder adds, every Every, what each model holds for that long.
|
||||
type Recorder struct {
|
||||
Inv Inventory
|
||||
// Models is each replica's model, by replica id.
|
||||
Models func() map[string]string
|
||||
Store Store
|
||||
Leader func() bool
|
||||
Every time.Duration
|
||||
Log *slog.Logger
|
||||
}
|
||||
|
||||
// Run records until ctx ends.
|
||||
func (r *Recorder) Run(ctx context.Context) {
|
||||
every := r.Every
|
||||
if every <= 0 {
|
||||
every = 5 * time.Minute
|
||||
}
|
||||
t := time.NewTicker(every)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case now := <-t.C:
|
||||
if r.Leader != nil && !r.Leader() {
|
||||
continue
|
||||
}
|
||||
held := Sample(r.Inv, r.Models(), every)
|
||||
if err := r.Store.AddHeld(ctx, now, held); err != nil && r.Log != nil {
|
||||
r.Log.Warn("could not record what the models held, for their prices", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sample is what each model holds now, for d: a GPU for each card a shard
|
||||
// was given, and the cores of a shard given cores and no GPU. A replica
|
||||
// whose model is not known (headroom kept for another, one just reaped) is
|
||||
// left out.
|
||||
func Sample(inv Inventory, models map[string]string, d time.Duration) []store.Held {
|
||||
by := map[string]*store.Held{}
|
||||
var order []string
|
||||
secs := int64(d.Seconds())
|
||||
for _, n := range inv.Nodes() {
|
||||
for _, a := range inv.Allocations(n.ID) {
|
||||
m, ok := models[a.ReplicaID]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
h := by[m]
|
||||
if h == nil {
|
||||
h = &store.Held{Model: m}
|
||||
by[m] = h
|
||||
order = append(order, m)
|
||||
}
|
||||
if g := len(a.GPUIndices); g > 0 {
|
||||
h.GPUSeconds += int64(g) * secs
|
||||
} else {
|
||||
h.CoreSeconds += int64(max(a.CPUCores, 1)) * secs
|
||||
}
|
||||
}
|
||||
}
|
||||
out := make([]store.Held, 0, len(order))
|
||||
for _, m := range order {
|
||||
out = append(out, *by[m])
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package held
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
type inv map[string][]inventory.Allocation
|
||||
|
||||
func (i inv) Nodes() []inventory.Node {
|
||||
var out []inventory.Node
|
||||
for id := range i {
|
||||
out = append(out, inventory.Node{ID: id})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (i inv) Allocations(id string) []inventory.Allocation { return i[id] }
|
||||
|
||||
func TestSampleCountsCardsAndCores(t *testing.T) {
|
||||
in := inv{
|
||||
"gpu1": {{ReplicaID: "big-1", GPUIndices: []int{0, 1}}, {ReplicaID: "headroom", GPUIndices: []int{2}}},
|
||||
"gpu2": {{ReplicaID: "big-2", GPUIndices: []int{0, 1}}},
|
||||
"cpu1": {{ReplicaID: "small-1", CPUCores: 4}, {ReplicaID: "tiny-1"}},
|
||||
}
|
||||
models := map[string]string{"big-1": "big", "big-2": "big", "small-1": "small", "tiny-1": "tiny"}
|
||||
got := map[string]store.Held{}
|
||||
for _, h := range Sample(in, models, 5*time.Minute) {
|
||||
got[h.Model] = h
|
||||
}
|
||||
if got["big"].GPUSeconds != 4*300 || got["small"].CoreSeconds != 4*300 || got["tiny"].CoreSeconds != 300 {
|
||||
t.Errorf("held %+v", got)
|
||||
}
|
||||
if len(got) != 3 {
|
||||
t.Errorf("headroom counted: %+v", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user