Ilker ManapandClaude Opus 5.5
8547528fb7
CI / test (push) Successful in 9m51s
auth.gpu_hour_cost (and core_hour_cost) is what an hour of a GPU (a core) costs the operator. The leader keeps, by the hour, the GPU- and core-hours each model's replicas hold (held_1h); llmgw ctl prices suggest spreads a model's over the tokens it served, an output token counted as four input ones and a cached prompt token as a tenth of one, and --apply writes the prices into auth.pricing. Idle hours are in the cost, so a model held and little used comes out dear; one held and billed nothing is shown with what it held. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
94 lines
2.3 KiB
Go
94 lines
2.3 KiB
Go
// Package held records, on the leader, what each model holds of the
|
|
// cluster: its replicas' GPUs and, for one on CPUs, its cores. Kept by the
|
|
// hour, it is the cost side of a token's price (llmgw ctl prices suggest).
|
|
package held
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// Inventory is where the allocations are.
|
|
type Inventory interface {
|
|
Nodes() []inventory.Node
|
|
Allocations(nodeID string) []inventory.Allocation
|
|
}
|
|
|
|
// Store keeps what was held.
|
|
type Store interface {
|
|
AddHeld(ctx context.Context, at time.Time, held []store.Held) error
|
|
}
|
|
|
|
// Recorder adds, every Every, what each model holds for that long.
|
|
type Recorder struct {
|
|
Inv Inventory
|
|
// Models is each replica's model, by replica id.
|
|
Models func() map[string]string
|
|
Store Store
|
|
Leader func() bool
|
|
Every time.Duration
|
|
Log *slog.Logger
|
|
}
|
|
|
|
// Run records until ctx ends.
|
|
func (r *Recorder) Run(ctx context.Context) {
|
|
every := r.Every
|
|
if every <= 0 {
|
|
every = 5 * time.Minute
|
|
}
|
|
t := time.NewTicker(every)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case now := <-t.C:
|
|
if r.Leader != nil && !r.Leader() {
|
|
continue
|
|
}
|
|
held := Sample(r.Inv, r.Models(), every)
|
|
if err := r.Store.AddHeld(ctx, now, held); err != nil && r.Log != nil {
|
|
r.Log.Warn("could not record what the models held, for their prices", "err", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Sample is what each model holds now, for d: a GPU for each card a shard
|
|
// was given, and the cores of a shard given cores and no GPU. A replica
|
|
// whose model is not known (headroom kept for another, one just reaped) is
|
|
// left out.
|
|
func Sample(inv Inventory, models map[string]string, d time.Duration) []store.Held {
|
|
by := map[string]*store.Held{}
|
|
var order []string
|
|
secs := int64(d.Seconds())
|
|
for _, n := range inv.Nodes() {
|
|
for _, a := range inv.Allocations(n.ID) {
|
|
m, ok := models[a.ReplicaID]
|
|
if !ok {
|
|
continue
|
|
}
|
|
h := by[m]
|
|
if h == nil {
|
|
h = &store.Held{Model: m}
|
|
by[m] = h
|
|
order = append(order, m)
|
|
}
|
|
if g := len(a.GPUIndices); g > 0 {
|
|
h.GPUSeconds += int64(g) * secs
|
|
} else {
|
|
h.CoreSeconds += int64(max(a.CPUCores, 1)) * secs
|
|
}
|
|
}
|
|
}
|
|
out := make([]store.Held, 0, len(order))
|
|
for _, m := range order {
|
|
out = append(out, *by[m])
|
|
}
|
|
return out
|
|
}
|