Ilker ManapandClaude Opus 5.5
8547528fb7
CI / test (push) Successful in 9m51s
auth.gpu_hour_cost (and core_hour_cost) is what an hour of a GPU (a core) costs the operator. The leader keeps, by the hour, the GPU- and core-hours each model's replicas hold (held_1h); llmgw ctl prices suggest spreads a model's over the tokens it served, an output token counted as four input ones and a cached prompt token as a tenth of one, and --apply writes the prices into auth.pricing. Idle hours are in the cost, so a model held and little used comes out dear; one held and billed nothing is shown with what it held. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
63 lines
1.7 KiB
Go
63 lines
1.7 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
)
|
|
|
|
// Held is what one model held of the cluster.
|
|
type Held struct {
|
|
Model string
|
|
GPUSeconds int64
|
|
CoreSeconds int64
|
|
}
|
|
|
|
// AddHeld adds what models held, during the hour at holds.
|
|
func (s *DB) AddHeld(ctx context.Context, at time.Time, held []Held) error {
|
|
bucket := at.UTC().Truncate(time.Hour).UnixMilli()
|
|
for _, h := range held {
|
|
if h.Model == "" || (h.GPUSeconds <= 0 && h.CoreSeconds <= 0) {
|
|
continue
|
|
}
|
|
if _, err := s.db.ExecContext(ctx,
|
|
`INSERT INTO held_1h (bucket, model, gpu_seconds, core_seconds) VALUES (?, ?, ?, ?)
|
|
ON CONFLICT(bucket, model) DO UPDATE SET
|
|
gpu_seconds = held_1h.gpu_seconds + excluded.gpu_seconds,
|
|
core_seconds = held_1h.core_seconds + excluded.core_seconds`,
|
|
bucket, h.Model, h.GPUSeconds, h.CoreSeconds); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// HeldSince is what each model held since the hour since falls in, and
|
|
// the first hour anything was recorded in (zero with nothing).
|
|
func (s *DB) HeldSince(ctx context.Context, since time.Time) ([]Held, time.Time, error) {
|
|
from := since.UTC().Truncate(time.Hour).UnixMilli()
|
|
rows, err := s.db.QueryContext(ctx,
|
|
`SELECT model, SUM(gpu_seconds), SUM(core_seconds), MIN(bucket) FROM held_1h
|
|
WHERE bucket >= ? GROUP BY model ORDER BY model`, from)
|
|
if err != nil {
|
|
return nil, time.Time{}, err
|
|
}
|
|
defer rows.Close()
|
|
var out []Held
|
|
var first int64
|
|
for rows.Next() {
|
|
var h Held
|
|
var b int64
|
|
if err := rows.Scan(&h.Model, &h.GPUSeconds, &h.CoreSeconds, &b); err != nil {
|
|
return nil, time.Time{}, err
|
|
}
|
|
if first == 0 || b < first {
|
|
first = b
|
|
}
|
|
out = append(out, h)
|
|
}
|
|
if first == 0 {
|
|
return out, time.Time{}, rows.Err()
|
|
}
|
|
return out, time.UnixMilli(first).UTC(), rows.Err()
|
|
}
|