Prices from what the cluster costs: ctl prices suggest
CI / test (push) Successful in 9m51s

auth.gpu_hour_cost (and core_hour_cost) is what an hour of a GPU (a core)
costs the operator. The leader keeps, by the hour, the GPU- and core-hours
each model's replicas hold (held_1h); llmgw ctl prices suggest spreads a
model's over the tokens it served, an output token counted as four input
ones and a cached prompt token as a tenth of one, and --apply writes the
prices into auth.pricing. Idle hours are in the cost, so a model held and
little used comes out dear; one held and billed nothing is shown with what
it held.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-15 06:54:55 +01:00
1 parent b8d3c084c5
commit 8547528fb7
22 files changed
+834 -1

No files matched your search

+5
View File
@@ -100,6 +100,10 @@ type API struct {
ctl *control.Controller
// slurmUsers are the Linux users made from the accounts; nil without a store.
slurmUsers SlurmUsers
// prices are suggested from what each model held and served; nil
// without a store.
prices PriceStore
priceCfg func() PriceConfig
// webhooks counts the settings' notify.webhooks; nil does not say.
webhooks func() int
}
@@ -143,6 +147,7 @@ func (a *API) Register(mux Mux) {
a.registerTenancy(mux)
a.registerTrain(mux)
a.registerSlurm(mux)
a.registerPrices(mux)
a.registerGuard(mux)
a.registerSettings(mux)
a.registerAuth(mux)
+108
View File
@@ -0,0 +1,108 @@
package adminapi
import (
"context"
"net/http"
"strconv"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// PriceStore is what a price is suggested from: what each model held of
// the cluster, and the tokens it served.
type PriceStore interface {
HeldSince(ctx context.Context, since time.Time) ([]store.Held, time.Time, error)
UsageByModel(ctx context.Context, since time.Time) ([]store.ModelUsage, error)
}
// PriceConfig is the cost basis and the prices in force.
type PriceConfig struct {
Basis accounting.Basis
Current map[string]accounting.Price
Default accounting.Price
}
// PriceSuggestions is a model's price from its cost, for each model.
type PriceSuggestions struct {
// Days is the window; RecordedFrom the first hour held is known for,
// later than the window's start on a cluster that began keeping it
// since, which makes the window shorter than asked.
Days int `json:"days"`
Since time.Time `json:"since"`
RecordedFrom time.Time `json:"recorded_from,omitempty"`
GPUHourCost float64 `json:"gpu_hour_cost"`
CoreHourCost float64 `json:"core_hour_cost"`
Models []PriceSuggestion `json:"models"`
}
// PriceSuggestion is one model's.
type PriceSuggestion struct {
accounting.Suggestion
Current accounting.Price `json:"current"`
}
// SetPrices gives the API what prices are suggested from.
func (a *API) SetPrices(st PriceStore, cfg func() PriceConfig) { a.prices, a.priceCfg = st, cfg }
func (a *API) registerPrices(mux Mux) {
mux.HandleFunc("GET /api/v1/prices/suggest", a.suggestPrices)
}
func (a *API) suggestPrices(w http.ResponseWriter, r *http.Request) {
if a.prices == nil || a.priceCfg == nil {
writeError(w, http.StatusNotImplemented, "prices are suggested from the store, which this control plane runs without")
return
}
days := 30
if v := r.URL.Query().Get("days"); v != "" {
n, err := strconv.Atoi(v)
if err != nil || n < 1 || n > 366 {
writeError(w, http.StatusBadRequest, "days is a whole number of days, 1 to 366")
return
}
days = n
}
cfg := a.priceCfg()
if cfg.Basis.GPUHour <= 0 && cfg.Basis.CoreHour <= 0 {
writeError(w, http.StatusConflict, "no cost to price from: set auth.gpu_hour_cost (and auth.core_hour_cost for "+
"models on CPUs), what an hour of one costs, in the units budgets are in")
return
}
since := time.Now().Add(-time.Duration(days) * 24 * time.Hour)
rows, first, err := a.prices.HeldSince(r.Context(), since)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
// Tokens from when holding was first recorded, so both sides cover the
// same hours.
from := since
if first.After(from) {
from = first
}
usage, err := a.prices.UsageByModel(r.Context(), from)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
held := map[string]accounting.Held{}
for _, h := range rows {
held[h.Model] = accounting.Held{GPUSeconds: h.GPUSeconds, CoreSeconds: h.CoreSeconds}
}
served := map[string]accounting.Served{}
for _, u := range usage {
served[u.Model] = accounting.Served{Prompt: u.Prompt, Completion: u.Completion}
}
out := PriceSuggestions{Days: days, Since: since.UTC(), RecordedFrom: first,
GPUHourCost: cfg.Basis.GPUHour, CoreHourCost: cfg.Basis.CoreHour, Models: []PriceSuggestion{}}
for _, s := range accounting.Suggest(held, served, cfg.Basis) {
cur, ok := cfg.Current[s.Model]
if !ok {
cur = cfg.Default
}
out.Models = append(out.Models, PriceSuggestion{Suggestion: s, Current: cur})
}
writeJSON(w, http.StatusOK, out)
}
+7
View File
@@ -408,6 +408,13 @@ var Routes = []Route{
"all away. The machines have them within a minute.",
Request: SlurmKeysRequest{}, Response: slurmsvc.User{}},
{Method: "GET", Path: "/api/v1/prices/suggest", Summary: "Each model's price, from what it held of the cluster",
Description: "days (30 if not given) is the window. A model's cost is the GPU- and core-hours its replicas held, idle " +
"ones too, at auth.gpu_hour_cost and auth.core_hour_cost; spread over its tokens, an output token four input " +
"ones and a cached prompt token a tenth of one, it is price, in micro-units a thousand tokens as " +
"auth.pricing takes them. current is the price in force. 409 with no cost set.",
Response: PriceSuggestions{}},
{Method: "POST", Path: "/api/v1/train/jobs", Summary: "Queue a LoRA adapter's training on a node",
Description: "The node trains when its agent is there, nothing else trains on it, and a card has gpu_memory_mib free; " +
"the pairs are NDJSON of {\"q\", \"a\"}. An adapter learns how answers sound far better than what they say.",
+93
View File
@@ -0,0 +1,93 @@
// Package held records, on the leader, what each model holds of the
// cluster: its replicas' GPUs and, for one on CPUs, its cores. Kept by the
// hour, it is the cost side of a token's price (llmgw ctl prices suggest).
package held
import (
"context"
"log/slog"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// Inventory is where the allocations are.
type Inventory interface {
Nodes() []inventory.Node
Allocations(nodeID string) []inventory.Allocation
}
// Store keeps what was held.
type Store interface {
AddHeld(ctx context.Context, at time.Time, held []store.Held) error
}
// Recorder adds, every Every, what each model holds for that long.
type Recorder struct {
Inv Inventory
// Models is each replica's model, by replica id.
Models func() map[string]string
Store Store
Leader func() bool
Every time.Duration
Log *slog.Logger
}
// Run records until ctx ends.
func (r *Recorder) Run(ctx context.Context) {
every := r.Every
if every <= 0 {
every = 5 * time.Minute
}
t := time.NewTicker(every)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case now := <-t.C:
if r.Leader != nil && !r.Leader() {
continue
}
held := Sample(r.Inv, r.Models(), every)
if err := r.Store.AddHeld(ctx, now, held); err != nil && r.Log != nil {
r.Log.Warn("could not record what the models held, for their prices", "err", err)
}
}
}
}
// Sample is what each model holds now, for d: a GPU for each card a shard
// was given, and the cores of a shard given cores and no GPU. A replica
// whose model is not known (headroom kept for another, one just reaped) is
// left out.
func Sample(inv Inventory, models map[string]string, d time.Duration) []store.Held {
by := map[string]*store.Held{}
var order []string
secs := int64(d.Seconds())
for _, n := range inv.Nodes() {
for _, a := range inv.Allocations(n.ID) {
m, ok := models[a.ReplicaID]
if !ok {
continue
}
h := by[m]
if h == nil {
h = &store.Held{Model: m}
by[m] = h
order = append(order, m)
}
if g := len(a.GPUIndices); g > 0 {
h.GPUSeconds += int64(g) * secs
} else {
h.CoreSeconds += int64(max(a.CPUCores, 1)) * secs
}
}
}
out := make([]store.Held, 0, len(order))
for _, m := range order {
out = append(out, *by[m])
}
return out
}
+40
View File
@@ -0,0 +1,40 @@
package held
import (
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
type inv map[string][]inventory.Allocation
func (i inv) Nodes() []inventory.Node {
var out []inventory.Node
for id := range i {
out = append(out, inventory.Node{ID: id})
}
return out
}
func (i inv) Allocations(id string) []inventory.Allocation { return i[id] }
func TestSampleCountsCardsAndCores(t *testing.T) {
in := inv{
"gpu1": {{ReplicaID: "big-1", GPUIndices: []int{0, 1}}, {ReplicaID: "headroom", GPUIndices: []int{2}}},
"gpu2": {{ReplicaID: "big-2", GPUIndices: []int{0, 1}}},
"cpu1": {{ReplicaID: "small-1", CPUCores: 4}, {ReplicaID: "tiny-1"}},
}
models := map[string]string{"big-1": "big", "big-2": "big", "small-1": "small", "tiny-1": "tiny"}
got := map[string]store.Held{}
for _, h := range Sample(in, models, 5*time.Minute) {
got[h.Model] = h
}
if got["big"].GPUSeconds != 4*300 || got["small"].CoreSeconds != 4*300 || got["tiny"].CoreSeconds != 300 {
t.Errorf("held %+v", got)
}
if len(got) != 3 {
t.Errorf("headroom counted: %+v", got)
}
}