auth.gpu_hour_cost (and core_hour_cost) is what an hour of a GPU (a core) costs the operator. The leader keeps, by the hour, the GPU- and core-hours each model's replicas hold (held_1h); llmgw ctl prices suggest spreads a model's over the tokens it served, an output token counted as four input ones and a cached prompt token as a tenth of one, and --apply writes the prices into auth.pricing. Idle hours are in the cost, so a model held and little used comes out dear; one held and billed nothing is shown with what it held. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b8d3c084c5
commit
8547528fb7
22 files changed
+834
-1
No files matched your search
@@ -100,6 +100,10 @@ type API struct {
|
||||
ctl *control.Controller
|
||||
// slurmUsers are the Linux users made from the accounts; nil without a store.
|
||||
slurmUsers SlurmUsers
|
||||
// prices are suggested from what each model held and served; nil
|
||||
// without a store.
|
||||
prices PriceStore
|
||||
priceCfg func() PriceConfig
|
||||
// webhooks counts the settings' notify.webhooks; nil does not say.
|
||||
webhooks func() int
|
||||
}
|
||||
@@ -143,6 +147,7 @@ func (a *API) Register(mux Mux) {
|
||||
a.registerTenancy(mux)
|
||||
a.registerTrain(mux)
|
||||
a.registerSlurm(mux)
|
||||
a.registerPrices(mux)
|
||||
a.registerGuard(mux)
|
||||
a.registerSettings(mux)
|
||||
a.registerAuth(mux)
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// PriceStore is what a price is suggested from: what each model held of
|
||||
// the cluster, and the tokens it served.
|
||||
type PriceStore interface {
|
||||
HeldSince(ctx context.Context, since time.Time) ([]store.Held, time.Time, error)
|
||||
UsageByModel(ctx context.Context, since time.Time) ([]store.ModelUsage, error)
|
||||
}
|
||||
|
||||
// PriceConfig is the cost basis and the prices in force.
|
||||
type PriceConfig struct {
|
||||
Basis accounting.Basis
|
||||
Current map[string]accounting.Price
|
||||
Default accounting.Price
|
||||
}
|
||||
|
||||
// PriceSuggestions is a model's price from its cost, for each model.
|
||||
type PriceSuggestions struct {
|
||||
// Days is the window; RecordedFrom the first hour held is known for,
|
||||
// later than the window's start on a cluster that began keeping it
|
||||
// since, which makes the window shorter than asked.
|
||||
Days int `json:"days"`
|
||||
Since time.Time `json:"since"`
|
||||
RecordedFrom time.Time `json:"recorded_from,omitempty"`
|
||||
GPUHourCost float64 `json:"gpu_hour_cost"`
|
||||
CoreHourCost float64 `json:"core_hour_cost"`
|
||||
Models []PriceSuggestion `json:"models"`
|
||||
}
|
||||
|
||||
// PriceSuggestion is one model's.
|
||||
type PriceSuggestion struct {
|
||||
accounting.Suggestion
|
||||
Current accounting.Price `json:"current"`
|
||||
}
|
||||
|
||||
// SetPrices gives the API what prices are suggested from.
|
||||
func (a *API) SetPrices(st PriceStore, cfg func() PriceConfig) { a.prices, a.priceCfg = st, cfg }
|
||||
|
||||
func (a *API) registerPrices(mux Mux) {
|
||||
mux.HandleFunc("GET /api/v1/prices/suggest", a.suggestPrices)
|
||||
}
|
||||
|
||||
func (a *API) suggestPrices(w http.ResponseWriter, r *http.Request) {
|
||||
if a.prices == nil || a.priceCfg == nil {
|
||||
writeError(w, http.StatusNotImplemented, "prices are suggested from the store, which this control plane runs without")
|
||||
return
|
||||
}
|
||||
days := 30
|
||||
if v := r.URL.Query().Get("days"); v != "" {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 1 || n > 366 {
|
||||
writeError(w, http.StatusBadRequest, "days is a whole number of days, 1 to 366")
|
||||
return
|
||||
}
|
||||
days = n
|
||||
}
|
||||
cfg := a.priceCfg()
|
||||
if cfg.Basis.GPUHour <= 0 && cfg.Basis.CoreHour <= 0 {
|
||||
writeError(w, http.StatusConflict, "no cost to price from: set auth.gpu_hour_cost (and auth.core_hour_cost for "+
|
||||
"models on CPUs), what an hour of one costs, in the units budgets are in")
|
||||
return
|
||||
}
|
||||
since := time.Now().Add(-time.Duration(days) * 24 * time.Hour)
|
||||
rows, first, err := a.prices.HeldSince(r.Context(), since)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
// Tokens from when holding was first recorded, so both sides cover the
|
||||
// same hours.
|
||||
from := since
|
||||
if first.After(from) {
|
||||
from = first
|
||||
}
|
||||
usage, err := a.prices.UsageByModel(r.Context(), from)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
held := map[string]accounting.Held{}
|
||||
for _, h := range rows {
|
||||
held[h.Model] = accounting.Held{GPUSeconds: h.GPUSeconds, CoreSeconds: h.CoreSeconds}
|
||||
}
|
||||
served := map[string]accounting.Served{}
|
||||
for _, u := range usage {
|
||||
served[u.Model] = accounting.Served{Prompt: u.Prompt, Completion: u.Completion}
|
||||
}
|
||||
out := PriceSuggestions{Days: days, Since: since.UTC(), RecordedFrom: first,
|
||||
GPUHourCost: cfg.Basis.GPUHour, CoreHourCost: cfg.Basis.CoreHour, Models: []PriceSuggestion{}}
|
||||
for _, s := range accounting.Suggest(held, served, cfg.Basis) {
|
||||
cur, ok := cfg.Current[s.Model]
|
||||
if !ok {
|
||||
cur = cfg.Default
|
||||
}
|
||||
out.Models = append(out.Models, PriceSuggestion{Suggestion: s, Current: cur})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
@@ -408,6 +408,13 @@ var Routes = []Route{
|
||||
"all away. The machines have them within a minute.",
|
||||
Request: SlurmKeysRequest{}, Response: slurmsvc.User{}},
|
||||
|
||||
{Method: "GET", Path: "/api/v1/prices/suggest", Summary: "Each model's price, from what it held of the cluster",
|
||||
Description: "days (30 if not given) is the window. A model's cost is the GPU- and core-hours its replicas held, idle " +
|
||||
"ones too, at auth.gpu_hour_cost and auth.core_hour_cost; spread over its tokens, an output token four input " +
|
||||
"ones and a cached prompt token a tenth of one, it is price, in micro-units a thousand tokens as " +
|
||||
"auth.pricing takes them. current is the price in force. 409 with no cost set.",
|
||||
Response: PriceSuggestions{}},
|
||||
|
||||
{Method: "POST", Path: "/api/v1/train/jobs", Summary: "Queue a LoRA adapter's training on a node",
|
||||
Description: "The node trains when its agent is there, nothing else trains on it, and a card has gpu_memory_mib free; " +
|
||||
"the pairs are NDJSON of {\"q\", \"a\"}. An adapter learns how answers sound far better than what they say.",
|
||||
|
||||
Reference in new issue
Block a user