auth.gpu_hour_cost (and core_hour_cost) is what an hour of a GPU (a core) costs the operator. The leader keeps, by the hour, the GPU- and core-hours each model's replicas hold (held_1h); llmgw ctl prices suggest spreads a model's over the tokens it served, an output token counted as four input ones and a cached prompt token as a tenth of one, and --apply writes the prices into auth.pricing. Idle hours are in the cost, so a model held and little used comes out dear; one held and billed nothing is shown with what it held. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b8d3c084c5
commit
8547528fb7
22 files changed
+834
-1
No files matched your search
@@ -36,6 +36,7 @@ import (
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/audit"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bundle"
|
||||
@@ -47,6 +48,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fleetmetrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/held"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
|
||||
@@ -433,6 +435,27 @@ func newServerCmd() *cobra.Command {
|
||||
api.SetAgentRestarter(agentSvc.RestartAgents)
|
||||
}
|
||||
api.SetAudit(st)
|
||||
// Prices from what the cluster costs: what each model
|
||||
// holds, kept by the hour on the leader.
|
||||
api.SetPrices(st, func() adminapi.PriceConfig {
|
||||
au := current.Load().Auth
|
||||
cur := make(map[string]accounting.Price, len(au.Pricing))
|
||||
for m, p := range au.Pricing {
|
||||
cur[m] = accountingPrice(p)
|
||||
}
|
||||
return adminapi.PriceConfig{Basis: accounting.Basis{GPUHour: au.GPUHourCost, CoreHour: au.CoreHourCost},
|
||||
Current: cur, Default: accountingPrice(au.DefaultPrice)}
|
||||
})
|
||||
go (&held.Recorder{Inv: admin.Inventory(), Store: st, Log: log,
|
||||
Models: func() map[string]string {
|
||||
m := map[string]string{}
|
||||
for _, rep := range admin.Reconciler().Replicas() {
|
||||
m[rep.ID] = rep.Model
|
||||
}
|
||||
return m
|
||||
},
|
||||
Leader: func() bool { return admin.Leadership() == nil || admin.Leadership().IsLeader() },
|
||||
}).Run(ctx)
|
||||
// Knowledge bases, with the cluster's own models to embed a
|
||||
// probe (a collection's dimension) and an operator's search.
|
||||
api.SetKB(&adminapi.KBAdmin{Store: st, Embed: embedder(rt.Replay)})
|
||||
@@ -1418,3 +1441,9 @@ func runPresetsPass(ctx context.Context, api *adminapi.API, lead *control.Leader
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// accountingPrice is a configured price as accounting has it.
|
||||
func accountingPrice(p config.Price) accounting.Price {
|
||||
return accounting.Price{PromptPer1k: p.PromptPer1k, CompletionPer1k: p.CompletionPer1k, CachedPer1k: p.CachedPer1k,
|
||||
PerImage: p.PerImage, PerMesh: p.PerMesh}
|
||||
}
|
||||
Reference in new issue
Block a user