auth.gpu_hour_cost (and core_hour_cost) is what an hour of a GPU (a core) costs the operator. The leader keeps, by the hour, the GPU- and core-hours each model's replicas hold (held_1h); llmgw ctl prices suggest spreads a model's over the tokens it served, an output token counted as four input ones and a cached prompt token as a tenth of one, and --apply writes the prices into auth.pricing. Idle hours are in the cost, so a model held and little used comes out dear; one held and billed nothing is shown with what it held. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b8d3c084c5
commit
8547528fb7
22 files changed
+834
-1
No files matched your search
@@ -190,6 +190,7 @@ func newCtlCmd() *cobra.Command {
|
||||
newKBCmd(client, &asJSON),
|
||||
newTrainCmd(client, &asJSON),
|
||||
newSlurmCmd(client, &asJSON),
|
||||
newPricesCmd(client, &asJSON),
|
||||
newConfigCmd(client, &asJSON),
|
||||
newAuditCmd(client, &asJSON),
|
||||
newNotifyCmd(client, &asJSON),
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"text/tabwriter"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
|
||||
)
|
||||
|
||||
func newPricesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "prices",
|
||||
Short: "What each model's tokens cost, from what the cluster costs",
|
||||
}
|
||||
var days int
|
||||
var apply bool
|
||||
var note string
|
||||
suggest := &cobra.Command{
|
||||
Use: "suggest",
|
||||
Short: "Each model's price from the GPU- and core-hours it held, over the tokens it served",
|
||||
Long: "Set auth.gpu_hour_cost first: what an hour of one GPU costs, in the units budgets are in (the\n" +
|
||||
"hardware written off over its years, power and room, the people who run it, over the GPUs and\n" +
|
||||
"the hours of a month); auth.core_hour_cost likewise for models on CPUs. Each model's cost is\n" +
|
||||
"the hours its replicas held, idle ones too, kept by the hour on the leader; spread over its\n" +
|
||||
"tokens, an output token counted as four input ones and a cached prompt token as a tenth of one,\n" +
|
||||
"it is its price. A model held and not used shows what that cost. --apply writes the prices\n" +
|
||||
"into auth.pricing, one settings version, as `ctl config set` does.",
|
||||
Example: " llmgw ctl config set auth.gpu_hour_cost 1.35\n" +
|
||||
" llmgw ctl prices suggest --days 30\n" +
|
||||
" llmgw ctl prices suggest --apply --note \"November's costs\"",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
var s adminapi.PriceSuggestions
|
||||
if err := client().get(cmd.Context(), "/api/v1/prices/suggest?days="+url.QueryEscape(strconv.Itoa(days)), &s); err != nil {
|
||||
return err
|
||||
}
|
||||
w := cmd.OutOrStdout()
|
||||
if *asJSON {
|
||||
return json.NewEncoder(w).Encode(s)
|
||||
}
|
||||
fmt.Fprintf(w, "cost: %g a GPU-hour, %g a core-hour; the last %d days", s.GPUHourCost, s.CoreHourCost, s.Days)
|
||||
if !s.RecordedFrom.IsZero() && s.RecordedFrom.After(s.Since) {
|
||||
fmt.Fprintf(w, " (held kept only since %s)", s.RecordedFrom.Local().Format("2006-01-02 15:04"))
|
||||
}
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "prices a million tokens: input / output / cached input, in budget units")
|
||||
tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
|
||||
fmt.Fprintln(tw, "MODEL\tGPU-H\tCORE-H\tCOST\tTOKENS IN/OUT\tSUGGESTED\tNOW\tNOTE")
|
||||
var changed []adminapi.PriceSuggestion
|
||||
for _, m := range s.Models {
|
||||
sug := "-"
|
||||
if !m.Price.Zero() {
|
||||
sug = perMillion(m.Price.PromptPer1k, m.Price.CompletionPer1k, m.Price.CachedPer1k)
|
||||
if m.Price.PromptPer1k != m.Current.PromptPer1k || m.Price.CompletionPer1k != m.Current.CompletionPer1k ||
|
||||
m.Price.CachedPer1k != m.Current.CachedPer1k {
|
||||
changed = append(changed, m)
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(tw, "%s\t%.1f\t%.1f\t%.2f\t%s/%s\t%s\t%s\t%s\n", m.Model, m.GPUHours, m.CoreHours, m.Cost,
|
||||
count(m.Prompt), count(m.Completion), sug,
|
||||
perMillion(m.Current.PromptPer1k, m.Current.CompletionPer1k, m.Current.CachedPer1k), m.Note)
|
||||
}
|
||||
if err := tw.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
if !apply {
|
||||
if len(changed) > 0 {
|
||||
fmt.Fprintf(w, "%d model(s) priced otherwise now; --apply writes the suggestions\n", len(changed))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if len(changed) == 0 {
|
||||
fmt.Fprintln(w, "nothing to change")
|
||||
return nil
|
||||
}
|
||||
return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
|
||||
for _, m := range changed {
|
||||
for k, v := range map[string]int64{"prompt_per_1k": m.Price.PromptPer1k,
|
||||
"completion_per_1k": m.Price.CompletionPer1k, "cached_per_1k": m.Price.CachedPer1k} {
|
||||
if err := doc.SetYAML([]string{"auth", "pricing", m.Model, k}, strconv.FormatInt(v, 10)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
},
|
||||
}
|
||||
suggest.Flags().IntVar(&days, "days", 30, "the window, in days")
|
||||
suggest.Flags().BoolVar(&apply, "apply", false, "write the suggested prices into auth.pricing")
|
||||
suggest.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version")
|
||||
cmd.AddCommand(suggest)
|
||||
return cmd
|
||||
}
|
||||
|
||||
// perMillion is micro-units a thousand tokens as budget units a million.
|
||||
func perMillion(in, out, cached int64) string {
|
||||
if in == 0 && out == 0 && cached == 0 {
|
||||
return "-"
|
||||
}
|
||||
f := func(v int64) string { return strconv.FormatFloat(float64(v)/1000, 'f', -1, 64) }
|
||||
return f(in) + " / " + f(out) + " / " + f(cached)
|
||||
}
|
||||
|
||||
func count(n int64) string {
|
||||
switch {
|
||||
case n >= 1e9:
|
||||
return fmt.Sprintf("%.1fB", float64(n)/1e9)
|
||||
case n >= 1e6:
|
||||
return fmt.Sprintf("%.1fM", float64(n)/1e6)
|
||||
case n >= 1e3:
|
||||
return fmt.Sprintf("%.1fk", float64(n)/1e3)
|
||||
}
|
||||
return strconv.FormatInt(n, 10)
|
||||
}
|
||||
@@ -36,6 +36,7 @@ import (
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/audit"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bundle"
|
||||
@@ -47,6 +48,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fleetmetrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/held"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
|
||||
@@ -433,6 +435,27 @@ func newServerCmd() *cobra.Command {
|
||||
api.SetAgentRestarter(agentSvc.RestartAgents)
|
||||
}
|
||||
api.SetAudit(st)
|
||||
// Prices from what the cluster costs: what each model
|
||||
// holds, kept by the hour on the leader.
|
||||
api.SetPrices(st, func() adminapi.PriceConfig {
|
||||
au := current.Load().Auth
|
||||
cur := make(map[string]accounting.Price, len(au.Pricing))
|
||||
for m, p := range au.Pricing {
|
||||
cur[m] = accountingPrice(p)
|
||||
}
|
||||
return adminapi.PriceConfig{Basis: accounting.Basis{GPUHour: au.GPUHourCost, CoreHour: au.CoreHourCost},
|
||||
Current: cur, Default: accountingPrice(au.DefaultPrice)}
|
||||
})
|
||||
go (&held.Recorder{Inv: admin.Inventory(), Store: st, Log: log,
|
||||
Models: func() map[string]string {
|
||||
m := map[string]string{}
|
||||
for _, rep := range admin.Reconciler().Replicas() {
|
||||
m[rep.ID] = rep.Model
|
||||
}
|
||||
return m
|
||||
},
|
||||
Leader: func() bool { return admin.Leadership() == nil || admin.Leadership().IsLeader() },
|
||||
}).Run(ctx)
|
||||
// Knowledge bases, with the cluster's own models to embed a
|
||||
// probe (a collection's dimension) and an operator's search.
|
||||
api.SetKB(&adminapi.KBAdmin{Store: st, Embed: embedder(rt.Replay)})
|
||||
@@ -1418,3 +1441,9 @@ func runPresetsPass(ctx context.Context, api *adminapi.API, lead *control.Leader
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// accountingPrice is a configured price as accounting has it.
|
||||
func accountingPrice(p config.Price) accounting.Price {
|
||||
return accounting.Price{PromptPer1k: p.PromptPer1k, CompletionPer1k: p.CompletionPer1k, CachedPer1k: p.CachedPer1k,
|
||||
PerImage: p.PerImage, PerMesh: p.PerMesh}
|
||||
}
|
||||
Reference in new issue
Block a user