// Package held records, on the leader, what each model holds of the // cluster: its replicas' GPUs and, for one on CPUs, its cores. Kept by the // hour, it is the cost side of a token's price (llmgw ctl prices suggest). package held import ( "context" "log/slog" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // Inventory is where the allocations are. type Inventory interface { Nodes() []inventory.Node Allocations(nodeID string) []inventory.Allocation } // Store keeps what was held. type Store interface { AddHeld(ctx context.Context, at time.Time, held []store.Held) error } // Recorder adds, every Every, what each model holds for that long. type Recorder struct { Inv Inventory // Models is each replica's model, by replica id. Models func() map[string]string Store Store Leader func() bool Every time.Duration Log *slog.Logger } // Run records until ctx ends. func (r *Recorder) Run(ctx context.Context) { every := r.Every if every <= 0 { every = 5 * time.Minute } t := time.NewTicker(every) defer t.Stop() for { select { case <-ctx.Done(): return case now := <-t.C: if r.Leader != nil && !r.Leader() { continue } held := Sample(r.Inv, r.Models(), every) if err := r.Store.AddHeld(ctx, now, held); err != nil && r.Log != nil { r.Log.Warn("could not record what the models held, for their prices", "err", err) } } } } // Sample is what each model holds now, for d: a GPU for each card a shard // was given, and the cores of a shard given cores and no GPU. A replica // whose model is not known (headroom kept for another, one just reaped) is // left out. func Sample(inv Inventory, models map[string]string, d time.Duration) []store.Held { by := map[string]*store.Held{} var order []string secs := int64(d.Seconds()) for _, n := range inv.Nodes() { for _, a := range inv.Allocations(n.ID) { m, ok := models[a.ReplicaID] if !ok { continue } h := by[m] if h == nil { h = &store.Held{Model: m} by[m] = h order = append(order, m) } if g := len(a.GPUIndices); g > 0 { h.GPUSeconds += int64(g) * secs } else { h.CoreSeconds += int64(max(a.CPUCores, 1)) * secs } } } out := make([]store.Held, 0, len(order)) for _, m := range order { out = append(out, *by[m]) } return out }