package control import ( "context" "encoding/json" "log/slog" "sync" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/bench" "gitea.mynodes.xyz/ilker/llmgw/internal/control/quota" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // QuotaStore is where capacity quotas and the tenancy tree are kept. type QuotaStore interface { CapacityQuotas(ctx context.Context) ([]store.CapacityQuota, error) ScopeParents(ctx context.Context) (map[string][]string, error) } // quotaRefresh is how stale the reconciler's view of quotas may get. A quota // changed by an operator takes effect within this; the reconciler asks many // times a pass and the store is not asked each time. const quotaRefresh = 30 * time.Second // QuotaPolicy returns what the reconciler asks for its quotas: the last // policy read from the store, read again once it is older than quotaRefresh. // A store that fails keeps the last good policy, so a database hiccup does // not drop every guarantee and limit at once. func QuotaPolicy(st QuotaStore, log *slog.Logger) func() *quota.Policy { var mu sync.Mutex var cur *quota.Policy var at time.Time return func() *quota.Policy { mu.Lock() defer mu.Unlock() if cur != nil && time.Since(at) < quotaRefresh { return cur } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() qs, err := st.CapacityQuotas(ctx) if err == nil { var parents map[string][]string if parents, err = st.ScopeParents(ctx); err == nil { cur, at = quota.New(qs, parents), time.Now() return cur } } log.Warn("could not read capacity quotas; keeping the last ones", "err", err) at = time.Now() // not again on every call while the store is down return cur } } // MeasurementStore is where benchmark results are kept. type MeasurementStore interface { Capacities(ctx context.Context) ([]store.StoredCapacity, error) } // Measurements returns what the reconciler asks for benchmark results, keyed // model/profile, read again once older than quotaRefresh; like QuotaPolicy it // keeps the last good answer when the store fails. func Measurements(st MeasurementStore, log *slog.Logger) func() map[string]reconciler.Measured { var mu sync.Mutex var cur map[string]reconciler.Measured var at time.Time return func() map[string]reconciler.Measured { mu.Lock() defer mu.Unlock() if cur != nil && time.Since(at) < quotaRefresh { return cur } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() at = time.Now() rows, err := st.Capacities(ctx) if err != nil { log.Warn("could not read benchmark results; keeping the last ones", "err", err) return cur } next := map[string]reconciler.Measured{} for _, row := range rows { var c bench.Capacity if json.Unmarshal(row.Measurement, &c) != nil { continue } next[row.Model+"/"+row.Profile] = reconciler.Measured{ MaxConcurrency: c.MaxConcurrencyAtSLO, TokPerSec: c.ThroughputTokPerSec} } cur = next return cur } }