Ilker ManapandClaude Opus 5.5
78e41460f9
Stored ctl bench results reach the reconciler: requests per replica past the measured SLO concurrency scale a model up, a replica is taken back only if the rest stay under 70% of theirs, and among profiles of equal priority the most measured tokens per GPU or core is tried first. ctl model target sizes a model for a throughput against the room there is. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
96 lines
3.0 KiB
Go
96 lines
3.0 KiB
Go
package control
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"log/slog"
|
|
"sync"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// QuotaStore is where capacity quotas and the tenancy tree are kept.
|
|
type QuotaStore interface {
|
|
CapacityQuotas(ctx context.Context) ([]store.CapacityQuota, error)
|
|
ScopeParents(ctx context.Context) (map[string][]string, error)
|
|
}
|
|
|
|
// quotaRefresh is how stale the reconciler's view of quotas may get. A quota
|
|
// changed by an operator takes effect within this; the reconciler asks many
|
|
// times a pass and the store is not asked each time.
|
|
const quotaRefresh = 30 * time.Second
|
|
|
|
// QuotaPolicy returns what the reconciler asks for its quotas: the last
|
|
// policy read from the store, read again once it is older than quotaRefresh.
|
|
// A store that fails keeps the last good policy, so a database hiccup does
|
|
// not drop every guarantee and limit at once.
|
|
func QuotaPolicy(st QuotaStore, log *slog.Logger) func() *quota.Policy {
|
|
var mu sync.Mutex
|
|
var cur *quota.Policy
|
|
var at time.Time
|
|
return func() *quota.Policy {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if cur != nil && time.Since(at) < quotaRefresh {
|
|
return cur
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
qs, err := st.CapacityQuotas(ctx)
|
|
if err == nil {
|
|
var parents map[string][]string
|
|
if parents, err = st.ScopeParents(ctx); err == nil {
|
|
cur, at = quota.New(qs, parents), time.Now()
|
|
return cur
|
|
}
|
|
}
|
|
log.Warn("could not read capacity quotas; keeping the last ones", "err", err)
|
|
at = time.Now() // not again on every call while the store is down
|
|
return cur
|
|
}
|
|
}
|
|
|
|
// MeasurementStore is where benchmark results are kept.
|
|
type MeasurementStore interface {
|
|
Capacities(ctx context.Context) ([]store.StoredCapacity, error)
|
|
}
|
|
|
|
// Measurements returns what the reconciler asks for benchmark results, keyed
|
|
// model/profile, read again once older than quotaRefresh; like QuotaPolicy it
|
|
// keeps the last good answer when the store fails.
|
|
func Measurements(st MeasurementStore, log *slog.Logger) func() map[string]reconciler.Measured {
|
|
var mu sync.Mutex
|
|
var cur map[string]reconciler.Measured
|
|
var at time.Time
|
|
return func() map[string]reconciler.Measured {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if cur != nil && time.Since(at) < quotaRefresh {
|
|
return cur
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
at = time.Now()
|
|
rows, err := st.Capacities(ctx)
|
|
if err != nil {
|
|
log.Warn("could not read benchmark results; keeping the last ones", "err", err)
|
|
return cur
|
|
}
|
|
next := map[string]reconciler.Measured{}
|
|
for _, row := range rows {
|
|
var c bench.Capacity
|
|
if json.Unmarshal(row.Measurement, &c) != nil {
|
|
continue
|
|
}
|
|
next[row.Model+"/"+row.Profile] = reconciler.Measured{
|
|
MaxConcurrency: c.MaxConcurrencyAtSLO, TokPerSec: c.ThroughputTokPerSec}
|
|
}
|
|
cur = next
|
|
return cur
|
|
}
|
|
}
|