ilkerandClaude Opus 5
b7417e1a69
The code behind plan §7.3-7.6: - breaker: folds cooldown and circuit breaking into one component (FailureThreshold=1 gives LiteLLM-style cooldown), with exponential backoff and half-open probes. - endpoint: live state registry. Inflight counting, staleness detection, drain rate derived from counters. A scrape failure does not trip the breaker — the engine may be serving fine while only its metrics endpoint is down. - scrape: parses the §7.6 metric contract. Errors when any of the mandatory three is missing; the metric name mapping is overridable, since names drift between versions (plan §14). - scheduler: the filter -> score -> pick pipeline. A "strategy" is now just a set of scorer weights; chat, batch and long-context are the same code with different weights. Scorers: queue, kv (hard penalty above the ceiling), inflight, load, weight, session (bounded-load consistent hashing). Pickers: max-score, p2c, softmax, round-robin. Decision carries the rationale — which scorer produced what, which filter dropped how many. Tuning the weights without that is guesswork. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
109 lines
2.5 KiB
Go
109 lines
2.5 KiB
Go
package scheduler
|
|
|
|
import (
|
|
"slices"
|
|
|
|
"github.com/cespare/xxhash/v2"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
)
|
|
|
|
// vnodesPerEndpoint is how many virtual nodes each endpoint places on the ring.
|
|
// More nodes improve the distribution; the only cost is the sort at pick time.
|
|
const vnodesPerEndpoint = 64
|
|
|
|
type vnode struct {
|
|
hash uint64
|
|
idx int // index into the eps slice
|
|
}
|
|
|
|
// pickBoundedLoad maps a key onto an endpoint by consistent hashing, walking
|
|
// the ring past any endpoint that is overloaded.
|
|
//
|
|
// "Overloaded" is measured by inflight count: an endpoint carrying more than
|
|
// loadFactor times the average is skipped. If every endpoint is overloaded it
|
|
// returns to the first target — insisting on the target beats losing affinity
|
|
// entirely, because at that point the whole pool is equally loaded anyway.
|
|
//
|
|
// It returns -1 only when there are no candidates.
|
|
func pickBoundedLoad(key string, eps []endpoint.Snapshot, loadFactor float64) int {
|
|
if len(eps) == 0 {
|
|
return -1
|
|
}
|
|
if len(eps) == 1 {
|
|
return 0
|
|
}
|
|
|
|
ring := buildRing(eps)
|
|
h := xxhash.Sum64String(key)
|
|
|
|
// Average load and the resulting ceiling.
|
|
totalInflight := 0
|
|
for _, e := range eps {
|
|
totalInflight += e.Inflight
|
|
}
|
|
avg := float64(totalInflight) / float64(len(eps))
|
|
ceiling := avg * loadFactor
|
|
if ceiling < 1 {
|
|
// Do not let the ceiling fall below 1 at very low load, or affinity
|
|
// would vanish entirely on an idle pool.
|
|
ceiling = 1
|
|
}
|
|
|
|
start, _ := slices.BinarySearchFunc(ring, h, func(v vnode, target uint64) int {
|
|
switch {
|
|
case v.hash < target:
|
|
return -1
|
|
case v.hash > target:
|
|
return 1
|
|
default:
|
|
return 0
|
|
}
|
|
})
|
|
|
|
first := -1
|
|
seen := make(map[int]struct{}, len(eps))
|
|
for i := range ring {
|
|
v := ring[(start+i)%len(ring)]
|
|
if _, dup := seen[v.idx]; dup {
|
|
continue
|
|
}
|
|
seen[v.idx] = struct{}{}
|
|
if first < 0 {
|
|
first = v.idx
|
|
}
|
|
if float64(eps[v.idx].Inflight) <= ceiling {
|
|
return v.idx
|
|
}
|
|
if len(seen) == len(eps) {
|
|
break
|
|
}
|
|
}
|
|
return first
|
|
}
|
|
|
|
func buildRing(eps []endpoint.Snapshot) []vnode {
|
|
ring := make([]vnode, 0, len(eps)*vnodesPerEndpoint)
|
|
var buf []byte
|
|
for i, e := range eps {
|
|
for v := range vnodesPerEndpoint {
|
|
buf = buf[:0]
|
|
buf = append(buf, e.ID...)
|
|
buf = append(buf, '#')
|
|
buf = append(buf, byte('0'+v/10), byte('0'+v%10))
|
|
ring = append(ring, vnode{hash: xxhash.Sum64(buf), idx: i})
|
|
}
|
|
}
|
|
slices.SortFunc(ring, func(a, b vnode) int {
|
|
switch {
|
|
case a.hash < b.hash:
|
|
return -1
|
|
case a.hash > b.hash:
|
|
return 1
|
|
default:
|
|
return 0
|
|
}
|
|
})
|
|
return ring
|
|
}
|