package scheduler import ( "slices" "github.com/cespare/xxhash/v2" "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" ) // vnodesPerEndpoint is how many virtual nodes each endpoint places on the ring. // More nodes improve the distribution; the only cost is the sort at pick time. const vnodesPerEndpoint = 64 type vnode struct { hash uint64 idx int // index into the eps slice } // pickBoundedLoad maps a key onto an endpoint by consistent hashing, walking // the ring past any endpoint that is overloaded. // // "Overloaded" is measured by inflight count: an endpoint carrying more than // loadFactor times the average is skipped. If every endpoint is overloaded it // returns to the first target — insisting on the target beats losing affinity // entirely, because at that point the whole pool is equally loaded anyway. // // It returns -1 only when there are no candidates. func pickBoundedLoad(key string, eps []endpoint.Snapshot, loadFactor float64) int { if len(eps) == 0 { return -1 } if len(eps) == 1 { return 0 } ring := buildRing(eps) h := xxhash.Sum64String(key) // Average load and the resulting ceiling. totalInflight := 0 for _, e := range eps { totalInflight += e.Inflight } avg := float64(totalInflight) / float64(len(eps)) ceiling := avg * loadFactor if ceiling < 1 { // Do not let the ceiling fall below 1 at very low load, or affinity // would vanish entirely on an idle pool. ceiling = 1 } start, _ := slices.BinarySearchFunc(ring, h, func(v vnode, target uint64) int { switch { case v.hash < target: return -1 case v.hash > target: return 1 default: return 0 } }) first := -1 seen := make(map[int]struct{}, len(eps)) for i := range ring { v := ring[(start+i)%len(ring)] if _, dup := seen[v.idx]; dup { continue } seen[v.idx] = struct{}{} if first < 0 { first = v.idx } if float64(eps[v.idx].Inflight) <= ceiling { return v.idx } if len(seen) == len(eps) { break } } return first } func buildRing(eps []endpoint.Snapshot) []vnode { ring := make([]vnode, 0, len(eps)*vnodesPerEndpoint) var buf []byte for i, e := range eps { for v := range vnodesPerEndpoint { buf = buf[:0] buf = append(buf, e.ID...) buf = append(buf, '#') buf = append(buf, byte('0'+v/10), byte('0'+v%10)) ring = append(ring, vnode{hash: xxhash.Sum64(buf), idx: i}) } } slices.SortFunc(ring, func(a, b vnode) int { switch { case a.hash < b.hash: return -1 case a.hash > b.hash: return 1 default: return 0 } }) return ring }