package scheduler import ( "gitea.mynodes.xyz/ilker/llmgw/internal/router/admission" "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" ) // AdapterView is where a LoRA adapter that loads on demand is, at the moment // of one decision. type AdapterView struct { // Holders are the replicas that have the adapter loaded, or are loading // it now — a request sent to one of those waits for that load rather // than starting a second. Holders map[string]bool // Free are the replicas with room for one more adapter without // unloading another. Free map[string]bool } // AdapterAffinity keeps the replicas that already hold the requested // adapter, while one of them has room for the request; failing that, the // replicas that can load it without unloading another; failing that, // everything. Requests for a model's own weights, and for adapters loaded at // start, are left alone. // // A filter rather than a scorer, unlike prefix affinity. A miss here is not // a slower prefill but a load — seconds, and perhaps another adapter thrown // out — and the pickers sample: P2C would send a third of the requests for an // adapter held by one of three replicas somewhere else, each a load. But it // gives way to load, by the same measure as pool priority: a holder that is // saturated does not keep its adapter's traffic, and the request goes where // the other scorers send it, which loads it there. That is how an adapter // in demand spreads to a second replica, and the only way it does. func AdapterAffinity(targetWaiting, kvCeiling float64) Filter { return adapterAffinityFilter{t: admission.Thresholds{TargetWaitingPerReplica: targetWaiting, KVCeiling: kvCeiling}} } type adapterAffinityFilter struct{ t admission.Thresholds } func (adapterAffinityFilter) Name() string { return "adapter-affinity" } func (f adapterAffinityFilter) Filter(c *Ctx, eps []endpoint.Snapshot) []endpoint.Snapshot { if c.Adapters == nil || c.Req.Adapter == "" { return eps } for _, set := range []map[string]bool{c.Adapters.Holders, c.Adapters.Free} { var keep []endpoint.Snapshot for _, e := range eps { if set[e.ID] && admission.Saturation([]endpoint.Snapshot{e}, 0, f.t) < 1 { keep = append(keep, e) } } if len(keep) > 0 { return keep } } return eps }