Files
llmgw/internal/canary/canary.go
T

80 lines
2.6 KiB
Go

// Package canary compares a model's candidate with the model it may
// replace, from what each side served, and says when to give up on it.
package canary
import (
"fmt"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// Side is what one side of a canary served since it began.
type Side struct {
Requests int64 `json:"requests"`
// Errors are answers of 500 and up: the engine's failures, not the
// client's mistakes.
Errors int64 `json:"errors"`
// TTFT is over streamed requests, Latency over the rest, both means in
// milliseconds with how many they are over.
TTFTMs float64 `json:"ttft_ms"`
TTFTCount int64 `json:"ttft_count"`
LatencyMs float64 `json:"latency_ms"`
LatencyCount int64 `json:"latency_count"`
}
// ErrorRate is errors over requests, 0 with none.
func (v Side) ErrorRate() float64 {
if v.Requests == 0 {
return 0
}
return float64(v.Errors) / float64(v.Requests)
}
// Add counts one finished request.
func (v *Side) Add(status int, streamed bool, ttft, took time.Duration) {
v.Requests++
if status >= 500 {
v.Errors++
}
switch {
case streamed && ttft > 0:
v.TTFTMs += (float64(ttft.Milliseconds()) - v.TTFTMs) / float64(v.TTFTCount+1)
v.TTFTCount++
case !streamed && status < 400:
v.LatencyMs += (float64(took.Milliseconds()) - v.LatencyMs) / float64(v.LatencyCount+1)
v.LatencyCount++
}
}
// Stat compares the two sides of one model's canary.
type Stat struct {
Canary string `json:"canary"`
Since time.Time `json:"since"`
Stable Side `json:"stable"`
Candidate Side `json:"candidate"`
}
// minTimed is how many timed requests a side needs before its mean counts.
const minTimed = 10
// Verdict says whether the candidate has done worse than the canary allows,
// and how. Nothing is decided before it has served MinRequests.
func Verdict(s Stat, cn catalog.Canary) (rollback bool, why string) {
minReq, maxErr, maxRatio := cn.Limits()
c, st := s.Candidate, s.Stable
if c.Requests < int64(minReq) {
return false, ""
}
if d := c.ErrorRate() - st.ErrorRate(); d > maxErr {
return true, fmt.Sprintf("errors in %.1f%% of its requests against %.1f%%", 100*c.ErrorRate(), 100*st.ErrorRate())
}
if c.TTFTCount >= minTimed && st.TTFTCount >= minTimed && st.TTFTMs > 0 && c.TTFTMs > maxRatio*st.TTFTMs {
return true, fmt.Sprintf("first token after %.0f ms on average against %.0f ms", c.TTFTMs, st.TTFTMs)
}
if c.LatencyCount >= minTimed && st.LatencyCount >= minTimed && st.LatencyMs > 0 && c.LatencyMs > maxRatio*st.LatencyMs {
return true, fmt.Sprintf("answers in %.0f ms on average against %.0f ms", c.LatencyMs, st.LatencyMs)
}
return false, ""
}