// Package canary compares a model's candidate with the model it may // replace, from what each side served, and says when to give up on it. package canary import ( "fmt" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" ) // Side is what one side of a canary served since it began. type Side struct { Requests int64 `json:"requests"` // Errors are answers of 500 and up: the engine's failures, not the // client's mistakes. Errors int64 `json:"errors"` // TTFT is over streamed requests, Latency over the rest, both means in // milliseconds with how many they are over. TTFTMs float64 `json:"ttft_ms"` TTFTCount int64 `json:"ttft_count"` LatencyMs float64 `json:"latency_ms"` LatencyCount int64 `json:"latency_count"` } // ErrorRate is errors over requests, 0 with none. func (v Side) ErrorRate() float64 { if v.Requests == 0 { return 0 } return float64(v.Errors) / float64(v.Requests) } // Add counts one finished request. func (v *Side) Add(status int, streamed bool, ttft, took time.Duration) { v.Requests++ if status >= 500 { v.Errors++ } switch { case streamed && ttft > 0: v.TTFTMs += (float64(ttft.Milliseconds()) - v.TTFTMs) / float64(v.TTFTCount+1) v.TTFTCount++ case !streamed && status < 400: v.LatencyMs += (float64(took.Milliseconds()) - v.LatencyMs) / float64(v.LatencyCount+1) v.LatencyCount++ } } // Stat compares the two sides of one model's canary. type Stat struct { Canary string `json:"canary"` Since time.Time `json:"since"` Stable Side `json:"stable"` Candidate Side `json:"candidate"` } // minTimed is how many timed requests a side needs before its mean counts. const minTimed = 10 // Verdict says whether the candidate has done worse than the canary allows, // and how. Nothing is decided before it has served MinRequests. func Verdict(s Stat, cn catalog.Canary) (rollback bool, why string) { minReq, maxErr, maxRatio := cn.Limits() c, st := s.Candidate, s.Stable if c.Requests < int64(minReq) { return false, "" } if d := c.ErrorRate() - st.ErrorRate(); d > maxErr { return true, fmt.Sprintf("errors in %.1f%% of its requests against %.1f%%", 100*c.ErrorRate(), 100*st.ErrorRate()) } if c.TTFTCount >= minTimed && st.TTFTCount >= minTimed && st.TTFTMs > 0 && c.TTFTMs > maxRatio*st.TTFTMs { return true, fmt.Sprintf("first token after %.0f ms on average against %.0f ms", c.TTFTMs, st.TTFTMs) } if c.LatencyCount >= minTimed && st.LatencyCount >= minTimed && st.LatencyMs > 0 && c.LatencyMs > maxRatio*st.LatencyMs { return true, fmt.Sprintf("answers in %.0f ms on average against %.0f ms", c.LatencyMs, st.LatencyMs) } return false, "" }