Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
80 lines
2.6 KiB
Go
80 lines
2.6 KiB
Go
// Package canary compares a model's candidate with the model it may
|
|
// replace, from what each side served, and says when to give up on it.
|
|
package canary
|
|
|
|
import (
|
|
"fmt"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
)
|
|
|
|
// Side is what one side of a canary served since it began.
|
|
type Side struct {
|
|
Requests int64 `json:"requests"`
|
|
// Errors are answers of 500 and up: the engine's failures, not the
|
|
// client's mistakes.
|
|
Errors int64 `json:"errors"`
|
|
// TTFT is over streamed requests, Latency over the rest, both means in
|
|
// milliseconds with how many they are over.
|
|
TTFTMs float64 `json:"ttft_ms"`
|
|
TTFTCount int64 `json:"ttft_count"`
|
|
LatencyMs float64 `json:"latency_ms"`
|
|
LatencyCount int64 `json:"latency_count"`
|
|
}
|
|
|
|
// ErrorRate is errors over requests, 0 with none.
|
|
func (v Side) ErrorRate() float64 {
|
|
if v.Requests == 0 {
|
|
return 0
|
|
}
|
|
return float64(v.Errors) / float64(v.Requests)
|
|
}
|
|
|
|
// Add counts one finished request.
|
|
func (v *Side) Add(status int, streamed bool, ttft, took time.Duration) {
|
|
v.Requests++
|
|
if status >= 500 {
|
|
v.Errors++
|
|
}
|
|
switch {
|
|
case streamed && ttft > 0:
|
|
v.TTFTMs += (float64(ttft.Milliseconds()) - v.TTFTMs) / float64(v.TTFTCount+1)
|
|
v.TTFTCount++
|
|
case !streamed && status < 400:
|
|
v.LatencyMs += (float64(took.Milliseconds()) - v.LatencyMs) / float64(v.LatencyCount+1)
|
|
v.LatencyCount++
|
|
}
|
|
}
|
|
|
|
// Stat compares the two sides of one model's canary.
|
|
type Stat struct {
|
|
Canary string `json:"canary"`
|
|
Since time.Time `json:"since"`
|
|
Stable Side `json:"stable"`
|
|
Candidate Side `json:"candidate"`
|
|
}
|
|
|
|
// minTimed is how many timed requests a side needs before its mean counts.
|
|
const minTimed = 10
|
|
|
|
// Verdict says whether the candidate has done worse than the canary allows,
|
|
// and how. Nothing is decided before it has served MinRequests.
|
|
func Verdict(s Stat, cn catalog.Canary) (rollback bool, why string) {
|
|
minReq, maxErr, maxRatio := cn.Limits()
|
|
c, st := s.Candidate, s.Stable
|
|
if c.Requests < int64(minReq) {
|
|
return false, ""
|
|
}
|
|
if d := c.ErrorRate() - st.ErrorRate(); d > maxErr {
|
|
return true, fmt.Sprintf("errors in %.1f%% of its requests against %.1f%%", 100*c.ErrorRate(), 100*st.ErrorRate())
|
|
}
|
|
if c.TTFTCount >= minTimed && st.TTFTCount >= minTimed && st.TTFTMs > 0 && c.TTFTMs > maxRatio*st.TTFTMs {
|
|
return true, fmt.Sprintf("first token after %.0f ms on average against %.0f ms", c.TTFTMs, st.TTFTMs)
|
|
}
|
|
if c.LatencyCount >= minTimed && st.LatencyCount >= minTimed && st.LatencyMs > 0 && c.LatencyMs > maxRatio*st.LatencyMs {
|
|
return true, fmt.Sprintf("answers in %.0f ms on average against %.0f ms", c.LatencyMs, st.LatencyMs)
|
|
}
|
|
return false, ""
|
|
}
|