SLO fields described
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5b92ae0277
commit
74429f1c00
3 files changed
+12
-3
No files matched your search
@@ -1747,10 +1747,15 @@ point and avoids depending on the script being on PATH.
|
||||
|
||||
`time.Duration`
|
||||
|
||||
TTFTp99 is the time to the first token 99 requests in 100 are within.
|
||||
|
||||
### `tpot_p99`
|
||||
|
||||
`time.Duration`
|
||||
|
||||
TPOTp99 is the time per output token, after the first, 99 streamed
|
||||
requests in 100 are within.
|
||||
|
||||
## Scaling
|
||||
|
||||
### `min`
|
||||
|
||||
@@ -39,9 +39,13 @@ type Scaling struct {
|
||||
func (s Scaling) OnDemand() bool { return s.Min == 0 && s.Max > 0 }
|
||||
|
||||
// SLO is the latency target a model is expected to meet. `ctl bench` measures
|
||||
// capacity against it (plan §6.2.1).
|
||||
// capacity against it (plan §6.2.1), and admission holds the lower bands
|
||||
// back while the model's recent p99 is past it.
|
||||
type SLO struct {
|
||||
// TTFTp99 is the time to the first token 99 requests in 100 are within.
|
||||
TTFTp99 time.Duration `koanf:"ttft_p99"`
|
||||
// TPOTp99 is the time per output token, after the first, 99 streamed
|
||||
// requests in 100 are within.
|
||||
TPOTp99 time.Duration `koanf:"tpot_p99"`
|
||||
}
|
||||
|
||||
|
||||
@@ -265,8 +265,8 @@ var Reference = []Block{
|
||||
{Key: "python", Type: "string", Doc: "Python is the interpreter of the SGLang virtualenv. SGLang is launched as\na module rather than a console script, which is the documented entry\npoint and avoids depending on the script being on PATH.", Nested: ""},
|
||||
}},
|
||||
{Package: "catalog", Name: "SLO", Title: "SLO", Doc: "", Fields: []Field{
|
||||
{Key: "ttft_p99", Type: "time.Duration", Doc: "", Nested: ""},
|
||||
{Key: "tpot_p99", Type: "time.Duration", Doc: "", Nested: ""},
|
||||
{Key: "ttft_p99", Type: "time.Duration", Doc: "TTFTp99 is the time to the first token 99 requests in 100 are within.", Nested: ""},
|
||||
{Key: "tpot_p99", Type: "time.Duration", Doc: "TPOTp99 is the time per output token, after the first, 99 streamed\nrequests in 100 are within.", Nested: ""},
|
||||
}},
|
||||
{Package: "catalog", Name: "Scaling", Title: "Scaling", Doc: "", Fields: []Field{
|
||||
{Key: "min", Type: "int", Doc: "", Nested: ""},
|
||||
|
||||
Reference in new issue
Block a user