SLO fields described

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 23:13:30 +02:00
1 parent 5b92ae0277
commit 74429f1c00
3 files changed
+12 -3

No files matched your search

+5
View File
@@ -1747,10 +1747,15 @@ point and avoids depending on the script being on PATH.
`time.Duration`
TTFTp99 is the time to the first token 99 requests in 100 are within.
### `tpot_p99`
`time.Duration`
TPOTp99 is the time per output token, after the first, 99 streamed
requests in 100 are within.
## Scaling
### `min`
+5 -1
View File
@@ -39,9 +39,13 @@ type Scaling struct {
func (s Scaling) OnDemand() bool { return s.Min == 0 && s.Max > 0 }
// SLO is the latency target a model is expected to meet. `ctl bench` measures
// capacity against it (plan §6.2.1).
// capacity against it (plan §6.2.1), and admission holds the lower bands
// back while the model's recent p99 is past it.
type SLO struct {
// TTFTp99 is the time to the first token 99 requests in 100 are within.
TTFTp99 time.Duration `koanf:"ttft_p99"`
// TPOTp99 is the time per output token, after the first, 99 streamed
// requests in 100 are within.
TPOTp99 time.Duration `koanf:"tpot_p99"`
}
+2 -2
View File
@@ -265,8 +265,8 @@ var Reference = []Block{
{Key: "python", Type: "string", Doc: "Python is the interpreter of the SGLang virtualenv. SGLang is launched as\na module rather than a console script, which is the documented entry\npoint and avoids depending on the script being on PATH.", Nested: ""},
}},
{Package: "catalog", Name: "SLO", Title: "SLO", Doc: "", Fields: []Field{
{Key: "ttft_p99", Type: "time.Duration", Doc: "", Nested: ""},
{Key: "tpot_p99", Type: "time.Duration", Doc: "", Nested: ""},
{Key: "ttft_p99", Type: "time.Duration", Doc: "TTFTp99 is the time to the first token 99 requests in 100 are within.", Nested: ""},
{Key: "tpot_p99", Type: "time.Duration", Doc: "TPOTp99 is the time per output token, after the first, 99 streamed\nrequests in 100 are within.", Nested: ""},
}},
{Package: "catalog", Name: "Scaling", Title: "Scaling", Doc: "", Fields: []Field{
{Key: "min", Type: "int", Doc: "", Nested: ""},