diff --git a/docs/reference.md b/docs/reference.md index 4910549..47cdb82 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -1747,10 +1747,15 @@ point and avoids depending on the script being on PATH. `time.Duration` +TTFTp99 is the time to the first token 99 requests in 100 are within. + ### `tpot_p99` `time.Duration` +TPOTp99 is the time per output token, after the first, 99 streamed +requests in 100 are within. + ## Scaling ### `min` diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index 9f4429c..f77d769 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -39,9 +39,13 @@ type Scaling struct { func (s Scaling) OnDemand() bool { return s.Min == 0 && s.Max > 0 } // SLO is the latency target a model is expected to meet. `ctl bench` measures -// capacity against it (plan §6.2.1). +// capacity against it (plan §6.2.1), and admission holds the lower bands +// back while the model's recent p99 is past it. type SLO struct { + // TTFTp99 is the time to the first token 99 requests in 100 are within. TTFTp99 time.Duration `koanf:"ttft_p99"` + // TPOTp99 is the time per output token, after the first, 99 streamed + // requests in 100 are within. TPOTp99 time.Duration `koanf:"tpot_p99"` } diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index 4647cd7..b362fbd 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -265,8 +265,8 @@ var Reference = []Block{ {Key: "python", Type: "string", Doc: "Python is the interpreter of the SGLang virtualenv. SGLang is launched as\na module rather than a console script, which is the documented entry\npoint and avoids depending on the script being on PATH.", Nested: ""}, }}, {Package: "catalog", Name: "SLO", Title: "SLO", Doc: "", Fields: []Field{ - {Key: "ttft_p99", Type: "time.Duration", Doc: "", Nested: ""}, - {Key: "tpot_p99", Type: "time.Duration", Doc: "", Nested: ""}, + {Key: "ttft_p99", Type: "time.Duration", Doc: "TTFTp99 is the time to the first token 99 requests in 100 are within.", Nested: ""}, + {Key: "tpot_p99", Type: "time.Duration", Doc: "TPOTp99 is the time per output token, after the first, 99 streamed\nrequests in 100 are within.", Nested: ""}, }}, {Package: "catalog", Name: "Scaling", Title: "Scaling", Doc: "", Fields: []Field{ {Key: "min", Type: "int", Doc: "", Nested: ""},