Stop using a liveness timeout as a generation budget
CI / test (push) Successful in 1m23s

response_header sounds like a check that the endpoint is alive, and for a
streaming request it is: headers arrive before generation starts. For a
non-streaming request the engine sends nothing until the completion is
finished, so the same setting is the total generation budget wearing a
liveness timeout's name.

At 60s that was invisible on GPUs and a hard cap of roughly a hundred
tokens on qwen2.5-coder-14b at 1.7 tokens a second. A request for 120
tokens failed at exactly one minute, and neither the setting's name nor the
error said why.

Two transports now: the short header timeout for streaming, where it still
catches a sick endpoint, and a generation budget for non-streaming,
defaulting to ten minutes. One number could not mean both things.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-16 23:15:00 +02:00
1 parent 3081c6f69b
commit db448a9e87
4 files changed
+117 -18

No files matched your search

+50 -14
View File
@@ -27,8 +27,20 @@ import (
type Config struct {
// DialTimeout bounds establishing the TCP connection.
DialTimeout time.Duration
// ResponseHeaderTimeout bounds the wait for upstream response headers.
// ResponseHeaderTimeout bounds the wait for upstream response headers on a
// streaming request, where headers arrive before generation begins and so
// are a liveness check.
ResponseHeaderTimeout time.Duration
// GenerationTimeout bounds a non-streaming request, where the engine sends
// no headers until the completion is finished.
//
// The two were one setting, and that made ResponseHeaderTimeout a total
// generation budget in disguise for every non-streaming request. At 60s it
// read as a sensible liveness guard and behaved as a cap of roughly a
// hundred tokens on a model generating 1.7 a second — a request for 120
// tokens failed at exactly one minute, and nothing in the name or the
// message said why. Zero falls back to TotalTimeout.
GenerationTimeout time.Duration
// FirstTokenTimeout bounds the wait for the first streamed event.
FirstTokenTimeout time.Duration
// IdleTimeout bounds the gap between streamed events. A stalled engine
@@ -49,6 +61,7 @@ func DefaultConfig() Config {
return Config{
DialTimeout: 5 * time.Second,
ResponseHeaderTimeout: 60 * time.Second,
GenerationTimeout: 10 * time.Minute,
FirstTokenTimeout: 120 * time.Second,
IdleTimeout: 60 * time.Second,
TotalTimeout: 30 * time.Minute,
@@ -74,6 +87,9 @@ func (c *Config) normalize() {
if c.TotalTimeout <= 0 {
c.TotalTimeout = d.TotalTimeout
}
if c.GenerationTimeout <= 0 {
c.GenerationTimeout = d.GenerationTimeout
}
if c.MaxIdleConnsPerHost <= 0 {
c.MaxIdleConnsPerHost = d.MaxIdleConnsPerHost
}
@@ -84,29 +100,42 @@ func (c *Config) normalize() {
// Proxy forwards requests to engine endpoints.
type Proxy struct {
cfg Config
cfg Config
// client is for streaming requests, where headers arrive before generation
// and a short header timeout is a genuine liveness check.
client *http.Client
// buffered is for non-streaming requests, where the engine sends nothing
// until the completion is done, so the header timeout is the generation
// budget. Two transports rather than one, because the same number cannot
// mean both things.
buffered *http.Client
}
// New builds a proxy.
func New(cfg Config) *Proxy {
cfg.normalize()
transport := func(headerTimeout time.Duration) *http.Transport {
return &http.Transport{
DialContext: (&net.Dialer{Timeout: cfg.DialTimeout}).DialContext,
ResponseHeaderTimeout: headerTimeout,
MaxIdleConnsPerHost: cfg.MaxIdleConnsPerHost,
MaxIdleConns: cfg.MaxIdleConnsPerHost * 8,
IdleConnTimeout: 90 * time.Second,
// Engines rarely speak h2c, and HTTP/2 over cleartext would
// need explicit setup anyway.
ForceAttemptHTTP2: false,
DisableCompression: true,
}
}
return &Proxy{
cfg: cfg,
buffered: &http.Client{
Transport: transport(cfg.GenerationTimeout),
},
client: &http.Client{
// No client timeout: streaming responses run for minutes and the
// per-phase watchdogs below bound them more precisely.
Transport: &http.Transport{
DialContext: (&net.Dialer{Timeout: cfg.DialTimeout}).DialContext,
ResponseHeaderTimeout: cfg.ResponseHeaderTimeout,
MaxIdleConnsPerHost: cfg.MaxIdleConnsPerHost,
MaxIdleConns: cfg.MaxIdleConnsPerHost * 8,
IdleConnTimeout: 90 * time.Second,
// Engines rarely speak h2c, and HTTP/2 over cleartext would
// need explicit setup anyway.
ForceAttemptHTTP2: false,
DisableCompression: true,
},
Transport: transport(cfg.ResponseHeaderTimeout),
},
}
}
@@ -175,7 +204,14 @@ func (p *Proxy) Do(ctx context.Context, w http.ResponseWriter, r Request) (Outco
req.Header.Set("Content-Type", "application/json")
req.ContentLength = int64(len(r.Body))
resp, err := p.client.Do(req)
// A non-streaming request waits for the whole completion before a single
// header arrives, so it is bounded by the generation budget rather than the
// liveness timeout.
client := p.client
if !r.Stream {
client = p.buffered
}
resp, err := client.Do(req)
if err != nil {
// The client going away is not the endpoint's fault; never blame it.
if ctx.Err() != nil && ctxCauseIsClient(ctx) {