From db448a9e8759426e8b0e5b49ba774809d42ac522 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 16 Sep 2026 23:15:00 +0200 Subject: [PATCH] Stop using a liveness timeout as a generation budget response_header sounds like a check that the endpoint is alive, and for a streaming request it is: headers arrive before generation starts. For a non-streaming request the engine sends nothing until the completion is finished, so the same setting is the total generation budget wearing a liveness timeout's name. At 60s that was invisible on GPUs and a hard cap of roughly a hundred tokens on qwen2.5-coder-14b at 1.7 tokens a second. A request for 120 tokens failed at exactly one minute, and neither the setting's name nor the error said why. Two transports now: the short header timeout for streaming, where it still catches a sick endpoint, and a generation budget for non-streaming, defaulting to ten minutes. One number could not mean both things. Co-Authored-By: Claude Opus 5 (1M context) --- internal/config/config.go | 15 +++++-- internal/router/proxy/proxy.go | 64 ++++++++++++++++++++++------- internal/router/proxy/proxy_test.go | 55 +++++++++++++++++++++++++ internal/router/router.go | 1 + 4 files changed, 117 insertions(+), 18 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index d26f9d05..aad506fa 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -213,11 +213,18 @@ type Band struct { // Timeouts bounds each phase of an upstream request. type Timeouts struct { - Dial time.Duration `koanf:"dial"` + Dial time.Duration `koanf:"dial"` + // ResponseHeader bounds the wait for headers on a streaming request. It is + // a liveness check there, because headers arrive before generation starts. ResponseHeader time.Duration `koanf:"response_header"` - FirstToken time.Duration `koanf:"first_token"` - Idle time.Duration `koanf:"idle"` - Total time.Duration `koanf:"total"` + // Generation bounds a non-streaming request, where the engine sends no + // headers until the completion is done. Setting response_header alone used + // to cap both, so 60s read as a liveness guard and behaved as a limit of + // about a hundred tokens on a slow model. + Generation time.Duration `koanf:"generation"` + FirstToken time.Duration `koanf:"first_token"` + Idle time.Duration `koanf:"idle"` + Total time.Duration `koanf:"total"` } // Scoring holds the thresholds the scorers normalize against. These want to be diff --git a/internal/router/proxy/proxy.go b/internal/router/proxy/proxy.go index 57a020c7..a57f10fe 100644 --- a/internal/router/proxy/proxy.go +++ b/internal/router/proxy/proxy.go @@ -27,8 +27,20 @@ import ( type Config struct { // DialTimeout bounds establishing the TCP connection. DialTimeout time.Duration - // ResponseHeaderTimeout bounds the wait for upstream response headers. + // ResponseHeaderTimeout bounds the wait for upstream response headers on a + // streaming request, where headers arrive before generation begins and so + // are a liveness check. ResponseHeaderTimeout time.Duration + // GenerationTimeout bounds a non-streaming request, where the engine sends + // no headers until the completion is finished. + // + // The two were one setting, and that made ResponseHeaderTimeout a total + // generation budget in disguise for every non-streaming request. At 60s it + // read as a sensible liveness guard and behaved as a cap of roughly a + // hundred tokens on a model generating 1.7 a second — a request for 120 + // tokens failed at exactly one minute, and nothing in the name or the + // message said why. Zero falls back to TotalTimeout. + GenerationTimeout time.Duration // FirstTokenTimeout bounds the wait for the first streamed event. FirstTokenTimeout time.Duration // IdleTimeout bounds the gap between streamed events. A stalled engine @@ -49,6 +61,7 @@ func DefaultConfig() Config { return Config{ DialTimeout: 5 * time.Second, ResponseHeaderTimeout: 60 * time.Second, + GenerationTimeout: 10 * time.Minute, FirstTokenTimeout: 120 * time.Second, IdleTimeout: 60 * time.Second, TotalTimeout: 30 * time.Minute, @@ -74,6 +87,9 @@ func (c *Config) normalize() { if c.TotalTimeout <= 0 { c.TotalTimeout = d.TotalTimeout } + if c.GenerationTimeout <= 0 { + c.GenerationTimeout = d.GenerationTimeout + } if c.MaxIdleConnsPerHost <= 0 { c.MaxIdleConnsPerHost = d.MaxIdleConnsPerHost } @@ -84,29 +100,42 @@ func (c *Config) normalize() { // Proxy forwards requests to engine endpoints. type Proxy struct { - cfg Config + cfg Config + // client is for streaming requests, where headers arrive before generation + // and a short header timeout is a genuine liveness check. client *http.Client + // buffered is for non-streaming requests, where the engine sends nothing + // until the completion is done, so the header timeout is the generation + // budget. Two transports rather than one, because the same number cannot + // mean both things. + buffered *http.Client } // New builds a proxy. func New(cfg Config) *Proxy { cfg.normalize() + transport := func(headerTimeout time.Duration) *http.Transport { + return &http.Transport{ + DialContext: (&net.Dialer{Timeout: cfg.DialTimeout}).DialContext, + ResponseHeaderTimeout: headerTimeout, + MaxIdleConnsPerHost: cfg.MaxIdleConnsPerHost, + MaxIdleConns: cfg.MaxIdleConnsPerHost * 8, + IdleConnTimeout: 90 * time.Second, + // Engines rarely speak h2c, and HTTP/2 over cleartext would + // need explicit setup anyway. + ForceAttemptHTTP2: false, + DisableCompression: true, + } + } return &Proxy{ cfg: cfg, + buffered: &http.Client{ + Transport: transport(cfg.GenerationTimeout), + }, client: &http.Client{ // No client timeout: streaming responses run for minutes and the // per-phase watchdogs below bound them more precisely. - Transport: &http.Transport{ - DialContext: (&net.Dialer{Timeout: cfg.DialTimeout}).DialContext, - ResponseHeaderTimeout: cfg.ResponseHeaderTimeout, - MaxIdleConnsPerHost: cfg.MaxIdleConnsPerHost, - MaxIdleConns: cfg.MaxIdleConnsPerHost * 8, - IdleConnTimeout: 90 * time.Second, - // Engines rarely speak h2c, and HTTP/2 over cleartext would - // need explicit setup anyway. - ForceAttemptHTTP2: false, - DisableCompression: true, - }, + Transport: transport(cfg.ResponseHeaderTimeout), }, } } @@ -175,7 +204,14 @@ func (p *Proxy) Do(ctx context.Context, w http.ResponseWriter, r Request) (Outco req.Header.Set("Content-Type", "application/json") req.ContentLength = int64(len(r.Body)) - resp, err := p.client.Do(req) + // A non-streaming request waits for the whole completion before a single + // header arrives, so it is bounded by the generation budget rather than the + // liveness timeout. + client := p.client + if !r.Stream { + client = p.buffered + } + resp, err := client.Do(req) if err != nil { // The client going away is not the endpoint's fault; never blame it. if ctx.Err() != nil && ctxCauseIsClient(ctx) { diff --git a/internal/router/proxy/proxy_test.go b/internal/router/proxy/proxy_test.go index 9861257a..6e9bc3bb 100644 --- a/internal/router/proxy/proxy_test.go +++ b/internal/router/proxy/proxy_test.go @@ -300,3 +300,58 @@ func TestHopByHopHeadersNotForwarded(t *testing.T) { t.Error("Traceparent was not forwarded; tracing would break") } } + +func TestANonStreamingRequestGetsTheGenerationBudget(t *testing.T) { + // The engine sends no headers until a non-streaming completion is done, so + // the header timeout is the generation budget in disguise. At 60s that read + // as a liveness guard and behaved as a cap of roughly a hundred tokens on a + // model generating 1.7 a second: a request for 120 tokens failed at exactly + // one minute, and nothing said why. + slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(200 * time.Millisecond) // "generating" + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"done"}}]}`)) + })) + defer slow.Close() + + p := New(Config{ + // Short enough that a shared setting would fail the request. + ResponseHeaderTimeout: 20 * time.Millisecond, + GenerationTimeout: 5 * time.Second, + TotalTimeout: 10 * time.Second, + }) + + rec := httptest.NewRecorder() + out, err := p.Do(t.Context(), rec, Request{URL: slow.URL, Body: []byte(`{}`), Stream: false}) + if err != nil { + t.Fatalf("a completion inside the generation budget failed: %v", err) + } + if out.Status != http.StatusOK { + t.Errorf("status %d", out.Status) + } +} + +func TestAStreamingRequestKeepsTheShortLivenessTimeout(t *testing.T) { + // The other half: for streaming, headers come before generation, so a slow + // header really does mean a sick endpoint and must still be caught. + dead := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(2 * time.Second) + w.WriteHeader(http.StatusOK) + })) + defer dead.Close() + + p := New(Config{ + ResponseHeaderTimeout: 50 * time.Millisecond, + GenerationTimeout: 10 * time.Second, + TotalTimeout: 10 * time.Second, + }) + + rec := httptest.NewRecorder() + start := time.Now() + if _, err := p.Do(t.Context(), rec, Request{URL: dead.URL, Body: []byte(`{}`), Stream: true}); err == nil { + t.Fatal("a stalled streaming endpoint was not caught") + } + if elapsed := time.Since(start); elapsed > time.Second { + t.Errorf("took %s; the streaming liveness timeout is not being applied", elapsed) + } +} diff --git a/internal/router/router.go b/internal/router/router.go index b879ded3..90a9a4ac 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -155,6 +155,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) { px: proxy.New(proxy.Config{ DialTimeout: cfg.Router.Timeouts.Dial, ResponseHeaderTimeout: cfg.Router.Timeouts.ResponseHeader, + GenerationTimeout: cfg.Router.Timeouts.Generation, FirstTokenTimeout: cfg.Router.Timeouts.FirstToken, IdleTimeout: cfg.Router.Timeouts.Idle, TotalTimeout: cfg.Router.Timeouts.Total,