response_header sounds like a check that the endpoint is alive, and for a streaming request it is: headers arrive before generation starts. For a non-streaming request the engine sends nothing until the completion is finished, so the same setting is the total generation budget wearing a liveness timeout's name. At 60s that was invisible on GPUs and a hard cap of roughly a hundred tokens on qwen2.5-coder-14b at 1.7 tokens a second. A request for 120 tokens failed at exactly one minute, and neither the setting's name nor the error said why. Two transports now: the short header timeout for streaming, where it still catches a sick endpoint, and a generation budget for non-streaming, defaulting to ten minutes. One number could not mean both things. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
3081c6f69b
commit
db448a9e87
4 files changed
+117
-18
No files matched your search
@@ -213,11 +213,18 @@ type Band struct {
|
||||
|
||||
// Timeouts bounds each phase of an upstream request.
|
||||
type Timeouts struct {
|
||||
Dial time.Duration `koanf:"dial"`
|
||||
Dial time.Duration `koanf:"dial"`
|
||||
// ResponseHeader bounds the wait for headers on a streaming request. It is
|
||||
// a liveness check there, because headers arrive before generation starts.
|
||||
ResponseHeader time.Duration `koanf:"response_header"`
|
||||
FirstToken time.Duration `koanf:"first_token"`
|
||||
Idle time.Duration `koanf:"idle"`
|
||||
Total time.Duration `koanf:"total"`
|
||||
// Generation bounds a non-streaming request, where the engine sends no
|
||||
// headers until the completion is done. Setting response_header alone used
|
||||
// to cap both, so 60s read as a liveness guard and behaved as a limit of
|
||||
// about a hundred tokens on a slow model.
|
||||
Generation time.Duration `koanf:"generation"`
|
||||
FirstToken time.Duration `koanf:"first_token"`
|
||||
Idle time.Duration `koanf:"idle"`
|
||||
Total time.Duration `koanf:"total"`
|
||||
}
|
||||
|
||||
// Scoring holds the thresholds the scorers normalize against. These want to be
|
||||
|
||||
@@ -27,8 +27,20 @@ import (
|
||||
type Config struct {
|
||||
// DialTimeout bounds establishing the TCP connection.
|
||||
DialTimeout time.Duration
|
||||
// ResponseHeaderTimeout bounds the wait for upstream response headers.
|
||||
// ResponseHeaderTimeout bounds the wait for upstream response headers on a
|
||||
// streaming request, where headers arrive before generation begins and so
|
||||
// are a liveness check.
|
||||
ResponseHeaderTimeout time.Duration
|
||||
// GenerationTimeout bounds a non-streaming request, where the engine sends
|
||||
// no headers until the completion is finished.
|
||||
//
|
||||
// The two were one setting, and that made ResponseHeaderTimeout a total
|
||||
// generation budget in disguise for every non-streaming request. At 60s it
|
||||
// read as a sensible liveness guard and behaved as a cap of roughly a
|
||||
// hundred tokens on a model generating 1.7 a second — a request for 120
|
||||
// tokens failed at exactly one minute, and nothing in the name or the
|
||||
// message said why. Zero falls back to TotalTimeout.
|
||||
GenerationTimeout time.Duration
|
||||
// FirstTokenTimeout bounds the wait for the first streamed event.
|
||||
FirstTokenTimeout time.Duration
|
||||
// IdleTimeout bounds the gap between streamed events. A stalled engine
|
||||
@@ -49,6 +61,7 @@ func DefaultConfig() Config {
|
||||
return Config{
|
||||
DialTimeout: 5 * time.Second,
|
||||
ResponseHeaderTimeout: 60 * time.Second,
|
||||
GenerationTimeout: 10 * time.Minute,
|
||||
FirstTokenTimeout: 120 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
TotalTimeout: 30 * time.Minute,
|
||||
@@ -74,6 +87,9 @@ func (c *Config) normalize() {
|
||||
if c.TotalTimeout <= 0 {
|
||||
c.TotalTimeout = d.TotalTimeout
|
||||
}
|
||||
if c.GenerationTimeout <= 0 {
|
||||
c.GenerationTimeout = d.GenerationTimeout
|
||||
}
|
||||
if c.MaxIdleConnsPerHost <= 0 {
|
||||
c.MaxIdleConnsPerHost = d.MaxIdleConnsPerHost
|
||||
}
|
||||
@@ -84,29 +100,42 @@ func (c *Config) normalize() {
|
||||
|
||||
// Proxy forwards requests to engine endpoints.
|
||||
type Proxy struct {
|
||||
cfg Config
|
||||
cfg Config
|
||||
// client is for streaming requests, where headers arrive before generation
|
||||
// and a short header timeout is a genuine liveness check.
|
||||
client *http.Client
|
||||
// buffered is for non-streaming requests, where the engine sends nothing
|
||||
// until the completion is done, so the header timeout is the generation
|
||||
// budget. Two transports rather than one, because the same number cannot
|
||||
// mean both things.
|
||||
buffered *http.Client
|
||||
}
|
||||
|
||||
// New builds a proxy.
|
||||
func New(cfg Config) *Proxy {
|
||||
cfg.normalize()
|
||||
transport := func(headerTimeout time.Duration) *http.Transport {
|
||||
return &http.Transport{
|
||||
DialContext: (&net.Dialer{Timeout: cfg.DialTimeout}).DialContext,
|
||||
ResponseHeaderTimeout: headerTimeout,
|
||||
MaxIdleConnsPerHost: cfg.MaxIdleConnsPerHost,
|
||||
MaxIdleConns: cfg.MaxIdleConnsPerHost * 8,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
// Engines rarely speak h2c, and HTTP/2 over cleartext would
|
||||
// need explicit setup anyway.
|
||||
ForceAttemptHTTP2: false,
|
||||
DisableCompression: true,
|
||||
}
|
||||
}
|
||||
return &Proxy{
|
||||
cfg: cfg,
|
||||
buffered: &http.Client{
|
||||
Transport: transport(cfg.GenerationTimeout),
|
||||
},
|
||||
client: &http.Client{
|
||||
// No client timeout: streaming responses run for minutes and the
|
||||
// per-phase watchdogs below bound them more precisely.
|
||||
Transport: &http.Transport{
|
||||
DialContext: (&net.Dialer{Timeout: cfg.DialTimeout}).DialContext,
|
||||
ResponseHeaderTimeout: cfg.ResponseHeaderTimeout,
|
||||
MaxIdleConnsPerHost: cfg.MaxIdleConnsPerHost,
|
||||
MaxIdleConns: cfg.MaxIdleConnsPerHost * 8,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
// Engines rarely speak h2c, and HTTP/2 over cleartext would
|
||||
// need explicit setup anyway.
|
||||
ForceAttemptHTTP2: false,
|
||||
DisableCompression: true,
|
||||
},
|
||||
Transport: transport(cfg.ResponseHeaderTimeout),
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -175,7 +204,14 @@ func (p *Proxy) Do(ctx context.Context, w http.ResponseWriter, r Request) (Outco
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.ContentLength = int64(len(r.Body))
|
||||
|
||||
resp, err := p.client.Do(req)
|
||||
// A non-streaming request waits for the whole completion before a single
|
||||
// header arrives, so it is bounded by the generation budget rather than the
|
||||
// liveness timeout.
|
||||
client := p.client
|
||||
if !r.Stream {
|
||||
client = p.buffered
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
// The client going away is not the endpoint's fault; never blame it.
|
||||
if ctx.Err() != nil && ctxCauseIsClient(ctx) {
|
||||
|
||||
@@ -300,3 +300,58 @@ func TestHopByHopHeadersNotForwarded(t *testing.T) {
|
||||
t.Error("Traceparent was not forwarded; tracing would break")
|
||||
}
|
||||
}
|
||||
|
||||
func TestANonStreamingRequestGetsTheGenerationBudget(t *testing.T) {
|
||||
// The engine sends no headers until a non-streaming completion is done, so
|
||||
// the header timeout is the generation budget in disguise. At 60s that read
|
||||
// as a liveness guard and behaved as a cap of roughly a hundred tokens on a
|
||||
// model generating 1.7 a second: a request for 120 tokens failed at exactly
|
||||
// one minute, and nothing said why.
|
||||
slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(200 * time.Millisecond) // "generating"
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"done"}}]}`))
|
||||
}))
|
||||
defer slow.Close()
|
||||
|
||||
p := New(Config{
|
||||
// Short enough that a shared setting would fail the request.
|
||||
ResponseHeaderTimeout: 20 * time.Millisecond,
|
||||
GenerationTimeout: 5 * time.Second,
|
||||
TotalTimeout: 10 * time.Second,
|
||||
})
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
out, err := p.Do(t.Context(), rec, Request{URL: slow.URL, Body: []byte(`{}`), Stream: false})
|
||||
if err != nil {
|
||||
t.Fatalf("a completion inside the generation budget failed: %v", err)
|
||||
}
|
||||
if out.Status != http.StatusOK {
|
||||
t.Errorf("status %d", out.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStreamingRequestKeepsTheShortLivenessTimeout(t *testing.T) {
|
||||
// The other half: for streaming, headers come before generation, so a slow
|
||||
// header really does mean a sick endpoint and must still be caught.
|
||||
dead := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(2 * time.Second)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer dead.Close()
|
||||
|
||||
p := New(Config{
|
||||
ResponseHeaderTimeout: 50 * time.Millisecond,
|
||||
GenerationTimeout: 10 * time.Second,
|
||||
TotalTimeout: 10 * time.Second,
|
||||
})
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
start := time.Now()
|
||||
if _, err := p.Do(t.Context(), rec, Request{URL: dead.URL, Body: []byte(`{}`), Stream: true}); err == nil {
|
||||
t.Fatal("a stalled streaming endpoint was not caught")
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed > time.Second {
|
||||
t.Errorf("took %s; the streaming liveness timeout is not being applied", elapsed)
|
||||
}
|
||||
}
|
||||
@@ -155,6 +155,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) {
|
||||
px: proxy.New(proxy.Config{
|
||||
DialTimeout: cfg.Router.Timeouts.Dial,
|
||||
ResponseHeaderTimeout: cfg.Router.Timeouts.ResponseHeader,
|
||||
GenerationTimeout: cfg.Router.Timeouts.Generation,
|
||||
FirstTokenTimeout: cfg.Router.Timeouts.FirstToken,
|
||||
IdleTimeout: cfg.Router.Timeouts.Idle,
|
||||
TotalTimeout: cfg.Router.Timeouts.Total,
|
||||
|
||||
Reference in new issue
Block a user