response_header sounds like a check that the endpoint is alive, and for a streaming request it is: headers arrive before generation starts. For a non-streaming request the engine sends nothing until the completion is finished, so the same setting is the total generation budget wearing a liveness timeout's name. At 60s that was invisible on GPUs and a hard cap of roughly a hundred tokens on qwen2.5-coder-14b at 1.7 tokens a second. A request for 120 tokens failed at exactly one minute, and neither the setting's name nor the error said why. Two transports now: the short header timeout for streaming, where it still catches a sick endpoint, and a generation budget for non-streaming, defaulting to ten minutes. One number could not mean both things. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
3081c6f69b
commit
db448a9e87
4 files changed
+117
-18
No files matched your search
@@ -300,3 +300,58 @@ func TestHopByHopHeadersNotForwarded(t *testing.T) {
|
||||
t.Error("Traceparent was not forwarded; tracing would break")
|
||||
}
|
||||
}
|
||||
|
||||
func TestANonStreamingRequestGetsTheGenerationBudget(t *testing.T) {
|
||||
// The engine sends no headers until a non-streaming completion is done, so
|
||||
// the header timeout is the generation budget in disguise. At 60s that read
|
||||
// as a liveness guard and behaved as a cap of roughly a hundred tokens on a
|
||||
// model generating 1.7 a second: a request for 120 tokens failed at exactly
|
||||
// one minute, and nothing said why.
|
||||
slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(200 * time.Millisecond) // "generating"
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"done"}}]}`))
|
||||
}))
|
||||
defer slow.Close()
|
||||
|
||||
p := New(Config{
|
||||
// Short enough that a shared setting would fail the request.
|
||||
ResponseHeaderTimeout: 20 * time.Millisecond,
|
||||
GenerationTimeout: 5 * time.Second,
|
||||
TotalTimeout: 10 * time.Second,
|
||||
})
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
out, err := p.Do(t.Context(), rec, Request{URL: slow.URL, Body: []byte(`{}`), Stream: false})
|
||||
if err != nil {
|
||||
t.Fatalf("a completion inside the generation budget failed: %v", err)
|
||||
}
|
||||
if out.Status != http.StatusOK {
|
||||
t.Errorf("status %d", out.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStreamingRequestKeepsTheShortLivenessTimeout(t *testing.T) {
|
||||
// The other half: for streaming, headers come before generation, so a slow
|
||||
// header really does mean a sick endpoint and must still be caught.
|
||||
dead := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(2 * time.Second)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer dead.Close()
|
||||
|
||||
p := New(Config{
|
||||
ResponseHeaderTimeout: 50 * time.Millisecond,
|
||||
GenerationTimeout: 10 * time.Second,
|
||||
TotalTimeout: 10 * time.Second,
|
||||
})
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
start := time.Now()
|
||||
if _, err := p.Do(t.Context(), rec, Request{URL: dead.URL, Body: []byte(`{}`), Stream: true}); err == nil {
|
||||
t.Fatal("a stalled streaming endpoint was not caught")
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed > time.Second {
|
||||
t.Errorf("took %s; the streaming liveness timeout is not being applied", elapsed)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user