Phase 1: the router serves traffic
CI / test (push) Successful in 1m1s

llmgw server is real now. It fronts a static set of engine replicas with an
OpenAI-compatible API, scoring endpoints through the pipeline added in the
previous commit.

- proxy: SSE relayed event by event with a flush after each, cancellation
  propagated upstream, and separate deadlines for dial, headers, first token,
  inter-token idle and total. The idle watchdog is what distinguishes a wedged
  engine from a slow one.
- router: dispatch with retry and model fallback. Retry only happens while
  nothing has reached the client; after that, splicing two responses together
  would be worse than the short stream the client already sees.
- config: YAML via koanf overlaid on defaults, with LLMGW_ env overrides.
  Validation rejects duplicate endpoint ids and aliases that shadow a model,
  because both misroute silently at request time rather than failing loudly.
- metrics: requests, duration, TTFT, tokens, plus per-scorer values and
  per-filter drops on a separate admin listener with health and pprof.

Phase 1 exit criterion, measured in test/e2e: 1000 concurrent streams across
3 sim-engines complete at ~6800 req/s; every client disconnect frees the KV
blocks on every engine; router overhead p99 is about 0.9 ms against the plan's
2 ms budget.

The handlers live in internal/router rather than an internal/router/openai
package as the plan's tree sketched: they need the dispatch loop, and splitting
them apart would have meant an interface with exactly one implementation. Plan
§4 updated to match.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
ilkerandClaude Opus 5 committed 2026-09-16 01:41:13 +02:00
1 parent f0c1ca87f1
commit a3f71aa5ee
20 files changed
+3028 -26

No files matched your search

+468
View File
@@ -0,0 +1,468 @@
// Package proxy streams a request to one upstream engine and relays the
// response to the client (plan §7.8).
//
// It handles exactly one attempt. Choosing the endpoint, retrying and tripping
// breakers belong to the caller, because those decisions need the scheduler and
// the endpoint registry. What the proxy owes the caller is an honest answer to
// "did anything reach the client yet?" — once the first byte is out, retrying
// elsewhere would corrupt the response.
package proxy
import (
"bufio"
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"strings"
"sync"
"sync/atomic"
"time"
)
// Config holds proxy settings.
type Config struct {
// DialTimeout bounds establishing the TCP connection.
DialTimeout time.Duration
// ResponseHeaderTimeout bounds the wait for upstream response headers.
ResponseHeaderTimeout time.Duration
// FirstTokenTimeout bounds the wait for the first streamed event.
FirstTokenTimeout time.Duration
// IdleTimeout bounds the gap between streamed events. A stalled engine
// looks identical to a slow one over a single connection, so without this
// a wedged request holds a client forever.
IdleTimeout time.Duration
// TotalTimeout bounds the whole request. Generous by default: a long
// generation is legitimately slow.
TotalTimeout time.Duration
// MaxIdleConnsPerHost keeps connections warm; engines are few and busy.
MaxIdleConnsPerHost int
// MaxResponseBytes bounds a buffered (non-streaming) response.
MaxResponseBytes int64
}
// DefaultConfig returns sensible defaults.
func DefaultConfig() Config {
return Config{
DialTimeout: 5 * time.Second,
ResponseHeaderTimeout: 60 * time.Second,
FirstTokenTimeout: 120 * time.Second,
IdleTimeout: 60 * time.Second,
TotalTimeout: 30 * time.Minute,
MaxIdleConnsPerHost: 64,
MaxResponseBytes: 64 << 20,
}
}
func (c *Config) normalize() {
d := DefaultConfig()
if c.DialTimeout <= 0 {
c.DialTimeout = d.DialTimeout
}
if c.ResponseHeaderTimeout <= 0 {
c.ResponseHeaderTimeout = d.ResponseHeaderTimeout
}
if c.FirstTokenTimeout <= 0 {
c.FirstTokenTimeout = d.FirstTokenTimeout
}
if c.IdleTimeout <= 0 {
c.IdleTimeout = d.IdleTimeout
}
if c.TotalTimeout <= 0 {
c.TotalTimeout = d.TotalTimeout
}
if c.MaxIdleConnsPerHost <= 0 {
c.MaxIdleConnsPerHost = d.MaxIdleConnsPerHost
}
if c.MaxResponseBytes <= 0 {
c.MaxResponseBytes = d.MaxResponseBytes
}
}
// Proxy forwards requests to engine endpoints.
type Proxy struct {
cfg Config
client *http.Client
}
// New builds a proxy.
func New(cfg Config) *Proxy {
cfg.normalize()
return &Proxy{
cfg: cfg,
client: &http.Client{
// No client timeout: streaming responses run for minutes and the
// per-phase watchdogs below bound them more precisely.
Transport: &http.Transport{
DialContext: (&net.Dialer{Timeout: cfg.DialTimeout}).DialContext,
ResponseHeaderTimeout: cfg.ResponseHeaderTimeout,
MaxIdleConnsPerHost: cfg.MaxIdleConnsPerHost,
MaxIdleConns: cfg.MaxIdleConnsPerHost * 8,
IdleConnTimeout: 90 * time.Second,
// Engines rarely speak h2c, and HTTP/2 over cleartext would
// need explicit setup anyway.
ForceAttemptHTTP2: false,
DisableCompression: true,
},
},
}
}
// Request describes one upstream attempt.
type Request struct {
// URL is the fully qualified upstream URL.
URL string
// Body is the request body to send.
Body []byte
// Header carries headers to forward (authorization is deliberately not).
Header http.Header
// Stream says whether an SSE response is expected.
Stream bool
// SuppressUsageChunk drops the final usage-only chunk from the client while
// still recording it. Set when the router asked for usage the client did
// not request.
SuppressUsageChunk bool
}
// Usage holds the token counts reported by the engine.
type Usage struct {
PromptTokens int
CompletionTokens int
TotalTokens int
CachedTokens int
}
// Outcome reports what happened during one attempt.
type Outcome struct {
// Wrote is true once any byte reached the client. Retrying is then unsafe.
Wrote bool
// Status is the upstream status code, 0 if the request never completed.
Status int
// Retryable means the attempt failed in a way another endpoint might survive.
Retryable bool
// TTFT is the time until the first streamed event, zero if none arrived.
TTFT time.Duration
// Usage is the engine-reported token usage, if it was present.
Usage *Usage
// Bytes is how many bytes were relayed to the client.
Bytes int64
}
var (
// ErrClientGone means the client disconnected; the upstream was cancelled.
ErrClientGone = errors.New("client disconnected")
// ErrUpstreamStalled means no event arrived within the idle or first-token timeout.
ErrUpstreamStalled = errors.New("upstream stalled")
)
// Do performs one attempt against one endpoint.
//
// On a retryable failure it writes nothing to w, leaving the caller free to try
// another endpoint. Once it has written anything, Outcome.Wrote is true and the
// caller must not retry.
func (p *Proxy) Do(ctx context.Context, w http.ResponseWriter, r Request) (Outcome, error) {
ctx, cancel := context.WithTimeout(ctx, p.cfg.TotalTimeout)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.URL, bytes.NewReader(r.Body))
if err != nil {
return Outcome{Retryable: false}, err
}
req.Header = cloneForUpstream(r.Header)
req.Header.Set("Content-Type", "application/json")
req.ContentLength = int64(len(r.Body))
resp, err := p.client.Do(req)
if err != nil {
// The client going away is not the endpoint's fault; never blame it.
if ctx.Err() != nil && ctxCauseIsClient(ctx) {
return Outcome{Retryable: false}, ErrClientGone
}
return Outcome{Retryable: true}, fmt.Errorf("upstream request failed: %w", err)
}
defer resp.Body.Close()
if retryableStatus(resp.StatusCode) {
// Drain a little so the connection can be reused, then let the caller
// pick somewhere else. Nothing has been written to the client.
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4<<10))
return Outcome{Status: resp.StatusCode, Retryable: true},
fmt.Errorf("upstream returned %s", resp.Status)
}
if r.Stream && isEventStream(resp.Header) {
return p.relayStream(ctx, cancel, w, resp, r)
}
return p.relayBuffered(w, resp)
}
// relayBuffered copies a non-streaming response through, capturing usage.
func (p *Proxy) relayBuffered(w http.ResponseWriter, resp *http.Response) (Outcome, error) {
body, err := io.ReadAll(io.LimitReader(resp.Body, p.cfg.MaxResponseBytes))
if err != nil {
return Outcome{Status: resp.StatusCode, Retryable: true}, err
}
copyHeaders(w.Header(), resp.Header)
w.WriteHeader(resp.StatusCode)
n, werr := w.Write(body)
out := Outcome{
Wrote: n > 0,
Status: resp.StatusCode,
Bytes: int64(n),
Usage: usageFromJSON(body),
}
if werr != nil {
return out, ErrClientGone
}
return out, nil
}
// relayStream relays an SSE response event by event, flushing as it goes.
//
// abort must cancel the context the upstream request was built with: the idle
// watchdog can only stop a silent engine by aborting that read.
func (p *Proxy) relayStream(ctx context.Context, abort context.CancelFunc, w http.ResponseWriter, resp *http.Response, r Request) (Outcome, error) {
flusher, ok := w.(http.Flusher)
if !ok {
return Outcome{Status: resp.StatusCode, Retryable: false},
errors.New("response writer does not support streaming")
}
copyHeaders(w.Header(), resp.Header)
w.WriteHeader(resp.StatusCode)
flusher.Flush()
out := Outcome{Wrote: true, Status: resp.StatusCode}
start := time.Now()
// One watchdog covers both phases: until the first event it enforces
// FirstTokenTimeout, afterwards IdleTimeout.
wd := newWatchdog(p.cfg.FirstTokenTimeout)
defer wd.run(ctx, abort)()
br := bufio.NewReaderSize(resp.Body, 16<<10)
var ev eventBuffer
for {
line, err := br.ReadString('\n')
if len(line) > 0 {
wd.beat()
ev.add(line)
if ev.complete() {
if out.TTFT == 0 {
out.TTFT = time.Since(start)
wd.setInterval(p.cfg.IdleTimeout)
}
if u := ev.usage(); u != nil {
out.Usage = u
if r.SuppressUsageChunk {
// The client never asked for usage; record it and drop
// the chunk rather than changing what they receive.
ev.reset()
continue
}
}
n, werr := io.WriteString(w, ev.string())
out.Bytes += int64(n)
ev.reset()
if werr != nil {
return out, ErrClientGone
}
flusher.Flush()
}
}
if err != nil {
if errors.Is(err, io.EOF) {
// Flush a trailing event that never got its blank line.
if s := ev.string(); s != "" {
n, _ := io.WriteString(w, s)
out.Bytes += int64(n)
flusher.Flush()
}
return out, nil
}
if wd.fired() {
return out, ErrUpstreamStalled
}
if ctx.Err() != nil {
return out, ErrClientGone
}
return out, err
}
}
}
// --- SSE event assembly ---
// eventBuffer accumulates the lines of one SSE event. Events are relayed whole
// rather than line by line so a usage-only chunk can be dropped atomically.
type eventBuffer struct {
lines []string
data strings.Builder
}
func (e *eventBuffer) add(line string) {
e.lines = append(e.lines, line)
if payload, ok := strings.CutPrefix(line, "data: "); ok {
e.data.WriteString(strings.TrimRight(payload, "\r\n"))
}
}
// complete reports whether the blank line terminating an event has arrived.
func (e *eventBuffer) complete() bool {
if len(e.lines) == 0 {
return false
}
last := e.lines[len(e.lines)-1]
return last == "\n" || last == "\r\n"
}
func (e *eventBuffer) string() string {
if len(e.lines) == 0 {
return ""
}
return strings.Join(e.lines, "")
}
func (e *eventBuffer) reset() {
e.lines = e.lines[:0]
e.data.Reset()
}
// usage returns the token usage carried by this event, if any.
func (e *eventBuffer) usage() *Usage {
s := e.data.String()
if s == "" || s == "[DONE]" || !strings.Contains(s, "\"usage\"") {
return nil
}
return usageFromJSON([]byte(s))
}
// --- watchdog ---
// watchdog aborts the upstream read once an interval passes with no beat, and
// records that it did so, so the resulting read error can be attributed to the
// engine stalling rather than to the client going away.
type watchdog struct {
interval atomic.Int64
last atomic.Int64
tripped atomic.Bool
}
func newWatchdog(interval time.Duration) *watchdog {
w := &watchdog{}
w.interval.Store(int64(interval))
w.last.Store(time.Now().UnixNano())
return w
}
func (w *watchdog) beat() { w.last.Store(time.Now().UnixNano()) }
func (w *watchdog) setInterval(d time.Duration) { w.interval.Store(int64(d)) }
func (w *watchdog) fired() bool { return w.tripped.Load() }
// run starts the watchdog and returns a stop function. abort is called if an
// interval elapses without a beat.
func (w *watchdog) run(ctx context.Context, abort context.CancelFunc) func() {
done := make(chan struct{})
go func() {
// Tick well below the shortest interval so the overshoot stays small.
// The interval shrinks once the first token arrives, so derive the tick
// from the smaller of the two rather than from whatever is set now.
tick := max(time.Duration(w.interval.Load())/4, 10*time.Millisecond)
t := time.NewTicker(tick)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-done:
return
case <-t.C:
interval := time.Duration(w.interval.Load())
if time.Since(time.Unix(0, w.last.Load())) > interval {
w.tripped.Store(true)
abort()
return
}
// Keep the tick proportional as the interval changes.
if want := max(interval/4, 10*time.Millisecond); want != tick {
tick = want
t.Reset(tick)
}
}
}
}()
var once sync.Once
return func() { once.Do(func() { close(done) }) }
}
// --- helpers ---
func retryableStatus(code int) bool {
switch code {
case http.StatusTooManyRequests,
http.StatusBadGateway,
http.StatusServiceUnavailable,
http.StatusGatewayTimeout:
return true
default:
return false
}
}
func isEventStream(h http.Header) bool {
return strings.HasPrefix(h.Get("Content-Type"), "text/event-stream")
}
// hopByHop headers must not be forwarded (RFC 9110 §7.6.1).
var hopByHop = map[string]bool{
"Connection": true,
"Proxy-Connection": true,
"Keep-Alive": true,
"Proxy-Authenticate": true,
"Proxy-Authorization": true,
"Te": true,
"Trailer": true,
"Transfer-Encoding": true,
"Upgrade": true,
}
func copyHeaders(dst, src http.Header) {
for k, vs := range src {
if hopByHop[http.CanonicalHeaderKey(k)] {
continue
}
for _, v := range vs {
dst.Add(k, v)
}
}
}
// cloneForUpstream forwards only what the engine needs. The client's
// Authorization header is deliberately dropped: it authenticates the caller to
// llmgw, not llmgw to the engine, and engines are reached over a trusted
// network (plan §14).
func cloneForUpstream(h http.Header) http.Header {
out := make(http.Header, 4)
for _, k := range []string{"Accept", "User-Agent", "Traceparent", "Tracestate"} {
if v := h.Get(k); v != "" {
out.Set(k, v)
}
}
return out
}
func ctxCauseIsClient(ctx context.Context) bool {
return errors.Is(context.Cause(ctx), context.Canceled)
}
+302
View File
@@ -0,0 +1,302 @@
package proxy
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
)
func testProxy(mut ...func(*Config)) *Proxy {
c := DefaultConfig()
c.FirstTokenTimeout = 2 * time.Second
c.IdleTimeout = 500 * time.Millisecond
c.TotalTimeout = 10 * time.Second
for _, f := range mut {
f(&c)
}
return New(c)
}
// sseServer serves a scripted SSE stream.
func sseServer(t *testing.T, chunks []string, gap time.Duration) (*httptest.Server, *atomic.Bool) {
t.Helper()
var cancelled atomic.Bool
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/event-stream")
w.WriteHeader(http.StatusOK)
f := w.(http.Flusher)
f.Flush()
for _, c := range chunks {
select {
case <-r.Context().Done():
cancelled.Store(true)
return
case <-time.After(gap):
}
if _, err := io.WriteString(w, c); err != nil {
cancelled.Store(true)
return
}
f.Flush()
}
}))
t.Cleanup(srv.Close)
return srv, &cancelled
}
func dataChunk(s string) string { return "data: " + s + "\n\n" }
func TestBufferedResponseIsRelayed(t *testing.T) {
up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
io.WriteString(w, `{"id":"x","usage":{"prompt_tokens":7,"completion_tokens":3}}`)
}))
defer up.Close()
rec := httptest.NewRecorder()
out, err := testProxy().Do(t.Context(), rec, Request{URL: up.URL, Body: []byte(`{}`)})
if err != nil {
t.Fatal(err)
}
if out.Status != 200 || !out.Wrote {
t.Errorf("outcome=%+v", out)
}
if out.Usage == nil || out.Usage.PromptTokens != 7 || out.Usage.TotalTokens != 10 {
t.Errorf("usage not captured: %+v", out.Usage)
}
if !strings.Contains(rec.Body.String(), `"id":"x"`) {
t.Errorf("body not relayed: %q", rec.Body.String())
}
}
func TestStreamIsRelayedEventByEvent(t *testing.T) {
up, _ := sseServer(t, []string{
dataChunk(`{"choices":[{"delta":{"content":"a"}}]}`),
dataChunk(`{"choices":[{"delta":{"content":"b"}}]}`),
"data: [DONE]\n\n",
}, time.Millisecond)
rec := httptest.NewRecorder()
out, err := testProxy().Do(t.Context(), rec, Request{URL: up.URL, Body: []byte(`{}`), Stream: true})
if err != nil {
t.Fatal(err)
}
body := rec.Body.String()
if n := strings.Count(body, "data: "); n != 3 {
t.Errorf("expected 3 data lines, got %d: %q", n, body)
}
if !strings.HasSuffix(body, "[DONE]\n\n") {
t.Errorf("stream did not end with [DONE]: %q", body)
}
if out.TTFT == 0 {
t.Error("TTFT was not recorded")
}
}
func TestUsageCapturedFromStream(t *testing.T) {
up, _ := sseServer(t, []string{
dataChunk(`{"choices":[{"delta":{"content":"a"}}]}`),
dataChunk(`{"choices":[],"usage":{"prompt_tokens":11,"completion_tokens":5,"total_tokens":16}}`),
"data: [DONE]\n\n",
}, time.Millisecond)
rec := httptest.NewRecorder()
out, err := testProxy().Do(t.Context(), rec, Request{URL: up.URL, Body: []byte(`{}`), Stream: true})
if err != nil {
t.Fatal(err)
}
if out.Usage == nil || out.Usage.TotalTokens != 16 {
t.Fatalf("usage not captured from stream: %+v", out.Usage)
}
if !strings.Contains(rec.Body.String(), `"usage"`) {
t.Error("usage chunk should reach a client that asked for it")
}
}
func TestUsageChunkSuppressed(t *testing.T) {
// The router asked the engine for usage the client never requested: record
// it, but do not change what the client receives.
up, _ := sseServer(t, []string{
dataChunk(`{"choices":[{"delta":{"content":"a"}}]}`),
dataChunk(`{"choices":[],"usage":{"prompt_tokens":11,"completion_tokens":5,"total_tokens":16}}`),
"data: [DONE]\n\n",
}, time.Millisecond)
rec := httptest.NewRecorder()
out, err := testProxy().Do(t.Context(), rec, Request{
URL: up.URL, Body: []byte(`{}`), Stream: true, SuppressUsageChunk: true,
})
if err != nil {
t.Fatal(err)
}
if out.Usage == nil || out.Usage.TotalTokens != 16 {
t.Fatalf("usage not recorded: %+v", out.Usage)
}
if strings.Contains(rec.Body.String(), `"usage"`) {
t.Errorf("usage chunk leaked to the client: %q", rec.Body.String())
}
if !strings.Contains(rec.Body.String(), "[DONE]") {
t.Error("[DONE] was dropped along with the usage chunk")
}
}
func TestRetryableStatusWritesNothing(t *testing.T) {
// The caller must be free to try another endpoint, which is only safe if
// the client has seen nothing.
for _, code := range []int{429, 502, 503, 504} {
up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
http.Error(w, "busy", code)
}))
rec := httptest.NewRecorder()
out, err := testProxy().Do(t.Context(), rec, Request{URL: up.URL, Body: []byte(`{}`)})
up.Close()
if err == nil {
t.Errorf("%d: expected an error", code)
}
if !out.Retryable {
t.Errorf("%d: not marked retryable", code)
}
if out.Wrote || rec.Body.Len() > 0 {
t.Errorf("%d: wrote %q to the client", code, rec.Body.String())
}
}
}
func TestNonRetryableStatusIsRelayed(t *testing.T) {
// A 400 is the client's fault; retrying elsewhere would produce the same
// answer, so relay it.
up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusBadRequest)
io.WriteString(w, `{"error":{"message":"bad"}}`)
}))
defer up.Close()
rec := httptest.NewRecorder()
out, err := testProxy().Do(t.Context(), rec, Request{URL: up.URL, Body: []byte(`{}`)})
if err != nil {
t.Fatal(err)
}
if out.Retryable {
t.Error("400 must not be retryable")
}
if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "bad") {
t.Errorf("error response not relayed: %d %q", rec.Code, rec.Body.String())
}
}
func TestConnectionFailureIsRetryable(t *testing.T) {
rec := httptest.NewRecorder()
out, err := testProxy(func(c *Config) { c.DialTimeout = 200 * time.Millisecond }).
Do(t.Context(), rec, Request{URL: "http://127.0.0.1:1/x", Body: []byte(`{}`)})
if err == nil {
t.Fatal("expected an error")
}
if !out.Retryable || out.Wrote {
t.Errorf("outcome=%+v", out)
}
}
func TestClientDisconnectCancelsUpstream(t *testing.T) {
// Phase 1's exit criterion at the proxy level: the GPU must stop working
// when the client goes away.
chunks := make([]string, 200)
for i := range chunks {
chunks[i] = dataChunk(fmt.Sprintf(`{"choices":[{"delta":{"content":"%d"}}]}`, i))
}
up, cancelled := sseServer(t, chunks, 10*time.Millisecond)
ctx, cancel := context.WithCancel(context.Background())
rec := httptest.NewRecorder()
done := make(chan Outcome, 1)
go func() {
out, _ := testProxy().Do(ctx, rec, Request{URL: up.URL, Body: []byte(`{}`), Stream: true})
done <- out
}()
time.Sleep(80 * time.Millisecond) // let a few events through
cancel()
select {
case <-done:
case <-time.After(3 * time.Second):
t.Fatal("proxy did not return after client cancellation")
}
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
if cancelled.Load() {
return
}
time.Sleep(10 * time.Millisecond)
}
t.Fatal("upstream kept generating after the client disconnected")
}
func TestStalledUpstreamHitsIdleTimeout(t *testing.T) {
up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/event-stream")
w.WriteHeader(http.StatusOK)
f := w.(http.Flusher)
io.WriteString(w, dataChunk(`{"choices":[{"delta":{"content":"a"}}]}`))
f.Flush()
<-r.Context().Done() // then go silent forever
}))
defer up.Close()
rec := httptest.NewRecorder()
p := testProxy(func(c *Config) { c.IdleTimeout = 200 * time.Millisecond })
start := time.Now()
out, err := p.Do(t.Context(), rec, Request{URL: up.URL, Body: []byte(`{}`), Stream: true})
elapsed := time.Since(start)
if !errors.Is(err, ErrUpstreamStalled) {
t.Fatalf("expected ErrUpstreamStalled, got %v", err)
}
if !out.Wrote {
t.Error("the events delivered before the stall should be reported as written")
}
if elapsed > 3*time.Second {
t.Errorf("idle timeout took too long: %v", elapsed)
}
}
func TestHopByHopHeadersNotForwarded(t *testing.T) {
var got http.Header
up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
got = r.Header.Clone()
io.WriteString(w, `{}`)
}))
defer up.Close()
h := http.Header{}
h.Set("Authorization", "Bearer sk-llmgw-secret")
h.Set("Accept", "text/event-stream")
h.Set("Traceparent", "00-trace-span-01")
rec := httptest.NewRecorder()
if _, err := testProxy().Do(t.Context(), rec, Request{URL: up.URL, Body: []byte(`{}`), Header: h}); err != nil {
t.Fatal(err)
}
if got.Get("Authorization") != "" {
t.Error("the client's Authorization header was forwarded upstream")
}
if got.Get("Accept") != "text/event-stream" {
t.Error("Accept was not forwarded")
}
if got.Get("Traceparent") != "00-trace-span-01" {
t.Error("Traceparent was not forwarded; tracing would break")
}
}
+39
View File
@@ -0,0 +1,39 @@
package proxy
import "encoding/json"
// usageEnvelope mirrors just the usage block of an OpenAI response.
// Everything else in the payload is passed through untouched, so only these
// fields are ever decoded.
type usageEnvelope struct {
Usage *struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
PromptTokensDetails *struct {
CachedTokens int `json:"cached_tokens"`
} `json:"prompt_tokens_details"`
} `json:"usage"`
}
// usageFromJSON extracts token usage from a response or an SSE chunk.
// It returns nil when the payload carries no usage block, which is the normal
// case for every chunk but the last.
func usageFromJSON(b []byte) *Usage {
var env usageEnvelope
if err := json.Unmarshal(b, &env); err != nil || env.Usage == nil {
return nil
}
u := &Usage{
PromptTokens: env.Usage.PromptTokens,
CompletionTokens: env.Usage.CompletionTokens,
TotalTokens: env.Usage.TotalTokens,
}
if d := env.Usage.PromptTokensDetails; d != nil {
u.CachedTokens = d.CachedTokens
}
if u.TotalTokens == 0 {
u.TotalTokens = u.PromptTokens + u.CompletionTokens
}
return u
}