Files
llmgw/internal/router/admin.go
T
ilkerandClaude Opus 5 a3f71aa5ee
CI / test (push) Successful in 1m1s
Phase 1: the router serves traffic
llmgw server is real now. It fronts a static set of engine replicas with an
OpenAI-compatible API, scoring endpoints through the pipeline added in the
previous commit.

- proxy: SSE relayed event by event with a flush after each, cancellation
  propagated upstream, and separate deadlines for dial, headers, first token,
  inter-token idle and total. The idle watchdog is what distinguishes a wedged
  engine from a slow one.
- router: dispatch with retry and model fallback. Retry only happens while
  nothing has reached the client; after that, splicing two responses together
  would be worse than the short stream the client already sees.
- config: YAML via koanf overlaid on defaults, with LLMGW_ env overrides.
  Validation rejects duplicate endpoint ids and aliases that shadow a model,
  because both misroute silently at request time rather than failing loudly.
- metrics: requests, duration, TTFT, tokens, plus per-scorer values and
  per-filter drops on a separate admin listener with health and pprof.

Phase 1 exit criterion, measured in test/e2e: 1000 concurrent streams across
3 sim-engines complete at ~6800 req/s; every client disconnect frees the KV
blocks on every engine; router overhead p99 is about 0.9 ms against the plan's
2 ms budget.

The handlers live in internal/router rather than an internal/router/openai
package as the plan's tree sketched: they need the dispatch loop, and splitting
them apart would have meant an interface with exactly one implementation. Plan
§4 updated to match.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 01:41:13 +02:00

107 lines
3.4 KiB
Go

package router
import (
"net/http"
"net/http/pprof"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
// AdminHandler returns the diagnostics surface: metrics, health and pprof.
//
// It is served on a separate listener so the client-facing port can be exposed
// without also exposing pprof (plan §9.1).
func (r *Router) AdminHandler() (http.Handler, error) {
reg := prometheus.NewRegistry()
reg.MustRegister(
collectors.NewGoCollector(),
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
)
for _, c := range r.m.collectors() {
if err := reg.Register(c); err != nil {
return nil, err
}
}
if err := reg.Register(newEndpointCollector(r)); err != nil {
return nil, err
}
mux := http.NewServeMux()
mux.Handle("GET /metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
})
mux.HandleFunc("GET /debug/pprof/", pprof.Index)
mux.HandleFunc("GET /debug/pprof/cmdline", pprof.Cmdline)
mux.HandleFunc("GET /debug/pprof/profile", pprof.Profile)
mux.HandleFunc("GET /debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("GET /debug/pprof/trace", pprof.Trace)
return mux, nil
}
// endpointCollector reports live endpoint state at scrape time rather than
// mirroring it into gauges on every request; the registry already holds it.
type endpointCollector struct {
r *Router
inflight *prometheus.Desc
waiting *prometheus.Desc
running *prometheus.Desc
kv *prometheus.Desc
stale *prometheus.Desc
drain *prometheus.Desc
breaker *prometheus.Desc
}
func newEndpointCollector(r *Router) *endpointCollector {
lbl := []string{"replica", "pool"}
return &endpointCollector{
r: r,
inflight: prometheus.NewDesc("llmgw_endpoint_inflight", "Requests the router has dispatched and not yet finished.", lbl, nil),
waiting: prometheus.NewDesc("llmgw_endpoint_waiting", "Engine queue depth, as last scraped.", lbl, nil),
running: prometheus.NewDesc("llmgw_endpoint_running", "Requests running on the engine, as last scraped.", lbl, nil),
kv: prometheus.NewDesc("llmgw_endpoint_kv_cache_usage", "Engine KV cache occupancy, 0..1.", lbl, nil),
drain: prometheus.NewDesc("llmgw_endpoint_drain_rate", "Observed completions per second.", lbl, nil),
stale: prometheus.NewDesc("llmgw_endpoint_metrics_stale", "1 when the endpoint's metrics are stale.", lbl, nil),
breaker: prometheus.NewDesc("llmgw_endpoint_breaker_open", "1 when the endpoint's circuit breaker is open.", lbl, nil),
}
}
func (c *endpointCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- c.inflight
ch <- c.waiting
ch <- c.running
ch <- c.kv
ch <- c.drain
ch <- c.stale
ch <- c.breaker
}
func (c *endpointCollector) Collect(ch chan<- prometheus.Metric) {
for _, s := range c.r.reg.Snapshots("") {
pool := s.Pool
if pool == "" {
pool = "none"
}
g := func(d *prometheus.Desc, v float64) {
ch <- prometheus.MustNewConstMetric(d, prometheus.GaugeValue, v, s.ID, pool)
}
g(c.inflight, float64(s.Inflight))
g(c.waiting, float64(s.Stats.Waiting))
g(c.running, float64(s.Stats.Running))
g(c.kv, s.Stats.KVCacheUsage)
g(c.drain, s.DrainRate)
g(c.stale, boolToFloat(s.Stale))
g(c.breaker, boolToFloat(s.Breaker.String() == "open"))
}
}
func boolToFloat(b bool) float64 {
if b {
return 1
}
return 0
}