Phase 1: the router serves traffic
llmgw server is real now. It fronts a static set of engine replicas with an OpenAI-compatible API, scoring endpoints through the pipeline added in the previous commit. - proxy: SSE relayed event by event with a flush after each, cancellation propagated upstream, and separate deadlines for dial, headers, first token, inter-token idle and total. The idle watchdog is what distinguishes a wedged engine from a slow one. - router: dispatch with retry and model fallback. Retry only happens while nothing has reached the client; after that, splicing two responses together would be worse than the short stream the client already sees. - config: YAML via koanf overlaid on defaults, with LLMGW_ env overrides. Validation rejects duplicate endpoint ids and aliases that shadow a model, because both misroute silently at request time rather than failing loudly. - metrics: requests, duration, TTFT, tokens, plus per-scorer values and per-filter drops on a separate admin listener with health and pprof. Phase 1 exit criterion, measured in test/e2e: 1000 concurrent streams across 3 sim-engines complete at ~6800 req/s; every client disconnect frees the KV blocks on every engine; router overhead p99 is about 0.9 ms against the plan's 2 ms budget. The handlers live in internal/router rather than an internal/router/openai package as the plan's tree sketched: they need the dispatch loop, and splitting them apart would have meant an interface with exactly one implementation. Plan §4 updated to match. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
f0c1ca87f1
commit
a3f71aa5ee
20 files changed
+3028
-26
No files matched your search
@@ -0,0 +1,157 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
|
||||
)
|
||||
|
||||
func newServerCmd() *cobra.Command {
|
||||
var (
|
||||
configPath string
|
||||
listen string
|
||||
adminListen string
|
||||
endpoints []string
|
||||
)
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "server",
|
||||
Short: "Run the router (phase 1; the control plane arrives in phase 3)",
|
||||
Long: "Serves the OpenAI API in front of a set of engine replicas.\n\n" +
|
||||
"In this phase the replica list is static: give it in the config file, or\n" +
|
||||
"pass --endpoint id=URL=model[,model] once per replica for a quick test.",
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
log := logger()
|
||||
|
||||
cfg, err := config.Load(configPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if listen != "" {
|
||||
cfg.Listen = listen
|
||||
}
|
||||
if adminListen != "" {
|
||||
cfg.AdminListen = adminListen
|
||||
}
|
||||
for _, spec := range endpoints {
|
||||
ep, err := parseEndpointFlag(spec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cfg.Endpoints = append(cfg.Endpoints, ep)
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(cfg.Endpoints) == 0 {
|
||||
return errors.New("no endpoints configured; use --endpoint or a config file")
|
||||
}
|
||||
|
||||
rt, err := router.New(cfg, log)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
rt.Start(ctx)
|
||||
defer rt.Stop()
|
||||
|
||||
adminHandler, err := rt.AdminHandler()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
apiSrv := &http.Server{
|
||||
Handler: rt.Handler(),
|
||||
ReadHeaderTimeout: 10 * time.Second,
|
||||
// No write timeout: responses stream for minutes and the proxy
|
||||
// enforces its own per-phase deadlines.
|
||||
}
|
||||
adminSrv := &http.Server{
|
||||
Handler: adminHandler,
|
||||
ReadHeaderTimeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
apiLn, err := net.Listen("tcp", cfg.Listen)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listening on %s: %w", cfg.Listen, err)
|
||||
}
|
||||
adminLn, err := net.Listen("tcp", cfg.AdminListen)
|
||||
if err != nil {
|
||||
_ = apiLn.Close()
|
||||
return fmt.Errorf("listening on %s: %w", cfg.AdminListen, err)
|
||||
}
|
||||
|
||||
log.Info("router listening",
|
||||
"api", apiLn.Addr().String(),
|
||||
"admin", adminLn.Addr().String(),
|
||||
"endpoints", len(cfg.Endpoints),
|
||||
"default_profile", cfg.Router.DefaultProfile,
|
||||
)
|
||||
|
||||
g, gctx := errgroup.WithContext(ctx)
|
||||
g.Go(func() error { return serve(apiSrv, apiLn) })
|
||||
g.Go(func() error { return serve(adminSrv, adminLn) })
|
||||
g.Go(func() error {
|
||||
<-gctx.Done()
|
||||
log.Info("shutting down")
|
||||
sctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
_ = adminSrv.Shutdown(sctx)
|
||||
return apiSrv.Shutdown(sctx)
|
||||
})
|
||||
return g.Wait()
|
||||
},
|
||||
}
|
||||
|
||||
f := cmd.Flags()
|
||||
f.StringVar(&configPath, "config", "", "configuration file (optional)")
|
||||
f.StringVar(&listen, "listen", "", "override the client-facing address")
|
||||
f.StringVar(&adminListen, "admin-listen", "", "override the metrics/pprof address")
|
||||
f.StringArrayVar(&endpoints, "endpoint", nil, "static endpoint: id=URL=model[,model] (repeatable)")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func serve(s *http.Server, ln net.Listener) error {
|
||||
if err := s.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseEndpointFlag decodes id=URL=model[,model].
|
||||
//
|
||||
// The URL contains '=' only in pathological cases, so splitting into exactly
|
||||
// three parts keeps the flag readable while staying unambiguous.
|
||||
func parseEndpointFlag(spec string) (config.Endpoint, error) {
|
||||
parts := strings.SplitN(spec, "=", 3)
|
||||
if len(parts) != 3 {
|
||||
return config.Endpoint{}, fmt.Errorf("--endpoint %q: expected id=URL=model[,model]", spec)
|
||||
}
|
||||
models := strings.Split(parts[2], ",")
|
||||
for i := range models {
|
||||
models[i] = strings.TrimSpace(models[i])
|
||||
}
|
||||
return config.Endpoint{
|
||||
ID: strings.TrimSpace(parts[0]),
|
||||
URL: strings.TrimRight(strings.TrimSpace(parts[1]), "/"),
|
||||
Models: models,
|
||||
Pool: "static",
|
||||
Weight: 1,
|
||||
}, nil
|
||||
}
|
||||
Reference in new issue
Block a user