llmgw server is real now. It fronts a static set of engine replicas with an OpenAI-compatible API, scoring endpoints through the pipeline added in the previous commit. - proxy: SSE relayed event by event with a flush after each, cancellation propagated upstream, and separate deadlines for dial, headers, first token, inter-token idle and total. The idle watchdog is what distinguishes a wedged engine from a slow one. - router: dispatch with retry and model fallback. Retry only happens while nothing has reached the client; after that, splicing two responses together would be worse than the short stream the client already sees. - config: YAML via koanf overlaid on defaults, with LLMGW_ env overrides. Validation rejects duplicate endpoint ids and aliases that shadow a model, because both misroute silently at request time rather than failing loudly. - metrics: requests, duration, TTFT, tokens, plus per-scorer values and per-filter drops on a separate admin listener with health and pprof. Phase 1 exit criterion, measured in test/e2e: 1000 concurrent streams across 3 sim-engines complete at ~6800 req/s; every client disconnect frees the KV blocks on every engine; router overhead p99 is about 0.9 ms against the plan's 2 ms budget. The handlers live in internal/router rather than an internal/router/openai package as the plan's tree sketched: they need the dispatch loop, and splitting them apart would have meant an interface with exactly one implementation. Plan §4 updated to match. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
45 lines
1.1 KiB
Go
45 lines
1.1 KiB
Go
package cli
|
|
|
|
import (
|
|
"errors"
|
|
|
|
"github.com/spf13/cobra"
|
|
)
|
|
|
|
// errNotImplemented marks commands belonging to phases not yet built.
|
|
var errNotImplemented = errors.New("not implemented yet (see llmgw-plan.md §11)")
|
|
|
|
func newAgentCmd() *cobra.Command {
|
|
var configPath string
|
|
cmd := &cobra.Command{
|
|
Use: "agent",
|
|
Short: "Node agent running on each GPU host (phase 2)",
|
|
RunE: func(_ *cobra.Command, _ []string) error {
|
|
return errNotImplemented
|
|
},
|
|
}
|
|
cmd.Flags().StringVar(&configPath, "config", "agent.yaml", "configuration file")
|
|
return cmd
|
|
}
|
|
|
|
func newCtlCmd() *cobra.Command {
|
|
cmd := &cobra.Command{
|
|
Use: "ctl",
|
|
Short: "Administrative CLI (phase 3)",
|
|
}
|
|
for _, sub := range []struct{ use, short string }{
|
|
{"nodes", "list nodes"},
|
|
{"models", "show the model catalog"},
|
|
{"replicas", "list replicas"},
|
|
{"explain-placement", "explain why a model could not be placed"},
|
|
{"bench", "measure profile capacity (§10)"},
|
|
} {
|
|
cmd.AddCommand(&cobra.Command{
|
|
Use: sub.use,
|
|
Short: sub.short,
|
|
RunE: func(_ *cobra.Command, _ []string) error { return errNotImplemented },
|
|
})
|
|
}
|
|
return cmd
|
|
}
|