diff --git a/deploy/examples/server.yaml b/deploy/examples/server.yaml index ce87346..052b291 100644 --- a/deploy/examples/server.yaml +++ b/deploy/examples/server.yaml @@ -7,7 +7,10 @@ # __ for nesting: LLMGW_ROUTER__RETRIES=3 listen: ":8080" # client-facing OpenAI API -admin_listen: ":9090" # metrics, health, pprof — do not expose this publicly +admin_listen: "127.0.0.1:9090" # dashboard, management API, pprof — keep it local +# Metrics only, on their own port, so Prometheus can reach them without pprof +# being reachable too. Empty means off; metrics stay on admin_listen either way. +# metrics_listen: "0.0.0.0:9091" # Where node agents connect. Separate from the client port so the fleet-facing # surface can be firewalled independently. agent_listen: ":7443" diff --git a/deploy/monitoring/prometheus.yml b/deploy/monitoring/prometheus.yml new file mode 100644 index 0000000..0af2a81 --- /dev/null +++ b/deploy/monitoring/prometheus.yml @@ -0,0 +1,45 @@ +# Scraping llmgw. +# +# Point this at metrics_listen, not admin_listen. They are different ports for +# a reason: the admin listener also carries pprof, which can dump the process +# heap, and API keys are in that heap in plaintext. +# +# In the control plane's configuration: +# +# admin_listen: "127.0.0.1:9090" # dashboard, management API, pprof +# metrics_listen: "0.0.0.0:9091" # metrics only, safe to reach +# +# One target for the whole fleet. Node and GPU figures are exported by the +# control plane, which already holds them — agents report hardware at +# registration and state on every heartbeat — so there is nothing to scrape on +# the nodes and no Prometheus configuration per machine. +global: + scrape_interval: 15s + +scrape_configs: + - job_name: llmgw + static_configs: + - targets: ["llmgw-ctl.internal:9091"] + labels: + cluster: primary + +# Worth alerting on, in rough order of how often it matters. +# +# A node that has stopped reporting: +# llmgw_node_last_seen_seconds > 60 +# +# A model that should be serving and is not. Covers a replica that will never +# start as well as one that has died, which look the same from a distance: +# llmgw_model_replicas_ready == 0 +# +# A card kept out of placement by a fault, which is otherwise invisible until +# a model will not fit: +# llmgw_gpu_healthy == 0 +# +# A fleet holding models nobody is asking for. Memory used says a model is +# loaded; utilisation says whether it is doing anything: +# avg_over_time(llmgw_gpu_utilization_percent[1h]) < 5 +# +# Requests queueing because the cluster is full, which is the pair of figures +# that needed two systems to correlate before these were on one endpoint: +# llmgw_admission_queue_depth > 0 diff --git a/docs/install.md b/docs/install.md index 437bba1..54ebf91 100644 --- a/docs/install.md +++ b/docs/install.md @@ -557,6 +557,34 @@ Or open `/chat` in a browser and use it without a key at all — the page carrie your session, and the request still goes through admission, rate limits, budget and accounting exactly as an API call does. +## Watching it + +The control plane exports both halves on one endpoint: what the cluster is +serving — requests, latency, time to first token, tokens, admission, per +endpoint — and what it is made of, per node and per GPU, including memory, +temperature, ECC faults and utilisation. + +Metrics get their own listener, because the admin one also carries pprof and +pprof can dump the process heap, where API keys sit in plaintext: + +```yaml +admin_listen: "127.0.0.1:9090" # dashboard, management API, pprof — keep local +metrics_listen: "0.0.0.0:9091" # metrics only, safe to reach +``` + +One scrape target for the whole fleet. The node and GPU figures come from the +control plane, which already holds them, so there is nothing to install on the +nodes and no Prometheus configuration per machine. + +A starting configuration, with the alerts worth having, is in +[deploy/monitoring/prometheus.yml](../deploy/monitoring/prometheus.yml). + +Two figures repay knowing about. `llmgw_gpu_utilization_percent` is absent +rather than zero when the driver does not report it — a flat zero across a +fleet means the measurement is missing, not that the fleet is idle. And +`llmgw_gpu_nvlink_peers` at zero everywhere means the topology is unknown, +which is what a binary built without the nvml tag reports. + ## When something does not work **The node is `ready` but every replica fails.** Almost always a missing engine diff --git a/internal/cli/server.go b/internal/cli/server.go index 7b82c42..5aeebc1 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -34,10 +34,11 @@ import ( func newServerCmd() *cobra.Command { var ( - configPath string - listen string - adminListen string - endpoints []string + configPath string + listen string + adminListen string + metricsListen string + endpoints []string ) cmd := &cobra.Command{ @@ -59,6 +60,9 @@ func newServerCmd() *cobra.Command { if adminListen != "" { cfg.AdminListen = adminListen } + if metricsListen != "" { + cfg.MetricsListen = metricsListen + } for _, spec := range endpoints { ep, err := parseEndpointFlag(spec) if err != nil { @@ -195,9 +199,35 @@ func newServerCmd() *cobra.Command { return fmt.Errorf("listening on %s: %w", cfg.AdminListen, err) } + // A metrics-only listener, when one is configured. Started after + // AdminHandler, which is where the registry it serves is built. + var metricsSrv *http.Server + var metricsLn net.Listener + if cfg.MetricsListen != "" { + h := rt.MetricsHandler() + if h == nil { + _ = apiLn.Close() + _ = adminLn.Close() + return fmt.Errorf("metrics handler is not ready; this is a wiring fault, not a configuration one") + } + metricsLn, err = net.Listen("tcp", cfg.MetricsListen) + if err != nil { + _ = apiLn.Close() + _ = adminLn.Close() + return fmt.Errorf("listening on %s: %w", cfg.MetricsListen, err) + } + metricsSrv = &http.Server{Handler: h, ReadHeaderTimeout: 10 * time.Second} + go func() { + if err := metricsSrv.Serve(metricsLn); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Error("metrics listener stopped", "err", err) + } + }() + } + log.Info("router listening", "api", apiLn.Addr().String(), "admin", adminLn.Addr().String(), + "metrics", metricsAddr(metricsLn), "endpoints", len(cfg.Endpoints), "default_profile", cfg.Router.DefaultProfile, ) @@ -252,6 +282,9 @@ func newServerCmd() *cobra.Command { if agentSrv != nil { _ = agentSrv.Shutdown(sctx) } + if metricsSrv != nil { + _ = metricsSrv.Shutdown(sctx) + } _ = adminSrv.Shutdown(sctx) return apiSrv.Shutdown(sctx) }) @@ -262,7 +295,8 @@ func newServerCmd() *cobra.Command { f := cmd.Flags() f.StringVar(&configPath, "config", "", "configuration file (optional)") f.StringVar(&listen, "listen", "", "override the client-facing address") - f.StringVar(&adminListen, "admin-listen", "", "override the metrics/pprof address") + f.StringVar(&adminListen, "admin-listen", "", "override the dashboard/pprof address") + f.StringVar(&metricsListen, "metrics-listen", "", "serve metrics only on this address, safe to expose") f.StringArrayVar(&endpoints, "endpoint", nil, "static endpoint: id=URL=model[,model] (repeatable)") return cmd @@ -308,6 +342,15 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live return ctl, svc, nil } +// metricsAddr renders the metrics listener for the startup line, saying so +// plainly when there is none rather than printing an empty string. +func metricsAddr(ln net.Listener) string { + if ln == nil { + return "off (admin listener only)" + } + return ln.Addr().String() +} + // dispatcherRef lets the controller be built before the service exists. type dispatcherRef struct { mu sync.RWMutex diff --git a/internal/config/config.go b/internal/config/config.go index 9267102..ab9a83f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -24,10 +24,27 @@ import ( type Server struct { // Listen is the client-facing OpenAI API address. Listen string `koanf:"listen"` - // AdminListen serves metrics, health and pprof. Kept separate so the - // client-facing port can be exposed without exposing diagnostics. + // AdminListen serves the dashboard, the management API, health and pprof. + // + // Loopback by default, and it should stay there: pprof can dump the process + // heap, and API keys are in that heap in plaintext. Every comment beside + // this setting has always said not to expose it, while the default bound + // every interface — a default that contradicts its own documentation is a + // footgun, so the default now matches the advice. AdminListen string `koanf:"admin_listen"` + // MetricsListen serves Prometheus metrics and nothing else. + // + // Separate from AdminListen because the reason that one is loopback is + // pprof, not metrics — and metrics are the one thing on it that something + // else genuinely needs to reach. Sharing a listener meant a fleet could + // either be monitored or keep its heap private, which is not a choice + // anybody should have to make. + // + // Empty means off, and metrics remain available on AdminListen either way, + // so nothing that scrapes it today has to change. + MetricsListen string `koanf:"metrics_listen"` + // AgentListen is where node agents connect. Separate from the client port // so the fleet-facing surface can be firewalled independently. AgentListen string `koanf:"agent_listen"` @@ -273,7 +290,7 @@ type Endpoint struct { func Default() Server { return Server{ Listen: ":8080", - AdminListen: ":9090", + AdminListen: "127.0.0.1:9090", AgentListen: ":7443", Control: Control{ Interval: 5 * time.Second, diff --git a/internal/router/admin.go b/internal/router/admin.go index bebf9e2..4e74cd2 100644 --- a/internal/router/admin.go +++ b/internal/router/admin.go @@ -47,8 +47,11 @@ func (r *Router) AdminHandler() (*http.ServeMux, error) { } } + // Kept, so anything already scraping the admin listener keeps working. + r.metricsHandler = promhttp.HandlerFor(reg, promhttp.HandlerOpts{}) + mux := http.NewServeMux() - mux.Handle("GET /metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) + mux.Handle("GET /metrics", r.metricsHandler) mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) @@ -144,3 +147,29 @@ func boolToFloat(b bool) float64 { } return 0 } + +// MetricsHandler serves metrics and nothing else, for a listener that can be +// reached from outside the machine. +// +// The whole reason the admin listener is loopback is pprof: it can dump the +// process heap, and API keys are in that heap. Metrics are the one thing on +// that listener something else genuinely needs, so they get a door of their +// own rather than forcing a choice between being monitored and keeping the +// heap private. +// +// Returns nil before AdminHandler has been called, because that is where the +// registry is assembled — the same ordering that silently published nothing +// once already, which is why this says so rather than leaving a caller to find +// out from an empty page. +func (r *Router) MetricsHandler() http.Handler { + if r.metricsHandler == nil { + return nil + } + mux := http.NewServeMux() + mux.Handle("GET /metrics", r.metricsHandler) + // Health too, so a scrape target can be probed without a second port. + mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + return mux +} diff --git a/internal/router/admin_test.go b/internal/router/admin_test.go index 99cc6e1..771dedd 100644 --- a/internal/router/admin_test.go +++ b/internal/router/admin_test.go @@ -44,3 +44,68 @@ func TestARegisteredCollectorReachesTheMetricsEndpoint(t *testing.T) { t.Error("a registered collector did not reach /metrics") } } + +func TestTheMetricsListenerServesMetricsAndNotPprof(t *testing.T) { + // The whole reason it exists. pprof can dump the process heap, and API keys + // are in that heap in plaintext — so the listener that can be reached from + // another machine must not carry it. Sharing one meant a fleet could be + // monitored or keep its heap private, and not both. + e := newFakeEngine(t, "m") + rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}})) + if _, err := rt.AdminHandler(); err != nil { + t.Fatal(err) + } + + h := rt.MetricsHandler() + if h == nil { + t.Fatal("no metrics handler after AdminHandler") + } + + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + if rec.Code != http.StatusOK { + t.Errorf("/metrics returned %d", rec.Code) + } + if !strings.Contains(rec.Body.String(), "llmgw_") { + t.Error("/metrics served nothing recognisable") + } + + for _, path := range []string{ + "/debug/pprof/", "/debug/pprof/heap", "/debug/pprof/cmdline", + "/debug/pprof/profile", "/debug/pprof/trace", "/api/v1/nodes", "/admin", + } { + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) + if rec.Code != http.StatusNotFound { + t.Errorf("%s returned %d on the metrics listener; it must not be reachable there", + path, rec.Code) + } + } +} + +func TestMetricsStayOnTheAdminListenerToo(t *testing.T) { + // Anything already scraping the admin listener keeps working; adding a door + // is not the same as moving one. + e := newFakeEngine(t, "m") + rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}})) + mux, err := rt.AdminHandler() + if err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + if rec.Code != http.StatusOK { + t.Errorf("/metrics on the admin listener returned %d", rec.Code) + } +} + +func TestTheMetricsHandlerIsNilBeforeTheRegistryExists(t *testing.T) { + // Ordering has bitten this exact area once already: a collector registered + // after AdminHandler published nothing at all, silently. Saying nil beats + // serving an empty page that looks like a working scrape target. + e := newFakeEngine(t, "m") + rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}})) + if rt.MetricsHandler() != nil { + t.Error("a metrics handler existed before the registry was built") + } +} diff --git a/internal/router/router.go b/internal/router/router.go index a3c8cdf..3b29da0 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -62,6 +62,10 @@ type Router struct { // activator loads a model that is configured but not currently running. activator Activator + // metricsHandler is built with the admin registry and reused by any + // listener that should serve metrics without serving pprof. + metricsHandler http.Handler + // landing configures the public page, which the router serves but does not // otherwise care about. landing config.Landing