The metrics existed and nothing could record them. They shared a listener with pprof, the management API and the dashboard, and pprof can dump the process heap where API keys sit in plaintext — so a fleet could be monitored or keep its heap private, and not both. That is not a choice anyone should have to make, and it is why every previous answer to "how do I scrape this" was a tunnel. metrics_listen serves metrics and health and nothing else, so it can be reached from wherever Prometheus lives. Metrics stay on the admin listener too: adding a door is not the same as moving one, and anything already scraping keeps working. While there: admin_listen defaulted to :9090, every interface, with a comment beside it saying not to expose it. A default that contradicts its own documentation is a footgun, so it is loopback now and someone who wants otherwise says so. The handler is nil until the registry is built, and says so rather than serving an empty page. Ordering has already caught this exact area once — a collector registered after AdminHandler published nothing at all, silently — and an empty scrape target looks like a working one. deploy/monitoring/prometheus.yml has a starting configuration and the alerts worth having: a node that has stopped reporting, a model that should be serving and is not, a card kept out of placement by a fault, and a fleet holding models nobody is asking for. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
ad37bee081
commit
1ea067d08e
8 files changed
+244
-10
No files matched your search
+48
-5
@@ -34,10 +34,11 @@ import (
|
||||
|
||||
func newServerCmd() *cobra.Command {
|
||||
var (
|
||||
configPath string
|
||||
listen string
|
||||
adminListen string
|
||||
endpoints []string
|
||||
configPath string
|
||||
listen string
|
||||
adminListen string
|
||||
metricsListen string
|
||||
endpoints []string
|
||||
)
|
||||
|
||||
cmd := &cobra.Command{
|
||||
@@ -59,6 +60,9 @@ func newServerCmd() *cobra.Command {
|
||||
if adminListen != "" {
|
||||
cfg.AdminListen = adminListen
|
||||
}
|
||||
if metricsListen != "" {
|
||||
cfg.MetricsListen = metricsListen
|
||||
}
|
||||
for _, spec := range endpoints {
|
||||
ep, err := parseEndpointFlag(spec)
|
||||
if err != nil {
|
||||
@@ -195,9 +199,35 @@ func newServerCmd() *cobra.Command {
|
||||
return fmt.Errorf("listening on %s: %w", cfg.AdminListen, err)
|
||||
}
|
||||
|
||||
// A metrics-only listener, when one is configured. Started after
|
||||
// AdminHandler, which is where the registry it serves is built.
|
||||
var metricsSrv *http.Server
|
||||
var metricsLn net.Listener
|
||||
if cfg.MetricsListen != "" {
|
||||
h := rt.MetricsHandler()
|
||||
if h == nil {
|
||||
_ = apiLn.Close()
|
||||
_ = adminLn.Close()
|
||||
return fmt.Errorf("metrics handler is not ready; this is a wiring fault, not a configuration one")
|
||||
}
|
||||
metricsLn, err = net.Listen("tcp", cfg.MetricsListen)
|
||||
if err != nil {
|
||||
_ = apiLn.Close()
|
||||
_ = adminLn.Close()
|
||||
return fmt.Errorf("listening on %s: %w", cfg.MetricsListen, err)
|
||||
}
|
||||
metricsSrv = &http.Server{Handler: h, ReadHeaderTimeout: 10 * time.Second}
|
||||
go func() {
|
||||
if err := metricsSrv.Serve(metricsLn); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
log.Error("metrics listener stopped", "err", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
log.Info("router listening",
|
||||
"api", apiLn.Addr().String(),
|
||||
"admin", adminLn.Addr().String(),
|
||||
"metrics", metricsAddr(metricsLn),
|
||||
"endpoints", len(cfg.Endpoints),
|
||||
"default_profile", cfg.Router.DefaultProfile,
|
||||
)
|
||||
@@ -252,6 +282,9 @@ func newServerCmd() *cobra.Command {
|
||||
if agentSrv != nil {
|
||||
_ = agentSrv.Shutdown(sctx)
|
||||
}
|
||||
if metricsSrv != nil {
|
||||
_ = metricsSrv.Shutdown(sctx)
|
||||
}
|
||||
_ = adminSrv.Shutdown(sctx)
|
||||
return apiSrv.Shutdown(sctx)
|
||||
})
|
||||
@@ -262,7 +295,8 @@ func newServerCmd() *cobra.Command {
|
||||
f := cmd.Flags()
|
||||
f.StringVar(&configPath, "config", "", "configuration file (optional)")
|
||||
f.StringVar(&listen, "listen", "", "override the client-facing address")
|
||||
f.StringVar(&adminListen, "admin-listen", "", "override the metrics/pprof address")
|
||||
f.StringVar(&adminListen, "admin-listen", "", "override the dashboard/pprof address")
|
||||
f.StringVar(&metricsListen, "metrics-listen", "", "serve metrics only on this address, safe to expose")
|
||||
f.StringArrayVar(&endpoints, "endpoint", nil, "static endpoint: id=URL=model[,model] (repeatable)")
|
||||
|
||||
return cmd
|
||||
@@ -308,6 +342,15 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
|
||||
return ctl, svc, nil
|
||||
}
|
||||
|
||||
// metricsAddr renders the metrics listener for the startup line, saying so
|
||||
// plainly when there is none rather than printing an empty string.
|
||||
func metricsAddr(ln net.Listener) string {
|
||||
if ln == nil {
|
||||
return "off (admin listener only)"
|
||||
}
|
||||
return ln.Addr().String()
|
||||
}
|
||||
|
||||
// dispatcherRef lets the controller be built before the service exists.
|
||||
type dispatcherRef struct {
|
||||
mu sync.RWMutex
|
||||
|
||||
Reference in new issue
Block a user