From c0421bc7ee79aa4ace0b4e7d1e3b88e5d803f82d Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 4 Oct 2026 10:02:44 +0200 Subject: [PATCH] Guard the admin API and pprof with a token only root and the server can read The admin listener answered any local process: models added, nodes upgraded, engine keys listed, the heap dumped. /api and /debug now demand the token the server writes 0600 to its state directory on first start; ctl, the console and node add send it (sudo, LLMGW_ADMIN_TOKEN or --token-file). pprof is off unless debug_pprof. admin_listen off loopback is refused unless admin_remote. Dashboard pages, /metrics and /health are unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/reference.md | 27 ++++++++ internal/cli/admintoken.go | 100 +++++++++++++++++++++++++++++ internal/cli/admintoken_test.go | 52 +++++++++++++++ internal/cli/ctl.go | 27 +++++--- internal/cli/node.go | 9 ++- internal/cli/server.go | 9 ++- internal/cli/serverinit_cluster.go | 11 ++-- internal/cli/tui_console.go | 16 +++-- internal/config/config.go | 12 ++++ internal/docgen/reference_gen.go | 3 + internal/router/adminauth.go | 71 ++++++++++++++++++++ internal/router/adminauth_test.go | 47 ++++++++++++++ 12 files changed, 365 insertions(+), 19 deletions(-) create mode 100644 internal/cli/admintoken.go create mode 100644 internal/cli/admintoken_test.go create mode 100644 internal/router/adminauth.go create mode 100644 internal/router/adminauth_test.go diff --git a/docs/reference.md b/docs/reference.md index b75d3d9..a657f37 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -173,6 +173,9 @@ rather than repeated. | `threads` | `Threads` | [Server](#server) | | `landing` | `Landing` | [Server](#server) | | `agent_tls` | `tlsconf.Config` | [Server](#server) | +| `admin_token_file` | `string` | [Server](#server) | +| `admin_remote` | `bool` | [Server](#server) | +| `debug_pprof` | `bool` | [Server](#server) | | `agent_insecure` | `bool` | [Server](#server) | | `monitoring` | `Monitoring` | [Server](#server) | | `tracing` | `Tracing` | [Server](#server) | @@ -1378,6 +1381,30 @@ trusted certificate, while this one is mutual and private. Takes a [tlsconf.Config](#tlsconfconfig) block. +### `admin_token_file` + +`string` + +AdminTokenFile holds the token the admin API's /api and /debug paths +demand. Empty is admin-token in the systemd unit's state directory +(/var/lib/llmgw), or secrets/admin-token beside this file for a server +run by hand. Made, 0600, when the server starts and it does not exist; +`sudo llmgw ctl` and the console read it from there. + +### `admin_remote` + +`bool` + +AdminRemote allows admin_listen on an address other than loopback. The +token still guards the API, but it then crosses the network in clear. + +### `debug_pprof` + +`bool` + +DebugPprof serves Go's profiler on the admin listener, behind the token. +Off by default: a heap profile holds the API keys requests presented. + ### `agent_insecure` `bool` diff --git a/internal/cli/admintoken.go b/internal/cli/admintoken.go new file mode 100644 index 0000000..79fe3ec --- /dev/null +++ b/internal/cli/admintoken.go @@ -0,0 +1,100 @@ +package cli + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "net" + "os" + "path/filepath" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/config" +) + +// The admin token: what the admin API's /api and /debug paths demand, kept +// in a file only root and the server can read. + +// defaultAdminTokenFile is where the server writes the token under its +// systemd unit — its state directory, the one place ProtectSystem=strict +// leaves it to write — and where clients look for it when told nothing. +const defaultAdminTokenFile = "/var/lib/llmgw/admin-token" + +// adminTokenPath is the token's file for a configuration: as it says; under +// systemd, in the unit's state directory; else secrets/admin-token beside +// the configuration, for a server run by hand. +func adminTokenPath(cfgPath string, cfg config.Server) string { + if cfg.AdminTokenFile != "" { + return cfg.AdminTokenFile + } + if dir := os.Getenv("STATE_DIRECTORY"); dir != "" { + return filepath.Join(strings.Split(dir, ":")[0], "admin-token") + } + return filepath.Join(filepath.Dir(cfgPath), "secrets", "admin-token") +} + +// ensureAdminToken reads the token, making it when there is none. +func ensureAdminToken(path string) (string, error) { + if b, err := os.ReadFile(path); err == nil { //nolint:gosec // the configured token file + if t := strings.TrimSpace(string(b)); len(t) >= 32 { + return t, nil + } + } else if !errors.Is(err, fs.ErrNotExist) { + return "", fmt.Errorf("reading the admin token %s: %w", path, err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return "", fmt.Errorf("making %s: %w", filepath.Dir(path), err) + } + raw := make([]byte, 32) + if _, err := rand.Read(raw); err != nil { + return "", err + } + t := hex.EncodeToString(raw) + if err := os.WriteFile(path, []byte(t+"\n"), 0o600); err != nil { + return "", fmt.Errorf("writing the admin token %s: %w", path, err) + } + return t, nil +} + +// readAdminToken is the token a client sends: LLMGW_ADMIN_TOKEN, else the +// first of these files it can read. Empty when none: the request then goes +// without, and the API says what is missing. +func readAdminToken(paths ...string) string { + if t := strings.TrimSpace(os.Getenv("LLMGW_ADMIN_TOKEN")); t != "" { + return t + } + for _, p := range append(paths, defaultAdminTokenFile) { + if p == "" { + continue + } + if b, err := os.ReadFile(p); err == nil { //nolint:gosec // the admin token file + if t := strings.TrimSpace(string(b)); t != "" { + return t + } + } + } + return "" +} + +// checkAdminListen refuses an admin listener off loopback unless the +// configuration says it is meant. +func checkAdminListen(cfg config.Server) error { + host, _, err := net.SplitHostPort(cfg.AdminListen) + if err != nil { + return fmt.Errorf("admin_listen %q: %w", cfg.AdminListen, err) + } + if host == "localhost" { + return nil + } + if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() { + return nil + } + if cfg.AdminRemote { + return nil + } + return fmt.Errorf("admin_listen %s is not loopback: the admin API adds models and upgrades nodes, and "+ + "its token would cross the network in clear; keep it on 127.0.0.1 and reach it by ssh, or set "+ + "admin_remote: true", cfg.AdminListen) +} diff --git a/internal/cli/admintoken_test.go b/internal/cli/admintoken_test.go new file mode 100644 index 0000000..a5cc307 --- /dev/null +++ b/internal/cli/admintoken_test.go @@ -0,0 +1,52 @@ +package cli + +import ( + "os" + "path/filepath" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/config" +) + +func TestTheAdminTokenIsMadeOnceReadableOnlyByItsOwner(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "server.yaml") + p := adminTokenPath(cfgPath, config.Default()) + if p != filepath.Join(dir, "secrets", "admin-token") { + t.Fatalf("path %s", p) + } + tok, err := ensureAdminToken(p) + if err != nil || len(tok) != 64 { + t.Fatalf("%q %v", tok, err) + } + if st, _ := os.Stat(p); st.Mode().Perm() != 0o600 { + t.Errorf("mode %v", st.Mode()) + } + again, _ := ensureAdminToken(p) + if again != tok { + t.Error("a second start made a new token") + } + t.Setenv("LLMGW_ADMIN_TOKEN", "") + if got := readAdminToken(p); got != tok { + t.Errorf("read %q", got) + } + t.Setenv("LLMGW_ADMIN_TOKEN", "from-env") + if got := readAdminToken(p); got != "from-env" { + t.Errorf("env: %q", got) + } +} + +func TestTheAdminListenerStaysOnLoopbackUnlessToldOtherwise(t *testing.T) { + cfg := config.Default() + for addr, ok := range map[string]bool{"127.0.0.1:9090": true, "localhost:9090": true, "[::1]:9090": true, + "0.0.0.0:9090": false, ":9090": false, "10.0.0.140:9090": false} { + cfg.AdminListen = addr + if err := checkAdminListen(cfg); (err == nil) != ok { + t.Errorf("%s: %v", addr, err) + } + } + cfg.AdminListen, cfg.AdminRemote = "10.0.0.140:9090", true + if err := checkAdminListen(cfg); err != nil { + t.Errorf("admin_remote: %v", err) + } +} diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index fc41efb..b54ab3c 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -19,17 +19,22 @@ import ( // ctlClient talks to a server's admin API. type ctlClient struct { - base string - http *http.Client + base string + http *http.Client + token string } -func newCtlClient(addr string) *ctlClient { +// newCtlClient talks to the admin API at addr, with the admin token from +// LLMGW_ADMIN_TOKEN or the first of tokenFiles it can read, then the usual +// place. +func newCtlClient(addr string, tokenFiles ...string) *ctlClient { if !strings.HasPrefix(addr, "http://") && !strings.HasPrefix(addr, "https://") { addr = "http://" + addr } return &ctlClient{ - base: strings.TrimRight(addr, "/"), - http: &http.Client{Timeout: 15 * time.Second}, + base: strings.TrimRight(addr, "/"), + http: &http.Client{Timeout: 15 * time.Second}, + token: readAdminToken(tokenFiles...), } } @@ -75,6 +80,9 @@ func (c *ctlClient) do(ctx context.Context, method, path string, body, out any) // Who made the change, kept with a stored model so a catalog that grew // over months can say where each entry came from. req.Header.Set("X-Llmgw-Actor", actor()) + if c.token != "" { + req.Header.Set("Authorization", "Bearer "+c.token) + } resp, err := c.http.Do(req) if err != nil { @@ -104,8 +112,9 @@ func (c *ctlClient) do(ctx context.Context, method, path string, body, out any) func newCtlCmd() *cobra.Command { var ( - server string - asJSON bool + server string + asJSON bool + tokenFile string ) cmd := &cobra.Command{ @@ -115,9 +124,11 @@ func newCtlCmd() *cobra.Command { "default) rather than the client-facing port.", } cmd.PersistentFlags().StringVar(&server, "server", "127.0.0.1:9090", "admin API address") + cmd.PersistentFlags().StringVar(&tokenFile, "token-file", "", + "the admin token's file (default: LLMGW_ADMIN_TOKEN, else "+defaultAdminTokenFile+")") cmd.PersistentFlags().BoolVar(&asJSON, "json", false, "emit JSON") - client := func() *ctlClient { return newCtlClient(server) } + client := func() *ctlClient { return newCtlClient(server, tokenFile) } cmd.AddCommand( newNodesCmd(client, &asJSON), diff --git a/internal/cli/node.go b/internal/cli/node.go index 92f73af..5499540 100644 --- a/internal/cli/node.go +++ b/internal/cli/node.go @@ -301,16 +301,21 @@ func (na *nodeAdd) runPlain(ctx context.Context, o onboard.Options, title string // this machine has it. func (na *nodeAdd) adminAPI() *ctlClient { addr := na.admin + token := "" + cfg, err := config.Load(na.serverCfg) + if err == nil { + token = adminTokenPath(na.serverCfg, cfg) + } if addr == "" { addr = "127.0.0.1:9090" - if cfg, err := config.Load(na.serverCfg); err == nil && cfg.AdminListen != "" { + if err == nil && cfg.AdminListen != "" { addr = cfg.AdminListen if strings.HasPrefix(addr, ":") { addr = "127.0.0.1" + addr } } } - return newCtlClient(addr) + return newCtlClient(addr, token) } // errNoAdmin is the watch giving up because the admin API is not here, which diff --git a/internal/cli/server.go b/internal/cli/server.go index b7ee8d8..adc9c7d 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -354,8 +354,15 @@ func newServerCmd() *cobra.Command { // No write timeout: responses stream for minutes and the proxy // enforces its own per-phase deadlines. } + if err := checkAdminListen(cfg); err != nil { + return err + } + adminToken, err := ensureAdminToken(adminTokenPath(configPath, cfg)) + if err != nil { + return err + } adminSrv := &http.Server{ - Handler: router.AdminGuard(adminMux), + Handler: router.AdminGuard(router.AdminAuth(adminToken, cfg.DebugPprof, adminMux)), ReadHeaderTimeout: 10 * time.Second, } diff --git a/internal/cli/serverinit_cluster.go b/internal/cli/serverinit_cluster.go index 2d9e492..e2b145a 100644 --- a/internal/cli/serverinit_cluster.go +++ b/internal/cli/serverinit_cluster.go @@ -552,11 +552,14 @@ func (in *serverInit) waitForStarter(ctx context.Context) error { } func (in *serverInit) adminAPI() *ctlClient { - addr := "127.0.0.1:9090" - if cfg, err := config.Load(in.configPath()); err == nil && cfg.AdminListen != "" { - addr = cfg.AdminListen + addr, token := "127.0.0.1:9090", "" + if cfg, err := config.Load(in.configPath()); err == nil { + token = adminTokenPath(in.configPath(), cfg) + if cfg.AdminListen != "" { + addr = cfg.AdminListen + } } - return newCtlClient(addr) + return newCtlClient(addr, token) } // waitFor polls until done says so, fails, or the time runs out. diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index 947b138..6d2fbae 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -1370,10 +1370,18 @@ func filepathBase(p string) string { // file is server.yaml on a control plane, agent.yaml on a node. func runConsole(ctx context.Context, out io.Writer, file string) error { addr := "127.0.0.1:9090" - if cfg, err := config.Load(file); err == nil && cfg.AdminListen != "" { - addr = cfg.AdminListen - if strings.HasPrefix(addr, ":") { - addr = "127.0.0.1" + addr + if cfg, err := config.Load(file); err == nil { + if cfg.AdminListen != "" { + addr = cfg.AdminListen + if strings.HasPrefix(addr, ":") { + addr = "127.0.0.1" + addr + } + } + // The admin token, for this screen and every command it runs in + // this process — `llmgw ctl` included — from this configuration's + // file rather than only the usual place. + if t := readAdminToken(adminTokenPath(file, cfg)); t != "" { + _ = os.Setenv("LLMGW_ADMIN_TOKEN", t) } } menu := cOverview diff --git a/internal/config/config.go b/internal/config/config.go index 6fbccd7..1892e95 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -65,6 +65,18 @@ type Server struct { // cover the client-facing one: that surface is fronted by a publicly // trusted certificate, while this one is mutual and private. AgentTLS tlsconf.Config `koanf:"agent_tls"` + // AdminTokenFile holds the token the admin API's /api and /debug paths + // demand. Empty is admin-token in the systemd unit's state directory + // (/var/lib/llmgw), or secrets/admin-token beside this file for a server + // run by hand. Made, 0600, when the server starts and it does not exist; + // `sudo llmgw ctl` and the console read it from there. + AdminTokenFile string `koanf:"admin_token_file"` + // AdminRemote allows admin_listen on an address other than loopback. The + // token still guards the API, but it then crosses the network in clear. + AdminRemote bool `koanf:"admin_remote"` + // DebugPprof serves Go's profiler on the admin listener, behind the token. + // Off by default: a heap profile holds the API keys requests presented. + DebugPprof bool `koanf:"debug_pprof"` // AgentInsecure lets a control plane run its agent port without // agent_tls: plaintext, and any machine that reaches it may register as // any node — be given replicas, and so be sent users' prompts — or, in diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index 88db6c2..dc4a2b0 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -213,6 +213,9 @@ var Reference = []Block{ {Key: "threads", Type: "Threads", Doc: "Threads keeps conversations and agent memory without API keys, for a\ngateway on one trusted machine. With auth on they live in the auth store,\nper project, and this block is ignored.", Nested: "config.Threads"}, {Key: "landing", Type: "Landing", Doc: "Landing configures the page served at \"/\".", Nested: "config.Landing"}, {Key: "agent_tls", Type: "tlsconf.Config", Doc: "AgentTLS authenticates the fleet-facing port. It deliberately does not\ncover the client-facing one: that surface is fronted by a publicly\ntrusted certificate, while this one is mutual and private.", Nested: "tlsconf.Config"}, + {Key: "admin_token_file", Type: "string", Doc: "AdminTokenFile holds the token the admin API's /api and /debug paths\ndemand. Empty is admin-token in the systemd unit's state directory\n(/var/lib/llmgw), or secrets/admin-token beside this file for a server\nrun by hand. Made, 0600, when the server starts and it does not exist;\n`sudo llmgw ctl` and the console read it from there.", Nested: ""}, + {Key: "admin_remote", Type: "bool", Doc: "AdminRemote allows admin_listen on an address other than loopback. The\ntoken still guards the API, but it then crosses the network in clear.", Nested: ""}, + {Key: "debug_pprof", Type: "bool", Doc: "DebugPprof serves Go's profiler on the admin listener, behind the token.\nOff by default: a heap profile holds the API keys requests presented.", Nested: ""}, {Key: "agent_insecure", Type: "bool", Doc: "AgentInsecure lets a control plane run its agent port without\nagent_tls: plaintext, and any machine that reaches it may register as\nany node — be given replicas, and so be sent users' prompts — or, in\nthe other direction, a machine in the path may tell every agent what\nto run. Refused unless said here, and then the port listens on\nloopback unless agent_listen names an address. For one machine, a\ntest, or a network nothing untrusted reaches.", Nested: ""}, {Key: "monitoring", Type: "Monitoring", Doc: "Monitoring is where the dashboard reads its history from.", Nested: "config.Monitoring"}, {Key: "tracing", Type: "Tracing", Doc: "Tracing sends a trace of each inference request to an OpenTelemetry\ncollector. Off unless an endpoint is set here or in\nOTEL_EXPORTER_OTLP_ENDPOINT.", Nested: "config.Tracing"}, diff --git a/internal/router/adminauth.go b/internal/router/adminauth.go new file mode 100644 index 0000000..284a986 --- /dev/null +++ b/internal/router/adminauth.go @@ -0,0 +1,71 @@ +package router + +import ( + "crypto/subtle" + "net/http" + "strings" +) + +// AdminAuth demands the admin token for what the admin listener can change +// or reveal: its API, which adds models, upgrades nodes and hands out engine +// keys, and pprof, whose heap holds the API keys requests presented. Being +// on loopback kept browsers out; it did not keep out every other process and +// user on the machine. +// +// The token lives in a file only root and the server can read, so `sudo +// llmgw ctl` and the console have it and nobody else does. pprof is served +// only when asked for in the configuration, and then also behind the token. +// The read-only pages — the cluster dashboard reached by SSH tunnel — and +// /metrics and /health are left as they were. +func AdminAuth(token string, pprof bool, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasPrefix(r.URL.Path, "/debug/"): + if !pprof { + http.NotFound(w, r) + return + } + case strings.HasPrefix(r.URL.Path, "/api/"): + default: + next.ServeHTTP(w, r) + return + } + if !tokenMatches(token, r.Header.Get("Authorization")) { + w.Header().Set("WWW-Authenticate", `Bearer realm="llmgw-admin"`) + writeJSONError(w, http.StatusUnauthorized, + "the admin API needs its token: run as root (sudo llmgw ...), or set LLMGW_ADMIN_TOKEN") + return + } + next.ServeHTTP(w, r) + }) +} + +func tokenMatches(token, header string) bool { + got, ok := strings.CutPrefix(header, "Bearer ") + if !ok || token == "" { + return false + } + return subtle.ConstantTimeCompare([]byte(strings.TrimSpace(got)), []byte(token)) == 1 +} + +func writeJSONError(w http.ResponseWriter, code int, msg string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _, _ = w.Write([]byte(`{"error":` + quoteJSON(msg) + `}`)) +} + +func quoteJSON(s string) string { + var b strings.Builder + b.WriteByte('"') + for _, c := range s { + switch c { + case '"', '\\': + b.WriteByte('\\') + b.WriteRune(c) + default: + b.WriteRune(c) + } + } + b.WriteByte('"') + return b.String() +} diff --git a/internal/router/adminauth_test.go b/internal/router/adminauth_test.go new file mode 100644 index 0000000..e24043d --- /dev/null +++ b/internal/router/adminauth_test.go @@ -0,0 +1,47 @@ +package router + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestTheAdminAPIAndProfilerWantTheTokenAndThePagesDoNot(t *testing.T) { + ok := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) + cases := []struct { + pprof bool + path, header string + want int + }{ + {false, "/api/v1/models", "", http.StatusUnauthorized}, + {false, "/api/v1/models", "Bearer wrong", http.StatusUnauthorized}, + {false, "/api/v1/models", "Bearer s3cret-token-of-enough-length-000000", http.StatusOK}, + {false, "/api/v1/replicas", "s3cret-token-of-enough-length-000000", http.StatusUnauthorized}, + {false, "/debug/pprof/heap", "Bearer s3cret-token-of-enough-length-000000", http.StatusNotFound}, + {true, "/debug/pprof/heap", "", http.StatusUnauthorized}, + {true, "/debug/pprof/heap", "Bearer s3cret-token-of-enough-length-000000", http.StatusOK}, + {false, "/", "", http.StatusOK}, + {false, "/nodes/n1", "", http.StatusOK}, + {false, "/metrics", "", http.StatusOK}, + } + for _, c := range cases { + h := AdminAuth("s3cret-token-of-enough-length-000000", c.pprof, ok) + r := httptest.NewRequest("GET", c.path, nil) + if c.header != "" { + r.Header.Set("Authorization", c.header) + } + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + if w.Code != c.want { + t.Errorf("pprof=%v %s %q: %d, want %d", c.pprof, c.path, c.header, w.Code, c.want) + } + } + // No token configured is no access, not open access. + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "/api/v1/models", nil) + r.Header.Set("Authorization", "Bearer ") + AdminAuth("", false, ok).ServeHTTP(w, r) + if w.Code != http.StatusUnauthorized { + t.Errorf("an empty token let a request in: %d", w.Code) + } +}