Guard the admin API and pprof with a token only root and the server can read

The admin listener answered any local process: models added, nodes
upgraded, engine keys listed, the heap dumped. /api and /debug now demand
the token the server writes 0600 to its state directory on first start;
ctl, the console and node add send it (sudo, LLMGW_ADMIN_TOKEN or
--token-file). pprof is off unless debug_pprof. admin_listen off loopback
is refused unless admin_remote. Dashboard pages, /metrics and /health are
unchanged.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

docs/reference.md

@@ -173,6 +173,9 @@ rather than repeated.
 | `threads` | `Threads` | [Server](#server) |
 | `landing` | `Landing` | [Server](#server) |
 | `agent_tls` | `tlsconf.Config` | [Server](#server) |
+| `admin_token_file` | `string` | [Server](#server) |
+| `admin_remote` | `bool` | [Server](#server) |
+| `debug_pprof` | `bool` | [Server](#server) |
 | `agent_insecure` | `bool` | [Server](#server) |
 | `monitoring` | `Monitoring` | [Server](#server) |
 | `tracing` | `Tracing` | [Server](#server) |
@@ -1378,6 +1381,30 @@ trusted certificate, while this one is mutual and private.
 
 Takes a [tlsconf.Config](#tlsconfconfig) block.
 
+### `admin_token_file`
+
+`string`
+
+AdminTokenFile holds the token the admin API's /api and /debug paths
+demand. Empty is admin-token in the systemd unit's state directory
+(/var/lib/llmgw), or secrets/admin-token beside this file for a server
+run by hand. Made, 0600, when the server starts and it does not exist;
+`sudo llmgw ctl` and the console read it from there.
+
+### `admin_remote`
+
+`bool`
+
+AdminRemote allows admin_listen on an address other than loopback. The
+token still guards the API, but it then crosses the network in clear.
+
+### `debug_pprof`
+
+`bool`
+
+DebugPprof serves Go's profiler on the admin listener, behind the token.
+Off by default: a heap profile holds the API keys requests presented.
+
 ### `agent_insecure`
 
 `bool`

internal/cli/admintoken.go

@@ -0,0 +1,100 @@
+package cli
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io/fs"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
+)
+
+// The admin token: what the admin API's /api and /debug paths demand, kept
+// in a file only root and the server can read.
+
+// defaultAdminTokenFile is where the server writes the token under its
+// systemd unit — its state directory, the one place ProtectSystem=strict
+// leaves it to write — and where clients look for it when told nothing.
+const defaultAdminTokenFile = "/var/lib/llmgw/admin-token"
+
+// adminTokenPath is the token's file for a configuration: as it says; under
+// systemd, in the unit's state directory; else secrets/admin-token beside
+// the configuration, for a server run by hand.
+func adminTokenPath(cfgPath string, cfg config.Server) string {
+	if cfg.AdminTokenFile != "" {
+		return cfg.AdminTokenFile
+	}
+	if dir := os.Getenv("STATE_DIRECTORY"); dir != "" {
+		return filepath.Join(strings.Split(dir, ":")[0], "admin-token")
+	}
+	return filepath.Join(filepath.Dir(cfgPath), "secrets", "admin-token")
+}
+
+// ensureAdminToken reads the token, making it when there is none.
+func ensureAdminToken(path string) (string, error) {
+	if b, err := os.ReadFile(path); err == nil { //nolint:gosec // the configured token file
+		if t := strings.TrimSpace(string(b)); len(t) >= 32 {
+			return t, nil
+		}
+	} else if !errors.Is(err, fs.ErrNotExist) {
+		return "", fmt.Errorf("reading the admin token %s: %w", path, err)
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		return "", fmt.Errorf("making %s: %w", filepath.Dir(path), err)
+	}
+	raw := make([]byte, 32)
+	if _, err := rand.Read(raw); err != nil {
+		return "", err
+	}
+	t := hex.EncodeToString(raw)
+	if err := os.WriteFile(path, []byte(t+"\n"), 0o600); err != nil {
+		return "", fmt.Errorf("writing the admin token %s: %w", path, err)
+	}
+	return t, nil
+}
+
+// readAdminToken is the token a client sends: LLMGW_ADMIN_TOKEN, else the
+// first of these files it can read. Empty when none: the request then goes
+// without, and the API says what is missing.
+func readAdminToken(paths ...string) string {
+	if t := strings.TrimSpace(os.Getenv("LLMGW_ADMIN_TOKEN")); t != "" {
+		return t
+	}
+	for _, p := range append(paths, defaultAdminTokenFile) {
+		if p == "" {
+			continue
+		}
+		if b, err := os.ReadFile(p); err == nil { //nolint:gosec // the admin token file
+			if t := strings.TrimSpace(string(b)); t != "" {
+				return t
+			}
+		}
+	}
+	return ""
+}
+
+// checkAdminListen refuses an admin listener off loopback unless the
+// configuration says it is meant.
+func checkAdminListen(cfg config.Server) error {
+	host, _, err := net.SplitHostPort(cfg.AdminListen)
+	if err != nil {
+		return fmt.Errorf("admin_listen %q: %w", cfg.AdminListen, err)
+	}
+	if host == "localhost" {
+		return nil
+	}
+	if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() {
+		return nil
+	}
+	if cfg.AdminRemote {
+		return nil
+	}
+	return fmt.Errorf("admin_listen %s is not loopback: the admin API adds models and upgrades nodes, and "+
+		"its token would cross the network in clear; keep it on 127.0.0.1 and reach it by ssh, or set "+
+		"admin_remote: true", cfg.AdminListen)
+}

internal/cli/admintoken_test.go

@@ -0,0 +1,52 @@
+package cli
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
+)
+
+func TestTheAdminTokenIsMadeOnceReadableOnlyByItsOwner(t *testing.T) {
+	dir := t.TempDir()
+	cfgPath := filepath.Join(dir, "server.yaml")
+	p := adminTokenPath(cfgPath, config.Default())
+	if p != filepath.Join(dir, "secrets", "admin-token") {
+		t.Fatalf("path %s", p)
+	}
+	tok, err := ensureAdminToken(p)
+	if err != nil || len(tok) != 64 {
+		t.Fatalf("%q %v", tok, err)
+	}
+	if st, _ := os.Stat(p); st.Mode().Perm() != 0o600 {
+		t.Errorf("mode %v", st.Mode())
+	}
+	again, _ := ensureAdminToken(p)
+	if again != tok {
+		t.Error("a second start made a new token")
+	}
+	t.Setenv("LLMGW_ADMIN_TOKEN", "")
+	if got := readAdminToken(p); got != tok {
+		t.Errorf("read %q", got)
+	}
+	t.Setenv("LLMGW_ADMIN_TOKEN", "from-env")
+	if got := readAdminToken(p); got != "from-env" {
+		t.Errorf("env: %q", got)
+	}
+}
+
+func TestTheAdminListenerStaysOnLoopbackUnlessToldOtherwise(t *testing.T) {
+	cfg := config.Default()
+	for addr, ok := range map[string]bool{"127.0.0.1:9090": true, "localhost:9090": true, "[::1]:9090": true,
+		"0.0.0.0:9090": false, ":9090": false, "10.0.0.140:9090": false} {
+		cfg.AdminListen = addr
+		if err := checkAdminListen(cfg); (err == nil) != ok {
+			t.Errorf("%s: %v", addr, err)
+		}
+	}
+	cfg.AdminListen, cfg.AdminRemote = "10.0.0.140:9090", true
+	if err := checkAdminListen(cfg); err != nil {
+		t.Errorf("admin_remote: %v", err)
+	}
+}

internal/cli/ctl.go

@@ -19,17 +19,22 @@ import (
 
 // ctlClient talks to a server's admin API.
 type ctlClient struct {
-	base string
-	http *http.Client
+	base  string
+	http  *http.Client
+	token string
 }
 
-func newCtlClient(addr string) *ctlClient {
+// newCtlClient talks to the admin API at addr, with the admin token from
+// LLMGW_ADMIN_TOKEN or the first of tokenFiles it can read, then the usual
+// place.
+func newCtlClient(addr string, tokenFiles ...string) *ctlClient {
 	if !strings.HasPrefix(addr, "http://") && !strings.HasPrefix(addr, "https://") {
 		addr = "http://" + addr
 	}
 	return &ctlClient{
-		base: strings.TrimRight(addr, "/"),
-		http: &http.Client{Timeout: 15 * time.Second},
+		base:  strings.TrimRight(addr, "/"),
+		http:  &http.Client{Timeout: 15 * time.Second},
+		token: readAdminToken(tokenFiles...),
 	}
 }
 
@@ -75,6 +80,9 @@ func (c *ctlClient) do(ctx context.Context, method, path string, body, out any)
 	// Who made the change, kept with a stored model so a catalog that grew
 	// over months can say where each entry came from.
 	req.Header.Set("X-Llmgw-Actor", actor())
+	if c.token != "" {
+		req.Header.Set("Authorization", "Bearer "+c.token)
+	}
 
 	resp, err := c.http.Do(req)
 	if err != nil {
@@ -104,8 +112,9 @@ func (c *ctlClient) do(ctx context.Context, method, path string, body, out any)
 
 func newCtlCmd() *cobra.Command {
 	var (
-		server string
-		asJSON bool
+		server    string
+		asJSON    bool
+		tokenFile string
 	)
 
 	cmd := &cobra.Command{
@@ -115,9 +124,11 @@ func newCtlCmd() *cobra.Command {
 			"default) rather than the client-facing port.",
 	}
 	cmd.PersistentFlags().StringVar(&server, "server", "127.0.0.1:9090", "admin API address")
+	cmd.PersistentFlags().StringVar(&tokenFile, "token-file", "",
+		"the admin token's file (default: LLMGW_ADMIN_TOKEN, else "+defaultAdminTokenFile+")")
 	cmd.PersistentFlags().BoolVar(&asJSON, "json", false, "emit JSON")
 
-	client := func() *ctlClient { return newCtlClient(server) }
+	client := func() *ctlClient { return newCtlClient(server, tokenFile) }
 
 	cmd.AddCommand(
 		newNodesCmd(client, &asJSON),

internal/cli/node.go

@@ -301,16 +301,21 @@ func (na *nodeAdd) runPlain(ctx context.Context, o onboard.Options, title string
 // this machine has it.
 func (na *nodeAdd) adminAPI() *ctlClient {
 	addr := na.admin
+	token := ""
+	cfg, err := config.Load(na.serverCfg)
+	if err == nil {
+		token = adminTokenPath(na.serverCfg, cfg)
+	}
 	if addr == "" {
 		addr = "127.0.0.1:9090"
-		if cfg, err := config.Load(na.serverCfg); err == nil && cfg.AdminListen != "" {
+		if err == nil && cfg.AdminListen != "" {
 			addr = cfg.AdminListen
 			if strings.HasPrefix(addr, ":") {
 				addr = "127.0.0.1" + addr
 			}
 		}
 	}
-	return newCtlClient(addr)
+	return newCtlClient(addr, token)
 }
 
 // errNoAdmin is the watch giving up because the admin API is not here, which

internal/cli/server.go

@@ -354,8 +354,15 @@ func newServerCmd() *cobra.Command {
 				// No write timeout: responses stream for minutes and the proxy
 				// enforces its own per-phase deadlines.
 			}
+			if err := checkAdminListen(cfg); err != nil {
+				return err
+			}
+			adminToken, err := ensureAdminToken(adminTokenPath(configPath, cfg))
+			if err != nil {
+				return err
+			}
 			adminSrv := &http.Server{
-				Handler:           router.AdminGuard(adminMux),
+				Handler:           router.AdminGuard(router.AdminAuth(adminToken, cfg.DebugPprof, adminMux)),
 				ReadHeaderTimeout: 10 * time.Second,
 			}
 

internal/cli/serverinit_cluster.go

@@ -552,11 +552,14 @@ func (in *serverInit) waitForStarter(ctx context.Context) error {
 }
 
 func (in *serverInit) adminAPI() *ctlClient {
-	addr := "127.0.0.1:9090"
-	if cfg, err := config.Load(in.configPath()); err == nil && cfg.AdminListen != "" {
-		addr = cfg.AdminListen
+	addr, token := "127.0.0.1:9090", ""
+	if cfg, err := config.Load(in.configPath()); err == nil {
+		token = adminTokenPath(in.configPath(), cfg)
+		if cfg.AdminListen != "" {
+			addr = cfg.AdminListen
+		}
 	}
-	return newCtlClient(addr)
+	return newCtlClient(addr, token)
 }
 
 // waitFor polls until done says so, fails, or the time runs out.

internal/cli/tui_console.go

@@ -1370,10 +1370,18 @@ func filepathBase(p string) string {
 // file is server.yaml on a control plane, agent.yaml on a node.
 func runConsole(ctx context.Context, out io.Writer, file string) error {
 	addr := "127.0.0.1:9090"
-	if cfg, err := config.Load(file); err == nil && cfg.AdminListen != "" {
-		addr = cfg.AdminListen
-		if strings.HasPrefix(addr, ":") {
-			addr = "127.0.0.1" + addr
+	if cfg, err := config.Load(file); err == nil {
+		if cfg.AdminListen != "" {
+			addr = cfg.AdminListen
+			if strings.HasPrefix(addr, ":") {
+				addr = "127.0.0.1" + addr
+			}
+		}
+		// The admin token, for this screen and every command it runs in
+		// this process — `llmgw ctl` included — from this configuration's
+		// file rather than only the usual place.
+		if t := readAdminToken(adminTokenPath(file, cfg)); t != "" {
+			_ = os.Setenv("LLMGW_ADMIN_TOKEN", t)
 		}
 	}
 	menu := cOverview

internal/config/config.go

@@ -65,6 +65,18 @@ type Server struct {
 	// cover the client-facing one: that surface is fronted by a publicly
 	// trusted certificate, while this one is mutual and private.
 	AgentTLS tlsconf.Config `koanf:"agent_tls"`
+	// AdminTokenFile holds the token the admin API's /api and /debug paths
+	// demand. Empty is admin-token in the systemd unit's state directory
+	// (/var/lib/llmgw), or secrets/admin-token beside this file for a server
+	// run by hand. Made, 0600, when the server starts and it does not exist;
+	// `sudo llmgw ctl` and the console read it from there.
+	AdminTokenFile string `koanf:"admin_token_file"`
+	// AdminRemote allows admin_listen on an address other than loopback. The
+	// token still guards the API, but it then crosses the network in clear.
+	AdminRemote bool `koanf:"admin_remote"`
+	// DebugPprof serves Go's profiler on the admin listener, behind the token.
+	// Off by default: a heap profile holds the API keys requests presented.
+	DebugPprof bool `koanf:"debug_pprof"`
 	// AgentInsecure lets a control plane run its agent port without
 	// agent_tls: plaintext, and any machine that reaches it may register as
 	// any node — be given replicas, and so be sent users' prompts — or, in

internal/docgen/reference_gen.go

@@ -213,6 +213,9 @@ var Reference = []Block{
 		{Key: "threads", Type: "Threads", Doc: "Threads keeps conversations and agent memory without API keys, for a\ngateway on one trusted machine. With auth on they live in the auth store,\nper project, and this block is ignored.", Nested: "config.Threads"},
 		{Key: "landing", Type: "Landing", Doc: "Landing configures the page served at \"/\".", Nested: "config.Landing"},
 		{Key: "agent_tls", Type: "tlsconf.Config", Doc: "AgentTLS authenticates the fleet-facing port. It deliberately does not\ncover the client-facing one: that surface is fronted by a publicly\ntrusted certificate, while this one is mutual and private.", Nested: "tlsconf.Config"},
+		{Key: "admin_token_file", Type: "string", Doc: "AdminTokenFile holds the token the admin API's /api and /debug paths\ndemand. Empty is admin-token in the systemd unit's state directory\n(/var/lib/llmgw), or secrets/admin-token beside this file for a server\nrun by hand. Made, 0600, when the server starts and it does not exist;\n`sudo llmgw ctl` and the console read it from there.", Nested: ""},
+		{Key: "admin_remote", Type: "bool", Doc: "AdminRemote allows admin_listen on an address other than loopback. The\ntoken still guards the API, but it then crosses the network in clear.", Nested: ""},
+		{Key: "debug_pprof", Type: "bool", Doc: "DebugPprof serves Go's profiler on the admin listener, behind the token.\nOff by default: a heap profile holds the API keys requests presented.", Nested: ""},
 		{Key: "agent_insecure", Type: "bool", Doc: "AgentInsecure lets a control plane run its agent port without\nagent_tls: plaintext, and any machine that reaches it may register as\nany node — be given replicas, and so be sent users' prompts — or, in\nthe other direction, a machine in the path may tell every agent what\nto run. Refused unless said here, and then the port listens on\nloopback unless agent_listen names an address. For one machine, a\ntest, or a network nothing untrusted reaches.", Nested: ""},
 		{Key: "monitoring", Type: "Monitoring", Doc: "Monitoring is where the dashboard reads its history from.", Nested: "config.Monitoring"},
 		{Key: "tracing", Type: "Tracing", Doc: "Tracing sends a trace of each inference request to an OpenTelemetry\ncollector. Off unless an endpoint is set here or in\nOTEL_EXPORTER_OTLP_ENDPOINT.", Nested: "config.Tracing"},

internal/router/adminauth.go

@@ -0,0 +1,71 @@
+package router
+
+import (
+	"crypto/subtle"
+	"net/http"
+	"strings"
+)
+
+// AdminAuth demands the admin token for what the admin listener can change
+// or reveal: its API, which adds models, upgrades nodes and hands out engine
+// keys, and pprof, whose heap holds the API keys requests presented. Being
+// on loopback kept browsers out; it did not keep out every other process and
+// user on the machine.
+//
+// The token lives in a file only root and the server can read, so `sudo
+// llmgw ctl` and the console have it and nobody else does. pprof is served
+// only when asked for in the configuration, and then also behind the token.
+// The read-only pages — the cluster dashboard reached by SSH tunnel — and
+// /metrics and /health are left as they were.
+func AdminAuth(token string, pprof bool, next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case strings.HasPrefix(r.URL.Path, "/debug/"):
+			if !pprof {
+				http.NotFound(w, r)
+				return
+			}
+		case strings.HasPrefix(r.URL.Path, "/api/"):
+		default:
+			next.ServeHTTP(w, r)
+			return
+		}
+		if !tokenMatches(token, r.Header.Get("Authorization")) {
+			w.Header().Set("WWW-Authenticate", `Bearer realm="llmgw-admin"`)
+			writeJSONError(w, http.StatusUnauthorized,
+				"the admin API needs its token: run as root (sudo llmgw ...), or set LLMGW_ADMIN_TOKEN")
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
+func tokenMatches(token, header string) bool {
+	got, ok := strings.CutPrefix(header, "Bearer ")
+	if !ok || token == "" {
+		return false
+	}
+	return subtle.ConstantTimeCompare([]byte(strings.TrimSpace(got)), []byte(token)) == 1
+}
+
+func writeJSONError(w http.ResponseWriter, code int, msg string) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(code)
+	_, _ = w.Write([]byte(`{"error":` + quoteJSON(msg) + `}`))
+}
+
+func quoteJSON(s string) string {
+	var b strings.Builder
+	b.WriteByte('"')
+	for _, c := range s {
+		switch c {
+		case '"', '\\':
+			b.WriteByte('\\')
+			b.WriteRune(c)
+		default:
+			b.WriteRune(c)
+		}
+	}
+	b.WriteByte('"')
+	return b.String()
+}

internal/router/adminauth_test.go

@@ -0,0 +1,47 @@
+package router
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestTheAdminAPIAndProfilerWantTheTokenAndThePagesDoNot(t *testing.T) {
+	ok := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })
+	cases := []struct {
+		pprof        bool
+		path, header string
+		want         int
+	}{
+		{false, "/api/v1/models", "", http.StatusUnauthorized},
+		{false, "/api/v1/models", "Bearer wrong", http.StatusUnauthorized},
+		{false, "/api/v1/models", "Bearer s3cret-token-of-enough-length-000000", http.StatusOK},
+		{false, "/api/v1/replicas", "s3cret-token-of-enough-length-000000", http.StatusUnauthorized},
+		{false, "/debug/pprof/heap", "Bearer s3cret-token-of-enough-length-000000", http.StatusNotFound},
+		{true, "/debug/pprof/heap", "", http.StatusUnauthorized},
+		{true, "/debug/pprof/heap", "Bearer s3cret-token-of-enough-length-000000", http.StatusOK},
+		{false, "/", "", http.StatusOK},
+		{false, "/nodes/n1", "", http.StatusOK},
+		{false, "/metrics", "", http.StatusOK},
+	}
+	for _, c := range cases {
+		h := AdminAuth("s3cret-token-of-enough-length-000000", c.pprof, ok)
+		r := httptest.NewRequest("GET", c.path, nil)
+		if c.header != "" {
+			r.Header.Set("Authorization", c.header)
+		}
+		w := httptest.NewRecorder()
+		h.ServeHTTP(w, r)
+		if w.Code != c.want {
+			t.Errorf("pprof=%v %s %q: %d, want %d", c.pprof, c.path, c.header, w.Code, c.want)
+		}
+	}
+	// No token configured is no access, not open access.
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest("GET", "/api/v1/models", nil)
+	r.Header.Set("Authorization", "Bearer ")
+	AdminAuth("", false, ok).ServeHTTP(w, r)
+	if w.Code != http.StatusUnauthorized {
+		t.Errorf("an empty token let a request in: %d", w.Code)
+	}
+}