Serve the cluster dashboard over the web, without serving pprof with it
CI / test (push) Successful in 1m21s
CI / test (push) Successful in 1m21s
Reaching it meant an SSH tunnel, because the page lived on the admin listener and that listener is bound to loopback. Opening the port was never the way to fix that: it also serves pprof, and pprof can dump the process heap, where every API key sits in plaintext. Exposing the port would have been exposing the key store. So the page moves rather than the port. The same dashboard is now mounted at /admin on the client-facing port, behind the session and CSRF machinery that already guards /profile, gated on a flag on the account. The admin listener keeps pprof and metrics and stays on loopback. One bit, not a role system: it grants looking. Everything that changes the cluster or issues a credential stays on the CLI, where it needs a shell rather than a session — which keeps the worst case of a stolen admin cookie to "saw the cluster" rather than "drained your fleet". Someone without the flag gets 404, not 403. A 403 confirms the page is there and that admin accounts exist here, which is free reconnaissance; 404 is what every other unknown path returns. The profile page does not show a link to a door the visitor cannot open. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
058ee382fa
commit
e519dc587c
12 files changed
+250
-14
No files matched your search
+32
-3
@@ -236,14 +236,43 @@ Tell them out of band and have them change it from their profile. Their sessions
|
||||
are closed — if they lost the password, the reason might be that someone else has
|
||||
it. Their API keys keep working; those are separate credentials.
|
||||
|
||||
## Seeing the cluster
|
||||
|
||||
The dashboard — nodes, replicas, models, why something did not get placed, usage
|
||||
per key — is served twice, and the difference matters.
|
||||
|
||||
On the **admin listener** (`127.0.0.1:9090` by default) it needs no
|
||||
authentication: having a shell on the machine is the credential.
|
||||
|
||||
```
|
||||
ssh -L 9090:127.0.0.1:9090 <control-plane> # then http://127.0.0.1:9090/
|
||||
```
|
||||
|
||||
That listener also serves pprof, which can dump the process heap — where API
|
||||
keys sit in plaintext. **Never expose that port.** It is bound to loopback for
|
||||
that reason.
|
||||
|
||||
On the **client-facing port** the same page is at `/admin`, behind a session and
|
||||
an admin flag, so it can be reached from a browser without the port carrying
|
||||
pprof along with it:
|
||||
|
||||
```
|
||||
llmgw users promote usr-k4mq… --store …
|
||||
```
|
||||
|
||||
Someone without the flag gets a 404 rather than a 403, because 403 confirms the
|
||||
page exists. The dashboard is read-only in both places: everything that changes
|
||||
the cluster or issues a credential stays on the CLI, which keeps the worst case
|
||||
of a stolen admin cookie to "saw the cluster".
|
||||
|
||||
## What this does not do
|
||||
|
||||
- **No email.** No approval notice, no self-service password reset, no address
|
||||
verification. The loop is manual on purpose at this size; if it stops being
|
||||
the right size, that is the thing to automate.
|
||||
- **No SSO.** Accounts are local, with local passwords.
|
||||
- **No roles.** Every approved account can do the same things: make its own keys
|
||||
and nothing else. Administration is the CLI, and having a shell here is what
|
||||
makes someone an administrator.
|
||||
- **No roles.** There is one flag, `is_admin`, and it grants looking at the
|
||||
dashboard. Everything else an administrator does is the CLI, and having a
|
||||
shell here is what makes someone one.
|
||||
- **Rate limits are per gateway instance.** With two instances each counts its own,
|
||||
so the effective limit doubles. v1 is single-instance by design.
|
||||
@@ -129,6 +129,11 @@ func newServerCmd() *cobra.Command {
|
||||
adminMux.HandleFunc("GET /", web.Dashboard(
|
||||
api, usage, version.Version, 5*time.Second, log))
|
||||
adminMux.Handle("GET /_static/", web.Static())
|
||||
|
||||
// The same page on the client-facing port at /admin, behind a
|
||||
// session and an admin flag. Handing the router the data rather
|
||||
// than the port keeps pprof where it is.
|
||||
rt.SetClusterView(api, usage)
|
||||
}
|
||||
|
||||
apiSrv := &http.Server{
|
||||
|
||||
+45
-4
@@ -32,7 +32,9 @@ func newUsersCmd() *cobra.Command {
|
||||
"Stop an account working, keeping it and its history"),
|
||||
newUserStateCmd(open, "reinstate", store.UserActive,
|
||||
"Let a suspended account work again"),
|
||||
newUserCreateCmd(open), newUserResetPasswordCmd(open))
|
||||
newUserCreateCmd(open), newUserResetPasswordCmd(open),
|
||||
newUserAdminCmd(open, "promote", true, "Let an account see the cluster dashboard at /admin"),
|
||||
newUserAdminCmd(open, "demote", false, "Take away access to the cluster dashboard"))
|
||||
return cmd
|
||||
}
|
||||
|
||||
@@ -65,11 +67,15 @@ func newUserListCmd(open userOpener) *cobra.Command {
|
||||
}
|
||||
|
||||
w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(w, "ID\tEMAIL\tNAME\tSTATE\tPROJECT\tSIGNED UP\tLAST LOGIN")
|
||||
fmt.Fprintln(w, "ID\tEMAIL\tNAME\tSTATE\tADMIN\tPROJECT\tSIGNED UP\tLAST LOGIN")
|
||||
now := time.Now()
|
||||
for _, u := range users {
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
|
||||
u.ID, u.Email, u.Name, u.State, dash(u.ProjectID),
|
||||
admin := "-"
|
||||
if u.IsAdmin {
|
||||
admin = "yes"
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
|
||||
u.ID, u.Email, u.Name, u.State, admin, dash(u.ProjectID),
|
||||
ago(u.CreatedAt, now), ago(u.LastLoginAt, now))
|
||||
}
|
||||
return w.Flush()
|
||||
@@ -142,6 +148,41 @@ func newUserStateCmd(open userOpener, verb string, state store.UserState, short
|
||||
}
|
||||
}
|
||||
|
||||
func newUserAdminCmd(open userOpener, verb string, admin bool, short string) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: verb + " <user-id>",
|
||||
Short: short,
|
||||
Long: "The dashboard is read-only, so this grants looking and nothing else.\n" +
|
||||
"Everything that changes the cluster or issues a credential stays here on\n" +
|
||||
"the CLI, where it needs a shell rather than a session — which keeps the\n" +
|
||||
"worst case of a stolen admin cookie to \"saw the cluster\".",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
s, err := open(cmd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
if err := s.SetAdmin(cmd.Context(), args[0], admin); err != nil {
|
||||
return err
|
||||
}
|
||||
u, err := s.UserByID(cmd.Context(), args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if admin {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s can now see /admin\n", u.Email)
|
||||
} else {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s can no longer see /admin\n", u.Email)
|
||||
cmd.PrintErrln("Their session stays open; the page is read-only. " +
|
||||
"Suspend the account to close it.")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newUserResetPasswordCmd(open userOpener) *cobra.Command {
|
||||
var password string
|
||||
cmd := &cobra.Command{
|
||||
|
||||
@@ -106,7 +106,16 @@ func (r *Router) Handler() http.Handler {
|
||||
// Signup, sign-in and the profile page. These are browser surfaces
|
||||
// authenticated by a session cookie, not by an API key, which is why
|
||||
// they are exempt below.
|
||||
web.NewAccountPages(r.accounts, r.store, r.store, version.Version, r.log).Register(mux)
|
||||
pages := web.NewAccountPages(r.accounts, r.store, r.store, version.Version, r.log)
|
||||
pages.Register(mux)
|
||||
|
||||
if r.clusterView != nil {
|
||||
// The cluster dashboard, reachable from a browser without exposing
|
||||
// the admin listener — which also carries pprof, and pprof can dump
|
||||
// the heap that holds every API key in plaintext.
|
||||
mux.HandleFunc("GET /admin", pages.AdminPage(web.Dashboard(
|
||||
r.clusterView, r.clusterUsage, version.Version, 5*time.Second, r.log)))
|
||||
}
|
||||
}
|
||||
|
||||
if r.auth == nil {
|
||||
@@ -117,7 +126,7 @@ func (r *Router) Handler() http.Handler {
|
||||
// stays open because a load balancer has no key.
|
||||
return auth.Middleware(r.auth, r.log,
|
||||
"/health", "/", "/_static/", "/claim/",
|
||||
"/signup", "/login", "/logout", "/profile", "/profile/")(mux)
|
||||
"/signup", "/login", "/logout", "/profile", "/profile/", "/admin")(mux)
|
||||
}
|
||||
|
||||
// parsedRequest is what the router needs from a request. Everything else in the
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/scheduler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/scrape"
|
||||
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
|
||||
)
|
||||
|
||||
// Router serves the OpenAI API in front of a set of engine replicas.
|
||||
@@ -57,6 +58,20 @@ type Router struct {
|
||||
store *storepkg.SQLite
|
||||
acct *accounting.Accountant
|
||||
recorder *accounting.Recorder
|
||||
|
||||
// clusterView is nil until the control plane hands it over, and stays nil
|
||||
// in a router-only deployment.
|
||||
clusterView web.Source
|
||||
clusterUsage web.UsageSource
|
||||
}
|
||||
|
||||
// SetClusterView gives the router the data the /admin page renders.
|
||||
//
|
||||
// Set by the server command once the control plane exists, because the router
|
||||
// is built first. Nil leaves /admin unmounted, which is what a router-only
|
||||
// deployment should have: no cluster to show.
|
||||
func (r *Router) SetClusterView(src web.Source, usage web.UsageSource) {
|
||||
r.clusterView, r.clusterUsage = src, usage
|
||||
}
|
||||
|
||||
// Auth exposes the authenticator so ctl and the batch writer can reach it.
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
-- Who may see the cluster.
|
||||
--
|
||||
-- Until now the answer was "whoever has a shell on the control plane", because
|
||||
-- the dashboard lived on a loopback-bound port. Serving it over the web needs
|
||||
-- an actual answer, and this is the smallest one that is not a lie: a flag on
|
||||
-- an account that already has a password, a session and a CSRF token.
|
||||
--
|
||||
-- Deliberately not a role system. One bit is enough for "can see the cluster"
|
||||
-- and impossible to get subtly wrong; roles arrive when there is a second
|
||||
-- distinction worth drawing.
|
||||
ALTER TABLE users ADD COLUMN is_admin INTEGER NOT NULL DEFAULT 0;
|
||||
@@ -53,7 +53,7 @@ func (s *SQLite) SessionByToken(ctx context.Context, token string) (Session, Use
|
||||
row := s.db.QueryRowContext(ctx,
|
||||
`SELECT s.user_id, s.csrf_hash, s.expires_at,
|
||||
u.id, u.email, u.name, u.password_hash, u.state, COALESCE(u.project_id, ''),
|
||||
u.created_at, u.approved_at, u.approved_by, u.last_login_at
|
||||
u.created_at, u.approved_at, u.approved_by, u.last_login_at, u.is_admin
|
||||
FROM sessions s JOIN users u ON u.id = s.user_id
|
||||
WHERE s.token_hash = ?`, HashKey(token))
|
||||
|
||||
@@ -67,7 +67,7 @@ func (s *SQLite) SessionByToken(ctx context.Context, token string) (Session, Use
|
||||
|
||||
if err := row.Scan(&sess.UserID, &sess.CSRFHash, &expires,
|
||||
&u.ID, &u.Email, &u.Name, &u.PasswordHash, &state, &u.ProjectID,
|
||||
&created, &approved, &approvedBy, &lastLogin); err != nil {
|
||||
&created, &approved, &approvedBy, &lastLogin, &u.IsAdmin); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Session{}, User{}, ErrNoSession
|
||||
}
|
||||
|
||||
+22
-2
@@ -40,6 +40,9 @@ type User struct {
|
||||
ApprovedAt time.Time
|
||||
ApprovedBy string
|
||||
LastLoginAt time.Time
|
||||
// IsAdmin lets this account see the cluster dashboard. It is not a role
|
||||
// system: one bit, and everything that changes the cluster stays on the CLI.
|
||||
IsAdmin bool
|
||||
}
|
||||
|
||||
// CanSignIn reports whether this account may open a session.
|
||||
@@ -53,7 +56,7 @@ func NormalizeEmail(email string) string {
|
||||
}
|
||||
|
||||
const userColumns = `id, email, name, password_hash, state, COALESCE(project_id, ''),
|
||||
created_at, approved_at, approved_by, last_login_at`
|
||||
created_at, approved_at, approved_by, last_login_at, is_admin`
|
||||
|
||||
// CreateUser records a new, unapproved account.
|
||||
func (s *SQLite) CreateUser(ctx context.Context, u User) error {
|
||||
@@ -128,6 +131,23 @@ func (s *SQLite) SetUserState(ctx context.Context, id string, state UserState) e
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetAdmin grants or withdraws access to the cluster dashboard.
|
||||
//
|
||||
// Withdrawing does not close their sessions: the page is read-only, so the
|
||||
// worst a stale session buys is a few more seconds of looking. Suspending the
|
||||
// account is what closes sessions, and that is the lever for someone who should
|
||||
// not be here at all.
|
||||
func (s *SQLite) SetAdmin(ctx context.Context, id string, admin bool) error {
|
||||
res, err := s.db.ExecContext(ctx, `UPDATE users SET is_admin = ? WHERE id = ?`, admin, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n, _ := res.RowsAffected(); n == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TouchLogin records a successful sign-in.
|
||||
func (s *SQLite) TouchLogin(ctx context.Context, id string, at time.Time) error {
|
||||
_, err := s.db.ExecContext(ctx, `UPDATE users SET last_login_at = ? WHERE id = ?`,
|
||||
@@ -184,7 +204,7 @@ func (s *SQLite) scanUser(sc scanner) (User, error) {
|
||||
var approvedBy sql.NullString
|
||||
|
||||
if err := sc.Scan(&u.ID, &u.Email, &u.Name, &u.PasswordHash, &state, &u.ProjectID,
|
||||
&created, &approved, &approvedBy, &lastLogin); err != nil {
|
||||
&created, &approved, &approvedBy, &lastLogin, &u.IsAdmin); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return User{}, ErrNotFound
|
||||
}
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// AdminPage wraps the dashboard in a session check for the public port.
|
||||
//
|
||||
// A visitor who is not an admin gets 404 rather than 403. 403 would confirm the
|
||||
// page exists and that admin accounts are a thing here, which is free
|
||||
// reconnaissance for anyone probing; 404 is what every other unknown path
|
||||
// returns.
|
||||
func (p *AccountPages) AdminPage(dashboard http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
noStore(w)
|
||||
_, user, ok := p.session(r)
|
||||
if !ok {
|
||||
// Not signed in at all: send them to sign in, which is useful and
|
||||
// reveals nothing — /login is public anyway.
|
||||
http.Redirect(w, r, "/login", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
if !user.IsAdmin {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
p.log.Info("admin viewed the cluster", "user", user.ID, "email", user.Email)
|
||||
dashboard(w, r)
|
||||
}
|
||||
}
|
||||
@@ -54,6 +54,7 @@ Tell whoever you asked for access that you have signed up.</p>
|
||||
<p class="sub">{{.User.Name}} · {{.User.Email}} · project {{.User.ProjectID}}</p>
|
||||
<nav>
|
||||
<a href="/">about this gateway</a>
|
||||
{{if .User.IsAdmin}}<a href="/admin">cluster</a>{{end}}
|
||||
<form method="post" action="/logout" class="inline">
|
||||
<input type="hidden" name="csrf" value="{{.CSRF}}">
|
||||
<button type="submit" class="link">Sign out</button>
|
||||
|
||||
+3
-1
@@ -1447,7 +1447,9 @@ The launch path is now built too:
|
||||
- [ ] External provider support (as a fallback)
|
||||
- [x] Read-only dashboard on the admin listener (nodes, replicas, models,
|
||||
placement failures, usage) and a public landing page
|
||||
- [ ] An admin UI that can change things, which needs roles this deliberately
|
||||
- [x] The dashboard reachable over the web at `/admin`, behind a session and an
|
||||
admin flag, without exposing the listener that carries pprof
|
||||
- [ ] An admin UI that can *change* things, which needs roles this deliberately
|
||||
does not have
|
||||
|
||||
---
|
||||
|
||||
@@ -5,12 +5,14 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
@@ -395,3 +397,74 @@ func TestAWrongCurrentPasswordChangesNothing(t *testing.T) {
|
||||
t.Errorf("a failed change signed the person out: %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheClusterPageIsOnlyForAdmins(t *testing.T) {
|
||||
// Serving the dashboard over the web needs an actual answer to "who may see
|
||||
// this", and the answer must not be "anyone with an account".
|
||||
path, _ := provision(t)
|
||||
h := start(t, 2, func(c *config.Server) {
|
||||
c.Auth = config.Auth{Enabled: true, Store: path, SignupOpen: true, CacheTTL: 50 * time.Millisecond}
|
||||
})
|
||||
h.rt.SetClusterView(fakeCluster{}, nil)
|
||||
// Rebuild the handler now that there is a cluster to show.
|
||||
srv := httptest.NewServer(h.rt.Handler())
|
||||
defer srv.Close()
|
||||
|
||||
// A stranger is sent to sign in, which reveals nothing: /login is public.
|
||||
anon := newBrowser(t, srv.URL)
|
||||
if code, _ := anon.get("/admin"); code != http.StatusSeeOther {
|
||||
t.Errorf("an anonymous visit returned %d", code)
|
||||
}
|
||||
|
||||
// An ordinary account gets 404, not 403. 403 would confirm the page exists
|
||||
// and that admin accounts are a thing here.
|
||||
user := newBrowser(t, srv.URL)
|
||||
id := signUpAndApproveAt(t, user, srv.URL, path, "ayse@example.com")
|
||||
code, body := user.get("/admin")
|
||||
if code != http.StatusNotFound {
|
||||
t.Errorf("a non-admin got %d, want 404", code)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(body), "forbidden") ||
|
||||
strings.Contains(strings.ToLower(body), "admin") {
|
||||
t.Error("the refusal admits the page exists")
|
||||
}
|
||||
// And the profile page does not advertise a door they cannot open.
|
||||
if _, profile := user.get("/profile"); strings.Contains(profile, `href="/admin"`) {
|
||||
t.Error("a non-admin is shown a link to the cluster")
|
||||
}
|
||||
|
||||
withStore(t, path, func(s *store.SQLite) {
|
||||
if err := s.SetAdmin(context.Background(), id, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
|
||||
code, body = user.get("/admin")
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("an admin got %d", code)
|
||||
}
|
||||
if !strings.Contains(body, "llmgw-node01") {
|
||||
t.Error("the page does not show the cluster")
|
||||
}
|
||||
if _, profile := user.get("/profile"); !strings.Contains(profile, `href="/admin"`) {
|
||||
t.Error("an admin is not shown the link")
|
||||
}
|
||||
}
|
||||
|
||||
// signUpAndApproveAt is signUpAndApprove against an explicitly given base URL,
|
||||
// for tests that rebuild the handler.
|
||||
func signUpAndApproveAt(t *testing.T, b *browser, base, path, email string) string {
|
||||
t.Helper()
|
||||
_ = base
|
||||
return signUpAndApprove(t, b, path, email)
|
||||
}
|
||||
|
||||
// fakeCluster is enough of a cluster for the page to render.
|
||||
type fakeCluster struct{}
|
||||
|
||||
func (fakeCluster) Nodes() []adminapi.NodeView {
|
||||
return []adminapi.NodeView{{ID: "llmgw-node01", State: "ready", GPUTotal: 8, GPUFree: 8}}
|
||||
}
|
||||
func (fakeCluster) Models() []adminapi.ModelView { return nil }
|
||||
func (fakeCluster) Replicas(string) []adminapi.ReplicaView { return nil }
|
||||
func (fakeCluster) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} }
|
||||
Reference in new issue
Block a user