diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go index 38fdd64..6d35f9f 100644 --- a/internal/auth/auth_test.go +++ b/internal/auth/auth_test.go @@ -380,3 +380,27 @@ func TestLastUsedIsBatchedNotWrittenPerRequest(t *testing.T) { t.Errorf("wrong key touched: %s", k.ID) } } + +func TestExemptingTheLandingPageDoesNotExemptEverything(t *testing.T) { + // A near-miss worth a test. "/" ends in a slash, so a prefix rule would + // match every path there is: authentication still installed, still logging, + // and letting the entire API through. + cases := []struct { + path string + want bool + }{ + {"/", true}, + {"/_static/base.css", true}, + {"/health", true}, + {"/v1/models", false}, + {"/v1/chat/completions", false}, + {"/healthz", false}, // not a prefix rule + {"/_staticx", false}, // prefix rules end at the slash + } + exempt := []string{"/health", "/", "/_static/"} + for _, c := range cases { + if got := isExempt(c.path, exempt); got != c.want { + t.Errorf("isExempt(%q) = %v, want %v", c.path, got, c.want) + } + } +} diff --git a/internal/auth/middleware.go b/internal/auth/middleware.go index fe050c1..3b74882 100644 --- a/internal/auth/middleware.go +++ b/internal/auth/middleware.go @@ -40,12 +40,21 @@ func Middleware(a *Authenticator, log *slog.Logger, exempt ...string) func(http. // isExempt reports whether a path is served without a key. // -// Health is the only one by default. Metrics live on the admin listener, which -// is a separate port precisely so it can be firewalled rather than +// An entry ending in "/" is a prefix rule, with one exception that matters +// enormously: "/" itself is exact. Treating it as a prefix would match every +// path there is, so exempting the landing page would silently exempt the whole +// API — authentication still installed, still logging, and letting everything +// through. +// +// Metrics and pprof are not in this list because they live on the admin +// listener, a separate port precisely so it can be firewalled rather than // authenticated. func isExempt(path string, exempt []string) bool { for _, e := range exempt { - if path == e || (strings.HasSuffix(e, "/") && strings.HasPrefix(path, e)) { + if path == e { + return true + } + if e != "/" && strings.HasSuffix(e, "/") && strings.HasPrefix(path, e) { return true } } diff --git a/internal/cli/server.go b/internal/cli/server.go index 1764159..2bd7f6c 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -26,6 +26,8 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" "gitea.mynodes.xyz/ilker/llmgw/internal/router" + "gitea.mynodes.xyz/ilker/llmgw/internal/version" + "gitea.mynodes.xyz/ilker/llmgw/internal/web" ) func newServerCmd() *cobra.Command { @@ -112,7 +114,21 @@ func newServerCmd() *cobra.Command { return err } if admin != nil { - adminapi.New(admin).Register(adminMux) + api := adminapi.New(admin) + api.Register(adminMux) + + // The dashboard lives here and only here. This listener is + // bound to loopback by default and also serves pprof, which can + // dump the process heap — where API keys sit in plaintext. + // Reaching the page from elsewhere is an SSH tunnel's job: + // ssh -L 9090:127.0.0.1:9090 + var usage web.UsageSource + if st := rt.Store(); st != nil { + usage = st + } + adminMux.HandleFunc("GET /", web.Dashboard( + api, usage, version.Version, 5*time.Second, log)) + adminMux.Handle("GET /_static/", web.Static()) } apiSrv := &http.Server{ diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 1ed5e77..88766e8 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -117,6 +117,15 @@ type RejectionView struct { // --- handlers --- func (a *API) listNodes(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, a.Nodes()) +} + +// Nodes builds the node view. +// +// Exported because the dashboard renders the same data server-side. Going back +// out through HTTP to fetch what is already in this process would add a way for +// the two to disagree. +func (a *API) Nodes() []NodeView { replicasOn := map[string]int{} for _, rep := range a.ctl.Reconciler().Replicas() { for _, n := range rep.NodeIDs() { @@ -150,10 +159,15 @@ func (a *API) listNodes(w http.ResponseWriter, _ *http.Request) { } out = append(out, v) } - writeJSON(w, http.StatusOK, out) + return out } func (a *API) listModels(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, a.Models()) +} + +// Models builds the catalog view. +func (a *API) Models() []ModelView { ready, total := map[string]int{}, map[string]int{} for _, rep := range a.ctl.Reconciler().Replicas() { total[rep.Model]++ @@ -175,12 +189,15 @@ func (a *API) listModels(w http.ResponseWriter, _ *http.Request) { } out = append(out, v) } - writeJSON(w, http.StatusOK, out) + return out } func (a *API) listReplicas(w http.ResponseWriter, r *http.Request) { - model := r.URL.Query().Get("model") + writeJSON(w, http.StatusOK, a.Replicas(r.URL.Query().Get("model"))) +} +// Replicas builds the replica view, optionally filtered to one model. +func (a *API) Replicas(model string) []ReplicaView { out := make([]ReplicaView, 0) for _, rep := range a.ctl.Reconciler().Replicas() { if model != "" && rep.Model != model { @@ -200,7 +217,7 @@ func (a *API) listReplicas(w http.ResponseWriter, r *http.Request) { } out = append(out, v) } - writeJSON(w, http.StatusOK, out) + return out } func (a *API) explainPlacement(w http.ResponseWriter, r *http.Request) { @@ -214,6 +231,15 @@ func (a *API) explainPlacement(w http.ResponseWriter, r *http.Request) { return } + writeJSON(w, http.StatusOK, a.Explain(model)) +} + +// Explain builds the placement explanation for one model. +// +// It does not check that the model exists; the handler above does that so it +// can answer 404, while the dashboard only ever asks about models it just read +// out of the catalog. +func (a *API) Explain(model string) ExplainView { v := ExplainView{Model: model} for _, rep := range a.ctl.Reconciler().Replicas() { if rep.Model == model && rep.State.Counts() { @@ -228,7 +254,7 @@ func (a *API) explainPlacement(w http.ResponseWriter, r *http.Request) { }) } } - writeJSON(w, http.StatusOK, v) + return v } // setNodeState implements cordon, uncordon and drain. diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 07784d8..f74e0d3 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -17,6 +17,8 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/router/admission" "gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy" "gitea.mynodes.xyz/ilker/llmgw/internal/store" + "gitea.mynodes.xyz/ilker/llmgw/internal/version" + "gitea.mynodes.xyz/ilker/llmgw/internal/web" ) // admit waits for the flow control layer to let the request through. With @@ -82,13 +84,20 @@ func (r *Router) Handler() http.Handler { mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) + + // The landing page says how to connect and nothing about what is here, so + // it is safe to serve without a key — and it has to be, because the person + // reading it is the one who does not have one yet. + mux.HandleFunc("GET /", web.Landing(version.Version, r.auth != nil)) + mux.Handle("GET /_static/", web.Static()) + if r.auth == nil { return mux } // Wrapping the whole mux rather than checking inside each handler: adding a // route must not be able to add an unauthenticated one by omission. Health // stays open because a load balancer has no key. - return auth.Middleware(r.auth, r.log, "/health")(mux) + return auth.Middleware(r.auth, r.log, "/health", "/", "/_static/")(mux) } // parsedRequest is what the router needs from a request. Everything else in the diff --git a/internal/store/sqlite.go b/internal/store/sqlite.go index d3a84e9..5e8a91a 100644 --- a/internal/store/sqlite.go +++ b/internal/store/sqlite.go @@ -504,3 +504,31 @@ func nowMillis() int64 { return time.Now().UnixMilli() } // SQLite satisfies Store. var _ Store = (*SQLite)(nil) + +// UsageSince summarizes activity per key. +// +// The join brings in the key's name and prefix, which is what a dashboard needs +// to show: a row identified only by an opaque id tells an operator nothing. +// Revoked keys still appear, because their usage still happened. +func (s *SQLite) UsageSince(ctx context.Context, since time.Time) ([]KeyUsage, error) { + return queryAll(ctx, s.db, ` + SELECT u.key_id, + COALESCE(k.name, ''), COALESCE(k.key_prefix, ''), u.project_id, + COUNT(*), SUM(u.prompt_tokens), SUM(u.completion_tokens), + SUM(u.cost_micros), + SUM(CASE WHEN u.status >= 400 THEN 1 ELSE 0 END), + MAX(u.ts) + FROM usage u + LEFT JOIN api_keys k ON k.id = u.key_id + WHERE u.ts >= ? + GROUP BY u.key_id, u.project_id + ORDER BY SUM(u.cost_micros) DESC, COUNT(*) DESC`, + []any{millis(since)}, func(rows *sql.Rows) (KeyUsage, error) { + var k KeyUsage + var last int64 + err := rows.Scan(&k.KeyID, &k.KeyName, &k.KeyPrefix, &k.ProjectID, + &k.Requests, &k.Prompt, &k.Completion, &k.CostMicros, &k.Errors, &last) + k.LastAt = fromMillis(last) + return k, err + }) +} diff --git a/internal/store/store.go b/internal/store/store.go index 673e3c4..0943491 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -193,6 +193,24 @@ type Store interface { // counter can be rebuilt after a restart. A restart must not reset a // budget, which it would if the counter lived only in memory. SpendSince(ctx context.Context, since time.Time) (map[string]int64, error) + // UsageSince summarizes activity per key. A reporting query: it scans and + // groups, so it belongs behind a dashboard refresh rather than a request. + UsageSince(ctx context.Context, since time.Time) ([]KeyUsage, error) Close() error } + +// KeyUsage summarizes one key's activity in a period, for the dashboard and +// `ctl`. It is a reporting query and never runs on the request path. +type KeyUsage struct { + KeyID string + KeyName string + KeyPrefix string + ProjectID string + Requests int64 + Prompt int64 + Completion int64 + CostMicros int64 + Errors int64 + LastAt time.Time +} diff --git a/internal/web/dashboard.go b/internal/web/dashboard.go new file mode 100644 index 0000000..8324daf --- /dev/null +++ b/internal/web/dashboard.go @@ -0,0 +1,116 @@ +package web + +import ( + "context" + "log/slog" + "net/http" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// DashboardData is the whole page. +type DashboardData struct { + Nodes []adminapi.NodeView + Models []adminapi.ModelView + Replicas []adminapi.ReplicaView + Unplaced []adminapi.ExplainView + Usage []store.KeyUsage + + NodeCount int + ModelCount int + ReadyReplicas int + TotalReplicas int + + AuthEnabled bool + RefreshSec int + Version string + Now string +} + +// Source is what the dashboard reads. An interface so the page can be rendered +// in a test without a control plane. +type Source interface { + Nodes() []adminapi.NodeView + Models() []adminapi.ModelView + Replicas(model string) []adminapi.ReplicaView + Explain(model string) adminapi.ExplainView +} + +// UsageSource is the accounting half, absent when authentication is off. +type UsageSource interface { + UsageSince(ctx context.Context, since time.Time) ([]store.KeyUsage, error) +} + +// Dashboard serves the read-only cluster view. +// +// It belongs on the admin listener and nowhere else. That listener is bound to +// loopback by default and also serves pprof, which can dump the process heap — +// where API keys live in plaintext. Exposing this page publicly would mean +// exposing that, so reaching it from elsewhere is an SSH tunnel's job: +// +// ssh -L 9090:127.0.0.1:9090 llmgw-ctl +// +// Read-only on purpose. Anything that mints a credential stays in `llmgw keys`, +// where it needs a shell on the machine rather than a browser session this +// project has no way to authenticate. +func Dashboard(src Source, usage UsageSource, version string, refresh time.Duration, + log *slog.Logger, +) http.HandlerFunc { + if log == nil { + log = slog.Default() + } + if refresh <= 0 { + refresh = 5 * time.Second + } + return func(w http.ResponseWriter, r *http.Request) { + d := DashboardData{ + RefreshSec: int(refresh.Seconds()), + Version: version, + Now: time.Now().UTC().Format(time.RFC3339), + AuthEnabled: usage != nil, + } + if src != nil { + d.Nodes = src.Nodes() + d.Models = src.Models() + d.Replicas = src.Replicas("") + d.NodeCount = len(d.Nodes) + d.ModelCount = len(d.Models) + d.TotalReplicas = len(d.Replicas) + for _, rep := range d.Replicas { + if rep.State == "ready" { + d.ReadyReplicas++ + } + } + d.Unplaced = unplaced(src, d.Models) + } + if usage != nil { + rows, err := usage.UsageSince(r.Context(), time.Now().Add(-24*time.Hour)) + if err != nil { + // A reporting query failing must not take the page with it: the + // cluster state above is what someone is most likely here for. + log.Warn("could not read usage for the dashboard", "err", err) + } + d.Usage = rows + } + render(w, "dashboard", d) + } +} + +// unplaced collects the placement explanations for models short of replicas. +// +// Only for those: asking why a satisfied model was not placed produces an empty +// answer, and a page full of empty answers hides the one that matters. +func unplaced(src Source, models []adminapi.ModelView) []adminapi.ExplainView { + var out []adminapi.ExplainView + for _, m := range models { + if m.Ready >= m.Min { + continue + } + if e := src.Explain(m.Name); len(e.Rejections) > 0 { + out = append(out, e) + } + } + return out +} diff --git a/internal/web/landing.go b/internal/web/landing.go new file mode 100644 index 0000000..216692e --- /dev/null +++ b/internal/web/landing.go @@ -0,0 +1,77 @@ +package web + +import ( + "net/http" + "strings" + "time" +) + +// LandingData is what the landing page shows. It is deliberately almost +// nothing: this page is served without a key, to anyone who can reach the +// address. +type LandingData struct { + BaseURL string + AuthEnabled bool + Version string + Now string +} + +// Landing serves the page at "/" on the client-facing port. +// +// Without it that path is a 404, which is what someone sees when they paste the +// address into a browser — an endpoint that looks broken rather than one that +// tells them how to use it. +// +// It lists no models and no cluster state. Everything that says what is here +// stays behind the key, including /v1/models. +func Landing(version string, authEnabled bool) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // Only the root: a catch-all would turn every mistyped path into this + // page, hiding genuine 404s from anyone debugging a client. + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + render(w, "landing", LandingData{ + BaseURL: baseURL(r), + AuthEnabled: authEnabled, + Version: version, + Now: time.Now().UTC().Format(time.RFC3339), + }) + } +} + +// baseURL reconstructs the address the client actually used. +// +// Behind a reverse proxy the scheme and host this process sees are the internal +// ones, so the forwarded headers are what make the copy-pasteable example on +// the page correct. They are attacker-controlled when nothing sets them, which +// is why they are only reflected into a displayed URL and never used for a +// decision. +func baseURL(r *http.Request) string { + scheme := "http" + switch { + case r.TLS != nil: + scheme = "https" + case r.Header.Get("X-Forwarded-Proto") != "": + scheme = firstValue(r.Header.Get("X-Forwarded-Proto")) + } + + host := r.Host + if fwd := r.Header.Get("X-Forwarded-Host"); fwd != "" { + host = firstValue(fwd) + } + if host == "" { + host = "localhost:8080" + } + return scheme + "://" + host +} + +// firstValue takes the first entry of a comma-separated forwarded header: a +// chain of proxies appends rather than replaces. +func firstValue(v string) string { + if i := strings.IndexByte(v, ','); i >= 0 { + v = v[:i] + } + return strings.TrimSpace(v) +} diff --git a/internal/web/templates/base.css b/internal/web/templates/base.css new file mode 100644 index 0000000..570132b --- /dev/null +++ b/internal/web/templates/base.css @@ -0,0 +1,51 @@ +/* Deliberately one small stylesheet with no build step. The single-binary goal + rules out a JS toolchain, and nothing here needs one. */ +:root { + color-scheme: light dark; + --bg: #fbfbfa; + --fg: #1c1c1a; + --muted: #6b6b66; + --line: #e3e3df; + --card: #ffffff; + --accent: #2f5d50; + --ok: #2f7a4f; + --warn: #9a6b1f; + --bad: #a33a32; +} +@media (prefers-color-scheme: dark) { + :root { + --bg: #17181a; --fg: #e8e8e4; --muted: #9a9a93; --line: #2c2e31; + --card: #1e2022; --accent: #7fb3a2; --ok: #6fbd8c; --warn: #d6a44e; --bad: #e07b72; + } +} +* { box-sizing: border-box; } +body { + margin: 0; background: var(--bg); color: var(--fg); + font: 15px/1.55 ui-sans-serif, -apple-system, "Segoe UI", Roboto, sans-serif; +} +.wrap { max-width: 1100px; margin: 0 auto; padding: 2.5rem 1.25rem 4rem; } +header { border-bottom: 1px solid var(--line); margin-bottom: 2rem; padding-bottom: 1.25rem; } +h1 { font-size: 1.5rem; margin: 0 0 .25rem; letter-spacing: -.01em; } +h2 { font-size: 1.05rem; margin: 2.25rem 0 .75rem; letter-spacing: -.005em; } +.sub { color: var(--muted); margin: 0; } +nav { margin-top: .75rem; display: flex; gap: 1rem; flex-wrap: wrap; } +nav a { color: var(--accent); text-decoration: none; font-size: .9rem; } +nav a:hover { text-decoration: underline; } +table { width: 100%; border-collapse: collapse; font-size: .875rem; background: var(--card); + border: 1px solid var(--line); border-radius: 6px; overflow: hidden; } +th, td { text-align: left; padding: .5rem .7rem; border-bottom: 1px solid var(--line); } +th { font-weight: 600; color: var(--muted); font-size: .78rem; text-transform: uppercase; + letter-spacing: .04em; } +tr:last-child td { border-bottom: none; } +td.num { text-align: right; font-variant-numeric: tabular-nums; } +code, pre { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: .85em; } +pre { background: var(--card); border: 1px solid var(--line); border-radius: 6px; + padding: .9rem 1rem; overflow-x: auto; } +.state { font-weight: 600; font-size: .8rem; } +.state-ready, .state-active { color: var(--ok); } +.state-pending, .state-starting, .state-draining, .state-cordoned { color: var(--warn); } +.state-failed, .state-lost, .state-unreachable, .state-revoked { color: var(--bad); } +.empty { color: var(--muted); font-style: italic; padding: .9rem 0; } +.note { color: var(--muted); font-size: .85rem; margin: .4rem 0 0; } +footer { margin-top: 3rem; padding-top: 1rem; border-top: 1px solid var(--line); + color: var(--muted); font-size: .8rem; display: flex; justify-content: space-between; gap: 1rem; } diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html new file mode 100644 index 0000000..2dd250d --- /dev/null +++ b/internal/web/templates/dashboard.html @@ -0,0 +1,95 @@ +{{define "dashboard"}} + + + +llmgw — cluster +
+
+

Cluster

+

{{.NodeCount}} nodes · {{.ReadyReplicas}} of {{.TotalReplicas}} replicas ready · {{.ModelCount}} models

+ +
+ +

Nodes

+{{if .Nodes}} + +{{range .Nodes}} + + + + + + + + +{{if .Unhealthy}}{{end}} +{{end}}
NodeStateGPUsClassesFabricsReplicasLast seenAgent
{{.ID}}{{if .MgmtAddr}} {{.MgmtAddr}}{{end}}{{.State}}{{.GPUFree}}/{{.GPUTotal}}{{join .Classes}}{{join .Fabrics}}{{.Replicas}}{{or .LastSeenAgo "—"}}{{or .AgentVersion "—"}}
{{join .Unhealthy}}
+{{else}}

No agents have registered.

{{end}} + +

Models

+{{if .Models}} + +{{range .Models}} + + + + + + +{{end}}
ModelAliasesReadyWantedProfilesFallbacks
{{.Name}}{{join .Aliases}}{{.Ready}}{{.Min}}–{{.Max}}{{join .Profiles}}{{join .Fallbacks}}
+{{else}}

The catalog is empty.

{{end}} + +{{if .Unplaced}} +

Why nothing was placed

+{{range .Unplaced}} +

{{.Model}}

+
{{range .Rejections}}{{.Profile}} @ {{or .Pool "-"}}{{if .Node}}/{{.Node}}{{end}}: {{.Reason}}
+{{end}}
+{{end}} +{{end}} + +

Replicas

+{{if .Replicas}} + +{{range .Replicas}} + + + + + + + +{{if .Err}}{{end}} +{{end}}
ReplicaModelStatePoolEndpointAgeNodes
{{.ID}}{{.Model}}{{.State}}{{or .Pool "—"}}{{if .Endpoint}}{{.Endpoint}}{{else}}—{{end}}{{duration .AgeSec}}{{range .Shards}}{{.Node}} {{end}}
{{.Err}}
+{{else}}

No replicas.

{{end}} + +

Usage — last 24 hours

+{{if .AuthEnabled}} + {{if .Usage}} + +{{range .Usage}} + + + + + + + + +{{end}}
KeyProjectRequestsErrorsPromptCompletionSpendLast
{{if .KeyName}}{{.KeyName}} {{.KeyPrefix}}{{else}}{{.KeyID}}{{end}}{{.ProjectID}}{{.Requests}}{{if .Errors}}{{.Errors}}{{else}}0{{end}}{{.Prompt}}{{.Completion}}{{units .CostMicros}}{{ago .LastAt}}
+

Spend is in units, whatever a unit was configured to mean. Usage is +written in batches, so the last few seconds may be missing.

+ {{else}}

No requests recorded in the last 24 hours.

{{end}} +{{else}} +

Authentication is off, so there is no identity to account against.

+{{end}} + +
+ llmgw {{.Version}} · refreshes every {{.RefreshSec}}s + {{.Now}} +
+
{{end}} diff --git a/internal/web/templates/landing.html b/internal/web/templates/landing.html new file mode 100644 index 0000000..7f316e3 --- /dev/null +++ b/internal/web/templates/landing.html @@ -0,0 +1,42 @@ +{{define "landing"}} + + +llmgw +
+
+

llmgw

+

An OpenAI-compatible endpoint in front of self-hosted models.

+
+ +

Point any OpenAI client at this address. The API is the one you already have a +client for; only the base URL changes.

+ +

Base URL

+
{{.BaseURL}}/v1
+ +

Example

+
curl {{.BaseURL}}/v1/chat/completions \
+  -H "Authorization: Bearer $LLMGW_API_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{"model": "MODEL", "messages": [{"role":"user","content":"hello"}]}'
+ +
from openai import OpenAI
+client = OpenAI(base_url="{{.BaseURL}}/v1", api_key=os.environ["LLMGW_API_KEY"])
+ +

Which models

+{{if .AuthEnabled}} +

GET /v1/models lists what is available to your key. It needs the key, +which is why this page does not list them: this page is served to anyone who can +reach the address.

+{{else}} +

GET /v1/models lists what is available.

+

This gateway has no API keys configured, so it serves anyone who can +reach it. That is a deployment choice and only safe on a trusted network.

+{{end}} + +

Keys

+

Keys are issued by an operator with llmgw keys create, and shown once. +There is no way to recover one: only its hash is stored.

+ + +
{{end}} diff --git a/internal/web/web.go b/internal/web/web.go new file mode 100644 index 0000000..e9512a1 --- /dev/null +++ b/internal/web/web.go @@ -0,0 +1,82 @@ +// Package web serves the two HTML surfaces: a landing page on the +// client-facing port and a read-only dashboard on the admin one. +// +// Server-rendered templates with one stylesheet, embedded in the binary. The +// single-binary goal rules out a JavaScript toolchain — a build step in CI, a +// node runtime on the runner, and megabytes of bundle — and nothing on either +// page needs one. The dashboard refreshes by reloading. +// +// The split between the two pages is a security boundary, not a layout choice. +// The landing page is served to anyone who can reach the address, so it +// contains no data at all: it says how to connect, not what is here. The +// dashboard shows the cluster and belongs on the admin listener, which is bound +// to loopback — see Dashboard. +package web + +import ( + "embed" + "fmt" + "html/template" + "net/http" + "strings" + "time" +) + +//go:embed templates +var files embed.FS + +var funcs = template.FuncMap{ + "join": func(v []string) string { + if len(v) == 0 { + return "—" + } + return strings.Join(v, ", ") + }, + "duration": func(sec int64) string { + return time.Duration(sec * int64(time.Second)).Round(time.Second).String() + }, + "ago": func(t time.Time) string { + if t.IsZero() { + return "never" + } + return time.Since(t).Round(time.Second).String() + " ago" + }, + // Spend is stored in micro-units so accumulation is exact; it is shown in + // whole units because that is the number an operator set a budget in. + "units": func(micros int64) string { + return fmt.Sprintf("%.2f", float64(micros)/1e6) + }, +} + +var tmpl = template.Must(template.New("").Funcs(funcs).ParseFS(files, "templates/*.html")) + +// Static serves the stylesheet, mounted on both listeners because both pages +// use it and it contains nothing. +// +// One file rather than a directory server: exposing a file server over the +// embedded templates would also serve the templates themselves, which is +// harmless today and the kind of thing that stops being harmless later. +func Static() http.Handler { + css, err := files.ReadFile("templates/base.css") + if err != nil { + panic("web: the stylesheet is missing from the binary: " + err.Error()) + } + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/_static/base.css" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "text/css; charset=utf-8") + w.Header().Set("Cache-Control", "public, max-age=3600") + _, _ = w.Write(css) + }) +} + +func render(w http.ResponseWriter, name string, data any) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + if err := tmpl.ExecuteTemplate(w, name, data); err != nil { + // The status is already written by then, so there is nothing honest to + // send; the page simply ends short. + http.Error(w, "template error", http.StatusInternalServerError) + } +} diff --git a/internal/web/web_test.go b/internal/web/web_test.go new file mode 100644 index 0000000..959cbff --- /dev/null +++ b/internal/web/web_test.go @@ -0,0 +1,233 @@ +package web + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +func get(t *testing.T, h http.Handler, target string, headers map[string]string) (int, string) { + t.Helper() + r := httptest.NewRequest(http.MethodGet, target, nil) + for k, v := range headers { + r.Header.Set(k, v) + } + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + return w.Code, w.Body.String() +} + +func TestLandingPageTellsYouHowToConnect(t *testing.T) { + code, body := get(t, Landing("v1.2.3", true), "http://llmgw.example/", nil) + if code != http.StatusOK { + t.Fatalf("status %d", code) + } + for _, want := range []string{"http://llmgw.example/v1", "Authorization: Bearer", "v1.2.3"} { + if !strings.Contains(body, want) { + t.Errorf("the page does not mention %q", want) + } + } +} + +func TestLandingPageSaysNothingAboutWhatIsHere(t *testing.T) { + // It is served without a key, to anyone who can reach the address. Anything + // on it is public, so nothing about the cluster belongs on it. + _, body := get(t, Landing("v1", true), "http://llmgw.example/", nil) + for _, leak := range []string{"llama", "qwen", "10.0.0.", "replica", "node01"} { + if strings.Contains(strings.ToLower(body), leak) { + t.Errorf("the landing page leaks %q", leak) + } + } +} + +func TestLandingPageWarnsWhenTheGatewayIsOpen(t *testing.T) { + // Someone who reaches this page on an unauthenticated gateway should learn + // that from the page rather than by discovering they can use it. + _, open := get(t, Landing("v1", false), "http://llmgw.example/", nil) + if !strings.Contains(open, "no API keys configured") { + t.Error("an open gateway does not say so") + } + _, closed := get(t, Landing("v1", true), "http://llmgw.example/", nil) + if strings.Contains(closed, "no API keys configured") { + t.Error("a closed gateway claims to be open") + } +} + +func TestTheExampleURLSurvivesAReverseProxy(t *testing.T) { + // The process sees http and an internal host; the client used https and a + // public name. An example naming the internal address is one nobody can + // paste anywhere. + _, body := get(t, Landing("v1", true), "http://10.0.0.140:8080/", map[string]string{ + "X-Forwarded-Proto": "https", + "X-Forwarded-Host": "llmgw.mynodes.xyz", + }) + if !strings.Contains(body, "https://llmgw.mynodes.xyz/v1") { + t.Error("the example does not use the address the client actually used") + } + if strings.Contains(body, "10.0.0.140") { + t.Error("the internal address leaked into the page") + } +} + +func TestForwardedHeaderChainsTakeTheFirstEntry(t *testing.T) { + // A chain of proxies appends rather than replaces. + _, body := get(t, Landing("v1", true), "http://internal/", map[string]string{ + "X-Forwarded-Proto": "https, http", + "X-Forwarded-Host": "llmgw.example, internal", + }) + if !strings.Contains(body, "https://llmgw.example/v1") { + t.Errorf("the base URL was built from the wrong entry") + } +} + +func TestOnlyTheRootIsTheLandingPage(t *testing.T) { + // A catch-all would turn every mistyped path into this page, hiding real + // 404s from anyone debugging a client. + code, _ := get(t, Landing("v1", true), "http://x/v1/chat/completionz", nil) + if code != http.StatusNotFound { + t.Errorf("a mistyped path returned %d", code) + } +} + +// fakeSource is a cluster for the dashboard to render. +type fakeSource struct{} + +func (fakeSource) Nodes() []adminapi.NodeView { + return []adminapi.NodeView{{ + ID: "llmgw-node01", State: "ready", GPUTotal: 8, GPUFree: 4, + Classes: []string{"h100-80g"}, Fabrics: []string{"ib-a"}, + Replicas: 1, LastSeenAgo: "2s", AgentVersion: "v0.5.0", + Unhealthy: []string{"gpu 3: ECC errors"}, + }} +} + +func (fakeSource) Models() []adminapi.ModelView { + return []adminapi.ModelView{ + {Name: "served", Ready: 1, Min: 1, Max: 2, Profiles: []string{"tp4"}}, + {Name: "stuck", Ready: 0, Min: 1, Max: 1, Profiles: []string{"tp8"}}, + } +} + +func (fakeSource) Replicas(string) []adminapi.ReplicaView { + return []adminapi.ReplicaView{{ + ID: "served-1", Model: "served", State: "ready", Pool: "ib-a/h100-80g", + Endpoint: "http://10.0.0.141:20000", AgeSec: 3600, + Shards: []adminapi.ShardView{{ID: "served-1-r0", Node: "llmgw-node01"}}, + }} +} + +func (fakeSource) Explain(model string) adminapi.ExplainView { + if model != "stuck" { + return adminapi.ExplainView{Model: model, Placed: true} + } + return adminapi.ExplainView{Model: "stuck", Rejections: []adminapi.RejectionView{ + {Profile: "tp8", Node: "llmgw-node01", Reason: "has 4 free h100-80g GPUs, needs 8"}, + }} +} + +type fakeUsage struct{ err error } + +func (f fakeUsage) UsageSince(context.Context, time.Time) ([]store.KeyUsage, error) { + if f.err != nil { + return nil, f.err + } + return []store.KeyUsage{{ + KeyID: "key-1", KeyName: "laptop", KeyPrefix: "sk-llmgw-ab12…", + ProjectID: "chat", Requests: 42, Errors: 2, + Prompt: 1000, Completion: 3000, CostMicros: 4_500_000, + LastAt: time.Now().Add(-time.Minute), + }}, nil +} + +func TestDashboardShowsTheCluster(t *testing.T) { + h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, nil) + code, body := get(t, h, "http://127.0.0.1:9090/", nil) + if code != http.StatusOK { + t.Fatalf("status %d", code) + } + for _, want := range []string{ + "llmgw-node01", "h100-80g", "served-1", "http://10.0.0.141:20000", + "ECC errors", "laptop", "4.50", // spend shown in units, not micro-units + } { + if !strings.Contains(body, want) { + t.Errorf("the dashboard does not show %q", want) + } + } +} + +func TestDashboardExplainsOnlyWhatIsUnplaced(t *testing.T) { + // Asking why a satisfied model was not placed gives an empty answer, and a + // page full of empty answers hides the one that matters. + h := Dashboard(fakeSource{}, nil, "v1", time.Second, nil) + _, body := get(t, h, "http://127.0.0.1:9090/", nil) + + if !strings.Contains(body, "needs 8") { + t.Error("the unplaced model has no explanation") + } + if strings.Count(body, "Why nothing was placed") != 1 { + t.Error("the explanation section is missing or duplicated") + } +} + +func TestDashboardSurvivesAFailingUsageQuery(t *testing.T) { + // A reporting query failing must not take the page with it: the cluster + // state is what someone is most likely here for. + h := Dashboard(fakeSource{}, fakeUsage{err: context.DeadlineExceeded}, "v1", time.Second, nil) + code, body := get(t, h, "http://127.0.0.1:9090/", nil) + if code != http.StatusOK { + t.Fatalf("status %d", code) + } + if !strings.Contains(body, "llmgw-node01") { + t.Error("the cluster state was lost with the usage query") + } +} + +func TestDashboardSaysWhenThereIsNothingToAccount(t *testing.T) { + h := Dashboard(fakeSource{}, nil, "v1", time.Second, nil) + _, body := get(t, h, "http://127.0.0.1:9090/", nil) + if !strings.Contains(body, "Authentication is off") { + t.Error("the page does not explain the empty usage section") + } +} + +func TestDashboardRendersAnEmptyCluster(t *testing.T) { + // The state a fresh install is in, and the one most likely to be looked at + // with a template that has never seen it. + h := Dashboard(emptySource{}, nil, "v1", time.Second, nil) + code, body := get(t, h, "http://127.0.0.1:9090/", nil) + if code != http.StatusOK { + t.Fatalf("status %d", code) + } + if !strings.Contains(body, "No agents have registered") { + t.Error("an empty cluster renders nothing useful") + } +} + +type emptySource struct{} + +func (emptySource) Nodes() []adminapi.NodeView { return nil } +func (emptySource) Models() []adminapi.ModelView { return nil } +func (emptySource) Replicas(string) []adminapi.ReplicaView { return nil } +func (emptySource) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} } + +func TestStaticIsNotAFileServer(t *testing.T) { + // A file server over the embedded templates would also serve the templates, + // which is harmless today and the kind of thing that stops being harmless. + h := Static() + if code, _ := get(t, h, "http://x/_static/base.css", nil); code != http.StatusOK { + t.Errorf("the stylesheet is not served: %d", code) + } + for _, path := range []string{ + "/_static/dashboard.html", "/_static/", "/_static/../web.go", + } { + if code, _ := get(t, h, "http://x"+path, nil); code == http.StatusOK { + t.Errorf("%s was served", path) + } + } +}