Pasting the gateway's address into a browser returned 404, which looks like a broken endpoint rather than one that tells you how to use it. Two pages, and the split between them is a security boundary rather than a layout choice. The landing page is served without a key — it has to be, because the person reading it is the one who does not have one yet — so it contains no data at all: it says how to connect, not what is here. Model names stay behind /v1/models. It reconstructs the address from the forwarded headers so the copy-pasteable example names the address the client actually used rather than the internal one; those headers are reflected into displayed text and never used for a decision. The dashboard shows nodes, models, replicas, placement failures and usage, and lives on the admin listener only. That listener is loopback-bound and also serves pprof, which can dump the process heap — where API keys sit in plaintext — so reaching the page from elsewhere is an SSH tunnel's job. It is read-only: anything that mints a credential stays in `llmgw keys`, where it needs a shell rather than a browser session this project has no way to authenticate. Mounting the landing page uncovered a hole in the exemption list. An entry ending in "/" was a prefix rule, and "/" ends in "/", so exempting the landing page would have exempted every path there is — authentication still installed, still logging, and letting the whole API through. "/" is now exact, with a test enumerating what must and must not be exempt. Server-rendered templates and one stylesheet, embedded. The single-binary goal rules out a JavaScript toolchain and nothing on either page needs one. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
0b93fb2883
commit
1839832b8b
14 files changed
+836
-10
No files matched your search
@@ -380,3 +380,27 @@ func TestLastUsedIsBatchedNotWrittenPerRequest(t *testing.T) {
|
||||
t.Errorf("wrong key touched: %s", k.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExemptingTheLandingPageDoesNotExemptEverything(t *testing.T) {
|
||||
// A near-miss worth a test. "/" ends in a slash, so a prefix rule would
|
||||
// match every path there is: authentication still installed, still logging,
|
||||
// and letting the entire API through.
|
||||
cases := []struct {
|
||||
path string
|
||||
want bool
|
||||
}{
|
||||
{"/", true},
|
||||
{"/_static/base.css", true},
|
||||
{"/health", true},
|
||||
{"/v1/models", false},
|
||||
{"/v1/chat/completions", false},
|
||||
{"/healthz", false}, // not a prefix rule
|
||||
{"/_staticx", false}, // prefix rules end at the slash
|
||||
}
|
||||
exempt := []string{"/health", "/", "/_static/"}
|
||||
for _, c := range cases {
|
||||
if got := isExempt(c.path, exempt); got != c.want {
|
||||
t.Errorf("isExempt(%q) = %v, want %v", c.path, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -40,12 +40,21 @@ func Middleware(a *Authenticator, log *slog.Logger, exempt ...string) func(http.
|
||||
|
||||
// isExempt reports whether a path is served without a key.
|
||||
//
|
||||
// Health is the only one by default. Metrics live on the admin listener, which
|
||||
// is a separate port precisely so it can be firewalled rather than
|
||||
// An entry ending in "/" is a prefix rule, with one exception that matters
|
||||
// enormously: "/" itself is exact. Treating it as a prefix would match every
|
||||
// path there is, so exempting the landing page would silently exempt the whole
|
||||
// API — authentication still installed, still logging, and letting everything
|
||||
// through.
|
||||
//
|
||||
// Metrics and pprof are not in this list because they live on the admin
|
||||
// listener, a separate port precisely so it can be firewalled rather than
|
||||
// authenticated.
|
||||
func isExempt(path string, exempt []string) bool {
|
||||
for _, e := range exempt {
|
||||
if path == e || (strings.HasSuffix(e, "/") && strings.HasPrefix(path, e)) {
|
||||
if path == e {
|
||||
return true
|
||||
}
|
||||
if e != "/" && strings.HasSuffix(e, "/") && strings.HasPrefix(path, e) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
+17
-1
@@ -26,6 +26,8 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
|
||||
)
|
||||
|
||||
func newServerCmd() *cobra.Command {
|
||||
@@ -112,7 +114,21 @@ func newServerCmd() *cobra.Command {
|
||||
return err
|
||||
}
|
||||
if admin != nil {
|
||||
adminapi.New(admin).Register(adminMux)
|
||||
api := adminapi.New(admin)
|
||||
api.Register(adminMux)
|
||||
|
||||
// The dashboard lives here and only here. This listener is
|
||||
// bound to loopback by default and also serves pprof, which can
|
||||
// dump the process heap — where API keys sit in plaintext.
|
||||
// Reaching the page from elsewhere is an SSH tunnel's job:
|
||||
// ssh -L 9090:127.0.0.1:9090 <host>
|
||||
var usage web.UsageSource
|
||||
if st := rt.Store(); st != nil {
|
||||
usage = st
|
||||
}
|
||||
adminMux.HandleFunc("GET /", web.Dashboard(
|
||||
api, usage, version.Version, 5*time.Second, log))
|
||||
adminMux.Handle("GET /_static/", web.Static())
|
||||
}
|
||||
|
||||
apiSrv := &http.Server{
|
||||
|
||||
@@ -117,6 +117,15 @@ type RejectionView struct {
|
||||
// --- handlers ---
|
||||
|
||||
func (a *API) listNodes(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, a.Nodes())
|
||||
}
|
||||
|
||||
// Nodes builds the node view.
|
||||
//
|
||||
// Exported because the dashboard renders the same data server-side. Going back
|
||||
// out through HTTP to fetch what is already in this process would add a way for
|
||||
// the two to disagree.
|
||||
func (a *API) Nodes() []NodeView {
|
||||
replicasOn := map[string]int{}
|
||||
for _, rep := range a.ctl.Reconciler().Replicas() {
|
||||
for _, n := range rep.NodeIDs() {
|
||||
@@ -150,10 +159,15 @@ func (a *API) listNodes(w http.ResponseWriter, _ *http.Request) {
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (a *API) listModels(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, a.Models())
|
||||
}
|
||||
|
||||
// Models builds the catalog view.
|
||||
func (a *API) Models() []ModelView {
|
||||
ready, total := map[string]int{}, map[string]int{}
|
||||
for _, rep := range a.ctl.Reconciler().Replicas() {
|
||||
total[rep.Model]++
|
||||
@@ -175,12 +189,15 @@ func (a *API) listModels(w http.ResponseWriter, _ *http.Request) {
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (a *API) listReplicas(w http.ResponseWriter, r *http.Request) {
|
||||
model := r.URL.Query().Get("model")
|
||||
writeJSON(w, http.StatusOK, a.Replicas(r.URL.Query().Get("model")))
|
||||
}
|
||||
|
||||
// Replicas builds the replica view, optionally filtered to one model.
|
||||
func (a *API) Replicas(model string) []ReplicaView {
|
||||
out := make([]ReplicaView, 0)
|
||||
for _, rep := range a.ctl.Reconciler().Replicas() {
|
||||
if model != "" && rep.Model != model {
|
||||
@@ -200,7 +217,7 @@ func (a *API) listReplicas(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (a *API) explainPlacement(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -214,6 +231,15 @@ func (a *API) explainPlacement(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, a.Explain(model))
|
||||
}
|
||||
|
||||
// Explain builds the placement explanation for one model.
|
||||
//
|
||||
// It does not check that the model exists; the handler above does that so it
|
||||
// can answer 404, while the dashboard only ever asks about models it just read
|
||||
// out of the catalog.
|
||||
func (a *API) Explain(model string) ExplainView {
|
||||
v := ExplainView{Model: model}
|
||||
for _, rep := range a.ctl.Reconciler().Replicas() {
|
||||
if rep.Model == model && rep.State.Counts() {
|
||||
@@ -228,7 +254,7 @@ func (a *API) explainPlacement(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
return v
|
||||
}
|
||||
|
||||
// setNodeState implements cordon, uncordon and drain.
|
||||
|
||||
@@ -17,6 +17,8 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
|
||||
)
|
||||
|
||||
// admit waits for the flow control layer to let the request through. With
|
||||
@@ -82,13 +84,20 @@ func (r *Router) Handler() http.Handler {
|
||||
mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
})
|
||||
|
||||
// The landing page says how to connect and nothing about what is here, so
|
||||
// it is safe to serve without a key — and it has to be, because the person
|
||||
// reading it is the one who does not have one yet.
|
||||
mux.HandleFunc("GET /", web.Landing(version.Version, r.auth != nil))
|
||||
mux.Handle("GET /_static/", web.Static())
|
||||
|
||||
if r.auth == nil {
|
||||
return mux
|
||||
}
|
||||
// Wrapping the whole mux rather than checking inside each handler: adding a
|
||||
// route must not be able to add an unauthenticated one by omission. Health
|
||||
// stays open because a load balancer has no key.
|
||||
return auth.Middleware(r.auth, r.log, "/health")(mux)
|
||||
return auth.Middleware(r.auth, r.log, "/health", "/", "/_static/")(mux)
|
||||
}
|
||||
|
||||
// parsedRequest is what the router needs from a request. Everything else in the
|
||||
|
||||
@@ -504,3 +504,31 @@ func nowMillis() int64 { return time.Now().UnixMilli() }
|
||||
|
||||
// SQLite satisfies Store.
|
||||
var _ Store = (*SQLite)(nil)
|
||||
|
||||
// UsageSince summarizes activity per key.
|
||||
//
|
||||
// The join brings in the key's name and prefix, which is what a dashboard needs
|
||||
// to show: a row identified only by an opaque id tells an operator nothing.
|
||||
// Revoked keys still appear, because their usage still happened.
|
||||
func (s *SQLite) UsageSince(ctx context.Context, since time.Time) ([]KeyUsage, error) {
|
||||
return queryAll(ctx, s.db, `
|
||||
SELECT u.key_id,
|
||||
COALESCE(k.name, ''), COALESCE(k.key_prefix, ''), u.project_id,
|
||||
COUNT(*), SUM(u.prompt_tokens), SUM(u.completion_tokens),
|
||||
SUM(u.cost_micros),
|
||||
SUM(CASE WHEN u.status >= 400 THEN 1 ELSE 0 END),
|
||||
MAX(u.ts)
|
||||
FROM usage u
|
||||
LEFT JOIN api_keys k ON k.id = u.key_id
|
||||
WHERE u.ts >= ?
|
||||
GROUP BY u.key_id, u.project_id
|
||||
ORDER BY SUM(u.cost_micros) DESC, COUNT(*) DESC`,
|
||||
[]any{millis(since)}, func(rows *sql.Rows) (KeyUsage, error) {
|
||||
var k KeyUsage
|
||||
var last int64
|
||||
err := rows.Scan(&k.KeyID, &k.KeyName, &k.KeyPrefix, &k.ProjectID,
|
||||
&k.Requests, &k.Prompt, &k.Completion, &k.CostMicros, &k.Errors, &last)
|
||||
k.LastAt = fromMillis(last)
|
||||
return k, err
|
||||
})
|
||||
}
|
||||
@@ -193,6 +193,24 @@ type Store interface {
|
||||
// counter can be rebuilt after a restart. A restart must not reset a
|
||||
// budget, which it would if the counter lived only in memory.
|
||||
SpendSince(ctx context.Context, since time.Time) (map[string]int64, error)
|
||||
// UsageSince summarizes activity per key. A reporting query: it scans and
|
||||
// groups, so it belongs behind a dashboard refresh rather than a request.
|
||||
UsageSince(ctx context.Context, since time.Time) ([]KeyUsage, error)
|
||||
|
||||
Close() error
|
||||
}
|
||||
|
||||
// KeyUsage summarizes one key's activity in a period, for the dashboard and
|
||||
// `ctl`. It is a reporting query and never runs on the request path.
|
||||
type KeyUsage struct {
|
||||
KeyID string
|
||||
KeyName string
|
||||
KeyPrefix string
|
||||
ProjectID string
|
||||
Requests int64
|
||||
Prompt int64
|
||||
Completion int64
|
||||
CostMicros int64
|
||||
Errors int64
|
||||
LastAt time.Time
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// DashboardData is the whole page.
|
||||
type DashboardData struct {
|
||||
Nodes []adminapi.NodeView
|
||||
Models []adminapi.ModelView
|
||||
Replicas []adminapi.ReplicaView
|
||||
Unplaced []adminapi.ExplainView
|
||||
Usage []store.KeyUsage
|
||||
|
||||
NodeCount int
|
||||
ModelCount int
|
||||
ReadyReplicas int
|
||||
TotalReplicas int
|
||||
|
||||
AuthEnabled bool
|
||||
RefreshSec int
|
||||
Version string
|
||||
Now string
|
||||
}
|
||||
|
||||
// Source is what the dashboard reads. An interface so the page can be rendered
|
||||
// in a test without a control plane.
|
||||
type Source interface {
|
||||
Nodes() []adminapi.NodeView
|
||||
Models() []adminapi.ModelView
|
||||
Replicas(model string) []adminapi.ReplicaView
|
||||
Explain(model string) adminapi.ExplainView
|
||||
}
|
||||
|
||||
// UsageSource is the accounting half, absent when authentication is off.
|
||||
type UsageSource interface {
|
||||
UsageSince(ctx context.Context, since time.Time) ([]store.KeyUsage, error)
|
||||
}
|
||||
|
||||
// Dashboard serves the read-only cluster view.
|
||||
//
|
||||
// It belongs on the admin listener and nowhere else. That listener is bound to
|
||||
// loopback by default and also serves pprof, which can dump the process heap —
|
||||
// where API keys live in plaintext. Exposing this page publicly would mean
|
||||
// exposing that, so reaching it from elsewhere is an SSH tunnel's job:
|
||||
//
|
||||
// ssh -L 9090:127.0.0.1:9090 llmgw-ctl
|
||||
//
|
||||
// Read-only on purpose. Anything that mints a credential stays in `llmgw keys`,
|
||||
// where it needs a shell on the machine rather than a browser session this
|
||||
// project has no way to authenticate.
|
||||
func Dashboard(src Source, usage UsageSource, version string, refresh time.Duration,
|
||||
log *slog.Logger,
|
||||
) http.HandlerFunc {
|
||||
if log == nil {
|
||||
log = slog.Default()
|
||||
}
|
||||
if refresh <= 0 {
|
||||
refresh = 5 * time.Second
|
||||
}
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
d := DashboardData{
|
||||
RefreshSec: int(refresh.Seconds()),
|
||||
Version: version,
|
||||
Now: time.Now().UTC().Format(time.RFC3339),
|
||||
AuthEnabled: usage != nil,
|
||||
}
|
||||
if src != nil {
|
||||
d.Nodes = src.Nodes()
|
||||
d.Models = src.Models()
|
||||
d.Replicas = src.Replicas("")
|
||||
d.NodeCount = len(d.Nodes)
|
||||
d.ModelCount = len(d.Models)
|
||||
d.TotalReplicas = len(d.Replicas)
|
||||
for _, rep := range d.Replicas {
|
||||
if rep.State == "ready" {
|
||||
d.ReadyReplicas++
|
||||
}
|
||||
}
|
||||
d.Unplaced = unplaced(src, d.Models)
|
||||
}
|
||||
if usage != nil {
|
||||
rows, err := usage.UsageSince(r.Context(), time.Now().Add(-24*time.Hour))
|
||||
if err != nil {
|
||||
// A reporting query failing must not take the page with it: the
|
||||
// cluster state above is what someone is most likely here for.
|
||||
log.Warn("could not read usage for the dashboard", "err", err)
|
||||
}
|
||||
d.Usage = rows
|
||||
}
|
||||
render(w, "dashboard", d)
|
||||
}
|
||||
}
|
||||
|
||||
// unplaced collects the placement explanations for models short of replicas.
|
||||
//
|
||||
// Only for those: asking why a satisfied model was not placed produces an empty
|
||||
// answer, and a page full of empty answers hides the one that matters.
|
||||
func unplaced(src Source, models []adminapi.ModelView) []adminapi.ExplainView {
|
||||
var out []adminapi.ExplainView
|
||||
for _, m := range models {
|
||||
if m.Ready >= m.Min {
|
||||
continue
|
||||
}
|
||||
if e := src.Explain(m.Name); len(e.Rejections) > 0 {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// LandingData is what the landing page shows. It is deliberately almost
|
||||
// nothing: this page is served without a key, to anyone who can reach the
|
||||
// address.
|
||||
type LandingData struct {
|
||||
BaseURL string
|
||||
AuthEnabled bool
|
||||
Version string
|
||||
Now string
|
||||
}
|
||||
|
||||
// Landing serves the page at "/" on the client-facing port.
|
||||
//
|
||||
// Without it that path is a 404, which is what someone sees when they paste the
|
||||
// address into a browser — an endpoint that looks broken rather than one that
|
||||
// tells them how to use it.
|
||||
//
|
||||
// It lists no models and no cluster state. Everything that says what is here
|
||||
// stays behind the key, including /v1/models.
|
||||
func Landing(version string, authEnabled bool) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
// Only the root: a catch-all would turn every mistyped path into this
|
||||
// page, hiding genuine 404s from anyone debugging a client.
|
||||
if r.URL.Path != "/" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
render(w, "landing", LandingData{
|
||||
BaseURL: baseURL(r),
|
||||
AuthEnabled: authEnabled,
|
||||
Version: version,
|
||||
Now: time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// baseURL reconstructs the address the client actually used.
|
||||
//
|
||||
// Behind a reverse proxy the scheme and host this process sees are the internal
|
||||
// ones, so the forwarded headers are what make the copy-pasteable example on
|
||||
// the page correct. They are attacker-controlled when nothing sets them, which
|
||||
// is why they are only reflected into a displayed URL and never used for a
|
||||
// decision.
|
||||
func baseURL(r *http.Request) string {
|
||||
scheme := "http"
|
||||
switch {
|
||||
case r.TLS != nil:
|
||||
scheme = "https"
|
||||
case r.Header.Get("X-Forwarded-Proto") != "":
|
||||
scheme = firstValue(r.Header.Get("X-Forwarded-Proto"))
|
||||
}
|
||||
|
||||
host := r.Host
|
||||
if fwd := r.Header.Get("X-Forwarded-Host"); fwd != "" {
|
||||
host = firstValue(fwd)
|
||||
}
|
||||
if host == "" {
|
||||
host = "localhost:8080"
|
||||
}
|
||||
return scheme + "://" + host
|
||||
}
|
||||
|
||||
// firstValue takes the first entry of a comma-separated forwarded header: a
|
||||
// chain of proxies appends rather than replaces.
|
||||
func firstValue(v string) string {
|
||||
if i := strings.IndexByte(v, ','); i >= 0 {
|
||||
v = v[:i]
|
||||
}
|
||||
return strings.TrimSpace(v)
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
/* Deliberately one small stylesheet with no build step. The single-binary goal
|
||||
rules out a JS toolchain, and nothing here needs one. */
|
||||
:root {
|
||||
color-scheme: light dark;
|
||||
--bg: #fbfbfa;
|
||||
--fg: #1c1c1a;
|
||||
--muted: #6b6b66;
|
||||
--line: #e3e3df;
|
||||
--card: #ffffff;
|
||||
--accent: #2f5d50;
|
||||
--ok: #2f7a4f;
|
||||
--warn: #9a6b1f;
|
||||
--bad: #a33a32;
|
||||
}
|
||||
@media (prefers-color-scheme: dark) {
|
||||
:root {
|
||||
--bg: #17181a; --fg: #e8e8e4; --muted: #9a9a93; --line: #2c2e31;
|
||||
--card: #1e2022; --accent: #7fb3a2; --ok: #6fbd8c; --warn: #d6a44e; --bad: #e07b72;
|
||||
}
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
margin: 0; background: var(--bg); color: var(--fg);
|
||||
font: 15px/1.55 ui-sans-serif, -apple-system, "Segoe UI", Roboto, sans-serif;
|
||||
}
|
||||
.wrap { max-width: 1100px; margin: 0 auto; padding: 2.5rem 1.25rem 4rem; }
|
||||
header { border-bottom: 1px solid var(--line); margin-bottom: 2rem; padding-bottom: 1.25rem; }
|
||||
h1 { font-size: 1.5rem; margin: 0 0 .25rem; letter-spacing: -.01em; }
|
||||
h2 { font-size: 1.05rem; margin: 2.25rem 0 .75rem; letter-spacing: -.005em; }
|
||||
.sub { color: var(--muted); margin: 0; }
|
||||
nav { margin-top: .75rem; display: flex; gap: 1rem; flex-wrap: wrap; }
|
||||
nav a { color: var(--accent); text-decoration: none; font-size: .9rem; }
|
||||
nav a:hover { text-decoration: underline; }
|
||||
table { width: 100%; border-collapse: collapse; font-size: .875rem; background: var(--card);
|
||||
border: 1px solid var(--line); border-radius: 6px; overflow: hidden; }
|
||||
th, td { text-align: left; padding: .5rem .7rem; border-bottom: 1px solid var(--line); }
|
||||
th { font-weight: 600; color: var(--muted); font-size: .78rem; text-transform: uppercase;
|
||||
letter-spacing: .04em; }
|
||||
tr:last-child td { border-bottom: none; }
|
||||
td.num { text-align: right; font-variant-numeric: tabular-nums; }
|
||||
code, pre { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: .85em; }
|
||||
pre { background: var(--card); border: 1px solid var(--line); border-radius: 6px;
|
||||
padding: .9rem 1rem; overflow-x: auto; }
|
||||
.state { font-weight: 600; font-size: .8rem; }
|
||||
.state-ready, .state-active { color: var(--ok); }
|
||||
.state-pending, .state-starting, .state-draining, .state-cordoned { color: var(--warn); }
|
||||
.state-failed, .state-lost, .state-unreachable, .state-revoked { color: var(--bad); }
|
||||
.empty { color: var(--muted); font-style: italic; padding: .9rem 0; }
|
||||
.note { color: var(--muted); font-size: .85rem; margin: .4rem 0 0; }
|
||||
footer { margin-top: 3rem; padding-top: 1rem; border-top: 1px solid var(--line);
|
||||
color: var(--muted); font-size: .8rem; display: flex; justify-content: space-between; gap: 1rem; }
|
||||
@@ -0,0 +1,95 @@
|
||||
{{define "dashboard"}}<!doctype html>
|
||||
<html lang="en"><head>
|
||||
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta http-equiv="refresh" content="{{.RefreshSec}}">
|
||||
<title>llmgw — cluster</title><link rel="stylesheet" href="/_static/base.css">
|
||||
</head><body><div class="wrap">
|
||||
<header>
|
||||
<h1>Cluster</h1>
|
||||
<p class="sub">{{.NodeCount}} nodes · {{.ReadyReplicas}} of {{.TotalReplicas}} replicas ready · {{.ModelCount}} models</p>
|
||||
<nav>
|
||||
<a href="/metrics">metrics</a>
|
||||
<a href="/debug/pprof/">pprof</a>
|
||||
<a href="/api/v1/nodes">api</a>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<h2>Nodes</h2>
|
||||
{{if .Nodes}}
|
||||
<table><tr><th>Node</th><th>State</th><th class="num">GPUs</th><th>Classes</th><th>Fabrics</th><th class="num">Replicas</th><th>Last seen</th><th>Agent</th></tr>
|
||||
{{range .Nodes}}<tr>
|
||||
<td><code>{{.ID}}</code>{{if .MgmtAddr}} <span class="note">{{.MgmtAddr}}</span>{{end}}</td>
|
||||
<td class="state state-{{.State}}">{{.State}}</td>
|
||||
<td class="num">{{.GPUFree}}/{{.GPUTotal}}</td>
|
||||
<td>{{join .Classes}}</td>
|
||||
<td>{{join .Fabrics}}</td>
|
||||
<td class="num">{{.Replicas}}</td>
|
||||
<td>{{or .LastSeenAgo "—"}}</td>
|
||||
<td><span class="note">{{or .AgentVersion "—"}}</span></td>
|
||||
</tr>{{if .Unhealthy}}<tr><td colspan="8" class="note">{{join .Unhealthy}}</td></tr>{{end}}
|
||||
{{end}}</table>
|
||||
{{else}}<p class="empty">No agents have registered.</p>{{end}}
|
||||
|
||||
<h2>Models</h2>
|
||||
{{if .Models}}
|
||||
<table><tr><th>Model</th><th>Aliases</th><th class="num">Ready</th><th class="num">Wanted</th><th>Profiles</th><th>Fallbacks</th></tr>
|
||||
{{range .Models}}<tr>
|
||||
<td><code>{{.Name}}</code></td>
|
||||
<td>{{join .Aliases}}</td>
|
||||
<td class="num">{{.Ready}}</td>
|
||||
<td class="num">{{.Min}}–{{.Max}}</td>
|
||||
<td>{{join .Profiles}}</td>
|
||||
<td>{{join .Fallbacks}}</td>
|
||||
</tr>{{end}}</table>
|
||||
{{else}}<p class="empty">The catalog is empty.</p>{{end}}
|
||||
|
||||
{{if .Unplaced}}
|
||||
<h2>Why nothing was placed</h2>
|
||||
{{range .Unplaced}}
|
||||
<p><code>{{.Model}}</code></p>
|
||||
<pre>{{range .Rejections}}{{.Profile}} @ {{or .Pool "-"}}{{if .Node}}/{{.Node}}{{end}}: {{.Reason}}
|
||||
{{end}}</pre>
|
||||
{{end}}
|
||||
{{end}}
|
||||
|
||||
<h2>Replicas</h2>
|
||||
{{if .Replicas}}
|
||||
<table><tr><th>Replica</th><th>Model</th><th>State</th><th>Pool</th><th>Endpoint</th><th class="num">Age</th><th>Nodes</th></tr>
|
||||
{{range .Replicas}}<tr>
|
||||
<td><code>{{.ID}}</code></td>
|
||||
<td>{{.Model}}</td>
|
||||
<td class="state state-{{.State}}">{{.State}}</td>
|
||||
<td>{{or .Pool "—"}}</td>
|
||||
<td>{{if .Endpoint}}<code>{{.Endpoint}}</code>{{else}}—{{end}}</td>
|
||||
<td class="num">{{duration .AgeSec}}</td>
|
||||
<td>{{range .Shards}}<code>{{.Node}}</code> {{end}}</td>
|
||||
</tr>{{if .Err}}<tr><td colspan="7" class="note">{{.Err}}</td></tr>{{end}}
|
||||
{{end}}</table>
|
||||
{{else}}<p class="empty">No replicas.</p>{{end}}
|
||||
|
||||
<h2>Usage <span class="note">— last 24 hours</span></h2>
|
||||
{{if .AuthEnabled}}
|
||||
{{if .Usage}}
|
||||
<table><tr><th>Key</th><th>Project</th><th class="num">Requests</th><th class="num">Errors</th><th class="num">Prompt</th><th class="num">Completion</th><th class="num">Spend</th><th>Last</th></tr>
|
||||
{{range .Usage}}<tr>
|
||||
<td>{{if .KeyName}}{{.KeyName}} <span class="note">{{.KeyPrefix}}</span>{{else}}<code>{{.KeyID}}</code>{{end}}</td>
|
||||
<td>{{.ProjectID}}</td>
|
||||
<td class="num">{{.Requests}}</td>
|
||||
<td class="num">{{if .Errors}}<span class="state-failed">{{.Errors}}</span>{{else}}0{{end}}</td>
|
||||
<td class="num">{{.Prompt}}</td>
|
||||
<td class="num">{{.Completion}}</td>
|
||||
<td class="num">{{units .CostMicros}}</td>
|
||||
<td><span class="note">{{ago .LastAt}}</span></td>
|
||||
</tr>{{end}}</table>
|
||||
<p class="note">Spend is in units, whatever a unit was configured to mean. Usage is
|
||||
written in batches, so the last few seconds may be missing.</p>
|
||||
{{else}}<p class="empty">No requests recorded in the last 24 hours.</p>{{end}}
|
||||
{{else}}
|
||||
<p class="empty">Authentication is off, so there is no identity to account against.</p>
|
||||
{{end}}
|
||||
|
||||
<footer>
|
||||
<span>llmgw {{.Version}} · refreshes every {{.RefreshSec}}s</span>
|
||||
<span>{{.Now}}</span>
|
||||
</footer>
|
||||
</div></body></html>{{end}}
|
||||
@@ -0,0 +1,42 @@
|
||||
{{define "landing"}}<!doctype html>
|
||||
<html lang="en"><head>
|
||||
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>llmgw</title><link rel="stylesheet" href="/_static/base.css">
|
||||
</head><body><div class="wrap">
|
||||
<header>
|
||||
<h1>llmgw</h1>
|
||||
<p class="sub">An OpenAI-compatible endpoint in front of self-hosted models.</p>
|
||||
</header>
|
||||
|
||||
<p>Point any OpenAI client at this address. The API is the one you already have a
|
||||
client for; only the base URL changes.</p>
|
||||
|
||||
<h2>Base URL</h2>
|
||||
<pre>{{.BaseURL}}/v1</pre>
|
||||
|
||||
<h2>Example</h2>
|
||||
<pre>curl {{.BaseURL}}/v1/chat/completions \
|
||||
-H "Authorization: Bearer $LLMGW_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model": "MODEL", "messages": [{"role":"user","content":"hello"}]}'</pre>
|
||||
|
||||
<pre>from openai import OpenAI
|
||||
client = OpenAI(base_url="{{.BaseURL}}/v1", api_key=os.environ["LLMGW_API_KEY"])</pre>
|
||||
|
||||
<h2>Which models</h2>
|
||||
{{if .AuthEnabled}}
|
||||
<p><code>GET /v1/models</code> lists what is available to your key. It needs the key,
|
||||
which is why this page does not list them: this page is served to anyone who can
|
||||
reach the address.</p>
|
||||
{{else}}
|
||||
<p><code>GET /v1/models</code> lists what is available.</p>
|
||||
<p class="note">This gateway has no API keys configured, so it serves anyone who can
|
||||
reach it. That is a deployment choice and only safe on a trusted network.</p>
|
||||
{{end}}
|
||||
|
||||
<h2>Keys</h2>
|
||||
<p>Keys are issued by an operator with <code>llmgw keys create</code>, and shown once.
|
||||
There is no way to recover one: only its hash is stored.</p>
|
||||
|
||||
<footer><span>llmgw {{.Version}}</span><span>{{.Now}}</span></footer>
|
||||
</div></body></html>{{end}}
|
||||
@@ -0,0 +1,82 @@
|
||||
// Package web serves the two HTML surfaces: a landing page on the
|
||||
// client-facing port and a read-only dashboard on the admin one.
|
||||
//
|
||||
// Server-rendered templates with one stylesheet, embedded in the binary. The
|
||||
// single-binary goal rules out a JavaScript toolchain — a build step in CI, a
|
||||
// node runtime on the runner, and megabytes of bundle — and nothing on either
|
||||
// page needs one. The dashboard refreshes by reloading.
|
||||
//
|
||||
// The split between the two pages is a security boundary, not a layout choice.
|
||||
// The landing page is served to anyone who can reach the address, so it
|
||||
// contains no data at all: it says how to connect, not what is here. The
|
||||
// dashboard shows the cluster and belongs on the admin listener, which is bound
|
||||
// to loopback — see Dashboard.
|
||||
package web
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"fmt"
|
||||
"html/template"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
//go:embed templates
|
||||
var files embed.FS
|
||||
|
||||
var funcs = template.FuncMap{
|
||||
"join": func(v []string) string {
|
||||
if len(v) == 0 {
|
||||
return "—"
|
||||
}
|
||||
return strings.Join(v, ", ")
|
||||
},
|
||||
"duration": func(sec int64) string {
|
||||
return time.Duration(sec * int64(time.Second)).Round(time.Second).String()
|
||||
},
|
||||
"ago": func(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return "never"
|
||||
}
|
||||
return time.Since(t).Round(time.Second).String() + " ago"
|
||||
},
|
||||
// Spend is stored in micro-units so accumulation is exact; it is shown in
|
||||
// whole units because that is the number an operator set a budget in.
|
||||
"units": func(micros int64) string {
|
||||
return fmt.Sprintf("%.2f", float64(micros)/1e6)
|
||||
},
|
||||
}
|
||||
|
||||
var tmpl = template.Must(template.New("").Funcs(funcs).ParseFS(files, "templates/*.html"))
|
||||
|
||||
// Static serves the stylesheet, mounted on both listeners because both pages
|
||||
// use it and it contains nothing.
|
||||
//
|
||||
// One file rather than a directory server: exposing a file server over the
|
||||
// embedded templates would also serve the templates themselves, which is
|
||||
// harmless today and the kind of thing that stops being harmless later.
|
||||
func Static() http.Handler {
|
||||
css, err := files.ReadFile("templates/base.css")
|
||||
if err != nil {
|
||||
panic("web: the stylesheet is missing from the binary: " + err.Error())
|
||||
}
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/_static/base.css" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/css; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", "public, max-age=3600")
|
||||
_, _ = w.Write(css)
|
||||
})
|
||||
}
|
||||
|
||||
func render(w http.ResponseWriter, name string, data any) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
if err := tmpl.ExecuteTemplate(w, name, data); err != nil {
|
||||
// The status is already written by then, so there is nothing honest to
|
||||
// send; the page simply ends short.
|
||||
http.Error(w, "template error", http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
func get(t *testing.T, h http.Handler, target string, headers map[string]string) (int, string) {
|
||||
t.Helper()
|
||||
r := httptest.NewRequest(http.MethodGet, target, nil)
|
||||
for k, v := range headers {
|
||||
r.Header.Set(k, v)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
return w.Code, w.Body.String()
|
||||
}
|
||||
|
||||
func TestLandingPageTellsYouHowToConnect(t *testing.T) {
|
||||
code, body := get(t, Landing("v1.2.3", true), "http://llmgw.example/", nil)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("status %d", code)
|
||||
}
|
||||
for _, want := range []string{"http://llmgw.example/v1", "Authorization: Bearer", "v1.2.3"} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("the page does not mention %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLandingPageSaysNothingAboutWhatIsHere(t *testing.T) {
|
||||
// It is served without a key, to anyone who can reach the address. Anything
|
||||
// on it is public, so nothing about the cluster belongs on it.
|
||||
_, body := get(t, Landing("v1", true), "http://llmgw.example/", nil)
|
||||
for _, leak := range []string{"llama", "qwen", "10.0.0.", "replica", "node01"} {
|
||||
if strings.Contains(strings.ToLower(body), leak) {
|
||||
t.Errorf("the landing page leaks %q", leak)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLandingPageWarnsWhenTheGatewayIsOpen(t *testing.T) {
|
||||
// Someone who reaches this page on an unauthenticated gateway should learn
|
||||
// that from the page rather than by discovering they can use it.
|
||||
_, open := get(t, Landing("v1", false), "http://llmgw.example/", nil)
|
||||
if !strings.Contains(open, "no API keys configured") {
|
||||
t.Error("an open gateway does not say so")
|
||||
}
|
||||
_, closed := get(t, Landing("v1", true), "http://llmgw.example/", nil)
|
||||
if strings.Contains(closed, "no API keys configured") {
|
||||
t.Error("a closed gateway claims to be open")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheExampleURLSurvivesAReverseProxy(t *testing.T) {
|
||||
// The process sees http and an internal host; the client used https and a
|
||||
// public name. An example naming the internal address is one nobody can
|
||||
// paste anywhere.
|
||||
_, body := get(t, Landing("v1", true), "http://10.0.0.140:8080/", map[string]string{
|
||||
"X-Forwarded-Proto": "https",
|
||||
"X-Forwarded-Host": "llmgw.mynodes.xyz",
|
||||
})
|
||||
if !strings.Contains(body, "https://llmgw.mynodes.xyz/v1") {
|
||||
t.Error("the example does not use the address the client actually used")
|
||||
}
|
||||
if strings.Contains(body, "10.0.0.140") {
|
||||
t.Error("the internal address leaked into the page")
|
||||
}
|
||||
}
|
||||
|
||||
func TestForwardedHeaderChainsTakeTheFirstEntry(t *testing.T) {
|
||||
// A chain of proxies appends rather than replaces.
|
||||
_, body := get(t, Landing("v1", true), "http://internal/", map[string]string{
|
||||
"X-Forwarded-Proto": "https, http",
|
||||
"X-Forwarded-Host": "llmgw.example, internal",
|
||||
})
|
||||
if !strings.Contains(body, "https://llmgw.example/v1") {
|
||||
t.Errorf("the base URL was built from the wrong entry")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyTheRootIsTheLandingPage(t *testing.T) {
|
||||
// A catch-all would turn every mistyped path into this page, hiding real
|
||||
// 404s from anyone debugging a client.
|
||||
code, _ := get(t, Landing("v1", true), "http://x/v1/chat/completionz", nil)
|
||||
if code != http.StatusNotFound {
|
||||
t.Errorf("a mistyped path returned %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeSource is a cluster for the dashboard to render.
|
||||
type fakeSource struct{}
|
||||
|
||||
func (fakeSource) Nodes() []adminapi.NodeView {
|
||||
return []adminapi.NodeView{{
|
||||
ID: "llmgw-node01", State: "ready", GPUTotal: 8, GPUFree: 4,
|
||||
Classes: []string{"h100-80g"}, Fabrics: []string{"ib-a"},
|
||||
Replicas: 1, LastSeenAgo: "2s", AgentVersion: "v0.5.0",
|
||||
Unhealthy: []string{"gpu 3: ECC errors"},
|
||||
}}
|
||||
}
|
||||
|
||||
func (fakeSource) Models() []adminapi.ModelView {
|
||||
return []adminapi.ModelView{
|
||||
{Name: "served", Ready: 1, Min: 1, Max: 2, Profiles: []string{"tp4"}},
|
||||
{Name: "stuck", Ready: 0, Min: 1, Max: 1, Profiles: []string{"tp8"}},
|
||||
}
|
||||
}
|
||||
|
||||
func (fakeSource) Replicas(string) []adminapi.ReplicaView {
|
||||
return []adminapi.ReplicaView{{
|
||||
ID: "served-1", Model: "served", State: "ready", Pool: "ib-a/h100-80g",
|
||||
Endpoint: "http://10.0.0.141:20000", AgeSec: 3600,
|
||||
Shards: []adminapi.ShardView{{ID: "served-1-r0", Node: "llmgw-node01"}},
|
||||
}}
|
||||
}
|
||||
|
||||
func (fakeSource) Explain(model string) adminapi.ExplainView {
|
||||
if model != "stuck" {
|
||||
return adminapi.ExplainView{Model: model, Placed: true}
|
||||
}
|
||||
return adminapi.ExplainView{Model: "stuck", Rejections: []adminapi.RejectionView{
|
||||
{Profile: "tp8", Node: "llmgw-node01", Reason: "has 4 free h100-80g GPUs, needs 8"},
|
||||
}}
|
||||
}
|
||||
|
||||
type fakeUsage struct{ err error }
|
||||
|
||||
func (f fakeUsage) UsageSince(context.Context, time.Time) ([]store.KeyUsage, error) {
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return []store.KeyUsage{{
|
||||
KeyID: "key-1", KeyName: "laptop", KeyPrefix: "sk-llmgw-ab12…",
|
||||
ProjectID: "chat", Requests: 42, Errors: 2,
|
||||
Prompt: 1000, Completion: 3000, CostMicros: 4_500_000,
|
||||
LastAt: time.Now().Add(-time.Minute),
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func TestDashboardShowsTheCluster(t *testing.T) {
|
||||
h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, nil)
|
||||
code, body := get(t, h, "http://127.0.0.1:9090/", nil)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("status %d", code)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"llmgw-node01", "h100-80g", "served-1", "http://10.0.0.141:20000",
|
||||
"ECC errors", "laptop", "4.50", // spend shown in units, not micro-units
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("the dashboard does not show %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDashboardExplainsOnlyWhatIsUnplaced(t *testing.T) {
|
||||
// Asking why a satisfied model was not placed gives an empty answer, and a
|
||||
// page full of empty answers hides the one that matters.
|
||||
h := Dashboard(fakeSource{}, nil, "v1", time.Second, nil)
|
||||
_, body := get(t, h, "http://127.0.0.1:9090/", nil)
|
||||
|
||||
if !strings.Contains(body, "needs 8") {
|
||||
t.Error("the unplaced model has no explanation")
|
||||
}
|
||||
if strings.Count(body, "Why nothing was placed") != 1 {
|
||||
t.Error("the explanation section is missing or duplicated")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDashboardSurvivesAFailingUsageQuery(t *testing.T) {
|
||||
// A reporting query failing must not take the page with it: the cluster
|
||||
// state is what someone is most likely here for.
|
||||
h := Dashboard(fakeSource{}, fakeUsage{err: context.DeadlineExceeded}, "v1", time.Second, nil)
|
||||
code, body := get(t, h, "http://127.0.0.1:9090/", nil)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("status %d", code)
|
||||
}
|
||||
if !strings.Contains(body, "llmgw-node01") {
|
||||
t.Error("the cluster state was lost with the usage query")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDashboardSaysWhenThereIsNothingToAccount(t *testing.T) {
|
||||
h := Dashboard(fakeSource{}, nil, "v1", time.Second, nil)
|
||||
_, body := get(t, h, "http://127.0.0.1:9090/", nil)
|
||||
if !strings.Contains(body, "Authentication is off") {
|
||||
t.Error("the page does not explain the empty usage section")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDashboardRendersAnEmptyCluster(t *testing.T) {
|
||||
// The state a fresh install is in, and the one most likely to be looked at
|
||||
// with a template that has never seen it.
|
||||
h := Dashboard(emptySource{}, nil, "v1", time.Second, nil)
|
||||
code, body := get(t, h, "http://127.0.0.1:9090/", nil)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("status %d", code)
|
||||
}
|
||||
if !strings.Contains(body, "No agents have registered") {
|
||||
t.Error("an empty cluster renders nothing useful")
|
||||
}
|
||||
}
|
||||
|
||||
type emptySource struct{}
|
||||
|
||||
func (emptySource) Nodes() []adminapi.NodeView { return nil }
|
||||
func (emptySource) Models() []adminapi.ModelView { return nil }
|
||||
func (emptySource) Replicas(string) []adminapi.ReplicaView { return nil }
|
||||
func (emptySource) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} }
|
||||
|
||||
func TestStaticIsNotAFileServer(t *testing.T) {
|
||||
// A file server over the embedded templates would also serve the templates,
|
||||
// which is harmless today and the kind of thing that stops being harmless.
|
||||
h := Static()
|
||||
if code, _ := get(t, h, "http://x/_static/base.css", nil); code != http.StatusOK {
|
||||
t.Errorf("the stylesheet is not served: %d", code)
|
||||
}
|
||||
for _, path := range []string{
|
||||
"/_static/dashboard.html", "/_static/", "/_static/../web.go",
|
||||
} {
|
||||
if code, _ := get(t, h, "http://x"+path, nil); code == http.StatusOK {
|
||||
t.Errorf("%s was served", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user