The metrics were exported and the only way to look at them was to stand up Prometheus beside the binary. For a project whose whole shape is one static binary with no dependencies, that is the wrong default: the common case is somebody wanting to know how full their cluster is, and that should not cost a monitoring stack. /admin now shows capacity per GPU class with a usage bar, free whole nodes per fabric — the number that decides whether a replica spanning nodes can land, since free cards on the wrong fabric are not capacity for a model that needs this one — and which replicas are holding hardware nobody is asking for. Nodes gain their memory and how many model files they can see. Taken from the same Summary that ctl status uses rather than derived in the page, because two implementations of the same arithmetic drift apart and the one on the screen is the one people would believe. A faulty card counts as neither used nor free, matching the scheduler. Counting it free is how a cluster looks like it has room it does not, and a page that disagreed with placement about that would be worse than no page. Prometheus stays supported and is now the answer to a narrower question: history, alerting, several clusters at once. The dashboard shows what is true now and keeps nothing, which is the honest division between them. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
481 lines
17 KiB
Go
481 lines
17 KiB
Go
package web
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
func get(t *testing.T, h http.Handler, target string, headers map[string]string) (int, string) {
|
|
t.Helper()
|
|
r := httptest.NewRequest(http.MethodGet, target, nil)
|
|
for k, v := range headers {
|
|
r.Header.Set(k, v)
|
|
}
|
|
w := httptest.NewRecorder()
|
|
h.ServeHTTP(w, r)
|
|
return w.Code, w.Body.String()
|
|
}
|
|
|
|
func TestLandingPageTellsYouHowToConnect(t *testing.T) {
|
|
code, body := get(t, Landing("v1.2.3", true, false, "", ""), "http://llmgw.example/", nil)
|
|
if code != http.StatusOK {
|
|
t.Fatalf("status %d", code)
|
|
}
|
|
for _, want := range []string{"http://llmgw.example/v1", "Authorization: Bearer", "v1.2.3"} {
|
|
if !strings.Contains(body, want) {
|
|
t.Errorf("the page does not mention %q", want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestLandingPageSaysNothingAboutWhatIsHere(t *testing.T) {
|
|
// It is served without a key, to anyone who can reach the address. Anything
|
|
// on it is public, so nothing about the cluster belongs on it.
|
|
_, body := get(t, Landing("v1", true, false, "", ""), "http://llmgw.example/", nil)
|
|
for _, leak := range []string{"llama", "qwen", "10.0.0.", "replica", "node01"} {
|
|
if strings.Contains(strings.ToLower(body), leak) {
|
|
t.Errorf("the landing page leaks %q", leak)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestLandingPageWarnsWhenTheGatewayIsOpen(t *testing.T) {
|
|
// Someone who reaches this page on an unauthenticated gateway should learn
|
|
// that from the page rather than by discovering they can use it.
|
|
_, open := get(t, Landing("v1", false, false, "", ""), "http://llmgw.example/", nil)
|
|
if !strings.Contains(open, "no API keys configured") {
|
|
t.Error("an open gateway does not say so")
|
|
}
|
|
_, closed := get(t, Landing("v1", true, false, "", ""), "http://llmgw.example/", nil)
|
|
if strings.Contains(closed, "no API keys configured") {
|
|
t.Error("a closed gateway claims to be open")
|
|
}
|
|
}
|
|
|
|
func TestTheExampleURLSurvivesAReverseProxy(t *testing.T) {
|
|
// The process sees http and an internal host; the client used https and a
|
|
// public name. An example naming the internal address is one nobody can
|
|
// paste anywhere.
|
|
_, body := get(t, Landing("v1", true, false, "", ""), "http://10.0.0.140:8080/", map[string]string{
|
|
"X-Forwarded-Proto": "https",
|
|
"X-Forwarded-Host": "llmgw.mynodes.xyz",
|
|
})
|
|
if !strings.Contains(body, "https://llmgw.mynodes.xyz/v1") {
|
|
t.Error("the example does not use the address the client actually used")
|
|
}
|
|
if strings.Contains(body, "10.0.0.140") {
|
|
t.Error("the internal address leaked into the page")
|
|
}
|
|
}
|
|
|
|
func TestForwardedHeaderChainsTakeTheFirstEntry(t *testing.T) {
|
|
// A chain of proxies appends rather than replaces.
|
|
_, body := get(t, Landing("v1", true, false, "", ""), "http://internal/", map[string]string{
|
|
"X-Forwarded-Proto": "https, http",
|
|
"X-Forwarded-Host": "llmgw.example, internal",
|
|
})
|
|
if !strings.Contains(body, "https://llmgw.example/v1") {
|
|
t.Errorf("the base URL was built from the wrong entry")
|
|
}
|
|
}
|
|
|
|
func TestOnlyTheRootIsTheLandingPage(t *testing.T) {
|
|
// A catch-all would turn every mistyped path into this page, hiding real
|
|
// 404s from anyone debugging a client.
|
|
code, _ := get(t, Landing("v1", true, false, "", ""), "http://x/v1/chat/completionz", nil)
|
|
if code != http.StatusNotFound {
|
|
t.Errorf("a mistyped path returned %d", code)
|
|
}
|
|
}
|
|
|
|
// fakeSource is a cluster for the dashboard to render.
|
|
type fakeSource struct{}
|
|
|
|
func (fakeSource) Nodes() []adminapi.NodeView {
|
|
return []adminapi.NodeView{{
|
|
ID: "llmgw-node01", State: "ready", GPUTotal: 8, GPUFree: 4,
|
|
Classes: []string{"h100-80g"}, Fabrics: []string{"ib-a"},
|
|
Replicas: 1, LastSeenAgo: "2s", AgentVersion: "v0.5.0",
|
|
Unhealthy: []string{"gpu 3: ECC errors"},
|
|
}}
|
|
}
|
|
|
|
func (fakeSource) Summary() adminapi.Summary {
|
|
return adminapi.Summary{
|
|
Nodes: adminapi.NodeTotals{Total: 1, Ready: 1},
|
|
GPUs: []adminapi.ClassUse{
|
|
{Class: "h100-80g", Total: 8, Used: 3, Unhealthy: 1},
|
|
},
|
|
Fabrics: []adminapi.FabricUse{
|
|
{Fabric: "ib-a", Nodes: 1, LargestFreeBlock: 0, NodesWithWeights: 1, Models: 2},
|
|
},
|
|
IdleReplicas: []adminapi.IdleReplica{
|
|
{ID: "served-abc", Model: "served", IdleSec: 3600, GPUs: 4},
|
|
},
|
|
}
|
|
}
|
|
|
|
func (fakeSource) Models() []adminapi.ModelView {
|
|
return []adminapi.ModelView{
|
|
{Name: "served", Ready: 1, Min: 1, Max: 2, Profiles: []string{"tp4"}},
|
|
{Name: "stuck", Ready: 0, Min: 1, Max: 1, Profiles: []string{"tp8"}},
|
|
}
|
|
}
|
|
|
|
func (fakeSource) Replicas(string) []adminapi.ReplicaView {
|
|
return []adminapi.ReplicaView{{
|
|
ID: "served-1", Model: "served", State: "ready", Pool: "ib-a/h100-80g",
|
|
Endpoint: "http://10.0.0.141:20000", AgeSec: 3600,
|
|
Shards: []adminapi.ShardView{{ID: "served-1-r0", Node: "llmgw-node01"}},
|
|
}}
|
|
}
|
|
|
|
func (fakeSource) Explain(model string) adminapi.ExplainView {
|
|
if model != "stuck" {
|
|
return adminapi.ExplainView{Model: model, Placed: true}
|
|
}
|
|
return adminapi.ExplainView{Model: "stuck", Rejections: []adminapi.RejectionView{
|
|
{Profile: "tp8", Node: "llmgw-node01", Reason: "has 4 free h100-80g GPUs, needs 8"},
|
|
}}
|
|
}
|
|
|
|
type fakeUsage struct {
|
|
err error
|
|
buckets []store.Bucket
|
|
}
|
|
|
|
func (f fakeUsage) UsageBuckets(context.Context, time.Time, time.Duration) ([]store.Bucket, error) {
|
|
if f.err != nil {
|
|
return nil, f.err
|
|
}
|
|
return f.buckets, nil
|
|
}
|
|
|
|
func (f fakeUsage) UsageSince(context.Context, time.Time) ([]store.KeyUsage, error) {
|
|
if f.err != nil {
|
|
return nil, f.err
|
|
}
|
|
return []store.KeyUsage{{
|
|
KeyID: "key-1", KeyName: "laptop", KeyPrefix: "sk-llmgw-ab12…",
|
|
ProjectID: "chat", Requests: 42, Errors: 2,
|
|
Prompt: 1000, Completion: 3000, CostMicros: 4_500_000,
|
|
LastAt: time.Now().Add(-time.Minute),
|
|
}}, nil
|
|
}
|
|
|
|
func TestDashboardShowsTheCluster(t *testing.T) {
|
|
h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, nil)
|
|
code, body := get(t, h, "http://127.0.0.1:9090/", nil)
|
|
if code != http.StatusOK {
|
|
t.Fatalf("status %d", code)
|
|
}
|
|
for _, want := range []string{
|
|
"llmgw-node01", "h100-80g", "served-1", "http://10.0.0.141:20000",
|
|
"ECC errors", "laptop", "4.50", // spend shown in units, not micro-units
|
|
} {
|
|
if !strings.Contains(body, want) {
|
|
t.Errorf("the dashboard does not show %q", want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDashboardExplainsOnlyWhatIsUnplaced(t *testing.T) {
|
|
// Asking why a satisfied model was not placed gives an empty answer, and a
|
|
// page full of empty answers hides the one that matters.
|
|
h := Dashboard(fakeSource{}, nil, "v1", time.Second, true, nil)
|
|
_, body := get(t, h, "http://127.0.0.1:9090/", nil)
|
|
|
|
if !strings.Contains(body, "needs 8") {
|
|
t.Error("the unplaced model has no explanation")
|
|
}
|
|
if strings.Count(body, "Why nothing was placed") != 1 {
|
|
t.Error("the explanation section is missing or duplicated")
|
|
}
|
|
}
|
|
|
|
func TestDashboardSurvivesAFailingUsageQuery(t *testing.T) {
|
|
// A reporting query failing must not take the page with it: the cluster
|
|
// state is what someone is most likely here for.
|
|
h := Dashboard(fakeSource{}, fakeUsage{err: context.DeadlineExceeded}, "v1", time.Second, true, nil)
|
|
code, body := get(t, h, "http://127.0.0.1:9090/", nil)
|
|
if code != http.StatusOK {
|
|
t.Fatalf("status %d", code)
|
|
}
|
|
if !strings.Contains(body, "llmgw-node01") {
|
|
t.Error("the cluster state was lost with the usage query")
|
|
}
|
|
}
|
|
|
|
func TestDashboardSaysWhenThereIsNothingToAccount(t *testing.T) {
|
|
h := Dashboard(fakeSource{}, nil, "v1", time.Second, true, nil)
|
|
_, body := get(t, h, "http://127.0.0.1:9090/", nil)
|
|
if !strings.Contains(body, "Authentication is off") {
|
|
t.Error("the page does not explain the empty usage section")
|
|
}
|
|
}
|
|
|
|
func TestDashboardRendersAnEmptyCluster(t *testing.T) {
|
|
// The state a fresh install is in, and the one most likely to be looked at
|
|
// with a template that has never seen it.
|
|
h := Dashboard(emptySource{}, nil, "v1", time.Second, true, nil)
|
|
code, body := get(t, h, "http://127.0.0.1:9090/", nil)
|
|
if code != http.StatusOK {
|
|
t.Fatalf("status %d", code)
|
|
}
|
|
if !strings.Contains(body, "No agents have registered") {
|
|
t.Error("an empty cluster renders nothing useful")
|
|
}
|
|
}
|
|
|
|
type emptySource struct{}
|
|
|
|
func (emptySource) Nodes() []adminapi.NodeView { return nil }
|
|
func (emptySource) Models() []adminapi.ModelView { return nil }
|
|
func (emptySource) Replicas(string) []adminapi.ReplicaView { return nil }
|
|
func (emptySource) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} }
|
|
func (emptySource) Summary() adminapi.Summary { return adminapi.Summary{} }
|
|
|
|
func TestStaticServesAssetsAndNothingElse(t *testing.T) {
|
|
// The assets live in their own embed. A file server over the tree that also
|
|
// carries the templates would serve those as well, and a directory listing
|
|
// would publish the shape of the tree to nobody's benefit.
|
|
h := Static()
|
|
for _, path := range []string{
|
|
"/_static/base.css",
|
|
"/_static/katex/katex.min.css",
|
|
"/_static/katex/katex.min.js",
|
|
"/_static/katex/auto-render.min.js",
|
|
"/_static/katex/fonts/KaTeX_Main-Regular.woff2",
|
|
} {
|
|
if code, _ := get(t, h, "http://x"+path, nil); code != http.StatusOK {
|
|
t.Errorf("%s is not served: %d", path, code)
|
|
}
|
|
}
|
|
for _, path := range []string{
|
|
"/_static/dashboard.html", "/_static/chat.html",
|
|
"/_static/", "/_static/katex/", "/_static/katex/fonts/",
|
|
"/_static/../web.go",
|
|
} {
|
|
if code, _ := get(t, h, "http://x"+path, nil); code == http.StatusOK {
|
|
t.Errorf("%s was served", path)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTheLandingPageSaysWhereToGetAKey(t *testing.T) {
|
|
// A page that explains the API and leaves the reader with no route to using
|
|
// it is where onboarding stops.
|
|
_, body := get(t, Landing("v1", true, false, "platform@example.com", ""), "http://x/", nil)
|
|
if !strings.Contains(body, `href="mailto:platform@example.com"`) {
|
|
t.Error("an address was not turned into a mailto link")
|
|
}
|
|
// With signup closed, the route is a person; the page still has to say what
|
|
// arriving looks like.
|
|
if !strings.Contains(body, "shown once") {
|
|
t.Error("the page does not say a key cannot be recovered")
|
|
}
|
|
}
|
|
|
|
func TestContactRenderingByKind(t *testing.T) {
|
|
cases := []struct{ in, text, href string }{
|
|
{"platform@example.com", "platform@example.com", "mailto:platform@example.com"},
|
|
{"mailto:ops@example.com", "mailto:ops@example.com", "mailto:ops@example.com"},
|
|
{"https://wiki.example/llmgw", "https://wiki.example/llmgw", "https://wiki.example/llmgw"},
|
|
{"http://wiki.example", "http://wiki.example", "http://wiki.example"},
|
|
// Not addresses and not URLs: shown, not linked.
|
|
{"#platform on Slack", "#platform on Slack", ""},
|
|
{"Ayse", "Ayse", ""},
|
|
{"", "", ""},
|
|
{" ", "", ""},
|
|
}
|
|
for _, c := range cases {
|
|
text, href := contactLink(c.in)
|
|
if text != c.text || href != c.href {
|
|
t.Errorf("contactLink(%q) = (%q, %q), want (%q, %q)", c.in, text, href, c.text, c.href)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestADangerousContactSchemeIsNeverLinked(t *testing.T) {
|
|
// This string comes from configuration, and configuration is not always
|
|
// written by whoever runs the gateway. A javascript: href on a page served
|
|
// to every visitor is a stored XSS with a very short path to it; escaping
|
|
// the text does not make the scheme safe.
|
|
for _, bad := range []string{
|
|
"javascript:alert(1)",
|
|
"JavaScript:alert(1)",
|
|
"data:text/html,<script>alert(1)</script>",
|
|
"vbscript:msgbox(1)",
|
|
"file:///etc/passwd",
|
|
} {
|
|
text, href := contactLink(bad)
|
|
if href != "" {
|
|
t.Errorf("contactLink(%q) produced href %q", bad, href)
|
|
}
|
|
if text == "" {
|
|
t.Errorf("contactLink(%q) dropped the text entirely", bad)
|
|
}
|
|
|
|
_, body := get(t, Landing("v1", true, false, bad, ""), "http://x/", nil)
|
|
if strings.Contains(body, "href=\"javascript") || strings.Contains(body, "href=\"data:") {
|
|
t.Errorf("%q reached the page as a link", bad)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTheNoticeIsEscaped(t *testing.T) {
|
|
_, body := get(t, Landing("v1", true, false, "", "<script>alert(1)</script>"), "http://x/", nil)
|
|
if strings.Contains(body, "<script>alert(1)</script>") {
|
|
t.Error("the notice was rendered as markup")
|
|
}
|
|
if !strings.Contains(body, "<script>") {
|
|
t.Error("the notice was dropped rather than escaped")
|
|
}
|
|
}
|
|
|
|
func TestWithoutAContactThePageStillSaysSomethingUseful(t *testing.T) {
|
|
_, body := get(t, Landing("v1", true, false, "", ""), "http://x/", nil)
|
|
if !strings.Contains(body, "whoever operates this gateway") {
|
|
t.Error("an unconfigured gateway leaves the reader with nothing")
|
|
}
|
|
}
|
|
|
|
func TestWithSignupOpenTheLandingPagePointsAtIt(t *testing.T) {
|
|
// The route from "I found this address" to "I have a key" has to be on the
|
|
// page, or the reader is stuck whatever else it says.
|
|
_, body := get(t, Landing("v1", true, true, "", ""), "http://x/", nil)
|
|
for _, want := range []string{`href="/signup"`, `href="/login"`, `href="/profile"`} {
|
|
if !strings.Contains(body, want) {
|
|
t.Errorf("the page does not link to %s", want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTheDashboardDrawsTrafficWhenThereIsSome(t *testing.T) {
|
|
now := time.Now().Truncate(time.Hour)
|
|
usage := fakeUsage{buckets: []store.Bucket{
|
|
{Start: now.Add(-2 * time.Hour), Requests: 40, Prompt: 4000, Completion: 900, LatencyMs: 120},
|
|
{Start: now.Add(-time.Hour)},
|
|
{Start: now, Requests: 95, Prompt: 9000, Completion: 2100, LatencyMs: 310},
|
|
}}
|
|
h := Dashboard(fakeSource{}, usage, "v1", time.Second, true, nil)
|
|
_, body := get(t, h, "http://127.0.0.1:9090/", nil)
|
|
|
|
if !strings.Contains(body, "<svg") {
|
|
t.Fatal("no chart was drawn")
|
|
}
|
|
// The empty hour in the middle must still be a slot, or the shape lies
|
|
// about when traffic happened.
|
|
if n := strings.Count(body, `<rect`); n < 4 {
|
|
t.Errorf("%d bars across three charts and three hours", n)
|
|
}
|
|
if !strings.Contains(body, "peak 95 requests/h") {
|
|
t.Error("the request peak is not labelled")
|
|
}
|
|
}
|
|
|
|
func TestTheDashboardSaysWhenThereIsNoTrafficYet(t *testing.T) {
|
|
h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, nil)
|
|
_, body := get(t, h, "http://127.0.0.1:9090/", nil)
|
|
if !strings.Contains(body, "No traffic recorded yet") {
|
|
t.Error("an empty history renders nothing at all")
|
|
}
|
|
}
|
|
|
|
func TestTheDashboardOffersOnlyLinksThatExistWhereItIsServed(t *testing.T) {
|
|
// The same page is served twice with different neighbours. Metrics, pprof
|
|
// and the management API exist only on the admin listener; account
|
|
// administration and the profile only on the client-facing one.
|
|
//
|
|
// Getting this wrong was not a 404: on the public port those paths fall
|
|
// through to the API-key middleware, so a visitor clicking "metrics" was
|
|
// told "no api key was provided", which is a far more confusing way to say
|
|
// the link does not belong there.
|
|
_, admin := get(t, Dashboard(fakeSource{}, nil, "v1", time.Second, true, nil),
|
|
"http://127.0.0.1:9090/", nil)
|
|
for _, want := range []string{`href="/metrics"`, `href="/debug/pprof/"`, `href="/api/v1/nodes"`} {
|
|
if !strings.Contains(admin, want) {
|
|
t.Errorf("the admin listener's page is missing %s", want)
|
|
}
|
|
}
|
|
if strings.Contains(admin, `href="/admin/users"`) {
|
|
t.Error("the admin listener's page links to a page only the public port serves")
|
|
}
|
|
|
|
_, public := get(t, Dashboard(fakeSource{}, nil, "v1", time.Second, false, nil),
|
|
"https://llmgw.example/admin", nil)
|
|
for _, unwanted := range []string{`href="/metrics"`, `href="/debug/pprof/"`, `href="/api/v1/nodes"`} {
|
|
if strings.Contains(public, unwanted) {
|
|
t.Errorf("the public page links to %s, which is not served there", unwanted)
|
|
}
|
|
}
|
|
if !strings.Contains(public, `href="/admin/users"`) {
|
|
t.Error("the public page does not link to account administration")
|
|
}
|
|
// And it says where the missing ones are, rather than pretending they do
|
|
// not exist.
|
|
if !strings.Contains(public, "admin listener") {
|
|
t.Error("the public page does not say where metrics and pprof live")
|
|
}
|
|
}
|
|
|
|
func TestTheDashboardShowsCapacityWithoutPrometheus(t *testing.T) {
|
|
// The whole point of putting this on the page: knowing how full a cluster
|
|
// is should not require standing up a monitoring stack beside it. The
|
|
// tables say what exists; this says how much of it is in use, which is the
|
|
// question asked first.
|
|
h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, nil)
|
|
code, page := get(t, h, "http://127.0.0.1:9090/", nil)
|
|
if code != http.StatusOK {
|
|
t.Fatalf("status %d", code)
|
|
}
|
|
for _, want := range []string{
|
|
"Capacity", "h100-80g", "3/8", // used of total
|
|
"Fabrics", "ib-a",
|
|
"Holding hardware nothing is asking for", "served-abc",
|
|
} {
|
|
if !strings.Contains(page, want) {
|
|
t.Errorf("the dashboard does not show %q", want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAFaultyCardIsShownAsNeitherUsedNorFree(t *testing.T) {
|
|
// Counting a faulty card as free is how a cluster looks like it has room it
|
|
// does not, and the page must not disagree with the scheduler about that.
|
|
h := Dashboard(fakeSource{}, fakeUsage{}, "v1", time.Second, true, nil)
|
|
code, page := get(t, h, "http://127.0.0.1:9090/", nil)
|
|
if code != http.StatusOK {
|
|
t.Fatalf("status %d", code)
|
|
}
|
|
if !strings.Contains(page, "Faulty") {
|
|
t.Error("faulty cards are not shown")
|
|
}
|
|
// 8 total, 3 used, 1 faulty, so 4 free.
|
|
if !strings.Contains(page, ">4<") {
|
|
t.Errorf("free count is wrong or missing:\n%s", page)
|
|
}
|
|
}
|
|
|
|
func TestAUsageMeterCannotOverrunItsColumn(t *testing.T) {
|
|
// The two counts it divides can disagree for an instant — a GPU freed
|
|
// between one read and the next — and a bar wider than its cell wrecks the
|
|
// table.
|
|
pct, ok := funcs["pct"].(func(int, int) int)
|
|
if !ok {
|
|
t.Fatal("the pct helper is missing")
|
|
}
|
|
for _, tc := range [][2]int{{0, 0}, {9, 8}, {-1, 8}, {100, 8}} {
|
|
if got := pct(tc[0], tc[1]); got < 0 || got > 100 {
|
|
t.Errorf("pct(%d,%d) = %d", tc[0], tc[1], got)
|
|
}
|
|
}
|
|
}
|