Ilker ManapandClaude Opus 5.5
e8a62b23b7
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
481 lines
16 KiB
Go
481 lines
16 KiB
Go
package web
|
||
|
||
import (
|
||
"fmt"
|
||
"io/fs"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"os"
|
||
"path/filepath"
|
||
"regexp"
|
||
"slices"
|
||
"sort"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
|
||
"golang.org/x/net/html"
|
||
)
|
||
|
||
// The keys a template looks up: {{t "key"}}, (th "key" .X), {{tn "key" .N}}.
|
||
var templateKey = regexp.MustCompile(`(?:\{\{-?|\(|\|)\s*(t|th|tn)\s+"([^"]+)"`)
|
||
|
||
// The keys Go code looks up: l.T("key"), lang.N("key", n), registerErrorKey(err, "key").
|
||
var (
|
||
goKey = regexp.MustCompile(`\.(T|TH|N)\(\s*"([^"]+)"\s*[,)]`)
|
||
goErrorKey = regexp.MustCompile(`registerErrorKey\([^,]+,\s*"([^"]+)"\)`)
|
||
goRegisterKey = regexp.MustCompile(`registerKeys\(([^)]*)\)`)
|
||
goStringLit = regexp.MustCompile(`"([^"]+)"`)
|
||
)
|
||
|
||
// usedKeys is every key the pages and handlers name, with whether it is a
|
||
// counted one (tn, N), which a language may give as key.other alone.
|
||
func usedKeys(t *testing.T) map[string]bool {
|
||
t.Helper()
|
||
keys := map[string]bool{}
|
||
err := fs.WalkDir(files, "templates", func(p string, d fs.DirEntry, err error) error {
|
||
if err != nil || d.IsDir() {
|
||
return err
|
||
}
|
||
b, err := fs.ReadFile(files, p)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
for _, m := range templateKey.FindAllStringSubmatch(string(b), -1) {
|
||
keys[m[2]] = keys[m[2]] || m[1] == "tn"
|
||
}
|
||
return nil
|
||
})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
gofiles, _ := filepath.Glob("*.go")
|
||
for _, name := range gofiles {
|
||
if strings.HasSuffix(name, "_test.go") {
|
||
continue
|
||
}
|
||
b, err := os.ReadFile(name)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for _, m := range goKey.FindAllStringSubmatch(string(b), -1) {
|
||
keys[m[2]] = keys[m[2]] || m[1] == "N"
|
||
}
|
||
for _, m := range goErrorKey.FindAllStringSubmatch(string(b), -1) {
|
||
keys[m[1]] = keys[m[1]] || false
|
||
}
|
||
for _, m := range goRegisterKey.FindAllStringSubmatch(string(b), -1) {
|
||
for _, s := range goStringLit.FindAllStringSubmatch(m[1], -1) {
|
||
if strings.HasSuffix(s[1], ".") {
|
||
continue // the prefix of keys built in a loop, listed by dynamicKeys
|
||
}
|
||
keys[s[1]] = keys[s[1]] || false
|
||
}
|
||
}
|
||
}
|
||
// And the ones built at run time, listed where they are built.
|
||
for _, k := range dynamicKeys {
|
||
keys[k] = keys[k] || false
|
||
}
|
||
for _, ek := range errorKeys {
|
||
keys[ek.key] = keys[ek.key] || false
|
||
}
|
||
return keys
|
||
}
|
||
|
||
// A key one language has and another lacks is a sentence that stays English
|
||
// on the second language's pages, which nobody notices until a customer does.
|
||
func TestEveryKeyUsedHasTextInEveryLanguage(t *testing.T) {
|
||
keys := usedKeys(t)
|
||
if len(keys) < 10 {
|
||
t.Fatalf("found only %d keys; the scan is broken", len(keys))
|
||
}
|
||
for _, l := range languages {
|
||
var missing []string
|
||
for k, counted := range keys {
|
||
if _, ok := l.msgs[k]; ok {
|
||
continue
|
||
}
|
||
if _, ok := l.msgs[k+".other"]; ok && counted {
|
||
continue
|
||
}
|
||
missing = append(missing, k)
|
||
}
|
||
sort.Strings(missing)
|
||
for _, k := range missing {
|
||
t.Errorf("%s has no text for %q", l.Code, k)
|
||
}
|
||
}
|
||
}
|
||
|
||
// fmtVerb finds the arguments a text takes, so a translation that drops or
|
||
// adds one is caught: Sprintf would print %!s(MISSING) on the page.
|
||
var fmtVerb = regexp.MustCompile(`%(?:\[\d+\])?[-+# 0]*\d*(?:\.\d+)?[a-zA-Z%]`)
|
||
|
||
func verbsOf(s string) []string {
|
||
var out []string
|
||
for _, v := range fmtVerb.FindAllString(s, -1) {
|
||
if v == "%%" {
|
||
continue
|
||
}
|
||
// %[2]s and %s are the same argument kind in a different place.
|
||
v = regexp.MustCompile(`\[\d+\]`).ReplaceAllString(v, "")
|
||
out = append(out, v[len(v)-1:])
|
||
}
|
||
slices.Sort(out)
|
||
return out
|
||
}
|
||
|
||
// Every language says everything English says, with the same arguments, and
|
||
// nothing English does not — a key left behind by a page that changed is
|
||
// text nobody can see and somebody will translate.
|
||
func TestEveryLanguageHasTheSameKeysAsEnglishWithTheSameArguments(t *testing.T) {
|
||
for _, l := range languages[1:] {
|
||
for k, en := range english.msgs {
|
||
if strings.HasPrefix(k, "_meta.") {
|
||
continue
|
||
}
|
||
v, ok := l.msgs[k]
|
||
if !ok {
|
||
// A singular is English's; a language without one says the
|
||
// count with key.other.
|
||
if strings.HasSuffix(k, ".one") {
|
||
continue
|
||
}
|
||
t.Errorf("%s lacks %q (English: %q)", l.Code, k, en)
|
||
continue
|
||
}
|
||
if !slices.Equal(verbsOf(v), verbsOf(en)) {
|
||
t.Errorf("%s %q takes %v where English takes %v: %q", l.Code, k, verbsOf(v), verbsOf(en), v)
|
||
}
|
||
}
|
||
for k := range l.msgs {
|
||
if _, ok := english.msgs[k]; !ok {
|
||
t.Errorf("%s has %q, which English does not", l.Code, k)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// templateAction is a template action, comments included.
|
||
var templateAction = regexp.MustCompile(`(?s)\{\{.*?\}\}`)
|
||
|
||
// englishWord is a run of letters in a template's text: after translation a
|
||
// page's own text is punctuation, figures and what the allow-list names.
|
||
var englishWord = regexp.MustCompile(`[A-Za-z][A-Za-z'’-]*[A-Za-z]`)
|
||
|
||
// untranslatable are words a page may carry in every language: the product,
|
||
// units, and the names of standards and hardware, which Turkish writes the
|
||
// same way. A word of prose does not belong here; put it in a catalogue.
|
||
var untranslatable = map[string]bool{
|
||
"llmgw": true, "ms": true, "MHz": true, "GB": true, "GiB": true, "MB": true, "TB": true,
|
||
"Wh": true, "GPU": true, "GPUs": true, "CPU": true, "CPUs": true, "API": true, "URL": true,
|
||
"CUDA": true, "NVLink": true, "PCIe": true, "ECC": true, "vLLM": true, "SSO": true,
|
||
"OIDC": true, "SCIM": true, "TLS": true, "HTTP": true, "HTTPS": true, "JSON": true,
|
||
"YAML": true, "KV": true,
|
||
}
|
||
|
||
// textAttributes are the attributes a person reads.
|
||
var textAttributes = map[string]bool{
|
||
"title": true, "placeholder": true, "aria-label": true, "alt": true,
|
||
"data-confirm": true, "label": true,
|
||
}
|
||
|
||
// skippedElements hold code, commands and scripts, which are the same in
|
||
// every language; translate="no" marks anything else that is.
|
||
var skippedElements = map[string]bool{
|
||
"script": true, "style": true, "pre": true, "code": true, "kbd": true, "samp": true,
|
||
}
|
||
|
||
var voidElements = map[string]bool{
|
||
"area": true, "base": true, "br": true, "col": true, "embed": true, "hr": true, "img": true,
|
||
"input": true, "link": true, "meta": true, "source": true, "track": true, "wbr": true,
|
||
}
|
||
|
||
// englishIn reports the English words left in a template's own text and in
|
||
// the attributes a person reads, with their lines.
|
||
func englishIn(src string) []string {
|
||
var found []string
|
||
lineAt := func(off int) int { return 1 + strings.Count(src[:min(off, len(src))], "\n") }
|
||
// report notes s when it has a word of prose in it, at the line of that
|
||
// word: raw is the source s came from, starting at off.
|
||
report := func(off int, raw, s string) {
|
||
for _, w := range englishWord.FindAllString(s, -1) {
|
||
if untranslatable[w] {
|
||
continue
|
||
}
|
||
if i := strings.Index(raw, w); i >= 0 {
|
||
off += i
|
||
}
|
||
found = append(found, fmt.Sprintf("line %d: %s", lineAt(off), strings.Join(strings.Fields(s), " ")))
|
||
return
|
||
}
|
||
}
|
||
|
||
// Literal strings inside actions that read as prose: a printf format or
|
||
// a dict value with a space and a word in it.
|
||
for _, loc := range templateAction.FindAllStringIndex(src, -1) {
|
||
a := src[loc[0]:loc[1]]
|
||
if strings.HasPrefix(strings.TrimLeft(a, "{-"), "/*") || strings.HasPrefix(strings.TrimLeft(a, "{- "), "/*") {
|
||
continue
|
||
}
|
||
for _, lit := range goStringLit.FindAllStringSubmatchIndex(a, -1) {
|
||
s := a[lit[2]:lit[3]]
|
||
if strings.Contains(s, " ") && englishWord.MatchString(s) {
|
||
report(loc[0]+lit[0], s, s)
|
||
}
|
||
}
|
||
}
|
||
|
||
// The text and attributes, with every action blanked to spaces of the
|
||
// same length so offsets still find lines.
|
||
blank := templateAction.ReplaceAllStringFunc(src, func(a string) string {
|
||
return strings.Map(func(r rune) rune {
|
||
if r == '\n' {
|
||
return '\n'
|
||
}
|
||
return ' '
|
||
}, a)
|
||
})
|
||
z := html.NewTokenizer(strings.NewReader(blank))
|
||
off := 0
|
||
var skip []string // the element being skipped, once per level of nesting
|
||
for {
|
||
tt := z.Next()
|
||
if tt == html.ErrorToken {
|
||
break
|
||
}
|
||
rawTok := string(z.Raw())
|
||
tok := z.Token()
|
||
switch tt {
|
||
case html.StartTagToken, html.SelfClosingTagToken:
|
||
name := tok.Data
|
||
if len(skip) > 0 {
|
||
if name == skip[0] && tt == html.StartTagToken && !voidElements[name] {
|
||
skip = append(skip, name)
|
||
}
|
||
break
|
||
}
|
||
no := skippedElements[name]
|
||
for _, a := range tok.Attr {
|
||
if a.Key == "translate" && a.Val == "no" {
|
||
no = true
|
||
}
|
||
if textAttributes[a.Key] {
|
||
report(off, rawTok, a.Val)
|
||
}
|
||
if a.Key == "value" && name == "input" {
|
||
for _, b := range tok.Attr {
|
||
if b.Key == "type" && (b.Val == "submit" || b.Val == "button") {
|
||
report(off, rawTok, a.Val)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if no && tt == html.StartTagToken && !voidElements[name] {
|
||
skip = append(skip, name)
|
||
}
|
||
case html.EndTagToken:
|
||
if len(skip) > 0 && tok.Data == skip[0] {
|
||
skip = skip[1:]
|
||
}
|
||
case html.TextToken:
|
||
if len(skip) == 0 {
|
||
report(off, rawTok, tok.Data)
|
||
}
|
||
}
|
||
off += len(rawTok)
|
||
}
|
||
return found
|
||
}
|
||
|
||
// No page keeps a sentence of its own in English: everything a person reads
|
||
// goes through the catalogue, so the next page added cannot quietly be
|
||
// English-only. A word that is the same in every language goes in
|
||
// untranslatable; a run of code or a name goes in an element with
|
||
// translate="no", which is what that attribute is for.
|
||
func TestNoTemplateTextIsLeftInEnglish(t *testing.T) {
|
||
err := fs.WalkDir(files, "templates", func(p string, d fs.DirEntry, err error) error {
|
||
if err != nil || d.IsDir() {
|
||
return err
|
||
}
|
||
b, err := fs.ReadFile(files, p)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
for _, f := range englishIn(string(b)) {
|
||
t.Errorf("%s %s", p, f)
|
||
}
|
||
return nil
|
||
})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
|
||
// The heuristic itself, on what it must and must not flag.
|
||
func TestTheEnglishCheckFindsProseAndLeavesCodeAlone(t *testing.T) {
|
||
src := `<p>{{t "a"}}</p>
|
||
<p>Sign in</p>
|
||
<pre>curl -H "Authorization: Bearer x"</pre>
|
||
<span translate="no">Qwen3 Coder</span>
|
||
<input placeholder="laptop, ci">
|
||
<code>llmgw ctl</code> {{.N}} GPUs
|
||
{{template "x" (dict "Title" (printf "Quota for %s" .Name))}}`
|
||
got := englishIn(src)
|
||
want := []string{"line 7: Quota for %s", "line 2: Sign in", "line 5: laptop, ci"}
|
||
if !slices.Equal(got, want) {
|
||
t.Errorf("got %q, want %q", got, want)
|
||
}
|
||
}
|
||
|
||
func TestThePortalAnswersInTheLanguageAskedFor(t *testing.T) {
|
||
defer func() { _ = SetDefaultLanguage("") }()
|
||
req := func(accept, cookie string) *http.Request {
|
||
r := httptest.NewRequest(http.MethodGet, "/", nil)
|
||
if accept != "" {
|
||
r.Header.Set("Accept-Language", accept)
|
||
}
|
||
if cookie != "" {
|
||
r.AddCookie(&http.Cookie{Name: langCookie, Value: cookie})
|
||
}
|
||
return r
|
||
}
|
||
for _, c := range []struct {
|
||
accept, cookie, def, want string
|
||
}{
|
||
{"", "", "", "en"},
|
||
{"tr-TR,tr;q=0.9,en;q=0.8", "", "", "tr"},
|
||
{"de-DE,tr;q=0.5,en;q=0.4", "", "", "tr"}, // the first one spoken
|
||
{"en;q=0.3,tr;q=0.9", "", "", "tr"}, // by weight, not order
|
||
{"tr-TR", "en", "", "en"}, // the choice outranks the browser
|
||
{"de, fr", "", "tr", "tr"}, // none spoken: the server's default
|
||
{"", "xx", "tr", "tr"}, // a cookie naming nothing is ignored
|
||
{"not a header;;", "", "", "en"}, // and so is a broken header
|
||
{"EN-gb", "", "tr", "en"}, // case and region do not matter
|
||
{"", "TR", "", "tr"},
|
||
} {
|
||
if err := SetDefaultLanguage(c.def); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if got := langOf(req(c.accept, c.cookie)).Code; got != c.want {
|
||
t.Errorf("Accept-Language %q, cookie %q, default %q: %s, want %s", c.accept, c.cookie, c.def, got, c.want)
|
||
}
|
||
}
|
||
if err := SetDefaultLanguage("xx"); err == nil || !strings.Contains(err.Error(), "en, tr") {
|
||
t.Errorf("an unknown default: %v", err)
|
||
}
|
||
}
|
||
|
||
func TestTurkishNumbersAndDatesAreWrittenTheTurkishWay(t *testing.T) {
|
||
tr := langByCode("tr")
|
||
day := time.Date(2026, 10, 4, 9, 5, 0, 0, time.UTC)
|
||
for _, c := range []struct{ got, want string }{
|
||
{tr.Num(1234.5, 1), "1.234,5"},
|
||
{tr.Num(-1234567.25, 2), "-1.234.567,25"},
|
||
{tr.Num(999, 0), "999"},
|
||
{tr.Int(1234), "1.234"},
|
||
{tr.Units(1_500_000), "1,50"},
|
||
{tr.Compact(1247), "1,2 bin"},
|
||
{tr.Compact(0.03), "0,03"},
|
||
{tr.Bytes(1536), "1,5 KiB"},
|
||
{tr.Latency(8800), "8,8 sn"},
|
||
{tr.Latency(111_000), "1 dk 51 sn"},
|
||
{tr.Date(day), "04.10.2026"},
|
||
{tr.DateTime(day), "04.10.2026 09:05"},
|
||
{tr.Format(day, "Mon 15:04"), "Paz 09:05"},
|
||
{tr.Format(day, "2 January"), "4 Ekim"},
|
||
{tr.Ago(time.Now().Add(-3 * time.Hour)), "3 saat önce"},
|
||
{tr.Ago(time.Time{}), "hiçbir zaman"},
|
||
// One is a count like any other in Turkish, not English's singular.
|
||
{tr.Ago(time.Now().Add(-time.Hour)), "1 saat önce"},
|
||
// English is as it always was.
|
||
{english.Num(1234.5, 1), "1234.5"},
|
||
{english.Compact(1247), "1.2k"},
|
||
{english.Date(day), "2026-10-04"},
|
||
{english.Ago(time.Now().Add(-time.Hour)), "1 hour ago"},
|
||
{english.Ago(time.Now().Add(-3 * time.Hour)), "3 hours ago"},
|
||
{english.Format(day, "Mon 15:04"), "Sun 09:05"},
|
||
} {
|
||
if c.got != c.want {
|
||
t.Errorf("%q, want %q", c.got, c.want)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestTheSwitcherRemembersTheChoiceAndGoesBack(t *testing.T) {
|
||
mux := http.NewServeMux()
|
||
mux.HandleFunc("GET /_lang/{code}", SwitchLanguage)
|
||
for _, c := range []struct{ code, referer, status, to string }{
|
||
{"tr", "http://gw.example/profile?tab=keys", "303", "/profile?tab=keys"},
|
||
{"en", "http://elsewhere.example/phish", "303", "/"}, // never off the site
|
||
{"tr", "", "303", "/"},
|
||
{"xx", "http://gw.example/", "404", ""},
|
||
} {
|
||
r := httptest.NewRequest(http.MethodGet, "http://gw.example/_lang/"+c.code, nil)
|
||
if c.referer != "" {
|
||
r.Header.Set("Referer", c.referer)
|
||
}
|
||
w := httptest.NewRecorder()
|
||
mux.ServeHTTP(w, r)
|
||
if itoa(w.Code) != c.status || w.Header().Get("Location") != c.to {
|
||
t.Errorf("%s from %q: %d to %q, want %s to %q", c.code, c.referer, w.Code, w.Header().Get("Location"), c.status, c.to)
|
||
}
|
||
if c.status == "303" && !strings.Contains(w.Header().Get("Set-Cookie"), langCookie+"="+c.code) {
|
||
t.Errorf("%s: no cookie: %q", c.code, w.Header().Get("Set-Cookie"))
|
||
}
|
||
}
|
||
}
|
||
|
||
// Every page offers every language, marks the one in use, and says which it
|
||
// is to the browser and to screen readers.
|
||
func TestEveryPageSaysItsLanguageAndOffersTheOthers(t *testing.T) {
|
||
for _, code := range []string{"en", "tr"} {
|
||
r := httptest.NewRequest(http.MethodGet, "/login", nil)
|
||
r.AddCookie(&http.Cookie{Name: langCookie, Value: code})
|
||
w := httptest.NewRecorder()
|
||
render(w, r, "login", loginData{})
|
||
body := w.Body.String()
|
||
if !strings.Contains(body, `<html lang="`+code+`">`) || w.Header().Get("Content-Language") != code {
|
||
t.Errorf("%s: the page does not say its language", code)
|
||
}
|
||
if !strings.Contains(body, `href="/_lang/tr"`) && code == "en" || !strings.Contains(body, `href="/_lang/en"`) && code == "tr" {
|
||
t.Errorf("%s: the other language is not offered", code)
|
||
}
|
||
if strings.Contains(body, `href="/_lang/`+code+`"`) {
|
||
t.Errorf("%s: the language in use is offered as a link", code)
|
||
}
|
||
}
|
||
}
|
||
|
||
// A catalogue with a key twice keeps one silently in JSON; ours refuses.
|
||
func TestACatalogueWithAKeyTwiceIsRefused(t *testing.T) {
|
||
if _, err := decodeCatalogue([]byte(`{"a": "x", "a": "y"}`)); err == nil {
|
||
t.Error("a key twice was accepted")
|
||
}
|
||
if m, err := decodeCatalogue([]byte(`{"a": "x", "b": "y"}`)); err != nil || len(m) != 2 {
|
||
t.Errorf("%v %v", m, err)
|
||
}
|
||
}
|
||
|
||
// The docs portal's own pages are translated; the documents themselves are
|
||
// English, and a Turkish reader is told so rather than left to find out.
|
||
func TestTheDocsPortalSpeaksTurkishAroundEnglishDocuments(t *testing.T) {
|
||
r := httptest.NewRequest(http.MethodGet, "/docs", nil)
|
||
r.Header.Set("Accept-Language", "tr")
|
||
w := httptest.NewRecorder()
|
||
Docs("v1", true)(w, r)
|
||
body := w.Body.String()
|
||
for _, want := range []string{`<html lang="tr">`, "llmgw belgeleri", "Tüm belgeler", "yalnızca İngilizcedir", `href="/_lang/en"`} {
|
||
if !strings.Contains(body, want) {
|
||
t.Errorf("the Turkish docs index lacks %q", want)
|
||
}
|
||
}
|
||
r = httptest.NewRequest(http.MethodGet, "/docs", nil)
|
||
w = httptest.NewRecorder()
|
||
Docs("v1", true)(w, r)
|
||
if strings.Contains(w.Body.String(), "İngilizce") || !strings.Contains(w.Body.String(), "Every document") {
|
||
t.Error("the English docs index is not as it was")
|
||
}
|
||
}
|