llmgw/internal/web/i18n_test.go

package web

import (
	"fmt"
	"io/fs"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"regexp"
	"slices"
	"sort"
	"strings"
	"testing"
	"time"

	"golang.org/x/net/html"
)

// The keys a template looks up: {{t "key"}}, (th "key" .X), {{tn "key" .N}}.
var templateKey = regexp.MustCompile(`(?:\{\{-?|\(|\|)\s*(t|th|tn)\s+"([^"]+)"`)

// The keys Go code looks up: l.T("key"), lang.N("key", n), registerErrorKey(err, "key").
var (
	goKey         = regexp.MustCompile(`\.(T|TH|N)\(\s*"([^"]+)"\s*[,)]`)
	goErrorKey    = regexp.MustCompile(`registerErrorKey\([^,]+,\s*"([^"]+)"\)`)
	goRegisterKey = regexp.MustCompile(`registerKeys\(([^)]*)\)`)
	goStringLit   = regexp.MustCompile(`"([^"]+)"`)
)

// usedKeys is every key the pages and handlers name, with whether it is a
// counted one (tn, N), which a language may give as key.other alone.
func usedKeys(t *testing.T) map[string]bool {
	t.Helper()
	keys := map[string]bool{}
	err := fs.WalkDir(files, "templates", func(p string, d fs.DirEntry, err error) error {
		if err != nil || d.IsDir() {
			return err
		}
		b, err := fs.ReadFile(files, p)
		if err != nil {
			return err
		}
		for _, m := range templateKey.FindAllStringSubmatch(string(b), -1) {
			keys[m[2]] = keys[m[2]] || m[1] == "tn"
		}
		return nil
	})
	if err != nil {
		t.Fatal(err)
	}
	gofiles, _ := filepath.Glob("*.go")
	for _, name := range gofiles {
		if strings.HasSuffix(name, "_test.go") {
			continue
		}
		b, err := os.ReadFile(name)
		if err != nil {
			t.Fatal(err)
		}
		for _, m := range goKey.FindAllStringSubmatch(string(b), -1) {
			keys[m[2]] = keys[m[2]] || m[1] == "N"
		}
		for _, m := range goErrorKey.FindAllStringSubmatch(string(b), -1) {
			keys[m[1]] = keys[m[1]] || false
		}
		for _, m := range goRegisterKey.FindAllStringSubmatch(string(b), -1) {
			for _, s := range goStringLit.FindAllStringSubmatch(m[1], -1) {
				if strings.HasSuffix(s[1], ".") {
					continue // the prefix of keys built in a loop, listed by dynamicKeys
				}
				keys[s[1]] = keys[s[1]] || false
			}
		}
	}
	// And the ones built at run time, listed where they are built.
	for _, k := range dynamicKeys {
		keys[k] = keys[k] || false
	}
	for _, ek := range errorKeys {
		keys[ek.key] = keys[ek.key] || false
	}
	return keys
}

// A key one language has and another lacks is a sentence that stays English
// on the second language's pages, which nobody notices until a customer does.
func TestEveryKeyUsedHasTextInEveryLanguage(t *testing.T) {
	keys := usedKeys(t)
	if len(keys) < 10 {
		t.Fatalf("found only %d keys; the scan is broken", len(keys))
	}
	for _, l := range languages {
		var missing []string
		for k, counted := range keys {
			if _, ok := l.msgs[k]; ok {
				continue
			}
			if _, ok := l.msgs[k+".other"]; ok && counted {
				continue
			}
			missing = append(missing, k)
		}
		sort.Strings(missing)
		for _, k := range missing {
			t.Errorf("%s has no text for %q", l.Code, k)
		}
	}
}

// fmtVerb finds the arguments a text takes, so a translation that drops or
// adds one is caught: Sprintf would print %!s(MISSING) on the page.
var fmtVerb = regexp.MustCompile(`%(?:\[\d+\])?[-+# 0]*\d*(?:\.\d+)?[a-zA-Z%]`)

func verbsOf(s string) []string {
	var out []string
	for _, v := range fmtVerb.FindAllString(s, -1) {
		if v == "%%" {
			continue
		}
		// %[2]s and %s are the same argument kind in a different place.
		v = regexp.MustCompile(`\[\d+\]`).ReplaceAllString(v, "")
		out = append(out, v[len(v)-1:])
	}
	slices.Sort(out)
	return out
}

// Every language says everything English says, with the same arguments, and
// nothing English does not — a key left behind by a page that changed is
// text nobody can see and somebody will translate.
func TestEveryLanguageHasTheSameKeysAsEnglishWithTheSameArguments(t *testing.T) {
	for _, l := range languages[1:] {
		for k, en := range english.msgs {
			if strings.HasPrefix(k, "_meta.") {
				continue
			}
			v, ok := l.msgs[k]
			if !ok {
				// A singular is English's; a language without one says the
				// count with key.other.
				if strings.HasSuffix(k, ".one") {
					continue
				}
				t.Errorf("%s lacks %q (English: %q)", l.Code, k, en)
				continue
			}
			if !slices.Equal(verbsOf(v), verbsOf(en)) {
				t.Errorf("%s %q takes %v where English takes %v: %q", l.Code, k, verbsOf(v), verbsOf(en), v)
			}
		}
		for k := range l.msgs {
			if _, ok := english.msgs[k]; !ok {
				t.Errorf("%s has %q, which English does not", l.Code, k)
			}
		}
	}
}

// templateAction is a template action, comments included.
var templateAction = regexp.MustCompile(`(?s)\{\{.*?\}\}`)

// englishWord is a run of letters in a template's text: after translation a
// page's own text is punctuation, figures and what the allow-list names.
var englishWord = regexp.MustCompile(`[A-Za-z][A-Za-z'’-]*[A-Za-z]`)

// untranslatable are words a page may carry in every language: the product,
// units, and the names of standards and hardware, which Turkish writes the
// same way. A word of prose does not belong here; put it in a catalogue.
var untranslatable = map[string]bool{
	"llmgw": true, "ms": true, "MHz": true, "GB": true, "GiB": true, "MB": true, "TB": true,
	"Wh": true, "GPU": true, "GPUs": true, "CPU": true, "CPUs": true, "API": true, "URL": true,
	"CUDA": true, "NVLink": true, "PCIe": true, "ECC": true, "vLLM": true, "SSO": true,
	"OIDC": true, "SCIM": true, "TLS": true, "HTTP": true, "HTTPS": true, "JSON": true,
	"YAML": true, "KV": true,
}

// textAttributes are the attributes a person reads.
var textAttributes = map[string]bool{
	"title": true, "placeholder": true, "aria-label": true, "alt": true,
	"data-confirm": true, "label": true,
}

// skippedElements hold code, commands and scripts, which are the same in
// every language; translate="no" marks anything else that is.
var skippedElements = map[string]bool{
	"script": true, "style": true, "pre": true, "code": true, "kbd": true, "samp": true,
}

var voidElements = map[string]bool{
	"area": true, "base": true, "br": true, "col": true, "embed": true, "hr": true, "img": true,
	"input": true, "link": true, "meta": true, "source": true, "track": true, "wbr": true,
}

// englishIn reports the English words left in a template's own text and in
// the attributes a person reads, with their lines.
func englishIn(src string) []string {
	var found []string
	lineAt := func(off int) int { return 1 + strings.Count(src[:min(off, len(src))], "\n") }
	// report notes s when it has a word of prose in it, at the line of that
	// word: raw is the source s came from, starting at off.
	report := func(off int, raw, s string) {
		for _, w := range englishWord.FindAllString(s, -1) {
			if untranslatable[w] {
				continue
			}
			if i := strings.Index(raw, w); i >= 0 {
				off += i
			}
			found = append(found, fmt.Sprintf("line %d: %s", lineAt(off), strings.Join(strings.Fields(s), " ")))
			return
		}
	}

	// Literal strings inside actions that read as prose: a printf format or
	// a dict value with a space and a word in it.
	for _, loc := range templateAction.FindAllStringIndex(src, -1) {
		a := src[loc[0]:loc[1]]
		if strings.HasPrefix(strings.TrimLeft(a, "{-"), "/*") || strings.HasPrefix(strings.TrimLeft(a, "{- "), "/*") {
			continue
		}
		for _, lit := range goStringLit.FindAllStringSubmatchIndex(a, -1) {
			s := a[lit[2]:lit[3]]
			if strings.Contains(s, " ") && englishWord.MatchString(s) {
				report(loc[0]+lit[0], s, s)
			}
		}
	}

	// The text and attributes, with every action blanked to spaces of the
	// same length so offsets still find lines.
	blank := templateAction.ReplaceAllStringFunc(src, func(a string) string {
		return strings.Map(func(r rune) rune {
			if r == '\n' {
				return '\n'
			}
			return ' '
		}, a)
	})
	z := html.NewTokenizer(strings.NewReader(blank))
	off := 0
	var skip []string // the element being skipped, once per level of nesting
	for {
		tt := z.Next()
		if tt == html.ErrorToken {
			break
		}
		rawTok := string(z.Raw())
		tok := z.Token()
		switch tt {
		case html.StartTagToken, html.SelfClosingTagToken:
			name := tok.Data
			if len(skip) > 0 {
				if name == skip[0] && tt == html.StartTagToken && !voidElements[name] {
					skip = append(skip, name)
				}
				break
			}
			no := skippedElements[name]
			for _, a := range tok.Attr {
				if a.Key == "translate" && a.Val == "no" {
					no = true
				}
				if textAttributes[a.Key] {
					report(off, rawTok, a.Val)
				}
				if a.Key == "value" && name == "input" {
					for _, b := range tok.Attr {
						if b.Key == "type" && (b.Val == "submit" || b.Val == "button") {
							report(off, rawTok, a.Val)
						}
					}
				}
			}
			if no && tt == html.StartTagToken && !voidElements[name] {
				skip = append(skip, name)
			}
		case html.EndTagToken:
			if len(skip) > 0 && tok.Data == skip[0] {
				skip = skip[1:]
			}
		case html.TextToken:
			if len(skip) == 0 {
				report(off, rawTok, tok.Data)
			}
		}
		off += len(rawTok)
	}
	return found
}

// No page keeps a sentence of its own in English: everything a person reads
// goes through the catalogue, so the next page added cannot quietly be
// English-only. A word that is the same in every language goes in
// untranslatable; a run of code or a name goes in an element with
// translate="no", which is what that attribute is for.
func TestNoTemplateTextIsLeftInEnglish(t *testing.T) {
	err := fs.WalkDir(files, "templates", func(p string, d fs.DirEntry, err error) error {
		if err != nil || d.IsDir() {
			return err
		}
		b, err := fs.ReadFile(files, p)
		if err != nil {
			return err
		}
		for _, f := range englishIn(string(b)) {
			t.Errorf("%s %s", p, f)
		}
		return nil
	})
	if err != nil {
		t.Fatal(err)
	}
}

// The heuristic itself, on what it must and must not flag.
func TestTheEnglishCheckFindsProseAndLeavesCodeAlone(t *testing.T) {
	src := `<p>{{t "a"}}</p>
<p>Sign in</p>
<pre>curl -H "Authorization: Bearer x"</pre>
<span translate="no">Qwen3 Coder</span>
<input placeholder="laptop, ci">
<code>llmgw ctl</code> {{.N}} GPUs
{{template "x" (dict "Title" (printf "Quota for %s" .Name))}}`
	got := englishIn(src)
	want := []string{"line 7: Quota for %s", "line 2: Sign in", "line 5: laptop, ci"}
	if !slices.Equal(got, want) {
		t.Errorf("got %q, want %q", got, want)
	}
}

func TestThePortalAnswersInTheLanguageAskedFor(t *testing.T) {
	defer func() { _ = SetDefaultLanguage("") }()
	req := func(accept, cookie string) *http.Request {
		r := httptest.NewRequest(http.MethodGet, "/", nil)
		if accept != "" {
			r.Header.Set("Accept-Language", accept)
		}
		if cookie != "" {
			r.AddCookie(&http.Cookie{Name: langCookie, Value: cookie})
		}
		return r
	}
	for _, c := range []struct {
		accept, cookie, def, want string
	}{
		{"", "", "", "en"},
		{"tr-TR,tr;q=0.9,en;q=0.8", "", "", "tr"},
		{"de-DE,tr;q=0.5,en;q=0.4", "", "", "tr"}, // the first one spoken
		{"en;q=0.3,tr;q=0.9", "", "", "tr"},       // by weight, not order
		{"tr-TR", "en", "", "en"},                 // the choice outranks the browser
		{"de, fr", "", "tr", "tr"},                // none spoken: the server's default
		{"", "xx", "tr", "tr"},                    // a cookie naming nothing is ignored
		{"not a header;;", "", "", "en"},          // and so is a broken header
		{"EN-gb", "", "tr", "en"},                 // case and region do not matter
		{"", "TR", "", "tr"},
	} {
		if err := SetDefaultLanguage(c.def); err != nil {
			t.Fatal(err)
		}
		if got := langOf(req(c.accept, c.cookie)).Code; got != c.want {
			t.Errorf("Accept-Language %q, cookie %q, default %q: %s, want %s", c.accept, c.cookie, c.def, got, c.want)
		}
	}
	if err := SetDefaultLanguage("xx"); err == nil || !strings.Contains(err.Error(), "en, tr") {
		t.Errorf("an unknown default: %v", err)
	}
}

func TestTurkishNumbersAndDatesAreWrittenTheTurkishWay(t *testing.T) {
	tr := langByCode("tr")
	day := time.Date(2026, 10, 4, 9, 5, 0, 0, time.UTC)
	for _, c := range []struct{ got, want string }{
		{tr.Num(1234.5, 1), "1.234,5"},
		{tr.Num(-1234567.25, 2), "-1.234.567,25"},
		{tr.Num(999, 0), "999"},
		{tr.Int(1234), "1.234"},
		{tr.Units(1_500_000), "1,50"},
		{tr.Compact(1247), "1,2 bin"},
		{tr.Compact(0.03), "0,03"},
		{tr.Bytes(1536), "1,5 KiB"},
		{tr.Latency(8800), "8,8 sn"},
		{tr.Latency(111_000), "1 dk 51 sn"},
		{tr.Date(day), "04.10.2026"},
		{tr.DateTime(day), "04.10.2026 09:05"},
		{tr.Format(day, "Mon 15:04"), "Paz 09:05"},
		{tr.Format(day, "2 January"), "4 Ekim"},
		{tr.Ago(time.Now().Add(-3 * time.Hour)), "3 saat önce"},
		{tr.Ago(time.Time{}), "hiçbir zaman"},
		// One is a count like any other in Turkish, not English's singular.
		{tr.Ago(time.Now().Add(-time.Hour)), "1 saat önce"},
		// English is as it always was.
		{english.Num(1234.5, 1), "1234.5"},
		{english.Compact(1247), "1.2k"},
		{english.Date(day), "2026-10-04"},
		{english.Ago(time.Now().Add(-time.Hour)), "1 hour ago"},
		{english.Ago(time.Now().Add(-3 * time.Hour)), "3 hours ago"},
		{english.Format(day, "Mon 15:04"), "Sun 09:05"},
	} {
		if c.got != c.want {
			t.Errorf("%q, want %q", c.got, c.want)
		}
	}
}

func TestTheSwitcherRemembersTheChoiceAndGoesBack(t *testing.T) {
	mux := http.NewServeMux()
	mux.HandleFunc("GET /_lang/{code}", SwitchLanguage)
	for _, c := range []struct{ code, referer, status, to string }{
		{"tr", "http://gw.example/profile?tab=keys", "303", "/profile?tab=keys"},
		{"en", "http://elsewhere.example/phish", "303", "/"}, // never off the site
		{"tr", "", "303", "/"},
		{"xx", "http://gw.example/", "404", ""},
	} {
		r := httptest.NewRequest(http.MethodGet, "http://gw.example/_lang/"+c.code, nil)
		if c.referer != "" {
			r.Header.Set("Referer", c.referer)
		}
		w := httptest.NewRecorder()
		mux.ServeHTTP(w, r)
		if itoa(w.Code) != c.status || w.Header().Get("Location") != c.to {
			t.Errorf("%s from %q: %d to %q, want %s to %q", c.code, c.referer, w.Code, w.Header().Get("Location"), c.status, c.to)
		}
		if c.status == "303" && !strings.Contains(w.Header().Get("Set-Cookie"), langCookie+"="+c.code) {
			t.Errorf("%s: no cookie: %q", c.code, w.Header().Get("Set-Cookie"))
		}
	}
}

// Every page offers every language, marks the one in use, and says which it
// is to the browser and to screen readers.
func TestEveryPageSaysItsLanguageAndOffersTheOthers(t *testing.T) {
	for _, code := range []string{"en", "tr"} {
		r := httptest.NewRequest(http.MethodGet, "/login", nil)
		r.AddCookie(&http.Cookie{Name: langCookie, Value: code})
		w := httptest.NewRecorder()
		render(w, r, "login", loginData{})
		body := w.Body.String()
		if !strings.Contains(body, `<html lang="`+code+`">`) || w.Header().Get("Content-Language") != code {
			t.Errorf("%s: the page does not say its language", code)
		}
		if !strings.Contains(body, `href="/_lang/tr"`) && code == "en" || !strings.Contains(body, `href="/_lang/en"`) && code == "tr" {
			t.Errorf("%s: the other language is not offered", code)
		}
		if strings.Contains(body, `href="/_lang/`+code+`"`) {
			t.Errorf("%s: the language in use is offered as a link", code)
		}
	}
}

// A catalogue with a key twice keeps one silently in JSON; ours refuses.
func TestACatalogueWithAKeyTwiceIsRefused(t *testing.T) {
	if _, err := decodeCatalogue([]byte(`{"a": "x", "a": "y"}`)); err == nil {
		t.Error("a key twice was accepted")
	}
	if m, err := decodeCatalogue([]byte(`{"a": "x", "b": "y"}`)); err != nil || len(m) != 2 {
		t.Errorf("%v %v", m, err)
	}
}

// The docs portal's own pages are translated; the documents themselves are
// English, and a Turkish reader is told so rather than left to find out.
func TestTheDocsPortalSpeaksTurkishAroundEnglishDocuments(t *testing.T) {
	r := httptest.NewRequest(http.MethodGet, "/docs", nil)
	r.Header.Set("Accept-Language", "tr")
	w := httptest.NewRecorder()
	Docs("v1", true)(w, r)
	body := w.Body.String()
	for _, want := range []string{`<html lang="tr">`, "llmgw belgeleri", "Tüm belgeler", "yalnızca İngilizcedir", `href="/_lang/en"`} {
		if !strings.Contains(body, want) {
			t.Errorf("the Turkish docs index lacks %q", want)
		}
	}
	r = httptest.NewRequest(http.MethodGet, "/docs", nil)
	w = httptest.NewRecorder()
	Docs("v1", true)(w, r)
	if strings.Contains(w.Body.String(), "İngilizce") || !strings.Contains(w.Body.String(), "Every document") {
		t.Error("the English docs index is not as it was")
	}
}