diff --git a/internal/router/chat.go b/internal/router/chat.go index 5cc39b3f..aebf9bdb 100644 --- a/internal/router/chat.go +++ b/internal/router/chat.go @@ -30,6 +30,13 @@ func (r *Router) registerChat(mux *http.ServeMux, pages *web.AccountPages) { web.Chat(user, r.modelsFor(req, user), version.Version)(w, req) })) mux.HandleFunc("POST /chat/send", pages.Guard(r.chatSend)) + // Behind the same guard as the rest of the page: it takes a POST body and + // returns a file, so it wants a session and the CSRF check for the same + // reasons /chat/send does. + mux.HandleFunc("POST /chat/download", pages.Guard( + func(w http.ResponseWriter, req *http.Request, _ store.User) { + web.CodeArchive(r.cfg.MaxBodyBytes)(w, req) + })) } // chatSend dispatches one turn on behalf of a signed-in person. diff --git a/internal/web/codeblocks.go b/internal/web/codeblocks.go new file mode 100644 index 00000000..25ce0698 --- /dev/null +++ b/internal/web/codeblocks.go @@ -0,0 +1,276 @@ +package web + +import ( + "fmt" + "path" + "strings" +) + +// CodeFile is one fenced block from a reply, with the path it should be +// written to. +type CodeFile struct { + Path string + Body string +} + +// maxCodeFiles and maxCodeBytes bound what one reply can turn into. +// +// The reply is already bounded by the token cap, so these are not the real +// defence; they are there so a change to that cap, or a model that emits +// something pathological, cannot turn a download into an archive nobody asked +// for. +const ( + maxCodeFiles = 64 + maxCodeBytes = 4 << 20 +) + +// ExtractCode pulls fenced code blocks out of a model reply and decides where +// each one belongs. +// +// The filename is whatever the model said, and models say it in several ways: +// on the fence itself (```go:main.go, ```main.go), or on the line above the +// fence as a heading or a comment. All of them are guesses, and a block whose +// name cannot be worked out still comes back — under a generated name, because +// dropping someone's code because it was unlabelled is worse than naming it +// badly. +func ExtractCode(reply string) []CodeFile { + lines := strings.Split(reply, "\n") + var out []CodeFile + used := map[string]int{} + + for i := 0; i < len(lines); i++ { + fence, info, ok := openingFence(lines[i]) + if !ok { + continue + } + var body []string + j := i + 1 + for ; j < len(lines); j++ { + if isClosingFence(lines[j], fence) { + break + } + body = append(body, lines[j]) + } + + lang, named := splitInfo(info) + if named == "" { + named = nameFromNearbyLine(lines, i) + } + p := safePath(named) + if p == "" { + p = fmt.Sprintf("snippet-%d%s", len(out)+1, extensionFor(lang)) + } + // Two blocks claiming one path would silently lose the first. Models + // do this when they show a file, then show it again with one line + // changed. + if n := used[p]; n > 0 { + ext := path.Ext(p) + p = fmt.Sprintf("%s-%d%s", strings.TrimSuffix(p, ext), n+1, ext) + } + used[p]++ + + out = append(out, CodeFile{Path: p, Body: strings.Join(body, "\n") + "\n"}) + if len(out) >= maxCodeFiles { + break + } + i = j + } + + total := 0 + for i := range out { + total += len(out[i].Body) + if total > maxCodeBytes { + return out[:i] + } + } + return out +} + +// openingFence recognises ``` and ~~~ fences of three characters or more, +// returning the fence itself so the matching close can be found. +func openingFence(line string) (fence, info string, ok bool) { + t := strings.TrimSpace(line) + for _, c := range []string{"`", "~"} { + if !strings.HasPrefix(t, strings.Repeat(c, 3)) { + continue + } + n := 0 + for n < len(t) && string(t[n]) == c { + n++ + } + info = strings.TrimSpace(t[n:]) + // An info string containing a backtick is not a fence at all; that is + // inline code with backticks in it. + if c == "`" && strings.Contains(info, "`") { + return "", "", false + } + return t[:n], info, true + } + return "", "", false +} + +func isClosingFence(line, fence string) bool { + t := strings.TrimSpace(line) + return strings.HasPrefix(t, fence) && strings.TrimRight(t, string(fence[0])) == "" +} + +// splitInfo separates the language from a filename on the fence line. +// +// Handles ```go:path/main.go, ```go path/main.go, ```path/main.go and bare +// ```go. A token is treated as a name rather than a language when it looks +// like one: it has a dot or a slash in it. +func splitInfo(info string) (lang, name string) { + if info == "" { + return "", "" + } + if l, n, found := strings.Cut(info, ":"); found { + return strings.TrimSpace(l), strings.TrimSpace(n) + } + fields := strings.Fields(info) + if len(fields) == 0 { + return "", "" + } + first := fields[0] + // Strip the attribute forms some tools use: title="x", file=x. + for _, f := range fields[1:] { + for _, k := range []string{"title=", "file=", "filename=", "name="} { + if v, ok := strings.CutPrefix(f, k); ok { + return first, strings.Trim(v, `"'`) + } + } + } + if looksLikePath(first) { + return "", first + } + if len(fields) > 1 && looksLikePath(fields[1]) { + return first, fields[1] + } + return first, "" +} + +func looksLikePath(s string) bool { + if strings.Contains(s, "/") || strings.Contains(s, ".") { + return true + } + // Files whose whole name is their type. ```Dockerfile is a language tag to + // a syntax highlighter and a filename to everyone else, and the reader + // wants a file called Dockerfile either way. + switch s { + case "Dockerfile", "Makefile", "Jenkinsfile", "Vagrantfile", "Procfile", "Gemfile", "Rakefile", "CMakeLists": + return true + } + return false +} + +// nameFromNearbyLine looks just above the fence for a filename. +// +// Models very often write the path as a short line of its own — as a heading, +// bold, in backticks, or as a comment — and then open the fence. Only the two +// lines immediately above are considered, because anything further away is +// prose that happens to mention a file. +func nameFromNearbyLine(lines []string, fence int) string { + for k := fence - 1; k >= 0 && k >= fence-2; k-- { + t := strings.TrimSpace(lines[k]) + if t == "" { + continue + } + t = strings.Trim(t, "*_#:` ") + t = strings.TrimPrefix(t, "//") + t = strings.TrimPrefix(t, "#") + for _, k := range []string{"File:", "file:", "FILE:", "Filename:", "filename:"} { + t = strings.TrimSpace(strings.TrimPrefix(t, k)) + } + t = strings.TrimSpace(t) + // One token that looks like a path. A sentence mentioning main.go is + // not a filename line. + if fields := strings.Fields(t); len(fields) == 1 && looksLikePath(fields[0]) { + return fields[0] + } + return "" + } + return "" +} + +// safePath turns a model-supplied name into something safe to write inside an +// archive, or "" when it cannot. +// +// This is the part that matters. The name comes from a language model and goes +// into a zip that someone will extract, so "../../.ssh/authorized_keys" and +// "/etc/passwd" and "C:\\Windows\\..." are all things to expect rather than +// things to hope against. Everything is forced to a relative path under the +// archive root, and anything that still escapes is rejected outright rather +// than repaired into something surprising. +func safePath(name string) string { + name = strings.TrimSpace(name) + if name == "" { + return "" + } + // Windows separators and drive letters, before anything else looks at it. + name = strings.ReplaceAll(name, `\`, "/") + if len(name) > 1 && name[1] == ':' { + name = name[2:] + } + name = strings.TrimPrefix(name, "/") + + cleaned := path.Clean(name) + if cleaned == "." || cleaned == "/" { + return "" + } + if strings.HasPrefix(cleaned, "../") || cleaned == ".." || strings.HasPrefix(cleaned, "/") { + return "" + } + for _, seg := range strings.Split(cleaned, "/") { + if seg == "" || seg == "." || seg == ".." { + return "" + } + // A NUL or a newline in a path is not a filename anyone meant. + if strings.ContainsAny(seg, "\x00\n\r") { + return "" + } + if len(seg) > 128 { + return "" + } + } + if len(cleaned) > 512 || strings.Count(cleaned, "/") > 16 { + return "" + } + return cleaned +} + +// extensionFor gives an unnamed block a plausible extension from its language. +func extensionFor(lang string) string { + switch strings.ToLower(lang) { + case "go": + return ".go" + case "python", "py": + return ".py" + case "javascript", "js": + return ".js" + case "typescript", "ts": + return ".ts" + case "rust", "rs": + return ".rs" + case "c": + return ".c" + case "cpp", "c++": + return ".cpp" + case "java": + return ".java" + case "sh", "bash", "shell": + return ".sh" + case "sql": + return ".sql" + case "yaml", "yml": + return ".yaml" + case "json": + return ".json" + case "html": + return ".html" + case "css": + return ".css" + case "": + return ".txt" + default: + return "." + strings.ToLower(lang) + } +} diff --git a/internal/web/codeblocks_test.go b/internal/web/codeblocks_test.go new file mode 100644 index 00000000..5a409726 --- /dev/null +++ b/internal/web/codeblocks_test.go @@ -0,0 +1,137 @@ +package web + +import ( + "strings" + "testing" +) + +func TestSafePathRefusesEverythingThatEscapes(t *testing.T) { + // The name comes from a language model and goes into an archive somebody + // extracts. These are things to expect, not things to hope against. + for _, bad := range []string{ + "../secrets.txt", + "../../.ssh/authorized_keys", + "a/../../b", + "..", + ".", + "", + " ", + "a/./../../b", + "foo\x00.go", + "foo\n../bar", + strings.Repeat("a", 200) + ".go", + strings.Repeat("a/", 20) + "x.go", + } { + if got := safePath(bad); got != "" { + t.Errorf("safePath(%q) = %q; it must be refused", bad, got) + } + } +} + +func TestSafePathKeepsOrdinaryNames(t *testing.T) { + for in, want := range map[string]string{ + "main.go": "main.go", + "cmd/server/main.go": "cmd/server/main.go", + "./pkg/x.go": "pkg/x.go", + "/src/app.py": "src/app.py", + `src\app.py`: "src/app.py", + "a/b/../c.txt": "a/c.txt", + "Dockerfile": "Dockerfile", + "deploy/k8s/deploy.yml": "deploy/k8s/deploy.yml", + } { + if got := safePath(in); got != want { + t.Errorf("safePath(%q) = %q, want %q", in, got, want) + } + } +} + +func TestExtractNamesFilesTheWayModelsWriteThem(t *testing.T) { + reply := "Here you go.\n\n" + + "```go:cmd/api/main.go\npackage main\n```\n\n" + + "**internal/store/store.go**\n" + + "```go\npackage store\n```\n\n" + + "```python scripts/run.py\nprint(1)\n```\n\n" + + "```Dockerfile\nFROM scratch\n```\n" + + files := ExtractCode(reply) + if len(files) != 4 { + t.Fatalf("got %d files: %+v", len(files), files) + } + want := []string{"cmd/api/main.go", "internal/store/store.go", "scripts/run.py", "Dockerfile"} + for i, w := range want { + if files[i].Path != w { + t.Errorf("file %d: path %q, want %q", i, files[i].Path, w) + } + } + if !strings.Contains(files[0].Body, "package main") { + t.Errorf("body lost: %q", files[0].Body) + } +} + +func TestAnUnnamedBlockIsStillReturned(t *testing.T) { + // Losing someone's code because the model did not label it is worse than + // naming it badly. + files := ExtractCode("```go\npackage main\n```\n```\nplain text\n```\n") + if len(files) != 2 { + t.Fatalf("got %d files", len(files)) + } + if files[0].Path != "snippet-1.go" || files[1].Path != "snippet-2.txt" { + t.Errorf("generated names wrong: %q, %q", files[0].Path, files[1].Path) + } +} + +func TestAnEscapingNameFallsBackRatherThanBeingRepaired(t *testing.T) { + // Silently rewriting ../../x.go into x.go would put a file somewhere the + // reader did not expect. It gets a generated name instead. + files := ExtractCode("```go:../../evil.go\npackage main\n```\n") + if len(files) != 1 { + t.Fatalf("got %d files", len(files)) + } + if strings.Contains(files[0].Path, "..") { + t.Fatalf("an escaping path survived: %q", files[0].Path) + } + if files[0].Path != "snippet-1.go" { + t.Errorf("path = %q, want the generated name", files[0].Path) + } +} + +func TestTwoBlocksClaimingOnePathBothSurvive(t *testing.T) { + // Models show a file, then show it again with one line changed. + files := ExtractCode("```go:main.go\nv1\n```\n```go:main.go\nv2\n```\n") + if len(files) != 2 { + t.Fatalf("got %d files", len(files)) + } + if files[0].Path == files[1].Path { + t.Fatalf("one overwrote the other: both are %q", files[0].Path) + } + if files[1].Path != "main-2.go" { + t.Errorf("second path = %q, want main-2.go", files[1].Path) + } +} + +func TestProseMentioningAFileIsNotAFilename(t *testing.T) { + reply := "You should now edit main.go and rebuild.\n```go\npackage main\n```\n" + files := ExtractCode(reply) + if len(files) != 1 { + t.Fatalf("got %d files", len(files)) + } + if files[0].Path != "snippet-1.go" { + t.Errorf("a sentence was read as a filename: %q", files[0].Path) + } +} + +func TestATextReplyWithNoCodeYieldsNothing(t *testing.T) { + if files := ExtractCode("No code here, just words about `main.go`.\n"); len(files) != 0 { + t.Errorf("got %d files from prose: %+v", len(files), files) + } +} + +func TestTheNumberOfFilesIsBounded(t *testing.T) { + var b strings.Builder + for range maxCodeFiles + 20 { + b.WriteString("```go\nx\n```\n") + } + if got := len(ExtractCode(b.String())); got > maxCodeFiles { + t.Errorf("returned %d files, over the %d cap", got, maxCodeFiles) + } +} diff --git a/internal/web/download.go b/internal/web/download.go new file mode 100644 index 00000000..8e3f828f --- /dev/null +++ b/internal/web/download.go @@ -0,0 +1,89 @@ +package web + +import ( + "archive/zip" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// CodeArchive turns one reply into a zip of the files it contains. +// +// Server-side because the interesting part is path handling, and a filename +// chosen by a language model is exactly the kind of input that wants +// archive/zip and a tested sanitiser rather than string concatenation in a +// browser. Nothing is stored: the reply arrives, the zip goes back, and +// neither is kept. +func CodeArchive(maxBody int64) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + noStore(w) + + var in struct { + Content string `json:"content"` + Name string `json:"name"` + } + body, err := io.ReadAll(io.LimitReader(r.Body, maxBody)) + if err != nil || json.Unmarshal(body, &in) != nil { + http.Error(w, "could not read the request", http.StatusBadRequest) + return + } + + files := ExtractCode(in.Content) + if len(files) == 0 { + http.Error(w, "there is no code in that reply to download", http.StatusBadRequest) + return + } + + // Headers before the first write: once a byte of zip is out, an error + // can only be delivered as a corrupt download. + w.Header().Set("Content-Type", "application/zip") + w.Header().Set("Content-Disposition", + fmt.Sprintf("attachment; filename=%q", archiveName(in.Name))) + // The archive is assembled from model output; nothing should sniff it + // as something else. + w.Header().Set("X-Content-Type-Options", "nosniff") + + zw := zip.NewWriter(w) + for _, f := range files { + hdr := &zip.FileHeader{Name: f.Path, Method: zip.Deflate, Modified: time.Now()} + hdr.SetMode(0o644) + out, err := zw.CreateHeader(hdr) + if err != nil { + // Half a zip is already on the wire; closing is the only + // honest end, and the browser will report a broken download. + _ = zw.Close() + return + } + if _, err := io.WriteString(out, f.Body); err != nil { + _ = zw.Close() + return + } + } + _ = zw.Close() + } +} + +// archiveName keeps the download recognisable without trusting what it is given. +func archiveName(name string) string { + name = strings.TrimSpace(name) + keep := strings.Map(func(r rune) rune { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + return r + case r == '-', r == '_': + return r + default: + return -1 + } + }, name) + if keep == "" { + keep = "code" + } + if len(keep) > 40 { + keep = keep[:40] + } + return keep + ".zip" +} diff --git a/internal/web/download_test.go b/internal/web/download_test.go new file mode 100644 index 00000000..d8df5d2f --- /dev/null +++ b/internal/web/download_test.go @@ -0,0 +1,135 @@ +package web + +import ( + "archive/zip" + "bytes" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func zipOf(t *testing.T, reply string) (*zip.Reader, *httptest.ResponseRecorder) { + t.Helper() + body := `{"content":` + jsonString(reply) + `,"name":"code"}` + r := httptest.NewRequest(http.MethodPost, "http://x/chat/download", strings.NewReader(body)) + w := httptest.NewRecorder() + CodeArchive(1<<20)(w, r) + if w.Code != http.StatusOK { + return nil, w + } + zr, err := zip.NewReader(bytes.NewReader(w.Body.Bytes()), int64(w.Body.Len())) + if err != nil { + t.Fatalf("the response is not a readable zip: %v", err) + } + return zr, w +} + +func jsonString(s string) string { + var b strings.Builder + b.WriteByte('"') + for _, r := range s { + switch r { + case '"': + b.WriteString(`\"`) + case '\\': + b.WriteString(`\\`) + case '\n': + b.WriteString(`\n`) + case '\t': + b.WriteString(`\t`) + default: + b.WriteRune(r) + } + } + b.WriteByte('"') + return b.String() +} + +func TestTheArchiveKeepsDirectories(t *testing.T) { + // The point of the feature: a reply is often a small tree, not one file. + zr, _ := zipOf(t, "```go:cmd/api/main.go\npackage main\n```\n"+ + "```go:internal/store/store.go\npackage store\n```\n"+ + "```yaml:deploy/k8s/deployment.yaml\nkind: Deployment\n```\n") + got := map[string]string{} + for _, f := range zr.File { + rc, err := f.Open() + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(rc) + rc.Close() + got[f.Name] = string(b) + } + for _, want := range []string{"cmd/api/main.go", "internal/store/store.go", "deploy/k8s/deployment.yaml"} { + if _, ok := got[want]; !ok { + t.Errorf("missing %s; archive has %v", want, keys(got)) + } + } + if !strings.Contains(got["cmd/api/main.go"], "package main") { + t.Errorf("contents lost: %q", got["cmd/api/main.go"]) + } +} + +func keys(m map[string]string) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} + +func TestNoEntryEscapesTheArchiveRoot(t *testing.T) { + // Every name in this archive was written by a language model, and someone + // is going to extract it with a tool that trusts the archive. + zr, _ := zipOf(t, "```go:../../.ssh/authorized_keys\nssh-rsa AAAA\n```\n"+ + "```go:/etc/cron.d/x\n* * * * * root sh\n```\n"+ + "```go:a/../../../b.go\npackage b\n```\n") + if zr == nil { + t.Fatal("nothing was produced") + } + for _, f := range zr.File { + if strings.HasPrefix(f.Name, "/") || strings.HasPrefix(f.Name, "../") || + strings.Contains(f.Name, "/../") || strings.Contains(f.Name, `\`) { + t.Errorf("entry escapes the root: %q", f.Name) + } + } +} + +func TestAReplyWithNoCodeIsRefusedRatherThanEmpty(t *testing.T) { + // An empty zip downloads successfully and tells the reader nothing. + _, w := zipOf(t, "There is no code here, only prose.") + if w.Code != http.StatusBadRequest { + t.Errorf("status %d; a reply with no code should be refused", w.Code) + } +} + +func TestTheDownloadIsSentAsAnAttachment(t *testing.T) { + _, w := zipOf(t, "```go:main.go\npackage main\n```\n") + if ct := w.Header().Get("Content-Type"); ct != "application/zip" { + t.Errorf("Content-Type %q", ct) + } + if cd := w.Header().Get("Content-Disposition"); !strings.HasPrefix(cd, "attachment;") { + t.Errorf("Content-Disposition %q; it must not render in the tab", cd) + } + if w.Header().Get("X-Content-Type-Options") != "nosniff" { + t.Error("the archive may be sniffed as another type") + } +} + +func TestTheArchiveNameIsNotTakenOnTrust(t *testing.T) { + for in, want := range map[string]string{ + "code": "code.zip", + "": "code.zip", + "../../etc/passwd": "etcpasswd.zip", + // Hyphens are kept on purpose, so "rm -rf" survives as "rm-rf". The + // point of this case is the quotes and slashes, which do not. + `a"; rm -rf /; x="`: "arm-rfx.zip", + strings.Repeat("x", 100): strings.Repeat("x", 40) + ".zip", + } { + if got := archiveName(in); got != want { + t.Errorf("archiveName(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/web/static/base.css b/internal/web/static/base.css index 419394ba..a6fd2a36 100644 --- a/internal/web/static/base.css +++ b/internal/web/static/base.css @@ -108,3 +108,5 @@ select { font: inherit; padding: .35rem .4rem; border: 1px solid var(--line); .bubble.you .katex { color: inherit; } /* A reply that stopped because it ran out of room, not because it was done. */ .note.truncated { color: var(--warn, #a15c00); margin: .2rem 0 .8rem; } +/* Offered under a reply that contains files worth keeping. */ +.link.download { display: inline-block; margin: .3rem 0 .8rem; } diff --git a/internal/web/templates/chat.html b/internal/web/templates/chat.html index 15b22fdc..c2572e6d 100644 --- a/internal/web/templates/chat.html +++ b/internal/web/templates/chat.html @@ -117,6 +117,39 @@ access to it.

} catch (e) { /* a reply without maths is better than no reply */ } } + // Fetched rather than linked, because the archive is built from a POST body + // and there is no URL that names it. The blob is revoked immediately after + // the click: it is the whole reply, and leaving it alive keeps it addressable + // for as long as the tab is open. + async function downloadCode(reply, btn) { + const was = btn.textContent; + btn.disabled = true; + btn.textContent = "Preparing…"; + try { + const resp = await fetch("/chat/download", { + method: "POST", + headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf }, + body: JSON.stringify({ content: reply, name: "code" }), + }); + if (!resp.ok) { + btn.textContent = "(" + resp.status + ") " + (await resp.text()).slice(0, 120); + return; + } + const url = URL.createObjectURL(await resp.blob()); + const a = document.createElement("a"); + a.href = url; + a.download = (resp.headers.get("Content-Disposition") || "") + .match(/filename="?([^"]+)"?/)?.[1] || "code.zip"; + a.click(); + URL.revokeObjectURL(url); + btn.textContent = was; + } catch (err) { + btn.textContent = "download failed"; + } finally { + btn.disabled = false; + } + } + form.onsubmit = async (e) => { e.preventDefault(); const text = input.value.trim(); @@ -173,6 +206,18 @@ access to it.

} if (reply) { messages.push({ role: "assistant", content: reply }); + // A reply with fenced code gets a download. The button carries the + // reply text; the server decides what the files are called, because + // those names come from the model and sanitising them is not a job + // for string handling in a page. + if (/^\s*(```|~~~)/m.test(reply)) { + const dl = document.createElement("button"); + dl.type = "button"; + dl.className = "link download"; + dl.textContent = "Download as .zip"; + dl.onclick = () => downloadCode(reply, dl); + out.after(dl); + } if (finish === "length") { // Appended as a sibling, never into the reply: out holds model text // placed with textContent, and this is ours.