The commonest convention of all, and the one the extractor missed. Asked for a two-file Go server, qwen2.5-coder-3b opened a plain ```go fence and wrote "// cmd/server/main.go" as the first line of the file — not on the fence, not above it. Both files downloaded as snippet-1.go and snippet-2.go, which is the feature failing at the first real use of it. Found by running it rather than by reading it: the unit tests covered the conventions I had thought of, and passed throughout. Only the first line or two, only when the line is a comment holding one token that looks like a path. "// this reads config.json at startup" is a sentence, "// Package main implements the server." is a doc comment, and a path comment further down the file is just a comment. The markers cover the languages people actually ask for, including block comments closed on the same line. The comment stays in the body. People write a path at the top of a file anyway, and editing model output to make a filename look tidier is a surprise nobody asked for. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
45d0dee470
commit
c174e46902
2 files changed
+103
No files matched your search
@@ -135,3 +135,59 @@ func TestTheNumberOfFilesIsBounded(t *testing.T) {
|
||||
t.Errorf("returned %d files, over the %d cap", got, maxCodeFiles)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPathCommentInsideTheBlockNamesTheFile(t *testing.T) {
|
||||
// Exactly what qwen2.5-coder-3b returned when asked for a two-file Go
|
||||
// server: a plain ```go fence with the path as the first line of the file.
|
||||
// Both files landed as snippet-1.go and snippet-2.go before this.
|
||||
reply := "```go\n// cmd/server/main.go\n\npackage main\n\nfunc main() {}\n```\n\n" +
|
||||
"```go\n// internal/handler/handler.go\n\npackage handler\n```\n"
|
||||
files := ExtractCode(reply)
|
||||
if len(files) != 2 {
|
||||
t.Fatalf("got %d files", len(files))
|
||||
}
|
||||
if files[0].Path != "cmd/server/main.go" || files[1].Path != "internal/handler/handler.go" {
|
||||
t.Fatalf("paths lost: %q, %q", files[0].Path, files[1].Path)
|
||||
}
|
||||
// The comment stays: people write those anyway, and rewriting model output
|
||||
// to tidy a filename is a surprise nobody asked for.
|
||||
if !strings.Contains(files[0].Body, "// cmd/server/main.go") {
|
||||
t.Error("the path comment was stripped from the body")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathCommentsAcrossLanguages(t *testing.T) {
|
||||
for _, tc := range []struct{ block, want string }{
|
||||
{"```python\n# scripts/run.py\nprint(1)\n```\n", "scripts/run.py"},
|
||||
{"```sql\n-- db/schema.sql\nSELECT 1;\n```\n", "db/schema.sql"},
|
||||
{"```html\n<!-- web/index.html -->\n<p>hi</p>\n```\n", "web/index.html"},
|
||||
{"```c\n/* src/main.c */\nint main(){}\n```\n", "src/main.c"},
|
||||
{"```go\n\n// pkg/x.go\npackage x\n```\n", "pkg/x.go"},
|
||||
} {
|
||||
files := ExtractCode(tc.block)
|
||||
if len(files) != 1 {
|
||||
t.Fatalf("%q: got %d files", tc.block, len(files))
|
||||
}
|
||||
if files[0].Path != tc.want {
|
||||
t.Errorf("%q: path %q, want %q", tc.block, files[0].Path, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOrdinaryCommentIsNotAPath(t *testing.T) {
|
||||
// A sentence that happens to mention a file is not a filename, and a
|
||||
// comment below the first couple of lines is just a comment.
|
||||
for _, block := range []string{
|
||||
"```go\n// this reads config.json at startup\npackage main\n```\n",
|
||||
"```go\npackage main\n\n// internal/x.go\nfunc main(){}\n```\n",
|
||||
"```go\n// Package main implements the server.\npackage main\n```\n",
|
||||
} {
|
||||
files := ExtractCode(block)
|
||||
if len(files) != 1 {
|
||||
t.Fatalf("got %d files", len(files))
|
||||
}
|
||||
if files[0].Path != "snippet-1.go" {
|
||||
t.Errorf("%q was read as a filename in:\n%s", files[0].Path, block)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user