The commonest convention of all, and the one the extractor missed. Asked for a two-file Go server, qwen2.5-coder-3b opened a plain ```go fence and wrote "// cmd/server/main.go" as the first line of the file — not on the fence, not above it. Both files downloaded as snippet-1.go and snippet-2.go, which is the feature failing at the first real use of it. Found by running it rather than by reading it: the unit tests covered the conventions I had thought of, and passed throughout. Only the first line or two, only when the line is a comment holding one token that looks like a path. "// this reads config.json at startup" is a sentence, "// Package main implements the server." is a doc comment, and a path comment further down the file is just a comment. The markers cover the languages people actually ask for, including block comments closed on the same line. The comment stays in the body. People write a path at the top of a file anyway, and editing model output to make a filename look tidier is a surprise nobody asked for. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
45d0dee470
commit
c174e46902
2 files changed
+103
No files matched your search
@@ -56,6 +56,9 @@ func ExtractCode(reply string) []CodeFile {
|
||||
if named == "" {
|
||||
named = nameFromNearbyLine(lines, i)
|
||||
}
|
||||
if named == "" {
|
||||
named = nameFromLeadingComment(body)
|
||||
}
|
||||
p := safePath(named)
|
||||
if p == "" {
|
||||
p = fmt.Sprintf("snippet-%d%s", len(out)+1, extensionFor(lang))
|
||||
@@ -191,6 +194,50 @@ func nameFromNearbyLine(lines []string, fence int) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// nameFromLeadingComment reads a path out of the first line or two of a block.
|
||||
//
|
||||
// The commonest convention of all, and the one this missed at first: the model
|
||||
// opens a plain ```go fence and writes "// cmd/server/main.go" as the first
|
||||
// line of the file. Asked for a two-file Go server, that is exactly what came
|
||||
// back, and both files landed in the archive as snippet-1.go and snippet-2.go.
|
||||
//
|
||||
// The comment is left in the body. A path comment at the top of a file is
|
||||
// something people write anyway, and editing model output to make a filename
|
||||
// look tidier is not worth the surprise.
|
||||
func nameFromLeadingComment(body []string) string {
|
||||
for n, line := range body {
|
||||
if n > 1 {
|
||||
return ""
|
||||
}
|
||||
t := strings.TrimSpace(line)
|
||||
if t == "" {
|
||||
continue
|
||||
}
|
||||
// The comment markers that start a line across the languages people
|
||||
// ask for. Block comments count only when closed on the same line.
|
||||
for _, open := range []string{"//", "#", "--", ";", "%", "<!--", "/*"} {
|
||||
rest, ok := strings.CutPrefix(t, open)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
rest = strings.TrimSuffix(strings.TrimSuffix(strings.TrimSpace(rest), "-->"), "*/")
|
||||
rest = strings.TrimSpace(rest)
|
||||
for _, k := range []string{"File:", "file:", "FILE:", "Filename:", "filename:"} {
|
||||
rest = strings.TrimSpace(strings.TrimPrefix(rest, k))
|
||||
}
|
||||
// One token that looks like a path, and nothing else on the line.
|
||||
// "// this reads config.json at startup" is a sentence.
|
||||
if fields := strings.Fields(rest); len(fields) == 1 && looksLikePath(fields[0]) {
|
||||
return fields[0]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
// A line of actual code: there is no path comment.
|
||||
return ""
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// safePath turns a model-supplied name into something safe to write inside an
|
||||
// archive, or "" when it cannot.
|
||||
//
|
||||
|
||||
@@ -135,3 +135,59 @@ func TestTheNumberOfFilesIsBounded(t *testing.T) {
|
||||
t.Errorf("returned %d files, over the %d cap", got, maxCodeFiles)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPathCommentInsideTheBlockNamesTheFile(t *testing.T) {
|
||||
// Exactly what qwen2.5-coder-3b returned when asked for a two-file Go
|
||||
// server: a plain ```go fence with the path as the first line of the file.
|
||||
// Both files landed as snippet-1.go and snippet-2.go before this.
|
||||
reply := "```go\n// cmd/server/main.go\n\npackage main\n\nfunc main() {}\n```\n\n" +
|
||||
"```go\n// internal/handler/handler.go\n\npackage handler\n```\n"
|
||||
files := ExtractCode(reply)
|
||||
if len(files) != 2 {
|
||||
t.Fatalf("got %d files", len(files))
|
||||
}
|
||||
if files[0].Path != "cmd/server/main.go" || files[1].Path != "internal/handler/handler.go" {
|
||||
t.Fatalf("paths lost: %q, %q", files[0].Path, files[1].Path)
|
||||
}
|
||||
// The comment stays: people write those anyway, and rewriting model output
|
||||
// to tidy a filename is a surprise nobody asked for.
|
||||
if !strings.Contains(files[0].Body, "// cmd/server/main.go") {
|
||||
t.Error("the path comment was stripped from the body")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathCommentsAcrossLanguages(t *testing.T) {
|
||||
for _, tc := range []struct{ block, want string }{
|
||||
{"```python\n# scripts/run.py\nprint(1)\n```\n", "scripts/run.py"},
|
||||
{"```sql\n-- db/schema.sql\nSELECT 1;\n```\n", "db/schema.sql"},
|
||||
{"```html\n<!-- web/index.html -->\n<p>hi</p>\n```\n", "web/index.html"},
|
||||
{"```c\n/* src/main.c */\nint main(){}\n```\n", "src/main.c"},
|
||||
{"```go\n\n// pkg/x.go\npackage x\n```\n", "pkg/x.go"},
|
||||
} {
|
||||
files := ExtractCode(tc.block)
|
||||
if len(files) != 1 {
|
||||
t.Fatalf("%q: got %d files", tc.block, len(files))
|
||||
}
|
||||
if files[0].Path != tc.want {
|
||||
t.Errorf("%q: path %q, want %q", tc.block, files[0].Path, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOrdinaryCommentIsNotAPath(t *testing.T) {
|
||||
// A sentence that happens to mention a file is not a filename, and a
|
||||
// comment below the first couple of lines is just a comment.
|
||||
for _, block := range []string{
|
||||
"```go\n// this reads config.json at startup\npackage main\n```\n",
|
||||
"```go\npackage main\n\n// internal/x.go\nfunc main(){}\n```\n",
|
||||
"```go\n// Package main implements the server.\npackage main\n```\n",
|
||||
} {
|
||||
files := ExtractCode(block)
|
||||
if len(files) != 1 {
|
||||
t.Fatalf("got %d files", len(files))
|
||||
}
|
||||
if files[0].Path != "snippet-1.go" {
|
||||
t.Errorf("%q was read as a filename in:\n%s", files[0].Path, block)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user