package train import ( "strings" "testing" ) func TestPairsFromATableByItsHeaderOrItsFirstColumns(t *testing.T) { for _, c := range []struct { name, file string want []Pair skipped int cols [2]string }{ {"headed.csv", "No;Cevap;Soru\n1;Yirmi gün.;İzin kaç gün?\n2;;Boş cevap\n3;Evet;Uzaktan çalışabilir miyim?\n", []Pair{{"İzin kaç gün?", "Yirmi gün."}, {"Uzaktan çalışabilir miyim?", "Evet"}}, 1, [2]string{"Soru", "Cevap"}}, {"plain.csv", "\ufeffİzin kaç gün?,\"Yirmi gün, yılda.\"\n\nMasraf?,Formla.\n", []Pair{{"İzin kaç gün?", "Yirmi gün, yılda."}, {"Masraf?", "Formla."}}, 0, [2]string{}}, {"tabs.tsv", "Question\tAnswer\nA?\tB.\n", []Pair{{"A?", "B."}}, 0, [2]string{"Question", "Answer"}}, {"d.jsonl", `{"q":"A?","a":"B."}` + "\n" + `{"question":"C?","answer":"D."}` + "\n" + `{"q":"E?"}` + "\n", []Pair{{"A?", "B."}, {"C?", "D."}}, 1, [2]string{}}, } { got, err := ReadPairs(c.name, []byte(c.file)) if err != nil { t.Errorf("%s: %v", c.name, err) continue } if len(got.Pairs) != len(c.want) || got.Skipped != c.skipped || got.Columns != c.cols { t.Errorf("%s: %+v", c.name, got) continue } for i := range c.want { if got.Pairs[i] != c.want[i] { t.Errorf("%s: pair %d = %+v, want %+v", c.name, i, got.Pairs[i], c.want[i]) } } } } func TestPairsRefusedWithAReason(t *testing.T) { for name, file := range map[string]string{ "a.csv": "only one column\n", "a.xls": "x", "a.pdf": "x", "a.json": "not json\n", } { if _, err := ReadPairs(name, []byte(file)); err == nil { t.Errorf("%s read", name) } } var b strings.Builder for range MaxPairs + 1 { b.WriteString("x,y\n") } if _, err := ReadPairs("big.csv", []byte(b.String())); err == nil || !strings.Contains(err.Error(), "more than") { t.Errorf("too many: %v", err) } } func TestNDJSONIsWhatAJobReads(t *testing.T) { got := NDJSON([]Pair{{"?", "a & b"}}) if got != `{"q":"?","a":"a & b"}`+"\n" { t.Errorf("%q", got) } }