package train import ( "bufio" "bytes" "encoding/csv" "encoding/json" "errors" "fmt" "path" "strings" "unicode/utf8" "gitea.mynodes.xyz/ilker/llmgw/internal/kb/extract" ) // Pairs from a file a person uploads: a spreadsheet of questions and // answers (Excel, CSV, tab-separated) or llmgw's own NDJSON of {"q", "a"}. // The columns are found by their header — Soru/Cevap, Question/Answer and // the like — or are the first two when the first row is not a header. // Pair is one question and the answer to learn for it. type Pair struct { Q string `json:"q"` A string `json:"a"` } // Read is what a file gave. type Read struct { Pairs []Pair // Skipped counts rows with a question and no answer, or the other way. Skipped int // Columns names the columns read, as the header has them; empty for // NDJSON and for a table with no header. Columns [2]string } // MaxPairs bounds one file: past it a job trains for hours on a page that // was meant to take minutes. const MaxPairs = 20000 var ( questionNames = []string{"soru", "sorular", "question", "questions", "q", "prompt", "instruction", "input", "girdi"} answerNames = []string{"cevap", "cevaplar", "yanıt", "yanit", "answer", "answers", "a", "response", "output", "completion", "çıktı", "cikti"} ) // ReadPairs reads a file's pairs; name says what kind it is. func ReadPairs(name string, b []byte) (Read, error) { var rows [][]string switch strings.ToLower(path.Ext(name)) { case ".xlsx", ".xlsm": r, err := extract.XLSXRows(b) if err != nil { return Read{}, fmt.Errorf("the Excel file could not be read: %w", err) } rows = r case ".jsonl", ".ndjson", ".json": return readNDJSON(b) case ".csv", ".tsv", ".txt": r, err := readDelimited(b) if err != nil { return Read{}, err } rows = r case ".xls": return Read{}, errors.New("an old Excel file (.xls): save it as .xlsx and choose it again") default: return Read{}, errors.New("choose an Excel (.xlsx), CSV or JSONL file") } return fromRows(rows) } func readDelimited(b []byte) ([][]string, error) { b = bytes.TrimPrefix(b, []byte("\xef\xbb\xbf")) if !utf8.Valid(b) { return nil, errors.New("the file is not UTF-8 text: save it as \"CSV UTF-8\"") } first, _, _ := strings.Cut(string(b), "\n") // Whichever separator the header line has most of: Excel writes ";" // where the decimal mark is a comma, as it is in Turkish. sep, most := ',', strings.Count(first, ",") for _, c := range []rune{';', '\t'} { if n := strings.Count(first, string(c)); n > most { sep, most = c, n } } r := csv.NewReader(bytes.NewReader(b)) r.Comma, r.FieldsPerRecord, r.LazyQuotes = sep, -1, true rows, err := r.ReadAll() if err != nil { return nil, fmt.Errorf("the file could not be read as a table: %w", err) } return rows, nil } func fromRows(rows [][]string) (Read, error) { var out Read qc, ac := 0, 1 if len(rows) > 0 { q, a := -1, -1 for i, c := range rows[0] { n := strings.ToLower(strings.TrimSpace(c)) switch { case q < 0 && contains(questionNames, n): q = i case a < 0 && contains(answerNames, n): a = i } } if q >= 0 && a >= 0 { qc, ac = q, a out.Columns = [2]string{strings.TrimSpace(rows[0][q]), strings.TrimSpace(rows[0][a])} rows = rows[1:] } } for _, r := range rows { var q, a string if qc < len(r) { q = strings.TrimSpace(r[qc]) } if ac < len(r) { a = strings.TrimSpace(r[ac]) } switch { case q == "" && a == "": case q == "" || a == "": out.Skipped++ default: out.Pairs = append(out.Pairs, Pair{Q: q, A: a}) } } return out, check(out) } func contains(names []string, n string) bool { for _, x := range names { if x == n { return true } } return false } func readNDJSON(b []byte) (Read, error) { var out Read sc := bufio.NewScanner(bytes.NewReader(b)) sc.Buffer(make([]byte, 64<<10), 1<<20) line := 0 for sc.Scan() { line++ l := bytes.TrimSpace(sc.Bytes()) if len(l) == 0 { continue } var p struct { Q, A string Question, Answer string Prompt, Completion string } if err := json.Unmarshal(l, &p); err != nil { return Read{}, fmt.Errorf("line %d is not JSON: %v", line, err) } q, a := first(p.Q, p.Question, p.Prompt), first(p.A, p.Answer, p.Completion) if q == "" || a == "" { out.Skipped++ continue } out.Pairs = append(out.Pairs, Pair{Q: q, A: a}) } if err := sc.Err(); err != nil { return Read{}, err } return out, check(out) } func first(s ...string) string { for _, x := range s { if x = strings.TrimSpace(x); x != "" { return x } } return "" } func check(r Read) error { switch { case len(r.Pairs) == 0: return errors.New("no question and answer found: the first column is the question and the second the answer, " + "or columns headed Soru and Cevap") case len(r.Pairs) > MaxPairs: return fmt.Errorf("%d pairs, more than the %d one training takes", len(r.Pairs), MaxPairs) } return nil } // NDJSON is the pairs as a job's data. func NDJSON(pairs []Pair) string { var b strings.Builder enc := json.NewEncoder(&b) enc.SetEscapeHTML(false) for _, p := range pairs { _ = enc.Encode(p) } return b.String() }