Files
llmgw/internal/kb/extract/ooxml.go
T
Ilker ManapandClaude Opus 5.5 a53b2f4be9 Customise a model from the portal: a file, a model, a button
A page for anyone in a project, built for the one who knows least: an
Excel, CSV or JSONL file of questions and answers, previewed; a model; a
name; who may use it; and a button. The weights to train on are found
from where the model's came (a GGUF repository's base model), the card's
memory from their size, the node as the one with the most free. When it
is done the adapter is published by itself (a llama.cpp model's replicas
replaced to load it) and is a model of its own in the chat and the API.

The project's by default; its members share it wider (confirmed, and
never wider than the knowledge bases its pairs came from) or delete it.
One trains at a time per project. ctl train start needs only --model and
--data now, and --publish publishes when done.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 11:31:12 +01:00

555 lines
13 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package extract
import (
"archive/zip"
"bytes"
"encoding/xml"
"errors"
"fmt"
"io"
"path"
"regexp"
"slices"
"sort"
"strconv"
"strings"
)
// maxMember is the most one member of an Office file or EPUB is inflated
// to: a zip bomb stops here rather than at the machine's memory.
const maxMember = 512 << 20
func openZip(b []byte) (*zip.Reader, error) {
return zip.NewReader(bytes.NewReader(b), int64(len(b)))
}
// member reads one file inside a zip, by its exact name; nil when absent.
func member(zr *zip.Reader, name string) ([]byte, error) {
for _, f := range zr.File {
if f.Name == name {
return readMember(f)
}
}
return nil, nil
}
func readMember(f *zip.File) ([]byte, error) {
rc, err := f.Open()
if err != nil {
return nil, err
}
defer rc.Close()
b, err := io.ReadAll(io.LimitReader(rc, maxMember+1))
if err != nil {
return nil, err
}
if len(b) > maxMember {
return nil, fmt.Errorf("%s inflates past %d MiB", f.Name, maxMember>>20)
}
return b, nil
}
// relID is an element's r:id, the namespaced id that names a relationship —
// not its own plain id, which sldId has as well.
func (e xmlEvent) relID() string {
for _, a := range e.attr {
if a.Name.Local == "id" && a.Name.Space != "" {
return a.Value
}
}
return ""
}
// xmlEvent is one step of walking an XML document: a start, an end, or
// character data, with the element's local name and attributes.
type xmlEvent struct {
start, end bool
name string
attr []xml.Attr
text string
}
func (e xmlEvent) get(local string) string {
for _, a := range e.attr {
if a.Name.Local == local {
return a.Value
}
}
return ""
}
// walkXML calls f for each element start and end and each run of character
// data. Namespaces are dropped: OOXML and ODF are read by local names.
func walkXML(b []byte, f func(xmlEvent)) error {
d := xml.NewDecoder(bytes.NewReader(b))
d.Strict = false
d.CharsetReader = func(cs string, in io.Reader) (io.Reader, error) {
all, err := io.ReadAll(in)
if err != nil {
return nil, err
}
return strings.NewReader(decodeCharset(all, cs)), nil
}
for {
tok, err := d.Token()
if errors.Is(err, io.EOF) {
return nil
}
if err != nil {
return err
}
switch t := tok.(type) {
case xml.StartElement:
f(xmlEvent{start: true, name: t.Name.Local, attr: t.Attr})
case xml.EndElement:
f(xmlEvent{end: true, name: t.Name.Local})
case xml.CharData:
f(xmlEvent{text: string(t)})
}
}
}
// headingLevel reads a heading level out of a paragraph style: Word's
// "Heading1", "Heading 2", "heading3", its Turkish "Balk1" (the id Word gives
// "Başlık 1"), "Title" and "Subtitle".
func headingLevel(style string) int {
s := strings.ToLower(strings.ReplaceAll(style, " ", ""))
switch s {
case "title", "konubaşlığı", "konubal":
return 1
case "subtitle", "altkonubaşlığı", "altkonubal":
return 2
}
for _, p := range []string{"heading", "balk", "başlık", "berschrift", "titre", "título", "titolo", "kop"} {
if rest, ok := strings.CutPrefix(s, p); ok {
if n, err := strconv.Atoi(rest); err == nil && n >= 1 && n <= 9 {
return min(n, 6)
}
}
}
return 0
}
// docxText reads a Word document: the body, then footnotes and endnotes.
// Paragraph styles give headings; tables are rows of tab-separated cells.
func docxText(b []byte) (string, error) {
zr, err := openZip(b)
if err != nil {
return "", err
}
body, err := member(zr, "word/document.xml")
if err != nil {
return "", err
}
if body == nil {
return "", errors.New("not a Word document: no word/document.xml")
}
var o out
if err := wordML(body, &o); err != nil {
return "", err
}
for _, extra := range []string{"word/footnotes.xml", "word/endnotes.xml"} {
if x, _ := member(zr, extra); x != nil {
_ = wordML(x, &o)
}
}
return o.String(), nil
}
func wordML(b []byte, o *out) error {
inText, inDel, inCell := false, 0, 0
level, list := 0, false
return walkXML(b, func(e xmlEvent) {
switch {
case e.start:
switch e.name {
case "p":
level, list = 0, false
case "pStyle":
level = headingLevel(e.get("val"))
if strings.Contains(strings.ToLower(e.get("val")), "list") {
list = true
}
case "numPr":
list = true
case "t":
inText = true
if level > 0 && inCell == 0 && o.line.Len() == 0 && o.prefix == "" {
o.heading(level)
} else if list && inCell == 0 && o.line.Len() == 0 && o.prefix == "" {
o.item()
}
case "delText":
// Tracked deletions are not the document's text.
inDel++
case "tc":
inCell++
case "tab":
o.raw("\t")
case "br", "cr":
o.newline()
}
case e.end:
switch e.name {
case "t":
inText = false
case "delText":
inDel--
case "p":
switch {
case inCell > 0:
// A cell's paragraphs stay in its row.
o.text(" ")
case list:
o.newline()
default:
o.para()
}
case "tc":
inCell--
o.tab()
case "tr":
o.newline()
case "tbl":
o.para()
}
default:
if inText && inDel == 0 {
o.raw(e.text)
}
}
})
}
// xlsxText reads a workbook: each sheet under its name as a heading, each
// row a line of tab-separated cells, shared strings resolved.
func xlsxText(b []byte) (string, error) {
zr, err := openZip(b)
if err != nil {
return "", err
}
shared := xlsxShared(zr)
sheets, err := xlsxSheets(zr)
if err != nil {
return "", err
}
var o out
for _, sh := range sheets {
data, err := member(zr, sh.path)
if err != nil || data == nil {
continue
}
o.heading(2)
o.text(sh.name)
o.para()
var cell strings.Builder
cellType, inV, inIs := "", false, false
first := true
_ = walkXML(data, func(e xmlEvent) {
switch {
case e.start && e.name == "row":
first = true
case e.end && e.name == "row":
o.newline()
case e.start && e.name == "c":
cellType = e.get("t")
cell.Reset()
case e.end && e.name == "c":
v := cell.String()
if cellType == "s" {
if i, err := strconv.Atoi(strings.TrimSpace(v)); err == nil && i >= 0 && i < len(shared) {
v = shared[i]
}
}
v = strings.TrimSpace(strings.ReplaceAll(v, "\n", " "))
if v != "" {
if !first {
o.tab()
}
o.text(v)
first = false
}
case e.start && e.name == "v":
inV = true
case e.end && e.name == "v":
inV = false
case e.start && e.name == "is":
inIs = true
case e.end && e.name == "is":
inIs = false
case !e.start && !e.end && (inV || inIs):
cell.WriteString(e.text)
}
})
o.para()
}
return o.String(), nil
}
// xlsxShared is a workbook's shared strings, which cells name by index.
func xlsxShared(zr *zip.Reader) []string {
var shared []string
ss, _ := member(zr, "xl/sharedStrings.xml")
if ss == nil {
return nil
}
var cur strings.Builder
inT, inRPh := false, 0
_ = walkXML(ss, func(e xmlEvent) {
switch {
case e.start && e.name == "si":
cur.Reset()
case e.end && e.name == "si":
shared = append(shared, cur.String())
case e.start && e.name == "rPh":
inRPh++ // phonetic guides, not the cell's text
case e.end && e.name == "rPh":
inRPh--
case e.start && e.name == "t":
inT = true
case e.end && e.name == "t":
inT = false
case !e.start && !e.end && inT && inRPh == 0:
cur.WriteString(e.text)
}
})
return shared
}
// XLSXRows reads a workbook's first sheet as rows of cells, each cell in
// its column (an empty one is ""), for a table such as questions and
// answers. Rows with nothing in them are left out.
func XLSXRows(b []byte) ([][]string, error) {
zr, err := openZip(b)
if err != nil {
return nil, err
}
shared := xlsxShared(zr)
sheets, err := xlsxSheets(zr)
if err != nil {
return nil, err
}
if len(sheets) == 0 {
return nil, errors.New("the workbook has no sheets")
}
data, err := member(zr, sheets[0].path)
if err != nil || data == nil {
return nil, fmt.Errorf("the workbook's first sheet, %s, could not be read", sheets[0].name)
}
var rows [][]string
var row []string
var cell strings.Builder
cellType, col, next := "", 0, 0
inV, inIs := false, false
_ = walkXML(data, func(e xmlEvent) {
switch {
case e.start && e.name == "row":
row, next = nil, 0
case e.end && e.name == "row":
if slices.ContainsFunc(row, func(s string) bool { return s != "" }) {
rows = append(rows, row)
}
case e.start && e.name == "c":
cellType = e.get("t")
col = next
if c, ok := columnOf(e.get("r")); ok {
col = c
}
next = col + 1
cell.Reset()
case e.end && e.name == "c":
v := cell.String()
if cellType == "s" {
if i, err := strconv.Atoi(strings.TrimSpace(v)); err == nil && i >= 0 && i < len(shared) {
v = shared[i]
}
}
if col < 0 || col > 16383 {
return
}
for len(row) <= col {
row = append(row, "")
}
row[col] = strings.TrimSpace(v)
case e.start && e.name == "v":
inV = true
case e.end && e.name == "v":
inV = false
case e.start && e.name == "is":
inIs = true
case e.end && e.name == "is":
inIs = false
case !e.start && !e.end && (inV || inIs):
cell.WriteString(e.text)
}
})
return rows, nil
}
// columnOf is a cell reference's column, from 0: "B7" is 1.
func columnOf(ref string) (int, bool) {
n, i := 0, 0
for ; i < len(ref) && ref[i] >= 'A' && ref[i] <= 'Z'; i++ {
n = n*26 + int(ref[i]-'A'+1)
}
if i == 0 {
return 0, false
}
return n - 1, true
}
type sheetRef struct{ name, path string }
// xlsxSheets lists a workbook's sheets in their order, with the part each is
// kept in.
func xlsxSheets(zr *zip.Reader) ([]sheetRef, error) {
wb, err := member(zr, "xl/workbook.xml")
if err != nil || wb == nil {
return nil, errors.New("not a workbook: no xl/workbook.xml")
}
rels := map[string]string{}
if r, _ := member(zr, "xl/_rels/workbook.xml.rels"); r != nil {
_ = walkXML(r, func(e xmlEvent) {
if e.start && e.name == "Relationship" {
rels[e.get("Id")] = e.get("Target")
}
})
}
var out []sheetRef
_ = walkXML(wb, func(e xmlEvent) {
if !e.start || e.name != "sheet" {
return
}
target := rels[e.relID()]
if target == "" {
return
}
p := path.Clean(path.Join("xl", target))
if strings.HasPrefix(target, "/") {
p = strings.TrimPrefix(target, "/")
}
out = append(out, sheetRef{name: e.get("name"), path: p})
})
return out, nil
}
var slideNum = regexp.MustCompile(`(\d+)\.xml$`)
func numbered(zr *zip.Reader, prefix string) []*zip.File {
var fs []*zip.File
for _, f := range zr.File {
if strings.HasPrefix(f.Name, prefix) && slideNum.MatchString(f.Name) && !strings.Contains(f.Name[len(prefix):], "/") {
fs = append(fs, f)
}
}
n := func(f *zip.File) int {
v, _ := strconv.Atoi(slideNum.FindStringSubmatch(f.Name)[1])
return v
}
sort.Slice(fs, func(i, j int) bool { return n(fs[i]) < n(fs[j]) })
return fs
}
// rels reads a part's relationships: id → target, and type → target, with
// targets resolved against the part's directory.
func rels(zr *zip.Reader, part string) (byID, byType map[string]string) {
byID, byType = map[string]string{}, map[string]string{}
dir, base := path.Split(part)
r, _ := member(zr, dir+"_rels/"+base+".rels")
if r == nil {
return byID, byType
}
_ = walkXML(r, func(e xmlEvent) {
if !e.start || e.name != "Relationship" || e.get("TargetMode") == "External" {
return
}
t := e.get("Target")
if strings.HasPrefix(t, "/") {
t = strings.TrimPrefix(t, "/")
} else {
t = path.Clean(path.Join(dir, t))
}
byID[e.get("Id")] = t
ty := e.get("Type")
byType[ty[strings.LastIndex(ty, "/")+1:]] = t
})
return byID, byType
}
// pptxText reads a presentation: each slide, in the presentation's order, as
// a heading, its text, and its speaker notes.
func pptxText(b []byte) (string, error) {
zr, err := openZip(b)
if err != nil {
return "", err
}
var order []string
if pres, _ := member(zr, "ppt/presentation.xml"); pres != nil {
ids, _ := rels(zr, "ppt/presentation.xml")
_ = walkXML(pres, func(e xmlEvent) {
if e.start && e.name == "sldId" {
if t := ids[e.relID()]; t != "" {
order = append(order, t)
}
}
})
}
if len(order) == 0 {
for _, f := range numbered(zr, "ppt/slides/slide") {
order = append(order, f.Name)
}
}
if len(order) == 0 {
return "", errors.New("not a presentation: no slides")
}
var o out
drawing := func(d []byte) {
inT, inField := false, 0
_ = walkXML(d, func(e xmlEvent) {
switch {
case e.start && e.name == "fld":
// Slide numbers and dates, not the slide's text.
inField++
case e.end && e.name == "fld":
inField--
case e.start && e.name == "t":
inT = true
case e.end && e.name == "t":
inT = false
case e.end && e.name == "p":
o.newline()
case e.start && e.name == "br":
o.newline()
case e.end && e.name == "sp":
o.para()
case e.end && e.name == "tc":
o.tab()
case e.end && e.name == "tr":
o.newline()
case !e.start && !e.end && inT && inField == 0:
o.raw(e.text)
}
})
}
for i, part := range order {
d, err := member(zr, part)
if err != nil || d == nil {
continue
}
o.heading(2)
o.text(fmt.Sprintf("Slide %d", i+1))
o.para()
drawing(d)
if _, byType := rels(zr, part); byType["notesSlide"] != "" {
if nd, _ := member(zr, byType["notesSlide"]); nd != nil {
o.para()
o.text("Notes:")
o.newline()
drawing(nd)
}
}
o.para()
}
return o.String(), nil
}