gen_odf_testdata.sh (11569B)
1 #!/bin/sh 2 # Regenerate the synthetic ODF / Word test files used by the "fulltext" 3 # tests in src/plugins/test_odf.c and src/plugins/test_ole2.c. 4 # 5 # Copyright (C) 2026 Christian Grothoff 6 # The generated files are dedicated to the public domain (CC0 1.0). 7 # 8 # The ODF packages (odf_fulltext.odt, .odp, .ods) are zips of 9 # hand-written XML, so every byte is owned by this script. The text 10 # document deliberately carries all the material the full-text 11 # extraction must leave out -- a running header and footer (in 12 # styles.xml), a footnote, a comment, page-number and page-count 13 # fields, tracked-change deletions, the alternative text of a frame -- 14 # next to the material it must keep: headings, a list, a table, a 15 # bookmark reference, a text frame, spaces/tabs/line breaks and 16 # character references. The presentation has speaker notes, the 17 # spreadsheet a cell comment. 18 # 19 # ole2_fulltext.doc is the text document (minus the tracked changes, 20 # which in Word format would need character-property parsing to 21 # drop, and minus the text frame, which Word stores outside of the 22 # main text) converted to Word 97 by LibreOffice, if it is installed: 23 # soffice --headless --convert-to doc 24 # Its header/footer, footnote, comment and PAGE/NUMPAGES fields then 25 # live where Word puts them, which is what test_ole2 checks. 26 # 27 # Needs python3; LibreOffice only for the .doc. 28 set -e 29 30 srcdir=$(dirname "$0") 31 outdir=${1:-"$srcdir/../src/plugins/testdata"} 32 33 python3 - "$outdir" <<'EOF' 34 import os 35 import sys 36 import zipfile 37 38 OUT = sys.argv[1] 39 DATE = (2026, 1, 2, 3, 4, 6) 40 41 NS = ('xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" ' 42 'xmlns:style="urn:oasis:names:tc:opendocument:xmlns:style:1.0" ' 43 'xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" ' 44 'xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" ' 45 'xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" ' 46 'xmlns:fo="urn:oasis:names:tc:opendocument:xmlns:' 47 'xsl-fo-compatible:1.0" ' 48 'xmlns:xlink="http://www.w3.org/1999/xlink" ' 49 'xmlns:dc="http://purl.org/dc/elements/1.1/" ' 50 'xmlns:meta="urn:oasis:names:tc:opendocument:xmlns:meta:1.0" ' 51 'xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:' 52 'svg-compatible:1.0" ' 53 'xmlns:presentation="urn:oasis:names:tc:opendocument:xmlns:' 54 'presentation:1.0" ' 55 'office:version="1.2"') 56 57 XML = '<?xml version="1.0" encoding="UTF-8"?>\n' 58 59 60 def meta(title): 61 return (XML + '<office:document-meta ' + NS + '><office:meta>' 62 '<meta:generator>gen_odf_testdata.sh</meta:generator>' 63 '<dc:title>' + title + '</dc:title>' 64 '</office:meta></office:document-meta>') 65 66 67 def manifest(mime): 68 return (XML + '<manifest:manifest xmlns:manifest=' 69 '"urn:oasis:names:tc:opendocument:xmlns:manifest:1.0" ' 70 'manifest:version="1.2">' 71 '<manifest:file-entry manifest:full-path="/" ' 72 'manifest:media-type="' + mime + '"/>' 73 '<manifest:file-entry manifest:full-path="content.xml" ' 74 'manifest:media-type="text/xml"/>' 75 '<manifest:file-entry manifest:full-path="styles.xml" ' 76 'manifest:media-type="text/xml"/>' 77 '<manifest:file-entry manifest:full-path="meta.xml" ' 78 'manifest:media-type="text/xml"/>' 79 '</manifest:manifest>') 80 81 82 def write(name, mime, title, content, styles): 83 with zipfile.ZipFile(os.path.join(OUT, name), 'w') as z: 84 def add(fn, data, method): 85 zi = zipfile.ZipInfo(fn, DATE) 86 zi.compress_type = method 87 zi.external_attr = 0o644 << 16 88 z.writestr(zi, data) 89 add('mimetype', mime, zipfile.ZIP_STORED) 90 add('content.xml', XML + content, zipfile.ZIP_DEFLATED) 91 add('styles.xml', XML + styles, zipfile.ZIP_DEFLATED) 92 add('meta.xml', meta(title), zipfile.ZIP_DEFLATED) 93 add('META-INF/manifest.xml', manifest(mime), zipfile.ZIP_DEFLATED) 94 95 96 def text_styles(): 97 return ('<office:document-styles ' + NS + '>' 98 '<office:automatic-styles>' 99 '<style:page-layout style:name="pm1"/>' 100 '</office:automatic-styles>' 101 '<office:master-styles>' 102 '<style:master-page style:name="Standard" ' 103 'style:page-layout-name="pm1">' 104 '<style:header><text:p>RUNNING HEADER TEXT</text:p>' 105 '</style:header>' 106 '<style:footer><text:p>Footer page ' 107 '<text:page-number text:select-page="current">1' 108 '</text:page-number></text:p></style:footer>' 109 '</style:master-page>' 110 '</office:master-styles></office:document-styles>') 111 112 113 def text_content(tracked, frame): 114 body = [] 115 if tracked: 116 body.append( 117 '<text:tracked-changes>' 118 '<text:changed-region text:id="ct1"><text:deletion>' 119 '<office:change-info><dc:creator>Editor</dc:creator>' 120 '<dc:date>2026-01-02T03:04:05</dc:date></office:change-info>' 121 '<text:p>DELETED TEXT</text:p>' 122 '</text:deletion></text:changed-region>' 123 '</text:tracked-changes>') 124 body.append('<text:h text:outline-level="1">Synthetic Heading</text:h>') 125 body.append( 126 '<text:p>First paragraph with spaces<text:s text:c="3"/>and' 127 '<text:tab/>a tab<text:line-break/>and a break; entities ' 128 '& <tag> ☺ €' 129 '<text:note text:id="ftn1" text:note-class="footnote">' 130 '<text:note-citation>1</text:note-citation><text:note-body>' 131 '<text:p>FOOTNOTE BODY TEXT</text:p></text:note-body></text:note>' 132 ' and the sentence ' 133 + ('<text:change text:change-id="ct1"/>' if tracked else '') 134 + 'continues' 135 '<office:annotation><dc:creator>Reviewer</dc:creator>' 136 '<dc:date>2026-01-02T03:04:05</dc:date>' 137 '<text:p>ANNOTATION TEXT</text:p></office:annotation>.</text:p>') 138 body.append( 139 '<text:p>This is page <text:page-number text:select-page="current">' 140 '7</text:page-number> of <text:page-count>9</text:page-count>' 141 ' pages.</text:p>') 142 body.append( 143 '<text:list><text:list-item><text:p>List item one</text:p>' 144 '</text:list-item><text:list-item><text:p>List item two</text:p>' 145 '</text:list-item></text:list>') 146 body.append( 147 '<table:table table:name="T1">' 148 '<table:table-column table:number-columns-repeated="2"/>' 149 '<table:table-row>' 150 '<table:table-cell office:value-type="string">' 151 '<text:p>Cell A1</text:p></table:table-cell>' 152 '<table:table-cell office:value-type="string">' 153 '<text:p>Cell B1</text:p></table:table-cell></table:table-row>' 154 '<table:table-row>' 155 '<table:table-cell office:value-type="string">' 156 '<text:p>Cell A2</text:p></table:table-cell>' 157 '<table:table-cell office:value-type="string">' 158 '<text:p>Cell B2</text:p></table:table-cell></table:table-row>' 159 '</table:table>') 160 body.append( 161 '<text:p><text:bookmark text:name="b1"/>See ' 162 '<text:bookmark-ref text:reference-format="text" text:ref-name="b1">' 163 'Chapter One</text:bookmark-ref> for details.</text:p>') 164 if frame: 165 body.append( 166 '<text:p>Before the frame<draw:frame draw:name="F1" ' 167 'text:anchor-type="as-char" svg:width="4cm" svg:height="1cm">' 168 '<draw:text-box><text:p>Frame caption text</text:p>' 169 '</draw:text-box><svg:title>ALT TITLE</svg:title>' 170 '<svg:desc>ALT TEXT</svg:desc></draw:frame>after the frame.' 171 '</text:p>') 172 body.append('<text:p>Last paragraph.</text:p>') 173 return ('<office:document-content ' + NS + '>' 174 '<office:automatic-styles/><office:body><office:text>' 175 + ''.join(body) 176 + '</office:text></office:body></office:document-content>') 177 178 179 write('odf_fulltext.odt', 180 'application/vnd.oasis.opendocument.text', 181 'Synthetic full text test', 182 text_content(True, True), 183 text_styles()) 184 185 # input for the Word conversion below; not part of the test data 186 write('ole2_fulltext.tmp.odt', 187 'application/vnd.oasis.opendocument.text', 188 'Synthetic full text test', 189 text_content(False, False), 190 text_styles()) 191 192 193 def slide(name, title, lines, notes): 194 return ('<draw:page draw:name="' + name + '" ' 195 'draw:master-page-name="Default">' 196 '<draw:frame presentation:class="title" svg:x="1cm" svg:y="1cm" ' 197 'svg:width="20cm" svg:height="3cm"><draw:text-box>' 198 '<text:p>' + title + '</text:p></draw:text-box></draw:frame>' 199 '<draw:frame presentation:class="outline" svg:x="1cm" ' 200 'svg:y="5cm" svg:width="20cm" svg:height="10cm"><draw:text-box>' 201 '<text:list>' 202 + ''.join('<text:list-item><text:p>' + ln 203 + '</text:p></text:list-item>' for ln in lines) 204 + '</text:list></draw:text-box></draw:frame>' 205 '<presentation:notes><draw:frame presentation:class="notes" ' 206 'svg:x="1cm" svg:y="1cm" svg:width="10cm" svg:height="5cm">' 207 '<draw:text-box><text:p>' + notes + '</text:p></draw:text-box>' 208 '</draw:frame></presentation:notes></draw:page>') 209 210 211 write('odf_fulltext.odp', 212 'application/vnd.oasis.opendocument.presentation', 213 'Synthetic slides', 214 '<office:document-content ' + NS + '><office:body><office:presentation>' 215 + slide('p1', 'First slide title', ['Bullet one', 'Bullet two'], 216 'SPEAKER NOTES ONE') 217 + slide('p2', 'Second slide title', ['Bullet three'], 218 'SPEAKER NOTES TWO') 219 + '</office:presentation></office:body></office:document-content>', 220 '<office:document-styles ' + NS + '><office:master-styles>' 221 '<style:master-page style:name="Default"/>' 222 '</office:master-styles></office:document-styles>') 223 224 write('odf_fulltext.ods', 225 'application/vnd.oasis.opendocument.spreadsheet', 226 'Synthetic sheet', 227 '<office:document-content ' + NS + '><office:body><office:spreadsheet>' 228 '<table:table table:name="Sheet1">' 229 '<table:table-column table:number-columns-repeated="2"/>' 230 '<table:table-row>' 231 '<table:table-cell office:value-type="string"><text:p>Name</text:p>' 232 '</table:table-cell>' 233 '<table:table-cell office:value-type="string"><text:p>Amount</text:p>' 234 '</table:table-cell></table:table-row>' 235 '<table:table-row>' 236 '<table:table-cell office:value-type="string">' 237 '<office:annotation><dc:creator>Reviewer</dc:creator>' 238 '<text:p>CELL COMMENT</text:p></office:annotation>' 239 '<text:p>Apples</text:p></table:table-cell>' 240 '<table:table-cell office:value-type="float" office:value="42">' 241 '<text:p>42</text:p></table:table-cell></table:table-row>' 242 '<table:table-row table:number-rows-repeated="3">' 243 '<table:table-cell table:number-columns-repeated="2"/>' 244 '</table:table-row>' 245 '</table:table></office:spreadsheet></office:body>' 246 '</office:document-content>', 247 '<office:document-styles ' + NS + '/>') 248 EOF 249 250 if command -v soffice > /dev/null 2>&1 251 then 252 tmp=$(mktemp -d) 253 soffice -env:UserInstallation="file://$tmp/profile" --headless \ 254 --convert-to doc:"MS Word 97" --outdir "$tmp" \ 255 "$outdir/ole2_fulltext.tmp.odt" > /dev/null 256 mv "$tmp/ole2_fulltext.tmp.doc" "$outdir/ole2_fulltext.doc" 257 rm -rf "$tmp" 258 else 259 echo "soffice not found, not regenerating ole2_fulltext.doc" >&2 260 fi 261 rm -f "$outdir/ole2_fulltext.tmp.odt"