libextractor

GNU libextractor
Log | Files | Refs | Submodules | README | LICENSE

gen_html_testdata.sh (2429B)

      1 #!/bin/sh
      2 # Regenerate src/plugins/testdata/html_fulltext.html, the fixture for the
      3 # html plugin's "fulltext" option.
      4 #
      5 # This file is in the public domain (CC0); so is the file it produces.
      6 #
      7 # Requirements: python3 (standard library only).  The page is written in
      8 # ISO-8859-1 (declared via http-equiv) so that the test also covers the
      9 # charset conversion; it carries all the page furniture the plugin must
     10 # drop (head, script, style, noscript, template, nav, page-level
     11 # header/footer, aside, landmark roles, hidden elements).  The expected
     12 # text is spelled out in src/plugins/test_html.c.
     13 #
     14 # Usage: contrib/gen_html_testdata.sh [output-directory]
     15 #        (default output directory: src/plugins/testdata)
     16 
     17 set -e
     18 OUT="${1:-$(dirname "$0")/../src/plugins/testdata}"
     19 mkdir -p "$OUT"
     20 python3 - "$OUT" <<'PYEOF'
     21 import os, sys
     22 
     23 OUT = sys.argv[1]
     24 
     25 PAGE = """<!DOCTYPE html>
     26 <html lang="de">
     27 <head>
     28 <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
     29 <title>Käse und Brot</title>
     30 <meta name="author" content="Christian Grothoff">
     31 <style>p { margin: 0; } nav { display: block; }</style>
     32 <script>document.write("<p>script text</p>");</script>
     33 </head>
     34 <body>
     35 <header><a href="/">Site banner</a></header>
     36 <nav><ul><li><a href="/">Home</a></li><li><a href="/a">About</a></li></ul></nav>
     37 <div role="navigation">Breadcrumbs &gt; here</div>
     38 <main>
     39 <article>
     40 <header><h1>Der Käse &amp; das Brot</h1></header>
     41 <p>Hello <b>world</b>, again&nbsp;here:
     42    &eacute;t&#233; &#x2014; done.</p>
     43 <p>Line one<br>line two<br><br>New paragraph after two breaks.</p>
     44 <div hidden>Hidden by attribute</div>
     45 <span aria-hidden="true">Decorative icon</span>
     46 <div style="display: none">Hidden by style</div>
     47 <ul><li>Item A</li><li>Item <i>B</i></li></ul>
     48 <table><tr><th>Cell 1</th><td>Cell 2</td></tr></table>
     49 <noscript>Please enable JavaScript</noscript>
     50 <template><p>Template text</p></template>
     51 <blockquote>Quoted text</blockquote>
     52 <p>Soft&shy;hyphen and <span>in</span>line elements.</p>
     53 <footer>Article byline</footer>
     54 </article>
     55 <aside>Sidebar text</aside>
     56 </main>
     57 <div role="contentinfo">Legal notice</div>
     58 <footer>Copyright 2026</footer>
     59 <script>var x = "</div>more script";</script>
     60 </body>
     61 </html>
     62 """
     63 
     64 with open (os.path.join (OUT, "html_fulltext.html"), "wb") as f:
     65   f.write (PAGE.encode ("iso-8859-1"))
     66 print ("html_fulltext.html: %d bytes"
     67        % os.path.getsize (os.path.join (OUT, "html_fulltext.html")))
     68 PYEOF