diff options
author | ng0 <ng0@n0.is> | 2019-11-12 17:28:30 +0000 |
---|---|---|
committer | ng0 <ng0@n0.is> | 2019-11-12 17:28:30 +0000 |
commit | 98690996582e0e511a8e8611b3db5080be6e75b0 (patch) | |
tree | 8b5ddd042d8d76d329e79451f70f14c4c52aafc3 /inc | |
parent | 4a4839075452c7527ce87775d08428e5cb433957 (diff) | |
download | www-98690996582e0e511a8e8611b3db5080be6e75b0.tar.gz www-98690996582e0e511a8e8611b3db5080be6e75b0.zip |
split up template.py, make site generation a class.
Diffstat (limited to 'inc')
-rw-r--r-- | inc/fileproc.py | 85 | ||||
-rw-r--r-- | inc/site.py | 127 | ||||
-rw-r--r-- | inc/sitemap.py | 20 | ||||
-rw-r--r-- | inc/sum.py | 35 | ||||
-rw-r--r-- | inc/textproc.py | 35 |
5 files changed, 302 insertions, 0 deletions
diff --git a/inc/fileproc.py b/inc/fileproc.py new file mode 100644 index 00000000..435078bc --- /dev/null +++ b/inc/fileproc.py | |||
@@ -0,0 +1,85 @@ | |||
1 | from pathlib import Path | ||
2 | |||
3 | def copy_files(kind, conf, locale, inlist, ptarget): | ||
4 | o = Path(ptarget) | ||
5 | for item in conf[inlist]: | ||
6 | i = Path(kind + "/" + item["file"]) | ||
7 | # print(i) | ||
8 | for t in item["targets"]: | ||
9 | d_loc = o / locale / t | ||
10 | d = o / t | ||
11 | # print(d) | ||
12 | if i.is_file() is not False: | ||
13 | d_loc.write_text(i.read_text()) | ||
14 | print("copied " + str(i) + " to " + str(d_loc) + "...") | ||
15 | d.write_text(i.read_text()) | ||
16 | print("copied " + str(i) + " to " + str(d) + "...") | ||
17 | |||
18 | |||
19 | def rm_rf(directory): | ||
20 | directory = Path(directory) | ||
21 | for child in directory.glob('*'): | ||
22 | if child.is_file(): | ||
23 | child.unlink() | ||
24 | else: | ||
25 | rm_rf(child) | ||
26 | # directory.rmdir() | ||
27 | |||
28 | |||
29 | def fileop(infile, outfile, action): | ||
30 | """ | ||
31 | infile: inputfile, Path object | ||
32 | outfile: outputfile, Path object | ||
33 | action: action if any, String | ||
34 | """ | ||
35 | i = Path(infile) | ||
36 | o = Path(outfile) | ||
37 | outdir = Path("rendered") | ||
38 | if i.is_file() is not False: | ||
39 | if action == "copy": | ||
40 | # Write content of i to o. | ||
41 | o.write_text(i.read_text()) | ||
42 | if action == "link": | ||
43 | o.symlink_to(i) | ||
44 | |||
45 | |||
46 | def write_name(filename, infile, locale, replacer): | ||
47 | return "./rendered/" + locale + "/" + infile.replace(replacer, | ||
48 | '').rstrip(".j2") | ||
49 | |||
50 | |||
51 | def localized(filename, locale, *args): | ||
52 | if len(args) == 0: | ||
53 | return "../" + locale + "/" + filename | ||
54 | ext = kwargs.get('ext', None) | ||
55 | if ext is not None: | ||
56 | lf = filename + "." + locale + "." + ext | ||
57 | lp = Path(lf) | ||
58 | if locale == "en" or not lp.is_file(): | ||
59 | return "../" + filename + "." + ext | ||
60 | else: | ||
61 | return "../" + lf | ||
62 | |||
63 | |||
64 | # This generates and switches sites generations, preventing | ||
65 | # in-place modification of the website. | ||
66 | # * save old generation directory name | ||
67 | # * jinja2 creates content in "rendered" (happened before calling this function) | ||
68 | # * calculate sum of "rendered" | ||
69 | # * move "rendered" to out/$sum | ||
70 | # * remove symlink "html_dir" | ||
71 | # * symlink out/$sum to "html_dir" | ||
72 | # * delete old generation directory | ||
73 | def generation_dir(htmldir): | ||
74 | oldgen = Path(htmldir).resolve() | ||
75 | # precondition: jinja2 has created the files in "rendered". | ||
76 | newgen = Path("rendered") | ||
77 | newgen_sum = walksum(newgen) | ||
78 | outdir = Path("out") | ||
79 | outdir.mkdir(parents=True, exist_ok=True) | ||
80 | newgen_target = Path("out") / newgen_sum | ||
81 | newgen.rename(newgen_target) | ||
82 | html = Path(htmldir) | ||
83 | html.unlink() | ||
84 | fileop(newgen, html, "link") | ||
85 | rm_rf(oldgen) | ||
diff --git a/inc/site.py b/inc/site.py new file mode 100644 index 00000000..67e5afa8 --- /dev/null +++ b/inc/site.py | |||
@@ -0,0 +1,127 @@ | |||
1 | import os | ||
2 | import os.path | ||
3 | import sys | ||
4 | import re | ||
5 | import gettext | ||
6 | import glob | ||
7 | import codecs | ||
8 | import jinja2 | ||
9 | import hashlib | ||
10 | from pathlib import Path, PurePosixPath | ||
11 | from ruamel.yaml import YAML | ||
12 | import inc.i18nfix | ||
13 | from inc.textproc import cut_news_text | ||
14 | from inc.fileproc import copy_files | ||
15 | |||
16 | |||
17 | class gen_site: | ||
18 | def load_config(self, name="www.yml"): | ||
19 | yaml = YAML(typ='safe') | ||
20 | site_configfile = Path(name) | ||
21 | return yaml.load(site_configfile) | ||
22 | |||
23 | def gen_abstract(self, conf, name, member, pages, length): | ||
24 | for item in conf[name]: | ||
25 | item[member] = cut_news_text(item[pages], length) | ||
26 | |||
27 | def run(self, root, conf, env): | ||
28 | # os.chdir("..") | ||
29 | print(os.getcwd()) | ||
30 | root = "../" + root | ||
31 | for in_file in glob.glob(root + "/*.j2"): | ||
32 | name, ext = re.match(r"(.*)\.([^.]+)$", | ||
33 | in_file.rstrip(".j2")).groups() | ||
34 | tmpl = env.get_template(in_file) | ||
35 | |||
36 | def self_localized(other_locale): | ||
37 | """ | ||
38 | Return URL for the current page in another locale. | ||
39 | """ | ||
40 | return "../" + other_locale + "/" + in_file.replace( | ||
41 | root + '/', '').rstrip(".j2") | ||
42 | |||
43 | def url_localized(filename): | ||
44 | if root == "news": | ||
45 | return "../../" + locale + "/" + filename | ||
46 | else: | ||
47 | return "../" + locale + "/" + filename | ||
48 | |||
49 | def url_static(filename): | ||
50 | if root == "news": | ||
51 | return "../../static/" + filename | ||
52 | else: | ||
53 | return "../static/" + filename | ||
54 | |||
55 | def url_dist(filename): | ||
56 | if root == "news": | ||
57 | return "../../dist/" + filename | ||
58 | else: | ||
59 | return "../dist/" + filename | ||
60 | |||
61 | def svg_localized(filename): | ||
62 | lf = filename + "." + locale + ".svg" | ||
63 | if locale == "en" or not Path(lf).is_file(): | ||
64 | return "../" + filename + ".svg" | ||
65 | else: | ||
66 | return "../" + lf | ||
67 | |||
68 | def url(x): | ||
69 | # TODO: look at the app root environment variable | ||
70 | # TODO: check if file exists | ||
71 | #if root == "news": | ||
72 | # return "../" + "../" + x | ||
73 | #else: | ||
74 | # return "../" + x | ||
75 | return "../" + x | ||
76 | |||
77 | for l in glob.glob("locale/*/"): | ||
78 | locale = os.path.basename(l[:-1]) | ||
79 | |||
80 | tr = gettext.translation("messages", | ||
81 | localedir="locale", | ||
82 | languages=[locale]) | ||
83 | |||
84 | tr.gettext = i18nfix.wrap_gettext(tr.gettext) | ||
85 | |||
86 | env.install_gettext_translations(tr, newstyle=True) | ||
87 | |||
88 | content = tmpl.render(lang=locale, | ||
89 | lang_full=conf["langs_full"][locale], | ||
90 | url=url, | ||
91 | meetingnotesdata=conf["meetingnotes"], | ||
92 | newsdata=conf["newsposts"], | ||
93 | videosdata=conf["videoslist"], | ||
94 | self_localized=self_localized, | ||
95 | url_localized=url_localized, | ||
96 | url_static=url_static, | ||
97 | url_dist=url_dist, | ||
98 | svg_localized=svg_localized, | ||
99 | filename=name + "." + ext) | ||
100 | |||
101 | if root == "news": | ||
102 | out_name = "./rendered/" + locale + "/" + root + "/" + in_file.replace( | ||
103 | root + '/', '').rstrip(".j2") | ||
104 | else: | ||
105 | out_name = "./rendered/" + locale + "/" + in_file.replace( | ||
106 | root + '/', '').rstrip(".j2") | ||
107 | |||
108 | outdir = Path("rendered") | ||
109 | if outdir.exists() is False: | ||
110 | sys.exit(1) | ||
111 | |||
112 | if root == "news": | ||
113 | langdir = outdir / locale / root | ||
114 | else: | ||
115 | langdir = outdir / locale | ||
116 | |||
117 | try: | ||
118 | langdir.mkdir(parents=True, exist_ok=True) | ||
119 | except e as FileNotFoundError: | ||
120 | print(e) | ||
121 | |||
122 | with codecs.open(out_name, "w", encoding='utf-8') as f: | ||
123 | try: | ||
124 | print(Path.cwd()) | ||
125 | f.write(content) | ||
126 | except e as Error: | ||
127 | print(e) | ||
diff --git a/inc/sitemap.py b/inc/sitemap.py new file mode 100644 index 00000000..e050c77d --- /dev/null +++ b/inc/sitemap.py | |||
@@ -0,0 +1,20 @@ | |||
1 | import os | ||
2 | from pathlib import Path, PurePosixPath | ||
3 | |||
4 | def sitemap_tree(path): | ||
5 | tree = dict(name=PurePosixPath(path).name, children=[]) | ||
6 | try: | ||
7 | mylist = os.listdir(path) | ||
8 | except OSError: | ||
9 | pass | ||
10 | else: | ||
11 | for name in mylist: | ||
12 | fn = os.path.join(path, name) | ||
13 | if os.path.isdir(fn): | ||
14 | tree['children'].append(sitemap_tree(fn)) | ||
15 | else: | ||
16 | np = os.path.join(name) | ||
17 | if np.startswith('/'): | ||
18 | np = np[1:] | ||
19 | tree['children'].append(dict(name=np)) | ||
20 | return tree | ||
diff --git a/inc/sum.py b/inc/sum.py new file mode 100644 index 00000000..9addf78f --- /dev/null +++ b/inc/sum.py | |||
@@ -0,0 +1,35 @@ | |||
1 | def sha256sum(_): | ||
2 | sha256 = hashlib.sha256() | ||
3 | with io.open(_, mode="rb") as fd: | ||
4 | content = fd.read() | ||
5 | sha256.update(content) | ||
6 | return sha256.hexdigest() | ||
7 | |||
8 | |||
9 | def walksum(_): | ||
10 | sha256 = hashlib.sha256() | ||
11 | x = Path(_) | ||
12 | if not x.exists(): | ||
13 | return -1 | ||
14 | try: | ||
15 | for root, directories, files in os.walk(_): | ||
16 | for names in sorted(files): | ||
17 | filepath = os.path.join(root, names) | ||
18 | try: | ||
19 | fl = open(filepath, 'rb') | ||
20 | except: | ||
21 | fl.close() | ||
22 | continue | ||
23 | while 1: | ||
24 | buf = fl.read(4096) | ||
25 | if not buf: | ||
26 | break | ||
27 | sha256.update(hashlib.sha256(buf).hexdigest()) | ||
28 | fl.close() | ||
29 | except: | ||
30 | import traceback | ||
31 | traceback.print_exc() | ||
32 | return -2 | ||
33 | return sha256.hexdigest() | ||
34 | |||
35 | |||
diff --git a/inc/textproc.py b/inc/textproc.py new file mode 100644 index 00000000..228518b2 --- /dev/null +++ b/inc/textproc.py | |||
@@ -0,0 +1,35 @@ | |||
1 | import html.parser | ||
2 | from bs4 import BeautifulSoup | ||
3 | |||
4 | class extractText(html.parser.HTMLParser): | ||
5 | def __init__(self): | ||
6 | super(extractText, self).__init__() | ||
7 | self.result = [] | ||
8 | def handle_data(self, data): | ||
9 | self.result.append(data) | ||
10 | def text_in(self): | ||
11 | return ''.join(self.result) | ||
12 | |||
13 | |||
14 | def html2text(html): | ||
15 | k = extractText() | ||
16 | k.feed(html) | ||
17 | return k.text_in() | ||
18 | |||
19 | |||
20 | def cut_text(filename, count): | ||
21 | with open(filename) as html: | ||
22 | soup = BeautifulSoup(html, features="lxml") | ||
23 | for script in soup(["script", "style"]): | ||
24 | script.extract() | ||
25 | k = [] | ||
26 | for i in soup.findAll('p')[1]: | ||
27 | k.append(i) | ||
28 | b = ''.join(str(e) for e in k) | ||
29 | text = html2text(b.replace("\n", "")) | ||
30 | textreduced = (text[:count] + '...') if len(text) > count else (text + '..') | ||
31 | return(textreduced) | ||
32 | |||
33 | |||
34 | def cut_news_text(filename, count): | ||
35 | return cut_text("news/" + filename + ".j2", count) | ||