Coverage for liitos/render.py: 81.43%
276 statements
« prev ^ index » next coverage.py v7.6.4, created at 2024-11-10 18:56:07 +00:00
1"""Render the concat document to pdf."""
3import json
4import os
5import pathlib
6import re
7import shutil
8import time
9from typing import Union, no_type_check
11import yaml
13import liitos.captions as cap
14import liitos.concat as con
15import liitos.description_lists as dsc
16import liitos.figures as fig
17import liitos.gather as gat
18import liitos.labels as lab
19import liitos.patch as pat
20import liitos.tables as tab
21import liitos.tools as too
22from liitos import (
23 CONTEXT,
24 ENCODING,
25 FROM_FORMAT_SPEC,
26 LATEX_PAYLOAD_NAME,
27 LOG_SEPARATOR,
28 OptionsType,
29 log,
30 parse_csl,
31)
33DOC_BASE = pathlib.Path('..', '..')
34STRUCTURE_PATH = DOC_BASE / 'structure.yml'
35IMAGES_FOLDER = 'images/'
36DIAGRAMS_FOLDER = 'diagrams/'
37PATCH_SPEC_NAME = 'patch.yml'
38INTER_PROCESS_SYNC_SECS = 0.1
39INTER_PROCESS_SYNC_ATTEMPTS = 10
40VENDORED_SVG_PAT = re.compile(r'^.+\]\([^.]+\.[^.]+\.svg\ .+$')
43@no_type_check
44def read_patches(folder_path: pathlib.Path, patches_path: pathlib.Path) -> tuple[list[tuple[str, str]], bool]:
45 """Ja ja."""
46 patches = []
47 need_patching = False
48 log.info(f'inspecting any patch spec file ({patches_path}) ...')
49 if patches_path.is_file() and patches_path.stat().st_size:
50 target_path = folder_path / PATCH_SPEC_NAME
51 shutil.copy(patches_path, target_path)
52 try:
53 with open(patches_path, 'rt', encoding=ENCODING) as handle:
54 patch_spec = yaml.safe_load(handle)
55 need_patching = True
56 except (OSError, UnicodeDecodeError) as err:
57 log.error(f'failed to load patch spec from ({patches_path}) with ({err}) - patching will be skipped')
58 need_patching = False
59 if need_patching: 59 ↛ 79line 59 didn't jump to line 79 because the condition on line 59 was always true
60 try:
61 patches = [(rep, lace) for rep, lace in patch_spec]
62 patch_pair_count = len(patches)
63 if not patch_pair_count: 63 ↛ 64line 63 didn't jump to line 64 because the condition on line 63 was never true
64 need_patching = False
65 log.warning('- ignoring empty patch spec')
66 else:
67 log.info(
68 f'- loaded {patch_pair_count} patch pair{"" if patch_pair_count == 1 else "s"}'
69 f' from patch spec file ({patches_path})'
70 )
71 except ValueError as err:
72 log.error(f'- failed to parse patch spec from ({patch_spec}) with ({err}) - patching will be skipped')
73 need_patching = False
74 else:
75 if patches_path.is_file(): 75 ↛ 76line 75 didn't jump to line 76 because the condition on line 75 was never true
76 log.warning(f'- ignoring empty patch spec file ({patches_path})')
77 else:
78 log.info(f'- no patch spec file ({patches_path}) detected')
79 return patches, need_patching
82@no_type_check
83def der(
84 doc_root: Union[str, pathlib.Path],
85 structure_name: str,
86 target_key: str,
87 facet_key: str,
88 options: OptionsType,
89) -> int:
90 """Later alligator."""
91 log.info(LOG_SEPARATOR)
92 log.info('entered render function ...')
93 target_code = target_key
94 facet_code = facet_key
95 if not facet_code.strip() or not target_code.strip(): 95 ↛ 96line 95 didn't jump to line 96 because the condition on line 95 was never true
96 log.error(f'render requires non-empty target ({target_code}) and facet ({facet_code}) codes')
97 return 2
98 log.info(f'parsed target ({target_code}) and facet ({facet_code}) from request')
100 from_format_spec = options.get('from_format_spec', FROM_FORMAT_SPEC)
101 filter_cs_list = parse_csl(options.get('filter_cs_list', ''))
102 if filter_cs_list: 102 ↛ 105line 102 didn't jump to line 105 because the condition on line 102 was always true
103 log.info(f'parsed from-format-spec ({from_format_spec}) and filters ({", ".join(filter_cs_list)}) from request')
104 else:
105 log.info(f'parsed from-format-spec ({from_format_spec}) and no filters from request')
107 structure, asset_map = gat.prelude(
108 doc_root=doc_root, structure_name=structure_name, target_key=target_key, facet_key=facet_key, command='render'
109 )
110 log.info(f'prelude teleported processor into the document root at ({os.getcwd()}/)')
112 rel_concat_folder_path = pathlib.Path('render/pdf/')
113 rel_concat_folder_path.mkdir(parents=True, exist_ok=True)
115 patches, need_patching = read_patches(rel_concat_folder_path, pathlib.Path(PATCH_SPEC_NAME))
117 os.chdir(rel_concat_folder_path)
118 log.info(f'render (this processor) teleported into the render/pdf location ({os.getcwd()}/)')
120 log.info(LOG_SEPARATOR)
121 log.info('Assessing the local version control status (compared to upstream) ...')
122 too.ensure_separate_log_lines(too.vcs_probe)
123 CONTEXT['builder_node_id'] = too.node_id()
124 log.info('Context noted with:')
125 log.info(f'- builder-node-id({CONTEXT.get("builder_node_id")})')
126 log.info(f'- source-hash({CONTEXT.get("source_hash")})')
127 log.info(f'- source-hint({CONTEXT.get("source_hint")})')
129 ok, aspect_map = too.load_target(target_code, facet_code)
130 if not ok or not aspect_map: 130 ↛ 131line 130 didn't jump to line 131 because the condition on line 130 was never true
131 return 0 if ok else 1
133 do_render = aspect_map.get('render', None)
134 if do_render is not None: 134 ↛ 137line 134 didn't jump to line 137 because the condition on line 134 was always true
135 log.info(f'found render instruction with value ({aspect_map["render"]})')
137 if do_render is None or do_render: 137 ↛ 140line 137 didn't jump to line 140 because the condition on line 137 was always true
138 log.info('we will render ...')
139 else:
140 log.warning('we will not render ...')
141 return 0xFADECAFE
143 log.info(LOG_SEPARATOR)
144 log.info('transforming SVG assets to high resolution PNG bitmaps ...')
145 for path_to_dir in (IMAGES_FOLDER, DIAGRAMS_FOLDER):
146 the_folder = pathlib.Path(path_to_dir)
147 if not the_folder.is_dir(): 147 ↛ 148line 147 didn't jump to line 148 because the condition on line 147 was never true
148 log.error(
149 f'svg-to-png directory ({the_folder}) in ({pathlib.Path().cwd()}) does not exist or is no directory'
150 )
151 continue
152 for svg in pathlib.Path(path_to_dir).iterdir():
153 if svg.is_file() and svg.suffix == '.svg':
154 png = str(svg).replace('.svg', '.png')
155 svg_to_png_command = ['svgexport', svg, png, '100%']
156 too.delegate(svg_to_png_command, 'svg-to-png')
158 special_patching = []
159 log.info(LOG_SEPARATOR)
160 log.info('rewriting src attribute values of SVG to PNG sources ...')
161 with open('document.md', 'rt', encoding=ENCODING) as handle:
162 lines = [line.rstrip() for line in handle.readlines()]
163 for slot, line in enumerate(lines):
164 if line.startswith(':
166 if '.svg' in line and line.count('.') >= 2: 166 ↛ 189line 166 didn't jump to line 189 because the condition on line 166 was always true
167 caption, src, alt, rest = con.parse_markdown_image(line)
168 stem, app_indicator, format_suffix = src.rsplit('.', 2)
169 log.info(f'- removing application indicator ({app_indicator}) from src ...')
170 if format_suffix != 'svg': 170 ↛ 171line 170 didn't jump to line 171 because the condition on line 170 was never true
171 log.warning(f' + format_suffix (.{format_suffix}) unexpected in <<{line.rstrip()}>> ...')
172 fine = f'{rest}'
173 log.info(f' transform[#{slot + 1}]: {line}')
174 log.info(f' into[#{slot + 1}]: {fine}')
175 lines[slot] = fine
176 dia_path_old = src.replace('.svg', '.png')
177 dia_path_new = f'{stem}.png'
178 dia_fine_rstrip = dia_path_new.rstrip()
179 if dia_path_old and dia_path_new: 179 ↛ 186line 179 didn't jump to line 186 because the condition on line 179 was always true
180 special_patching.append((dia_path_old, dia_path_new))
181 log.info(
182 f'post-action[#{slot + 1}]: adding to queue for sync move: ({dia_path_old})'
183 f' -> ({dia_path_new})'
184 )
185 else:
186 log.warning(f'- old: {src.rstrip()}')
187 log.warning(f'- new: {dia_fine_rstrip}')
188 continue
189 if '.svg' in line:
190 fine = line.replace('.svg', '.png')
191 log.info(f' transform[#{slot + 1}]: {line}')
192 log.info(f' into[#{slot + 1}]: {fine}')
193 lines[slot] = fine
194 continue
195 with open('document.md', 'wt', encoding=ENCODING) as handle:
196 handle.write('\n'.join(lines))
198 log.info(LOG_SEPARATOR)
199 log.info('ensure diagram files can be found when patched ...')
200 if special_patching:
201 for old, mew in special_patching:
202 source_asset = pathlib.Path(old)
203 target_asset = pathlib.Path(mew)
204 log.info(f'- moving: ({source_asset}) -> ({target_asset})')
205 present = False
206 remaining_attempts = INTER_PROCESS_SYNC_ATTEMPTS
207 while remaining_attempts > 0 and not present: 207 ↛ 220line 207 didn't jump to line 220 because the condition on line 207 was always true
208 try:
209 present = source_asset.is_file()
210 except Exception as ex:
211 log.error(f' * probing for resource ({old}) failed with ({ex}) ... continuing')
212 log.info(
213 f' + resource ({old}) is{" " if present else " NOT "}present at ({source_asset})'
214 f' - attempt {11 - remaining_attempts} of {INTER_PROCESS_SYNC_ATTEMPTS} ...'
215 )
216 if present: 216 ↛ 218line 216 didn't jump to line 218 because the condition on line 216 was always true
217 break
218 time.sleep(INTER_PROCESS_SYNC_SECS)
219 remaining_attempts -= 1
220 if not source_asset.is_file(): 220 ↛ 221line 220 didn't jump to line 221 because the condition on line 220 was never true
221 log.warning(
222 f'- resource ({old}) still not present at ({source_asset})'
223 f' as seen from ({os.getcwd()}) after {remaining_attempts} attempts'
224 f' and ({round(remaining_attempts * INTER_PROCESS_SYNC_SECS, 0) :.0f} seconds waiting)'
225 )
226 shutil.move(source_asset, target_asset)
227 else:
228 log.info('post-action queue (from reference renaming) is empty - nothing to move')
229 log.info(LOG_SEPARATOR)
231 # prototyping >>>
232 fmt_spec = from_format_spec
233 in_doc = 'document.md'
234 out_doc = 'ast-no-filter.json'
235 markdown_to_ast_command = [
236 'pandoc',
237 '--verbose',
238 '-f',
239 fmt_spec,
240 '-t',
241 'json',
242 in_doc,
243 '-o',
244 out_doc,
245 ]
246 log.info(LOG_SEPARATOR)
247 log.info(f'executing ({" ".join(markdown_to_ast_command)}) ...')
248 if code := too.delegate(markdown_to_ast_command, 'markdown-to-ast'): 248 ↛ 249line 248 didn't jump to line 249 because the condition on line 248 was never true
249 return code
251 log.info(LOG_SEPARATOR)
253 mermaid_caption_map = too.mermaid_captions_from_json_ast(out_doc)
254 log.info(LOG_SEPARATOR)
255 # no KISS too.ensure_separate_log_lines(json.dumps, [mermaid_caption_map, 2])
256 for line in json.dumps(mermaid_caption_map, indent=2).split('\n'):
257 for fine in line.split('\n'):
258 log.info(fine)
259 log.info(LOG_SEPARATOR)
261 # <<< prototyping
263 fmt_spec = from_format_spec
264 in_doc = 'document.md'
265 out_doc = LATEX_PAYLOAD_NAME
266 markdown_to_latex_command = [
267 'pandoc',
268 '--verbose',
269 '-f',
270 fmt_spec,
271 '-t',
272 'latex',
273 in_doc,
274 '-o',
275 out_doc,
276 ]
277 if filter_cs_list: 277 ↛ 280line 277 didn't jump to line 280 because the condition on line 277 was always true
278 filters = [added_prefix for expr in filter_cs_list for added_prefix in ('--filter', expr)]
279 markdown_to_latex_command += filters
280 log.info(LOG_SEPARATOR)
281 log.info(f'executing ({" ".join(markdown_to_latex_command)}) ...')
282 if code := too.delegate(markdown_to_latex_command, 'markdown-to-latex'): 282 ↛ 283line 282 didn't jump to line 283 because the condition on line 282 was never true
283 return code
285 log.info(LOG_SEPARATOR)
286 log.info(f'load text lines from intermediate {LATEX_PAYLOAD_NAME} file before internal transforms ...')
287 with open(LATEX_PAYLOAD_NAME, 'rt', encoding=ENCODING) as handle:
288 lines = [line.rstrip() for line in handle.readlines()]
290 patch_counter = 1
291 if options.get('table_caption_below', False): 291 ↛ 292line 291 didn't jump to line 292 because the condition on line 291 was never true
292 lines = too.execute_filter(
293 cap.weave,
294 head='move any captions below tables ...',
295 backup=f'document-before-caps-patch-{patch_counter}.tex.txt',
296 label='captions-below-tables',
297 text_lines=lines,
298 lookup=None,
299 )
300 patch_counter += 1
301 else:
302 log.info('NOT moving captions below tables!')
304 lines = too.execute_filter(
305 lab.inject,
306 head='inject stem (derived from file name) labels ...',
307 backup=f'document-before-inject-stem-label-patch-{patch_counter}.tex.txt',
308 label='inject-stem-derived-labels',
309 text_lines=lines,
310 lookup=mermaid_caption_map,
311 )
312 patch_counter += 1
314 lines = too.execute_filter(
315 fig.scale,
316 head='scale figures ...',
317 backup=f'document-before-scale-figures-patch-{patch_counter}.tex.txt',
318 label='inject-scale-figures',
319 text_lines=lines,
320 lookup=None,
321 )
322 patch_counter += 1
324 lines = too.execute_filter(
325 dsc.options,
326 head='add options to descriptions (definition lists) ...',
327 backup=f'document-before-description-options-patch-{patch_counter}.tex.txt',
328 label='inject-description-options',
329 text_lines=lines,
330 lookup=None,
331 )
332 patch_counter += 1
334 if options.get('patch_tables', False): 334 ↛ 335line 334 didn't jump to line 335 because the condition on line 334 was never true
335 lookup_tunnel = {'table_style': 'ugly' if options.get('table_uglify', False) else 'readable'}
336 lines = too.execute_filter(
337 tab.patch,
338 head='patching tables EXPERIMENTAL (table-shape) ...',
339 backup=f'document-before-table-shape-patch-{patch_counter}.tex.txt',
340 label='changed-table-shape',
341 text_lines=lines,
342 lookup=lookup_tunnel,
343 )
344 patch_counter += 1
345 else:
346 log.info(LOG_SEPARATOR)
347 log.info('not patching tables but commenting out (ignoring) any columns command (table-shape) ...')
348 patched_lines = [f'%IGNORED_{v}' if v.startswith(r'\columns=') else v for v in lines]
349 patched_lines = [f'%IGNORED_{v}' if v.startswith(r'\tablefontsize=') else v for v in patched_lines]
350 log.info('diff of the (ignore-table-shape-if-not-patched) filter result:')
351 too.log_unified_diff(lines, patched_lines)
352 lines = patched_lines
353 log.info(LOG_SEPARATOR)
355 if need_patching:
356 log.info(LOG_SEPARATOR)
357 log.info('apply user patches ...')
358 doc_before_user_patch = f'document-before-user-patch-{patch_counter}.tex.txt'
359 patch_counter += 1
360 with open(doc_before_user_patch, 'wt', encoding=ENCODING) as handle:
361 handle.write('\n'.join(lines))
362 patched_lines = pat.apply(patches, lines)
363 with open(LATEX_PAYLOAD_NAME, 'wt', encoding=ENCODING) as handle:
364 handle.write('\n'.join(patched_lines))
365 log.info('diff of the (user-patches) filter result:')
366 too.log_unified_diff(lines, patched_lines)
367 lines = patched_lines
368 else:
369 log.info(LOG_SEPARATOR)
370 log.info('skipping application of user patches ...')
372 log.info(LOG_SEPARATOR)
373 log.info(f'Internal text line buffer counts {len(lines)} lines')
375 log.info(LOG_SEPARATOR)
376 log.info('cp -a driver.tex this.tex ...')
377 source_asset = 'driver.tex'
378 target_asset = 'this.tex'
379 shutil.copy(source_asset, target_asset)
381 latex_to_pdf_command = ['lualatex', '--shell-escape', 'this.tex']
382 log.info(LOG_SEPARATOR)
383 log.info('1/3) lualatex --shell-escape this.tex ...')
384 if code := too.delegate(latex_to_pdf_command, 'latex-to-pdf(1/3)'): 384 ↛ 385line 384 didn't jump to line 385 because the condition on line 384 was never true
385 return code
387 log.info(LOG_SEPARATOR)
388 log.info('2/3) lualatex --shell-escape this.tex ...')
389 if code := too.delegate(latex_to_pdf_command, 'latex-to-pdf(2/3)'): 389 ↛ 390line 389 didn't jump to line 390 because the condition on line 389 was never true
390 return code
392 log.info(LOG_SEPARATOR)
393 log.info('3/3) lualatex --shell-escape this.tex ...')
394 if code := too.delegate(latex_to_pdf_command, 'latex-to-pdf(3/3)'): 394 ↛ 395line 394 didn't jump to line 395 because the condition on line 394 was never true
395 return code
397 if str(options.get('label', '')).strip(): 397 ↛ 398line 397 didn't jump to line 398 because the condition on line 397 was never true
398 labeling_call = str(options['label']).strip().split()
399 labeling_call.extend(
400 [
401 '--key-value-pairs',
402 (
403 f'BuilderNodeID={CONTEXT["builder_node_id"]}'
404 f',SourceHash={CONTEXT.get("source_hash", "no-source-hash-given")}'
405 f',SourceHint={CONTEXT.get("source_hint", "no-source-hint-given")}'
406 ),
407 ]
408 )
409 log.info(LOG_SEPARATOR)
410 log.info(f'Labeling the resulting pdf file per ({" ".join(labeling_call)})')
411 too.delegate(labeling_call, 'label-pdf')
412 log.info(LOG_SEPARATOR)
414 log.info(LOG_SEPARATOR)
415 log.info('Moving stuff around (result phase) ...')
416 source_asset = 'this.pdf'
417 target_asset = '../index.pdf'
418 shutil.copy(source_asset, target_asset)
420 log.info(LOG_SEPARATOR)
421 log.info('Deliverable taxonomy: ...')
422 too.report_taxonomy(pathlib.Path(target_asset))
424 pdffonts_command = ['pdffonts', target_asset]
425 too.delegate(pdffonts_command, 'assess-pdf-fonts')
427 log.info(LOG_SEPARATOR)
428 log.info('done.')
429 log.info(LOG_SEPARATOR)
431 return 0