Coverage for liitos/render.py: 81.64%
280 statements
« prev ^ index » next coverage.py v7.6.10, created at 2025-01-05 17:22:35 +00:00
1"""Render the concat document to pdf."""
3import json
4import os
5import pathlib
6import re
7import shutil
8import time
9from typing import Union, no_type_check
11import yaml
13import liitos.captions as cap
14import liitos.concat as con
15import liitos.description_lists as dsc
16import liitos.figures as fig
17import liitos.gather as gat
18import liitos.labels as lab
19import liitos.patch as pat
20import liitos.tables as tab
21import liitos.tools as too
22from liitos import (
23 CONTEXT,
24 ENCODING,
25 FROM_FORMAT_SPEC,
26 LATEX_PAYLOAD_NAME,
27 LOG_SEPARATOR,
28 OptionsType,
29 log,
30 parse_csl,
31)
33DOC_BASE = pathlib.Path('..', '..')
34STRUCTURE_PATH = DOC_BASE / 'structure.yml'
35IMAGES_FOLDER = 'images/'
36DIAGRAMS_FOLDER = 'diagrams/'
37PATCH_SPEC_NAME = 'patch.yml'
38INTER_PROCESS_SYNC_SECS = 0.1
39INTER_PROCESS_SYNC_ATTEMPTS = 10
40VENDORED_SVG_PAT = re.compile(r'^.+\]\([^.]+\.[^.]+\.svg\ .+$')
43@no_type_check
44def read_patches(folder_path: pathlib.Path, patches_path: pathlib.Path) -> tuple[list[tuple[str, str]], bool]:
45 """Ja ja."""
46 patches = []
47 need_patching = False
48 log.info(f'inspecting any patch spec file ({patches_path}) ...')
49 if patches_path.is_file() and patches_path.stat().st_size:
50 target_path = folder_path / PATCH_SPEC_NAME
51 shutil.copy(patches_path, target_path)
52 try:
53 with open(patches_path, 'rt', encoding=ENCODING) as handle:
54 patch_spec = yaml.safe_load(handle)
55 need_patching = True
56 except (OSError, UnicodeDecodeError) as err:
57 log.error(f'failed to load patch spec from ({patches_path}) with ({err}) - patching will be skipped')
58 need_patching = False
59 if need_patching: 59 ↛ 79line 59 didn't jump to line 79 because the condition on line 59 was always true
60 try:
61 patches = [(rep, lace) for rep, lace in patch_spec]
62 patch_pair_count = len(patches)
63 if not patch_pair_count: 63 ↛ 64line 63 didn't jump to line 64 because the condition on line 63 was never true
64 need_patching = False
65 log.warning('- ignoring empty patch spec')
66 else:
67 log.info(
68 f'- loaded {patch_pair_count} patch pair{"" if patch_pair_count == 1 else "s"}'
69 f' from patch spec file ({patches_path})'
70 )
71 except ValueError as err:
72 log.error(f'- failed to parse patch spec from ({patch_spec}) with ({err}) - patching will be skipped')
73 need_patching = False
74 else:
75 if patches_path.is_file(): 75 ↛ 76line 75 didn't jump to line 76 because the condition on line 75 was never true
76 log.warning(f'- ignoring empty patch spec file ({patches_path})')
77 else:
78 log.info(f'- no patch spec file ({patches_path}) detected')
79 return patches, need_patching
82@no_type_check
83def der(
84 doc_root: Union[str, pathlib.Path],
85 structure_name: str,
86 target_key: str,
87 facet_key: str,
88 options: OptionsType,
89) -> int:
90 """Later alligator."""
91 log.info(LOG_SEPARATOR)
92 log.info('entered render function ...')
93 target_code = target_key
94 facet_code = facet_key
95 if not facet_code.strip() or not target_code.strip(): 95 ↛ 96line 95 didn't jump to line 96 because the condition on line 95 was never true
96 log.error(f'render requires non-empty target ({target_code}) and facet ({facet_code}) codes')
97 return 2
98 log.info(f'parsed target ({target_code}) and facet ({facet_code}) from request')
100 from_format_spec = options.get('from_format_spec', FROM_FORMAT_SPEC)
101 filter_cs_list = parse_csl(options.get('filter_cs_list', ''))
102 if filter_cs_list: 102 ↛ 105line 102 didn't jump to line 105 because the condition on line 102 was always true
103 log.info(f'parsed from-format-spec ({from_format_spec}) and filters ({", ".join(filter_cs_list)}) from request')
104 else:
105 log.info(f'parsed from-format-spec ({from_format_spec}) and no filters from request')
107 structure, asset_map = gat.prelude(
108 doc_root=doc_root, structure_name=structure_name, target_key=target_key, facet_key=facet_key, command='render'
109 )
110 log.info(f'prelude teleported processor into the document root at ({os.getcwd()}/)')
112 rel_concat_folder_path = pathlib.Path('render/pdf/')
113 rel_concat_folder_path.mkdir(parents=True, exist_ok=True)
115 patches, need_patching = read_patches(rel_concat_folder_path, pathlib.Path(PATCH_SPEC_NAME))
117 os.chdir(rel_concat_folder_path)
118 log.info(f'render (this processor) teleported into the render/pdf location ({os.getcwd()}/)')
120 log.info(LOG_SEPARATOR)
121 log.info('Assessing the local version control status (compared to upstream) ...')
122 too.ensure_separate_log_lines(too.vcs_probe)
123 CONTEXT['builder_node_id'] = too.node_id()
124 log.info('Context noted with:')
125 log.info(f'- builder-node-id({CONTEXT.get("builder_node_id")})')
126 log.info(f'- source-hash({CONTEXT.get("source_hash")})')
127 log.info(f'- source-hint({CONTEXT.get("source_hint")})')
129 ok, aspect_map = too.load_target(target_code, facet_code)
130 if not ok or not aspect_map: 130 ↛ 131line 130 didn't jump to line 131 because the condition on line 130 was never true
131 return 0 if ok else 1
133 do_render = aspect_map.get('render', None)
134 if do_render is not None: 134 ↛ 137line 134 didn't jump to line 137 because the condition on line 134 was always true
135 log.info(f'found render instruction with value ({aspect_map["render"]})')
137 if do_render is None or do_render: 137 ↛ 140line 137 didn't jump to line 140 because the condition on line 137 was always true
138 log.info('we will render ...')
139 else:
140 log.warning('we will not render ...')
141 return 0xFADECAFE
143 log.info(LOG_SEPARATOR)
144 log.info('transforming SVG assets to high resolution PNG bitmaps ...')
145 for path_to_dir in (IMAGES_FOLDER, DIAGRAMS_FOLDER):
146 the_folder = pathlib.Path(path_to_dir)
147 if not the_folder.is_dir():
148 log.info(
149 f'svg-to-png directory ({the_folder}) in ({pathlib.Path().cwd()}) does not exist or is no directory'
150 f' - trying to create {the_folder}'
151 )
152 try:
153 the_folder.mkdir(parents=True, exist_ok=True)
154 except FileExistsError as err:
155 log.error(f'failed to create {the_folder} - detail: {err}')
156 continue
157 for svg in pathlib.Path(path_to_dir).iterdir():
158 if svg.is_file() and svg.suffix == '.svg':
159 png = str(svg).replace('.svg', '.png')
160 svg_to_png_command = ['svgexport', svg, png, '100%']
161 too.delegate(svg_to_png_command, 'svg-to-png')
163 special_patching = []
164 log.info(LOG_SEPARATOR)
165 log.info('rewriting src attribute values of SVG to PNG sources ...')
166 with open('document.md', 'rt', encoding=ENCODING) as handle:
167 lines = [line.rstrip() for line in handle.readlines()]
168 for slot, line in enumerate(lines):
169 if line.startswith(':
171 if '.svg' in line and line.count('.') >= 2: 171 ↛ 194line 171 didn't jump to line 194 because the condition on line 171 was always true
172 caption, src, alt, rest = con.parse_markdown_image(line)
173 stem, app_indicator, format_suffix = src.rsplit('.', 2)
174 log.info(f'- removing application indicator ({app_indicator}) from src ...')
175 if format_suffix != 'svg': 175 ↛ 176line 175 didn't jump to line 176 because the condition on line 175 was never true
176 log.warning(f' + format_suffix (.{format_suffix}) unexpected in <<{line.rstrip()}>> ...')
177 fine = f'{rest}'
178 log.info(f' transform[#{slot + 1}]: {line}')
179 log.info(f' into[#{slot + 1}]: {fine}')
180 lines[slot] = fine
181 dia_path_old = src.replace('.svg', '.png')
182 dia_path_new = f'{stem}.png'
183 dia_fine_rstrip = dia_path_new.rstrip()
184 if dia_path_old and dia_path_new: 184 ↛ 191line 184 didn't jump to line 191 because the condition on line 184 was always true
185 special_patching.append((dia_path_old, dia_path_new))
186 log.info(
187 f'post-action[#{slot + 1}]: adding to queue for sync move: ({dia_path_old})'
188 f' -> ({dia_path_new})'
189 )
190 else:
191 log.warning(f'- old: {src.rstrip()}')
192 log.warning(f'- new: {dia_fine_rstrip}')
193 continue
194 if '.svg' in line:
195 fine = line.replace('.svg', '.png')
196 log.info(f' transform[#{slot + 1}]: {line}')
197 log.info(f' into[#{slot + 1}]: {fine}')
198 lines[slot] = fine
199 continue
200 with open('document.md', 'wt', encoding=ENCODING) as handle:
201 handle.write('\n'.join(lines))
203 log.info(LOG_SEPARATOR)
204 log.info('ensure diagram files can be found when patched ...')
205 if special_patching:
206 for old, mew in special_patching:
207 source_asset = pathlib.Path(old)
208 target_asset = pathlib.Path(mew)
209 log.info(f'- moving: ({source_asset}) -> ({target_asset})')
210 present = False
211 remaining_attempts = INTER_PROCESS_SYNC_ATTEMPTS
212 while remaining_attempts > 0 and not present: 212 ↛ 225line 212 didn't jump to line 225 because the condition on line 212 was always true
213 try:
214 present = source_asset.is_file()
215 except Exception as ex:
216 log.error(f' * probing for resource ({old}) failed with ({ex}) ... continuing')
217 log.info(
218 f' + resource ({old}) is{" " if present else " NOT "}present at ({source_asset})'
219 f' - attempt {11 - remaining_attempts} of {INTER_PROCESS_SYNC_ATTEMPTS} ...'
220 )
221 if present: 221 ↛ 223line 221 didn't jump to line 223 because the condition on line 221 was always true
222 break
223 time.sleep(INTER_PROCESS_SYNC_SECS)
224 remaining_attempts -= 1
225 if not source_asset.is_file(): 225 ↛ 226line 225 didn't jump to line 226 because the condition on line 225 was never true
226 log.warning(
227 f'- resource ({old}) still not present at ({source_asset})'
228 f' as seen from ({os.getcwd()}) after {remaining_attempts} attempts'
229 f' and ({round(remaining_attempts * INTER_PROCESS_SYNC_SECS, 0) :.0f} seconds waiting)'
230 )
231 shutil.move(source_asset, target_asset)
232 else:
233 log.info('post-action queue (from reference renaming) is empty - nothing to move')
234 log.info(LOG_SEPARATOR)
236 # prototyping >>>
237 fmt_spec = from_format_spec
238 in_doc = 'document.md'
239 out_doc = 'ast-no-filter.json'
240 markdown_to_ast_command = [
241 'pandoc',
242 '--verbose',
243 '-f',
244 fmt_spec,
245 '-t',
246 'json',
247 in_doc,
248 '-o',
249 out_doc,
250 ]
251 log.info(LOG_SEPARATOR)
252 log.info(f'executing ({" ".join(markdown_to_ast_command)}) ...')
253 if code := too.delegate(markdown_to_ast_command, 'markdown-to-ast'): 253 ↛ 254line 253 didn't jump to line 254 because the condition on line 253 was never true
254 return code
256 log.info(LOG_SEPARATOR)
258 mermaid_caption_map = too.mermaid_captions_from_json_ast(out_doc)
259 log.info(LOG_SEPARATOR)
260 # no KISS too.ensure_separate_log_lines(json.dumps, [mermaid_caption_map, 2])
261 for line in json.dumps(mermaid_caption_map, indent=2).split('\n'):
262 for fine in line.split('\n'):
263 log.info(fine)
264 log.info(LOG_SEPARATOR)
266 # <<< prototyping
268 fmt_spec = from_format_spec
269 in_doc = 'document.md'
270 out_doc = LATEX_PAYLOAD_NAME
271 markdown_to_latex_command = [
272 'pandoc',
273 '--verbose',
274 '-f',
275 fmt_spec,
276 '-t',
277 'latex',
278 in_doc,
279 '-o',
280 out_doc,
281 ]
282 if filter_cs_list: 282 ↛ 285line 282 didn't jump to line 285 because the condition on line 282 was always true
283 filters = [added_prefix for expr in filter_cs_list for added_prefix in ('--filter', expr)]
284 markdown_to_latex_command += filters
285 log.info(LOG_SEPARATOR)
286 log.info(f'executing ({" ".join(markdown_to_latex_command)}) ...')
287 if code := too.delegate(markdown_to_latex_command, 'markdown-to-latex'): 287 ↛ 288line 287 didn't jump to line 288 because the condition on line 287 was never true
288 return code
290 log.info(LOG_SEPARATOR)
291 log.info(f'load text lines from intermediate {LATEX_PAYLOAD_NAME} file before internal transforms ...')
292 with open(LATEX_PAYLOAD_NAME, 'rt', encoding=ENCODING) as handle:
293 lines = [line.rstrip() for line in handle.readlines()]
295 patch_counter = 1
296 if options.get('table_caption_below', False): 296 ↛ 297line 296 didn't jump to line 297 because the condition on line 296 was never true
297 lines = too.execute_filter(
298 cap.weave,
299 head='move any captions below tables ...',
300 backup=f'document-before-caps-patch-{patch_counter}.tex.txt',
301 label='captions-below-tables',
302 text_lines=lines,
303 lookup=None,
304 )
305 patch_counter += 1
306 else:
307 log.info('NOT moving captions below tables!')
309 lines = too.execute_filter(
310 lab.inject,
311 head='inject stem (derived from file name) labels ...',
312 backup=f'document-before-inject-stem-label-patch-{patch_counter}.tex.txt',
313 label='inject-stem-derived-labels',
314 text_lines=lines,
315 lookup=mermaid_caption_map,
316 )
317 patch_counter += 1
319 lines = too.execute_filter(
320 fig.scale,
321 head='scale figures ...',
322 backup=f'document-before-scale-figures-patch-{patch_counter}.tex.txt',
323 label='inject-scale-figures',
324 text_lines=lines,
325 lookup=None,
326 )
327 patch_counter += 1
329 lines = too.execute_filter(
330 dsc.options,
331 head='add options to descriptions (definition lists) ...',
332 backup=f'document-before-description-options-patch-{patch_counter}.tex.txt',
333 label='inject-description-options',
334 text_lines=lines,
335 lookup=None,
336 )
337 patch_counter += 1
339 if options.get('patch_tables', False): 339 ↛ 340line 339 didn't jump to line 340 because the condition on line 339 was never true
340 lookup_tunnel = {'table_style': 'ugly' if options.get('table_uglify', False) else 'readable'}
341 lines = too.execute_filter(
342 tab.patch,
343 head='patching tables EXPERIMENTAL (table-shape) ...',
344 backup=f'document-before-table-shape-patch-{patch_counter}.tex.txt',
345 label='changed-table-shape',
346 text_lines=lines,
347 lookup=lookup_tunnel,
348 )
349 patch_counter += 1
350 else:
351 log.info(LOG_SEPARATOR)
352 log.info('not patching tables but commenting out (ignoring) any columns command (table-shape) ...')
353 patched_lines = [f'%IGNORED_{v}' if v.startswith(r'\columns=') else v for v in lines]
354 patched_lines = [f'%IGNORED_{v}' if v.startswith(r'\tablefontsize=') else v for v in patched_lines]
355 log.info('diff of the (ignore-table-shape-if-not-patched) filter result:')
356 too.log_unified_diff(lines, patched_lines)
357 lines = patched_lines
358 log.info(LOG_SEPARATOR)
360 if need_patching:
361 log.info(LOG_SEPARATOR)
362 log.info('apply user patches ...')
363 doc_before_user_patch = f'document-before-user-patch-{patch_counter}.tex.txt'
364 patch_counter += 1
365 with open(doc_before_user_patch, 'wt', encoding=ENCODING) as handle:
366 handle.write('\n'.join(lines))
367 patched_lines = pat.apply(patches, lines)
368 with open(LATEX_PAYLOAD_NAME, 'wt', encoding=ENCODING) as handle:
369 handle.write('\n'.join(patched_lines))
370 log.info('diff of the (user-patches) filter result:')
371 too.log_unified_diff(lines, patched_lines)
372 lines = patched_lines
373 else:
374 log.info(LOG_SEPARATOR)
375 log.info('skipping application of user patches ...')
377 log.info(LOG_SEPARATOR)
378 log.info(f'Internal text line buffer counts {len(lines)} lines')
380 log.info(LOG_SEPARATOR)
381 log.info('cp -a driver.tex this.tex ...')
382 source_asset = 'driver.tex'
383 target_asset = 'this.tex'
384 shutil.copy(source_asset, target_asset)
386 latex_to_pdf_command = ['lualatex', '--shell-escape', 'this.tex']
387 log.info(LOG_SEPARATOR)
388 log.info('1/3) lualatex --shell-escape this.tex ...')
389 if code := too.delegate(latex_to_pdf_command, 'latex-to-pdf(1/3)'): 389 ↛ 390line 389 didn't jump to line 390 because the condition on line 389 was never true
390 return code
392 log.info(LOG_SEPARATOR)
393 log.info('2/3) lualatex --shell-escape this.tex ...')
394 if code := too.delegate(latex_to_pdf_command, 'latex-to-pdf(2/3)'): 394 ↛ 395line 394 didn't jump to line 395 because the condition on line 394 was never true
395 return code
397 log.info(LOG_SEPARATOR)
398 log.info('3/3) lualatex --shell-escape this.tex ...')
399 if code := too.delegate(latex_to_pdf_command, 'latex-to-pdf(3/3)'): 399 ↛ 400line 399 didn't jump to line 400 because the condition on line 399 was never true
400 return code
402 if str(options.get('label', '')).strip(): 402 ↛ 403line 402 didn't jump to line 403 because the condition on line 402 was never true
403 labeling_call = str(options['label']).strip().split()
404 labeling_call.extend(
405 [
406 '--key-value-pairs',
407 (
408 f'BuilderNodeID={CONTEXT["builder_node_id"]}'
409 f',SourceHash={CONTEXT.get("source_hash", "no-source-hash-given")}'
410 f',SourceHint={CONTEXT.get("source_hint", "no-source-hint-given")}'
411 ),
412 ]
413 )
414 log.info(LOG_SEPARATOR)
415 log.info(f'Labeling the resulting pdf file per ({" ".join(labeling_call)})')
416 too.delegate(labeling_call, 'label-pdf')
417 log.info(LOG_SEPARATOR)
419 log.info(LOG_SEPARATOR)
420 log.info('Moving stuff around (result phase) ...')
421 source_asset = 'this.pdf'
422 target_asset = '../index.pdf'
423 shutil.copy(source_asset, target_asset)
425 log.info(LOG_SEPARATOR)
426 log.info('Deliverable taxonomy: ...')
427 too.report_taxonomy(pathlib.Path(target_asset))
429 pdffonts_command = ['pdffonts', target_asset]
430 too.delegate(pdffonts_command, 'assess-pdf-fonts')
432 log.info(LOG_SEPARATOR)
433 log.info('done.')
434 log.info(LOG_SEPARATOR)
436 return 0