Coverage for liitos/render.py: 81.62%
283 statements
« prev ^ index » next coverage.py v7.10.6, created at 2025-08-31 13:07:35 +00:00
1"""Render the concat document to pdf."""
3import json
4import os
5import pathlib
6import re
7import shutil
8import time
9from typing import Union, no_type_check
11import yaml
13import liitos.captions as cap
14import liitos.concat as con
15import liitos.description_lists as dsc
16import liitos.figures as fig
17import liitos.gather as gat
18import liitos.labels as lab
19import liitos.patch as pat
20import liitos.tables as tab
21import liitos.tools as too
22from liitos import (
23 CONTEXT,
24 ENCODING,
25 FROM_FORMAT_SPEC,
26 LATEX_PAYLOAD_NAME,
27 LOG_SEPARATOR,
28 OptionsType,
29 log,
30 parse_csl,
31)
33DOC_BASE = pathlib.Path('..', '..')
34STRUCTURE_PATH = DOC_BASE / 'structure.yml'
35IMAGES_FOLDER = 'images/'
36DIAGRAMS_FOLDER = 'diagrams/'
37PATCH_SPEC_NAME = 'patch.yml'
38INTER_PROCESS_SYNC_SECS = 0.1
39INTER_PROCESS_SYNC_ATTEMPTS = 10
40VENDORED_SVG_PAT = re.compile(r'^.+\]\([^.]+\.[^.]+\.svg\ .+$')
43@no_type_check
44def read_patches(folder_path: pathlib.Path, patches_path: pathlib.Path) -> tuple[list[tuple[str, str]], bool]:
45 """Obtain any search-replace pairs from user patching file."""
46 patches = []
47 need_patching = False
48 log.info(f'inspecting any patch spec file ({patches_path}) ...')
49 if patches_path.is_file() and patches_path.stat().st_size:
50 target_path = folder_path / PATCH_SPEC_NAME
51 shutil.copy(patches_path, target_path)
52 try:
53 with open(patches_path, 'rt', encoding=ENCODING) as handle:
54 patch_spec = yaml.safe_load(handle)
55 need_patching = True
56 except (OSError, UnicodeDecodeError) as err:
57 log.error(f'failed to load patch spec from ({patches_path}) with ({err}) - patching will be skipped')
58 need_patching = False
59 if need_patching: 59 ↛ 79line 59 didn't jump to line 79 because the condition on line 59 was always true
60 try:
61 patches = [(rep, lace) for rep, lace in patch_spec]
62 patch_pair_count = len(patches)
63 if not patch_pair_count: 63 ↛ 64line 63 didn't jump to line 64 because the condition on line 63 was never true
64 need_patching = False
65 log.warning('- ignoring empty patch spec')
66 else:
67 log.info(
68 f'- loaded {patch_pair_count} patch pair{"" if patch_pair_count == 1 else "s"}'
69 f' from patch spec file ({patches_path})'
70 )
71 except ValueError as err:
72 log.error(f'- failed to parse patch spec from ({patch_spec}) with ({err}) - patching will be skipped')
73 need_patching = False
74 else:
75 if patches_path.is_file(): 75 ↛ 76line 75 didn't jump to line 76 because the condition on line 75 was never true
76 log.warning(f'- ignoring empty patch spec file ({patches_path})')
77 else:
78 log.info(f'- no patch spec file ({patches_path}) detected')
79 return patches, need_patching
82@no_type_check
83def der(
84 doc_root: Union[str, pathlib.Path],
85 structure_name: str,
86 target_key: str,
87 facet_key: str,
88 options: OptionsType,
89) -> int:
90 """Render the document as PDF, eventually."""
91 log.info(LOG_SEPARATOR)
92 log.info('entered render function ...')
93 target_code = target_key
94 facet_code = facet_key
95 if not facet_code.strip() or not target_code.strip(): 95 ↛ 96line 95 didn't jump to line 96 because the condition on line 95 was never true
96 log.error(f'render requires non-empty target ({target_code}) and facet ({facet_code}) codes')
97 return 2
98 log.info(f'parsed target ({target_code}) and facet ({facet_code}) from request')
100 from_format_spec = options.get('from_format_spec', FROM_FORMAT_SPEC)
101 filter_cs_list = parse_csl(options.get('filter_cs_list', ''))
102 if filter_cs_list: 102 ↛ 105line 102 didn't jump to line 105 because the condition on line 102 was always true
103 log.info(f'parsed from-format-spec ({from_format_spec}) and filters ({", ".join(filter_cs_list)}) from request')
104 else:
105 log.info(f'parsed from-format-spec ({from_format_spec}) and no filters from request')
107 structure, asset_map = gat.prelude(
108 doc_root=doc_root, structure_name=structure_name, target_key=target_key, facet_key=facet_key, command='render'
109 )
110 log.info(f'prelude teleported processor into the document root at ({os.getcwd()}/)')
112 rel_concat_folder_path = pathlib.Path('render/pdf/')
113 rel_concat_folder_path.mkdir(parents=True, exist_ok=True)
115 patches, need_patching = read_patches(rel_concat_folder_path, pathlib.Path(PATCH_SPEC_NAME))
117 os.chdir(rel_concat_folder_path)
118 log.info(f'render (this processor) teleported into the render/pdf location ({os.getcwd()}/)')
120 log.info(LOG_SEPARATOR)
121 log.warning('Assessing the local version control status (compared to upstream) ...')
122 too.ensure_separate_log_lines(too.vcs_probe, log.warning)
123 CONTEXT['builder_node_id'] = too.node_id()
124 log.warning('Context noted with:')
125 log.warning(f'- builder-node-id({CONTEXT.get("builder_node_id")})')
126 log.warning(f'- source-hash({CONTEXT.get("source_hash")})')
127 log.warning(f'- source-hint({CONTEXT.get("source_hint")})')
129 ok, aspect_map = too.load_target(target_code, facet_code)
130 if not ok or not aspect_map: 130 ↛ 131line 130 didn't jump to line 131 because the condition on line 130 was never true
131 return 0 if ok else 1
133 is_quiet = options.get('quiet', False)
134 do_render = aspect_map.get('render', None)
135 if do_render is not None: 135 ↛ 138line 135 didn't jump to line 138 because the condition on line 135 was always true
136 log.info(f'found render instruction with value ({aspect_map["render"]})')
138 if do_render is None or do_render or options['force']: 138 ↛ 142line 138 didn't jump to line 142 because the condition on line 138 was always true
139 why = 'default-render' if do_render is None else ('render-true' if do_render else 'render-force')
140 log.warning(f'we will render ({why=}) ...')
141 else:
142 log.warning('we will not render ...')
143 return 0xFADECAFE
145 log.info(LOG_SEPARATOR)
146 log.info('transforming SVG assets to high resolution PNG bitmaps ...')
147 for path_to_dir in (IMAGES_FOLDER, DIAGRAMS_FOLDER):
148 the_folder = pathlib.Path(path_to_dir)
149 if not the_folder.is_dir():
150 log.info(
151 f'svg-to-png directory ({the_folder}) in ({pathlib.Path().cwd()}) does not exist or is no directory'
152 f' - trying to create {the_folder}'
153 )
154 try:
155 the_folder.mkdir(parents=True, exist_ok=True)
156 except FileExistsError as err:
157 log.error(f'failed to create {the_folder} - detail: {err}')
158 continue
159 for svg in pathlib.Path(path_to_dir).iterdir():
160 if svg.is_file() and svg.suffix == '.svg':
161 png = str(svg).replace('.svg', '.png')
162 svg_to_png_command = ['svgexport', svg, png, '100%']
163 too.delegate(svg_to_png_command, 'svg-to-png', is_quiet=is_quiet)
165 special_patching = []
166 log.info(LOG_SEPARATOR)
167 log.info('rewriting src attribute values of SVG to PNG sources ...')
168 with open('document.md', 'rt', encoding=ENCODING) as handle:
169 lines = [line.rstrip() for line in handle.readlines()]
170 for slot, line in enumerate(lines):
171 if line.startswith(':
173 if '.svg' in line and line.count('.') >= 2: 173 ↛ 195line 173 didn't jump to line 195 because the condition on line 173 was always true
174 caption, src, alt, rest = con.parse_markdown_image(line)
175 stem, app_indicator, format_suffix = src.rsplit('.', 2)
176 log.info(f'- removing application indicator ({app_indicator}) from src ...')
177 if format_suffix != 'svg': 177 ↛ 178line 177 didn't jump to line 178 because the condition on line 177 was never true
178 log.warning(f' + format_suffix (.{format_suffix}) unexpected in <<{line.rstrip()}>> ...')
179 fine = f'{rest}'
180 log.info(f' transform[#{slot + 1}]: {line}')
181 log.info(f' into[#{slot + 1}]: {fine}')
182 lines[slot] = fine
183 dia_path_old = src.replace('.svg', '.png')
184 dia_path_new = f'{stem}.png'
185 if dia_path_old and dia_path_new: 185 ↛ 192line 185 didn't jump to line 192 because the condition on line 185 was always true
186 special_patching.append((dia_path_old, dia_path_new))
187 log.info(
188 f'post-action[#{slot + 1}]: adding to queue for sync move: ({dia_path_old})'
189 f' -> ({dia_path_new})'
190 )
191 else:
192 log.warning(f'- old: {src.rstrip()}')
193 log.warning(f'- new: {dia_path_new.rstrip()}')
194 continue
195 if '.svg' in line:
196 fine = line.replace('.svg', '.png')
197 log.info(f' transform[#{slot + 1}]: {line}')
198 log.info(f' into[#{slot + 1}]: {fine}')
199 lines[slot] = fine
200 continue
201 with open('document.md', 'wt', encoding=ENCODING) as handle:
202 handle.write('\n'.join(lines))
204 log.info(LOG_SEPARATOR)
205 log.info('ensure diagram files can be found when patched ...')
206 if special_patching:
207 for old, mew in special_patching:
208 source_asset = pathlib.Path(old)
209 target_asset = pathlib.Path(mew)
210 log.info(f'- moving: ({source_asset}) -> ({target_asset})')
211 present = False
212 remaining_attempts = INTER_PROCESS_SYNC_ATTEMPTS
213 while remaining_attempts > 0 and not present: 213 ↛ 226line 213 didn't jump to line 226 because the condition on line 213 was always true
214 try:
215 present = source_asset.is_file()
216 except Exception as ex:
217 log.error(f' * probing for resource ({old}) failed with ({ex}) ... continuing')
218 log.info(
219 f' + resource ({old}) is{" " if present else " NOT "}present at ({source_asset})'
220 f' - attempt {11 - remaining_attempts} of {INTER_PROCESS_SYNC_ATTEMPTS} ...'
221 )
222 if present: 222 ↛ 224line 222 didn't jump to line 224 because the condition on line 222 was always true
223 break
224 time.sleep(INTER_PROCESS_SYNC_SECS)
225 remaining_attempts -= 1
226 if not source_asset.is_file(): 226 ↛ 227line 226 didn't jump to line 227 because the condition on line 226 was never true
227 log.warning(
228 f'- resource ({old}) still not present at ({source_asset})'
229 f' as seen from ({os.getcwd()}) after {remaining_attempts} attempts'
230 f' and ({round(remaining_attempts * INTER_PROCESS_SYNC_SECS, 0) :.0f} seconds waiting)'
231 )
232 elif target_asset.is_file(): 232 ↛ 234line 232 didn't jump to line 234 because the condition on line 232 was always true
233 log.warning(f'overwriting existing {target_asset} from {source_asset}')
234 shutil.move(source_asset, target_asset)
235 else:
236 log.info('post-action queue (from reference renaming) is empty - nothing to move')
237 log.info(LOG_SEPARATOR)
239 fmt_spec = from_format_spec
240 in_doc = 'document.md'
241 out_doc = 'ast-no-filter.json'
242 markdown_to_ast_command = [
243 'pandoc',
244 '--verbose',
245 '-f',
246 fmt_spec,
247 '-t',
248 'json',
249 in_doc,
250 '-o',
251 out_doc,
252 ]
253 log.info(LOG_SEPARATOR)
254 log.info(f'executing ({" ".join(markdown_to_ast_command)}) ...')
255 if code := too.delegate(markdown_to_ast_command, 'markdown-to-ast', is_quiet=is_quiet): 255 ↛ 256line 255 didn't jump to line 256 because the condition on line 255 was never true
256 return code
258 log.info(LOG_SEPARATOR)
260 mermaid_caption_map = too.mermaid_captions_from_json_ast(out_doc)
261 log.info(LOG_SEPARATOR)
262 # no KISS too.ensure_separate_log_lines(json.dumps, [mermaid_caption_map, 2])
263 for line in json.dumps(mermaid_caption_map, indent=2).split('\n'):
264 for fine in line.split('\n'):
265 log.info(fine)
266 log.info(LOG_SEPARATOR)
268 fmt_spec = from_format_spec
269 in_doc = 'document.md'
270 out_doc = LATEX_PAYLOAD_NAME
271 markdown_to_latex_command = [
272 'pandoc',
273 '--verbose',
274 '-f',
275 fmt_spec,
276 '-t',
277 'latex',
278 in_doc,
279 '-o',
280 out_doc,
281 ]
282 if filter_cs_list: 282 ↛ 285line 282 didn't jump to line 285 because the condition on line 282 was always true
283 filters = [added_prefix for expr in filter_cs_list for added_prefix in ('--filter', expr)]
284 markdown_to_latex_command += filters
285 log.info(LOG_SEPARATOR)
286 log.warning(f'executing ({" ".join(markdown_to_latex_command)}) ...')
287 if code := too.delegate(markdown_to_latex_command, 'markdown-to-latex', is_quiet=is_quiet): 287 ↛ 288line 287 didn't jump to line 288 because the condition on line 287 was never true
288 return code
290 log.info(LOG_SEPARATOR)
291 log.info(f'load text lines from intermediate {LATEX_PAYLOAD_NAME} file before internal transforms ...')
292 with open(LATEX_PAYLOAD_NAME, 'rt', encoding=ENCODING) as handle:
293 lines = [line.rstrip() for line in handle.readlines()]
295 patch_counter = 1
296 if options.get('table_caption_below', False): 296 ↛ 297line 296 didn't jump to line 297 because the condition on line 296 was never true
297 lines = too.execute_filter(
298 cap.weave,
299 head='move any captions below tables ...',
300 backup=f'document-before-caps-patch-{patch_counter}.tex.txt',
301 label='captions-below-tables',
302 text_lines=lines,
303 lookup=None,
304 )
305 patch_counter += 1
306 else:
307 log.info('NOT moving captions below tables!')
309 lines = too.execute_filter(
310 lab.inject,
311 head='inject stem (derived from file name) labels ...',
312 backup=f'document-before-inject-stem-label-patch-{patch_counter}.tex.txt',
313 label='inject-stem-derived-labels',
314 text_lines=lines,
315 lookup=mermaid_caption_map,
316 )
317 patch_counter += 1
319 lines = too.execute_filter(
320 fig.scale,
321 head='scale figures ...',
322 backup=f'document-before-scale-figures-patch-{patch_counter}.tex.txt',
323 label='inject-scale-figures',
324 text_lines=lines,
325 lookup=None,
326 )
327 patch_counter += 1
329 lines = too.execute_filter(
330 dsc.options,
331 head='add options to descriptions (definition lists) ...',
332 backup=f'document-before-description-options-patch-{patch_counter}.tex.txt',
333 label='inject-description-options',
334 text_lines=lines,
335 lookup=None,
336 )
337 patch_counter += 1
339 if options.get('patch_tables', False): 339 ↛ 340line 339 didn't jump to line 340 because the condition on line 339 was never true
340 lookup_tunnel = {'table_style': 'ugly' if options.get('table_uglify', False) else 'readable'}
341 lines = too.execute_filter(
342 tab.patch,
343 head='patching tables EXPERIMENTAL (table-shape) ...',
344 backup=f'document-before-table-shape-patch-{patch_counter}.tex.txt',
345 label='changed-table-shape',
346 text_lines=lines,
347 lookup=lookup_tunnel,
348 )
349 patch_counter += 1
350 else:
351 log.info(LOG_SEPARATOR)
352 log.info('not patching tables but commenting out (ignoring) any columns command (table-shape) ...')
353 patched_lines = [f'%IGNORED_{v}' if v.startswith(r'\columns=') else v for v in lines]
354 patched_lines = [f'%IGNORED_{v}' if v.startswith(r'\tablefontsize=') else v for v in patched_lines]
355 log.info('diff of the (ignore-table-shape-if-not-patched) filter result:')
356 too.log_unified_diff(lines, patched_lines)
357 lines = patched_lines
358 log.info(LOG_SEPARATOR)
360 if need_patching:
361 log.info(LOG_SEPARATOR)
362 log.info('apply user patches ...')
363 doc_before_user_patch = f'document-before-user-patch-{patch_counter}.tex.txt'
364 patch_counter += 1
365 with open(doc_before_user_patch, 'wt', encoding=ENCODING) as handle:
366 handle.write('\n'.join(lines))
367 patched_lines = pat.apply(patches, lines)
368 with open(LATEX_PAYLOAD_NAME, 'wt', encoding=ENCODING) as handle:
369 handle.write('\n'.join(patched_lines))
370 log.info('diff of the (user-patches) filter result:')
371 too.log_unified_diff(lines, patched_lines)
372 lines = patched_lines
373 else:
374 log.info(LOG_SEPARATOR)
375 log.info('skipping application of user patches ...')
377 log.info(LOG_SEPARATOR)
378 log.info(f'Internal text line buffer counts {len(lines)} lines')
380 log.info(LOG_SEPARATOR)
381 log.info('cp -a driver.tex this.tex ...')
382 source_asset = 'driver.tex'
383 target_asset = 'this.tex'
384 shutil.copy(source_asset, target_asset)
386 latex_to_pdf_command = ['lualatex', '--shell-escape', 'this.tex']
387 log.info(LOG_SEPARATOR)
388 log.warning('1/3) lualatex --shell-escape this.tex ...')
389 if code := too.delegate(latex_to_pdf_command, 'latex-to-pdf(1/3)', is_quiet=is_quiet): 389 ↛ 390line 389 didn't jump to line 390 because the condition on line 389 was never true
390 return code
392 log.info(LOG_SEPARATOR)
393 log.warning('2/3) lualatex --shell-escape this.tex ...')
394 if code := too.delegate(latex_to_pdf_command, 'latex-to-pdf(2/3)', is_quiet=is_quiet): 394 ↛ 395line 394 didn't jump to line 395 because the condition on line 394 was never true
395 return code
397 log.info(LOG_SEPARATOR)
398 log.warning('3/3) lualatex --shell-escape this.tex ...')
399 if code := too.delegate(latex_to_pdf_command, 'latex-to-pdf(3/3)', is_quiet=is_quiet): 399 ↛ 400line 399 didn't jump to line 400 because the condition on line 399 was never true
400 return code
402 if str(options.get('label', '')).strip(): 402 ↛ 403line 402 didn't jump to line 403 because the condition on line 402 was never true
403 labeling_call = str(options['label']).strip().split()
404 labeling_call.extend(
405 [
406 '--key-value-pairs',
407 (
408 f'BuilderNodeID={CONTEXT["builder_node_id"]}'
409 f',SourceHash={CONTEXT.get("source_hash", "no-source-hash-given")}'
410 f',SourceHint={CONTEXT.get("source_hint", "no-source-hint-given")}'
411 ),
412 ]
413 )
414 log.info(LOG_SEPARATOR)
415 log.warning(f'Labeling the resulting pdf file per ({" ".join(labeling_call)})')
416 too.delegate(labeling_call, 'label-pdf', is_quiet=is_quiet)
417 log.info(LOG_SEPARATOR)
419 log.info(LOG_SEPARATOR)
420 log.warning('Moving stuff around (result phase) ...')
421 source_asset = 'this.pdf'
422 target_asset = '../index.pdf'
423 shutil.copy(source_asset, target_asset)
425 log.info(LOG_SEPARATOR)
426 log.warning('Deliverable taxonomy: ...')
427 too.report_taxonomy(pathlib.Path(target_asset))
429 pdffonts_command = ['pdffonts', target_asset]
430 too.delegate(pdffonts_command, 'assess-pdf-fonts', is_quiet=is_quiet)
432 log.info(LOG_SEPARATOR)
433 log.warning('done.')
434 log.info(LOG_SEPARATOR)
436 return 0