latexml/post.rs
1//! Post-processing pipeline API.
2//!
3//! Provides a public interface to the LaTeXML post-processing pipeline
4//! (Scan → Bibliography → CrossRef → Graphics → Split → MathML → XSLT → HTML5 fixups).
5//! Used by both the `latexml_oxide` binary and the `cortex_worker` binary.
6
7use latexml_core::{
8 Info, Warn,
9 common::error::{emit_error, note_progress},
10 s,
11 telemetry::{self, Phase},
12};
13use latexml_post::{
14 document::{PostDocument, PostDocumentOptions},
15 object_db::ObjectDB,
16 processor::Processor,
17};
18use once_cell::sync::Lazy;
19
20// Process-once cached env var (see WISDOM #56 — getenv hot-path race).
21static POST_AUDIT: Lazy<bool> = Lazy::new(|| std::env::var("LATEXML_POST_AUDIT").is_ok());
22
23/// Start an audit timer when `LATEXML_POST_AUDIT` is set. Module-level (not a
24/// closure) so [`render_spilled_page`] — shared by the serial driver and the
25/// parallel render worker — can use the identical instrumentation.
26fn audit_start(name: &str) -> Option<(String, std::time::Instant)> {
27 if *POST_AUDIT {
28 Some((name.to_string(), std::time::Instant::now()))
29 } else {
30 None
31 }
32}
33
34/// Report an audit timer started by [`audit_start`] (no-op when audit is off).
35fn audit_end(started: Option<(String, std::time::Instant)>) {
36 if let Some((name, t0)) = started {
37 let ms = t0.elapsed().as_millis();
38 Info!("audit", "phase", s!("{} took {}ms", name, ms));
39 }
40}
41
42/// Options for the post-processing pipeline.
43pub struct PostOptions<'a> {
44 pub pmml: bool,
45 pub cmml: bool,
46 pub keep_xmath: bool,
47 pub stylesheet: Option<&'a str>,
48 pub destination: Option<&'a str>,
49 /// Base directory of the original LaTeX source (`--sourcedirectory`; Perl
50 /// Config.pm L147, passed to Post as `sourceDirectory` in LaTeXML.pm L429).
51 /// Used to locate graphics/resources the post-phase copies. When `None`,
52 /// the caller defaults it to the source file's own directory.
53 pub source_directory: Option<&'a str>,
54 /// Root directory of the generated site (`--sitedirectory`; Perl Config.pm
55 /// L146, passed to Post as `siteDirectory` in LaTeXML.pm L430). Establishes
56 /// the base against which cross-document/resource URLs are made relative.
57 /// When `None`, Post defaults it to the destination's directory (Perl
58 /// Config.pm L466-469; `document.rs` mirrors that fallback).
59 pub site_directory: Option<&'a str>,
60 /// Extra resource search paths (the `--path` flag): directories searched
61 /// for `--css`/`--javascript` files (and other post resources) to copy into
62 /// the destination, in addition to the document's own paths, the current
63 /// directory, and the binary's embedded resource table.
64 pub search_paths: &'a [String],
65 pub nodefaultresources: bool,
66 pub css_files: &'a [String],
67 pub js_files: &'a [String],
68 pub noinvisibletimes: bool,
69 /// Remap styled alphanumerics to Unicode Plane-1 (Perl `--plane1`, default on).
70 pub plane1: bool,
71 /// Remap only the poorly-supported variants, to their simpler form
72 /// (Perl `--hackplane1`); implies `plane1`.
73 pub hackplane1: bool,
74 pub mathtex: bool,
75 /// Cross-reference URL style (Perl `--urlstyle`; `CrossRef::generateURL`).
76 /// Defaults to [`UrlStyle::File`](latexml_post::crossref::UrlStyle::File) at
77 /// every non-CLI construction site (the binary threads the parsed `--urlstyle`
78 /// value here).
79 pub url_style: latexml_post::crossref::UrlStyle,
80 pub navigationtoc: Option<&'a str>,
81 pub schemadocs: bool,
82 pub split: bool,
83 pub split_xpath: Option<String>,
84 pub split_naming: Option<&'a str>,
85 pub xslt_parameters: &'a [String],
86 /// If > 0, try the vector-SVG converters (mutool → pdftocairo) for PDF
87 /// graphics smaller than this many KB (vector-preservation path). Fall
88 /// back to the raster ImageMagick `convert`/`gs` → PNG path on failure
89 /// or timeout. Tracks upstream brucemiller/LaTeXML#902.
90 pub graphics_svg_threshold_kb: u32,
91 /// Convert `\includegraphics` figures to web images (Perl `graphicimages!`
92 /// → `dographics`, default on). When `false` (`--nographicimages`) the
93 /// Graphics post-phase is skipped entirely: the output keeps the raw
94 /// `<ltx:graphics>` references untouched — useful for faster runs or hosts
95 /// without the image tools.
96 pub graphicimages: bool,
97 /// Timestamp string embedded in the generated page (Perl `--timestamp`,
98 /// XSLT `TIMESTAMP` param). `None` → no timestamp emitted (the deterministic
99 /// default; Perl instead defaults to the current time). Perl's
100 /// `--timestamp=0` "omit" case is normalized to `None` by the caller.
101 pub timestamp: Option<&'a str>,
102 /// Favicon resource for the generated site (Perl `--icon`, XSLT `ICON`
103 /// param): emitted as a `<link rel="icon">` and copied to the destination.
104 pub icon: Option<&'a str>,
105 /// Output extraction mode (Perl `LaTeXML::Util::Pack::whatsout`).
106 /// `Document` (default) → serialize the full post-processed
107 /// document; `Fragment` → embeddable HTML snippet via
108 /// `latexml_post::extract::get_embeddable`; `Math` → math subtree
109 /// via `get_math`. Applied to each `PostDocument` in the final
110 /// serialization loop.
111 pub whatsout: latexml_post::extract::Whatsout,
112}
113
114/// The built-in XSLT stylesheet for an output format (logical embedded-resource
115/// name; Perl `Config.pm` L543-551). Returns `None` for a format with no
116/// default (e.g. `xml`), so a caller keeps `--stylesheet` / no post.
117///
118/// SINGLE SOURCE OF TRUTH shared by the CLI (`bin/latexml_oxide.rs` picks
119/// `effective_stylesheet` when `--stylesheet` is unset) and the library
120/// (`crate::api::convert_to_html`), so the two can never disagree on which
121/// sheet an `html5`/`xhtml`/`epub` job gets.
122pub fn default_stylesheet(format: Option<&str>) -> Option<&'static str> {
123 match format {
124 Some("html5") => Some("resources/XSLT/LaTeXML-html5.xsl"),
125 Some("html") | Some("xhtml") => Some("resources/XSLT/LaTeXML-all-xhtml.xsl"),
126 Some("epub") | Some("epub3") => Some("resources/XSLT/LaTeXML-epub3.xsl"),
127 _ => None,
128 }
129}
130
131/// Emit a post-processing error: capture it into the active log buffer (so it
132/// reaches `cortex.log`) AND bump the Error counter so `Status:conversion`
133/// reflects post-phase failures (Perl `LaTeXML.pm` L633: `max(core, post)`).
134///
135/// Deliberately NOT the `Error!` macro: that expands to a too-many-errors
136/// `Fatal!(…)` → `return Err(error::Error)`, which only typechecks inside the
137/// digester's `Result<_, error::Error>` functions. The post-processing entry
138/// points return `String`/`Result<_, ()>`, so we log+count directly with no
139/// control-flow side effect.
140fn post_error(object: &str, message: &str) { emit_error("post", object, message); }
141
142/// Result of [`run_post_processing_logged`]: the serialized HTML plus the
143/// post-phase log text and status code, mirroring Perl `LaTeXML.pm`'s
144/// `convert_post`, which flushes the log AFTER post and folds the post status
145/// into the final `max(core, post)` verdict (L631-634).
146pub struct PostOutcome {
147 /// Serialized post-processed output (same value as [`run_post_processing`]).
148 pub html: String,
149 /// Log text captured during the post phase only — Graphics/MathML/XSLT
150 /// `Info!`/`Warn!`/`post_error` lines. Append to the core conversion log.
151 pub log: String,
152 /// Status code read from the shared REPORT counter AFTER post. Because the
153 /// counter is NOT reset between core and post, this is already the combined
154 /// core+post code; callers still `max()` it with the core code as a
155 /// belt-and-suspenders guard for paths that force a fatal outside REPORT
156 /// (e.g. a `catch_unwind`-trapped panic).
157 pub status_code: usize,
158}
159
160/// Run post-processing while capturing its log into a fresh thread-local
161/// LOG_BUFFER. `Converter::convert()` already flushed the *core* log into its
162/// own response, leaving the buffer unbound — so without re-binding here every
163/// post-phase `Info!`/`Warn!`/`post_error` (incl. a silent EPS→PNG failure)
164/// reaches stderr only and never the persisted `--log`/`cortex.log`. Perl's
165/// single `flush_log()` after `convert_post` sweeps up the post log for ALL
166/// consumers; this is the Rust equivalent, shared by the `latexml_oxide` and
167/// `cortex_worker` binaries so their logs reach parity. See SYNC_STATUS task 5.
168pub fn run_post_processing_logged(xml: &str, opts: &PostOptions) -> PostOutcome {
169 run_post_processing_impl_logged(PostInput::Xml(xml), opts)
170}
171
172/// File-input variant of [`run_post_processing_logged`]: parses the XML from
173/// disk via libxml2's streaming reader rather than from an in-memory `String`.
174/// Used for already-converted `.xml` inputs, where slurping the file would keep
175/// a whole extra copy of a multi-hundred-MB document resident alongside the DOM.
176pub fn run_post_processing_from_file_logged(path: &str, opts: &PostOptions) -> PostOutcome {
177 run_post_processing_impl_logged(PostInput::File(path), opts)
178}
179
180fn run_post_processing_impl_logged(input: PostInput, opts: &PostOptions) -> PostOutcome {
181 latexml_core::util::logger::bind_log();
182 let html = run_post_processing_impl(input, opts);
183 let log = latexml_core::util::logger::flush_log();
184 let status_code = latexml_core::common::error::get_status_code();
185 PostOutcome { html, log, status_code }
186}
187
188/// Where the post-pipeline reads its already-converted LaTeXML XML from.
189#[derive(Clone, Copy)]
190enum PostInput<'a> {
191 /// XML already resident in memory (the TeX→XML conversion result). The common
192 /// path — the 2.8M-doc arXiv fleet and every TeX conversion feed this.
193 Xml(&'a str),
194 /// A path to an XML file, parsed via libxml2's streaming file reader
195 /// (`xmlReadIO`). Avoids vivifying a very large document as a Rust `String`
196 /// on top of the ~11× DOM it becomes.
197 File(&'a str),
198}
199
200/// Run the post-processing pipeline on in-memory XML output.
201///
202/// Executes: Split → Scan → MakeBibliography → CrossRef → Graphics → MathML → XSLT → HTML5 fixups.
203pub fn run_post_processing(xml: &str, opts: &PostOptions) -> String {
204 run_post_processing_impl(PostInput::Xml(xml), opts)
205}
206
207/// File-input variant of [`run_post_processing`].
208pub fn run_post_processing_from_file(path: &str, opts: &PostOptions) -> String {
209 run_post_processing_impl(PostInput::File(path), opts)
210}
211
212/// The in-memory parse ceiling: libxml2's `xmlReadMemory` takes the buffer
213/// length as a C `int`, so a handoff string of `i32::MAX` bytes or more
214/// CANNOT be parsed from memory — it fails with "Document too large for
215/// i32". First witness to cross it: the 131 MB book's 2.68 GB core XML,
216/// single-invocation `.tex → .htm` (laptop UAT 2026-07-31; the streamed
217/// assembly exists only as serialized text, so post must re-parse it).
218/// Above the ceiling the handoff spills to a temp file beside the
219/// destination and takes the `PostInput::File` arm — the streaming-reader
220/// path that already parses this very document in the two-invocation flow.
221///
222/// Env-overridable for tests only (`LATEXML_POST_MEM_PARSE_LIMIT`): a real
223/// 2-GiB-plus fixture is not something CI can allocate, so the guard test
224/// drives the spill path with a tiny limit instead.
225fn post_mem_parse_limit() -> usize {
226 std::env::var("LATEXML_POST_MEM_PARSE_LIMIT")
227 .ok()
228 .and_then(|v| v.parse::<usize>().ok())
229 .unwrap_or(i32::MAX as usize)
230}
231
232/// The size at which a *split* run prefers the streaming split front-end
233/// (`latexml_post::stream_split`) over the whole-DOM parse. Default 1 GiB: a
234/// core XML that large parses to a ~20+ GB DOM (measured ~16 GB for 614 MB),
235/// where the streaming front-end peaks at one content subtree. Test override:
236/// `LATEXML_POST_STREAM_THRESHOLD` (bytes).
237fn stream_split_threshold() -> u64 {
238 std::env::var("LATEXML_POST_STREAM_THRESHOLD")
239 .ok()
240 .and_then(|v| v.parse::<u64>().ok())
241 .unwrap_or(1 << 30)
242}
243
244/// Gate for the streaming split: `LATEXML_POST_STREAM_SPLIT=1` forces it on
245/// (any size), `=0` disables it, unset engages it automatically for files at
246/// or above [`stream_split_threshold`] — where the whole-DOM parse is beyond
247/// commodity-RAM hosts anyway (the 131 MB witness's 2.68 GB core XML OOM'd a
248/// 31 GB laptop *during the parse*, 0 pages written).
249fn stream_split_gate(path: &str) -> bool {
250 match std::env::var("LATEXML_POST_STREAM_SPLIT").as_deref() {
251 Ok("0") => false,
252 Ok(_) => true,
253 Err(_) => std::fs::metadata(path)
254 .map(|m| m.len() >= stream_split_threshold())
255 .unwrap_or(false),
256 }
257}
258
259/// `--splitnaming` → `SplitNaming`, shared by the DOM and streaming split
260/// front-ends so the two cannot disagree.
261fn resolve_split_naming(split_naming: Option<&str>) -> latexml_post::split::SplitNaming {
262 use latexml_post::split::SplitNaming;
263 match split_naming {
264 Some("id") | None => SplitNaming::Id,
265 Some("idrelative") => SplitNaming::IdRelative,
266 Some("label") => SplitNaming::Label,
267 Some("labelrelative") => SplitNaming::LabelRelative,
268 Some(other) => {
269 Warn!(
270 "post",
271 "split",
272 format!("Unknown splitnaming '{other}', using 'id'")
273 );
274 SplitNaming::Id
275 },
276 }
277}
278
279/// Destination directory for a page pathname (empty parent → ".", mirroring
280/// `PostDocument`'s internal derivation).
281fn destination_directory_of(destination: &str) -> Option<String> {
282 std::path::Path::new(destination).parent().map(|p| {
283 let s = p.to_string_lossy();
284 if s.is_empty() {
285 ".".to_string()
286 } else {
287 s.into_owned()
288 }
289 })
290}
291
292/// The two placeholder probes recorded per page WHILE it is parsed, so the
293/// ObjectDB-baton sweeps (MakeIndex, MakeBibliography) re-read only the
294/// handful of pages that actually carry work.
295fn page_placeholder_probes(d: &PostDocument) -> (bool, bool) {
296 (
297 !d.findnodes("//ltx:index[not(ltx:indexlist)] | //ltx:glossary[not(ltx:glossarylist)]")
298 .is_empty(),
299 !d.findnodes("//ltx:bibliography").is_empty(),
300 )
301}
302
303/// Everything the shared pipeline tail (MakeIndex → sweeps → page-major
304/// render) needs from a front-end: spilled pages plus the scanned ObjectDB.
305/// Produced by BOTH front-ends — the streaming split and the whole-DOM
306/// parse+Split+Scan.
307struct PostFront {
308 spilled_pages: Vec<SpilledPage>,
309 db: ObjectDB,
310 svg_fragments: Vec<(String, String)>,
311 intent_literal: bool,
312}
313
314/// Attempt the streaming split front-end (`latexml_post::stream_split`).
315///
316/// `Ok(None)` = not applicable (the union matched no pages, or the stream
317/// could not be split faithfully — already reported) → the caller falls back
318/// to the whole-DOM pipeline. `Err(())` = a hard failure after pages were in
319/// flight (I/O, resource ceiling), already reported via `post_error`; the
320/// caller bails out.
321fn try_streaming_front(
322 path: &str,
323 split_xpath: &str,
324 split_naming: Option<&str>,
325 destination: Option<&str>,
326 page_opts: &PostDocumentOptions,
327 spill_dir: &std::path::Path,
328) -> Result<Option<PostFront>, ()> {
329 let naming = resolve_split_naming(split_naming);
330 telemetry::phase_enter(Phase::Split);
331 let outcome =
332 latexml_post::stream_split::stream_split(path, split_xpath, naming, destination, spill_dir);
333 telemetry::phase_exit();
334 let outcome = match outcome {
335 Ok(Some(o)) => o,
336 Ok(None) => return Ok(None),
337 Err(e) => {
338 // Fail toward flagging: the whole-DOM fallback may well OOM on an input
339 // this large, but a loud failure beats a silently wrong result — and
340 // beats the whole-DOM fallback, whose parse of a threshold-sized file
341 // is a guaranteed OOM on commodity RAM (witnessed: a mid-stream error
342 // fell back into a 2.68 GB whole-DOM parse that grew past a 26 GB
343 // ceiling with zero pages to show). The unions this front-end cannot
344 // evaluate never reach here — the caller gates on `supports_union`.
345 post_error("split", &format!("streaming split failed: {e}"));
346 return Err(());
347 },
348 };
349 // The engagement marker the parity/threshold tests assert on.
350 Info!(
351 "post",
352 "stream-split",
353 s!(
354 "streaming split engaged for '{}': {} pages",
355 path,
356 outcome.pages.len()
357 )
358 );
359 note_progress(&format!("Split into {} documents", outcome.pages.len()));
360 let intent_literal = outcome
361 .latexml_pis
362 .iter()
363 .any(|pi| pi.contains("package=\"ar5iv"));
364 let svg_fragments = extract_svg_fragments(&outcome.picture_xml);
365
366 // Pre-order Scan sweep over the spilled pages — the streaming equivalent of
367 // `run_phase(docs, Scan)` + the driver's spill loop, ONE page resident at a
368 // time. The order is semantic, not cosmetic: SITE_ROOT is taken from the
369 // first page scanned (the root page), an ancestor must be in the DB before
370 // its descendants (Scan's parent inference falls back to SITE_ROOT
371 // otherwise), and each parent's `children` list follows scan order — all
372 // three feed CrossRef's navigation.
373 telemetry::phase_enter(Phase::PostScan);
374 let mut scanner = latexml_post::scan::Scan::new(ObjectDB::new());
375 let mut spilled: Vec<SpilledPage> = Vec::with_capacity(outcome.pages.len());
376 for page in &outcome.pages {
377 if let Err(e) = latexml_core::stomach::check_timeout() {
378 // FATAL, not Error: a scan stopped partway means the delivered site
379 // would be silently truncated (user decision 2026-08-02 — a resource
380 // stop that truncates the deliverable carries fatal severity).
381 Info!(
382 "post",
383 "stopped",
384 s!(
385 "Scan stopped after {} of {} pages",
386 spilled.len(),
387 outcome.pages.len()
388 )
389 );
390 e.log_fatal();
391 telemetry::phase_exit();
392 return Err(());
393 }
394 let path_str = page.path.to_string_lossy().into_owned();
395 let page_doc = match PostDocument::new_from_file(&path_str, page_opts.clone()) {
396 Ok(mut d) => {
397 d.destination = Some(page.destination.clone());
398 d.destination_directory = destination_directory_of(&page.destination);
399 d
400 },
401 Err(e) => {
402 post_error("post", &format!("cannot read a streamed page: {e}"));
403 telemetry::phase_exit();
404 return Err(());
405 },
406 };
407 // Scan's only page mutation: a root without an id gets
408 // `xml:id="Document"`. Detect it so the spill is refreshed (pass B
409 // re-parses pages from disk and must see what Scan registered).
410 let root_unnamed = page_doc
411 .get_document_element()
412 .and_then(|r| latexml_post::document::get_xml_id(&r))
413 .is_none();
414 let nodes = scanner.to_process(&page_doc);
415 let processed = if nodes.is_empty() {
416 vec![page_doc]
417 } else {
418 match scanner.process(page_doc, nodes) {
419 Ok(p) => p,
420 Err(e) => {
421 post_error("Scan", &format!("Scan failed: {e}"));
422 telemetry::phase_exit();
423 return Err(());
424 },
425 }
426 };
427 for d in processed {
428 if root_unnamed && let Err(e) = std::fs::write(&page.path, d.to_xml_string()) {
429 post_error("post", &format!("cannot refresh a disk-staged page: {e}"));
430 telemetry::phase_exit();
431 return Err(());
432 }
433 let (needs_index, needs_bib) = page_placeholder_probes(&d);
434 spilled.push(SpilledPage {
435 needs_index,
436 needs_bib,
437 path: page.path.clone(),
438 destination: Some(page.destination.clone()),
439 destination_directory: destination_directory_of(&page.destination),
440 });
441 // The page's DOM drops here — one resident at a time.
442 }
443 }
444 telemetry::phase_exit();
445 Ok(Some(PostFront {
446 spilled_pages: spilled,
447 db: scanner.db,
448 svg_fragments,
449 intent_literal,
450 }))
451}
452
453/// Write an oversized handoff to a temp file for the streaming file parser.
454/// Beside the destination when there is one (writing to the destination
455/// directory is always allowed), else the system temp dir.
456fn spill_oversized_xml(xml: &str, opts: &PostOptions) -> std::io::Result<std::path::PathBuf> {
457 let dir = opts
458 .destination
459 .and_then(|d| {
460 std::path::Path::new(d)
461 .parent()
462 .map(std::path::Path::to_path_buf)
463 })
464 .filter(|d| !d.as_os_str().is_empty())
465 .unwrap_or_else(std::env::temp_dir);
466 let path = dir.join(format!("latexml-post-handoff-{}.xml", std::process::id()));
467 std::fs::write(&path, xml)?;
468 Ok(path)
469}
470
471fn run_post_processing_impl(input: PostInput, opts: &PostOptions) -> String {
472 // Oversized in-memory handoff → spill + streaming file parse (see
473 // `post_mem_parse_limit`). The temp file is removed on every exit path by
474 // the guard; a spill failure falls through to the memory parse, whose own
475 // error reporting then names the real problem.
476 let mut _spill_cleanup: Option<std::path::PathBuf> = None;
477 let spilled_path: Option<String>;
478 // A handoff below the i32 ceiling but at/above the streaming-split
479 // threshold also spills when splitting is on: only the `File` arm can take
480 // the streaming split, and a memory parse of a multi-GB handoff is the OOM
481 // this work removes.
482 let spill_for_streaming = |len: usize| opts.split && len as u64 >= stream_split_threshold();
483 let input = match input {
484 PostInput::Xml(xml)
485 if xml.len() >= post_mem_parse_limit() || spill_for_streaming(xml.len()) =>
486 {
487 match spill_oversized_xml(xml, opts) {
488 Ok(path) => {
489 // Operationally worth a line: a multi-GB temp file just appeared
490 // beside the destination. Also the guard test's engagement proof —
491 // without it a silently-failing spill would fall back to the memory
492 // parse and the test would pass vacuously.
493 latexml_core::Info!(
494 "post",
495 "spill",
496 format!(
497 "oversized handoff ({} bytes) staged to {}",
498 xml.len(),
499 path.display()
500 )
501 );
502 spilled_path = Some(path.to_string_lossy().into_owned());
503 _spill_cleanup = Some(path);
504 PostInput::File(spilled_path.as_deref().expect("just set"))
505 },
506 Err(e) => {
507 latexml_core::Info!(
508 "post",
509 "spill",
510 format!("could not stage an oversized handoff to disk ({e}); parsing from memory")
511 );
512 PostInput::Xml(xml)
513 },
514 }
515 },
516 other => other,
517 };
518 let result = run_post_processing_inner(input, opts);
519 if let Some(path) = _spill_cleanup {
520 let _ = std::fs::remove_file(path);
521 }
522 result
523}
524
525fn run_post_processing_inner(input: PostInput, opts: &PostOptions) -> String {
526 let PostOptions {
527 pmml,
528 cmml,
529 keep_xmath,
530 stylesheet,
531 destination,
532 source_directory,
533 site_directory,
534 search_paths,
535 nodefaultresources,
536 css_files,
537 js_files,
538 noinvisibletimes,
539 plane1,
540 hackplane1,
541 mathtex,
542 url_style,
543 navigationtoc,
544 schemadocs,
545 split,
546 ref split_xpath,
547 split_naming,
548 xslt_parameters,
549 graphics_svg_threshold_kb,
550 graphicimages,
551 timestamp,
552 icon,
553 whatsout,
554 } = *opts;
555
556 let mut doc_opts = PostDocumentOptions::default();
557 if let Some(dest) = destination {
558 doc_opts.destination = Some(dest.to_string());
559 }
560 if let Some(src_dir) = source_directory {
561 doc_opts.source_directory = Some(src_dir.to_string());
562 let mut sp = doc_opts.searchpaths.take().unwrap_or_default();
563 sp.push(src_dir.to_string());
564 doc_opts.searchpaths = Some(sp);
565 }
566 // --sitedirectory (Perl LaTeXML.pm L430 `siteDirectory`): the site root for
567 // relativizing cross-document/resource URLs. When unset, `PostDocument::new`
568 // falls back to the destination's directory (document.rs, Perl Config.pm L466).
569 if let Some(site_dir) = site_directory {
570 doc_opts.site_directory = Some(site_dir.to_string());
571 }
572 // `raw_xml` holds the in-memory serialization — `Some` only for `Xml` input.
573 // `File` input streams from disk and never holds it, so `raw_xml` is `None`
574 // and the three raw-string features (empty-doc substitution, SVG extraction,
575 // ar5iv sniff) plus the on-failure echo are re-derived from the parsed doc.
576 //
577 // Perl LaTeXML.pm L330-336: an empty core result (e.g. after a Fatal) is still
578 // post-processed against a bare <document/> root ("important for utility
579 // features such as packing .zip archives"). Mirror that for empty in-memory
580 // input; a File is never empty (an empty file simply errors on parse below).
581 let raw_xml: Option<&str> = match input {
582 PostInput::Xml(xml) if xml.trim().is_empty() => Some("<document/>"),
583 PostInput::Xml(xml) => Some(xml),
584 PostInput::File(_) => None,
585 };
586 // Best-effort output when a phase bails: echo the original XML for in-memory
587 // input; for File input the source is on disk (and a failure here is itself
588 // the case we are hardening against), so return empty and rely on the logged
589 // Error + status code.
590 let fallback = || raw_xml.map(str::to_string).unwrap_or_default();
591
592 // Pass B re-parses each spilled page and must see the SAME searchpaths and
593 // site directory as this parse does.
594 let page_opts = doc_opts.clone();
595
596 // The per-page spill directory, created up front: BOTH front-ends (the
597 // streaming split and the whole-DOM parse) write their pages into it.
598 // Beside the destination when there is one — the same volume the output
599 // lands on (and NOT the system temp dir, which can be a RAM-backed tmpfs:
600 // a 40k-page document spills the whole core XML's worth of pages here).
601 let mut spill_builder = tempfile::Builder::new();
602 spill_builder.prefix(".latexml-post-pages-");
603 let page_spill = match destination
604 .and_then(|d| std::path::Path::new(d).parent())
605 .filter(|p| !p.as_os_str().is_empty() && p.is_dir())
606 {
607 Some(dest_dir) => spill_builder.tempdir_in(dest_dir),
608 None => spill_builder.tempdir(),
609 };
610 let page_spill = match page_spill {
611 Ok(dir) => dir,
612 Err(e) => {
613 post_error(
614 "post",
615 &format!("cannot create the page staging directory: {e}"),
616 );
617 return fallback();
618 },
619 };
620
621 // ---- Front-end selection --------------------------------------------------
622 //
623 // For a *file* input that will be split and is past the streaming threshold
624 // (or force-enabled), the streaming split front-end produces the spilled
625 // pages + scanned ObjectDB without ever parsing the whole document — the
626 // whole-DOM parse of a 2.68 GB core XML needs ~30+ GB and OOM'd a 31 GB
627 // host before Split ran (laptop UAT 2026-07-31). Everything downstream of
628 // this block is shared between the two front-ends.
629 let streamed: Option<PostFront> = if let PostInput::File(path) = input
630 && split
631 && destination.is_some()
632 && let Some(xpath) = split_xpath.as_deref()
633 && latexml_post::stream_split::supports_union(xpath)
634 && stream_split_gate(path)
635 {
636 match try_streaming_front(
637 path,
638 xpath,
639 split_naming,
640 destination,
641 &page_opts,
642 page_spill.path(),
643 ) {
644 Ok(front) => front,
645 Err(()) => return fallback(),
646 }
647 } else {
648 None
649 };
650
651 let front = if let Some(front) = streamed {
652 front
653 } else {
654 // ==== Whole-DOM front-end (the pre-existing pipeline, indentation kept) ====
655 telemetry::phase_enter(Phase::PostXmlParse);
656 let t_parse = audit_start("PostDocument parse");
657 let parsed = match input {
658 // `raw_xml` is `Some` for `Xml` input (the two arms above), so the default
659 // is dead — kept only to avoid an `unwrap`.
660 PostInput::Xml(_) => {
661 PostDocument::new_from_string(raw_xml.unwrap_or("<document/>"), doc_opts)
662 },
663 PostInput::File(path) => PostDocument::new_from_file(path, doc_opts),
664 };
665 let doc = match parsed {
666 Ok(d) => d,
667 Err(e) => {
668 post_error("parse", &format!("failed to parse XML: {e}"));
669 telemetry::phase_exit();
670 return fallback();
671 },
672 };
673 audit_end(t_parse);
674 telemetry::phase_exit();
675
676 // SVG extraction reads the pre-post XML before any in-tree mutation; do it
677 // once up-front so the regex-based fragment table is valid for every split
678 // sub-document below. In-memory input scans the string; File input locates
679 // pictures via a limit-safe `//ltx:picture` DOM query and serializes only
680 // those (never the whole file) — most large documents have none.
681 let t_svg = audit_start("SVG extraction");
682 let svg_fragments = match raw_xml {
683 Some(xml) => extract_svg_fragments(xml),
684 None => extract_svg_fragments_from_doc(&doc),
685 };
686 audit_end(t_svg);
687
688 // ar5iv sniff: the ar5iv package emits literal-intent MathML. In-memory input
689 // checks the raw string; file input checks the parsed doc's `<?latexml
690 // package="ar5iv"?>` PI. Computed here — before Split moves `doc` into `docs`.
691 let intent_literal = match raw_xml {
692 Some(xml) => xml.contains("package=\"ar5iv"),
693 None => doc
694 .processing_instructions()
695 .iter()
696 .any(|pi| pi.contains("package=\"ar5iv")),
697 };
698
699 // Perl-faithful pipeline order (latexmlpost L223-242):
700 // Split → Scan → MakeBibliography → CrossRef → Graphics → ...
701 // Split runs FIRST so each downstream pass sees the per-page
702 // destination. With the previous order (Scan before Split), every
703 // entry's `location` was the root document and CrossRef built every
704 // ref as a within-page anchor — the user-visible TOC links pointed
705 // at `#Ch1` instead of `Ch1.html`.
706 //
707 // Phase 1: Split (only attributes time when --split is on)
708 let mut docs: Vec<PostDocument> = if split {
709 telemetry::phase_enter(Phase::Split);
710 let result = if let Some(xpath) = split_xpath {
711 let naming = resolve_split_naming(split_naming);
712 let mut splitter = latexml_post::split::Split::new(xpath, naming, false);
713 let split_nodes = splitter.to_process(&doc);
714 match splitter.process(doc, split_nodes) {
715 Ok(docs) => {
716 if docs.len() > 1 {
717 note_progress(&format!("Split into {} documents", docs.len()));
718 }
719 Ok(docs)
720 },
721 Err(e) => {
722 post_error("split", &format!("Split failed: {e}"));
723 Err(())
724 },
725 }
726 } else {
727 Ok(vec![doc])
728 };
729 telemetry::phase_exit();
730 match result {
731 Ok(ds) => ds,
732 Err(()) => return fallback(),
733 }
734 } else {
735 vec![doc]
736 };
737
738 // Helper: run one processor across every doc in a Vec, mirroring
739 // Perl Post.pm:50-65's `foreach $proc { @newdocs = ... }` loop.
740 // Each processor's per-doc `process` may return multiple docs (only
741 // Split actually does, and we've already run Split above), so the
742 // inner result is flattened back into `docs`.
743 fn run_phase<P: Processor + ?Sized>(
744 docs: Vec<PostDocument>,
745 proc: &mut P,
746 label: &'static str,
747 ) -> Result<Vec<PostDocument>, ()> {
748 let mut out: Vec<PostDocument> = Vec::with_capacity(docs.len());
749 for d in docs {
750 // Cooperative memory check per document, so a phase that grows across
751 // many documents degrades gracefully instead of being hard-killed.
752 //
753 // MEASURED LIMIT, so nobody reads more into this than it delivers: on the
754 // 614 MB witness at `--max-memory 6000` the ceiling is breached BEFORE
755 // this loop is reached — the one-time parse + split DOM alone exceeds it,
756 // so the run still ends at the hard watchdog (exit 137, zero pages). No
757 // cooperative check inside the phases can catch that; bounding it is
758 // task #147 (streaming the split parse). What this DOES cover is the
759 // document whose growth happens across the phase itself.
760 if let Err(e) = latexml_core::stomach::check_timeout() {
761 // FATAL, not Error — see the render-loop stop below.
762 Info!("post", "stopped", s!("{label} stopped mid-phase"));
763 e.log_fatal();
764 return Err(());
765 }
766 let nodes = proc.to_process(&d);
767 if nodes.is_empty() {
768 out.push(d);
769 continue;
770 }
771 match proc.process(d, nodes) {
772 Ok(processed) => out.extend(processed),
773 Err(e) => {
774 post_error(label, &format!("{label} failed: {e}"));
775 return Err(());
776 },
777 }
778 }
779 Ok(out)
780 }
781
782 // Phase 2: Scan — runs on EACH sub-document so its entries register
783 // the per-page `location` and `pageid`. Single shared ObjectDB so
784 // the later CrossRef pass can resolve cross-doc refs.
785 let mut scanner = latexml_post::scan::Scan::new(ObjectDB::new());
786 telemetry::phase_enter(Phase::PostScan);
787 let t_scan = audit_start("Scan");
788 docs = match run_phase(docs, &mut scanner, "Scan") {
789 Ok(d) => d,
790 Err(()) => {
791 telemetry::phase_exit();
792 return fallback();
793 },
794 };
795 audit_end(t_scan);
796 telemetry::phase_exit();
797
798 // ---- Pass A / Pass B boundary -------------------------------------------
799 //
800 // Scan is the ONLY phase that needs global knowledge: it visits every page
801 // and produces strings in the ObjectDB. Everything after it is page-local.
802 // The driver used to be phase-major (`for each phase { for each page }`),
803 // which keeps every page alive at every boundary — and a live page is not
804 // cheap: each is its own `xmlDoc` with its own dictionary, id table and
805 // lazily-built caches, measured at ~1.6 MB of per-document overhead. Across
806 // the 40,201 pages of a 614 MB core XML that alone grew RSS from 16 GB to
807 // 80 GB *during Scan*, wrote zero pages, and was killed by the ceiling.
808 //
809 // So: spill each scanned page to disk (into the shared `page_spill` dir
810 // created before front-end selection), free it, and render page-by-page
811 // below. Peak becomes the one-time split DOM plus ONE page. Spilling (rather
812 // than re-deriving pages from the source) keeps Scan's per-page
813 // `location`/`pageid` registration byte-identical — those semantics were
814 // themselves the fix for empty cross-document TOCs, so they are the wrong
815 // thing to restructure here.
816 //
817 // (path, destination, destination_directory) — the metadata that does NOT
818 // survive an XML round-trip and must be restored in pass B.
819 // Plus a flag per page: does it carry an index/glossary/bibliography
820 // placeholder? Recorded HERE, while the page is already parsed, so the two
821 // baton sweeps below visit only the handful of pages that have work instead
822 // of re-parsing every page twice (~80 k redundant parses on a 40 k-page
823 // document).
824 let mut spilled_pages = Vec::with_capacity(docs.len());
825 for (i, d) in docs.drain(..).enumerate() {
826 let path = page_spill.path().join(format!("page-{i:07}.xml"));
827 if let Err(e) = std::fs::write(&path, d.to_xml_string()) {
828 post_error("post", &format!("cannot stage page {i} to disk: {e}"));
829 return fallback();
830 }
831 let (needs_index, needs_bib) = page_placeholder_probes(&d);
832 spilled_pages.push(SpilledPage {
833 needs_index,
834 needs_bib,
835 path,
836 destination: d.get_destination().map(String::from),
837 destination_directory: d.get_destination_directory().map(String::from),
838 });
839 // `d` drops here: one page's worth of DOM and caches released before the
840 // next is touched.
841 }
842 PostFront {
843 spilled_pages,
844 db: scanner.db,
845 svg_fragments,
846 intent_literal,
847 }
848 }; // ==== end of the whole-DOM front-end ====
849 let PostFront {
850 spilled_pages,
851 db,
852 svg_fragments,
853 intent_literal,
854 } = front;
855
856 // Phase 2b: MakeIndex (Perl LaTeXML.pm L466-470)
857 // Runs BEFORE MakeBibliography in Perl. Populates `<ltx:indexlist>`
858 // inside `<ltx:index>` placeholders and `<ltx:glossarylist>` inside
859 // `<ltx:glossary>` placeholders, using the `GLOSSARY:*` / `INDEX:*`
860 // entries Scan registered in the ObjectDB. Without this pass, glossary
861 // sections render empty in HTML (witness: tests/structure/glossary.tex).
862 let mut indexer = latexml_post::make_index::MakeIndex::new(db, false, false);
863
864 // Phase 3: MakeBibliography
865 //
866 // A raw `.bib` is converted by a recursive BibTeX session (Perl
867 // `MakeBibliography.pm::convertBibliography`), which needs this crate's
868 // model loader — so `latexml_post` declares the hook and we fill it in here,
869 // on the thread that is about to run the pipeline.
870 crate::bib_session::install();
871
872 // The output file extension CrossRef strips for `--urlstyle` (Perl
873 // `extension => $$opts{extension}`, LaTeXML.pm L479; derived in Config.pm
874 // L408-420 from the destination's extension, else the format). Without it the
875 // `server`/`negotiated` styles would strip the wrong suffix (the `xml` default)
876 // and leave an html site's links untouched.
877 let out_extension = destination
878 .and_then(|d| std::path::Path::new(d).extension().and_then(|e| e.to_str()))
879 .map(str::to_string)
880 .unwrap_or_else(|| {
881 if stylesheet.is_some_and(|s| s.contains("html")) {
882 "html".to_string()
883 } else {
884 "xml".to_string()
885 }
886 });
887
888 // Phase 4: CrossRef — built only once the bibliography sweep has handed the
889 // ObjectDB baton on (see `sweep_pages`), so nothing holds two owners of it.
890 let build_crossref = |db| {
891 let mut crossref = latexml_post::crossref::CrossRef::new(db, url_style, true);
892 crossref.set_extension(&out_extension);
893 if let Some(navtoc) = navigationtoc {
894 crossref.set_navigation_toc(navtoc);
895 }
896 crossref
897 };
898
899 // Phase 5: Graphics (Perl `graphicimages!` → `dographics`, default on).
900 // `--nographicimages` skips the phase wholesale, leaving the raw
901 // `<ltx:graphics>` references in the output untouched.
902 let graphics_proc = graphicimages.then(|| {
903 latexml_post::graphics::Graphics::new(None, true)
904 .with_svg_threshold_kb(graphics_svg_threshold_kb)
905 });
906
907 // Phase 3: MathML + XSLT
908 let post = latexml_post::Post::new();
909 let mut processors: Vec<Box<dyn Processor>> = Vec::new();
910
911 // `intent_literal` was computed up-front (before Split consumed `doc`).
912
913 // Parallel P+C markup: when both formats are requested, the Content-MathML
914 // processor is a *secondary* of the Presentation primary, folded into one
915 // `<m:semantics>`/`<m:annotation-xml>` by `combine_parallel` — NOT a second
916 // independent pass (which left the content tree as an orphan `<apply>` sibling
917 // of `<m:semantics>`, rendered as stray text by browsers). Mirrors Perl
918 // `MathProcessor`'s primary→secondary parallel model.
919 if pmml {
920 let mut presentation = latexml_post::mathml::MathML::new_presentation()
921 .with_keep_xmath(keep_xmath)
922 .with_invisible_times(!noinvisibletimes)
923 .with_plane1(plane1, hackplane1)
924 .with_mathtex(mathtex)
925 .with_intent_literal(intent_literal);
926 if cmml {
927 presentation = presentation.with_secondaries(vec![Box::new(
928 latexml_post::mathml::MathML::new_content()
929 .with_keep_xmath(keep_xmath)
930 .with_invisible_times(!noinvisibletimes)
931 .with_plane1(plane1, hackplane1)
932 .secondary(),
933 )]);
934 }
935 processors.push(Box::new(presentation));
936 } else if cmml {
937 // Content-only output (no presentation primary).
938 processors.push(Box::new(
939 latexml_post::mathml::MathML::new_content()
940 .with_keep_xmath(keep_xmath)
941 .with_invisible_times(!noinvisibletimes)
942 .with_plane1(plane1, hackplane1),
943 ));
944 }
945 // Clones of the fully-resolved XSLT inputs for the parallel-render worker
946 // manifest — `XSLT::new` consumes the real map/vec below, and by the time
947 // the pass-B driver decides between serial and parallel they are gone.
948 let mut manifest_xslt_params: Vec<(String, String)> = Vec::new();
949 let mut manifest_searchpaths: Vec<String> = Vec::new();
950 if let Some(xsl_path) = stylesheet {
951 let mut searchpaths = vec![".".to_string()];
952 if let Ok(exe) = std::env::current_exe()
953 && let Some(project_root) = exe
954 .parent()
955 .and_then(|p| p.parent())
956 .and_then(|p| p.parent())
957 {
958 searchpaths.insert(0, project_root.display().to_string());
959 }
960 // Prepend the user's `--path` directories so `--css`/`--javascript`
961 // resources are found there (and take priority over `.`/project root)
962 // when copy_param_resources searches for them.
963 for p in search_paths.iter().rev() {
964 searchpaths.insert(0, p.clone());
965 }
966 let mut xslt_params = rustc_hash::FxHashMap::default();
967
968 // When `--schemadocs` is on, auto-prepend the rustdoc-styled
969 // theme assets so callers don't need to repeat them on every
970 // invocation. Idempotent: skipped if the user has already
971 // listed the same basename via `--css` / `--javascript`.
972 let prepend = |list: &[String], extra: &str| -> Vec<String> {
973 if list
974 .iter()
975 .any(|p| std::path::Path::new(p).file_name().and_then(|s| s.to_str()) == Some(extra))
976 {
977 list.to_vec()
978 } else {
979 let mut out = Vec::with_capacity(list.len() + 1);
980 out.push(extra.to_string());
981 out.extend_from_slice(list);
982 out
983 }
984 };
985 let css_effective: Vec<String> = if schemadocs {
986 prepend(css_files, latexml_post::schema_docs::THEME_CSS_BASENAME)
987 } else {
988 css_files.to_vec()
989 };
990 let js_effective: Vec<String> = if schemadocs {
991 prepend(js_files, latexml_post::schema_docs::THEME_JS_BASENAME)
992 } else {
993 js_files.to_vec()
994 };
995 if !css_effective.is_empty() {
996 xslt_params.insert(
997 "CSS".to_string(),
998 format!("\"{}\"", css_effective.join("|")),
999 );
1000 }
1001 if !js_effective.is_empty() {
1002 xslt_params.insert(
1003 "JAVASCRIPT".to_string(),
1004 format!("\"{}\"", js_effective.join("|")),
1005 );
1006 }
1007 if let Some(navtoc) = navigationtoc {
1008 xslt_params.insert("NAVIGATIONTOC".to_string(), format!("\"{}\"", navtoc));
1009 }
1010 if let Some(ts) = timestamp {
1011 xslt_params.insert("TIMESTAMP".to_string(), format!("\"{}\"", ts));
1012 }
1013 if let Some(icon) = icon {
1014 xslt_params.insert("ICON".to_string(), format!("\"{}\"", icon));
1015 }
1016 // Perl `LaTeXML.pm:562` unconditionally seeds `LATEXML_VERSION => '$LaTeXML::VERSION'`,
1017 // which drives `LaTeXML-common.xsl`'s `LaTeXML_identifier` generator stamp
1018 // (`<!--Generated by LaTeXML oxide (version X) ...-->`) and BookML's `utils.xsl`
1019 // `b:version-leq($LATEXML_VERSION,…)`. We expose OUR own crate `X.Y.Z` (#320). Inserted
1020 // BEFORE the user-override loop so `--xsltparameter LATEXML_VERSION=…` still wins (this
1021 // is how Perl's test suite pins `version TEST` for version-independent goldens).
1022 xslt_params.insert(
1023 "LATEXML_VERSION".to_string(),
1024 format!("\"{}\"", crate::core_interface::LATEXML_VERSION),
1025 );
1026 for param in xslt_parameters {
1027 if let Some((key, value)) = param.split_once('=') {
1028 xslt_params.insert(key.to_string(), format!("\"{}\"", value));
1029 }
1030 }
1031 manifest_xslt_params = xslt_params
1032 .iter()
1033 .map(|(k, v)| (k.clone(), v.clone()))
1034 .collect();
1035 manifest_searchpaths = searchpaths.clone();
1036 match latexml_post::xslt::XSLT::new(
1037 xsl_path,
1038 xslt_params,
1039 nodefaultresources,
1040 None,
1041 searchpaths,
1042 ) {
1043 Ok(xslt) => processors.push(Box::new(xslt)),
1044 Err(e) => post_error("xslt", &format!("XSLT error: {e}")),
1045 }
1046 }
1047
1048 // process_chain attributes per-processor inside latexml_post::Post::
1049 // process_chain (MathmlPres / MathmlCont / Xslt). No outer phase wrap
1050 // needed; the inner per-processor guards cover their own time.
1051 //
1052 // Perl-faithful: pass ALL split docs to ProcessChain at once. Perl's
1053 // `Post::ProcessChain_internal` (Post.pm L41-67) seeds `@docs = ($doc)`
1054 // and lets each processor return a (possibly multi-element) list which
1055 // becomes the input to the next processor. We've already produced the
1056 // post-Split list above; ProcessChain just needs to fan MathML/XSLT
1057 // across it.
1058 let is_html_out = stylesheet.is_some_and(|s| s.contains("html"));
1059
1060 // ---- Pass B: render one page at a time, then FREE it --------------------
1061 //
1062 // Document-major, not phase-major: each page goes through MakeIndex ->
1063 // MakeBibliography -> CrossRef -> Graphics -> MathML -> XSLT -> write, and
1064 // is dropped before the next is parsed. Every one of these phases is
1065 // page-local; only Scan (pass A, above) needed the whole document, and it
1066 // left its results in the ObjectDB as strings. Peak is therefore ONE page
1067 // plus the index, whatever the page count.
1068 //
1069 // Failure semantics change deliberately: pages that finish are already on
1070 // disk, so a later failure leaves a partial site rather than nothing. That
1071 // matches how a resource Fatal already keeps partial core output.
1072 // (The per-page pipeline itself lives in `render_spilled_page`, shared with
1073 // the process-parallel worker in `crate::render_workers`.)
1074
1075 // The ObjectDB is a BATON: MakeIndex, MakeBibliography and CrossRef each take
1076 // it by value in turn, so they cannot be alive simultaneously. Each therefore
1077 // gets its own sweep over the spilled pages — still ONE page resident at a
1078 // time. A page is only re-spilled when its phase actually had work
1079 // (`to_process` non-empty), which for index/bibliography placeholders is a
1080 // handful of pages out of tens of thousands, so the sweeps cost a parse each
1081 // and almost no writes.
1082 fn sweep_pages(
1083 pages: &[SpilledPage],
1084 opts: &PostDocumentOptions,
1085 proc: &mut dyn Processor,
1086 label: &'static str,
1087 selector: fn(&SpilledPage) -> bool,
1088 ) -> Result<(), ()> {
1089 for page_meta in pages.iter().filter(|p| selector(p)) {
1090 let (path, dest, dest_dir) = (
1091 &page_meta.path,
1092 &page_meta.destination,
1093 &page_meta.destination_directory,
1094 );
1095 let path_str = path.to_string_lossy().into_owned();
1096 let mut page = PostDocument::new_from_file(&path_str, opts.clone()).map_err(|e| {
1097 post_error("post", &format!("cannot re-read a disk-staged page: {e}"));
1098 })?;
1099 page.destination = dest.clone();
1100 page.destination_directory = dest_dir.clone();
1101 let nodes = proc.to_process(&page);
1102 if nodes.is_empty() {
1103 continue; // untouched: the spill on disk is still current
1104 }
1105 let processed = proc.process(page, nodes).map_err(|e| {
1106 post_error(label, &format!("{label} failed: {e}"));
1107 })?;
1108 // A phase may split one page into several (an index page beside its
1109 // host); the extras are written by their own destination in pass B, so
1110 // re-spill each under the original path only for the first and append
1111 // the rest as additional spills is NOT needed here: MakeIndex and
1112 // MakeBibliography fill placeholders in place and return the same page.
1113 for d in processed.iter().take(1) {
1114 if let Err(e) = std::fs::write(path, d.to_xml_string()) {
1115 post_error("post", &format!("cannot re-stage a page to disk: {e}"));
1116 return Err(());
1117 }
1118 }
1119 }
1120 Ok(())
1121 }
1122
1123 if sweep_pages(&spilled_pages, &page_opts, &mut indexer, "MakeIndex", |p| {
1124 p.needs_index
1125 })
1126 .is_err()
1127 {
1128 return fallback();
1129 }
1130 let mut bibmaker = latexml_post::make_bibliography::MakeBibliography::new(indexer.db, false);
1131 telemetry::phase_enter(Phase::Bibliography);
1132 let t_bib = audit_start("MakeBibliography");
1133 let bib_result = sweep_pages(
1134 &spilled_pages,
1135 &page_opts,
1136 &mut bibmaker,
1137 "MakeBibliography",
1138 |p| p.needs_bib,
1139 );
1140 audit_end(t_bib);
1141 telemetry::phase_exit();
1142 if bib_result.is_err() {
1143 return fallback();
1144 }
1145 // ---- Parallel pass B: process-level page-range workers (design §6) ------
1146 //
1147 // Env-gated (`LATEXML_RENDER_JOBS`, default 1 = the serial path below,
1148 // unchanged), and only past a minimum page count — below it the db save +
1149 // child spawn overhead beats the win. The ObjectDB is COMPLETE here (Scan,
1150 // MakeIndex and MakeBibliography have all run), so it is saved once as a
1151 // SQLite file and each worker attaches it readonly; the workers run the
1152 // identical `render_spilled_page` pipeline over contiguous page ranges and
1153 // the parent folds their diagnostics deterministically, in chunk order.
1154 let render_jobs = crate::render_workers::render_jobs();
1155 if render_jobs > 1 && spilled_pages.len() >= crate::render_workers::MIN_PAGES_FOR_PARALLEL {
1156 let page_jobs: Vec<crate::render_workers::PageJob> = spilled_pages
1157 .iter()
1158 .map(|p| crate::render_workers::PageJob {
1159 path: p.path.clone(),
1160 destination: p.destination.clone(),
1161 destination_directory: p.destination_directory.clone(),
1162 })
1163 .collect();
1164 let manifest = crate::render_workers::RenderManifest {
1165 // Filled in by `parallel_render` once the db is saved.
1166 dbfile: std::path::PathBuf::new(),
1167 navigation_toc: navigationtoc.map(String::from),
1168 url_style: url_style.as_cli().to_string(),
1169 out_extension: out_extension.clone(),
1170 graphicimages,
1171 graphics_svg_threshold_kb,
1172 pmml,
1173 cmml,
1174 keep_xmath,
1175 invisible_times: !noinvisibletimes,
1176 plane1,
1177 hackplane1,
1178 mathtex,
1179 intent_literal,
1180 stylesheet: stylesheet.map(String::from),
1181 xslt_params: manifest_xslt_params,
1182 nodefaultresources,
1183 searchpaths: manifest_searchpaths,
1184 is_html_out,
1185 svg_fragments: svg_fragments.clone(),
1186 schemadocs,
1187 whatsout: whatsout.as_cli().to_string(),
1188 page_opts: (&page_opts).into(),
1189 pages: Vec::new(),
1190 };
1191 let t_pages = audit_start("render_pages");
1192 let outcome = crate::render_workers::parallel_render(
1193 manifest,
1194 page_jobs,
1195 render_jobs,
1196 page_spill.path(),
1197 &bibmaker.db,
1198 );
1199 audit_end(t_pages);
1200 if let Some(result) = outcome {
1201 Info!(
1202 "post",
1203 "parallel-render",
1204 s!(
1205 "parallel render finished: {} of {} page(s) written",
1206 result.pages_rendered,
1207 spilled_pages.len()
1208 )
1209 );
1210 drop(page_spill);
1211 return result.main_output.unwrap_or_else(fallback);
1212 }
1213 // Setup failed BEFORE any worker spawned (already reported): fall through
1214 // to the serial render — the spilled pages are untouched on disk.
1215 }
1216
1217 let crossref = build_crossref(bibmaker.db);
1218 let ctx = PageRenderCtx {
1219 page_opts: page_opts.clone(),
1220 is_html_out,
1221 svg_fragments,
1222 schemadocs,
1223 whatsout,
1224 };
1225 let mut procs = PageProcessors {
1226 crossref,
1227 graphics: graphics_proc,
1228 post,
1229 processors,
1230 };
1231
1232 let mut main_output: Option<String> = None;
1233 let t_pages = audit_start("render_pages");
1234 let mut pages_rendered = 0usize;
1235 for SpilledPage {
1236 path,
1237 destination: dest,
1238 destination_directory: dest_dir,
1239 ..
1240 } in spilled_pages
1241 {
1242 // The cooperative memory guard, at the one seam post has: a page boundary.
1243 //
1244 // Nothing in latexml_post or this driver consulted it — `check_timeout`
1245 // appeared ZERO times across both — so an oversized post run was killed by
1246 // the hard watchdog at 100% of the ceiling (exit 137, `Fatal:oom:rss …
1247 // exceeded the … ceiling`) with nothing on disk, where the design promise
1248 // is a named Fatal at 75% plus whatever finished. Measured on the 131 MB
1249 // witness at `--max-memory 48000`: post died at 48,002 MiB having written
1250 // ZERO pages. Same shape as the Build phase before it got a guard.
1251 //
1252 // Stopping HERE is cheap and honest: pages already rendered are already
1253 // written, so the run ends with a real, if partial, site.
1254 if let Err(e) = latexml_core::stomach::check_timeout() {
1255 // FATAL, not Error (user decision 2026-08-02): a render stopped at
1256 // page N of M is a TRUNCATED deliverable — the same severity class as
1257 // the core's cheap-partial handoff. The pages already written stay on
1258 // disk (a real, if partial, site), the verdict says failed, and the
1259 // combined status the framework reads is 3. Witness that motivated
1260 // it: the joint 131 MB run stopped at 56,088 of 115,519 pages with
1261 // Error-class severity and process exit 0.
1262 Info!(
1263 "post",
1264 "stopped",
1265 s!("render stopped after {pages_rendered} page(s); partial site preserved")
1266 );
1267 e.log_fatal();
1268 break;
1269 }
1270 // Hand freed page memory back to the OS, exactly as core pass 1 does
1271 // (core_interface.rs, same rationale): every page cycles a full DOM +
1272 // XSLT result through GLIBC's heap (libxml2/libxslt allocate via libc,
1273 // not the Rust allocator), and glibc keeps freed mid-heap pages mapped —
1274 // the render loop's measured ~152 KB/page RSS growth on the 131 MB
1275 // witness is dominated by exactly this class, and it is what pushed the
1276 // single-invocation run over the memory fuse at page 56,088 of 115,519.
1277 // Rate-limited: a trim walks the heap.
1278 if pages_rendered > 0 && pages_rendered.is_multiple_of(512) {
1279 #[cfg(target_os = "linux")]
1280 unsafe {
1281 libc::malloc_trim(0);
1282 }
1283 #[cfg(not(feature = "dhat-heap"))]
1284 unsafe {
1285 libmimalloc_sys::mi_collect(true);
1286 }
1287 }
1288 // Retention triage probe (`LATEXML_POST_MEMDIAG=1`): splits the render
1289 // loop's RSS growth into live-C (libxml2/libxslt allocations still
1290 // reachable), free-but-mapped C, and "everything else" (Rust/mimalloc +
1291 // mmap). One run with this told the ~144 KB/page term apart from the
1292 // three candidates the first fixes addressed.
1293 #[cfg(target_os = "linux")]
1294 if pages_rendered > 0
1295 && pages_rendered.is_multiple_of(1024)
1296 && std::env::var("LATEXML_POST_MEMDIAG").is_ok()
1297 {
1298 let mi = unsafe { libc::mallinfo2() };
1299 let rss_mb = latexml_core::watchdog::process_rss_kb().unwrap_or(0) / 1024;
1300 eprintln!(
1301 "memdiag: pages={} RSS={}MB C-live={}MB C-free={}MB",
1302 pages_rendered,
1303 rss_mb,
1304 mi.uordblks / (1024 * 1024),
1305 mi.fordblks / (1024 * 1024)
1306 );
1307 }
1308 let outputs = match render_spilled_page(&path, &mut procs, &ctx, dest, dest_dir) {
1309 Ok(o) => o,
1310 Err(()) => return fallback(),
1311 };
1312 for (dest, output) in outputs {
1313 if let Some(path) = dest.as_deref() {
1314 if let Some(parent) = std::path::Path::new(path).parent()
1315 && !parent.as_os_str().is_empty()
1316 {
1317 let _ = std::fs::create_dir_all(parent);
1318 }
1319 pages_rendered += 1;
1320 if let Err(e) = std::fs::write(path, &output) {
1321 post_error("write", &format!("failed to write page {path}: {e}"));
1322 }
1323 }
1324 if main_output.is_none() {
1325 main_output = Some(output);
1326 }
1327 }
1328 }
1329 audit_end(t_pages);
1330 drop(page_spill);
1331
1332 main_output.unwrap_or_else(fallback)
1333}
1334
1335/// The per-page processor set for pass B — everything [`render_spilled_page`]
1336/// mutates. Bundled so the serial driver and the process-parallel worker
1337/// ([`crate::render_workers`]) share ONE pipeline implementation and cannot
1338/// drift.
1339pub(crate) struct PageProcessors {
1340 pub(crate) crossref: latexml_post::crossref::CrossRef,
1341 pub(crate) graphics: Option<latexml_post::graphics::Graphics>,
1342 pub(crate) post: latexml_post::Post,
1343 pub(crate) processors: Vec<Box<dyn Processor>>,
1344}
1345
1346/// The read-only per-run context for pass B page rendering, shared by every
1347/// page of one conversion (serial driver or one parallel worker).
1348pub(crate) struct PageRenderCtx {
1349 /// Clone-source for each page's `PostDocument` parse — carries the same
1350 /// searchpaths/site-directory the pass-A parse used.
1351 pub(crate) page_opts: PostDocumentOptions,
1352 pub(crate) is_html_out: bool,
1353 pub(crate) svg_fragments: Vec<(String, String)>,
1354 pub(crate) schemadocs: bool,
1355 pub(crate) whatsout: latexml_post::extract::Whatsout,
1356}
1357
1358/// Run one page-local phase (CrossRef / Graphics) over one page, mirroring
1359/// `run_phase` for a single document. Failures are reported via [`post_error`].
1360fn run_page_phase(
1361 doc: PostDocument,
1362 proc: &mut dyn Processor,
1363 label: &'static str,
1364) -> Result<Vec<PostDocument>, ()> {
1365 let nodes = proc.to_process(&doc);
1366 if nodes.is_empty() {
1367 return Ok(vec![doc]);
1368 }
1369 proc.process(doc, nodes).map_err(|e| {
1370 post_error(label, &format!("{label} failed: {e}"));
1371 })
1372}
1373
1374/// The pass-B per-page pipeline: parse one spilled page (deleting the spill
1375/// file as soon as it is parsed), run CrossRef → Graphics → MathML/XSLT, and
1376/// return the finalized `(destination, output)` pairs for the caller to write.
1377/// Exactly the body the serial render loop ran in place before the
1378/// process-parallel split — phase order, telemetry and audit calls included —
1379/// so the serial path stays byte-identical (guard:
1380/// `118_streaming_split_parity`). `Err(())` means the failure was already
1381/// reported via [`post_error`] (or a phase's own diagnostics).
1382pub(crate) fn render_spilled_page(
1383 path: &std::path::Path,
1384 procs: &mut PageProcessors,
1385 ctx: &PageRenderCtx,
1386 destination: Option<String>,
1387 destination_directory: Option<String>,
1388) -> Result<Vec<(Option<String>, String)>, ()> {
1389 let path_str = path.to_string_lossy().into_owned();
1390 let page = match PostDocument::new_from_file(&path_str, ctx.page_opts.clone()) {
1391 Ok(mut d) => {
1392 d.destination = destination;
1393 d.destination_directory = destination_directory;
1394 d
1395 },
1396 Err(e) => {
1397 post_error("post", &format!("cannot re-read a disk-staged page: {e}"));
1398 return Err(());
1399 },
1400 };
1401 // Free the spill as soon as it is parsed: for a 40k-page document the
1402 // spill area is the size of the core XML, and it need not outlive its page.
1403 let _ = std::fs::remove_file(path);
1404
1405 // CrossRef resolves this page's references out of the completed ObjectDB,
1406 // and its navigation/TOC work reads the DB rather than sibling pages — so
1407 // it is page-local once pass A has finished.
1408 telemetry::phase_enter(Phase::Crossref);
1409 let t_xref = audit_start("CrossRef");
1410 let xref = run_page_phase(page, &mut procs.crossref, "CrossRef");
1411 audit_end(t_xref);
1412 telemetry::phase_exit();
1413 let mut staged = xref?;
1414 if let Some(gfx) = procs.graphics.as_mut() {
1415 telemetry::phase_enter(Phase::Graphics);
1416 let t_gfx = audit_start("Graphics");
1417 let mut next = Vec::with_capacity(staged.len());
1418 let mut failed = false;
1419 for d in staged.drain(..) {
1420 match run_page_phase(d, gfx, "Graphics") {
1421 Ok(out) => next.extend(out),
1422 Err(()) => {
1423 failed = true;
1424 break;
1425 },
1426 }
1427 }
1428 audit_end(t_gfx);
1429 telemetry::phase_exit();
1430 if failed {
1431 return Err(());
1432 }
1433 staged = next;
1434 }
1435
1436 // Harvest the Graphics-resolved image for every `<ltx:graphics>` that lives
1437 // inside a `<ltx:picture>`, keyed by `xml:id`. The picture's serialized SVG
1438 // fragment (captured in pass A, BEFORE Graphics ran) carries the RAW
1439 // `<graphics>` with no `@imagesrc`; the imagesrc set here on the page DOM is
1440 // then discarded when XSLT collapses the picture to an empty placeholder span.
1441 // So we snapshot the resolution now and re-inject it into the fragment at
1442 // splice time (`finalize_html5`). This restores Perl's Graphics→SVG order
1443 // (`LaTeXML.pm` L493 before L502), which our pass-A/pass-B split inverts.
1444 // arXiv/html_feedback#1291 (witness arXiv:2311.14363v2, Inkscape `.pdf_tex`
1445 // figures). Only meaningful for the HTML splice path.
1446 let resolved_graphics = if ctx.is_html_out {
1447 harvest_resolved_graphics(&staged)
1448 } else {
1449 rustc_hash::FxHashMap::default()
1450 };
1451
1452 let rendered = match procs.post.process_chain(staged, &mut procs.processors) {
1453 Ok(r) => r,
1454 Err(e) => {
1455 post_error("convert", &format!("Post-processing failed: {e}"));
1456 return Err(());
1457 },
1458 };
1459
1460 let mut outputs = Vec::with_capacity(rendered.len());
1461 for doc in rendered {
1462 let dest = doc.get_destination().map(String::from);
1463 let output = latexml_post::extract::serialize_whatsout(&doc, ctx.whatsout);
1464 let output = if ctx.is_html_out {
1465 finalize_html5(output, &ctx.svg_fragments, &resolved_graphics)
1466 } else {
1467 output
1468 };
1469 let output = if ctx.schemadocs && ctx.is_html_out {
1470 latexml_post::schema_docs::process_page(&output)
1471 } else {
1472 output
1473 };
1474 outputs.push((dest, output));
1475 // `doc` drops here — the whole point.
1476 }
1477 Ok(outputs)
1478}
1479
1480/// One page spilled between the post pipeline's two passes: where its XML
1481/// lives, the metadata that does NOT survive an XML round-trip, and whether it
1482/// carries the placeholders the two ObjectDB-baton sweeps look for (recorded in
1483/// pass A, while the page is already parsed).
1484struct SpilledPage {
1485 path: std::path::PathBuf,
1486 destination: Option<String>,
1487 destination_directory: Option<String>,
1488 needs_index: bool,
1489 needs_bib: bool,
1490}
1491
1492/// Re-derive SVG fragments from a parsed document (the file-input equivalent of
1493/// [`extract_svg_fragments`]).
1494///
1495/// `//ltx:picture` is a *typed* descendant query, which libxml2 evaluates
1496/// without materializing `descendant-or-self::node()` — so it stays under the
1497/// 10M node-set ceiling even on a 600 MB document. Most large documents have no
1498/// pictures at all (the fast, zero-serialization path); when they do, only the
1499/// picture subtrees are serialized and handed to the shared string extractor,
1500/// never the whole file.
1501fn extract_svg_fragments_from_doc(doc: &PostDocument) -> Vec<(String, String)> {
1502 let pictures = doc.findnodes("//ltx:picture");
1503 if pictures.is_empty() {
1504 return Vec::new();
1505 }
1506 let joined: String = pictures.iter().map(|p| doc.node_to_string(p)).collect();
1507 extract_svg_fragments(&joined)
1508}
1509
1510/// A picture-nested `<ltx:graphics>` that the Graphics phase resolved to a web
1511/// image, harvested (by `xml:id`) from the post-Graphics page DOM before XSLT
1512/// discards it. Re-injected into the picture's SVG fragment at splice time so
1513/// the `<foreignObject>` carries a real `<img>`/`<object>` instead of the raw
1514/// `<graphics>` the pass-A snapshot froze. arXiv/html_feedback#1291.
1515struct ResolvedGraphic {
1516 imagesrc: String,
1517 imagewidth: Option<String>,
1518 imageheight: Option<String>,
1519 /// The `ltx_img_landscape|portrait|square` token Graphics adds (aspect ratio).
1520 aspect_class: Option<String>,
1521 /// Mirrors the XSLT `alt`: `@description`, else "Refer to caption" when inside
1522 /// a captioned figure, else "[Uncaptioned image]".
1523 alt: String,
1524 is_svg: bool,
1525}
1526
1527/// Harvest [`ResolvedGraphic`]s for every `<ltx:graphics>` inside a
1528/// `<ltx:picture>` that Graphics gave an `@imagesrc`, keyed by `xml:id`.
1529/// The `alt`/`object`-vs-`img` decisions mirror `LaTeXML-misc-xhtml.xsl`'s
1530/// `ltx:graphics` templates (L142 raster → `<img>`, L192 `.svg` → `<object>`).
1531fn harvest_resolved_graphics(
1532 docs: &[PostDocument],
1533) -> rustc_hash::FxHashMap<String, ResolvedGraphic> {
1534 let mut map = rustc_hash::FxHashMap::default();
1535 for d in docs {
1536 for g in d.findnodes("//ltx:graphics[@imagesrc][ancestor::ltx:picture]") {
1537 // xml:id is namespace-bound; the bare string key silently misses it
1538 // (docs/archive/XMLID_ACCESSOR_AUDIT_2026-06-08.md).
1539 let Some(id) = g.get_attribute_ns("id", latexml_core::common::xml::XML_NS) else {
1540 continue;
1541 };
1542 let imagesrc = g.get_attribute("imagesrc").unwrap_or_default();
1543 if imagesrc.is_empty() {
1544 continue;
1545 }
1546 let is_svg = imagesrc.ends_with(".svg");
1547 let aspect_class = g.get_attribute("class").and_then(|c| {
1548 c.split_whitespace()
1549 .find(|t| t.starts_with("ltx_img_"))
1550 .map(str::to_string)
1551 });
1552 let alt = if let Some(desc) = g.get_attribute("description") {
1553 desc
1554 } else if !d
1555 .findnodes_at("ancestor::ltx:figure[1]/ltx:caption", Some(&g))
1556 .is_empty()
1557 {
1558 "Refer to caption".to_string()
1559 } else {
1560 "[Uncaptioned image]".to_string()
1561 };
1562 map.insert(id, ResolvedGraphic {
1563 imagesrc,
1564 imagewidth: g.get_attribute("imagewidth"),
1565 imageheight: g.get_attribute("imageheight"),
1566 aspect_class,
1567 alt,
1568 is_svg,
1569 });
1570 }
1571 }
1572 map
1573}
1574
1575/// Escape a string for use inside a double-quoted HTML attribute value.
1576fn escape_attr(s: &str) -> String {
1577 s.replace('&', "&")
1578 .replace('<', "<")
1579 .replace('>', ">")
1580 .replace('"', """)
1581}
1582
1583/// A self-closing `<graphics …/>` element — the frozen pre-Graphics placeholder
1584/// the picture-SVG path carries until splice time. Shared by the two consumers:
1585/// `convert_picture_children_to_svg` (strip it from a makebox `<text>` copy /
1586/// re-emit it as `<foreignObject>`) and `resolve_fragment_graphics` (rewrite it
1587/// to the resolved `<img>`/`<object>`).
1588static GRAPHICS_SELF_CLOSE_RE: std::sync::LazyLock<regex::Regex> =
1589 std::sync::LazyLock::new(|| regex::Regex::new(r"<graphics\b[^>]*/>").unwrap());
1590
1591/// Render a resolved graphic as the HTML the XSLT would have produced for the
1592/// `<ltx:graphics>` — `<object>` for an `.svg` imagesrc (interactivity), `<img>`
1593/// otherwise. Attribute set and order (src/data, id, class, width, height,
1594/// alt/aria-label) mirror `LaTeXML-misc-xhtml.xsl` (`add_id` carries the
1595/// graphic's `xml:id` onto the element, as Perl's SVG.pm→XSLT chain does).
1596fn render_resolved_graphic(id: &str, g: &ResolvedGraphic, constrain: bool) -> String {
1597 let id = format!(" id=\"{}\"", escape_attr(id));
1598 let class = match &g.aspect_class {
1599 Some(c) => format!(" class=\"ltx_graphics {c}\""),
1600 None => " class=\"ltx_graphics\"".to_string(),
1601 };
1602 let w = g
1603 .imagewidth
1604 .as_deref()
1605 .map(|w| format!(" width=\"{}\"", escape_attr(w)))
1606 .unwrap_or_default();
1607 let h = g
1608 .imageheight
1609 .as_deref()
1610 .map(|h| format!(" height=\"{}\"", escape_attr(h)))
1611 .unwrap_or_default();
1612 let src = escape_attr(&g.imagesrc);
1613 let alt = escape_attr(&g.alt);
1614 // Constrain the image to the `<foreignObject>` box the SVG already sized. The
1615 // enclosing `<foreignObject overflow="visible">` (SVG.pm L148-183 port) takes
1616 // its size from the graphic's own width/imagewidth in the picture's scaled
1617 // coordinate space; without this the browser instead draws the `<img>` at the
1618 // raster's *natural* pixel size (the `width`/`height` px attrs), which spills
1619 // out of that box — the giant-image regression (arXiv:2510.17772 Fig 7, every
1620 // picture-nested `\includegraphics`, e.g. overpic). `100%` fills the sized box;
1621 // `object-fit:contain` preserves the aspect ratio.
1622 //
1623 // `constrain` is false only for a DEGENERATE picture SVG (sub-pixel outer size,
1624 // the `\unitlength`-not-applied core gap — SYNC_STATUS "picture-SVG unitlength
1625 // sizing", witness arXiv:2311.14363v2 Inkscape `.pdf_tex`). There the box is
1626 // ~0px, so a `100%`/`object-fit` constraint would collapse the image to nothing
1627 // (worse than the pre-existing overflow leak); leave it unconstrained until the
1628 // core sizing is fixed. `resolve_fragment_graphics` makes the call.
1629 let style = if constrain {
1630 " style=\"width:100%;height:100%;object-fit:contain\""
1631 } else {
1632 ""
1633 };
1634 if g.is_svg {
1635 // The XSLT uses aria-label (not alt) on <object>, and a <p> fallback; the
1636 // aria-label alone is enough inside the SVG foreignObject.
1637 format!(
1638 "<object type=\"image/svg+xml\" data=\"{src}\"{id}{class}{style}{w}{h} aria-label=\"{alt}\"></object>"
1639 )
1640 } else {
1641 format!("<img src=\"{src}\"{id}{class}{style}{w}{h} alt=\"{alt}\">")
1642 }
1643}
1644
1645/// Rewrite each raw `<graphics .../>` in a picture's SVG fragment into the
1646/// Graphics-resolved `<img>`/`<object>`, matched by its own `xml:id`. A picture
1647/// can hold several (e.g. an Inkscape `.pdf_tex` overlay: `page=1` background +
1648/// `page=2` labels). An id with no resolved entry is left as-is (a genuinely
1649/// unconverted / missing image, exactly as before).
1650fn resolve_fragment_graphics(
1651 fragment: &str,
1652 resolved: &rustc_hash::FxHashMap<String, ResolvedGraphic>,
1653) -> String {
1654 use std::sync::LazyLock;
1655 static XMLID_RE: LazyLock<regex::Regex> =
1656 LazyLock::new(|| regex::Regex::new(r#"xml:id="([^"]+)""#).unwrap());
1657 static SVG_DIM_RE: LazyLock<regex::Regex> = LazyLock::new(|| {
1658 regex::Regex::new(r#"<svg\b[^>]*\bwidth="([0-9.]+)"[^>]*\bheight="([0-9.]+)""#).unwrap()
1659 });
1660 if resolved.is_empty() || !fragment.contains("<graphics") {
1661 return fragment.to_string();
1662 }
1663 // Constrain the resolved images to their `<foreignObject>` boxes (object-fit)
1664 // ONLY when the picture's outer `<svg>` has a real coordinate extent. A
1665 // sub-pixel outer SVG is the `\unitlength`-not-applied core gap (SYNC_STATUS
1666 // "picture-SVG unitlength sizing", arXiv:2311.14363v2): its foreignObjects
1667 // collapse in the browser, so a `100%` constraint would zero the image. Below
1668 // `DEGENERATE_SVG_PX` we leave images unconstrained (the pre-existing overflow
1669 // behavior) rather than hide them. A well-sized picture (overpic, ~186px+) is
1670 // far above the floor; a degenerate one is ~1px.
1671 const DEGENERATE_SVG_PX: f64 = 4.0;
1672 let constrain = SVG_DIM_RE
1673 .captures(fragment)
1674 .and_then(|c| {
1675 let w = c[1].parse::<f64>().ok()?;
1676 let h = c[2].parse::<f64>().ok()?;
1677 Some(w >= DEGENERATE_SVG_PX && h >= DEGENERATE_SVG_PX)
1678 })
1679 // No parseable outer <svg> dims → assume a real picture and constrain.
1680 .unwrap_or(true);
1681 GRAPHICS_SELF_CLOSE_RE
1682 .replace_all(fragment, |caps: ®ex::Captures| {
1683 let tag = &caps[0];
1684 match XMLID_RE.captures(tag) {
1685 Some(c) => match resolved.get(&c[1]) {
1686 Some(g) => render_resolved_graphic(&c[1], g, constrain),
1687 None => tag.to_string(),
1688 },
1689 None => tag.to_string(),
1690 }
1691 })
1692 .to_string()
1693}
1694
1695/// Apply HTML5 cleanup (XML prolog strip, void-element fixes) and inject SVG
1696/// fragments into empty `ltx_picture` spans. Pulled out of `run_post_processing`
1697/// so it can run on every split sub-document, not just the first.
1698///
1699/// `resolved_graphics` carries the Graphics-phase image resolution for
1700/// picture-nested `<graphics>` (see [`harvest_resolved_graphics`]); the raw
1701/// `<graphics>` frozen into each fragment during pass-A extraction is rewritten
1702/// to the resolved `<img>`/`<object>` here. arXiv/html_feedback#1291.
1703fn finalize_html5(
1704 output: String,
1705 svg_fragments: &[(String, String)],
1706 resolved_graphics: &rustc_hash::FxHashMap<String, ResolvedGraphic>,
1707) -> String {
1708 use std::sync::LazyLock;
1709 // Cached at first call — regex compile is the slow part of `Regex::new`,
1710 // and finalize_html5 runs on every (sub-)document in the post-pipeline.
1711 static XML_PROLOG_RE: LazyLock<regex::Regex> =
1712 LazyLock::new(|| regex::Regex::new(r"^<\?xml[^?]*\?>\s*").unwrap());
1713 static NON_VOID_SELF_CLOSE_RE: LazyLock<regex::Regex> = LazyLock::new(|| {
1714 regex::Regex::new(
1715 r"<(span|div|p|a|td|th|tr|section|article|figure|figcaption|pre|code|em|strong|b|i|u|sub|sup|small|cite)(\s[^>]*)?/>",
1716 ).unwrap()
1717 });
1718 static VOID_CLOSE_RE: LazyLock<regex::Regex> = LazyLock::new(|| {
1719 regex::Regex::new(r"</(br|img|hr|input|meta|link|col|area|base|source|track|wbr|embed|param)>")
1720 .unwrap()
1721 });
1722 static VOID_SELF_CLOSE_RE: LazyLock<regex::Regex> = LazyLock::new(|| {
1723 regex::Regex::new(
1724 r"<(br|img|hr|input|meta|link|col|area|base|source|track|wbr|embed|param)(\s[^>]*?)\s*/>",
1725 )
1726 .unwrap()
1727 });
1728
1729 let _gp_html5 = telemetry::phase(Phase::Html5Fixups);
1730 // Strip <?xml version...?> prolog: HTML5 must NOT have an XML declaration.
1731 // libxml2's to_string() includes it by default; we strip it here.
1732 let output = XML_PROLOG_RE.replace(&output, "").to_string();
1733 let output = NON_VOID_SELF_CLOSE_RE
1734 .replace_all(&output, "<$1$2></$1>")
1735 .to_string();
1736 let output = VOID_CLOSE_RE.replace_all(&output, "").to_string();
1737 let mut output = VOID_SELF_CLOSE_RE
1738 .replace_all(&output, "<$1$2>")
1739 .to_string();
1740 // SVG fragment injection into the empty `ltx_picture` placeholder spans.
1741 //
1742 // WHY this is a string splice and not a DOM operation (issue #398).
1743 // The SVG must land AFTER the XSLT stage (it is collected separately by
1744 // `extract_svg_fragments` / the streaming split), and the obvious DOM
1745 // alternative — re-parse the SVG fragment and append it under the placeholder
1746 // node before serialization — was deliberately abandoned: it tripped a
1747 // libxml2 use-after-free in `PostDocument` cleanup (see the note on
1748 // `extract_svg_fragments`). So we splice into the serialized string instead.
1749 //
1750 // The original splice hard-coded the placeholder as `<span id="…"
1751 // class="ltx_picture"…>` — coupled to id-before-class order AND double quotes,
1752 // neither guaranteed by a serializer. Measured (issue #398): on every LIVE
1753 // input the coupling holds — the XSLT emits `id` (add_id) then `class`
1754 // (add_classes is the first thing add_attributes does), and libxml2 preserves
1755 // insertion order and always double-quotes — so no real break was
1756 // demonstrable; the fragility is latent. We keep the string splice (the DOM
1757 // refactor is not worth re-entering the use-after-free for a latent concern)
1758 // but make the MATCH robust so a future serializer/XSLT change cannot silently
1759 // break it: match any empty `<span>` whose class contains `ltx_picture`,
1760 // regardless of attribute order or quote style, and look the fragment up by
1761 // the id found inside.
1762 //
1763 // WHEN TO REVISIT (do the real DOM refactor and delete this splice):
1764 // * once the fork's `PostDocument` cleanup no longer use-after-frees on an
1765 // inserted subtree (rust-libxml; we have fixed adjacent UAF/NULL-deref
1766 // bugs there this year) — then inject the SVG as child nodes of the
1767 // placeholder in the post-XSLT DOM (`doc`, before `serialize_whatsout`),
1768 // which also lets the void-element normalization above move to a proper
1769 // HTML serializer;
1770 // * or if a demonstrated attribute-order/quote break ever appears (none
1771 // today) — that would raise the priority from latent to real.
1772 if !svg_fragments.is_empty() {
1773 static EMPTY_PICTURE_SPAN: LazyLock<regex::Regex> = LazyLock::new(|| {
1774 // An empty `<span …></span>` carrying `ltx_picture` as a whole class
1775 // token, in a single- or double-quoted class value, with the other
1776 // attributes (notably `id`) in ANY position.
1777 regex::Regex::new(
1778 r#"<span\b(?P<attrs>[^>]*\bclass\s*=\s*(?:"[^"]*\bltx_picture\b[^"]*"|'[^']*\bltx_picture\b[^']*')[^>]*)></span>"#,
1779 )
1780 .unwrap()
1781 });
1782 static SPAN_ID: LazyLock<regex::Regex> =
1783 LazyLock::new(|| regex::Regex::new(r#"\bid\s*=\s*(?:"([^"]+)"|'([^']+)')"#).unwrap());
1784 output = EMPTY_PICTURE_SPAN
1785 .replace_all(&output, |caps: ®ex::Captures| {
1786 let attrs = &caps["attrs"];
1787 let id = SPAN_ID
1788 .captures(attrs)
1789 .and_then(|c| c.get(1).or_else(|| c.get(2)))
1790 .map(|m| m.as_str());
1791 match id.and_then(|id| svg_fragments.iter().find(|(fid, _)| fid == id)) {
1792 // Preserve the placeholder's own attributes verbatim; only fill it.
1793 // The fragment's raw `<graphics>` (frozen pre-Graphics) is rewritten
1794 // to the resolved image here — Perl's Graphics→SVG order (#1291).
1795 Some((_, svg_html)) => {
1796 let filled = resolve_fragment_graphics(svg_html, resolved_graphics);
1797 format!("<span{attrs}>{filled}</span>")
1798 },
1799 // A picture span we have no fragment for — leave it untouched.
1800 None => caps[0].to_string(),
1801 }
1802 })
1803 .to_string();
1804 }
1805 output
1806}
1807
1808/// Extract SVG fragments from intermediate LaTeXML XML.
1809///
1810/// Finds `<picture>` elements, converts their children to inline SVG HTML.
1811/// Uses a lightweight regex+string approach (no libxml2) to avoid the
1812/// use-after-free crash in PostDocument cleanup.
1813///
1814/// Returns (picture_id, svg_html) pairs for post-XSLT injection.
1815fn extract_svg_fragments(xml: &str) -> Vec<(String, String)> {
1816 use std::sync::LazyLock;
1817 // Fast-fail: most documents have no `<picture>` elements (tikz / pgf
1818 // is uncommon in the canvas). Skip the backtracking lazy-match
1819 // regex (`(?s)...(.*?)`) when `<picture` doesn't appear as a literal
1820 // substring. `str::contains` is a SIMD-accelerated byte search and
1821 // takes microseconds even on ~MB inputs.
1822 if !xml.contains("<picture") {
1823 return Vec::new();
1824 }
1825 static PICTURE_RE: LazyLock<regex::Regex> =
1826 LazyLock::new(|| regex::Regex::new(r#"(?s)<picture([^>]*)>(.*?)</picture>"#).unwrap());
1827 static ID_RE: LazyLock<regex::Regex> =
1828 LazyLock::new(|| regex::Regex::new(r#"xml:id="([^"]+)""#).unwrap());
1829 static WIDTH_RE: LazyLock<regex::Regex> =
1830 LazyLock::new(|| regex::Regex::new(r#"width="([^"]+)""#).unwrap());
1831 static HEIGHT_RE: LazyLock<regex::Regex> =
1832 LazyLock::new(|| regex::Regex::new(r#"height="([^"]+)""#).unwrap());
1833 let mut fragments = Vec::new();
1834 let picture_re = &*PICTURE_RE;
1835 let id_re = &*ID_RE;
1836 let width_re = &*WIDTH_RE;
1837 let height_re = &*HEIGHT_RE;
1838
1839 for pic_caps in picture_re.captures_iter(xml) {
1840 let attrs = &pic_caps[1];
1841 let content = &pic_caps[2];
1842 let id = id_re
1843 .captures(attrs)
1844 .map(|c| c[1].to_string())
1845 .unwrap_or_default();
1846 let width = width_re.captures(attrs).and_then(|c| parse_tex_dim(&c[1]));
1847 let height = height_re.captures(attrs).and_then(|c| parse_tex_dim(&c[1]));
1848
1849 if id.is_empty() || content.trim().is_empty() {
1850 continue;
1851 }
1852
1853 let w = width.unwrap_or(100.0);
1854 let h = height.unwrap_or(100.0);
1855
1856 // Build inline SVG: coordinate system has y-flip (TeX origin bottom-left, SVG top-left)
1857 let mut svg_content = format!(
1858 r#"<svg xmlns="http://www.w3.org/2000/svg" version="1.1" width="{w:.2}" height="{h:.2}" overflow="visible">"#,
1859 );
1860 svg_content.push_str(&format!(
1861 r#"<g transform="translate(0,{h:.2}) scale(1,-1)">"#,
1862 ));
1863
1864 // Convert ltx picture children to SVG elements
1865 svg_content.push_str(&convert_picture_children_to_svg(content));
1866
1867 svg_content.push_str("</g></svg>");
1868 fragments.push((id, svg_content));
1869 }
1870 fragments
1871}
1872
1873/// Convert LaTeXML picture children (g, line, text, circle, etc.) to SVG elements.
1874fn convert_picture_children_to_svg(content: &str) -> String {
1875 use std::sync::LazyLock;
1876 static G_RE: LazyLock<regex::Regex> =
1877 LazyLock::new(|| regex::Regex::new(r#"(?s)<g([^>]*)>(.*?)</g>"#).unwrap());
1878 static TRANSFORM_RE: LazyLock<regex::Regex> =
1879 LazyLock::new(|| regex::Regex::new(r#"transform="([^"]+)""#).unwrap());
1880 static LINE_RE: LazyLock<regex::Regex> =
1881 LazyLock::new(|| regex::Regex::new(r#"<line\s+points="([^"]+)"([^/]*)/?>"#).unwrap());
1882 static CIRCLE_RE: LazyLock<regex::Regex> =
1883 LazyLock::new(|| regex::Regex::new(r#"<circle([^/]*)/?>"#).unwrap());
1884 static ELLIPSE_RE: LazyLock<regex::Regex> =
1885 LazyLock::new(|| regex::Regex::new(r#"<ellipse([^/]*)/?>"#).unwrap());
1886 static RECT_RE: LazyLock<regex::Regex> =
1887 LazyLock::new(|| regex::Regex::new(r#"<rect([^/]*)/?>"#).unwrap());
1888 static POLYGON_RE: LazyLock<regex::Regex> =
1889 LazyLock::new(|| regex::Regex::new(r#"<polygon([^/]*)/?>"#).unwrap());
1890 static PATH_RE: LazyLock<regex::Regex> =
1891 LazyLock::new(|| regex::Regex::new(r#"<path([^/]*)/?>"#).unwrap());
1892 static BEZIER_RE: LazyLock<regex::Regex> =
1893 LazyLock::new(|| regex::Regex::new(r#"<bezier\s+points="([^"]+)"([^/]*)/?>"#).unwrap());
1894 static TEXT_RE: LazyLock<regex::Regex> =
1895 LazyLock::new(|| regex::Regex::new(r#"(?s)<text([^>]*)>(.*?)</text>"#).unwrap());
1896 // Foreign (non-SVG) content that a picture makebox can carry: a math label or
1897 // an embedded graphic. Perl's SVG.pm wraps these in <foreignObject>. The
1898 // `<graphics …/>` matcher is the module-level `GRAPHICS_SELF_CLOSE_RE` (shared
1899 // with `resolve_fragment_graphics`).
1900 static MATH_RE: LazyLock<regex::Regex> =
1901 LazyLock::new(|| regex::Regex::new(r#"(?s)<Math\b[^>]*>.*?</Math>"#).unwrap());
1902
1903 let mut svg = String::new();
1904
1905 for g_caps in G_RE.captures_iter(content) {
1906 let g_attrs = &g_caps[1];
1907 let g_content = &g_caps[2];
1908
1909 // Extract transform
1910 let transform = TRANSFORM_RE.captures(g_attrs).map(|c| c[1].to_string());
1911
1912 if let Some(t) = &transform {
1913 svg.push_str(&format!(r#"<g transform="{t}">"#));
1914 } else {
1915 svg.push_str("<g>");
1916 }
1917
1918 // <line points="x1,y1 x2,y2" stroke="..." stroke-width="..."/>
1919 for line_caps in LINE_RE.captures_iter(g_content) {
1920 let points = &line_caps[1];
1921 let rest_attrs = &line_caps[2];
1922 let coords: Vec<&str> = points.split_whitespace().collect();
1923 if coords.len() >= 2 {
1924 let p1: Vec<&str> = coords[0].split(',').collect();
1925 let p2: Vec<&str> = coords[1].split(',').collect();
1926 if p1.len() == 2 && p2.len() == 2 {
1927 svg.push_str(&format!(
1928 r#"<line x1="{}" y1="{}" x2="{}" y2="{}"{}/>"#,
1929 p1[0], p1[1], p2[0], p2[1], rest_attrs
1930 ));
1931 }
1932 }
1933 }
1934
1935 // <circle cx="..." cy="..." r="..." .../>
1936 for circle_caps in CIRCLE_RE.captures_iter(g_content) {
1937 svg.push_str(&format!("<circle{}/>", &circle_caps[1]));
1938 }
1939
1940 // <ellipse cx="..." cy="..." rx="..." ry="..." .../>
1941 for ellipse_caps in ELLIPSE_RE.captures_iter(g_content) {
1942 svg.push_str(&format!("<ellipse{}/>", &ellipse_caps[1]));
1943 }
1944
1945 // <rect x="..." y="..." width="..." height="..." .../>
1946 for rect_caps in RECT_RE.captures_iter(g_content) {
1947 svg.push_str(&format!("<rect{}/>", &rect_caps[1]));
1948 }
1949
1950 // <polygon points="..." .../>
1951 for polygon_caps in POLYGON_RE.captures_iter(g_content) {
1952 svg.push_str(&format!("<polygon{}/>", &polygon_caps[1]));
1953 }
1954
1955 // <path d="..." .../>
1956 for path_caps in PATH_RE.captures_iter(g_content) {
1957 svg.push_str(&format!("<path{}/>", &path_caps[1]));
1958 }
1959
1960 // <bezier points="x1,y1 x2,y2 x3,y3 x4,y4" .../>
1961 // Convert to SVG cubic bezier path
1962 for bez_caps in BEZIER_RE.captures_iter(g_content) {
1963 let points = &bez_caps[1];
1964 let rest = &bez_caps[2];
1965 let coords: Vec<&str> = points.split_whitespace().collect();
1966 if coords.len() >= 4 {
1967 // SVG cubic bezier: M x0,y0 C x1,y1 x2,y2 x3,y3
1968 let d = format!(
1969 "M {} C {} {} {}",
1970 coords[0], coords[1], coords[2], coords[3]
1971 );
1972 svg.push_str(&format!(r#"<path d="{d}"{rest} fill="none"/>"#));
1973 } else if coords.len() >= 3 {
1974 // Quadratic bezier: M x0,y0 Q x1,y1 x2,y2
1975 let d = format!("M {} Q {} {}", coords[0], coords[1], coords[2]);
1976 svg.push_str(&format!(r#"<path d="{d}"{rest} fill="none"/>"#));
1977 }
1978 }
1979
1980 // <arc .../> — arc segments (rarely used, stub for now)
1981 // <wedge .../> — filled wedges (rarely used, stub for now)
1982
1983 // <text>...</text> — wrap in SVG text with y-flip correction. A picture
1984 // makebox (e.g. overpic's `\put(0,0){\makebox(0,0)[bl]{\includegraphics}}`)
1985 // serializes its graphic/math INSIDE the `<text>`; strip those here — the
1986 // foreign-content block below emits each `<graphics>`/`<Math>` once, as a
1987 // faithful `<foreignObject>`. Left in, the graphic renders a SECOND time,
1988 // unconstrained, inside an SVG `<text>` (the overpic double-image,
1989 // arXiv:2510.17772 Fig 7). Plain-text labels are untouched.
1990 for text_caps in TEXT_RE.captures_iter(g_content) {
1991 let text_attrs = &text_caps[1];
1992 let text_content = GRAPHICS_SELF_CLOSE_RE.replace_all(&text_caps[2], "");
1993 let text_content = MATH_RE.replace_all(&text_content, "");
1994 // The overpic BACKGROUND makebox wraps only the graphic, so after the strip
1995 // there is no label left — skip the wrapper rather than emit a dead
1996 // `<text></text>` (the graphic still renders via the foreign block below).
1997 if text_content.trim().is_empty() {
1998 continue;
1999 }
2000 svg.push_str(&format!(
2001 r#"<g transform="scale(1,-1)"><text{text_attrs}>{text_content}</text></g>"#,
2002 ));
2003 }
2004
2005 // Foreign content (a `<Math>` label or a `<graphics>` image inside a picture
2006 // makebox) must survive into the SVG wrapped in a `<foreignObject>`, not be
2007 // dropped. Mirrors `SVG.pm::convertNode`
2008 // (LaTeXML/lib/LaTeXML/Post/SVG.pm:148-183): a non-SVG child takes its size
2009 // from its own width/imagewidth, else the containing `<g>`'s
2010 // innerwidth/width (defaults 1pt, non-zero required), and is placed in a
2011 // y-flipped `<g><foreignObject overflow="visible">`. html_feedback#74
2012 // (arXiv:0810.1673v3 Fig 2 — xfig math labels + `\epsfig` image).
2013 //
2014 // NOTE: `latexml_post::svg::SVG::convert_foreign` is the *canonical*,
2015 // DOM-based faithful port (it already handles this). This string-level
2016 // reimplementation exists only because the active SVG path splices serialized
2017 // strings post-XSLT to dodge a libxml2 PostDocument-cleanup UAF (see the
2018 // `finalize_html5` splice note). When that path can carry a live DOM, this
2019 // block and its siblings above are deleted in favor of the `svg.rs` Processor.
2020 for node in MATH_RE
2021 .find_iter(g_content)
2022 .chain(GRAPHICS_SELF_CLOSE_RE.find_iter(g_content))
2023 .map(|m| m.as_str())
2024 {
2025 let width = svg_attr(node, "width")
2026 .or_else(|| svg_attr(node, "imagewidth"))
2027 .or_else(|| svg_attr(g_attrs, "innerwidth"))
2028 .or_else(|| svg_attr(g_attrs, "width"))
2029 .unwrap_or_else(|| "1pt".to_string());
2030 let height = svg_attr(node, "height")
2031 .or_else(|| svg_attr(node, "imageheight"))
2032 .or_else(|| svg_attr(g_attrs, "innerheight"))
2033 .or_else(|| svg_attr(g_attrs, "height"))
2034 .unwrap_or_else(|| "1pt".to_string());
2035 let depth = svg_attr(node, "depth")
2036 .or_else(|| svg_attr(g_attrs, "innerdepth"))
2037 .or_else(|| svg_attr(g_attrs, "depth"))
2038 .unwrap_or_else(|| "0pt".to_string());
2039 let px_w = parse_tex_dim(&width).unwrap_or(1.0);
2040 let px_h = parse_tex_dim(&height).unwrap_or(1.0);
2041 let px_d = parse_tex_dim(&depth).unwrap_or(0.0);
2042 // Perl: y = to_px(height) + to_px(depth); flip so foreign content is upright.
2043 let y = px_h + px_d;
2044 svg.push_str(&format!(
2045 r#"<g transform="translate(0,{y:.2}) scale(1,-1)"><foreignObject width="{px_w:.2}" height="{px_h:.2}" overflow="visible">{node}</foreignObject></g>"#
2046 ));
2047 }
2048
2049 svg.push_str("</g>");
2050 }
2051
2052 // Also handle direct children not inside <g> (e.g. top-level <bezier>, <line>)
2053 // These appear directly inside <picture> without a <g> wrapper
2054 let direct_bezier_re =
2055 regex::Regex::new(r#"(?m)^\s*<bezier\s+points="([^"]+)"([^/]*)/?>"#).unwrap();
2056 for bez_caps in direct_bezier_re.captures_iter(content) {
2057 let points = &bez_caps[1];
2058 let rest = &bez_caps[2];
2059 let coords: Vec<&str> = points.split_whitespace().collect();
2060 if coords.len() >= 4 {
2061 let d = format!(
2062 "M {} C {} {} {}",
2063 coords[0], coords[1], coords[2], coords[3]
2064 );
2065 svg.push_str(&format!(r#"<path d="{d}"{rest} fill="none"/>"#));
2066 } else if coords.len() >= 3 {
2067 let d = format!("M {} Q {} {}", coords[0], coords[1], coords[2]);
2068 svg.push_str(&format!(r#"<path d="{d}"{rest} fill="none"/>"#));
2069 }
2070 }
2071
2072 svg
2073}
2074
2075/// Extract a `name="value"` attribute from a serialized element/attribute
2076/// string. Matches `name` only at an attribute boundary (string start,
2077/// whitespace, or after `<`), so a query for `width` does NOT match
2078/// `innerwidth="…"`. Returns the first such value.
2079fn svg_attr(s: &str, name: &str) -> Option<String> {
2080 let pat = format!("{name}=\"");
2081 let bytes = s.as_bytes();
2082 let mut from = 0;
2083 while let Some(rel) = s[from..].find(&pat) {
2084 let at = from + rel;
2085 let boundary = at == 0 || bytes[at - 1].is_ascii_whitespace() || bytes[at - 1] == b'<';
2086 if boundary {
2087 let vstart = at + pat.len();
2088 return s[vstart..]
2089 .find('"')
2090 .map(|end| s[vstart..vstart + end].to_string());
2091 }
2092 from = at + pat.len();
2093 }
2094 None
2095}
2096
2097/// Parse a TeX dimension string (e.g. "100.0pt") to pixels.
2098fn parse_tex_dim(s: &str) -> Option<f64> {
2099 let s = s.trim();
2100 if let Some(rest) = s.strip_suffix("pt") {
2101 rest.parse::<f64>().ok().map(|v| v * 96.0 / 72.27)
2102 } else if let Some(rest) = s.strip_suffix("px") {
2103 rest.parse::<f64>().ok()
2104 } else {
2105 s.parse::<f64>().ok()
2106 }
2107}
2108
2109#[cfg(test)]
2110mod finalize_html5_splice_tests {
2111 //! #398: the `ltx_picture` SVG splice must fill the placeholder span
2112 //! regardless of the serialized attribute ORDER or QUOTE style — the coupling
2113 //! the original `<span id="…" class="ltx_picture"…>` regex had. No live input
2114 //! perturbs the order today (the XSLT emits id-then-class, libxml2 preserves
2115 //! order and double-quotes), so these craft the perturbations directly to keep
2116 //! the match robust against a future serializer/XSLT change. The old regex
2117 //! FAILED cases 2-4 (class-first / single-quotes / an attribute between id and
2118 //! class); the hardened match passes them.
2119 use super::finalize_html5;
2120
2121 fn splice(span: &str) -> String {
2122 finalize_html5(
2123 span.to_string(),
2124 &[("p1".to_string(), "<svg>OK</svg>".to_string())],
2125 &rustc_hash::FxHashMap::default(),
2126 )
2127 }
2128
2129 #[test]
2130 fn splice_is_attribute_order_and_quote_agnostic() {
2131 // 1. Canonical (id-first, double-quote) — the only shape seen live.
2132 assert!(splice(r#"<span id="p1" class="ltx_picture"></span>"#).contains("<svg>OK</svg>"));
2133 // 2. class BEFORE id — the old regex required id-then-class.
2134 assert!(splice(r#"<span class="ltx_picture" id="p1"></span>"#).contains("<svg>OK</svg>"));
2135 // 3. single quotes — the old regex hard-coded double quotes.
2136 assert!(splice(r#"<span id='p1' class='ltx_picture'></span>"#).contains("<svg>OK</svg>"));
2137 // 4. an attribute BETWEEN id and class — the old regex needed them adjacent.
2138 assert!(
2139 splice(r#"<span id="p1" style="color:red" class="ltx_picture"></span>"#)
2140 .contains("<svg>OK</svg>")
2141 );
2142 // The placeholder's own attributes are preserved (only the content is filled).
2143 assert!(
2144 splice(r#"<span class="ltx_picture" id="p1"></span>"#).contains(r#"class="ltx_picture""#)
2145 );
2146 }
2147
2148 #[test]
2149 fn splice_leaves_non_targets_alone() {
2150 // Not a picture span → untouched.
2151 assert_eq!(
2152 splice(r#"<span id="p1" class="ltx_note"></span>"#),
2153 r#"<span id="p1" class="ltx_note"></span>"#
2154 );
2155 // A picture span with an id we have NO fragment for → untouched.
2156 assert_eq!(
2157 splice(r#"<span id="other" class="ltx_picture"></span>"#),
2158 r#"<span id="other" class="ltx_picture"></span>"#
2159 );
2160 // `ltx_picture` must be a whole class token, not a substring.
2161 assert_eq!(
2162 splice(r#"<span id="p1" class="ltx_picturewide"></span>"#),
2163 r#"<span id="p1" class="ltx_picturewide"></span>"#
2164 );
2165 }
2166}
2167
2168#[cfg(test)]
2169mod picture_graphics_resolution_tests {
2170 //! arXiv/html_feedback#1291 (witness arXiv:2311.14363v2): a `<graphics>` inside
2171 //! a `{picture}` (Inkscape `.pdf_tex` figures) must carry the Graphics-resolved
2172 //! `<img>`/`<object>` into its `<foreignObject>`, not the raw `<graphics>` the
2173 //! pass-A snapshot froze before Graphics ran. `finalize_html5` re-injects the
2174 //! resolution harvested from the post-Graphics page DOM.
2175 use rustc_hash::FxHashMap;
2176
2177 use super::{ResolvedGraphic, finalize_html5};
2178
2179 fn resolved(pairs: Vec<(&str, ResolvedGraphic)>) -> FxHashMap<String, ResolvedGraphic> {
2180 pairs.into_iter().map(|(k, v)| (k.to_string(), v)).collect()
2181 }
2182
2183 /// A picture placeholder span + its SVG fragment carrying two raw `<graphics>`
2184 /// (an Inkscape overlay: `page=1` background + `page=2` labels).
2185 fn placeholder_and_fragment() -> (String, (String, String)) {
2186 let span = r#"<span id="p1" class="ltx_picture"></span>"#.to_string();
2187 let fragment = (
2188 "p1".to_string(),
2189 concat!(
2190 r#"<svg><g><foreignObject width="100" height="120">"#,
2191 r#"<graphics graphic="bg.pdf" options="page=1" xml:id="p1.g1"/></foreignObject>"#,
2192 r#"<foreignObject width="100" height="120">"#,
2193 r#"<graphics graphic="bg.pdf" options="page=2" xml:id="p1.g2"/></foreignObject>"#,
2194 r#"</g></svg>"#,
2195 )
2196 .to_string(),
2197 );
2198 (span, fragment)
2199 }
2200
2201 #[test]
2202 fn raster_graphics_becomes_img_inside_foreignobject() {
2203 let (span, fragment) = placeholder_and_fragment();
2204 let map = resolved(vec![
2205 ("p1.g1", ResolvedGraphic {
2206 imagesrc: "bg.png".to_string(),
2207 imagewidth: Some("100".to_string()),
2208 imageheight: Some("120".to_string()),
2209 aspect_class: Some("ltx_img_portrait".to_string()),
2210 alt: "Refer to caption".to_string(),
2211 is_svg: false,
2212 }),
2213 ("p1.g2", ResolvedGraphic {
2214 imagesrc: "bg-2.png".to_string(),
2215 imagewidth: None,
2216 imageheight: None,
2217 aspect_class: None,
2218 alt: "[Uncaptioned image]".to_string(),
2219 is_svg: false,
2220 }),
2221 ]);
2222 let out = finalize_html5(span, &[fragment], &map);
2223 // Both nested graphics are resolved to <img>, each by its own xml:id.
2224 assert!(
2225 out.contains(r#"<img src="bg.png" id="p1.g1" class="ltx_graphics ltx_img_portrait" style="width:100%;height:100%;object-fit:contain" width="100" height="120" alt="Refer to caption">"#),
2226 "page=1 graphic not resolved to <img> (with foreignObject-box constraint):\n{out}"
2227 );
2228 assert!(
2229 out.contains(
2230 r#"<img src="bg-2.png" id="p1.g2" class="ltx_graphics" style="width:100%;height:100%;object-fit:contain" alt="[Uncaptioned image]">"#
2231 ),
2232 "page=2 graphic not resolved to <img>:\n{out}"
2233 );
2234 // The raw <graphics> element must be gone (the #1291 defect).
2235 assert!(
2236 !out.contains("<graphics"),
2237 "raw <graphics> survived into HTML (missing image):\n{out}"
2238 );
2239 }
2240
2241 #[test]
2242 fn svg_imagesrc_becomes_object() {
2243 let (span, fragment) = placeholder_and_fragment();
2244 let map = resolved(vec![("p1.g1", ResolvedGraphic {
2245 imagesrc: "x1.svg".to_string(),
2246 imagewidth: Some("100".to_string()),
2247 imageheight: Some("120".to_string()),
2248 aspect_class: Some("ltx_img_square".to_string()),
2249 alt: "Refer to caption".to_string(),
2250 is_svg: true,
2251 })]);
2252 let out = finalize_html5(span, &[fragment], &map);
2253 assert!(
2254 out.contains(r#"<object type="image/svg+xml" data="x1.svg" id="p1.g1" class="ltx_graphics ltx_img_square" style="width:100%;height:100%;object-fit:contain" width="100" height="120" aria-label="Refer to caption"></object>"#),
2255 "svg graphic not resolved to <object>:\n{out}"
2256 );
2257 // p1.g2 has no map entry → its raw <graphics> is left as-is (genuinely
2258 // unconverted), never silently promoted to a broken <img>.
2259 assert!(
2260 out.contains(r#"<graphics graphic="bg.pdf" options="page=2" xml:id="p1.g2"/>"#),
2261 "unresolved graphic should be left raw:\n{out}"
2262 );
2263 }
2264
2265 #[test]
2266 fn empty_map_leaves_raw_graphics_untouched() {
2267 // Non-HTML / graphicimages-off path: no resolution, no rewrite.
2268 let (span, fragment) = placeholder_and_fragment();
2269 let out = finalize_html5(span, &[fragment], &FxHashMap::default());
2270 assert!(out.contains(r#"<graphics graphic="bg.pdf" options="page=1" xml:id="p1.g1"/>"#));
2271 }
2272
2273 #[test]
2274 fn degenerate_picture_svg_leaves_image_unconstrained() {
2275 // A picture whose outer <svg> is sub-pixel (the `\unitlength`-not-applied
2276 // core gap, arXiv:2311.14363v2 Inkscape `.pdf_tex`): its foreignObject boxes
2277 // collapse in the browser, so a `100%`/object-fit constraint would zero the
2278 // image. The resolved <img> must therefore carry NO size constraint here
2279 // (leaving the pre-existing overflow behavior), unlike a well-sized picture.
2280 let span = r#"<span id="p1" class="ltx_picture"></span>"#.to_string();
2281 let fragment = (
2282 "p1".to_string(),
2283 concat!(
2284 r#"<svg xmlns="http://www.w3.org/2000/svg" width="1.33" height="0.95" overflow="visible">"#,
2285 r#"<g><foreignObject width="458.28" height="328.24" overflow="visible">"#,
2286 r#"<graphics graphic="x5.pdf" xml:id="p1.g1"/></foreignObject></g></svg>"#,
2287 )
2288 .to_string(),
2289 );
2290 let map = resolved(vec![("p1.g1", ResolvedGraphic {
2291 imagesrc: "x5.png".to_string(),
2292 imagewidth: None,
2293 imageheight: None,
2294 aspect_class: None,
2295 alt: "Refer to caption".to_string(),
2296 is_svg: false,
2297 })]);
2298 let out = finalize_html5(span, &[fragment], &map);
2299 assert!(
2300 out.contains(r#"<img src="x5.png" id="p1.g1" class="ltx_graphics" alt="Refer to caption">"#),
2301 "degenerate-SVG image must be resolved but UNCONSTRAINED (no object-fit):\n{out}"
2302 );
2303 assert!(
2304 !out.contains("object-fit"),
2305 "a sub-pixel picture SVG must not get an object-fit constraint (would hide the image):\n{out}"
2306 );
2307 }
2308}
2309
2310#[cfg(test)]
2311mod picture_svg_foreign_content_tests {
2312 //! html_feedback#74 (arXiv:0810.1673v3 Fig 2): a `<Math>` or `<graphics>`
2313 //! inside a `{picture}` `\makebox` must survive into the SVG, wrapped in a
2314 //! `<foreignObject>` (as Perl's `SVG.pm` does), not be dropped.
2315 //! `convert_picture_children_to_svg` handled the drawing primitives + `<text>`
2316 //! but had NO case for foreign content, so xfig/pstricks figures silently lost
2317 //! their math labels and `\epsfig` images. Plain text is the baseline; math and
2318 //! graphics are the canaries.
2319 use super::convert_picture_children_to_svg;
2320
2321 #[test]
2322 fn plain_text_label_still_renders() {
2323 // The shape emitted for `\put(50,50){\makebox(0,0){Hello}}` — already worked.
2324 let input = r#"<g transform="translate(50,50)"><text>Hello</text></g>"#;
2325 let out = convert_picture_children_to_svg(input);
2326 assert!(
2327 out.contains("Hello"),
2328 "plain-text picture label regressed: {out}"
2329 );
2330 }
2331
2332 #[test]
2333 fn math_label_survives_as_foreignobject() {
2334 // The shape emitted for `\put(20,20){\makebox(0,0){$X^{2}$}}`.
2335 let input = r#"<g transform="translate(20,20)" innerwidth="13.55pt" innerheight="8.14pt"><Math mode="inline" tex="X^{2}"><XMath><XMTok>X</XMTok></XMath></Math></g>"#;
2336 let out = convert_picture_children_to_svg(input);
2337 assert!(
2338 out.contains("foreignObject") && out.contains("Math"),
2339 "math inside a picture makebox was dropped, not wrapped in <foreignObject>: {out}"
2340 );
2341 }
2342
2343 #[test]
2344 fn embedded_graphic_survives_as_foreignobject() {
2345 // The shape emitted for `\epsfig{file=link.ps}` inside a picture.
2346 let input = r#"<g transform="translate(0,0)" innerwidth="277pt" innerheight="277pt"><graphics candidates="link.ps" graphic="link.ps"/></g>"#;
2347 let out = convert_picture_children_to_svg(input);
2348 assert!(
2349 out.contains("foreignObject") && out.contains("graphic"),
2350 "an embedded picture graphic was dropped, not wrapped in <foreignObject>: {out}"
2351 );
2352 }
2353
2354 #[test]
2355 fn graphic_inside_a_makebox_text_is_not_double_emitted() {
2356 // overpic's `\put(0,0){\makebox(0,0)[bl]{\includegraphics{img}}}` serializes
2357 // the graphic INSIDE a `<text>`. It must render ONCE — as a `<foreignObject>`
2358 // — never a second time inside an SVG `<text>` (the giant double-image,
2359 // arXiv:2510.17772 Fig 7). The plain "A" label beside it must survive.
2360 let input = r#"<g transform="translate(0,0)" innerwidth="277pt" innerheight="277pt"><text>A<graphics graphic="img.png" xml:id="p1.g1"/></text></g>"#;
2361 let out = convert_picture_children_to_svg(input);
2362 assert_eq!(
2363 out.matches("<graphics").count(),
2364 1,
2365 "picture graphic emitted more than once (double-image): {out}"
2366 );
2367 assert!(
2368 out.contains("foreignObject"),
2369 "the single copy must be the faithful <foreignObject>, not inside <text>: {out}"
2370 );
2371 // The <graphics> must be gone from any <text> node: no `<graphic` between a
2372 // <text> and its </text>.
2373 let text_seg = out.split("<text").nth(1).unwrap_or("");
2374 let text_inner = text_seg.split("</text>").next().unwrap_or("");
2375 assert!(
2376 !text_inner.contains("<graphic"),
2377 "graphic left inside <text> (will double-render): {out}"
2378 );
2379 assert!(
2380 out.contains(">A<") || text_inner.contains('A'),
2381 "plain label dropped: {out}"
2382 );
2383 }
2384
2385 #[test]
2386 fn background_makebox_graphic_only_emits_no_empty_text() {
2387 // The overpic BACKGROUND — `\put(0,0){\makebox(0,0)[bl]{\usebox\OVP@box}}` —
2388 // wraps ONLY the graphic, so the `<text>` has no sibling label. After the
2389 // graphic is stripped from the text copy, the leftover is empty: emit the
2390 // graphic once (as `<foreignObject>`) and NO dead `<text></text>` wrapper.
2391 let input = r#"<g transform="translate(0,0)" innerwidth="277pt" innerheight="277pt"><text><graphics graphic="bg.png" xml:id="p1.g0"/></text></g>"#;
2392 let out = convert_picture_children_to_svg(input);
2393 assert_eq!(
2394 out.matches("<graphics").count(),
2395 1,
2396 "background graphic emitted more than once: {out}"
2397 );
2398 assert!(
2399 out.contains("foreignObject"),
2400 "the graphic must survive as a <foreignObject>: {out}"
2401 );
2402 assert!(
2403 !out.contains("<text>") && !out.contains("<text "),
2404 "an empty <text> wrapper was emitted for a label-less makebox: {out}"
2405 );
2406 }
2407}