latexml_post/
pack.rs

1//! Output bundling — port of `LaTeXML::Post::Pack`.
2//!
3//! Separates final-bundle concerns from the post-processing pipeline.
4//! Both `latexml_oxide` and `cortex_worker` previously inlined their own
5//! `pack_output_zip` / `pack_output_zip_with_resources` + `add_dir_to_zip`
6//! copies; this module is the single source of truth so the two binaries
7//! produce byte-identical bundle layouts.
8//!
9//! Bundle layout (zip):
10//! ```text
11//! <html_filename>            — post-processed HTML
12//! <resource_dir>/...         — every non-`.html` file under the staging
13//!                               dir (Graphics-converted PNG/SVG, CSS, …),
14//!                               preserving subdirectories.
15//! <log_filename>             — log text, if `log_filename` is set and
16//!                               `log` is non-empty.
17//! status                     — single-line status string.
18//! telemetry.json             — single-line JSON per-job telemetry, only
19//!                               written when `telemetry_json` is set
20//!                               (cortex_worker canvas runs).
21//! ```
22
23use std::{
24  fs::File,
25  io::{self, BufWriter, Write},
26  path::Path,
27};
28
29use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions};
30
31/// Write-buffer size for the zip output. 64 KiB matches the typical
32/// compressed-block size from miniz_oxide/flate2 on HTML+image
33/// content; smaller buffers (8 KiB) cause one syscall per block,
34/// larger (1 MiB) waste RSS without improving throughput. Measured on
35/// 1910.01256 (3 PNG + 2 SVG + 2 CSS + HTML = 1.2 MB zip): unbuffered
36/// → ~70 write() syscalls; 64 KiB buffer → ~12.
37const ZIP_WRITE_BUF: usize = 64 * 1024;
38
39/// Options for [`pack_archive`].
40pub struct PackOptions<'a> {
41  /// Destination zip path.
42  pub zip_path:          &'a str,
43  /// Name to use for the HTML entry inside the zip (e.g. `paper.html`).
44  /// Conventionally `<stem>.html` where `stem` is the source TeX name.
45  pub html_filename:     &'a str,
46  /// Post-processed HTML content.
47  pub html:              &'a str,
48  /// Name for the log entry; pass `None` to skip writing a log entry.
49  pub log_filename:      Option<&'a str>,
50  /// Log content. Skipped if empty even when `log_filename` is set.
51  pub log:               &'a str,
52  /// Single-line status string. Always written as `status`.
53  pub status:            &'a str,
54  /// Resource staging directory (typically a `TempDir`). Every
55  /// non-`.html` file under it is bundled, preserving subdirectories.
56  /// Pass `None` to skip resource bundling.
57  pub resource_dir:      Option<&'a Path>,
58  /// Optional per-job telemetry JSON line. When `Some`, written as
59  /// `telemetry.json` inside the zip. Used by `cortex_worker` canvas
60  /// runs; `benchmark_canvas.sh` extracts this member and appends to
61  /// `<output_dir>/telemetry.jsonl`. See `docs/performance/TELEMETRY.md`.
62  pub telemetry_json:    Option<&'a str>,
63  /// `SOURCE_DATE_EPOCH` (Unix seconds, UTC). When `Some`, every zip
64  /// member's last-modified time is pinned to it for reproducible
65  /// archives — Perl `Pack/Zip.pm` L113-115
66  /// (`setLastModFileDateTimeFromUnix`). `None` lets the zip crate use
67  /// its default write timestamp.
68  pub source_date_epoch: Option<u64>,
69}
70
71/// Pack the post-processing outputs into a zip archive.
72///
73/// Returns an `io::Result` rather than `crate::processor::PostError`
74/// because callers (binary mains) are already `Box<dyn Error>`-typed.
75///
76/// **IO performance:** the underlying file is wrapped in a 64 KiB
77/// `BufWriter` before handing it to `ZipWriter`. The zip crate's
78/// internal deflate output is small chunks (per-block from miniz);
79/// without buffering each chunk would be its own `write()` syscall.
80/// Measured ~6× fewer syscalls on 1910.01256 (7 resource files + HTML).
81pub fn pack_archive(opts: &PackOptions) -> io::Result<()> {
82  let file = File::create(opts.zip_path)?;
83  let buf_file = BufWriter::with_capacity(ZIP_WRITE_BUF, file);
84  let mut zip = ZipWriter::new(buf_file);
85  let mut zip_options =
86    SimpleFileOptions::default().compression_method(CompressionMethod::Deflated);
87  // Reproducible archives: pin every member's mod-time to SOURCE_DATE_EPOCH
88  // when provided (Perl Pack/Zip.pm L113-115). DOS/zip timestamps only
89  // span 1980-2107, so out-of-range epochs are silently left at the
90  // crate default — matching the spirit of `setLastModFileDateTimeFromUnix`
91  // (which would clamp) without failing the whole archive.
92  if let Some(epoch) = opts.source_date_epoch {
93    if let Some(dt) = epoch_to_zip_datetime(epoch) {
94      zip_options = zip_options.last_modified_time(dt);
95    }
96  }
97
98  // HTML first, so `unzip -l` shows the main artifact at the top.
99  zip
100    .start_file(opts.html_filename, zip_options)
101    .map_err(io_err)?;
102  zip.write_all(opts.html.as_bytes())?;
103
104  // Resource files (Graphics-converted PNG/SVG, injected CSS, etc.).
105  if let Some(dir) = opts.resource_dir {
106    if dir.exists() {
107      add_dir_to_zip(&mut zip, dir, dir, &zip_options)?;
108    }
109  }
110
111  // Log entry.
112  if let Some(log_name) = opts.log_filename {
113    if !opts.log.is_empty() {
114      zip.start_file(log_name, zip_options).map_err(io_err)?;
115      zip.write_all(opts.log.as_bytes())?;
116    }
117  }
118
119  // Status.
120  zip.start_file("status", zip_options).map_err(io_err)?;
121  zip.write_all(opts.status.as_bytes())?;
122
123  // Telemetry (cortex_worker only).
124  if let Some(tjson) = opts.telemetry_json {
125    zip
126      .start_file("telemetry.json", zip_options)
127      .map_err(io_err)?;
128    zip.write_all(tjson.as_bytes())?;
129  }
130
131  zip.finish().map_err(io_err)?;
132  Ok(())
133}
134
135/// Recursively add files from `dir` to a ZIP archive, preserving the
136/// directory structure relative to `base`.
137///
138/// Two skip rules apply:
139///  * `.html` files — the post-processed HTML is added separately by [`pack_archive`] (and the
140///    staging dir may hold a stray copy written there for the Graphics processor's relative paths).
141///  * [`is_excluded_archive_entry`] — Perl `Pack/Zip.pm`'s `ARCHIVE_EXT_EXCLUDE` (source
142///    `.tex`/`.bib`, nested archives, dotfiles, editor backups). Applied per-basename, matching
143///    Perl's `addTree` filter `sub { !/$ext_exclude/ }`.
144///
145/// Each source file is wrapped in a 64 KiB `BufReader` to amortise
146/// `read()` syscalls on the input side (the `io::copy` 8 KiB default
147/// chunk would otherwise issue ~ceil(filesize/8K) reads per resource).
148fn add_dir_to_zip<W: Write + io::Seek>(
149  zip: &mut ZipWriter<W>,
150  dir: &Path,
151  base: &Path,
152  options: &SimpleFileOptions,
153) -> io::Result<()> {
154  for entry in std::fs::read_dir(dir)? {
155    let entry = entry?;
156    let path = entry.path();
157    let rel = path.strip_prefix(base).unwrap_or(&path);
158    // Zip entry names use '/' by spec (APPNOTE 4.4.17.1); Path on Windows
159    // yields '\'. Replace only the platform separator: on Unix that's a
160    // no-op ('/'→'/') that leaves a literal '\' — a legal Unix filename
161    // byte — untouched; on Windows it rewrites '\'→'/'.
162    let name = rel
163      .to_string_lossy()
164      .replace(std::path::MAIN_SEPARATOR, "/");
165    let basename = entry.file_name().to_string_lossy().to_string();
166
167    if path.is_dir() {
168      // Perl's `addTree` filter excludes whole subtrees whose *directory*
169      // name matches (e.g. a nested `.git`); honour the same per-basename
170      // rule before recursing.
171      if !is_excluded_archive_entry(&basename) {
172        add_dir_to_zip(zip, &path, base, options)?;
173      }
174    } else if !name.ends_with(".html") && !is_excluded_archive_entry(&basename) {
175      zip.start_file(&name, *options).map_err(io_err)?;
176      let f = File::open(&path)?;
177      let mut buf_reader = io::BufReader::with_capacity(ZIP_WRITE_BUF, f);
178      io::copy(&mut buf_reader, zip)?;
179    }
180  }
181  Ok(())
182}
183
184/// Whether a bundle entry should be excluded from the archive — port of
185/// Perl `Pack/Zip.pm` `$ARCHIVE_EXT_EXCLUDE`
186/// (`qr/(?:^\.)|(?:\.(?:zip|gz|epub|tex|bib|mobi|cache)$)|(?:~$)/`),
187/// applied to the file's basename:
188///  * hidden dotfiles (`^\.`),
189///  * editor backups (`~$`),
190///  * nested archives / source / cache files (`.zip`, `.gz`, `.epub`, `.tex`, `.bib`, `.mobi`,
191///    `.cache`).
192fn is_excluded_archive_entry(basename: &str) -> bool {
193  if basename.starts_with('.') || basename.ends_with('~') {
194    return true;
195  }
196  // Suffix test on the lowercase extension (Perl anchors `$`, i.e. the
197  // final extension). `rsplit('.')` yields the extension before any dot.
198  match basename.rsplit_once('.') {
199    Some((_, ext)) => matches!(
200      ext.to_ascii_lowercase().as_str(),
201      "zip" | "gz" | "epub" | "tex" | "bib" | "mobi" | "cache"
202    ),
203    None => false,
204  }
205}
206
207/// Convert a Unix epoch (seconds, UTC) into a zip [`zip::DateTime`].
208///
209/// DOS/zip timestamps only represent 1980-01-01..=2107; epochs outside
210/// that window return `None` (caller falls back to the crate default).
211/// Pure civil-date arithmetic (Howard Hinnant's `civil_from_days`) so we
212/// don't pull in a date-time crate just for `SOURCE_DATE_EPOCH`.
213fn epoch_to_zip_datetime(epoch: u64) -> Option<zip::DateTime> {
214  let days = (epoch / 86_400) as i64;
215  let secs_of_day = (epoch % 86_400) as u32;
216  let (hour, minute, second) = (
217    (secs_of_day / 3600) as u8,
218    ((secs_of_day % 3600) / 60) as u8,
219    (secs_of_day % 60) as u8,
220  );
221  // civil_from_days: days since 1970-01-01 → (year, month, day).
222  let z = days + 719_468;
223  let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
224  let doe = z - era * 146_097; // [0, 146096]
225  let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146_096) / 365; // [0, 399]
226  let year = yoe + era * 400;
227  let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); // [0, 365]
228  let mp = (5 * doy + 2) / 153; // [0, 11]
229  let day = (doy - (153 * mp + 2) / 5 + 1) as u8; // [1, 31]
230  let month = (if mp < 10 { mp + 3 } else { mp - 9 }) as u8; // [1, 12]
231  let year = year + i64::from(month <= 2);
232  if !(1980..=2107).contains(&year) {
233    return None;
234  }
235  zip::DateTime::from_date_and_time(year as u16, month, day, hour, minute, second).ok()
236}
237
238/// Convert a `zip::result::ZipError` into an `io::Error` so the caller
239/// signature can stay `io::Result`. The zip crate wraps `io::Error`
240/// already; we just re-wrap unrecognized kinds as `Other`.
241fn io_err(e: zip::result::ZipError) -> io::Error {
242  match e {
243    zip::result::ZipError::Io(inner) => inner,
244    other => io::Error::other(other.to_string()),
245  }
246}
247
248#[cfg(test)]
249mod tests {
250  use std::io::Read;
251
252  use super::*;
253
254  /// Read back the set of entry names from a zip on disk.
255  fn zip_entry_names(zip_path: &Path) -> Vec<String> {
256    let f = File::open(zip_path).expect("open zip");
257    let mut archive = zip::ZipArchive::new(f).expect("parse zip");
258    (0..archive.len())
259      .map(|i| archive.by_index(i).expect("entry").name().to_string())
260      .collect()
261  }
262
263  #[test]
264  fn excludes_perl_archive_ext_set() {
265    // Perl Zip.pm ARCHIVE_EXT_EXCLUDE = qr/(?:^\.)|(?:\.(?:zip|gz|epub|
266    // tex|bib|mobi|cache)$)|(?:~$)/ — applied to the basename.
267    assert!(is_excluded_archive_entry("paper.tex"));
268    assert!(is_excluded_archive_entry("refs.bib"));
269    assert!(is_excluded_archive_entry("bundle.zip"));
270    assert!(is_excluded_archive_entry("page.gz"));
271    assert!(is_excluded_archive_entry("book.epub"));
272    assert!(is_excluded_archive_entry("book.mobi"));
273    assert!(is_excluded_archive_entry("LaTeXML.cache"));
274    assert!(is_excluded_archive_entry(".hidden"));
275    assert!(is_excluded_archive_entry("backup~"));
276    // Kept: real bundle resources.
277    assert!(!is_excluded_archive_entry("fig1.png"));
278    assert!(!is_excluded_archive_entry("diagram.svg"));
279    assert!(!is_excluded_archive_entry("LaTeXML.css"));
280    assert!(!is_excluded_archive_entry("logo.jpg"));
281  }
282
283  #[test]
284  fn pack_archive_bundles_resources_minus_excluded() {
285    let staging = tempfile::tempdir().expect("tempdir");
286    let p = staging.path();
287    // Resources that SHOULD be bundled.
288    std::fs::write(p.join("fig1.png"), b"PNGDATA").unwrap();
289    std::fs::write(p.join("LaTeXML.css"), b"body{}").unwrap();
290    std::fs::create_dir(p.join("sub")).unwrap();
291    std::fs::write(p.join("sub").join("img.svg"), b"<svg/>").unwrap();
292    // Resources that must be EXCLUDED.
293    std::fs::write(p.join("paper.tex"), b"\\documentclass{article}").unwrap();
294    std::fs::write(p.join("refs.bib"), b"@book{x}").unwrap();
295    std::fs::write(p.join("LaTeXML.cache"), b"cache").unwrap();
296    std::fs::write(p.join(".hidden"), b"secret").unwrap();
297    std::fs::write(p.join("backup~"), b"old").unwrap();
298    // The HTML is added separately by pack_archive; a stray copy in
299    // the staging dir must not be double-added.
300    std::fs::write(p.join("doc.html"), b"<html>staging copy</html>").unwrap();
301
302    let out = tempfile::tempdir().expect("out dir");
303    let zip_path = out.path().join("bundle.zip");
304    let zip_path_str = zip_path.to_string_lossy().to_string();
305
306    pack_archive(&PackOptions {
307      zip_path:          &zip_path_str,
308      html_filename:     "doc.html",
309      html:              "<html>real document</html>",
310      log_filename:      Some("doc.log"),
311      log:               "log line",
312      status:            "Status:conversion:0",
313      resource_dir:      Some(p),
314      telemetry_json:    None,
315      source_date_epoch: None,
316    })
317    .expect("pack archive");
318
319    let names = zip_entry_names(&zip_path);
320    // Bundled resources present.
321    assert!(names.iter().any(|n| n == "fig1.png"), "names: {names:?}");
322    assert!(names.iter().any(|n| n == "LaTeXML.css"), "names: {names:?}");
323    assert!(
324      names.iter().any(|n| n == "sub/img.svg"),
325      "subdir resource missing; names: {names:?}"
326    );
327    // Core entries present.
328    assert!(names.iter().any(|n| n == "doc.html"));
329    assert!(names.iter().any(|n| n == "doc.log"));
330    assert!(names.iter().any(|n| n == "status"));
331    // Excluded resources absent.
332    for forbidden in [
333      "paper.tex",
334      "refs.bib",
335      "LaTeXML.cache",
336      ".hidden",
337      "backup~",
338    ] {
339      assert!(
340        !names.iter().any(|n| n == forbidden),
341        "{forbidden} must be excluded; names: {names:?}"
342      );
343    }
344    // Exactly one doc.html (the real one), not the staging copy too.
345    assert_eq!(
346      names.iter().filter(|n| n.as_str() == "doc.html").count(),
347      1,
348      "doc.html must not be double-added; names: {names:?}"
349    );
350    // And the real HTML, not the staging copy, is what got stored.
351    let f = File::open(&zip_path).unwrap();
352    let mut archive = zip::ZipArchive::new(f).unwrap();
353    let mut html_entry = archive.by_name("doc.html").unwrap();
354    let mut body = String::new();
355    html_entry.read_to_string(&mut body).unwrap();
356    assert_eq!(body, "<html>real document</html>");
357  }
358
359  #[test]
360  fn source_date_epoch_sets_member_timestamp() {
361    // Perl Zip.pm L113-115: when SOURCE_DATE_EPOCH is set, every member
362    // gets that fixed mod-time for reproducible archives. 2021-01-01
363    // 00:00:00 UTC = 1609459200.
364    let staging = tempfile::tempdir().expect("tempdir");
365    std::fs::write(staging.path().join("fig.png"), b"x").unwrap();
366    let out = tempfile::tempdir().expect("out");
367    let zip_path = out.path().join("ts.zip");
368    let zip_path_str = zip_path.to_string_lossy().to_string();
369    pack_archive(&PackOptions {
370      zip_path:          &zip_path_str,
371      html_filename:     "d.html",
372      html:              "<html/>",
373      log_filename:      None,
374      log:               "",
375      status:            "ok",
376      resource_dir:      Some(staging.path()),
377      telemetry_json:    None,
378      source_date_epoch: Some(1_609_459_200),
379    })
380    .expect("pack");
381
382    let f = File::open(&zip_path).unwrap();
383    let mut archive = zip::ZipArchive::new(f).unwrap();
384    let entry = archive.by_name("fig.png").unwrap();
385    let dt = entry.last_modified().expect("has mod time");
386    assert_eq!(dt.year(), 2021, "year");
387    assert_eq!(dt.month(), 1, "month");
388    assert_eq!(dt.day(), 1, "day");
389  }
390}
latexml_post/pack.rs

latexml_post/
pack.rs
