Skip to main content

latexml_core/
dump_writer.rs

1//! Writer for Rust-native kernel dump files.
2//!
3//! Serializes the State diff (from snapshot/diff_from_snapshot) into a
4//! compact text format that can be loaded back efficiently.
5//!
6//! This is the "DumpFile" equivalent from Perl's TeX_Job.pool.ltxml.
7//! The workflow follows Perl's Makefile.PL "make formats" scaffold:
8//!   1. take_snapshot() — before processing latex.ltx
9//!   2. Process latex.ltx through the engine
10//!   3. diff_snapshot() — find what changed
11//!   4. write_dump() — serialize the diff
12//!   5. At runtime: load_dump() restores the state
13
14use std::{io::Write, path::Path};
15
16use rustc_hash::FxHashMap as HashMap;
17
18use crate::definition::Definition; // trait for get_expansion(), get_cs(), etc.
19use crate::{
20  common::{arena, numeric_ops::NumericOps, store::Stored},
21  state::TableName,
22};
23
24/// Write a state diff to a dump file.
25/// Returns the number of entries successfully written.
26///
27/// Writes in three ordered sections, matching Perl's
28/// `TeX_Job.pool.ltxml::DumpFile` which separates
29/// `@cmds_early` / `@cmds` / `@cmds_late`:
30///
31///   1. **cmds_early** — M:PA / M:MPA let-aliases whose **target pre-existed** in the bootstrap
32///      snapshot (e.g. `\tex_let:D → \let`, `\tex_def:D → \def`). Applied first because their
33///      targets are always available (they're in the bootstrap pool we loaded unconditionally
34///      before this dump).
35///   2. **cmds** (regular) — V / M:E / M:T / R / C / LC / UC / SC / MC / DC entries: data,
36///      expandable definitions, registers, codes.
37///   3. **cmds_late** — M:PA / M:MPA let-aliases whose **target is defined by this dump**. Applied
38///      last so the target is installed before the alias fires.
39///
40/// The early/late split requires the caller (ini_tex) to have staged a
41/// snapshot via `state::stage_snapshot("bootstrap")`. If no snapshot is
42/// available, all aliases are emitted as `late` (safe — they just wait
43/// until after regular entries).
44pub fn write_dump(
45  path: &Path,
46  entries: &[(TableName, arena::SymStr, Stored)],
47) -> Result<usize, String> {
48  let mut file =
49    std::fs::File::create(path).map_err(|e| format!("Failed to create dump file: {}", e))?;
50
51  writeln!(file, "# latexml-oxide kernel dump v3").ok();
52  writeln!(
53    file,
54    "# Generated by dump_writer, {} entries",
55    entries.len()
56  )
57  .ok();
58  writeln!(file, "# Format: <table>\\t<key>\\t<data>").ok();
59  writeln!(file, "#   <table> = M | V | C | LC | UC | SC | MC | DC").ok();
60  writeln!(file, "#   <data> depends on <table>:").ok();
61  writeln!(
62    file,
63    "#     M/E:   E\\t<cs>\\t<nargs>\\t<flags>\\t<body>\\t<proto>\\t<v3_params>"
64  )
65  .ok();
66  writeln!(
67    file,
68    "#            v3_params encodes each Parameter structurally so Until:/Match:"
69  )
70  .ok();
71  writeln!(
72    file,
73    "#            delimiter tokens round-trip (see docs/archive/DUMP_FORMAT_PERL_ANALYSIS_2026-04-30.md)."
74  )
75  .ok();
76  writeln!(
77    file,
78    "#     M/PA:  PA\\t<target_cs>   (MPA for math primitives)"
79  )
80  .ok();
81  writeln!(file, "#     M/T:   T\\t<meaning_token>").ok();
82  writeln!(
83    file,
84    "#     M/R:   R\\t<cs>\\t<rtype>\\t<value>[\\t<mathglyph>]"
85  )
86  .ok();
87  writeln!(
88    file,
89    "#     M/N:   N                     (None-meaning sentinel)"
90  )
91  .ok();
92  writeln!(
93    file,
94    "#     V:     <type>\\t<value>       (B,I,S,CH,T,TK,N,F,G,MG,D,MD,VD)"
95  )
96  .ok();
97  writeln!(file, "#     C:                  CC\\t<u8>   (catcode)").ok();
98  writeln!(
99    file,
100    "#     DC/LC/UC/SC/MC:     CH\\t<u32>  (char-indexed table entries)"
101  )
102  .ok();
103  writeln!(file, "#   <flags> = subset of {{L, P}} (long, protected)").ok();
104  writeln!(
105    file,
106    "#   All keys and proto are url-encoded (%09 for tab etc.); token body"
107  )
108  .ok();
109  writeln!(
110    file,
111    "#   tokens use the form <catcode_int>:<url_encoded_text> joined by commas."
112  )
113  .ok();
114
115  // Read the bootstrap snapshot so we can classify each alias's target
116  // as pre-existing (→ early) or defined-during-load (→ late).
117  let bootstrap_snap = crate::state::get_staged_snapshot("bootstrap");
118
119  // Pre-serialize + partition. Skipped keys (ver@..., unserializable
120  // values) fall out naturally.
121  let mut early_aliases: Vec<(String, String, String)> = Vec::new();
122  let mut regular: Vec<(String, String, String)> = Vec::new();
123  let mut late_aliases: Vec<(String, String, String)> = Vec::new();
124  let mut skipped = 0usize;
125
126  // expl3 implements intarrays by stashing values in `\fontdimen<idx>\<font>`
127  // slots, picking `cmr10` at various tiny `at <N>sp` instantiations to
128  // get one font instance per intarray. Each slot is normally written as
129  // an individual `V\tfontdimen_fontinfo_<font> at <Nsp>_<idx>\tD\t<val>`
130  // record; that produces ~89k V-records (≈40% of the dump) for an
131  // initialized expl3 + LaTeX kernel. We group them by (font,size) and
132  // emit a single `IA` record per intarray with the values RLE-encoded
133  // — same in-memory state after replay, ~10× smaller on disk.
134  // See `docs/archive/PERL_LOADFORMAT_AUDIT.md` ("Fontdimen/intarray storage").
135  let mut fontdimen_groups: HashMap<String, Vec<(u32, i64)>> = HashMap::default();
136
137  for (table, key, value) in entries {
138    let table_code = table_to_code(*table);
139    let key_str = arena::with(*key, |s| s.to_string());
140
141    // Skip \ver@ package version markers (runtime metadata, not definitions)
142    if key_str.starts_with("\\ver@") {
143      skipped += 1;
144      continue;
145    }
146
147    // Skip MAX_ERRORS — `ini_tex.rs:189` raises it to 1_000_000 during
148    // dump-build (so raw latex.ltx + expl3-code.tex can run through
149    // transient errors). It must NOT leak into runtime conversions:
150    // a 1M cap lets runaway error cascades (e.g. AmS-TeX `\cases` mis-
151    // parse → 1M `\hbox`/`&` per paper, math0205073) bypass the
152    // 10000 default. dump_reader has a parallel filter for the
153    // already-shipped dumps; this filter keeps future regenerations
154    // clean.
155    if matches!(*table, TableName::Value) && key_str == "MAX_ERRORS" {
156      skipped += 1;
157      continue;
158    }
159
160    // Skip \@currname / \@currext file-IO bookkeeping. These are
161    // assigned per-document during `\input` by `read_input_file_recursive`
162    // (see `binding/content.rs:262-263`) to the literal filename's
163    // tokens, so the snapshot captures the LAST opened file's name
164    // ("plain.tex" / "latex.ltx"). Perl's `TeX_FileIO.pool.ltxml:28-29`
165    // initializes them via `Let('\@currname','\lx@empty')` before any
166    // file load, and Perl's plain_dump.pool.ltxml omits them — so
167    // post-dump they remain at the `\lx@empty` baseline. Matching that
168    // behavior here keeps us file-IO-state-agnostic in the dump.
169    if matches!(*table, TableName::Meaning) && (key_str == "\\@currname" || key_str == "\\@currext")
170    {
171      skipped += 1;
172      continue;
173    }
174
175    // Skip token-list register VALUES whose body contains a self-`\the`
176    // reference. latex.ltx L10564-10580 sets `\frozen@everymath` =
177    // `{... \the\everymath}` then allocates a NEW `\everymath` via
178    // `\newtoks`. Our `\newtoks` doesn't fully detach the new register's
179    // slot from the frozen alias's, so the body lands on `\everymath`'s
180    // value slot — `\the\everymath` then expands recursively at math
181    // entry, blowing the token limit. Until the `\newtoks`/`\let`
182    // slot-aliasing is fixed at the dialect level, the safest cure is
183    // to drop the corrupted self-referential capture; runtime
184    // re-evaluation of latex.ltx's hook chain (or LaTeXML's own
185    // `\everymath`/`\everydisplay` setup) repopulates correctly.
186    if matches!(*table, TableName::Value)
187      && matches!(
188        key_str.as_str(),
189        "\\everymath"
190          | "\\everydisplay"
191          | "\\everyhbox"
192          | "\\everyvbox"
193          | "\\everycr"
194          | "\\everyjob"
195          | "\\everypar"
196          | "\\everyeof"
197      )
198    {
199      // Confirm the loop pattern before dropping (don't suppress
200      // valid hook contributions on engines with sound aliasing).
201      if let Stored::Tokens(tks) = value {
202        let body = tks.unlist_ref();
203        let needle_cs = key_str.clone();
204        let has_self_the = body.iter().any(|t| t.with_str(|s| s == "\\the"))
205          && body.iter().any(|t| t.with_str(|s| s == needle_cs));
206        if has_self_the {
207          skipped += 1;
208          continue;
209        }
210      }
211    }
212
213    // Perl #2771 (2026-03-13) + upstream IGNORED_SYMBOLS: these are
214    // runtime-only Value entries — control-flow counters, and large
215    // registered-rule tables that can't meaningfully round-trip through
216    // the dump (they hold closures, cross-refs, or re-populate during
217    // engine init anyway). Match Perl's TeX_Job.pool.ltxml list.
218    if matches!(*table, TableName::Value)
219      && matches!(
220        key_str.as_str(),
221        "if_count"
222          | "absorb_count"
223          | "if_stack"
224          | "DOCUMENT_REWRITE_RULES"
225          | "PARAMETER_TYPES"
226          | "TAG_PROPERTIES"
227          | "MATH_LIGATURES"
228          | "TEXT_LIGATURES"
229      )
230    {
231      skipped += 1;
232      continue;
233    }
234
235    // Perl IGNORED_SYMBOLS: meaning:\lnot, meaning:\to are skipped because
236    // pre-2017 TeXlive let-aliased \lnot → \neg and \to → \rightarrow,
237    // which would gratuitously diverge tests across TL versions.
238    // Mirror Perl `TeX_Job.pool.ltxml` IGNORED_SYMBOLS L104-107.
239    if matches!(*table, TableName::Meaning) && matches!(key_str.as_str(), "\\lnot" | "\\to") {
240      skipped += 1;
241      continue;
242    }
243
244    // Intarray slot consolidation — see fontdimen_groups comment above.
245    if matches!(*table, TableName::Value)
246      && let Some((prefix, idx)) = parse_fontdimen_key(&key_str)
247      && let Stored::Dimension(d) = value
248    {
249      fontdimen_groups
250        .entry(prefix.to_string())
251        .or_default()
252        .push((idx, d.0));
253      continue;
254    }
255
256    let Some(serialized) = serialize_stored(value) else {
257      skipped += 1;
258      continue;
259    };
260
261    // A let-alias entry is an M row whose serialized form starts with
262    // `PA\t` or `MPA\t`. Nothing else produces those prefixes.
263    let is_alias = matches!(*table, TableName::Meaning)
264      && (serialized.starts_with("PA\t") || serialized.starts_with("MPA\t"));
265
266    let row = (table_code.to_string(), url_encode(&key_str), serialized);
267    if is_alias {
268      // Extract the target CS from "PA\t<target>" / "MPA\t<target>"
269      // (we already filtered self-aliases in serialize_stored, so the
270      //  target is always a different CS than the key).
271      let target_raw = row.2.split('\t').nth(1).unwrap_or("");
272      let target = crate::dump_reader::url_decode(target_raw);
273      let target_sym = arena::pin(&target);
274      // Target pre-existed iff it was in the bootstrap meaning table.
275      let pre_existed = bootstrap_snap
276        .as_ref()
277        .is_some_and(|snap| snap.contains_key(&(TableName::Meaning, target_sym)));
278      if pre_existed {
279        early_aliases.push(row);
280      } else {
281        late_aliases.push(row);
282      }
283    } else {
284      regular.push(row);
285    }
286  }
287
288  // Emit one IA record per intarray group. Fall back to individual V
289  // records for any non-dense group (defensive: dump_reader only knows
290  // how to expand contiguous 1..N runs).
291  let mut ia_count = 0usize;
292  let mut ia_fallback_v = 0usize;
293  for (prefix, mut slots) in fontdimen_groups.into_iter() {
294    slots.sort_by_key(|s| s.0);
295    let dense = slots
296      .iter()
297      .enumerate()
298      .all(|(i, (idx, _))| *idx == (i as u32 + 1));
299    if !dense {
300      eprintln!(
301        "[dump_writer] non-dense intarray {:?} ({} slots) — emitting as individual V records",
302        prefix,
303        slots.len()
304      );
305      for (idx, val) in &slots {
306        let key = format!("{}_{}", prefix, idx);
307        regular.push(("V".to_string(), url_encode(&key), format!("D\t{}", val)));
308        ia_fallback_v += 1;
309      }
310      continue;
311    }
312    let values: Vec<i64> = slots.into_iter().map(|(_, v)| v).collect();
313    let rle = rle_encode_i64(&values);
314    let body = format!("{}\t{}", values.len(), rle);
315    regular.push(("IA".to_string(), url_encode(&prefix), body));
316    ia_count += 1;
317  }
318  if ia_count > 0 || ia_fallback_v > 0 {
319    eprintln!(
320      "[dump_writer] intarray consolidation: {} IA records, {} V fallbacks",
321      ia_count, ia_fallback_v
322    );
323  }
324
325  writeln!(
326    file,
327    "# Section 0: early let-aliases ({}) — target in bootstrap",
328    early_aliases.len()
329  )
330  .ok();
331  for (t, k, v) in &early_aliases {
332    writeln!(file, "{}\t{}\t{}", t, k, v).map_err(|e| format!("Write error: {}", e))?;
333  }
334  writeln!(file, "# Section 1: regular entries ({})", regular.len()).ok();
335  for (t, k, v) in &regular {
336    writeln!(file, "{}\t{}\t{}", t, k, v).map_err(|e| format!("Write error: {}", e))?;
337  }
338  writeln!(
339    file,
340    "# Section 2: late let-aliases ({}) — target in Section 1",
341    late_aliases.len()
342  )
343  .ok();
344  for (t, k, v) in &late_aliases {
345    writeln!(file, "{}\t{}\t{}", t, k, v).map_err(|e| format!("Write error: {}", e))?;
346  }
347
348  let count = early_aliases.len() + regular.len() + late_aliases.len();
349  writeln!(file, "# Written: {}, Skipped: {}", count, skipped).ok();
350  eprintln!(
351    "[dump_writer] Wrote {} entries to {} ({} skipped; {} early + {} regular + {} late)",
352    count,
353    path.display(),
354    skipped,
355    early_aliases.len(),
356    regular.len(),
357    late_aliases.len(),
358  );
359
360  Ok(count)
361}
362
363fn table_to_code(t: TableName) -> &'static str {
364  match t {
365    TableName::Meaning => "M",
366    TableName::Value => "V",
367    TableName::Catcode => "C",
368    TableName::Mathcode => "MC",
369    TableName::Sfcode => "SC",
370    TableName::Lccode => "LC",
371    TableName::Uccode => "UC",
372    TableName::Delcode => "DC",
373    TableName::Stash => "ST",
374    TableName::StashActive => "SA",
375  }
376}
377
378/// Serialize a Stored value to a type-tag + data string.
379/// Returns None if the value can't be serialized.
380fn serialize_stored(stored: &Stored) -> Option<String> {
381  match stored {
382    Stored::None => Some("N".to_string()),
383    Stored::Bool(b) => Some(format!("B\t{}", if *b { "1" } else { "0" })),
384    Stored::Int(i) => Some(format!("I\t{}", i)),
385    Stored::String(s) => Some(format!(
386      "S\t{}",
387      url_encode(&arena::with(*s, |s| s.to_string()))
388    )),
389    Stored::Charcode(c) => Some(format!("CH\t{}", c)),
390    Stored::Catcode(cc) => Some(format!("CC\t{}", u8::from(*cc))),
391    Stored::Token(t) => Some(format!("T\t{}", serialize_token(t))),
392    Stored::Tokens(tks) => {
393      let tok_strs: Vec<String> = tks.unlist_ref().iter().map(serialize_token).collect();
394      Some(format!("TK\t{}", tok_strs.join(",")))
395    },
396    Stored::Expandable(exp) => {
397      use crate::common::object::Object;
398      let cs_name = exp.get_cs().with_str(url_encode);
399      let nargs = exp.get_num_args();
400      let mut flags = String::new();
401      if exp.is_long {
402        flags.push('L');
403      }
404      if exp.is_protected() {
405        flags.push('P');
406      }
407      // Two parallel parameter encodings:
408      //
409      //   - `proto` (v2, 5th field) — `Parameters::stringify()`, a flat space-separated prototype
410      //     string. Round-trips simple specs but not `Until:\end{verbatim}` or other
411      //     brace-in-delimiter forms.
412      //   - `v3_params` (v3, 6th field) — per-Parameter structured record carrying (name, spec,
413      //     flags, extras). Modeled on Perl's `P(type, spec, extra=>[T(...)])`; see
414      //     `docs/archive/DUMP_FORMAT_PERL_ANALYSIS_2026-04-30.md`. Bypasses `parse_parameters` at load
415      //     time so delimited params round-trip intact.
416      //
417      // We emit both so older readers still work, and newer readers can
418      // prefer v3 transparently.
419      let proto = exp
420        .get_parameters()
421        .map(|p| p.stringify())
422        .unwrap_or_default();
423      let proto_encoded = url_encode(&proto);
424      let v3_params = exp
425        .get_parameters()
426        .map(serialize_parameters_v3)
427        .unwrap_or_default();
428      match exp.get_expansion() {
429        Some(crate::definition::ExpansionBody::Tokens(tks)) => {
430          let tok_strs: Vec<String> = tks.unlist_ref().iter().map(serialize_token).collect();
431          Some(format!(
432            "E\t{}\t{}\t{}\t{}\t{}\t{}",
433            cs_name,
434            nargs,
435            flags,
436            tok_strs.join(","),
437            proto_encoded,
438            v3_params
439          ))
440        },
441        // No body at all (e.g. `DefMacro!("\\@gobble{}", None)`): serializes
442        // as an E-entry with empty token body. The reader reconstructs an
443        // Expandable with empty Tokens + the original paramlist, which at
444        // expansion time reads and discards its arguments — semantically
445        // identical to the no-body form's runtime behavior (expandable.rs
446        // `invoke` matches `None` → reads args, returns NO_TOKENS).
447        None => Some(format!(
448          "E\t{}\t{}\t{}\t\t{}\t{}",
449          cs_name, nargs, flags, proto_encoded, v3_params
450        )),
451        // Closure-based body — can't serialize the closure itself, BUT
452        // if the dump entry is a `\let`-alias to a closure-Expandable
453        // primitive (e.g. `\let \tex_expandafter:D \expandafter`), we
454        // CAN capture it via PA. Mirrors the `Stored::Primitive` /
455        // `Stored::Conditional` arms below. Without this, every
456        // `\let \X \expandafter`-style alias is silently dropped from
457        // the dump — the cause of `\tex_expandafter:D`,
458        // `\tex_unexpanded:D`, `\tex_the:D` being missing in
459        // `latex.dump.txt` even though expl3-code.tex L357+ aliases them.
460        //
461        // The dump_reader's add-only policy at load-time skips entries
462        // whose key is already defined (the canonical CS lives in the
463        // compiled engine), so self-aliases are no-ops. Real let-aliases
464        // (key != cs) replay via `state::let_i`.
465        Some(crate::definition::ExpansionBody::Closure(_)) => Some(format!("PA\t{}", cs_name)),
466      }
467    },
468    // Closure-based primitives can't be serialized directly — but if the entry's
469    // CS differs from the primitive's own CS, it's a `\let`-alias, and we CAN
470    // capture the alias so the dump reader can replay `\let <key> <target>` at
471    // load time. This is how `\tex_let:D`, `\tex_def:D`, and hundreds of other
472    // expl3-renamed primitives stay reachable across dump boundaries (the
473    // defining assignment `\let \tex_let:D \let` in expl3-code.tex needs to
474    // survive the dump for `\usepackage{expl3}`'s guard `\ifx\csname
475    // tex_let:D\endcsname\relax` to short-circuit).
476    //
477    // NOTE: the KEY we're saving IS the dump entry's first column; the VALUE
478    // we write is the primitive's OWN CS name. Dump reader compares these:
479    // if equal, it's the "primary" primitive (already in bindings) and skipped;
480    // if different, it's an alias — replay `\let` at load time.
481    Stored::Primitive(p) => {
482      // Perl `Core/Dumper.pm::dump_primitive` (L383-389): if a primitive has
483      // a `font` directive AND no replacement closure (i.e. defined by `\font`),
484      // emit `FD(<cs>, <fontID>)` instead of generic Primitive serialization.
485      // `font_id` is set on `\font`-defined primitives by `engine/tex_fonts.rs`'s
486      // post-define hook (the Rust path) — it carries the value-table key under
487      // which the Stored::Font lives. The reader installs a synthesized
488      // Primitive whose before_digest looks up that font and merges, mirroring
489      // `LaTeXML::Core::Definition::FontDef::invoke` (FontDef.pm L38-45).
490      if let Some(fid) = p.font_id {
491        let fid_str = arena::with(fid, |s| s.to_string());
492        return Some(format!("FD\t{}", url_encode(&fid_str)));
493      }
494      let target_cs = p.cs.with_str(url_encode);
495      Some(format!("PA\t{}", target_cs))
496    },
497    Stored::Font(f) => {
498      // Perl `dump_font` (Core/Dumper.pm L281-284) emits `F(... components ...)`.
499      // Rust's `Font` is a flat options struct, so we serialize the non-None
500      // fields as `key=value` pairs separated by `\x1f` (unit-separator), with
501      // values url-encoded for tab/newline-safety. Reader inverts in dump_reader.rs.
502      let mut parts = Vec::with_capacity(8);
503      if let Some(ref name) = f.name {
504        parts.push(format!("name={}", url_encode(name)));
505      }
506      if let Some(size) = f.size {
507        parts.push(format!("size={}", size));
508      }
509      if let Some(ref family) = f.family {
510        parts.push(format!("family={}", url_encode(family)));
511      }
512      if let Some(ref series) = f.series {
513        parts.push(format!("series={}", url_encode(series)));
514      }
515      if let Some(ref shape) = f.shape {
516        parts.push(format!("shape={}", url_encode(shape)));
517      }
518      if let Some(ref encoding) = f.encoding {
519        parts.push(format!("encoding={}", url_encode(encoding)));
520      }
521      if let Some(ref language) = f.language {
522        parts.push(format!("language={}", url_encode(language)));
523      }
524      if let Some(ref mathstyle) = f.mathstyle {
525        parts.push(format!("mathstyle={}", url_encode(mathstyle)));
526      }
527      if let Some(ref opacity) = f.opacity {
528        parts.push(format!("opacity={}", url_encode(opacity)));
529      }
530      if let Some(scale) = f.scale {
531        parts.push(format!("scale={}", scale));
532      }
533      if let Some(b) = f.emph {
534        parts.push(format!("emph={}", if b { 1 } else { 0 }));
535      }
536      if let Some(b) = f.scripted {
537        parts.push(format!("scripted={}", if b { 1 } else { 0 }));
538      }
539      if let Some(step) = f.mathstylestep {
540        parts.push(format!("mathstylestep={}", step));
541      }
542      if let Some(flags) = f.flags {
543        parts.push(format!("flags={}", flags));
544      }
545      Some(format!("F\t{}", parts.join("\x1f")))
546    },
547    Stored::MathPrimitive(p) => {
548      let target_cs = p.cs.with_str(url_encode);
549      Some(format!("MPA\t{}", target_cs))
550    },
551    // Conditionals — same logic as Primitives. \ifx, \ifnum, \if etc. are
552    // stored as Stored::Conditional with their canonical CS field. When
553    // expl3-code.tex does `\let \if_meaning:w \ifx`, the new entry shares
554    // the same Conditional Rc, with cs = \ifx. Capturing this as a PA
555    // alias lets the dump reader replay `\let \if_meaning:w \ifx` at load
556    // time. Without this, every expl3 conditional alias is silently lost
557    // from the dump and `\if_meaning:w`-style invocations fail at runtime.
558    Stored::Conditional(c) => {
559      let target_cs = c.cs.with_str(url_encode);
560      Some(format!("PA\t{}", target_cs))
561    },
562    // Constructors — `\cr`, `\noalign`, `\mathchoice`, etc. defined via
563    // `DefConstructor!` carry replacement / before_digest / after_digest
564    // closures that the dump format can't serialize. But the canonical
565    // CS lives in the compiled engine, so a `\let \tex_cr:D \cr`-style
566    // alias can round-trip as `PA\t<cs>` and replay via `state::let_i`.
567    // Self-aliases are no-ops at load (key already defined).
568    Stored::Constructor(c) => {
569      let target_cs = c.cs.with_str(url_encode);
570      Some(format!("PA\t{}", target_cs))
571    },
572    Stored::Register(reg) => {
573      // Register: serialize as R\tCS\tTYPE\tVALUE[\tMATHGLYPH]
574      //
575      // Closure-backed registers can't be serialized directly — but if the
576      // dump entry is a `\let`-alias to a closure-Register primitive
577      // (e.g. `\let \tex_dimexpr:D \dimexpr`), we CAN capture it via PA.
578      // Mirrors the `Stored::Primitive` / closure-`Expandable` arms above.
579      // Without this every `\let \tex_dimexpr:D \dimexpr`-style alias is
580      // silently dropped from the dump — the cause of `\tex_dimexpr:D`,
581      // `\tex_cr:D`, `\tex_dp:D`, `\tex_ht:D`, `\tex_wd:D` (and ~430
582      // engine-specific others) being missing in `latex.dump.txt` even
583      // though expl3-code.tex L280-700 aliases them. The dump_reader's
584      // add-only policy at load-time skips entries whose key is already
585      // defined (the canonical CS lives in the compiled engine), so
586      // self-aliases are no-ops. Real let-aliases (key != cs) replay via
587      // `state::let_i` — so the engine-only register's behavior follows.
588      if reg.getter.is_some() || reg.setter.is_some() {
589        let target_cs = reg.cs.with_str(url_encode);
590        return Some(format!("PA\t{}", target_cs));
591      }
592      let cs_name = reg.cs.with_str(url_encode);
593      let rtype = match reg.register_type {
594        crate::definition::register::RegisterType::Number => "N",
595        crate::definition::register::RegisterType::Dimension => "D",
596        crate::definition::register::RegisterType::Glue => "G",
597        crate::definition::register::RegisterType::MuGlue => "MG",
598        crate::definition::register::RegisterType::Tokens => "TK",
599        crate::definition::register::RegisterType::CharDef => "CD",
600        _ => return None,
601      };
602      let value_str = match &reg.value {
603        Some(crate::definition::register::RegisterValue::Number(n)) => n.value_of().to_string(),
604        Some(crate::definition::register::RegisterValue::Dimension(d)) => d.value_of().to_string(),
605        Some(crate::definition::register::RegisterValue::Glue(g)) => {
606          let mut s = g.skip.to_string();
607          if let Some(p) = g.plus {
608            s.push_str(&format!(",p{}", p));
609          }
610          if let Some(ref pf) = g.pfill {
611            s.push_str(&format!(",pf{}", fillcode_index(pf)));
612          }
613          if let Some(m) = g.minus {
614            s.push_str(&format!(",m{}", m));
615          }
616          if let Some(ref mf) = g.mfill {
617            s.push_str(&format!(",mf{}", fillcode_index(mf)));
618          }
619          s
620        },
621        Some(crate::definition::register::RegisterValue::Tokens(tks)) => {
622          let tok_strs: Vec<String> = tks.unlist_ref().iter().map(serialize_token).collect();
623          tok_strs.join(",")
624        },
625        _ => "0".to_string(),
626      };
627      // Serialize address if different from cs name. Allocated registers
628      // (`\newcount\m@ne`/`\newdimen\p@`/etc.) point at low-level `\count<n>`/
629      // `\dimen<n>` slots whose runtime values live there, NOT at the alias
630      // CS name. Without the address, dump_reader would `assign_internal` at
631      // the CS name's slot (default value), and `\the\m@ne` would yield 0
632      // instead of -1. Mirror Perl's `R(C(...),undef,...,address=>'\\count22')`
633      // serialization (LaTeXML/Core/Dumper.pm). Format extension:
634      // `R\t<cs>\t<rtype>\t<value>\t<mathglyph_or_empty>\t<address>`.
635      // If address == cs_name we still serialize an empty 6th field for
636      // forward-compat parsers; the reader accepts both 4/5/6-field forms.
637      let mut s = format!("R\t{}\t{}\t{}", cs_name, rtype, value_str);
638      let mathglyph_str = match reg.mathglyph {
639        Some(glyph) => format!("{}", glyph as u32),
640        None => String::new(),
641      };
642      let cs_decoded = arena::with(reg.cs.get_sym(), |s| s.to_string());
643      let address_field = if reg.address.is_empty() || reg.address == cs_decoded {
644        String::new()
645      } else {
646        url_encode(&reg.address)
647      };
648      // Only emit extended fields when needed (keep current format size for
649      // the common case)
650      if !mathglyph_str.is_empty() || !address_field.is_empty() {
651        s.push_str(&format!("\t{}", mathglyph_str));
652      }
653      if !address_field.is_empty() {
654        s.push_str(&format!("\t{}", address_field));
655      }
656      Some(s)
657    },
658    // "Nm" (Number) — distinct from "I" (raw Int) so the reader knows
659    // to install Stored::Number, matching the type expected by register
660    // slots. Round-tripping Number through "I" silently downgraded
661    // values to Stored::Int and broke `\count` register reads.
662    Stored::Number(n) => Some(format!("Nm\t{}", n.value_of())),
663    Stored::Float(f) => Some(format!("F\t{}", f.0)),
664    Stored::Dimension(d) => Some(format!("D\t{}", d.0)),
665    Stored::Glue(g) => {
666      let mut s = format!("G\t{}", g.skip);
667      if let Some(p) = g.plus {
668        s.push_str(&format!(",p{}", p));
669      }
670      if let Some(ref pf) = g.pfill {
671        s.push_str(&format!(",pf{}", fillcode_index(pf)));
672      }
673      if let Some(m) = g.minus {
674        s.push_str(&format!(",m{}", m));
675      }
676      if let Some(ref mf) = g.mfill {
677        s.push_str(&format!(",mf{}", fillcode_index(mf)));
678      }
679      Some(s)
680    },
681    Stored::MuDimension(d) => Some(format!("MD\t{}", d.0)),
682    Stored::MuGlue(g) => {
683      let mut s = format!("MG\t{}", g.skip);
684      if let Some(p) = g.plus {
685        s.push_str(&format!(",p{}", p));
686      }
687      if let Some(ref pf) = g.pfill {
688        s.push_str(&format!(",pf{}", fillcode_index(pf)));
689      }
690      if let Some(m) = g.minus {
691        s.push_str(&format!(",m{}", m));
692      }
693      if let Some(ref mf) = g.mfill {
694        s.push_str(&format!(",mf{}", fillcode_index(mf)));
695      }
696      Some(s)
697    },
698    Stored::VecDequeStored(vd) if vd.is_empty() => Some("VD\t".to_string()),
699    _ => None,
700  }
701}
702
703fn fillcode_index(fc: &crate::common::glue::FillCode) -> usize {
704  use crate::common::glue::FillCode::*;
705  match fc {
706    Fil => 1,
707    Fill => 2,
708    Filll => 3,
709  }
710}
711
712fn serialize_token(t: &crate::token::Token) -> String {
713  let cc: u8 = t.get_catcode().into();
714  let text = t.with_str(url_encode);
715  format!("{}:{}", cc, text)
716}
717
718/// Serialize a `Parameters` list in the v3 structured format.
719///
720/// Layout (ASCII control-byte delimited so nothing collides with
721/// url-encoded printable content):
722/// - `\x1e` between Parameters (RS, Record Separator)
723/// - `\x1f` between fields of a single Parameter (US, Unit Separator)
724/// - `\x1d` between Tokens lists inside `extras` (GS, Group Separator)
725///
726/// Per-parameter fields: `<name>\x1f<spec>\x1f<flags>\x1f<extras>`
727///   - `<name>` — `Parameter.name` (Perl's `type`), url-encoded.
728///   - `<spec>` — `Parameter.spec`, url-encoded.
729///   - `<flags>` — semicolon-separated `n=1` (novalue) / `o=1` (optional).
730///   - `<extras>` — `<Tokens1>\x1d<Tokens2>…` where each `<TokensN>` is the same comma-joined
731///     `<catcode>:<text>` encoding used for E-entry bodies. Empty extras → empty field.
732///
733/// The reader (`dump_reader::parse_parameters_v3`) hands the decoded
734/// `(name, spec, Some(extras))` triple directly to `Parameter::new`,
735/// bypassing `parse_parameters` entirely — so `Until:\end{verbatim}`
736/// and `Match:…` round-trip with their catcoded delimiter tokens intact.
737pub(crate) fn serialize_parameters_v3(params: &crate::parameter::Parameters) -> String {
738  let mut out = String::new();
739  for (i, p) in params.get_parameters().iter().enumerate() {
740    if i > 0 {
741      out.push('\x1e');
742    }
743    let name = arena::with(p.name, url_encode);
744    let spec = arena::with(p.spec, url_encode);
745    let mut flags = String::new();
746    if p.novalue {
747      flags.push_str("n=1");
748    }
749    if p.optional {
750      if !flags.is_empty() {
751        flags.push(';');
752      }
753      flags.push_str("o=1");
754    }
755    let extras = p
756      .extra
757      .iter()
758      .map(|tks| {
759        tks
760          .clone()
761          .unlist()
762          .iter()
763          .map(serialize_token)
764          .collect::<Vec<_>>()
765          .join(",")
766      })
767      .collect::<Vec<_>>()
768      .join("\x1d");
769    out.push_str(&name);
770    out.push('\x1f');
771    out.push_str(&spec);
772    out.push('\x1f');
773    out.push_str(&flags);
774    out.push('\x1f');
775    out.push_str(&extras);
776  }
777  out
778}
779
780fn url_encode(s: &str) -> String {
781  let mut result = String::with_capacity(s.len());
782  for ch in s.chars() {
783    match ch {
784      '\t' => result.push_str("%09"),
785      '\n' => result.push_str("%0A"),
786      '\r' => result.push_str("%0D"),
787      '%' => result.push_str("%25"),
788      ',' => result.push_str("%2C"),
789      _ if ch.is_ascii_control() => {
790        result.push_str(&format!("%{:02X}", ch as u8));
791      },
792      _ => result.push(ch),
793    }
794  }
795  result
796}
797
798/// Recognize expl3's intarray-as-fontdimen storage keys, e.g.
799/// `fontdimen_fontinfo_cmr10 at 15sp_12737` → ("fontdimen_fontinfo_cmr10 at 15sp", 12737).
800/// Returns None for other Value keys (so the regular V-record path applies).
801fn parse_fontdimen_key(key: &str) -> Option<(&str, u32)> {
802  // Cheap gate first to keep the hot path fast on non-fontdimen keys.
803  if !key.starts_with("fontdimen_fontinfo_") {
804    return None;
805  }
806  // The last `_<digits>` tail is the slot index; everything before is the
807  // per-intarray prefix. Be defensive: an all-letter tail (e.g. a future
808  // non-indexed `fontdimen_fontinfo_*` key) should not match.
809  let last_us = key.rfind('_')?;
810  let (prefix, tail) = (&key[..last_us], &key[last_us + 1..]);
811  let index: u32 = tail.parse().ok()?;
812  Some((prefix, index))
813}
814
815/// Run-length encode a slice of i64 values as a comma-separated list:
816/// each run is either `<v>` (single) or `<v>x<n>` (count ≥ 2). Decoder
817/// in `dump_reader::rle_decode_i64` is the inverse.
818fn rle_encode_i64(values: &[i64]) -> String {
819  let mut out = String::new();
820  let mut i = 0;
821  while i < values.len() {
822    let v = values[i];
823    let mut count = 1usize;
824    while i + count < values.len() && values[i + count] == v {
825      count += 1;
826    }
827    if !out.is_empty() {
828      out.push(',');
829    }
830    if count == 1 {
831      out.push_str(&v.to_string());
832    } else {
833      out.push_str(&format!("{}x{}", v, count));
834    }
835    i += count;
836  }
837  out
838}
839
840#[cfg(test)]
841mod tests {
842  use super::*;
843  use crate::{
844    parameter::{Parameter, Parameters},
845    token::{Catcode, Token},
846    tokens::Tokens,
847  };
848
849  /// Construct a Parameter without calling `init()` — `init()` needs live
850  /// state (PARAMETER_TYPES table), which we don't set up in unit tests.
851  /// The full test suite exercises the `init()` path via dump load.
852  fn raw_param(name: &str, spec: &str, extra: Vec<Tokens>) -> Parameter {
853    Parameter {
854      name: arena::pin(name),
855      spec: arena::pin(spec),
856      extra,
857      ..Parameter::default()
858    }
859  }
860
861  /// A Parameters list with two `Plain {}` args encodes as two records
862  /// separated by RS, each `Plain<US>{}<US><US>`. Verifies baseline layout.
863  #[test]
864  fn v3_encoding_plain() {
865    let ps = Parameters::new(vec![
866      raw_param("Plain", "{}", vec![]),
867      raw_param("Plain", "{}", vec![]),
868    ]);
869    let s = serialize_parameters_v3(&ps);
870    assert_eq!(s, "Plain\x1f{}\x1f\x1f\x1ePlain\x1f{}\x1f\x1f");
871  }
872
873  /// `Until:<delim>` with a brace-containing delimiter. The test covers
874  /// the case that livelocked 00_tokenize under v2: the delimiter tokens
875  /// must serialize as catcoded tokens (not as the raw spec string), so
876  /// `{`/`}` survive as CC_BEGIN / CC_END rather than being re-parsed as
877  /// spec-level braces.
878  #[test]
879  fn v3_encoding_until_with_braces() {
880    let delim = Tokens::new(vec![
881      Token {
882        text: arena::pin("\\end"),
883        code: Catcode::CS,
884        #[cfg(feature = "token-locators")]
885        loc: 0,
886      },
887      Token {
888        text: arena::pin("{"),
889        code: Catcode::BEGIN,
890        #[cfg(feature = "token-locators")]
891        loc: 0,
892      },
893      Token {
894        text: arena::pin("verbatim"),
895        code: Catcode::LETTER,
896        #[cfg(feature = "token-locators")]
897        loc: 0,
898      },
899      Token {
900        text: arena::pin("}"),
901        code: Catcode::END,
902        #[cfg(feature = "token-locators")]
903        loc: 0,
904      },
905    ]);
906    let ps = Parameters::new(vec![raw_param("Until", "Until:\\end{verbatim}", vec![
907      delim,
908    ])]);
909    let s = serialize_parameters_v3(&ps);
910    // Expected: Until<US>Until:\end{verbatim}<US><US>16:\end,1:{,11:verbatim,2:}
911    assert!(s.starts_with("Until\x1fUntil:\\end{verbatim}\x1f\x1f"));
912    assert!(s.contains("16:\\end,1:{,11:verbatim,2:}"));
913  }
914
915  /// `novalue` flag encodes as `n=1` in the flags field.
916  #[test]
917  fn v3_encoding_novalue_flag() {
918    let mut p = raw_param("Match", "Match:abc", vec![]);
919    p.novalue = true;
920    let ps = Parameters::new(vec![p]);
921    let s = serialize_parameters_v3(&ps);
922    assert_eq!(s, "Match\x1fMatch:abc\x1fn=1\x1f");
923  }
924
925  /// `optional` flag encodes as `o=1`; both flags together as `n=1;o=1`.
926  #[test]
927  fn v3_encoding_both_flags() {
928    let mut p = raw_param("OptionalMatch", "OptionalMatch:x", vec![]);
929    p.novalue = true;
930    p.optional = true;
931    let ps = Parameters::new(vec![p]);
932    let s = serialize_parameters_v3(&ps);
933    assert_eq!(s, "OptionalMatch\x1fOptionalMatch:x\x1fn=1;o=1\x1f");
934  }
935
936  /// Empty Parameters serializes to empty string.
937  #[test]
938  fn v3_encoding_empty() {
939    let ps = Parameters::new(vec![]);
940    assert_eq!(serialize_parameters_v3(&ps), "");
941  }
942
943  /// Multiple Tokens inside `extras` (as in `Match` with multiple choices)
944  /// are separated by GS. Verifies the three-level delimiter scheme.
945  #[test]
946  fn v3_encoding_multiple_extra_tokens() {
947    let t1 = Tokens::new(vec![Token {
948      text: arena::pin("a"),
949      code: Catcode::LETTER,
950      #[cfg(feature = "token-locators")]
951      loc: 0,
952    }]);
953    let t2 = Tokens::new(vec![Token {
954      text: arena::pin("b"),
955      code: Catcode::LETTER,
956      #[cfg(feature = "token-locators")]
957      loc: 0,
958    }]);
959    let ps = Parameters::new(vec![raw_param("Match", "Match:ab", vec![t1, t2])]);
960    let s = serialize_parameters_v3(&ps);
961    // Expected extras field: "11:a<GS>11:b"
962    assert_eq!(s, "Match\x1fMatch:ab\x1f\x1f11:a\x1d11:b");
963  }
964
965  // --- RLE encoder tests (intarray consolidation) ---
966
967  #[test]
968  fn rle_empty_slice() {
969    assert_eq!(rle_encode_i64(&[]), "");
970  }
971
972  #[test]
973  fn rle_single_value() {
974    assert_eq!(rle_encode_i64(&[5]), "5");
975  }
976
977  #[test]
978  fn rle_two_distinct() {
979    assert_eq!(rle_encode_i64(&[1, 2]), "1,2");
980  }
981
982  #[test]
983  fn rle_run_of_two() {
984    assert_eq!(rle_encode_i64(&[5, 5]), "5x2");
985  }
986
987  #[test]
988  fn rle_long_run() {
989    assert_eq!(rle_encode_i64(&[218; 10000]), "218x10000");
990  }
991
992  #[test]
993  fn rle_mixed_runs() {
994    assert_eq!(rle_encode_i64(&[1, 2, 2, 3, 3, 3, 1]), "1,2x2,3x3,1");
995  }
996
997  #[test]
998  fn rle_negative() {
999    assert_eq!(rle_encode_i64(&[-5]), "-5");
1000    assert_eq!(rle_encode_i64(&[-5, -5, -5]), "-5x3");
1001  }
1002
1003  #[test]
1004  fn rle_extreme_values() {
1005    assert_eq!(rle_encode_i64(&[i64::MIN]), i64::MIN.to_string());
1006    assert_eq!(rle_encode_i64(&[i64::MAX]), i64::MAX.to_string());
1007    assert_eq!(rle_encode_i64(&[0; 3]), "0x3");
1008  }
1009
1010  // --- parse_fontdimen_key tests ---
1011
1012  #[test]
1013  fn fontdimen_key_standard() {
1014    assert_eq!(
1015      parse_fontdimen_key("fontdimen_fontinfo_cmr10 at 15sp_12737"),
1016      Some(("fontdimen_fontinfo_cmr10 at 15sp", 12737))
1017    );
1018  }
1019
1020  #[test]
1021  fn fontdimen_key_index_one() {
1022    assert_eq!(
1023      parse_fontdimen_key("fontdimen_fontinfo_cmr10 at 5sp_1"),
1024      Some(("fontdimen_fontinfo_cmr10 at 5sp", 1))
1025    );
1026  }
1027
1028  #[test]
1029  fn fontdimen_key_non_fontdimen() {
1030    assert_eq!(parse_fontdimen_key("count@"), None);
1031    assert_eq!(parse_fontdimen_key("\\@oddpage"), None);
1032  }
1033
1034  #[test]
1035  fn fontdimen_key_wrong_prefix() {
1036    // Missing the `_fontinfo_` segment — must not match.
1037    assert_eq!(parse_fontdimen_key("fontdimen_cmr10_15"), None);
1038  }
1039
1040  #[test]
1041  fn fontdimen_key_non_numeric_tail() {
1042    // No trailing digits after last `_` ⇒ no index ⇒ no match.
1043    assert_eq!(
1044      parse_fontdimen_key("fontdimen_fontinfo_cmr10 at 15sp_abc"),
1045      None
1046    );
1047  }
1048
1049  #[test]
1050  fn fontdimen_key_empty_tail() {
1051    // Trailing underscore with no digits ⇒ no match.
1052    assert_eq!(
1053      parse_fontdimen_key("fontdimen_fontinfo_cmr10 at 15sp_"),
1054      None
1055    );
1056  }
1057}