latexml_core/
dump_codegen.rs

1//! Generate Rust source code from a kernel dump file.
2//!
3//! Reads the text dump produced by `dump_writer.rs` and emits a `.rs` file
4//! containing typed static data tables. The compiler checks every entry at
5//! build time — no runtime parsing, no text format errors possible.
6//!
7//! Usage:
8//!   1. Generate dump: `latexml_oxide --init=latex.ltx --dest=/tmp/dump.tmp`
9//!   2. Generate Rust: `dump_codegen::generate_rs("/tmp/dump.tmp", "latex_dump.rs")`
10//!   3. Place in `latexml_package/src/engine/latex_dump.rs`
11//!   4. Load at runtime via `latex_dump::load_definitions()`
12
13use std::{io::Write, path::Path};
14
15/// Value entries to skip (runtime-specific or cause regressions).
16const SKIP_VALUES: &[&str] = &[
17  "INTERPRETING_DEFINITIONS",
18  "if_count",
19  "absorb_count",
20  "_loaded",
21  "INCLUDE_COMMENTS",
22  "INCLUDE_STYLES",
23  "\\everyjob",
24  "\\toks",
25  "input_file:",
26  "output_file:",
27  "texsys",
28];
29
30fn should_skip_value(key: &str) -> bool {
31  SKIP_VALUES
32    .iter()
33    .any(|skip| key == *skip || key.starts_with(skip) || key.ends_with(skip) || key.contains(skip))
34}
35
36/// Escape a string for use as a Rust string literal.
37fn rust_escape(s: &str) -> String {
38  if s.contains("\"#")
39    || s.contains('\r')
40    || s
41      .chars()
42      .any(|c| c.is_ascii_control() && c != '\n' && c != '\t')
43  {
44    let mut out = String::with_capacity(s.len() + 8);
45    out.push('"');
46    for ch in s.chars() {
47      match ch {
48        '\\' => out.push_str("\\\\"),
49        '"' => out.push_str("\\\""),
50        '\n' => out.push_str("\\n"),
51        '\r' => out.push_str("\\r"),
52        '\t' => out.push_str("\\t"),
53        c if c.is_ascii_control() => out.push_str(&format!("\\x{:02x}", c as u8)),
54        c => out.push(c),
55      }
56    }
57    out.push('"');
58    out
59  } else {
60    format!("r#\"{}\"#", s)
61  }
62}
63
64fn url_decode(s: &str) -> String {
65  let mut result = String::with_capacity(s.len());
66  let mut chars = s.chars();
67  while let Some(ch) = chars.next() {
68    if ch == '%' {
69      let hex: String = chars.by_ref().take(2).collect();
70      if let Ok(byte) = u8::from_str_radix(&hex, 16) {
71        result.push(byte as char);
72      }
73    } else {
74      result.push(ch);
75    }
76  }
77  result
78}
79
80// ---- Parsed entry types ----
81
82struct BoolEntry {
83  key: String,
84  val: bool,
85}
86struct IntEntry {
87  key: String,
88  val: i64,
89}
90struct StrEntry {
91  key: String,
92  val: String,
93}
94struct DimEntry {
95  key: String,
96  val: i64,
97}
98struct GlueEntry {
99  key:   String,
100  skip:  i64,
101  plus:  Option<i64>,
102  pfill: u8, // 0=None, 1=Fil, 2=Fill, 3=Filll
103  minus: Option<i64>,
104  mfill: u8,
105}
106struct CharcodeEntry {
107  key: String,
108  val: u16,
109}
110struct CatcodeEntry {
111  key: String,
112  val: u8,
113}
114struct TokenEntry {
115  key:  String,
116  cc:   u8,
117  text: String,
118}
119struct TokenListEntry {
120  key:       String,
121  tok_start: u32,
122  tok_count: u16,
123}
124struct ExpandEntry {
125  cs:        String,
126  nargs:     u8,
127  flags:     u8, // bit 0=long, bit 1=protected
128  tok_start: u32,
129  tok_count: u16,
130}
131struct LetAliasEntry {
132  key:     String, // e.g. "\\tex_let:D"
133  target:  String, // e.g. "\\let"
134  // Reserved for future use: when we start emitting MathPrimitive-specific
135  // assignment calls, this flag distinguishes PA (ordinary Primitive alias)
136  // from MPA (MathPrimitive alias). Today `state::let_i` handles both by
137  // copying the target's Stored meaning, so the flag is advisory.
138  #[allow(dead_code)]
139  is_math: bool,
140}
141struct TokData {
142  cc:   u8,
143  text: String,
144}
145
146/// All categorized entries from a dump file.
147#[derive(Default)]
148struct DumpData {
149  // Value table
150  bools:         Vec<BoolEntry>,
151  ints:          Vec<IntEntry>,
152  strings:       Vec<StrEntry>,
153  dims:          Vec<DimEntry>,
154  glues:         Vec<GlueEntry>,
155  mudims:        Vec<DimEntry>,
156  muglues:       Vec<GlueEntry>,
157  charcodes:     Vec<CharcodeEntry>,
158  catcode_vals:  Vec<CatcodeEntry>,
159  single_tokens: Vec<TokenEntry>,
160  token_lists:   Vec<TokenListEntry>,
161  nones:         Vec<String>,
162  vecdeques:     Vec<String>,
163
164  // Code tables
165  catcodes:  Vec<CatcodeEntry>,
166  lccodes:   Vec<CharcodeEntry>,
167  uccodes:   Vec<CharcodeEntry>,
168  sfcodes:   Vec<CharcodeEntry>,
169  delcodes:  Vec<CharcodeEntry>,
170  mathcodes: Vec<CharcodeEntry>,
171
172  // Meaning table
173  let_defs:    Vec<TokenEntry>,
174  expandables: Vec<ExpandEntry>,
175  let_aliases: Vec<LetAliasEntry>, // PA / MPA entries
176
177  // Token pools (flattened)
178  value_tokens:  Vec<TokData>,
179  expand_tokens: Vec<TokData>,
180}
181
182impl DumpData {
183  fn total_entries(&self) -> usize {
184    self.bools.len()
185      + self.ints.len()
186      + self.strings.len()
187      + self.dims.len()
188      + self.glues.len()
189      + self.mudims.len()
190      + self.muglues.len()
191      + self.charcodes.len()
192      + self.catcode_vals.len()
193      + self.single_tokens.len()
194      + self.token_lists.len()
195      + self.nones.len()
196      + self.vecdeques.len()
197      + self.catcodes.len()
198      + self.lccodes.len()
199      + self.uccodes.len()
200      + self.sfcodes.len()
201      + self.delcodes.len()
202      + self.mathcodes.len()
203      + self.let_defs.len()
204      + self.expandables.len()
205      + self.let_aliases.len()
206  }
207}
208
209/// Parse dump content into categorized entries.
210fn parse_dump(content: &str) -> DumpData {
211  let mut data = DumpData::default();
212
213  for line in content.lines() {
214    let line = line.trim();
215    if line.is_empty() || line.starts_with('#') {
216      continue;
217    }
218    let parts: Vec<&str> = line.splitn(3, '\t').collect();
219    if parts.len() < 2 {
220      continue;
221    }
222    let table = parts[0];
223    let key = url_decode(parts[1]);
224    let rest = if parts.len() > 2 { parts[2] } else { "" };
225
226    match table {
227      "V" => parse_value(&mut data, &key, rest),
228      "M" => parse_meaning(&mut data, &key, rest),
229      "C" => {
230        let val_parts: Vec<&str> = rest.splitn(2, '\t').collect();
231        if val_parts.len() >= 2
232          && let Ok(v) = val_parts[1].parse::<u8>()
233        {
234          data.catcodes.push(CatcodeEntry { key, val: v });
235        }
236      },
237      "LC" => {
238        let val_parts: Vec<&str> = rest.splitn(2, '\t').collect();
239        if val_parts.len() >= 2
240          && let Ok(v) = val_parts[1].parse::<u16>()
241        {
242          data.lccodes.push(CharcodeEntry { key, val: v });
243        }
244      },
245      "UC" => {
246        let val_parts: Vec<&str> = rest.splitn(2, '\t').collect();
247        if val_parts.len() >= 2
248          && let Ok(v) = val_parts[1].parse::<u16>()
249        {
250          data.uccodes.push(CharcodeEntry { key, val: v });
251        }
252      },
253      "SC" => {
254        let val_parts: Vec<&str> = rest.splitn(2, '\t').collect();
255        if val_parts.len() >= 2
256          && let Ok(v) = val_parts[1].parse::<u16>()
257        {
258          data.sfcodes.push(CharcodeEntry { key, val: v });
259        }
260      },
261      "DC" => {
262        let val_parts: Vec<&str> = rest.splitn(2, '\t').collect();
263        if val_parts.len() >= 2
264          && let Ok(v) = val_parts[1].parse::<u16>()
265        {
266          data.delcodes.push(CharcodeEntry { key, val: v });
267        }
268      },
269      "MC" => {
270        let val_parts: Vec<&str> = rest.splitn(2, '\t').collect();
271        if val_parts.len() >= 2
272          && let Ok(v) = val_parts[1].parse::<u16>()
273        {
274          data.mathcodes.push(CharcodeEntry { key, val: v });
275        }
276      },
277      _ => {}, // Skip unknown tables
278    }
279  }
280
281  data
282}
283
284fn parse_value(data: &mut DumpData, key: &str, rest: &str) {
285  // Skip \ver@ and other runtime-specific entries
286  if key.starts_with("\\ver@") || should_skip_value(key) {
287    return;
288  }
289  let parts: Vec<&str> = rest.splitn(2, '\t').collect();
290  if parts.is_empty() {
291    return;
292  }
293  let val_data = parts.get(1).copied().unwrap_or("");
294
295  match parts[0] {
296    "N" => data.nones.push(key.to_string()),
297    "B" => data.bools.push(BoolEntry {
298      key: key.to_string(),
299      val: val_data == "1",
300    }),
301    "I" => {
302      if let Ok(v) = val_data.parse::<i64>() {
303        data.ints.push(IntEntry { key: key.to_string(), val: v });
304      }
305    },
306    "S" => data.strings.push(StrEntry {
307      key: key.to_string(),
308      val: url_decode(val_data),
309    }),
310    "CH" => {
311      if let Ok(v) = val_data.parse::<u16>() {
312        data
313          .charcodes
314          .push(CharcodeEntry { key: key.to_string(), val: v });
315      }
316    },
317    "CC" => {
318      if let Ok(v) = val_data.parse::<u8>() {
319        data
320          .catcode_vals
321          .push(CatcodeEntry { key: key.to_string(), val: v });
322      }
323    },
324    "D" => {
325      if let Ok(v) = val_data.parse::<i64>() {
326        data.dims.push(DimEntry { key: key.to_string(), val: v });
327      }
328    },
329    "G" => {
330      if let Some(g) = parse_glue_data(key, val_data) {
331        data.glues.push(g);
332      }
333    },
334    "MD" => {
335      if let Ok(v) = val_data.parse::<i64>() {
336        data.mudims.push(DimEntry { key: key.to_string(), val: v });
337      }
338    },
339    "MG" => {
340      if let Some(g) = parse_glue_data(key, val_data) {
341        data.muglues.push(g);
342      }
343    },
344    "T" => {
345      if let Some((cc, text)) = parse_tok(val_data) {
346        data
347          .single_tokens
348          .push(TokenEntry { key: key.to_string(), cc, text });
349      }
350    },
351    "TK" => {
352      if val_data.is_empty() {
353        data.token_lists.push(TokenListEntry {
354          key:       key.to_string(),
355          tok_start: data.value_tokens.len() as u32,
356          tok_count: 0,
357        });
358      } else {
359        let start = data.value_tokens.len() as u32;
360        let mut count = 0u16;
361        for tok_s in val_data.split(',') {
362          if let Some((cc, text)) = parse_tok(tok_s) {
363            data.value_tokens.push(TokData { cc, text });
364            count += 1;
365          }
366        }
367        data.token_lists.push(TokenListEntry {
368          key:       key.to_string(),
369          tok_start: start,
370          tok_count: count,
371        });
372      }
373    },
374    "VD" => data.vecdeques.push(key.to_string()),
375    _ => {},
376  }
377}
378
379fn parse_meaning(data: &mut DumpData, key: &str, rest: &str) {
380  let parts: Vec<&str> = rest.splitn(2, '\t').collect();
381  if parts.is_empty() {
382    return;
383  }
384  match parts[0] {
385    "N" => {}, // Skip None meanings
386    "E" => {
387      let eparts: Vec<&str> = parts.get(1).unwrap_or(&"").splitn(4, '\t').collect();
388      if eparts.len() < 4 {
389        return;
390      }
391      let nargs: u8 = eparts[1].parse().unwrap_or(0);
392      let flags_str = eparts[2];
393      let tok_data = eparts[3];
394
395      let mut flags: u8 = 0;
396      if flags_str.contains('L') {
397        flags |= 1;
398      }
399      if flags_str.contains('P') {
400        flags |= 2;
401      }
402
403      let start = data.expand_tokens.len() as u32;
404      let mut count = 0u16;
405      if !tok_data.is_empty() {
406        for tok_s in tok_data.split(',') {
407          if let Some((cc, text)) = parse_tok(tok_s) {
408            data.expand_tokens.push(TokData { cc, text });
409            count += 1;
410          }
411        }
412      }
413      data.expandables.push(ExpandEntry {
414        cs: key.to_string(),
415        nargs,
416        flags,
417        tok_start: start,
418        tok_count: count,
419      });
420    },
421    "T" => {
422      let tok_s = parts.get(1).unwrap_or(&"");
423      if let Some((cc, text)) = parse_tok(tok_s) {
424        data
425          .let_defs
426          .push(TokenEntry { key: key.to_string(), cc, text });
427      }
428    },
429    "PA" | "MPA" => {
430      // Primitive alias: \let <key> = <target>.
431      // Emit as a call to state::let_i at load time; target must already
432      // be defined (guaranteed by the (d.2) early/late ordering).
433      let target = url_decode(parts.get(1).unwrap_or(&""));
434      if target.is_empty() || target == key {
435        return; // self-alias or malformed: skip
436      }
437      data.let_aliases.push(LetAliasEntry {
438        key: key.to_string(),
439        target,
440        is_math: parts[0] == "MPA",
441      });
442    },
443    _ => {},
444  }
445}
446
447fn parse_tok(s: &str) -> Option<(u8, String)> {
448  let (cc_str, text) = s.split_once(':')?;
449  let cc: u8 = cc_str.parse().ok()?;
450  Some((cc, url_decode(text)))
451}
452
453fn parse_glue_data(key: &str, s: &str) -> Option<GlueEntry> {
454  let mut skip = 0i64;
455  let mut plus = None;
456  let mut pfill = 0u8;
457  let mut minus = None;
458  let mut mfill = 0u8;
459
460  for (i, part) in s.split(',').enumerate() {
461    if i == 0 {
462      skip = part.parse().ok()?;
463    } else if let Some(rest) = part.strip_prefix("pf") {
464      pfill = rest.parse().unwrap_or(0);
465    } else if let Some(rest) = part.strip_prefix('p') {
466      plus = Some(rest.parse().ok()?);
467    } else if let Some(rest) = part.strip_prefix("mf") {
468      mfill = rest.parse().unwrap_or(0);
469    } else if let Some(rest) = part.strip_prefix('m') {
470      minus = Some(rest.parse().ok()?);
471    }
472  }
473
474  Some(GlueEntry {
475    key: key.to_string(),
476    skip,
477    plus,
478    pfill,
479    minus,
480    mfill,
481  })
482}
483
484// ---- Code generation ----
485
486/// Generate a Rust source file with typed static data tables from a text dump.
487/// Every entry is compiler-checked. Zero runtime parsing needed.
488pub fn generate_rs(dump_path: &Path, output_path: &Path) -> Result<usize, String> {
489  let content =
490    std::fs::read_to_string(dump_path).map_err(|e| format!("Failed to read dump: {}", e))?;
491
492  let data = parse_dump(&content);
493  let total = data.total_entries();
494
495  let mut out =
496    std::fs::File::create(output_path).map_err(|e| format!("Failed to create output: {}", e))?;
497
498  // Header
499  writeln!(
500    out,
501    "//! Auto-generated kernel dump — static data tables.\n\
502     //! DO NOT EDIT — regenerate with: `cargo run --release --bin latexml_oxide -- --init=latex.ltx`\n\
503     //!\n\
504     //! {total} entries, compiler-checked at build time. Zero runtime parsing.\n\
505     #![allow(unused)]\n\
506     #![allow(clippy::type_complexity)]\n"
507  )
508  .map_err(we)?;
509
510  // Imports
511  writeln!(
512    out,
513    "use std::collections::VecDeque;\n\
514     \n\
515     use latexml_core::common::arena;\n\
516     use latexml_core::common::def_parser::parse_parameters;\n\
517     use latexml_core::common::dimension::Dimension;\n\
518     use latexml_core::common::glue::{{FillCode, Glue}};\n\
519     use latexml_core::common::mudimension::MuDimension;\n\
520     use latexml_core::common::muglue::MuGlue;\n\
521     use latexml_core::common::store::Stored;\n\
522     use latexml_core::definition::expandable::{{Expandable, ExpandableOptions}};\n\
523     use latexml_core::state;\n\
524     use latexml_core::state::Scope;\n\
525     use latexml_core::token::{{Catcode, Token}};\n\
526     use latexml_core::tokens::Tokens;\n"
527  )
528  .map_err(we)?;
529
530  // Local struct for expandable definitions
531  writeln!(
532    out,
533    "struct ExpandDef {{\n\
534     {s}cs: &'static str,\n\
535     {s}nargs: u8,\n\
536     {s}flags: u8, // bit 0 = long, bit 1 = protected\n\
537     {s}tok_start: u32,\n\
538     {s}tok_count: u16,\n\
539     }}\n",
540    s = "  "
541  )
542  .map_err(we)?;
543
544  // Emit static arrays
545  emit_bool_array(&mut out, "BOOL_VALUES", &data.bools)?;
546  emit_int_array(&mut out, "INT_VALUES", &data.ints)?;
547  emit_str_array(&mut out, "STRING_VALUES", &data.strings)?;
548  emit_dim_array(&mut out, "DIMENSION_VALUES", &data.dims)?;
549  emit_glue_array(&mut out, "GLUE_VALUES", &data.glues)?;
550  emit_dim_array(&mut out, "MUDIM_VALUES", &data.mudims)?;
551  emit_glue_array(&mut out, "MUGLUE_VALUES", &data.muglues)?;
552  emit_charcode_array(&mut out, "CHARCODE_VALUES", &data.charcodes)?;
553  emit_catcode_array(&mut out, "CATCODE_VALUE_ENTRIES", &data.catcode_vals)?;
554  emit_token_array(&mut out, "SINGLE_TOKEN_VALUES", &data.single_tokens)?;
555  emit_none_array(&mut out, "NONE_VALUES", &data.nones)?;
556  emit_none_array(&mut out, "VECDEQUE_VALUES", &data.vecdeques)?;
557
558  // Token list values (V/TK) — entries reference VALUE_TOKENS pool
559  emit_token_list_array(&mut out, "TOKEN_LIST_VALUES", &data.token_lists)?;
560  emit_tok_pool(&mut out, "VALUE_TOKENS", &data.value_tokens)?;
561
562  // Code tables
563  emit_catcode_array(&mut out, "CATCODE_TABLE", &data.catcodes)?;
564  emit_charcode_array(&mut out, "LCCODE_TABLE", &data.lccodes)?;
565  emit_charcode_array(&mut out, "UCCODE_TABLE", &data.uccodes)?;
566  emit_charcode_array(&mut out, "SFCODE_TABLE", &data.sfcodes)?;
567  emit_charcode_array(&mut out, "DELCODE_TABLE", &data.delcodes)?;
568  emit_charcode_array(&mut out, "MATHCODE_TABLE", &data.mathcodes)?;
569
570  // Meaning table
571  emit_token_array(&mut out, "LET_DEFINITIONS", &data.let_defs)?;
572  emit_expand_array(&mut out, "EXPANDABLE_DEFS", &data.expandables)?;
573  emit_tok_pool(&mut out, "EXPAND_TOKENS", &data.expand_tokens)?;
574  emit_let_alias_array(&mut out, "LET_ALIAS_DEFS", &data.let_aliases)?;
575
576  // load_definitions function
577  emit_load_fn(&mut out, &data)?;
578
579  Info!(
580    "dump_codegen",
581    "generated",
582    s!(
583      "Generated {} entries to {} (static data tables)",
584      total,
585      output_path.display()
586    )
587  );
588
589  Ok(total)
590}
591
592fn we(e: std::io::Error) -> String { format!("Write error: {}", e) }
593
594fn emit_bool_array(
595  out: &mut std::fs::File,
596  name: &str,
597  entries: &[BoolEntry],
598) -> Result<(), String> {
599  writeln!(out, "static {name}: &[(&str, bool)] = &[").map_err(we)?;
600  for e in entries {
601    writeln!(out, "  ({}, {}),", rust_escape(&e.key), e.val).map_err(we)?;
602  }
603  writeln!(out, "];\n").map_err(we)
604}
605
606fn emit_int_array(out: &mut std::fs::File, name: &str, entries: &[IntEntry]) -> Result<(), String> {
607  writeln!(out, "static {name}: &[(&str, i64)] = &[").map_err(we)?;
608  for e in entries {
609    writeln!(out, "  ({}, {}),", rust_escape(&e.key), e.val).map_err(we)?;
610  }
611  writeln!(out, "];\n").map_err(we)
612}
613
614fn emit_str_array(out: &mut std::fs::File, name: &str, entries: &[StrEntry]) -> Result<(), String> {
615  writeln!(out, "static {name}: &[(&str, &str)] = &[").map_err(we)?;
616  for e in entries {
617    writeln!(out, "  ({}, {}),", rust_escape(&e.key), rust_escape(&e.val)).map_err(we)?;
618  }
619  writeln!(out, "];\n").map_err(we)
620}
621
622fn emit_dim_array(out: &mut std::fs::File, name: &str, entries: &[DimEntry]) -> Result<(), String> {
623  writeln!(out, "static {name}: &[(&str, i64)] = &[").map_err(we)?;
624  for e in entries {
625    writeln!(out, "  ({}, {}),", rust_escape(&e.key), e.val).map_err(we)?;
626  }
627  writeln!(out, "];\n").map_err(we)
628}
629
630fn emit_glue_array(
631  out: &mut std::fs::File,
632  name: &str,
633  entries: &[GlueEntry],
634) -> Result<(), String> {
635  // (key, skip, plus, pfill, minus, mfill)
636  writeln!(
637    out,
638    "static {name}: &[(&str, i64, Option<i64>, u8, Option<i64>, u8)] = &["
639  )
640  .map_err(we)?;
641  for e in entries {
642    let plus_s = match e.plus {
643      Some(v) => format!("Some({v})"),
644      None => "None".to_string(),
645    };
646    let minus_s = match e.minus {
647      Some(v) => format!("Some({v})"),
648      None => "None".to_string(),
649    };
650    writeln!(
651      out,
652      "  ({}, {}, {}, {}, {}, {}),",
653      rust_escape(&e.key),
654      e.skip,
655      plus_s,
656      e.pfill,
657      minus_s,
658      e.mfill
659    )
660    .map_err(we)?;
661  }
662  writeln!(out, "];\n").map_err(we)
663}
664
665fn emit_charcode_array(
666  out: &mut std::fs::File,
667  name: &str,
668  entries: &[CharcodeEntry],
669) -> Result<(), String> {
670  writeln!(out, "static {name}: &[(&str, u16)] = &[").map_err(we)?;
671  for e in entries {
672    writeln!(out, "  ({}, {}),", rust_escape(&e.key), e.val).map_err(we)?;
673  }
674  writeln!(out, "];\n").map_err(we)
675}
676
677fn emit_catcode_array(
678  out: &mut std::fs::File,
679  name: &str,
680  entries: &[CatcodeEntry],
681) -> Result<(), String> {
682  writeln!(out, "static {name}: &[(&str, u8)] = &[").map_err(we)?;
683  for e in entries {
684    writeln!(out, "  ({}, {}),", rust_escape(&e.key), e.val).map_err(we)?;
685  }
686  writeln!(out, "];\n").map_err(we)
687}
688
689fn emit_token_array(
690  out: &mut std::fs::File,
691  name: &str,
692  entries: &[TokenEntry],
693) -> Result<(), String> {
694  writeln!(out, "static {name}: &[(&str, u8, &str)] = &[").map_err(we)?;
695  for e in entries {
696    writeln!(
697      out,
698      "  ({}, {}, {}),",
699      rust_escape(&e.key),
700      e.cc,
701      rust_escape(&e.text)
702    )
703    .map_err(we)?;
704  }
705  writeln!(out, "];\n").map_err(we)
706}
707
708/// Emit a static array of (key, target) pairs for primitive let-aliases.
709/// The `is_math` field on `LetAliasEntry` is currently advisory only —
710/// `state::let_i` installs whatever meaning the target token has, so the
711/// runtime distinction between Primitive and MathPrimitive is handled by
712/// `Stored` unification rather than the loader.
713fn emit_let_alias_array(
714  out: &mut std::fs::File,
715  name: &str,
716  entries: &[LetAliasEntry],
717) -> Result<(), String> {
718  writeln!(out, "static {name}: &[(&str, &str)] = &[").map_err(we)?;
719  for e in entries {
720    writeln!(
721      out,
722      "  ({}, {}),",
723      rust_escape(&e.key),
724      rust_escape(&e.target)
725    )
726    .map_err(we)?;
727  }
728  writeln!(out, "];\n").map_err(we)
729}
730
731fn emit_none_array(out: &mut std::fs::File, name: &str, entries: &[String]) -> Result<(), String> {
732  writeln!(out, "static {name}: &[&str] = &[").map_err(we)?;
733  for e in entries {
734    writeln!(out, "  {},", rust_escape(e)).map_err(we)?;
735  }
736  writeln!(out, "];\n").map_err(we)
737}
738
739fn emit_token_list_array(
740  out: &mut std::fs::File,
741  name: &str,
742  entries: &[TokenListEntry],
743) -> Result<(), String> {
744  writeln!(out, "static {name}: &[(&str, u32, u16)] = &[").map_err(we)?;
745  for e in entries {
746    writeln!(
747      out,
748      "  ({}, {}, {}),",
749      rust_escape(&e.key),
750      e.tok_start,
751      e.tok_count
752    )
753    .map_err(we)?;
754  }
755  writeln!(out, "];\n").map_err(we)
756}
757
758fn emit_tok_pool(out: &mut std::fs::File, name: &str, tokens: &[TokData]) -> Result<(), String> {
759  writeln!(out, "static {name}: &[(u8, &str)] = &[").map_err(we)?;
760  for t in tokens {
761    writeln!(out, "  ({}, {}),", t.cc, rust_escape(&t.text)).map_err(we)?;
762  }
763  writeln!(out, "];\n").map_err(we)
764}
765
766fn emit_expand_array(
767  out: &mut std::fs::File,
768  name: &str,
769  entries: &[ExpandEntry],
770) -> Result<(), String> {
771  writeln!(out, "static {name}: &[ExpandDef] = &[").map_err(we)?;
772  for e in entries {
773    writeln!(
774      out,
775      "  ExpandDef {{ cs: {}, nargs: {}, flags: {}, tok_start: {}, tok_count: {} }},",
776      rust_escape(&e.cs),
777      e.nargs,
778      e.flags,
779      e.tok_start,
780      e.tok_count
781    )
782    .map_err(we)?;
783  }
784  writeln!(out, "];\n").map_err(we)
785}
786
787/// Emit the load_definitions() function that iterates over all static arrays.
788fn emit_load_fn(out: &mut std::fs::File, data: &DumpData) -> Result<(), String> {
789  let s = "  ";
790  writeln!(
791    out,
792    "/// Load the precompiled kernel definitions into the global state.\n\
793     /// Perl equivalent: LoadFormat → LoadPool(format_dump)\n\
794     pub fn load_definitions() -> latexml_core::common::error::Result<()> {{"
795  )
796  .map_err(we)?;
797
798  // Bool values
799  if !data.bools.is_empty() {
800    writeln!(
801      out,
802      "{s}for &(key, val) in BOOL_VALUES {{\n\
803       {s}{s}state::assign_value(key, Stored::Bool(val), Some(Scope::Global));\n\
804       {s}}}"
805    )
806    .map_err(we)?;
807  }
808
809  // Int values
810  if !data.ints.is_empty() {
811    writeln!(
812      out,
813      "{s}for &(key, val) in INT_VALUES {{\n\
814       {s}{s}state::assign_value(key, Stored::Int(val), Some(Scope::Global));\n\
815       {s}}}"
816    )
817    .map_err(we)?;
818  }
819
820  // String values
821  if !data.strings.is_empty() {
822    writeln!(
823      out,
824      "{s}for &(key, val) in STRING_VALUES {{\n\
825       {s}{s}state::assign_value(key, Stored::from(val), Some(Scope::Global));\n\
826       {s}}}"
827    )
828    .map_err(we)?;
829  }
830
831  // Dimension values
832  if !data.dims.is_empty() {
833    writeln!(
834      out,
835      "{s}for &(key, val) in DIMENSION_VALUES {{\n\
836       {s}{s}state::assign_value(key, Stored::Dimension(Dimension(val)), Some(Scope::Global));\n\
837       {s}}}"
838    )
839    .map_err(we)?;
840  }
841
842  // Glue values
843  if !data.glues.is_empty() {
844    writeln!(
845      out,
846      "{s}for &(key, skip, plus, pfill, minus, mfill) in GLUE_VALUES {{\n\
847       {s}{s}state::assign_value(key, Stored::Glue(Glue {{\n\
848       {s}{s}{s}skip, plus, pfill: FillCode::new(pfill as usize),\n\
849       {s}{s}{s}minus, mfill: FillCode::new(mfill as usize),\n\
850       {s}{s}}}), Some(Scope::Global));\n\
851       {s}}}"
852    )
853    .map_err(we)?;
854  }
855
856  // MuDimension values
857  if !data.mudims.is_empty() {
858    writeln!(
859      out,
860      "{s}for &(key, val) in MUDIM_VALUES {{\n\
861       {s}{s}state::assign_value(key, Stored::MuDimension(MuDimension(val)), Some(Scope::Global));\n\
862       {s}}}"
863    )
864    .map_err(we)?;
865  }
866
867  // MuGlue values
868  if !data.muglues.is_empty() {
869    writeln!(
870      out,
871      "{s}for &(key, skip, plus, pfill, minus, mfill) in MUGLUE_VALUES {{\n\
872       {s}{s}state::assign_value(key, Stored::MuGlue(MuGlue {{\n\
873       {s}{s}{s}skip, plus, pfill: FillCode::new(pfill as usize),\n\
874       {s}{s}{s}minus, mfill: FillCode::new(mfill as usize),\n\
875       {s}{s}}}), Some(Scope::Global));\n\
876       {s}}}"
877    )
878    .map_err(we)?;
879  }
880
881  // Charcode values
882  if !data.charcodes.is_empty() {
883    writeln!(
884      out,
885      "{s}for &(key, val) in CHARCODE_VALUES {{\n\
886       {s}{s}state::assign_value(key, Stored::Charcode(val), Some(Scope::Global));\n\
887       {s}}}"
888    )
889    .map_err(we)?;
890  }
891
892  // Catcode values (in value table, not catcode table)
893  if !data.catcode_vals.is_empty() {
894    writeln!(
895      out,
896      "{s}for &(key, val) in CATCODE_VALUE_ENTRIES {{\n\
897       {s}{s}state::assign_value(key, Stored::Catcode(Catcode::from(val)), Some(Scope::Global));\n\
898       {s}}}"
899    )
900    .map_err(we)?;
901  }
902
903  // Single token values
904  if !data.single_tokens.is_empty() {
905    writeln!(
906      out,
907      "{s}for &(key, cc, text) in SINGLE_TOKEN_VALUES {{\n\
908       {s}{s}let tok = Token {{ text: arena::pin(text), code: Catcode::from(cc) }};\n\
909       {s}{s}state::assign_value(key, Stored::Token(tok), Some(Scope::Global));\n\
910       {s}}}"
911    )
912    .map_err(we)?;
913  }
914
915  // Token list values
916  if !data.token_lists.is_empty() {
917    writeln!(
918      out,
919      "{s}for &(key, tok_start, tok_count) in TOKEN_LIST_VALUES {{\n\
920       {s}{s}let toks: Vec<Token> = VALUE_TOKENS[tok_start as usize..][..tok_count as usize]\n\
921       {s}{s}{s}.iter()\n\
922       {s}{s}{s}.map(|&(cc, text)| Token {{ text: arena::pin(text), code: Catcode::from(cc) }})\n\
923       {s}{s}{s}.collect();\n\
924       {s}{s}state::assign_value(key, Stored::Tokens(Tokens::from(toks)), Some(Scope::Global));\n\
925       {s}}}"
926    )
927    .map_err(we)?;
928  }
929
930  // None values
931  if !data.nones.is_empty() {
932    writeln!(
933      out,
934      "{s}for &key in NONE_VALUES {{\n\
935       {s}{s}state::assign_value(key, Stored::None, Some(Scope::Global));\n\
936       {s}}}"
937    )
938    .map_err(we)?;
939  }
940
941  // VecDeque values
942  if !data.vecdeques.is_empty() {
943    writeln!(
944      out,
945      "{s}for &key in VECDEQUE_VALUES {{\n\
946       {s}{s}state::assign_value(key, Stored::VecDequeStored(VecDeque::new()), Some(Scope::Global));\n\
947       {s}}}"
948    )
949    .map_err(we)?;
950  }
951
952  // Catcode table
953  if !data.catcodes.is_empty() {
954    writeln!(
955      out,
956      "{s}for &(key, val) in CATCODE_TABLE {{\n\
957       {s}{s}if let Some(ch) = key.chars().next() {{\n\
958       {s}{s}{s}state::assign_catcode(ch, Catcode::from(val), Some(Scope::Global));\n\
959       {s}{s}}}\n\
960       {s}}}"
961    )
962    .map_err(we)?;
963  }
964
965  // LC/UC/SF/DC/MC code tables
966  for (array_name, fn_name) in [
967    ("LCCODE_TABLE", "assign_lccode"),
968    ("UCCODE_TABLE", "assign_uccode"),
969    ("SFCODE_TABLE", "assign_sfcode"),
970    ("DELCODE_TABLE", "assign_delcode"),
971    ("MATHCODE_TABLE", "assign_mathcode"),
972  ] {
973    let entries_exist = match array_name {
974      "LCCODE_TABLE" => !data.lccodes.is_empty(),
975      "UCCODE_TABLE" => !data.uccodes.is_empty(),
976      "SFCODE_TABLE" => !data.sfcodes.is_empty(),
977      "DELCODE_TABLE" => !data.delcodes.is_empty(),
978      "MATHCODE_TABLE" => !data.mathcodes.is_empty(),
979      _ => false,
980    };
981    if entries_exist {
982      writeln!(
983        out,
984        "{s}for &(key, val) in {array_name} {{\n\
985         {s}{s}if let Some(ch) = key.chars().next() {{\n\
986         {s}{s}{s}state::{fn_name}(ch, val, Some(Scope::Global));\n\
987         {s}{s}}}\n\
988         {s}}}"
989      )
990      .map_err(we)?;
991    }
992  }
993
994  // Let definitions (M/T)
995  if !data.let_defs.is_empty() {
996    writeln!(
997      out,
998      "{s}for &(cs_name, cc, text) in LET_DEFINITIONS {{\n\
999       {s}{s}let cs = Token {{ text: arena::pin(cs_name), code: Catcode::CS }};\n\
1000       {s}{s}if !state::has_meaning(&cs) {{\n\
1001       {s}{s}{s}let tok = Token {{ text: arena::pin(text), code: Catcode::from(cc) }};\n\
1002       {s}{s}{s}state::assign_meaning(&cs, tok, Some(Scope::Global));\n\
1003       {s}{s}}}\n\
1004       {s}}}"
1005    )
1006    .map_err(we)?;
1007  }
1008
1009  // Expandable definitions (M/E) — the bulk (77% of entries)
1010  if !data.expandables.is_empty() {
1011    writeln!(
1012      out,
1013      "{s}for def in EXPANDABLE_DEFS {{\n\
1014       {s}{s}let cs = Token {{ text: arena::pin(def.cs), code: Catcode::CS }};\n\
1015       {s}{s}if state::has_meaning(&cs) {{ continue; }}\n\
1016       {s}{s}let toks: Vec<Token> = EXPAND_TOKENS[def.tok_start as usize..][..def.tok_count as usize]\n\
1017       {s}{s}{s}.iter()\n\
1018       {s}{s}{s}.map(|&(cc, text)| Token {{ text: arena::pin(text), code: Catcode::from(cc) }})\n\
1019       {s}{s}{s}.collect();\n\
1020       {s}{s}let params = if def.nargs > 0 {{\n\
1021       {s}{s}{s}let proto = \"{{}}\".repeat(def.nargs as usize);\n\
1022       {s}{s}{s}// init_flag=true: engine is up when this table is consumed\n\
1023       {s}{s}{s}// at runtime, so Parameter::init() can resolve readers\n\
1024       {s}{s}{s}// via PARAMETER_TYPES; without init the Plain reader\n\
1025       {s}{s}{s}// falls back to mock_reader and invocation fails.\n\
1026       {s}{s}{s}parse_parameters(&proto, &cs, true).ok().flatten()\n\
1027       {s}{s}}} else {{ None }};\n\
1028       {s}{s}let opts = Some(ExpandableOptions {{\n\
1029       {s}{s}{s}long: def.flags & 1 != 0,\n\
1030       {s}{s}{s}protected: def.flags & 2 != 0,\n\
1031       {s}{s}{s}nopack_parameters: true,\n\
1032       {s}{s}{s}..ExpandableOptions::default()\n\
1033       {s}{s}}});\n\
1034       {s}{s}if let Ok(exp) = Expandable::new(cs, params, Some(Tokens::from(toks).into()), opts) {{\n\
1035       {s}{s}{s}state::install_definition(exp, Some(Scope::Global));\n\
1036       {s}{s}}}\n\
1037       {s}}}"
1038    )
1039    .map_err(we)?;
1040  }
1041
1042  // Primitive-alias entries (M/PA, M/MPA). The write pass ordered these
1043  // so that `target` is either a bootstrap primitive (always present) or
1044  // an entry defined earlier in this pass. Skip self-aliases and any
1045  // alias whose key already has a meaning (add-only parity).
1046  if !data.let_aliases.is_empty() {
1047    writeln!(
1048      out,
1049      "{s}for &(key_s, target_s) in LET_ALIAS_DEFS {{\n\
1050       {s}{s}if key_s == target_s {{ continue; }}\n\
1051       {s}{s}let key_tok = Token {{ text: arena::pin(key_s), code: Catcode::CS }};\n\
1052       {s}{s}if state::has_meaning(&key_tok) {{ continue; }}\n\
1053       {s}{s}let target_tok = Token {{ text: arena::pin(target_s), code: Catcode::CS }};\n\
1054       {s}{s}if !state::has_meaning(&target_tok) {{ continue; }}\n\
1055       {s}{s}state::let_i(&key_tok, &target_tok, Some(Scope::Global));\n\
1056       {s}}}"
1057    )
1058    .map_err(we)?;
1059  }
1060
1061  writeln!(out, "\n{s}Ok(())\n}}").map_err(we)?;
1062
1063  Ok(())
1064}
latexml_core/dump_codegen.rs

latexml_core/
dump_codegen.rs
