latexml_core/common/
cleaners.rs

1use std::borrow::Cow;
2
3use once_cell::sync::Lazy;
4use regex::Regex;
5use unicode_normalization::UnicodeNormalization;
6use unidecode::unidecode;
7
8use crate::binding::def::dialect::{
9  DIRTY_ID_IDIOM_RE, LEADING_PROTOCOL_RE, NON_ID_CHARSET_RE, SPACES_RE, TILDE_NOISE_RE,
10  TRAILING_SLASH_RE,
11};
12
13static TRAILING_PUNCT_RE: Lazy<Regex> = Lazy::new(|| Regex::new(r"[\.,;]+$").unwrap());
14static NON_ALNUM_RE: Lazy<Regex> = Lazy::new(|| Regex::new(r"[^a-zA-Z0-9]").unwrap());
15
16//======================================================================
17// Cleaners
18//======================================================================
19
20static RMLETTERS: [char; 7] = ['i', 'v', 'x', 'l', 'c', 'd', 'm'];
21/// auxiliary helper for `roman`
22pub fn roman_aux<T: Into<i64>>(stuff: T) -> String {
23  let mut n: i64 = stuff.into();
24  if n <= 0 {
25    return String::new();
26  }
27  let mut div = 1000;
28  let mut s: String = if n >= div {
29    String::from_utf8(vec![b'm'; (n / div) as usize]).unwrap()
30  } else {
31    String::new()
32  };
33  let mut p = 4;
34  while n % div != 0 {
35    n %= div;
36    div /= 10;
37    let mut d = n / div;
38    if d % 5 == 4 {
39      s.push(RMLETTERS[p]);
40      d += 1;
41    }
42    if d > 4 {
43      s.push(RMLETTERS[p + (d / 5) as usize]);
44      d %= 5;
45    }
46    if d != 0 {
47      s.push_str(&String::from_utf8(vec![RMLETTERS[p] as u8; d as usize]).unwrap());
48    }
49    // silly, but i'm postponing rewriting the entire method for now, just porting over from Perl
50    if p > 2 {
51      p -= 2;
52    } else {
53      p = 0;
54    }
55  }
56  s
57}
58
59/// cleans a string down to characters acceptable for an id attribute
60pub fn clean_id(key: &str) -> String {
61  let cleaned = Cow::Borrowed(key.trim()); // Trim leading/trailing whitespace
62  let cleaned_1 = SPACES_RE.replace_all(&cleaned, ""); // remove all spaces
63  // Remove common idiom:
64  // Perl parity: CleanID strips `${}^{foo}$` down to just `foo`.
65  // The regex captures that inner content as named group `label`; the
66  // replacement must reference it by the correct name (`$inner` was a
67  // stale typo that silently erased the captured text).
68  let cleaned_2 = DIRTY_ID_IDIOM_RE.replace_all(&cleaned_1, "$label");
69  // transform some forbidden chars
70  let cleaned_3 = cleaned_2
71    .replace(':', "..") // No colons!
72    .replace('@', "-at-")
73    .replace('*', "-star-")
74    .replace('$', "-dollar-")
75    .replace(',', "-comma-")
76    .replace('%', "-pct-")
77    .replace('&', "-amp-");
78  let cleaned_4 = unidecode(&cleaned_3);
79  let cleaned_5 = NON_ID_CHARSET_RE.replace_all(&cleaned_4, ""); // remove everything else.
80  let out = cleaned_5.as_ref();
81  // Perl parity (Package.pm CleanID): XML ids must start with a letter or `_`
82  // (since we already replaced `:` with `..`). Prepend "X" when the cleaned
83  // key starts with anything else — protects against leading `.`, `-`, or
84  // digits, which would otherwise produce invalid id attributes.
85  match out.chars().next() {
86    Some(c) if c.is_ascii_alphabetic() || c == '_' => out.to_string(),
87    Some(_) => format!("X{out}"),
88    None => String::new(),
89  }
90}
91/// cleans a string down to characters acceptable for a label attribute
92pub fn clean_label<'a>(label: &'a str, prefix_opt: Option<&str>) -> Cow<'a, str> {
93  let key = label.trim(); // Trim leading/trailing, in any case
94  let cleaned_1 = SPACES_RE.replace_all(key, "_"); // spaces to underscores
95  let prefix = prefix_opt.unwrap_or("LABEL");
96  if prefix.is_empty() {
97    cleaned_1
98  } else {
99    Cow::Owned(s!("{}:{}", prefix, cleaned_1))
100  }
101}
102
103/// Clean string for use in index keys (Perl: CleanIndexKey)
104/// Applies NFC normalization and removes trailing punctuation.
105pub fn clean_index_key(key: &str) -> String {
106  let trimmed = key.trim();
107  let normalized: String = trimmed.nfc().collect();
108  TRAILING_PUNCT_RE.replace(&normalized, "").to_string()
109}
110
111/// Clean string for use as a CSS class name (Perl: CleanClassName)
112/// Decomposes to NFD, removes non-alphanumeric chars, recomposes to NFC.
113pub fn clean_class_name(key: &str) -> String {
114  let trimmed = key.trim();
115  let decomposed: String = trimmed.nfd().collect();
116  let cleaned = NON_ALNUM_RE.replace_all(&decomposed, "");
117  cleaned.nfc().collect()
118}
119
120/// cleans a string down to characters acceptable for a bibliography key
121pub fn clean_bib_key(key: &str) -> String {
122  // Originally lc() here, but let's preserve case till Postproc.
123  let trimmed = key.trim();
124  SPACES_RE.replace_all(trimmed, "").to_string()
125}
126
127/// Return the bibkey in a form to ACTUALLY lookup (Perl: NormalizeBibKey)
128/// Usually use clean_bib_key to preserve key in the original form (case)
129pub fn normalize_bib_key(key: &str) -> String { clean_bib_key(key).to_lowercase() }
130
131/// Split comma-separated text into trimmed tokens (Perl: TrimmedCommaList)
132pub fn trimmed_comma_list(text: &str) -> Vec<String> {
133  let trimmed = text.trim();
134  if trimmed.is_empty() {
135    return Vec::new();
136  }
137  trimmed.split(',').map(|s| s.trim().to_string()).collect()
138}
139
140/// cleans a string down to characters acceptable for a URL
141pub fn clean_url(url: &str) -> String {
142  let cleaned = url.trim(); // Trim leading/trailing whitespace
143  TILDE_NOISE_RE.replace_all(cleaned, "~").to_string()
144}
145
146/// builds a complete url from fragments
147pub fn compose_url(base: &str, url: &str, fragid_opt: Option<&str>) -> String {
148  let base = TRAILING_SLASH_RE.replace(base, ""); //  remove trailing /
149  let fragid = fragid_opt.unwrap_or("");
150  let base: String = if !base.is_empty() && !LEADING_PROTOCOL_RE.is_match(url) {
151    // already has protocol, so is absolute url
152    base.to_string() + if url.starts_with('/') { "" } else { "/" } // else start w/base, possibly /
153  } else {
154    String::new()
155  };
156  let fragid: String = if !fragid.is_empty() {
157    s!("#{}", clean_id(fragid))
158  } else {
159    String::new()
160  };
161  clean_url(&(base + url + &fragid))
162}
163
164#[cfg(test)]
165mod tests {
166  use super::*;
167
168  #[test]
169  fn clean_id_preserves_alpha_start() {
170    assert_eq!(clean_id("foo"), "foo");
171    assert_eq!(clean_id("Foo_bar"), "Foo_bar");
172    assert_eq!(clean_id("_underscore"), "_underscore");
173  }
174
175  #[test]
176  fn clean_id_prepends_x_for_non_alpha_start() {
177    // Leading digit, dot, or hyphen is invalid in XML ids — prepend X.
178    assert_eq!(clean_id("1foo"), "X1foo");
179    assert_eq!(clean_id(".foo"), "X.foo");
180    assert_eq!(clean_id("-foo"), "X-foo");
181  }
182
183  #[test]
184  fn clean_id_after_colon_replacement() {
185    // `:` becomes `..`, so ":foo" → "..foo" → needs X prefix.
186    assert_eq!(clean_id(":foo"), "X..foo");
187  }
188
189  #[test]
190  fn clean_id_empty_stays_empty() {
191    assert_eq!(clean_id(""), "");
192    assert_eq!(clean_id("   "), "");
193  }
194
195  #[test]
196  fn clean_id_dirty_idiom_preserves_label() {
197    // Perl: $key =~ s/\$\{\}\^\{(.*?)\}\$/$1/g; retains the captured
198    // content. Common TeX idiom from latex generates ${}^{foo}$.
199    assert_eq!(clean_id("${}^{foo}$"), "foo");
200    assert_eq!(clean_id("bar${}^{tag}$"), "bartag");
201  }
202
203  #[test]
204  fn roman_aux_non_positive() {
205    assert_eq!(roman_aux(0i64), "");
206    assert_eq!(roman_aux(-1i64), "");
207  }
208
209  #[test]
210  fn roman_aux_basic() {
211    assert_eq!(roman_aux(1i64), "i");
212    assert_eq!(roman_aux(1000i64), "m");
213    assert_eq!(roman_aux(1999i64), "mcmxcix");
214  }
215
216  #[test]
217  fn clean_label_default_prefix() {
218    // Spaces become underscores; default prefix is "LABEL:".
219    assert_eq!(clean_label("foo bar", None), "LABEL:foo_bar");
220    assert_eq!(clean_label("simple", None), "LABEL:simple");
221  }
222
223  #[test]
224  fn clean_label_custom_prefix() {
225    assert_eq!(clean_label("thm:main", Some("REF")), "REF:thm:main");
226  }
227
228  #[test]
229  fn clean_label_empty_prefix_skips_colon() {
230    // Empty prefix means no prefix at all (not an empty-prefix colon).
231    let out = clean_label("foo bar", Some(""));
232    assert_eq!(out, "foo_bar");
233  }
234
235  #[test]
236  fn clean_label_trims_whitespace() {
237    // Leading/trailing whitespace trimmed before space-to-underscore.
238    assert_eq!(clean_label("  foo  ", None), "LABEL:foo");
239  }
240
241  #[test]
242  fn clean_class_name_basic() {
243    // clean_class_name strips spaces, converts to lowercase, removes
244    // non-class-safe chars.
245    let out = clean_class_name("foo");
246    assert!(out.contains("foo"), "got {out:?}");
247  }
248
249  #[test]
250  fn clean_bib_key_basic() {
251    // Bib keys are case-preserved but trimmed/cleaned.
252    let out = clean_bib_key("Author:2020");
253    assert!(!out.is_empty());
254  }
255
256  #[test]
257  fn normalize_bib_key_case_insensitive() {
258    // normalize_bib_key should produce the same output for case variants.
259    let a = normalize_bib_key("Author2020");
260    let b = normalize_bib_key("AUTHOR2020");
261    assert_eq!(a, b, "normalize_bib_key folds case (got {a:?} vs {b:?})");
262  }
263
264  #[test]
265  fn trimmed_comma_list_basic() {
266    let out = trimmed_comma_list("a, b ,c,  d");
267    assert_eq!(out, vec!["a", "b", "c", "d"]);
268  }
269
270  #[test]
271  fn trimmed_comma_list_handles_empty_segments() {
272    // Leading/trailing/internal empty comma positions — behavior may
273    // retain empty tokens or drop them depending on the implementation;
274    // just assert consistency with non-empty entries.
275    let out = trimmed_comma_list(",a,,b,");
276    assert!(
277      out.contains(&"a".to_string()) && out.contains(&"b".to_string()),
278      "got {out:?}"
279    );
280  }
281
282  #[test]
283  fn clean_url_removes_quotes_and_whitespace() {
284    // clean_url is lenient — it trims and normalizes. At minimum it
285    // must not break a well-formed URL.
286    let canonical = "http://example.com/path";
287    assert_eq!(clean_url(canonical), canonical);
288  }
289
290  #[test]
291  fn clean_index_key_trims_trailing_punct() {
292    // Per docstring: Applies NFC + strips trailing punctuation.
293    let out = clean_index_key("topic.");
294    assert_eq!(out, "topic", "got {out:?}");
295  }
296}
latexml_core/common/cleaners.rs

latexml_core/common/
cleaners.rs
