latexml_core/util/char_cursor.rs
1//! A forward cursor over `&str` whose position is always a char boundary.
2//!
3//! # Why this exists
4//!
5//! `&str` carries exactly one invariant the compiler enforces — *valid UTF-8*.
6//! A `usize` used to index it carries **none**. So every `&s[a..b]` is an
7//! unchecked assertion that both ends are char boundaries, and when the
8//! assertion is wrong the program does not return an error, it **panics**.
9//!
10//! That combination is unusually hostile here. This is a Perl port, and Perl
11//! strings are sequences of *characters*: `pos`, `substr` and `\G` have no
12//! boundary concept at all. Every hand-rolled `as_bytes()` + `i += 1` scanner
13//! translated from Perl therefore introduces an invariant the original never
14//! had, silently, with no type-level trace — and passes every ASCII fixture
15//! forever. Witness 2605.22125: a `.bib` title containing `\“` aborted the whole
16//! document, and the code had been live for months.
17//!
18//! # The rule the bug teaches
19//!
20//! The panic happens at the slice, but the defect is always in the **advance**:
21//!
22//! | advance | safe? |
23//! |---|---|
24//! | scan to an ASCII delimiter | always — an ASCII byte is never a UTF-8 continuation byte |
25//! | by `char::len_utf8()` | always |
26//! | a fixed count past an unclassified byte | **never** |
27//!
28//! So the fix is not to check slices, it is to remove the ability to advance
29//! wrongly. This cursor exposes no byte-count advance at all, which makes
30//! [`slice_from`](crate::util::char_cursor::CharCursor::slice_from) infallible and the whole class
31//! unrepresentable in code written against it.
32//!
33//! Rust guidelines `anti-index-over-iter` / `perf-iter-over-index`: prefer the
34//! iterator std already provides (`char_indices`) over manual indexing. This is
35//! a thin, self-documenting wrapper over exactly that — not a new abstraction.
36//!
37//! # Cost
38//!
39//! A `Peekable<CharIndices>` and the source reference; no allocation, one pass,
40//! the same traversal a byte walker made. `char_indices` also decodes each
41//! character once instead of re-decoding at every slice.
42
43/// Forward cursor over a `&str`, positioned only ever at char boundaries.
44///
45/// # Examples
46///
47/// ```
48/// use latexml_core::util::char_cursor::CharCursor;
49///
50/// // Take a run of ASCII letters, then whatever single character follows —
51/// // even a 3-byte one. A byte walker would split it; this cannot.
52/// let mut cur = CharCursor::new("word“tail");
53/// let start = cur.pos();
54/// cur.take_while(char::is_alphanumeric);
55/// assert_eq!(cur.slice_from(start), "word");
56/// assert_eq!(cur.next(), Some('“'));
57/// ```
58pub struct CharCursor<'a> {
59 src: &'a str,
60 iter: std::iter::Peekable<std::str::CharIndices<'a>>,
61 /// Byte offset of the next character, or `src.len()` at the end. Always a
62 /// char boundary: it only ever comes from `CharIndices`.
63 pos: usize,
64}
65
66impl<'a> CharCursor<'a> {
67 /// Start at the beginning of `src`.
68 #[inline]
69 pub fn new(src: &'a str) -> Self {
70 Self {
71 src,
72 iter: src.char_indices().peekable(),
73 pos: 0,
74 }
75 }
76
77 /// Byte offset of the next character — a valid boundary, and the mark to
78 /// hand to [`slice_from`](Self::slice_from).
79 #[inline]
80 pub fn pos(&self) -> usize { self.pos }
81
82 /// The next character, without consuming it.
83 #[inline]
84 pub fn peek(&mut self) -> Option<char> { self.iter.peek().map(|&(_, c)| c) }
85
86 /// The character *after* the next one, without consuming anything.
87 ///
88 /// This is the `i + 1 < len` lookahead a byte walker spells out by hand, with
89 /// no arithmetic on indices — the arithmetic is where the bug lives.
90 #[inline]
91 pub fn peek_second(&mut self) -> Option<char> {
92 let mut probe = self.iter.clone();
93 probe.next();
94 probe.next().map(|(_, c)| c)
95 }
96
97 /// Consume and return the next character, advancing by its full width.
98 #[inline]
99 #[allow(clippy::should_implement_trait)] // deliberately not `Iterator`: see below
100 pub fn next(&mut self) -> Option<char> {
101 let (i, c) = self.iter.next()?;
102 self.pos = i + c.len_utf8();
103 Some(c)
104 }
105
106 /// Consume characters while `pred` holds.
107 #[inline]
108 pub fn take_while(&mut self, mut pred: impl FnMut(char) -> bool) {
109 while self.peek().is_some_and(&mut pred) {
110 self.next();
111 }
112 }
113
114 /// `true` once the input is exhausted.
115 #[inline]
116 pub fn is_done(&mut self) -> bool { self.peek().is_none() }
117
118 /// The text between a previous [`pos`](Self::pos) mark and the current
119 /// position.
120 ///
121 /// Infallible — both ends came from `CharIndices`, so both are char
122 /// boundaries. That is the entire point of the type.
123 ///
124 /// # Panics
125 ///
126 /// Only if `mark` did not come from this cursor's [`pos`](Self::pos), or is
127 /// ahead of the current position. Both are caller bugs, not input-dependent.
128 #[inline]
129 pub fn slice_from(&self, mark: usize) -> &'a str { &self.src[mark..self.pos] }
130
131 /// The remaining, unconsumed text.
132 #[inline]
133 pub fn rest(&self) -> &'a str { &self.src[self.pos..] }
134}
135
136// NOTE: deliberately NOT an `Iterator` impl. `Iterator` would hand callers
137// `by_ref().take_while(..)`, `zip`, `enumerate` and friends, all of which
138// consume the item that fails the predicate — the cursor's whole job is that
139// `peek`/`pos` stay in lockstep so a mark remains meaningful.
140
141#[cfg(test)]
142mod tests {
143 use super::*;
144
145 #[test]
146 fn positions_are_always_char_boundaries() {
147 // One representative per UTF-8 encoded width, interleaved with ASCII so
148 // marks land on both sides of every multi-byte character.
149 let src = "a é b “ c 𝔄 d";
150 let mut cur = CharCursor::new(src);
151 while !cur.is_done() {
152 assert!(
153 src.is_char_boundary(cur.pos()),
154 "cursor stopped at byte {} which is not a boundary of {src:?}",
155 cur.pos()
156 );
157 // The invariant that matters: slicing at any reachable position is safe.
158 let _ = cur.slice_from(0);
159 let _ = cur.rest();
160 cur.next();
161 }
162 assert_eq!(cur.pos(), src.len());
163 }
164
165 #[test]
166 fn slice_from_returns_exactly_the_consumed_text() {
167 let mut cur = CharCursor::new("“quoted” rest");
168 let start = cur.pos();
169 cur.take_while(|c| c != ' ');
170 assert_eq!(cur.slice_from(start), "“quoted”");
171 assert_eq!(cur.rest(), " rest");
172 }
173
174 #[test]
175 fn peek_does_not_advance_and_peek_second_looks_past_it() {
176 let mut cur = CharCursor::new("𝔄b");
177 assert_eq!(cur.peek(), Some('𝔄'));
178 assert_eq!(cur.peek(), Some('𝔄'), "peek must not consume");
179 assert_eq!(cur.pos(), 0, "peek must not advance");
180 assert_eq!(cur.peek_second(), Some('b'));
181 assert_eq!(cur.pos(), 0, "peek_second must not advance");
182 assert_eq!(cur.next(), Some('𝔄'));
183 assert_eq!(cur.pos(), 4, "a 4-byte char advances by 4");
184 }
185
186 #[test]
187 fn empty_and_exhausted_are_well_behaved() {
188 let mut cur = CharCursor::new("");
189 assert!(cur.is_done());
190 assert_eq!(cur.next(), None);
191 assert_eq!(cur.pos(), 0);
192 assert_eq!(cur.slice_from(0), "");
193
194 let mut cur = CharCursor::new("é");
195 assert_eq!(cur.next(), Some('é'));
196 assert_eq!(cur.next(), None, "past the end stays None");
197 assert_eq!(cur.pos(), 2, "position does not run past the end");
198 }
199}