diff --git a/crates/typos/src/tokens.rs b/crates/typos/src/tokens.rs index 3c4afdf..c36db17 100644 --- a/crates/typos/src/tokens.rs +++ b/crates/typos/src/tokens.rs @@ -374,7 +374,7 @@ mod parser { ::Item: AsChar + Copy, ::Item: AsChar + Copy, { - preceded(char('\\'), take_while(is_xid_continue))(input) + preceded(take_while1(is_escape), take_while(is_xid_continue))(input) } fn printf(input: T) -> IResult @@ -496,6 +496,12 @@ mod parser { unicode_xid::UnicodeXID::is_xid_continue(c) } + #[inline] + fn is_escape(i: impl AsChar + Copy) -> bool { + let c = i.as_char(); + c == '\\' + } + #[inline] fn is_digit_sep(chr: char) -> bool { // `_`: number literal separator in Rust and other languages @@ -990,10 +996,10 @@ mod test { fn tokenize_c_escape() { let parser = TokenizerBuilder::new().build(); - let input = "Hello \\Hello \\ World"; + let input = "Hello \\Hello \\ \\\\ World"; let expected: Vec = vec![ Identifier::new_unchecked("Hello", Case::None, 0), - Identifier::new_unchecked("World", Case::None, 15), + Identifier::new_unchecked("World", Case::None, 18), ]; let actual: Vec<_> = parser.parse_bytes(input.as_bytes()).collect(); assert_eq!(expected, actual);