diff --git a/crates/typos/src/tokens.rs b/crates/typos/src/tokens.rs index da61571..0ccb5c2 100644 --- a/crates/typos/src/tokens.rs +++ b/crates/typos/src/tokens.rs @@ -152,7 +152,7 @@ mod parser { fn identifier(input: T) -> IResult where - T: nom::InputTakeAtPosition, + T: nom::InputTakeAtPosition + std::fmt::Debug, ::Item: AsChar + Copy, { // Generally a language would be `{XID_Start}{XID_Continue}*` but going with only @@ -191,18 +191,39 @@ mod parser { terminated(url_literal, sep1), c_escape, printf, - sep1, + other, )))(input) } fn sep1(input: T) -> IResult where - T: nom::InputTakeAtPosition, + T: nom::InputTakeAtPosition + std::fmt::Debug, ::Item: AsChar + Copy, { take_while1(is_ignore_char)(input) } + fn other(input: T) -> IResult + where + T: nom::InputTakeAtPosition + + nom::InputTake + + nom::InputIter + + nom::InputLength + + nom::Slice> + + nom::Slice> + + nom::Offset + + Clone + + PartialEq + + std::fmt::Debug, + ::Item: AsChar + Copy, + ::Item: AsChar + Copy, + { + recognize(tuple(( + satisfy(|c| !is_xid_continue(c)), + take_while(is_ignore_char), + )))(input) + } + fn ordinal_literal(input: T) -> IResult where T: nom::InputTakeAtPosition @@ -212,7 +233,8 @@ mod parser { + nom::Offset + nom::Slice> + nom::Slice> - + Clone, + + Clone + + std::fmt::Debug, ::Item: AsChar + Copy, ::Item: AsChar + Copy, { @@ -237,7 +259,7 @@ mod parser { fn dec_literal(input: T) -> IResult where - T: nom::InputTakeAtPosition, + T: nom::InputTakeAtPosition + std::fmt::Debug, ::Item: AsChar + Copy, { take_while1(is_dec_digit_with_sep)(input) @@ -250,7 +272,8 @@ mod parser { + nom::InputIter + nom::InputLength + nom::Slice> - + Clone, + + Clone + + std::fmt::Debug, ::Item: AsChar + Copy, ::Item: AsChar + Copy, { @@ -269,7 +292,8 @@ mod parser { + nom::Offset + nom::Slice> + nom::Slice> - + Clone, + + Clone + + std::fmt::Debug, ::Item: AsChar + Copy, ::Item: AsChar + Copy, { @@ -295,7 +319,8 @@ mod parser { + nom::Offset + nom::Slice> + nom::Slice> - + Clone, + + Clone + + std::fmt::Debug, ::Item: AsChar + Copy, ::Item: AsChar + Copy, { @@ -325,8 +350,8 @@ mod parser { + nom::Offset + nom::Slice> + nom::Slice> - + std::fmt::Debug - + Clone, + + Clone + + std::fmt::Debug, ::Item: AsChar + Copy, ::Item: AsChar + Copy, { @@ -359,8 +384,8 @@ mod parser { + nom::Offset + nom::Slice> + nom::Slice> - + std::fmt::Debug - + Clone, + + Clone + + std::fmt::Debug, ::Item: AsChar + Copy, ::Item: AsChar + Copy, { @@ -380,8 +405,8 @@ mod parser { + nom::Offset + nom::Slice> + nom::Slice> - + std::fmt::Debug - + Clone, + + Clone + + std::fmt::Debug, ::Item: AsChar + Copy, ::Item: AsChar + Copy, { @@ -412,8 +437,8 @@ mod parser { + nom::Offset + nom::Slice> + nom::Slice> - + std::fmt::Debug - + Clone, + + Clone + + std::fmt::Debug, ::Item: AsChar + Copy, ::Item: AsChar + Copy, { @@ -433,8 +458,8 @@ mod parser { + nom::Offset + nom::Slice> + nom::Slice> - + std::fmt::Debug - + Clone, + + Clone + + std::fmt::Debug, ::Item: AsChar + Copy, ::Item: AsChar + Copy, { @@ -1159,6 +1184,22 @@ mod test { assert_eq!(expected, actual); } + #[test] + fn tokenize_template() { + let parser = TokenizerBuilder::new().build(); + + let input = "Hello {{% foo %}} world!"; + let expected: Vec = vec![ + Identifier::new_unchecked("Hello", Case::None, 0), + Identifier::new_unchecked("foo", Case::None, 10), + Identifier::new_unchecked("world", Case::None, 18), + ]; + let actual: Vec<_> = parser.parse_bytes(input.as_bytes()).collect(); + assert_eq!(expected, actual); + let actual: Vec<_> = parser.parse_str(input).collect(); + assert_eq!(expected, actual); + } + #[test] fn split_ident() { let cases = [