actions/typos
1
0
Fork 0
mirror of https://github.com/crate-ci/typos.git synced 2024-11-25 10:31:02 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(parser): Don't stop on almost-printfs

Browse source 
When we added support for printf interopolation, we had to adjust our
separator matching to not eat the start of printf interpolation.

When doing so, I overlooked the need to still eat it in the catch-all.
If we don't, we then try to read `%` as part of the identifier and bail
out early.

Fixes #411
This commit is contained in:
Ed Page 2022-01-26 09:39:20 -06:00
parent 4b2e66487c
commit 3c78d65462
1 changed files with 59 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

77
crates/typos/src/tokens.rs
View file

@ -152,7 +152,7 @@ mod parser {
fn identifier<T>(input: T) -> IResult<T, T>
where
T: nom::InputTakeAtPosition,
T: nom::InputTakeAtPosition + std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
{
// Generally a language would be `{XID_Start}{XID_Continue}*` but going with only
@ -191,18 +191,39 @@ mod parser {
terminated(url_literal, sep1),
c_escape,
printf,
sep1,
other,
)))(input)
}
fn sep1<T>(input: T) -> IResult<T, T>
where
T: nom::InputTakeAtPosition,
T: nom::InputTakeAtPosition + std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
{
take_while1(is_ignore_char)(input)
}
fn other<T>(input: T) -> IResult<T, T>
where
T: nom::InputTakeAtPosition
+ nom::InputTake
+ nom::InputIter
+ nom::InputLength
+ nom::Slice<std::ops::RangeFrom<usize>>
+ nom::Slice<std::ops::RangeTo<usize>>
+ nom::Offset
+ Clone
+ PartialEq
+ std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
<T as nom::InputIter>::Item: AsChar + Copy,
{
recognize(tuple((
satisfy(|c| !is_xid_continue(c)),
take_while(is_ignore_char),
)))(input)
}
fn ordinal_literal<T>(input: T) -> IResult<T, T>
where
T: nom::InputTakeAtPosition
@ -212,7 +233,8 @@ mod parser {
+ nom::Offset
+ nom::Slice<std::ops::RangeTo<usize>>
+ nom::Slice<std::ops::RangeFrom<usize>>
+ Clone,
+ Clone
+ std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
<T as nom::InputIter>::Item: AsChar + Copy,
{
@ -237,7 +259,7 @@ mod parser {
fn dec_literal<T>(input: T) -> IResult<T, T>
where
T: nom::InputTakeAtPosition,
T: nom::InputTakeAtPosition + std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
{
take_while1(is_dec_digit_with_sep)(input)
@ -250,7 +272,8 @@ mod parser {
+ nom::InputIter
+ nom::InputLength
+ nom::Slice<std::ops::RangeFrom<usize>>
+ Clone,
+ Clone
+ std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
<T as nom::InputIter>::Item: AsChar + Copy,
{
@ -269,7 +292,8 @@ mod parser {
+ nom::Offset
+ nom::Slice<std::ops::RangeTo<usize>>
+ nom::Slice<std::ops::RangeFrom<usize>>
+ Clone,
+ Clone
+ std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
<T as nom::InputIter>::Item: AsChar + Copy,
{
@ -295,7 +319,8 @@ mod parser {
+ nom::Offset
+ nom::Slice<std::ops::RangeTo<usize>>
+ nom::Slice<std::ops::RangeFrom<usize>>
+ Clone,
+ Clone
+ std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
<T as nom::InputIter>::Item: AsChar + Copy,
{
@ -325,8 +350,8 @@ mod parser {
+ nom::Offset
+ nom::Slice<std::ops::RangeTo<usize>>
+ nom::Slice<std::ops::RangeFrom<usize>>
+ std::fmt::Debug
+ Clone,
+ Clone
+ std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
<T as nom::InputIter>::Item: AsChar + Copy,
{
@ -359,8 +384,8 @@ mod parser {
+ nom::Offset
+ nom::Slice<std::ops::RangeTo<usize>>
+ nom::Slice<std::ops::RangeFrom<usize>>
+ std::fmt::Debug
+ Clone,
+ Clone
+ std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
<T as nom::InputIter>::Item: AsChar + Copy,
{
@ -380,8 +405,8 @@ mod parser {
+ nom::Offset
+ nom::Slice<std::ops::RangeTo<usize>>
+ nom::Slice<std::ops::RangeFrom<usize>>
+ std::fmt::Debug
+ Clone,
+ Clone
+ std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
<T as nom::InputIter>::Item: AsChar + Copy,
{
@ -412,8 +437,8 @@ mod parser {
+ nom::Offset
+ nom::Slice<std::ops::RangeTo<usize>>
+ nom::Slice<std::ops::RangeFrom<usize>>
+ std::fmt::Debug
+ Clone,
+ Clone
+ std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
<T as nom::InputIter>::Item: AsChar + Copy,
{
@ -433,8 +458,8 @@ mod parser {
+ nom::Offset
+ nom::Slice<std::ops::RangeTo<usize>>
+ nom::Slice<std::ops::RangeFrom<usize>>
+ std::fmt::Debug
+ Clone,
+ Clone
+ std::fmt::Debug,
<T as nom::InputTakeAtPosition>::Item: AsChar + Copy,
<T as nom::InputIter>::Item: AsChar + Copy,
{
@ -1159,6 +1184,22 @@ mod test {
assert_eq!(expected, actual);
}
#[test]
fn tokenize_template() {
let parser = TokenizerBuilder::new().build();
let input = "Hello {{% foo %}} world!";
let expected: Vec<Identifier> = vec![
Identifier::new_unchecked("Hello", Case::None, 0),
Identifier::new_unchecked("foo", Case::None, 10),
Identifier::new_unchecked("world", Case::None, 18),
];
let actual: Vec<_> = parser.parse_bytes(input.as_bytes()).collect();
assert_eq!(expected, actual);
let actual: Vec<_> = parser.parse_str(input).collect();
assert_eq!(expected, actual);
}
#[test]
fn split_ident() {
let cases = [