actions/typos
1
0
Fork 0
mirror of https://github.com/crate-ci/typos.git synced 2025-01-25 07:58:58 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor(typos): Switch to BStr for better debugging

Browse source
This commit is contained in:
Ed Page 2023-07-14 13:28:36 -05:00
parent 4fd4537856
commit 6f40717c8f
1 changed files with 8 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
crates/typos/src/tokens.rs
View file

@ -1,4 +1,5 @@
use bstr::ByteSlice; use bstr::ByteSlice;
use winnow::BStr;
/// Define rules for tokenizaing a buffer. /// Define rules for tokenizaing a buffer.
#[derive(Debug, Clone, PartialEq, Eq, Hash)] #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@ -48,7 +49,9 @@ impl Tokenizer {
let iter = if self.unicode && !ByteSlice::is_ascii(content.as_bytes()) { let iter = if self.unicode && !ByteSlice::is_ascii(content.as_bytes()) {
itertools::Either::Left(unicode_parser::iter_identifiers(content)) itertools::Either::Left(unicode_parser::iter_identifiers(content))
} else { } else {
itertools::Either::Right(ascii_parser::iter_identifiers(content.as_bytes())) itertools::Either::Right(ascii_parser::iter_identifiers(BStr::new(
content.as_bytes(),
)))
}; };
iter.map(move |identifier| self.transform(identifier, content.as_bytes())) iter.map(move |identifier| self.transform(identifier, content.as_bytes()))
} }
@ -58,7 +61,7 @@ impl Tokenizer {
let iter = Utf8Chunks::new(content).flat_map(unicode_parser::iter_identifiers); let iter = Utf8Chunks::new(content).flat_map(unicode_parser::iter_identifiers);
itertools::Either::Left(iter) itertools::Either::Left(iter)
} else { } else {
itertools::Either::Right(ascii_parser::iter_identifiers(content)) itertools::Either::Right(ascii_parser::iter_identifiers(BStr::new(content)))
}; };
iter.map(move |identifier| self.transform(identifier, content)) iter.map(move |identifier| self.transform(identifier, content))
} }
@ -630,7 +633,9 @@ mod unicode_parser {
mod ascii_parser { mod ascii_parser {
use super::parser::next_identifier; use super::parser::next_identifier;
pub(crate) fn iter_identifiers(mut input: &[u8]) -> impl Iterator<Item = &str> { use winnow::BStr;
pub(crate) fn iter_identifiers(mut input: &BStr) -> impl Iterator<Item = &str> {
std::iter::from_fn(move || match next_identifier(input) { std::iter::from_fn(move || match next_identifier(input) {
Ok((i, o)) => { Ok((i, o)) => {
input = i; input = i;