actions/typos
1
0
Fork 0
mirror of https://github.com/crate-ci/typos.git synced 2024-11-25 02:20:58 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor(parser): Rename bytes-parser

Browse source
This commit is contained in:
Ed Page 2019-07-16 19:38:54 -06:00
parent d247d68c37
commit 6da830572a
3 changed files with 15 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
benches/tokenize.rs
View file

@ -6,34 +6,34 @@ mod data;
#[bench]
fn symbol_parse_empty(b: &mut test::Bencher) {
b.iter(|| typos::tokens::Identifier::parse(data::EMPTY.as_bytes()).last());
b.iter(|| typos::tokens::Identifier::parse_bytes(data::EMPTY.as_bytes()).last());
}
#[bench]
fn symbol_parse_no_tokens(b: &mut test::Bencher) {
b.iter(|| typos::tokens::Identifier::parse(data::NO_TOKENS.as_bytes()).last());
b.iter(|| typos::tokens::Identifier::parse_bytes(data::NO_TOKENS.as_bytes()).last());
}
#[bench]
fn symbol_parse_single_token(b: &mut test::Bencher) {
b.iter(|| {
typos::tokens::Identifier::parse(data::SINGLE_TOKEN.as_bytes()).last();
typos::tokens::Identifier::parse_bytes(data::SINGLE_TOKEN.as_bytes()).last();
});
}
#[bench]
fn symbol_parse_sherlock(b: &mut test::Bencher) {
b.iter(|| typos::tokens::Identifier::parse(data::SHERLOCK.as_bytes()).last());
b.iter(|| typos::tokens::Identifier::parse_bytes(data::SHERLOCK.as_bytes()).last());
}
#[bench]
fn symbol_parse_code(b: &mut test::Bencher) {
b.iter(|| typos::tokens::Identifier::parse(data::CODE.as_bytes()).last());
b.iter(|| typos::tokens::Identifier::parse_bytes(data::CODE.as_bytes()).last());
}
#[bench]
fn symbol_parse_corpus(b: &mut test::Bencher) {
b.iter(|| typos::tokens::Identifier::parse(data::CORPUS.as_bytes()).last());
b.iter(|| typos::tokens::Identifier::parse_bytes(data::CORPUS.as_bytes()).last());
}
#[bench]

2
src/lib.rs
View file

@ -34,7 +34,7 @@ pub fn process_file(
for (line_idx, line) in buffer.lines().enumerate() {
let line_num = line_idx + 1;
for ident in tokens::Identifier::parse(line) {
for ident in tokens::Identifier::parse_bytes(line) {
if !ignore_hex && is_hex(ident.token()) {
continue;
}

16
src/tokens.rs
View file

@ -14,7 +14,7 @@ pub struct Identifier<'t> {
impl<'t> Identifier<'t> {
pub fn new(token: &'t str, offset: usize) -> Result<Self, failure::Error> {
let mut itr = Self::parse(token.as_bytes());
let mut itr = Self::parse_bytes(token.as_bytes());
let mut item = itr
.next()
.ok_or_else(|| failure::format_err!("Invalid ident (none found): {:?}", token))?;
@ -38,7 +38,7 @@ impl<'t> Identifier<'t> {
Self { token, offset }
}
pub fn parse(content: &[u8]) -> impl Iterator<Item = Identifier<'_>> {
pub fn parse_bytes(content: &[u8]) -> impl Iterator<Item = Identifier<'_>> {
lazy_static::lazy_static! {
// Getting false positives for this lint
#[allow(clippy::invalid_regex)]
@ -242,7 +242,7 @@ mod test {
fn tokenize_empty_is_empty() {
let input = b"";
let expected: Vec<Identifier> = vec![];
let actual: Vec<_> = Identifier::parse(input).collect();
let actual: Vec<_> = Identifier::parse_bytes(input).collect();
assert_eq!(expected, actual);
}
@ -250,7 +250,7 @@ mod test {
fn tokenize_word_is_word() {
let input = b"word";
let expected: Vec<Identifier> = vec![Identifier::new_unchecked("word", 0)];
let actual: Vec<_> = Identifier::parse(input).collect();
let actual: Vec<_> = Identifier::parse_bytes(input).collect();
assert_eq!(expected, actual);
}
@ -261,7 +261,7 @@ mod test {
Identifier::new_unchecked("A", 0),
Identifier::new_unchecked("B", 2),
];
let actual: Vec<_> = Identifier::parse(input).collect();
let actual: Vec<_> = Identifier::parse_bytes(input).collect();
assert_eq!(expected, actual);
}
@ -272,7 +272,7 @@ mod test {
Identifier::new_unchecked("A", 0),
Identifier::new_unchecked("B", 2),
];
let actual: Vec<_> = Identifier::parse(input).collect();
let actual: Vec<_> = Identifier::parse_bytes(input).collect();
assert_eq!(expected, actual);
}
@ -283,7 +283,7 @@ mod test {
Identifier::new_unchecked("A", 0),
Identifier::new_unchecked("B", 3),
];
let actual: Vec<_> = Identifier::parse(input).collect();
let actual: Vec<_> = Identifier::parse_bytes(input).collect();
assert_eq!(expected, actual);
}
@ -291,7 +291,7 @@ mod test {
fn tokenize_underscore_doesnt_separate() {
let input = b"A_B";
let expected: Vec<Identifier> = vec![Identifier::new_unchecked("A_B", 0)];
let actual: Vec<_> = Identifier::parse(input).collect();
let actual: Vec<_> = Identifier::parse_bytes(input).collect();
assert_eq!(expected, actual);
}