actions/typos
1
0
Fork 0
mirror of https://github.com/crate-ci/typos.git synced 2024-11-22 09:01:04 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor(parser): Rename bytes-parser

Browse source
This commit is contained in:
Ed Page 2019-07-16 19:38:54 -06:00
parent d247d68c37
commit 6da830572a
3 changed files with 15 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
benches/tokenize.rs
View file

@ -6,34 +6,34 @@ mod data;
#[bench] #[bench]
fn symbol_parse_empty(b: &mut test::Bencher) { fn symbol_parse_empty(b: &mut test::Bencher) {
b.iter(|| typos::tokens::Identifier::parse(data::EMPTY.as_bytes()).last()); b.iter(|| typos::tokens::Identifier::parse_bytes(data::EMPTY.as_bytes()).last());
} }
#[bench] #[bench]
fn symbol_parse_no_tokens(b: &mut test::Bencher) { fn symbol_parse_no_tokens(b: &mut test::Bencher) {
b.iter(|| typos::tokens::Identifier::parse(data::NO_TOKENS.as_bytes()).last()); b.iter(|| typos::tokens::Identifier::parse_bytes(data::NO_TOKENS.as_bytes()).last());
} }
#[bench] #[bench]
fn symbol_parse_single_token(b: &mut test::Bencher) { fn symbol_parse_single_token(b: &mut test::Bencher) {
b.iter(|| { b.iter(|| {
typos::tokens::Identifier::parse(data::SINGLE_TOKEN.as_bytes()).last(); typos::tokens::Identifier::parse_bytes(data::SINGLE_TOKEN.as_bytes()).last();
}); });
} }
#[bench] #[bench]
fn symbol_parse_sherlock(b: &mut test::Bencher) { fn symbol_parse_sherlock(b: &mut test::Bencher) {
b.iter(|| typos::tokens::Identifier::parse(data::SHERLOCK.as_bytes()).last()); b.iter(|| typos::tokens::Identifier::parse_bytes(data::SHERLOCK.as_bytes()).last());
} }
#[bench] #[bench]
fn symbol_parse_code(b: &mut test::Bencher) { fn symbol_parse_code(b: &mut test::Bencher) {
b.iter(|| typos::tokens::Identifier::parse(data::CODE.as_bytes()).last()); b.iter(|| typos::tokens::Identifier::parse_bytes(data::CODE.as_bytes()).last());
} }
#[bench] #[bench]
fn symbol_parse_corpus(b: &mut test::Bencher) { fn symbol_parse_corpus(b: &mut test::Bencher) {
b.iter(|| typos::tokens::Identifier::parse(data::CORPUS.as_bytes()).last()); b.iter(|| typos::tokens::Identifier::parse_bytes(data::CORPUS.as_bytes()).last());
} }
#[bench] #[bench]

2
src/lib.rs
View file

@ -34,7 +34,7 @@ pub fn process_file(
for (line_idx, line) in buffer.lines().enumerate() { for (line_idx, line) in buffer.lines().enumerate() {
let line_num = line_idx + 1; let line_num = line_idx + 1;
for ident in tokens::Identifier::parse(line) { for ident in tokens::Identifier::parse_bytes(line) {
if !ignore_hex && is_hex(ident.token()) { if !ignore_hex && is_hex(ident.token()) {
continue; continue;
} }

16
src/tokens.rs
View file

@ -14,7 +14,7 @@ pub struct Identifier<'t> {
impl<'t> Identifier<'t> { impl<'t> Identifier<'t> {
pub fn new(token: &'t str, offset: usize) -> Result<Self, failure::Error> { pub fn new(token: &'t str, offset: usize) -> Result<Self, failure::Error> {
let mut itr = Self::parse(token.as_bytes()); let mut itr = Self::parse_bytes(token.as_bytes());
let mut item = itr let mut item = itr
.next() .next()
.ok_or_else(|| failure::format_err!("Invalid ident (none found): {:?}", token))?; .ok_or_else(|| failure::format_err!("Invalid ident (none found): {:?}", token))?;
@ -38,7 +38,7 @@ impl<'t> Identifier<'t> {
Self { token, offset } Self { token, offset }
} }
pub fn parse(content: &[u8]) -> impl Iterator<Item = Identifier<'_>> { pub fn parse_bytes(content: &[u8]) -> impl Iterator<Item = Identifier<'_>> {
lazy_static::lazy_static! { lazy_static::lazy_static! {
// Getting false positives for this lint // Getting false positives for this lint
#[allow(clippy::invalid_regex)] #[allow(clippy::invalid_regex)]
@ -242,7 +242,7 @@ mod test {
fn tokenize_empty_is_empty() { fn tokenize_empty_is_empty() {
let input = b""; let input = b"";
let expected: Vec<Identifier> = vec![]; let expected: Vec<Identifier> = vec![];
let actual: Vec<_> = Identifier::parse(input).collect(); let actual: Vec<_> = Identifier::parse_bytes(input).collect();
assert_eq!(expected, actual); assert_eq!(expected, actual);
} }
@ -250,7 +250,7 @@ mod test {
fn tokenize_word_is_word() { fn tokenize_word_is_word() {
let input = b"word"; let input = b"word";
let expected: Vec<Identifier> = vec![Identifier::new_unchecked("word", 0)]; let expected: Vec<Identifier> = vec![Identifier::new_unchecked("word", 0)];
let actual: Vec<_> = Identifier::parse(input).collect(); let actual: Vec<_> = Identifier::parse_bytes(input).collect();
assert_eq!(expected, actual); assert_eq!(expected, actual);
} }
@ -261,7 +261,7 @@ mod test {
Identifier::new_unchecked("A", 0), Identifier::new_unchecked("A", 0),
Identifier::new_unchecked("B", 2), Identifier::new_unchecked("B", 2),
]; ];
let actual: Vec<_> = Identifier::parse(input).collect(); let actual: Vec<_> = Identifier::parse_bytes(input).collect();
assert_eq!(expected, actual); assert_eq!(expected, actual);
} }
@ -272,7 +272,7 @@ mod test {
Identifier::new_unchecked("A", 0), Identifier::new_unchecked("A", 0),
Identifier::new_unchecked("B", 2), Identifier::new_unchecked("B", 2),
]; ];
let actual: Vec<_> = Identifier::parse(input).collect(); let actual: Vec<_> = Identifier::parse_bytes(input).collect();
assert_eq!(expected, actual); assert_eq!(expected, actual);
} }
@ -283,7 +283,7 @@ mod test {
Identifier::new_unchecked("A", 0), Identifier::new_unchecked("A", 0),
Identifier::new_unchecked("B", 3), Identifier::new_unchecked("B", 3),
]; ];
let actual: Vec<_> = Identifier::parse(input).collect(); let actual: Vec<_> = Identifier::parse_bytes(input).collect();
assert_eq!(expected, actual); assert_eq!(expected, actual);
} }
@ -291,7 +291,7 @@ mod test {
fn tokenize_underscore_doesnt_separate() { fn tokenize_underscore_doesnt_separate() {
let input = b"A_B"; let input = b"A_B";
let expected: Vec<Identifier> = vec![Identifier::new_unchecked("A_B", 0)]; let expected: Vec<Identifier> = vec![Identifier::new_unchecked("A_B", 0)];
let actual: Vec<_> = Identifier::parse(input).collect(); let actual: Vec<_> = Identifier::parse_bytes(input).collect();
assert_eq!(expected, actual); assert_eq!(expected, actual);
} }