diff --git a/benches/corrections.rs b/benches/corrections.rs index b849986..8e0d751 100644 --- a/benches/corrections.rs +++ b/benches/corrections.rs @@ -4,12 +4,12 @@ extern crate test; #[bench] fn load_corrections(b: &mut test::Bencher) { - b.iter(|| typos::Dictionary::new()); + b.iter(|| typos::BuiltIn::new()); } #[bench] fn correct_word_hit(b: &mut test::Bencher) { - let corrections = typos::Dictionary::new(); + let corrections = typos::BuiltIn::new(); let input = typos::tokens::Word::new("successs", 0).unwrap(); assert_eq!( corrections.correct_word(input), @@ -20,7 +20,7 @@ fn correct_word_hit(b: &mut test::Bencher) { #[bench] fn correct_word_miss(b: &mut test::Bencher) { - let corrections = typos::Dictionary::new(); + let corrections = typos::BuiltIn::new(); let input = typos::tokens::Word::new("success", 0).unwrap(); assert_eq!(corrections.correct_word(input), None); b.iter(|| corrections.correct_word(input)); diff --git a/benches/file.rs b/benches/file.rs index 5f69285..0783e86 100644 --- a/benches/file.rs +++ b/benches/file.rs @@ -12,18 +12,10 @@ fn process_empty(b: &mut test::Bencher) { let sample_path = temp.child("sample"); sample_path.write_str(data::EMPTY).unwrap(); - let corrections = typos::Dictionary::new(); - b.iter(|| { - typos::process_file( - sample_path.path(), - &corrections, - true, - true, - true, - false, - typos::report::print_silent, - ) - }); + let corrections = typos::BuiltIn::new(); + let parser = typos::tokens::Parser::new(); + let checks = typos::checks::CheckSettings::new().build(&corrections, &parser); + b.iter(|| checks.check_file(sample_path.path(), typos::report::print_silent)); temp.close().unwrap(); } @@ -34,18 +26,10 @@ fn process_no_tokens(b: &mut test::Bencher) { let sample_path = temp.child("sample"); sample_path.write_str(data::NO_TOKENS).unwrap(); - let corrections = typos::Dictionary::new(); - b.iter(|| { - typos::process_file( - sample_path.path(), - &corrections, - true, - true, - true, - false, - typos::report::print_silent, - ) - }); + let corrections = typos::BuiltIn::new(); + let parser = typos::tokens::Parser::new(); + let checks = typos::checks::CheckSettings::new().build(&corrections, &parser); + b.iter(|| checks.check_file(sample_path.path(), typos::report::print_silent)); temp.close().unwrap(); } @@ -56,18 +40,10 @@ fn process_single_token(b: &mut test::Bencher) { let sample_path = temp.child("sample"); sample_path.write_str(data::SINGLE_TOKEN).unwrap(); - let corrections = typos::Dictionary::new(); - b.iter(|| { - typos::process_file( - sample_path.path(), - &corrections, - true, - true, - true, - false, - typos::report::print_silent, - ) - }); + let corrections = typos::BuiltIn::new(); + let parser = typos::tokens::Parser::new(); + let checks = typos::checks::CheckSettings::new().build(&corrections, &parser); + b.iter(|| checks.check_file(sample_path.path(), typos::report::print_silent)); temp.close().unwrap(); } @@ -78,18 +54,10 @@ fn process_sherlock(b: &mut test::Bencher) { let sample_path = temp.child("sample"); sample_path.write_str(data::SHERLOCK).unwrap(); - let corrections = typos::Dictionary::new(); - b.iter(|| { - typos::process_file( - sample_path.path(), - &corrections, - true, - true, - true, - false, - typos::report::print_silent, - ) - }); + let corrections = typos::BuiltIn::new(); + let parser = typos::tokens::Parser::new(); + let checks = typos::checks::CheckSettings::new().build(&corrections, &parser); + b.iter(|| checks.check_file(sample_path.path(), typos::report::print_silent)); temp.close().unwrap(); } @@ -100,18 +68,10 @@ fn process_code(b: &mut test::Bencher) { let sample_path = temp.child("sample"); sample_path.write_str(data::CODE).unwrap(); - let corrections = typos::Dictionary::new(); - b.iter(|| { - typos::process_file( - sample_path.path(), - &corrections, - true, - true, - true, - false, - typos::report::print_silent, - ) - }); + let corrections = typos::BuiltIn::new(); + let parser = typos::tokens::Parser::new(); + let checks = typos::checks::CheckSettings::new().build(&corrections, &parser); + b.iter(|| checks.check_file(sample_path.path(), typos::report::print_silent)); temp.close().unwrap(); } @@ -122,18 +82,10 @@ fn process_corpus(b: &mut test::Bencher) { let sample_path = temp.child("sample"); sample_path.write_str(data::CORPUS).unwrap(); - let corrections = typos::Dictionary::new(); - b.iter(|| { - typos::process_file( - sample_path.path(), - &corrections, - true, - true, - true, - false, - typos::report::print_silent, - ) - }); + let corrections = typos::BuiltIn::new(); + let parser = typos::tokens::Parser::new(); + let checks = typos::checks::CheckSettings::new().build(&corrections, &parser); + b.iter(|| checks.check_file(sample_path.path(), typos::report::print_silent)); temp.close().unwrap(); } diff --git a/benches/tokenize.rs b/benches/tokenize.rs index 75b1b9c..dea743a 100644 --- a/benches/tokenize.rs +++ b/benches/tokenize.rs @@ -6,60 +6,66 @@ mod data; #[bench] fn symbol_parse_empty(b: &mut test::Bencher) { - b.iter(|| typos::tokens::Identifier::parse_bytes(data::EMPTY.as_bytes()).last()); + let parser = typos::tokens::Parser::new(); + b.iter(|| parser.parse_bytes(data::EMPTY.as_bytes()).last()); } #[bench] fn symbol_parse_no_tokens(b: &mut test::Bencher) { - b.iter(|| typos::tokens::Identifier::parse_bytes(data::NO_TOKENS.as_bytes()).last()); + let parser = typos::tokens::Parser::new(); + b.iter(|| parser.parse_bytes(data::NO_TOKENS.as_bytes()).last()); } #[bench] fn symbol_parse_single_token(b: &mut test::Bencher) { + let parser = typos::tokens::Parser::new(); b.iter(|| { - typos::tokens::Identifier::parse_bytes(data::SINGLE_TOKEN.as_bytes()).last(); + parser.parse_bytes(data::SINGLE_TOKEN.as_bytes()).last(); }); } #[bench] fn symbol_parse_sherlock(b: &mut test::Bencher) { - b.iter(|| typos::tokens::Identifier::parse_bytes(data::SHERLOCK.as_bytes()).last()); + let parser = typos::tokens::Parser::new(); + b.iter(|| parser.parse_bytes(data::SHERLOCK.as_bytes()).last()); } #[bench] fn symbol_parse_code(b: &mut test::Bencher) { - b.iter(|| typos::tokens::Identifier::parse_bytes(data::CODE.as_bytes()).last()); + let parser = typos::tokens::Parser::new(); + b.iter(|| parser.parse_bytes(data::CODE.as_bytes()).last()); } #[bench] fn symbol_parse_corpus(b: &mut test::Bencher) { - b.iter(|| typos::tokens::Identifier::parse_bytes(data::CORPUS.as_bytes()).last()); + let parser = typos::tokens::Parser::new(); + b.iter(|| parser.parse_bytes(data::CORPUS.as_bytes()).last()); } #[bench] fn symbol_split_lowercase_short(b: &mut test::Bencher) { let input = "abcabcabcabc"; - let symbol = typos::tokens::Identifier::new(input, 0).unwrap(); + let symbol = typos::tokens::Identifier::new_unchecked(input, 0); b.iter(|| symbol.split().last()); } #[bench] fn symbol_split_lowercase_long(b: &mut test::Bencher) { let input = "abcabcabcabc".repeat(90); - let symbol = typos::tokens::Identifier::new(&input, 0).unwrap(); + let symbol = typos::tokens::Identifier::new_unchecked(&input, 0); b.iter(|| symbol.split().last()); } #[bench] fn symbol_split_mixed_short(b: &mut test::Bencher) { let input = "abcABCAbc123"; - let symbol = typos::tokens::Identifier::new(input, 0).unwrap(); + let symbol = typos::tokens::Identifier::new_unchecked(input, 0); b.iter(|| symbol.split().last()); } #[bench] fn symbol_split_mixed_long(b: &mut test::Bencher) { let input = "abcABCAbc123".repeat(90); - let symbol = typos::tokens::Identifier::new(&input, 0).unwrap(); + let symbol = typos::tokens::Identifier::new_unchecked(&input, 0); b.iter(|| symbol.split().last()); } diff --git a/src/checks.rs b/src/checks.rs new file mode 100644 index 0000000..308d724 --- /dev/null +++ b/src/checks.rs @@ -0,0 +1,171 @@ +use std::fs::File; +use std::io::Read; + +use bstr::ByteSlice; + +use crate::report; +use crate::tokens; +use crate::Dictionary; + +pub struct CheckSettings { + check_filenames: bool, + check_files: bool, + binary: bool, +} + +impl CheckSettings { + pub fn new() -> Self { + Default::default() + } + + pub fn check_filenames(&mut self, yes: bool) -> &mut Self { + self.check_filenames = yes; + self + } + + pub fn check_files(&mut self, yes: bool) -> &mut Self { + self.check_files = yes; + self + } + + pub fn binary(&mut self, yes: bool) -> &mut Self { + self.binary = yes; + self + } + + pub fn build<'d, 'p>( + &self, + dictionary: &'d Dictionary, + parser: &'p tokens::Parser, + ) -> Checks<'d, 'p> { + Checks { + dictionary, + parser, + check_filenames: self.check_filenames, + check_files: self.check_files, + binary: self.binary, + } + } +} + +impl Default for CheckSettings { + fn default() -> Self { + Self { + check_filenames: true, + check_files: true, + binary: false, + } + } +} + +pub struct Checks<'d, 'p> { + dictionary: &'d Dictionary, + parser: &'p tokens::Parser, + check_filenames: bool, + check_files: bool, + binary: bool, +} + +impl<'d, 'p> Checks<'d, 'p> { + pub fn check_filename( + &self, + path: &std::path::Path, + report: report::Report, + ) -> Result { + let mut typos_found = false; + + if !self.check_filenames { + return Ok(typos_found); + } + + for part in path.components().filter_map(|c| c.as_os_str().to_str()) { + for ident in self.parser.parse(part) { + if let Some(correction) = self.dictionary.correct_ident(ident) { + let msg = report::FilenameCorrection { + path, + typo: ident.token(), + correction, + non_exhaustive: (), + }; + report(msg.into()); + typos_found = true; + } + for word in ident.split() { + if let Some(correction) = self.dictionary.correct_word(word) { + let msg = report::FilenameCorrection { + path, + typo: word.token(), + correction, + non_exhaustive: (), + }; + report(msg.into()); + typos_found = true; + } + } + } + } + + Ok(typos_found) + } + + pub fn check_file( + &self, + path: &std::path::Path, + report: report::Report, + ) -> Result { + let mut typos_found = false; + + if !self.check_files { + return Ok(typos_found); + } + + let mut buffer = Vec::new(); + File::open(path)?.read_to_end(&mut buffer)?; + if !self.binary && buffer.find_byte(b'\0').is_some() { + let msg = report::BinaryFile { + path, + non_exhaustive: (), + }; + report(msg.into()); + return Ok(typos_found); + } + + for (line_idx, line) in buffer.lines().enumerate() { + let line_num = line_idx + 1; + for ident in self.parser.parse_bytes(line) { + if let Some(correction) = self.dictionary.correct_ident(ident) { + let col_num = ident.offset(); + let msg = report::Correction { + path, + line, + line_num, + col_num, + typo: ident.token(), + correction, + non_exhaustive: (), + }; + typos_found = true; + report(msg.into()); + } + for word in ident.split() { + if let Some(correction) = self.dictionary.correct_word(word) { + let col_num = word.offset(); + let msg = report::Correction { + path, + line, + line_num, + col_num, + typo: word.token(), + correction, + non_exhaustive: (), + }; + typos_found = true; + report(msg.into()); + } + } + } + } + + Ok(typos_found) + } +} diff --git a/src/dict.rs b/src/dict.rs index 1861757..1ff7359 100644 --- a/src/dict.rs +++ b/src/dict.rs @@ -4,12 +4,21 @@ use unicase::UniCase; use crate::tokens::Case; -#[derive(Default)] -pub struct Dictionary {} +pub trait Dictionary { + fn correct_ident<'s, 'w>( + &'s self, + _ident: crate::tokens::Identifier<'w>, + ) -> Option>; -impl Dictionary { + fn correct_word<'s, 'w>(&'s self, word: crate::tokens::Word<'w>) -> Option>; +} + +#[derive(Default)] +pub struct BuiltIn {} + +impl BuiltIn { pub fn new() -> Self { - Dictionary {} + Self {} } pub fn correct_ident<'s, 'w>( @@ -25,6 +34,19 @@ impl Dictionary { } } +impl Dictionary for BuiltIn { + fn correct_ident<'s, 'w>( + &'s self, + ident: crate::tokens::Identifier<'w>, + ) -> Option> { + BuiltIn::correct_ident(self, ident) + } + + fn correct_word<'s, 'w>(&'s self, word: crate::tokens::Word<'w>) -> Option> { + BuiltIn::correct_word(self, word) + } +} + fn map_lookup( map: &'static phf::Map, &'static str>, key: &str, diff --git a/src/lib.rs b/src/lib.rs index 7ef5ea5..b5201df 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,119 +4,8 @@ extern crate serde_derive; mod dict; mod dict_codegen; +pub mod checks; pub mod report; pub mod tokens; pub use crate::dict::*; - -use std::fs::File; -use std::io::Read; - -use bstr::ByteSlice; - -pub fn process_file( - path: &std::path::Path, - dictionary: &Dictionary, - check_filenames: bool, - check_files: bool, - ignore_hex: bool, - binary: bool, - report: report::Report, -) -> Result { - let mut typos_found = false; - - if check_filenames { - for part in path.components().filter_map(|c| c.as_os_str().to_str()) { - for ident in tokens::Identifier::parse(part) { - if !ignore_hex && is_hex(ident.token()) { - continue; - } - if let Some(correction) = dictionary.correct_ident(ident) { - let msg = report::FilenameCorrection { - path, - typo: ident.token(), - correction, - non_exhaustive: (), - }; - report(msg.into()); - typos_found = true; - } - for word in ident.split() { - if let Some(correction) = dictionary.correct_word(word) { - let msg = report::FilenameCorrection { - path, - typo: word.token(), - correction, - non_exhaustive: (), - }; - report(msg.into()); - typos_found = true; - } - } - } - } - } - - if check_files { - let mut buffer = Vec::new(); - File::open(path)?.read_to_end(&mut buffer)?; - if !binary && buffer.find_byte(b'\0').is_some() { - let msg = report::BinaryFile { - path, - non_exhaustive: (), - }; - report(msg.into()); - return Ok(typos_found); - } - - for (line_idx, line) in buffer.lines().enumerate() { - let line_num = line_idx + 1; - for ident in tokens::Identifier::parse_bytes(line) { - if !ignore_hex && is_hex(ident.token()) { - continue; - } - if let Some(correction) = dictionary.correct_ident(ident) { - let col_num = ident.offset(); - let msg = report::Correction { - path, - line, - line_num, - col_num, - typo: ident.token(), - correction, - non_exhaustive: (), - }; - typos_found = true; - report(msg.into()); - } - for word in ident.split() { - if let Some(correction) = dictionary.correct_word(word) { - let col_num = word.offset(); - let msg = report::Correction { - path, - line, - line_num, - col_num, - typo: word.token(), - correction, - non_exhaustive: (), - }; - typos_found = true; - report(msg.into()); - } - } - } - } - } - - Ok(typos_found) -} - -fn is_hex(ident: &str) -> bool { - lazy_static::lazy_static! { - // `_`: number literal separator in Rust and other languages - // `'`: number literal separator in C++ - static ref HEX: regex::Regex = regex::Regex::new(r#"^0[xX][0-9a-fA-F_']+$"#).unwrap(); - } - HEX.is_match(ident) -} diff --git a/src/main.rs b/src/main.rs index 2e662a7..8673f7e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -255,12 +255,23 @@ fn run() -> Result { let mut builder = get_logging(options.verbose.log_level()); builder.init(); - let dictionary = typos::Dictionary::new(); let check_filenames = options.check_filenames().unwrap_or(true); let check_files = options.check_files().unwrap_or(true); let ignore_hex = options.ignore_hex().unwrap_or(true); let binary = options.binary().unwrap_or(false); + let dictionary = typos::BuiltIn::new(); + + let parser = typos::tokens::ParserBuilder::new() + .ignore_hex(ignore_hex) + .build(); + + let checks = typos::checks::CheckSettings::new() + .check_filenames(check_filenames) + .check_files(check_files) + .binary(binary) + .build(&dictionary, &parser); + let first_path = &options .path .get(0) @@ -279,15 +290,10 @@ fn run() -> Result { for entry in walk.build() { let entry = entry?; if entry.file_type().map(|t| t.is_file()).unwrap_or(true) { - if typos::process_file( - entry.path(), - &dictionary, - check_filenames, - check_files, - ignore_hex, - binary, - options.format.report(), - )? { + if checks.check_filename(entry.path(), options.format.report())? { + typos_found = true; + } + if checks.check_file(entry.path(), options.format.report())? { typos_found = true; } } diff --git a/src/tokens.rs b/src/tokens.rs index 2d8c09a..e91824a 100644 --- a/src/tokens.rs +++ b/src/tokens.rs @@ -6,6 +6,80 @@ pub enum Case { None, } +#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] +pub struct ParserBuilder { + ignore_hex: bool, +} + +impl ParserBuilder { + pub fn new() -> Self { + Default::default() + } + + pub fn ignore_hex(&mut self, yes: bool) -> &mut Self { + self.ignore_hex = yes; + self + } + + pub fn build(&self) -> Parser { + let pattern = r#"\b(\p{Alphabetic}|\d|_|')+\b"#; + let words_str = regex::Regex::new(pattern).unwrap(); + let words_bytes = regex::bytes::Regex::new(pattern).unwrap(); + Parser { + words_str, + words_bytes, + ignore_hex: self.ignore_hex, + } + } +} + +#[derive(Debug, Clone)] +pub struct Parser { + words_str: regex::Regex, + words_bytes: regex::bytes::Regex, + ignore_hex: bool, +} + +impl Parser { + pub fn new() -> Self { + ParserBuilder::default().build() + } + + pub fn parse<'c>(&'c self, content: &'c str) -> impl Iterator> { + let ignore_hex = self.ignore_hex; + self.words_str + .find_iter(content) + .filter(move |m| !ignore_hex || !is_hex(m.as_str().as_bytes())) + .map(|m| Identifier::new_unchecked(m.as_str(), m.start())) + } + + pub fn parse_bytes<'c>(&'c self, content: &'c [u8]) -> impl Iterator> { + let ignore_hex = self.ignore_hex; + self.words_bytes + .find_iter(content) + .filter(move |m| !ignore_hex || !is_hex(m.as_bytes())) + .filter_map(|m| { + let s = std::str::from_utf8(m.as_bytes()).ok(); + s.map(|s| Identifier::new_unchecked(s, m.start())) + }) + } +} + +impl Default for Parser { + fn default() -> Self { + Self::new() + } +} + +fn is_hex(ident: &[u8]) -> bool { + lazy_static::lazy_static! { + // `_`: number literal separator in Rust and other languages + // `'`: number literal separator in C++ + static ref HEX: regex::bytes::Regex = regex::bytes::Regex::new(r#"^0[xX][0-9a-fA-F_']+$"#).unwrap(); + } + HEX.is_match(ident) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Identifier<'t> { token: &'t str, @@ -13,54 +87,10 @@ pub struct Identifier<'t> { } impl<'t> Identifier<'t> { - pub fn new(token: &'t str, offset: usize) -> Result { - let mut itr = Self::parse_bytes(token.as_bytes()); - let mut item = itr - .next() - .ok_or_else(|| failure::format_err!("Invalid ident (none found): {:?}", token))?; - if item.offset != 0 { - return Err(failure::format_err!( - "Invalid ident (padding found): {:?}", - token - )); - } - item.offset += offset; - if itr.next().is_some() { - return Err(failure::format_err!( - "Invalid ident (contains more than one): {:?}", - token - )); - } - Ok(item) - } - - pub(crate) fn new_unchecked(token: &'t str, offset: usize) -> Self { + pub fn new_unchecked(token: &'t str, offset: usize) -> Self { Self { token, offset } } - pub fn parse(content: &str) -> impl Iterator> { - lazy_static::lazy_static! { - // Getting false positives for this lint - #[allow(clippy::invalid_regex)] - static ref SPLIT: regex::Regex = regex::Regex::new(r#"\b(\p{Alphabetic}|\d|_|')+\b"#).unwrap(); - } - SPLIT - .find_iter(content) - .map(|m| Identifier::new_unchecked(m.as_str(), m.start())) - } - - pub fn parse_bytes(content: &[u8]) -> impl Iterator> { - lazy_static::lazy_static! { - // Getting false positives for this lint - #[allow(clippy::invalid_regex)] - static ref SPLIT: regex::bytes::Regex = regex::bytes::Regex::new(r#"\b(\p{Alphabetic}|\d|_|')+\b"#).unwrap(); - } - SPLIT.find_iter(content).filter_map(|m| { - let s = std::str::from_utf8(m.as_bytes()).ok(); - s.map(|s| Identifier::new_unchecked(s, m.start())) - }) - } - pub fn token(&self) -> &str { self.token } @@ -87,7 +117,6 @@ pub struct Word<'t> { impl<'t> Word<'t> { pub fn new(token: &'t str, offset: usize) -> Result { - Identifier::new(token, offset)?; let mut itr = split_ident(token, 0); let mut item = itr .next() @@ -108,7 +137,7 @@ impl<'t> Word<'t> { Ok(item) } - pub(crate) fn new_unchecked(token: &'t str, case: Case, offset: usize) -> Self { + pub fn new_unchecked(token: &'t str, case: Case, offset: usize) -> Self { Self { token, case, @@ -251,70 +280,113 @@ mod test { #[test] fn tokenize_empty_is_empty() { + let parser = Parser::new(); + let input = ""; let expected: Vec = vec![]; - let actual: Vec<_> = Identifier::parse_bytes(input.as_bytes()).collect(); + let actual: Vec<_> = parser.parse_bytes(input.as_bytes()).collect(); assert_eq!(expected, actual); - let actual: Vec<_> = Identifier::parse(input).collect(); + let actual: Vec<_> = parser.parse(input).collect(); assert_eq!(expected, actual); } #[test] fn tokenize_word_is_word() { + let parser = Parser::new(); + let input = "word"; let expected: Vec = vec![Identifier::new_unchecked("word", 0)]; - let actual: Vec<_> = Identifier::parse_bytes(input.as_bytes()).collect(); + let actual: Vec<_> = parser.parse_bytes(input.as_bytes()).collect(); assert_eq!(expected, actual); - let actual: Vec<_> = Identifier::parse(input).collect(); + let actual: Vec<_> = parser.parse(input).collect(); assert_eq!(expected, actual); } #[test] fn tokenize_space_separated_words() { + let parser = Parser::new(); + let input = "A B"; let expected: Vec = vec![ Identifier::new_unchecked("A", 0), Identifier::new_unchecked("B", 2), ]; - let actual: Vec<_> = Identifier::parse_bytes(input.as_bytes()).collect(); + let actual: Vec<_> = parser.parse_bytes(input.as_bytes()).collect(); assert_eq!(expected, actual); - let actual: Vec<_> = Identifier::parse(input).collect(); + let actual: Vec<_> = parser.parse(input).collect(); assert_eq!(expected, actual); } #[test] fn tokenize_dot_separated_words() { + let parser = Parser::new(); + let input = "A.B"; let expected: Vec = vec![ Identifier::new_unchecked("A", 0), Identifier::new_unchecked("B", 2), ]; - let actual: Vec<_> = Identifier::parse_bytes(input.as_bytes()).collect(); + let actual: Vec<_> = parser.parse_bytes(input.as_bytes()).collect(); assert_eq!(expected, actual); - let actual: Vec<_> = Identifier::parse(input).collect(); + let actual: Vec<_> = parser.parse(input).collect(); assert_eq!(expected, actual); } #[test] fn tokenize_namespace_separated_words() { + let parser = Parser::new(); + let input = "A::B"; let expected: Vec = vec![ Identifier::new_unchecked("A", 0), Identifier::new_unchecked("B", 3), ]; - let actual: Vec<_> = Identifier::parse_bytes(input.as_bytes()).collect(); + let actual: Vec<_> = parser.parse_bytes(input.as_bytes()).collect(); assert_eq!(expected, actual); - let actual: Vec<_> = Identifier::parse(input).collect(); + let actual: Vec<_> = parser.parse(input).collect(); assert_eq!(expected, actual); } #[test] fn tokenize_underscore_doesnt_separate() { + let parser = Parser::new(); + let input = "A_B"; let expected: Vec = vec![Identifier::new_unchecked("A_B", 0)]; - let actual: Vec<_> = Identifier::parse_bytes(input.as_bytes()).collect(); + let actual: Vec<_> = parser.parse_bytes(input.as_bytes()).collect(); assert_eq!(expected, actual); - let actual: Vec<_> = Identifier::parse(input).collect(); + let actual: Vec<_> = parser.parse(input).collect(); + assert_eq!(expected, actual); + } + + #[test] + fn tokenize_ignore_hex_enabled() { + let parser = ParserBuilder::new().ignore_hex(true).build(); + + let input = "Hello 0xDEADBEEF World"; + let expected: Vec = vec![ + Identifier::new_unchecked("Hello", 0), + Identifier::new_unchecked("World", 17), + ]; + let actual: Vec<_> = parser.parse_bytes(input.as_bytes()).collect(); + assert_eq!(expected, actual); + let actual: Vec<_> = parser.parse(input).collect(); + assert_eq!(expected, actual); + } + + #[test] + fn tokenize_ignore_hex_disabled() { + let parser = ParserBuilder::new().ignore_hex(false).build(); + + let input = "Hello 0xDEADBEEF World"; + let expected: Vec = vec![ + Identifier::new_unchecked("Hello", 0), + Identifier::new_unchecked("0xDEADBEEF", 6), + Identifier::new_unchecked("World", 17), + ]; + let actual: Vec<_> = parser.parse_bytes(input.as_bytes()).collect(); + assert_eq!(expected, actual); + let actual: Vec<_> = parser.parse(input).collect(); assert_eq!(expected, actual); } @@ -375,7 +447,7 @@ mod test { ), ]; for (input, expected) in cases.iter() { - let ident = Identifier::new(input, 0).unwrap(); + let ident = Identifier::new_unchecked(input, 0); let result: Vec<_> = ident.split().map(|w| (w.token, w.case, w.offset)).collect(); assert_eq!(&result, expected); }