actions/typos
1
0
Fork 0
mirror of https://github.com/crate-ci/typos.git synced 2024-12-25 17:12:16 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

perf: Speed up identifier splitting

Browse source 
Before
```
test process_code         ... bench:      25,627 ns/iter (+/- 2,062)
test process_corpus       ... bench:  20,192,253 ns/iter (+/- 603,029)
test process_empty        ... bench:       7,418 ns/iter (+/- 707)
test process_no_tokens    ... bench:       8,788 ns/iter (+/- 1,065)
test process_sherlock     ... bench:      30,420 ns/iter (+/- 2,699)
test process_single_token ... bench:       9,426 ns/iter (+/- 811)
test symbol_split_lowercase_long  ... bench:       2,763 ns/iter (+/- 246)
test symbol_split_lowercase_short ... bench:         110 ns/iter (+/- 12)
test symbol_split_mixed_long      ... bench:       7,373 ns/iter (+/- 1,111)
test symbol_split_mixed_short     ... bench:         357 ns/iter (+/- 86)
```

After
```
test process_code         ... bench:      20,973 ns/iter (+/- 1,717)
test process_corpus       ... bench:  15,826,059 ns/iter (+/- 1,016,628)
test process_empty        ... bench:       7,364 ns/iter (+/- 616)
test process_no_tokens    ... bench:       8,858 ns/iter (+/- 632)
test process_sherlock     ... bench:      24,707 ns/iter (+/- 3,482)
test process_single_token ... bench:       9,339 ns/iter (+/- 706)
test symbol_split_lowercase_long  ... bench:       2,727 ns/iter (+/- 151)
test symbol_split_lowercase_short ... bench:          46 ns/iter (+/- 2)
test symbol_split_mixed_long      ... bench:       5,753 ns/iter (+/- 441)
test symbol_split_mixed_short     ... bench:          76 ns/iter (+/- 3)
```

Fixes #33
This commit is contained in:
Ed Page 2019-10-25 14:36:08 -06:00
parent 2ae1a0bca6
commit 979b42ed6f
1 changed files with 89 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

150
typos/src/tokens.rs
View file

@ -238,72 +238,100 @@ impl WordMode {
}
}
fn split_ident(ident: &str, offset: usize) -> impl Iterator<Item = Word<'_>> {
let mut result = vec![];
struct SplitIdent<'s> {
ident: &'s str,
offset: usize,
let mut char_indices = ident.char_indices().peekable();
let mut start = 0;
let mut start_mode = WordMode::Boundary;
let mut last_mode = WordMode::Boundary;
while let Some((i, c)) = char_indices.next() {
let cur_mode = WordMode::classify(c);
if cur_mode == WordMode::Boundary {
assert!(start_mode == WordMode::Boundary);
continue;
}
if start_mode == WordMode::Boundary {
start_mode = cur_mode;
start = i;
}
char_indices: std::iter::Peekable<std::str::CharIndices<'s>>,
start: usize,
start_mode: WordMode,
last_mode: WordMode,
}
if let Some(&(next_i, next)) = char_indices.peek() {
// The mode including the current character, assuming the current character does
// not result in a word boundary.
let next_mode = WordMode::classify(next);
match (last_mode, cur_mode, next_mode) {
// cur_mode is last of current word
(_, _, WordMode::Boundary)
| (_, WordMode::Lowercase, WordMode::Number)
| (_, WordMode::Uppercase, WordMode::Number)
| (_, WordMode::Number, WordMode::Lowercase)
| (_, WordMode::Number, WordMode::Uppercase)
| (_, WordMode::Lowercase, WordMode::Uppercase) => {
let case = start_mode.case(cur_mode);
result.push(Word::new_unchecked(
&ident[start..next_i],
case,
start + offset,
));
start = next_i;
start_mode = WordMode::Boundary;
last_mode = WordMode::Boundary;
}
// cur_mode is start of next word
(WordMode::Uppercase, WordMode::Uppercase, WordMode::Lowercase) => {
result.push(Word::new_unchecked(
&ident[start..i],
Case::Scream,
start + offset,
));
start = i;
start_mode = cur_mode;
last_mode = WordMode::Boundary;
}
// No word boundary
(_, _, _) => {
last_mode = cur_mode;
}
}
} else {
// Collect trailing characters as a word
let case = start_mode.case(cur_mode);
result.push(Word::new_unchecked(&ident[start..], case, start + offset));
break;
impl<'s> SplitIdent<'s> {
fn new(ident: &'s str, offset: usize) -> Self {
Self {
ident,
offset,
char_indices: ident.char_indices().peekable(),
start: 0,
start_mode: WordMode::Boundary,
last_mode: WordMode::Boundary,
}
}
}
result.into_iter()
impl<'s> Iterator for SplitIdent<'s> {
type Item = Word<'s>;
fn next(&mut self) -> Option<Word<'s>> {
while let Some((i, c)) = self.char_indices.next() {
let cur_mode = WordMode::classify(c);
if cur_mode == WordMode::Boundary {
assert!(self.start_mode == WordMode::Boundary);
continue;
}
if self.start_mode == WordMode::Boundary {
self.start_mode = cur_mode;
self.start = i;
}
if let Some(&(next_i, next)) = self.char_indices.peek() {
// The mode including the current character, assuming the current character does
// not result in a word boundary.
let next_mode = WordMode::classify(next);
match (self.last_mode, cur_mode, next_mode) {
// cur_mode is last of current word
(_, _, WordMode::Boundary)
| (_, WordMode::Lowercase, WordMode::Number)
| (_, WordMode::Uppercase, WordMode::Number)
| (_, WordMode::Number, WordMode::Lowercase)
| (_, WordMode::Number, WordMode::Uppercase)
| (_, WordMode::Lowercase, WordMode::Uppercase) => {
let case = self.start_mode.case(cur_mode);
let result = Word::new_unchecked(
&self.ident[self.start..next_i],
case,
self.start + self.offset,
);
self.start = next_i;
self.start_mode = WordMode::Boundary;
self.last_mode = WordMode::Boundary;
return Some(result);
}
// cur_mode is start of next word
(WordMode::Uppercase, WordMode::Uppercase, WordMode::Lowercase) => {
let result = Word::new_unchecked(
&self.ident[self.start..i],
Case::Scream,
self.start + self.offset,
);
self.start = i;
self.start_mode = cur_mode;
self.last_mode = WordMode::Boundary;
return Some(result);
}
// No word boundary
(_, _, _) => {
self.last_mode = cur_mode;
}
}
} else {
// Collect trailing characters as a word
let case = self.start_mode.case(cur_mode);
let result =
Word::new_unchecked(&self.ident[self.start..], case, self.start + self.offset);
return Some(result);
}
}
None
}
}
fn split_ident(ident: &str, offset: usize) -> impl Iterator<Item = Word<'_>> {
SplitIdent::new(ident, offset)
}
#[cfg(test)]