actions/typos
1
0
Fork 0
mirror of https://github.com/crate-ci/typos.git synced 2024-11-22 00:51:11 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(parse): Track the case of each word

Browse source
This commit is contained in:
Ed Page 2019-06-22 11:47:51 -06:00
parent 80aeed1b43
commit 881fce5114
1 changed files with 117 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

139
src/tokens.rs
View file

@ -1,3 +1,11 @@
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Case {
Title,
Lower,
Scream,
None,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Symbol<'t> {
token: &'t str,
@ -46,6 +54,10 @@ impl<'t> Symbol<'t> {
self.token
}
pub fn case(&self) -> Case {
Case::None
}
pub fn offset(&self) -> usize {
self.offset
}
@ -58,6 +70,7 @@ impl<'t> Symbol<'t> {
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Word<'t> {
token: &'t str,
case: Case,
offset: usize,
}
@ -84,14 +97,22 @@ impl<'t> Word<'t> {
Ok(item)
}
pub(crate) fn new_unchecked(token: &'t str, offset: usize) -> Self {
Self { token, offset }
pub(crate) fn new_unchecked(token: &'t str, case: Case, offset: usize) -> Self {
Self {
token,
case,
offset,
}
}
pub fn token(&self) -> &str {
self.token
}
pub fn case(&self) -> Case {
self.case
}
pub fn offset(&self) -> usize {
self.offset
}
@ -127,6 +148,22 @@ impl WordMode {
WordMode::Boundary
}
}
fn case(self, last: WordMode) -> Case {
match (self, last) {
(WordMode::Uppercase, WordMode::Uppercase) => Case::Scream,
(WordMode::Uppercase, WordMode::Lowercase) => Case::Title,
(WordMode::Lowercase, WordMode::Lowercase) => Case::Lower,
(WordMode::Number, WordMode::Number) => Case::None,
(WordMode::Number, _)
| (_, WordMode::Number)
| (WordMode::Boundary, _)
| (_, WordMode::Boundary)
| (WordMode::Lowercase, WordMode::Uppercase) => {
unreachable!("Invalid case combination: ({:?}, {:?})", self, last)
}
}
}
}
fn split_symbol(symbol: &str, offset: usize) -> impl Iterator<Item = Word<'_>> {
@ -135,6 +172,7 @@ fn split_symbol(symbol: &str, offset: usize) -> impl Iterator<Item = Word<'_>> {
let mut char_indices = symbol.char_indices().peekable();
let mut start = 0;
let mut start_mode = WordMode::Boundary;
let mut last_mode = WordMode::Boundary;
while let Some((i, c)) = char_indices.next() {
let cur_mode = WordMode::classify(c);
if cur_mode == WordMode::Boundary {
@ -143,13 +181,16 @@ fn split_symbol(symbol: &str, offset: usize) -> impl Iterator<Item = Word<'_>> {
}
continue;
}
if start_mode == WordMode::Boundary {
start_mode = cur_mode;
}
if let Some(&(next_i, next)) = char_indices.peek() {
// The mode including the current character, assuming the current character does
// not result in a word boundary.
let next_mode = WordMode::classify(next);
match (start_mode, cur_mode, next_mode) {
match (last_mode, cur_mode, next_mode) {
// cur_mode is last of current word
(_, _, WordMode::Boundary)
| (_, WordMode::Lowercase, WordMode::Number)
@ -157,24 +198,36 @@ fn split_symbol(symbol: &str, offset: usize) -> impl Iterator<Item = Word<'_>> {
| (_, WordMode::Number, WordMode::Lowercase)
| (_, WordMode::Number, WordMode::Uppercase)
| (_, WordMode::Lowercase, WordMode::Uppercase) => {
result.push(Word::new_unchecked(&symbol[start..next_i], start + offset));
let case = start_mode.case(cur_mode);
result.push(Word::new_unchecked(
&symbol[start..next_i],
case,
start + offset,
));
start = next_i;
start_mode = WordMode::Boundary;
last_mode = WordMode::Boundary;
}
// cur_mode is start of next word
(WordMode::Uppercase, WordMode::Uppercase, WordMode::Lowercase) => {
result.push(Word::new_unchecked(&symbol[start..i], start + offset));
result.push(Word::new_unchecked(
&symbol[start..i],
Case::Scream,
start + offset,
));
start = i;
start_mode = WordMode::Boundary;
start_mode = cur_mode;
last_mode = WordMode::Boundary;
}
// No word boundary
(_, _, _) => {
start_mode = cur_mode;
last_mode = cur_mode;
}
}
} else {
// Collect trailing characters as a word
result.push(Word::new_unchecked(&symbol[start..], start + offset));
let case = start_mode.case(cur_mode);
result.push(Word::new_unchecked(&symbol[start..], case, start + offset));
break;
}
}
@ -240,23 +293,65 @@ mod test {
#[test]
fn split_symbol() {
let cases = [
("lowercase", &["lowercase"] as &[&str]),
("Class", &["Class"]),
("MyClass", &["My", "Class"]),
("MyC", &["My", "C"]),
("HTML", &["HTML"]),
("PDFLoader", &["PDF", "Loader"]),
("AString", &["A", "String"]),
("SimpleXMLParser", &["Simple", "XML", "Parser"]),
("vimRPCPlugin", &["vim", "RPC", "Plugin"]),
("GL11Version", &["GL", "11", "Version"]),
("99Bottles", &["99", "Bottles"]),
("May5", &["May", "5"]),
("BFG9000", &["BFG", "9000"]),
(
"lowercase",
&[("lowercase", Case::Lower, 0usize)] as &[(&str, Case, usize)],
),
("Class", &[("Class", Case::Title, 0)]),
(
"MyClass",
&[("My", Case::Title, 0), ("Class", Case::Title, 2)],
),
("MyC", &[("My", Case::Title, 0), ("C", Case::Scream, 2)]),
("HTML", &[("HTML", Case::Scream, 0)]),
(
"PDFLoader",
&[("PDF", Case::Scream, 0), ("Loader", Case::Title, 3)],
),
(
"AString",
&[("A", Case::Scream, 0), ("String", Case::Title, 1)],
),
(
"SimpleXMLParser",
&[
("Simple", Case::Title, 0),
("XML", Case::Scream, 6),
("Parser", Case::Title, 9),
],
),
(
"vimRPCPlugin",
&[
("vim", Case::Lower, 0),
("RPC", Case::Scream, 3),
("Plugin", Case::Title, 6),
],
),
(
"GL11Version",
&[
("GL", Case::Scream, 0),
("11", Case::None, 2),
("Version", Case::Title, 4),
],
),
(
"99Bottles",
&[("99", Case::None, 0), ("Bottles", Case::Title, 2)],
),
("May5", &[("May", Case::Title, 0), ("5", Case::None, 3)]),
(
"BFG9000",
&[("BFG", Case::Scream, 0), ("9000", Case::None, 3)],
),
];
for (input, expected) in cases.iter() {
let symbol = Symbol::new(input, 0).unwrap();
let result: Vec<_> = symbol.split().map(|w| w.token).collect();
let result: Vec<_> = symbol
.split()
.map(|w| (w.token, w.case, w.offset))
.collect();
assert_eq!(&result, expected);
}
}