From 881fce5114e9d16517be911445682b14cc4eb837 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Sat, 22 Jun 2019 11:47:51 -0600 Subject: [PATCH] feat(parse): Track the case of each word --- src/tokens.rs | 139 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 117 insertions(+), 22 deletions(-) diff --git a/src/tokens.rs b/src/tokens.rs index 0da2b07..d9c0f72 100644 --- a/src/tokens.rs +++ b/src/tokens.rs @@ -1,3 +1,11 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Case { + Title, + Lower, + Scream, + None, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Symbol<'t> { token: &'t str, @@ -46,6 +54,10 @@ impl<'t> Symbol<'t> { self.token } + pub fn case(&self) -> Case { + Case::None + } + pub fn offset(&self) -> usize { self.offset } @@ -58,6 +70,7 @@ impl<'t> Symbol<'t> { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Word<'t> { token: &'t str, + case: Case, offset: usize, } @@ -84,14 +97,22 @@ impl<'t> Word<'t> { Ok(item) } - pub(crate) fn new_unchecked(token: &'t str, offset: usize) -> Self { - Self { token, offset } + pub(crate) fn new_unchecked(token: &'t str, case: Case, offset: usize) -> Self { + Self { + token, + case, + offset, + } } pub fn token(&self) -> &str { self.token } + pub fn case(&self) -> Case { + self.case + } + pub fn offset(&self) -> usize { self.offset } @@ -127,6 +148,22 @@ impl WordMode { WordMode::Boundary } } + + fn case(self, last: WordMode) -> Case { + match (self, last) { + (WordMode::Uppercase, WordMode::Uppercase) => Case::Scream, + (WordMode::Uppercase, WordMode::Lowercase) => Case::Title, + (WordMode::Lowercase, WordMode::Lowercase) => Case::Lower, + (WordMode::Number, WordMode::Number) => Case::None, + (WordMode::Number, _) + | (_, WordMode::Number) + | (WordMode::Boundary, _) + | (_, WordMode::Boundary) + | (WordMode::Lowercase, WordMode::Uppercase) => { + unreachable!("Invalid case combination: ({:?}, {:?})", self, last) + } + } + } } fn split_symbol(symbol: &str, offset: usize) -> impl Iterator> { @@ -135,6 +172,7 @@ fn split_symbol(symbol: &str, offset: usize) -> impl Iterator> { let mut char_indices = symbol.char_indices().peekable(); let mut start = 0; let mut start_mode = WordMode::Boundary; + let mut last_mode = WordMode::Boundary; while let Some((i, c)) = char_indices.next() { let cur_mode = WordMode::classify(c); if cur_mode == WordMode::Boundary { @@ -143,13 +181,16 @@ fn split_symbol(symbol: &str, offset: usize) -> impl Iterator> { } continue; } + if start_mode == WordMode::Boundary { + start_mode = cur_mode; + } if let Some(&(next_i, next)) = char_indices.peek() { // The mode including the current character, assuming the current character does // not result in a word boundary. let next_mode = WordMode::classify(next); - match (start_mode, cur_mode, next_mode) { + match (last_mode, cur_mode, next_mode) { // cur_mode is last of current word (_, _, WordMode::Boundary) | (_, WordMode::Lowercase, WordMode::Number) @@ -157,24 +198,36 @@ fn split_symbol(symbol: &str, offset: usize) -> impl Iterator> { | (_, WordMode::Number, WordMode::Lowercase) | (_, WordMode::Number, WordMode::Uppercase) | (_, WordMode::Lowercase, WordMode::Uppercase) => { - result.push(Word::new_unchecked(&symbol[start..next_i], start + offset)); + let case = start_mode.case(cur_mode); + result.push(Word::new_unchecked( + &symbol[start..next_i], + case, + start + offset, + )); start = next_i; start_mode = WordMode::Boundary; + last_mode = WordMode::Boundary; } // cur_mode is start of next word (WordMode::Uppercase, WordMode::Uppercase, WordMode::Lowercase) => { - result.push(Word::new_unchecked(&symbol[start..i], start + offset)); + result.push(Word::new_unchecked( + &symbol[start..i], + Case::Scream, + start + offset, + )); start = i; - start_mode = WordMode::Boundary; + start_mode = cur_mode; + last_mode = WordMode::Boundary; } // No word boundary (_, _, _) => { - start_mode = cur_mode; + last_mode = cur_mode; } } } else { // Collect trailing characters as a word - result.push(Word::new_unchecked(&symbol[start..], start + offset)); + let case = start_mode.case(cur_mode); + result.push(Word::new_unchecked(&symbol[start..], case, start + offset)); break; } } @@ -240,23 +293,65 @@ mod test { #[test] fn split_symbol() { let cases = [ - ("lowercase", &["lowercase"] as &[&str]), - ("Class", &["Class"]), - ("MyClass", &["My", "Class"]), - ("MyC", &["My", "C"]), - ("HTML", &["HTML"]), - ("PDFLoader", &["PDF", "Loader"]), - ("AString", &["A", "String"]), - ("SimpleXMLParser", &["Simple", "XML", "Parser"]), - ("vimRPCPlugin", &["vim", "RPC", "Plugin"]), - ("GL11Version", &["GL", "11", "Version"]), - ("99Bottles", &["99", "Bottles"]), - ("May5", &["May", "5"]), - ("BFG9000", &["BFG", "9000"]), + ( + "lowercase", + &[("lowercase", Case::Lower, 0usize)] as &[(&str, Case, usize)], + ), + ("Class", &[("Class", Case::Title, 0)]), + ( + "MyClass", + &[("My", Case::Title, 0), ("Class", Case::Title, 2)], + ), + ("MyC", &[("My", Case::Title, 0), ("C", Case::Scream, 2)]), + ("HTML", &[("HTML", Case::Scream, 0)]), + ( + "PDFLoader", + &[("PDF", Case::Scream, 0), ("Loader", Case::Title, 3)], + ), + ( + "AString", + &[("A", Case::Scream, 0), ("String", Case::Title, 1)], + ), + ( + "SimpleXMLParser", + &[ + ("Simple", Case::Title, 0), + ("XML", Case::Scream, 6), + ("Parser", Case::Title, 9), + ], + ), + ( + "vimRPCPlugin", + &[ + ("vim", Case::Lower, 0), + ("RPC", Case::Scream, 3), + ("Plugin", Case::Title, 6), + ], + ), + ( + "GL11Version", + &[ + ("GL", Case::Scream, 0), + ("11", Case::None, 2), + ("Version", Case::Title, 4), + ], + ), + ( + "99Bottles", + &[("99", Case::None, 0), ("Bottles", Case::Title, 2)], + ), + ("May5", &[("May", Case::Title, 0), ("5", Case::None, 3)]), + ( + "BFG9000", + &[("BFG", Case::Scream, 0), ("9000", Case::None, 3)], + ), ]; for (input, expected) in cases.iter() { let symbol = Symbol::new(input, 0).unwrap(); - let result: Vec<_> = symbol.split().map(|w| w.token).collect(); + let result: Vec<_> = symbol + .split() + .map(|w| (w.token, w.case, w.offset)) + .collect(); assert_eq!(&result, expected); } }