feat(parse): Track the case of each word

This commit is contained in:
Ed Page 2019-06-22 11:47:51 -06:00
parent 80aeed1b43
commit 881fce5114

View file

@ -1,3 +1,11 @@
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Case {
Title,
Lower,
Scream,
None,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Symbol<'t> { pub struct Symbol<'t> {
token: &'t str, token: &'t str,
@ -46,6 +54,10 @@ impl<'t> Symbol<'t> {
self.token self.token
} }
pub fn case(&self) -> Case {
Case::None
}
pub fn offset(&self) -> usize { pub fn offset(&self) -> usize {
self.offset self.offset
} }
@ -58,6 +70,7 @@ impl<'t> Symbol<'t> {
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Word<'t> { pub struct Word<'t> {
token: &'t str, token: &'t str,
case: Case,
offset: usize, offset: usize,
} }
@ -84,14 +97,22 @@ impl<'t> Word<'t> {
Ok(item) Ok(item)
} }
pub(crate) fn new_unchecked(token: &'t str, offset: usize) -> Self { pub(crate) fn new_unchecked(token: &'t str, case: Case, offset: usize) -> Self {
Self { token, offset } Self {
token,
case,
offset,
}
} }
pub fn token(&self) -> &str { pub fn token(&self) -> &str {
self.token self.token
} }
pub fn case(&self) -> Case {
self.case
}
pub fn offset(&self) -> usize { pub fn offset(&self) -> usize {
self.offset self.offset
} }
@ -127,6 +148,22 @@ impl WordMode {
WordMode::Boundary WordMode::Boundary
} }
} }
fn case(self, last: WordMode) -> Case {
match (self, last) {
(WordMode::Uppercase, WordMode::Uppercase) => Case::Scream,
(WordMode::Uppercase, WordMode::Lowercase) => Case::Title,
(WordMode::Lowercase, WordMode::Lowercase) => Case::Lower,
(WordMode::Number, WordMode::Number) => Case::None,
(WordMode::Number, _)
| (_, WordMode::Number)
| (WordMode::Boundary, _)
| (_, WordMode::Boundary)
| (WordMode::Lowercase, WordMode::Uppercase) => {
unreachable!("Invalid case combination: ({:?}, {:?})", self, last)
}
}
}
} }
fn split_symbol(symbol: &str, offset: usize) -> impl Iterator<Item = Word<'_>> { fn split_symbol(symbol: &str, offset: usize) -> impl Iterator<Item = Word<'_>> {
@ -135,6 +172,7 @@ fn split_symbol(symbol: &str, offset: usize) -> impl Iterator<Item = Word<'_>> {
let mut char_indices = symbol.char_indices().peekable(); let mut char_indices = symbol.char_indices().peekable();
let mut start = 0; let mut start = 0;
let mut start_mode = WordMode::Boundary; let mut start_mode = WordMode::Boundary;
let mut last_mode = WordMode::Boundary;
while let Some((i, c)) = char_indices.next() { while let Some((i, c)) = char_indices.next() {
let cur_mode = WordMode::classify(c); let cur_mode = WordMode::classify(c);
if cur_mode == WordMode::Boundary { if cur_mode == WordMode::Boundary {
@ -143,13 +181,16 @@ fn split_symbol(symbol: &str, offset: usize) -> impl Iterator<Item = Word<'_>> {
} }
continue; continue;
} }
if start_mode == WordMode::Boundary {
start_mode = cur_mode;
}
if let Some(&(next_i, next)) = char_indices.peek() { if let Some(&(next_i, next)) = char_indices.peek() {
// The mode including the current character, assuming the current character does // The mode including the current character, assuming the current character does
// not result in a word boundary. // not result in a word boundary.
let next_mode = WordMode::classify(next); let next_mode = WordMode::classify(next);
match (start_mode, cur_mode, next_mode) { match (last_mode, cur_mode, next_mode) {
// cur_mode is last of current word // cur_mode is last of current word
(_, _, WordMode::Boundary) (_, _, WordMode::Boundary)
| (_, WordMode::Lowercase, WordMode::Number) | (_, WordMode::Lowercase, WordMode::Number)
@ -157,24 +198,36 @@ fn split_symbol(symbol: &str, offset: usize) -> impl Iterator<Item = Word<'_>> {
| (_, WordMode::Number, WordMode::Lowercase) | (_, WordMode::Number, WordMode::Lowercase)
| (_, WordMode::Number, WordMode::Uppercase) | (_, WordMode::Number, WordMode::Uppercase)
| (_, WordMode::Lowercase, WordMode::Uppercase) => { | (_, WordMode::Lowercase, WordMode::Uppercase) => {
result.push(Word::new_unchecked(&symbol[start..next_i], start + offset)); let case = start_mode.case(cur_mode);
result.push(Word::new_unchecked(
&symbol[start..next_i],
case,
start + offset,
));
start = next_i; start = next_i;
start_mode = WordMode::Boundary; start_mode = WordMode::Boundary;
last_mode = WordMode::Boundary;
} }
// cur_mode is start of next word // cur_mode is start of next word
(WordMode::Uppercase, WordMode::Uppercase, WordMode::Lowercase) => { (WordMode::Uppercase, WordMode::Uppercase, WordMode::Lowercase) => {
result.push(Word::new_unchecked(&symbol[start..i], start + offset)); result.push(Word::new_unchecked(
&symbol[start..i],
Case::Scream,
start + offset,
));
start = i; start = i;
start_mode = WordMode::Boundary; start_mode = cur_mode;
last_mode = WordMode::Boundary;
} }
// No word boundary // No word boundary
(_, _, _) => { (_, _, _) => {
start_mode = cur_mode; last_mode = cur_mode;
} }
} }
} else { } else {
// Collect trailing characters as a word // Collect trailing characters as a word
result.push(Word::new_unchecked(&symbol[start..], start + offset)); let case = start_mode.case(cur_mode);
result.push(Word::new_unchecked(&symbol[start..], case, start + offset));
break; break;
} }
} }
@ -240,23 +293,65 @@ mod test {
#[test] #[test]
fn split_symbol() { fn split_symbol() {
let cases = [ let cases = [
("lowercase", &["lowercase"] as &[&str]), (
("Class", &["Class"]), "lowercase",
("MyClass", &["My", "Class"]), &[("lowercase", Case::Lower, 0usize)] as &[(&str, Case, usize)],
("MyC", &["My", "C"]), ),
("HTML", &["HTML"]), ("Class", &[("Class", Case::Title, 0)]),
("PDFLoader", &["PDF", "Loader"]), (
("AString", &["A", "String"]), "MyClass",
("SimpleXMLParser", &["Simple", "XML", "Parser"]), &[("My", Case::Title, 0), ("Class", Case::Title, 2)],
("vimRPCPlugin", &["vim", "RPC", "Plugin"]), ),
("GL11Version", &["GL", "11", "Version"]), ("MyC", &[("My", Case::Title, 0), ("C", Case::Scream, 2)]),
("99Bottles", &["99", "Bottles"]), ("HTML", &[("HTML", Case::Scream, 0)]),
("May5", &["May", "5"]), (
("BFG9000", &["BFG", "9000"]), "PDFLoader",
&[("PDF", Case::Scream, 0), ("Loader", Case::Title, 3)],
),
(
"AString",
&[("A", Case::Scream, 0), ("String", Case::Title, 1)],
),
(
"SimpleXMLParser",
&[
("Simple", Case::Title, 0),
("XML", Case::Scream, 6),
("Parser", Case::Title, 9),
],
),
(
"vimRPCPlugin",
&[
("vim", Case::Lower, 0),
("RPC", Case::Scream, 3),
("Plugin", Case::Title, 6),
],
),
(
"GL11Version",
&[
("GL", Case::Scream, 0),
("11", Case::None, 2),
("Version", Case::Title, 4),
],
),
(
"99Bottles",
&[("99", Case::None, 0), ("Bottles", Case::Title, 2)],
),
("May5", &[("May", Case::Title, 0), ("5", Case::None, 3)]),
(
"BFG9000",
&[("BFG", Case::Scream, 0), ("9000", Case::None, 3)],
),
]; ];
for (input, expected) in cases.iter() { for (input, expected) in cases.iter() {
let symbol = Symbol::new(input, 0).unwrap(); let symbol = Symbol::new(input, 0).unwrap();
let result: Vec<_> = symbol.split().map(|w| w.token).collect(); let result: Vec<_> = symbol
.split()
.map(|w| (w.token, w.case, w.offset))
.collect();
assert_eq!(&result, expected); assert_eq!(&result, expected);
} }
} }