mirror of
https://github.com/crate-ci/typos.git
synced 2024-11-22 09:01:04 -05:00
feat(parse): Track the case of each word
This commit is contained in:
parent
80aeed1b43
commit
881fce5114
1 changed files with 117 additions and 22 deletions
139
src/tokens.rs
139
src/tokens.rs
|
@ -1,3 +1,11 @@
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum Case {
|
||||||
|
Title,
|
||||||
|
Lower,
|
||||||
|
Scream,
|
||||||
|
None,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub struct Symbol<'t> {
|
pub struct Symbol<'t> {
|
||||||
token: &'t str,
|
token: &'t str,
|
||||||
|
@ -46,6 +54,10 @@ impl<'t> Symbol<'t> {
|
||||||
self.token
|
self.token
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn case(&self) -> Case {
|
||||||
|
Case::None
|
||||||
|
}
|
||||||
|
|
||||||
pub fn offset(&self) -> usize {
|
pub fn offset(&self) -> usize {
|
||||||
self.offset
|
self.offset
|
||||||
}
|
}
|
||||||
|
@ -58,6 +70,7 @@ impl<'t> Symbol<'t> {
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub struct Word<'t> {
|
pub struct Word<'t> {
|
||||||
token: &'t str,
|
token: &'t str,
|
||||||
|
case: Case,
|
||||||
offset: usize,
|
offset: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -84,14 +97,22 @@ impl<'t> Word<'t> {
|
||||||
Ok(item)
|
Ok(item)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn new_unchecked(token: &'t str, offset: usize) -> Self {
|
pub(crate) fn new_unchecked(token: &'t str, case: Case, offset: usize) -> Self {
|
||||||
Self { token, offset }
|
Self {
|
||||||
|
token,
|
||||||
|
case,
|
||||||
|
offset,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn token(&self) -> &str {
|
pub fn token(&self) -> &str {
|
||||||
self.token
|
self.token
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn case(&self) -> Case {
|
||||||
|
self.case
|
||||||
|
}
|
||||||
|
|
||||||
pub fn offset(&self) -> usize {
|
pub fn offset(&self) -> usize {
|
||||||
self.offset
|
self.offset
|
||||||
}
|
}
|
||||||
|
@ -127,6 +148,22 @@ impl WordMode {
|
||||||
WordMode::Boundary
|
WordMode::Boundary
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn case(self, last: WordMode) -> Case {
|
||||||
|
match (self, last) {
|
||||||
|
(WordMode::Uppercase, WordMode::Uppercase) => Case::Scream,
|
||||||
|
(WordMode::Uppercase, WordMode::Lowercase) => Case::Title,
|
||||||
|
(WordMode::Lowercase, WordMode::Lowercase) => Case::Lower,
|
||||||
|
(WordMode::Number, WordMode::Number) => Case::None,
|
||||||
|
(WordMode::Number, _)
|
||||||
|
| (_, WordMode::Number)
|
||||||
|
| (WordMode::Boundary, _)
|
||||||
|
| (_, WordMode::Boundary)
|
||||||
|
| (WordMode::Lowercase, WordMode::Uppercase) => {
|
||||||
|
unreachable!("Invalid case combination: ({:?}, {:?})", self, last)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn split_symbol(symbol: &str, offset: usize) -> impl Iterator<Item = Word<'_>> {
|
fn split_symbol(symbol: &str, offset: usize) -> impl Iterator<Item = Word<'_>> {
|
||||||
|
@ -135,6 +172,7 @@ fn split_symbol(symbol: &str, offset: usize) -> impl Iterator<Item = Word<'_>> {
|
||||||
let mut char_indices = symbol.char_indices().peekable();
|
let mut char_indices = symbol.char_indices().peekable();
|
||||||
let mut start = 0;
|
let mut start = 0;
|
||||||
let mut start_mode = WordMode::Boundary;
|
let mut start_mode = WordMode::Boundary;
|
||||||
|
let mut last_mode = WordMode::Boundary;
|
||||||
while let Some((i, c)) = char_indices.next() {
|
while let Some((i, c)) = char_indices.next() {
|
||||||
let cur_mode = WordMode::classify(c);
|
let cur_mode = WordMode::classify(c);
|
||||||
if cur_mode == WordMode::Boundary {
|
if cur_mode == WordMode::Boundary {
|
||||||
|
@ -143,13 +181,16 @@ fn split_symbol(symbol: &str, offset: usize) -> impl Iterator<Item = Word<'_>> {
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if start_mode == WordMode::Boundary {
|
||||||
|
start_mode = cur_mode;
|
||||||
|
}
|
||||||
|
|
||||||
if let Some(&(next_i, next)) = char_indices.peek() {
|
if let Some(&(next_i, next)) = char_indices.peek() {
|
||||||
// The mode including the current character, assuming the current character does
|
// The mode including the current character, assuming the current character does
|
||||||
// not result in a word boundary.
|
// not result in a word boundary.
|
||||||
let next_mode = WordMode::classify(next);
|
let next_mode = WordMode::classify(next);
|
||||||
|
|
||||||
match (start_mode, cur_mode, next_mode) {
|
match (last_mode, cur_mode, next_mode) {
|
||||||
// cur_mode is last of current word
|
// cur_mode is last of current word
|
||||||
(_, _, WordMode::Boundary)
|
(_, _, WordMode::Boundary)
|
||||||
| (_, WordMode::Lowercase, WordMode::Number)
|
| (_, WordMode::Lowercase, WordMode::Number)
|
||||||
|
@ -157,24 +198,36 @@ fn split_symbol(symbol: &str, offset: usize) -> impl Iterator<Item = Word<'_>> {
|
||||||
| (_, WordMode::Number, WordMode::Lowercase)
|
| (_, WordMode::Number, WordMode::Lowercase)
|
||||||
| (_, WordMode::Number, WordMode::Uppercase)
|
| (_, WordMode::Number, WordMode::Uppercase)
|
||||||
| (_, WordMode::Lowercase, WordMode::Uppercase) => {
|
| (_, WordMode::Lowercase, WordMode::Uppercase) => {
|
||||||
result.push(Word::new_unchecked(&symbol[start..next_i], start + offset));
|
let case = start_mode.case(cur_mode);
|
||||||
|
result.push(Word::new_unchecked(
|
||||||
|
&symbol[start..next_i],
|
||||||
|
case,
|
||||||
|
start + offset,
|
||||||
|
));
|
||||||
start = next_i;
|
start = next_i;
|
||||||
start_mode = WordMode::Boundary;
|
start_mode = WordMode::Boundary;
|
||||||
|
last_mode = WordMode::Boundary;
|
||||||
}
|
}
|
||||||
// cur_mode is start of next word
|
// cur_mode is start of next word
|
||||||
(WordMode::Uppercase, WordMode::Uppercase, WordMode::Lowercase) => {
|
(WordMode::Uppercase, WordMode::Uppercase, WordMode::Lowercase) => {
|
||||||
result.push(Word::new_unchecked(&symbol[start..i], start + offset));
|
result.push(Word::new_unchecked(
|
||||||
|
&symbol[start..i],
|
||||||
|
Case::Scream,
|
||||||
|
start + offset,
|
||||||
|
));
|
||||||
start = i;
|
start = i;
|
||||||
start_mode = WordMode::Boundary;
|
start_mode = cur_mode;
|
||||||
|
last_mode = WordMode::Boundary;
|
||||||
}
|
}
|
||||||
// No word boundary
|
// No word boundary
|
||||||
(_, _, _) => {
|
(_, _, _) => {
|
||||||
start_mode = cur_mode;
|
last_mode = cur_mode;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Collect trailing characters as a word
|
// Collect trailing characters as a word
|
||||||
result.push(Word::new_unchecked(&symbol[start..], start + offset));
|
let case = start_mode.case(cur_mode);
|
||||||
|
result.push(Word::new_unchecked(&symbol[start..], case, start + offset));
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -240,23 +293,65 @@ mod test {
|
||||||
#[test]
|
#[test]
|
||||||
fn split_symbol() {
|
fn split_symbol() {
|
||||||
let cases = [
|
let cases = [
|
||||||
("lowercase", &["lowercase"] as &[&str]),
|
(
|
||||||
("Class", &["Class"]),
|
"lowercase",
|
||||||
("MyClass", &["My", "Class"]),
|
&[("lowercase", Case::Lower, 0usize)] as &[(&str, Case, usize)],
|
||||||
("MyC", &["My", "C"]),
|
),
|
||||||
("HTML", &["HTML"]),
|
("Class", &[("Class", Case::Title, 0)]),
|
||||||
("PDFLoader", &["PDF", "Loader"]),
|
(
|
||||||
("AString", &["A", "String"]),
|
"MyClass",
|
||||||
("SimpleXMLParser", &["Simple", "XML", "Parser"]),
|
&[("My", Case::Title, 0), ("Class", Case::Title, 2)],
|
||||||
("vimRPCPlugin", &["vim", "RPC", "Plugin"]),
|
),
|
||||||
("GL11Version", &["GL", "11", "Version"]),
|
("MyC", &[("My", Case::Title, 0), ("C", Case::Scream, 2)]),
|
||||||
("99Bottles", &["99", "Bottles"]),
|
("HTML", &[("HTML", Case::Scream, 0)]),
|
||||||
("May5", &["May", "5"]),
|
(
|
||||||
("BFG9000", &["BFG", "9000"]),
|
"PDFLoader",
|
||||||
|
&[("PDF", Case::Scream, 0), ("Loader", Case::Title, 3)],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"AString",
|
||||||
|
&[("A", Case::Scream, 0), ("String", Case::Title, 1)],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"SimpleXMLParser",
|
||||||
|
&[
|
||||||
|
("Simple", Case::Title, 0),
|
||||||
|
("XML", Case::Scream, 6),
|
||||||
|
("Parser", Case::Title, 9),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"vimRPCPlugin",
|
||||||
|
&[
|
||||||
|
("vim", Case::Lower, 0),
|
||||||
|
("RPC", Case::Scream, 3),
|
||||||
|
("Plugin", Case::Title, 6),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"GL11Version",
|
||||||
|
&[
|
||||||
|
("GL", Case::Scream, 0),
|
||||||
|
("11", Case::None, 2),
|
||||||
|
("Version", Case::Title, 4),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"99Bottles",
|
||||||
|
&[("99", Case::None, 0), ("Bottles", Case::Title, 2)],
|
||||||
|
),
|
||||||
|
("May5", &[("May", Case::Title, 0), ("5", Case::None, 3)]),
|
||||||
|
(
|
||||||
|
"BFG9000",
|
||||||
|
&[("BFG", Case::Scream, 0), ("9000", Case::None, 3)],
|
||||||
|
),
|
||||||
];
|
];
|
||||||
for (input, expected) in cases.iter() {
|
for (input, expected) in cases.iter() {
|
||||||
let symbol = Symbol::new(input, 0).unwrap();
|
let symbol = Symbol::new(input, 0).unwrap();
|
||||||
let result: Vec<_> = symbol.split().map(|w| w.token).collect();
|
let result: Vec<_> = symbol
|
||||||
|
.split()
|
||||||
|
.map(|w| (w.token, w.case, w.offset))
|
||||||
|
.collect();
|
||||||
assert_eq!(&result, expected);
|
assert_eq!(&result, expected);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue