mirror of
https://github.com/crate-ci/typos.git
synced 2024-11-21 16:41:01 -05:00
refactor(dict): Change typos-dict to trie
This is +/- 15%, depending on the benchmark.
This commit is contained in:
Ed Page
parent
fa1119aa47
commit
c8d1058a71
6 changed files with 91836 additions and 33843 deletions
113
Cargo.lock
generated
113
Cargo.lock
generated
|
|
@ -23,7 +23,7 @@ version = "0.7.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98"
|
||||
dependencies = [
|
||||
"getrandom 0.2.3",
|
||||
"getrandom",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
]
|
||||
|
|
@ -578,17 +578,6 @@ version = "1.1.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7"
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.1.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi 0.9.0+wasi-snapshot-preview1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.3"
|
||||
|
|
@ -597,7 +586,7 @@ checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753"
|
|||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi 0.10.2+wasi-snapshot-preview1",
|
||||
"wasi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -913,26 +902,6 @@ dependencies = [
|
|||
"phf_shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_codegen"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815"
|
||||
dependencies = [
|
||||
"phf_generator",
|
||||
"phf_shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526"
|
||||
dependencies = [
|
||||
"phf_shared",
|
||||
"rand 0.7.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.8.0"
|
||||
|
|
@ -1066,20 +1035,6 @@ version = "0.5.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8"
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
|
||||
dependencies = [
|
||||
"getrandom 0.1.16",
|
||||
"libc",
|
||||
"rand_chacha 0.2.2",
|
||||
"rand_core 0.5.1",
|
||||
"rand_hc 0.2.0",
|
||||
"rand_pcg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.3"
|
||||
|
|
@ -1087,19 +1042,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rand_chacha 0.3.0",
|
||||
"rand_core 0.6.2",
|
||||
"rand_hc 0.3.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core 0.5.1",
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
"rand_hc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1109,16 +1054,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core 0.6.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
|
||||
dependencies = [
|
||||
"getrandom 0.1.16",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1127,16 +1063,7 @@ version = "0.6.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7"
|
||||
dependencies = [
|
||||
"getrandom 0.2.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_hc"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
|
||||
dependencies = [
|
||||
"rand_core 0.5.1",
|
||||
"getrandom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1145,16 +1072,7 @@ version = "0.3.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73"
|
||||
dependencies = [
|
||||
"rand_core 0.6.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_pcg"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429"
|
||||
dependencies = [
|
||||
"rand_core 0.5.1",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1400,7 +1318,7 @@ checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
|
|||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"rand 0.8.3",
|
||||
"rand",
|
||||
"redox_syscall",
|
||||
"remove_dir_all",
|
||||
"winapi",
|
||||
|
|
@ -1548,8 +1466,8 @@ dependencies = [
|
|||
name = "typos-dict"
|
||||
version = "0.5.0"
|
||||
dependencies = [
|
||||
"dictgen",
|
||||
"log",
|
||||
"phf",
|
||||
"unicase",
|
||||
]
|
||||
|
||||
|
|
@ -1559,9 +1477,8 @@ version = "1.3.0"
|
|||
dependencies = [
|
||||
"codegenrs",
|
||||
"csv",
|
||||
"dictgen",
|
||||
"itertools 0.10.0",
|
||||
"phf",
|
||||
"phf_codegen",
|
||||
"structopt",
|
||||
"unicase",
|
||||
]
|
||||
|
|
@ -1646,7 +1563,7 @@ version = "0.8.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
|
||||
dependencies = [
|
||||
"getrandom 0.2.3",
|
||||
"getrandom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1705,12 +1622,6 @@ dependencies = [
|
|||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.9.0+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.10.2+wasi-snapshot-preview1"
|
||||
|
|
|
|||
|
|
@ -15,6 +15,6 @@ azure-devops = { project = "crate-ci", pipeline = "typos" }
|
|||
codecov = { repository = "crate-ci/typos" }
|
||||
|
||||
[dependencies]
|
||||
phf = { version = "0.8", features = ["unicase"] }
|
||||
unicase = "2.5"
|
||||
log = "0.4"
|
||||
dictgen = { version = "0.1", path = "../dictgen" }
|
||||
|
|
|
|||
|
|
@ -18,10 +18,9 @@ codecov = { repository = "crate-ci/typos" }
|
|||
disable-release = true
|
||||
|
||||
[dependencies]
|
||||
phf = { version = "0.8", features = ["unicase"] }
|
||||
phf_codegen = "0.8"
|
||||
csv = "1.1"
|
||||
itertools = "0.10"
|
||||
unicase = "2.5"
|
||||
codegenrs = "1.0"
|
||||
structopt = "0.3"
|
||||
dictgen = { version = "0.1", path = "../../dictgen" }
|
||||
|
|
|
|||
|
|
@ -11,17 +11,7 @@ fn generate<W: std::io::Write>(file: &mut W) {
|
|||
.unwrap();
|
||||
writeln!(file, "#![allow(clippy::unreadable_literal)]",).unwrap();
|
||||
writeln!(file).unwrap();
|
||||
writeln!(file, "use unicase::UniCase;").unwrap();
|
||||
|
||||
let mut smallest = usize::MAX;
|
||||
let mut largest = usize::MIN;
|
||||
|
||||
writeln!(
|
||||
file,
|
||||
"pub static WORD_DICTIONARY: phf::Map<unicase::UniCase<&'static str>, &'static [&'static str]> = "
|
||||
)
|
||||
.unwrap();
|
||||
let mut builder = phf_codegen::Map::new();
|
||||
let records: Vec<_> = csv::ReaderBuilder::new()
|
||||
.has_headers(false)
|
||||
.flexible(true)
|
||||
|
|
@ -29,25 +19,20 @@ fn generate<W: std::io::Write>(file: &mut W) {
|
|||
.records()
|
||||
.map(|r| r.unwrap())
|
||||
.collect();
|
||||
for record in &records {
|
||||
let mut record_fields = record.iter();
|
||||
let key = record_fields.next().unwrap();
|
||||
smallest = std::cmp::min(smallest, key.len());
|
||||
largest = std::cmp::max(largest, key.len());
|
||||
let value = format!(
|
||||
"&[{}]",
|
||||
itertools::join(record_fields.map(|field| format!(r#""{}""#, field)), ", ")
|
||||
);
|
||||
builder.entry(unicase::UniCase::new(&record[0]), &value);
|
||||
}
|
||||
let codegenned = builder.build();
|
||||
writeln!(file, "{}", codegenned).unwrap();
|
||||
writeln!(file, ";").unwrap();
|
||||
writeln!(file).unwrap();
|
||||
writeln!(
|
||||
dictgen::generate_trie(
|
||||
file,
|
||||
"pub const WORD_RANGE: std::ops::RangeInclusive<usize> = {}..={};",
|
||||
smallest, largest
|
||||
"WORD",
|
||||
"&'static [&'static str]",
|
||||
records.iter().map(|record| {
|
||||
let mut record_fields = record.iter();
|
||||
let key = record_fields.next().unwrap();
|
||||
let value = format!(
|
||||
"&[{}]",
|
||||
itertools::join(record_fields.map(|field| format!(r#""{}""#, field)), ", ")
|
||||
);
|
||||
(key, value)
|
||||
}),
|
||||
64,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
22
src/dict.rs
22
src/dict.rs
|
|
@ -55,11 +55,7 @@ impl BuiltIn {
|
|||
impl BuiltIn {
|
||||
// Not using `Status` to avoid the allocations
|
||||
fn correct_with_dict(&self, word: unicase::UniCase<&str>) -> Option<&'static [&'static str]> {
|
||||
if typos_dict::WORD_RANGE.contains(&word.len()) {
|
||||
map_lookup(&typos_dict::WORD_DICTIONARY, word)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
typos_dict::WORD_TRIE.find(&word).copied()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -175,22 +171,6 @@ impl typos::Dictionary for BuiltIn {
|
|||
}
|
||||
}
|
||||
|
||||
fn map_lookup<V: Clone>(
|
||||
map: &'static phf::Map<UniCase<&'static str>, V>,
|
||||
key: unicase::UniCase<&str>,
|
||||
) -> Option<V> {
|
||||
// This transmute should be safe as `get` will not store the reference with
|
||||
// the expanded lifetime. This is due to `Borrow` being overly strict and
|
||||
// can't have an impl for `&'static str` to `Borrow<&'a str>`.
|
||||
//
|
||||
//
|
||||
// See https://github.com/rust-lang/rust/issues/28853#issuecomment-158735548
|
||||
unsafe {
|
||||
let key = ::std::mem::transmute::<_, unicase::UniCase<&'static str>>(key);
|
||||
map.get(&key).cloned()
|
||||
}
|
||||
}
|
||||
|
||||
fn case_correct(correction: &mut Cow<'_, str>, case: Case) {
|
||||
match case {
|
||||
Case::Lower | Case::None => (),
|
||||
|
|
|
|||
Loading…
Reference in a new issue