actions/typos
1
0
Fork 0
mirror of https://github.com/crate-ci/typos.git synced 2024-11-21 16:41:01 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

perf: Multi-threaded spell checking

Browse source 
Fixes #7
This commit is contained in:
Ed Page 2019-10-25 16:37:09 -06:00 committed by Ed Page
parent 333762f55c
commit 6b8047ee44
3 changed files with 481 additions and 427 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

785
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

6
benchsuite/benchsuite.sh
View file

@ -90,12 +90,16 @@ function bench_dir() {
echo "$name: $version" >> $output echo "$name: $version" >> $output
echo "" >> $output echo "" >> $output
rg_command="" rg_command=""
rg_j1_command=""
if [[ ! -z $rg_path ]]; then if [[ ! -z $rg_path ]]; then
rg_command="$rg_path bin $path" rg_command="$rg_path bin $path"
rg_j1_command="$rg_path --threads 1 bin $path"
fi fi
typos_command="" typos_command=""
typos_j1_command=""
if [[ ! -z $typos_path ]]; then if [[ ! -z $typos_path ]]; then
typos_command="$typos_path $path" typos_command="$typos_path $path"
typos_j1_command="$typos_path --threads 1 $path"
fi fi
misspell_rs_command="" misspell_rs_command=""
if [[ ! -z $misspell_rs_path ]]; then if [[ ! -z $misspell_rs_path ]]; then
@ -110,7 +114,7 @@ function bench_dir() {
if [[ ! -z $codespell_path ]]; then if [[ ! -z $codespell_path ]]; then
codespell_command="$codespell_path $path" codespell_command="$codespell_path $path"
fi fi
hyperfine --warmup 1 -i --export-json $report_prefix-rg.json --export-markdown $report_prefix-rg.md "$rg_command" "$typos_command" "$misspell_rs_command" "$misspell_go_command" "$codespell_command" hyperfine --warmup 1 -i --export-json $report_prefix-rg.json --export-markdown $report_prefix-rg.md "$rg_command" "$rg_j1_command" "$typos_command" "$typos_j1_command" "$misspell_rs_command" "$misspell_go_command" "$codespell_command"
cat $report_prefix-rg.md >> $output cat $report_prefix-rg.md >> $output
fi fi
echo "" >> $output echo "" >> $output

117
src/main.rs
View file

@ -3,6 +3,7 @@
extern crate clap; extern crate clap;
use std::io::Write; use std::io::Write;
use std::sync::atomic;
use structopt::StructOpt; use structopt::StructOpt;
@ -275,7 +276,7 @@ impl config::WalkSource for WalkArgs {
} }
} }
trait Checks { trait Checks: Send + Sync {
fn check_filename( fn check_filename(
&self, &self,
path: &std::path::Path, path: &std::path::Path,
@ -392,7 +393,7 @@ fn check_path(
checks: &dyn Checks, checks: &dyn Checks,
parser: &typos::tokens::Parser, parser: &typos::tokens::Parser,
dictionary: &dyn typos::Dictionary, dictionary: &dyn typos::Dictionary,
) -> Result<(bool, bool), anyhow::Error> { ) -> (bool, bool) {
let mut typos_found = false; let mut typos_found = false;
let mut errors_found = false; let mut errors_found = false;
@ -408,7 +409,35 @@ fn check_path(
} }
} }
Ok((typos_found, errors_found)) (typos_found, errors_found)
}
fn check_path_parallel(
walk: ignore::WalkParallel,
format: Format,
checks: &dyn Checks,
parser: &typos::tokens::Parser,
dictionary: &dyn typos::Dictionary,
) -> (bool, bool) {
let typos_found = atomic::AtomicBool::new(false);
let errors_found = atomic::AtomicBool::new(false);
walk.run(|| {
Box::new(|entry: Result<ignore::DirEntry, ignore::Error>| {
match check_entry(entry, format, checks, parser, dictionary) {
Ok(true) => typos_found.store(true, atomic::Ordering::Relaxed),
Err(err) => {
let msg = typos::report::Error::new(err.to_string());
format.report()(msg.into());
errors_found.store(true, atomic::Ordering::Relaxed);
}
_ => (),
}
ignore::WalkState::Continue
})
});
(typos_found.into_inner(), errors_found.into_inner())
} }
fn check_entry( fn check_entry(
@ -489,24 +518,56 @@ fn run() -> Result<i32, anyhow::Error> {
.git_ignore(config.files.ignore_vcs()) .git_ignore(config.files.ignore_vcs())
.git_exclude(config.files.ignore_vcs()) .git_exclude(config.files.ignore_vcs())
.parents(config.files.ignore_parent()); .parents(config.files.ignore_parent());
let single_threaded = args.threads == 1;
if args.files { if args.files {
for entry in walk.build() { if single_threaded {
match entry { for entry in walk.build() {
Ok(entry) => { match entry {
let msg = typos::report::File::new(entry.path()); Ok(entry) => {
args.format.report()(msg.into()); let msg = typos::report::File::new(entry.path());
} args.format.report()(msg.into());
Err(err) => { }
let msg = typos::report::Error::new(err.to_string()); Err(err) => {
args.format.report()(msg.into()); let msg = typos::report::Error::new(err.to_string());
errors_found = true args.format.report()(msg.into());
errors_found = true
}
} }
} }
} else {
let format = args.format;
let atomic_errors = atomic::AtomicBool::new(errors_found);
walk.build_parallel().run(|| {
Box::new(|entry: Result<ignore::DirEntry, ignore::Error>| {
match entry {
Ok(entry) => {
let msg = typos::report::File::new(entry.path());
format.report()(msg.into());
}
Err(err) => {
let msg = typos::report::Error::new(err.to_string());
format.report()(msg.into());
atomic_errors.store(true, atomic::Ordering::Relaxed);
}
}
ignore::WalkState::Continue
})
});
errors_found = atomic_errors.into_inner();
} }
} else if args.identifiers { } else if args.identifiers {
let checks = settings.build_identifier_parser(); let checks = settings.build_identifier_parser();
let (cur_typos, cur_errors) = let (cur_typos, cur_errors) = if single_threaded {
check_path(walk.build(), args.format, &checks, &parser, &dictionary)?; check_path(walk.build(), args.format, &checks, &parser, &dictionary)
} else {
check_path_parallel(
walk.build_parallel(),
args.format,
&checks,
&parser,
&dictionary,
)
};
if cur_typos { if cur_typos {
typos_found = true; typos_found = true;
} }
@ -515,8 +576,17 @@ fn run() -> Result<i32, anyhow::Error> {
} }
} else if args.words { } else if args.words {
let checks = settings.build_word_parser(); let checks = settings.build_word_parser();
let (cur_typos, cur_errors) = let (cur_typos, cur_errors) = if single_threaded {
check_path(walk.build(), args.format, &checks, &parser, &dictionary)?; check_path(walk.build(), args.format, &checks, &parser, &dictionary)
} else {
check_path_parallel(
walk.build_parallel(),
args.format,
&checks,
&parser,
&dictionary,
)
};
if cur_typos { if cur_typos {
typos_found = true; typos_found = true;
} }
@ -525,8 +595,17 @@ fn run() -> Result<i32, anyhow::Error> {
} }
} else { } else {
let checks = settings.build_checks(); let checks = settings.build_checks();
let (cur_typos, cur_errors) = let (cur_typos, cur_errors) = if single_threaded {
check_path(walk.build(), args.format, &checks, &parser, &dictionary)?; check_path(walk.build(), args.format, &checks, &parser, &dictionary)
} else {
check_path_parallel(
walk.build_parallel(),
args.format,
&checks,
&parser,
&dictionary,
)
};
if cur_typos { if cur_typos {
typos_found = true; typos_found = true;
} }