Merge pull request #88 from epage/threads

perf: Multi-threaded spell checking

azure-pipelines.yml

@@ -16,7 +16,7 @@ stages:
   jobs:
   - template: default.yml@templates
     parameters:
-      minrust: 1.36.0
+      minrust: 1.40.0
       codecov_token: $(CODECOV_TOKEN_SECRET)
 - stage: codegen
   displayName: Verify Code-gen

benchsuite/benchsuite.sh

@@ -90,12 +90,16 @@ function bench_dir() {
     echo "$name: $version" >> $output
     echo "" >> $output
     rg_command=""
+    rg_j1_command=""
     if [[ ! -z $rg_path ]]; then
       rg_command="$rg_path bin $path"
+      rg_j1_command="$rg_path --threads 1 bin $path"
     fi
     typos_command=""
+    typos_j1_command=""
     if [[ ! -z $typos_path ]]; then
       typos_command="$typos_path $path"
+      typos_j1_command="$typos_path --threads 1 $path"
     fi
     misspell_rs_command=""
     if [[ ! -z $misspell_rs_path ]]; then
@@ -110,7 +114,7 @@ function bench_dir() {
     if [[ ! -z $codespell_path ]]; then
       codespell_command="$codespell_path $path"
     fi
-    hyperfine --warmup 1 -i --export-json $report_prefix-rg.json --export-markdown $report_prefix-rg.md "$rg_command" "$typos_command" "$misspell_rs_command" "$misspell_go_command" "$codespell_command"
+    hyperfine --warmup 1 -i --export-json $report_prefix-rg.json --export-markdown $report_prefix-rg.md "$rg_command" "$rg_j1_command" "$typos_command" "$typos_j1_command" "$misspell_rs_command" "$misspell_go_command" "$codespell_command"
     cat $report_prefix-rg.md >> $output
   fi
   echo "" >> $output

src/main.rs

@@ -3,6 +3,7 @@
 extern crate clap;
 
 use std::io::Write;
+use std::sync::atomic;
 
 use structopt::StructOpt;
 
@@ -79,6 +80,10 @@ struct Args {
     )]
     pub format: Format,
 
+    #[structopt(short = "j", long = "threads", default_value = "0")]
+    /// The approximate number of threads to use.
+    threads: usize,
+
     #[structopt(flatten)]
     config: ConfigArgs,
 
@@ -86,6 +91,16 @@ struct Args {
     verbose: clap_verbosity_flag::Verbosity,
 }
 
+impl Args {
+    pub fn infer(mut self) -> Self {
+        if self.path.len() == 1 && self.path[0].is_file() {
+            self.threads = 1;
+        }
+
+        self
+    }
+}
+
 #[derive(Debug, StructOpt)]
 #[structopt(rename_all = "kebab-case")]
 pub struct FileArgs {
@@ -261,7 +276,7 @@ impl config::WalkSource for WalkArgs {
     }
 }
 
-trait Checks {
+trait Checks: Send + Sync {
     fn check_filename(
         &self,
         path: &std::path::Path,
@@ -378,7 +393,7 @@ fn check_path(
     checks: &dyn Checks,
     parser: &typos::tokens::Parser,
     dictionary: &dyn typos::Dictionary,
-) -> Result<(bool, bool), anyhow::Error> {
+) -> (bool, bool) {
     let mut typos_found = false;
     let mut errors_found = false;
 
@@ -394,7 +409,35 @@ fn check_path(
         }
     }
 
-    Ok((typos_found, errors_found))
+    (typos_found, errors_found)
+}
+
+fn check_path_parallel(
+    walk: ignore::WalkParallel,
+    format: Format,
+    checks: &dyn Checks,
+    parser: &typos::tokens::Parser,
+    dictionary: &dyn typos::Dictionary,
+) -> (bool, bool) {
+    let typos_found = atomic::AtomicBool::new(false);
+    let errors_found = atomic::AtomicBool::new(false);
+
+    walk.run(|| {
+        Box::new(|entry: Result<ignore::DirEntry, ignore::Error>| {
+            match check_entry(entry, format, checks, parser, dictionary) {
+                Ok(true) => typos_found.store(true, atomic::Ordering::Relaxed),
+                Err(err) => {
+                    let msg = typos::report::Error::new(err.to_string());
+                    format.report()(msg.into());
+                    errors_found.store(true, atomic::Ordering::Relaxed);
+                }
+                _ => (),
+            }
+            ignore::WalkState::Continue
+        })
+    });
+
+    (typos_found.into_inner(), errors_found.into_inner())
 }
 
 fn check_entry(
@@ -421,16 +464,15 @@ fn check_entry(
 }
 
 fn run() -> Result<i32, anyhow::Error> {
-    let args = Args::from_args();
+    let args = Args::from_args().infer();
 
     init_logging(args.verbose.log_level());
 
-    let mut config = config::Config::default();
+    let config = if let Some(path) = args.custom_config.as_ref() {
-    if let Some(path) = args.custom_config.as_ref() {
+        config::Config::from_file(path)?
-        let custom = config::Config::from_file(path)?;
+    } else {
-        config.update(&custom);
+        config::Config::default()
-    }
+    };
-    let config = config;
 
     let mut typos_found = false;
     let mut errors_found = false;
@@ -468,30 +510,63 @@ fn run() -> Result<i32, anyhow::Error> {
             .binary(config.files.binary());
 
         let mut walk = ignore::WalkBuilder::new(path);
-        walk.hidden(config.files.ignore_hidden())
+        walk.threads(args.threads)
+            .hidden(config.files.ignore_hidden())
             .ignore(config.files.ignore_dot())
             .git_global(config.files.ignore_global())
             .git_ignore(config.files.ignore_vcs())
             .git_exclude(config.files.ignore_vcs())
             .parents(config.files.ignore_parent());
+        let single_threaded = args.threads == 1;
         if args.files {
-            for entry in walk.build() {
+            if single_threaded {
-                match entry {
+                for entry in walk.build() {
-                    Ok(entry) => {
+                    match entry {
-                        let msg = typos::report::File::new(entry.path());
+                        Ok(entry) => {
-                        args.format.report()(msg.into());
+                            let msg = typos::report::File::new(entry.path());
-                    }
+                            args.format.report()(msg.into());
-                    Err(err) => {
+                        }
-                        let msg = typos::report::Error::new(err.to_string());
+                        Err(err) => {
-                        args.format.report()(msg.into());
+                            let msg = typos::report::Error::new(err.to_string());
-                        errors_found = true
+                            args.format.report()(msg.into());
+                            errors_found = true
+                        }
                     }
                 }
+            } else {
+                let format = args.format;
+                let atomic_errors = atomic::AtomicBool::new(errors_found);
+                walk.build_parallel().run(|| {
+                    Box::new(|entry: Result<ignore::DirEntry, ignore::Error>| {
+                        match entry {
+                            Ok(entry) => {
+                                let msg = typos::report::File::new(entry.path());
+                                format.report()(msg.into());
+                            }
+                            Err(err) => {
+                                let msg = typos::report::Error::new(err.to_string());
+                                format.report()(msg.into());
+                                atomic_errors.store(true, atomic::Ordering::Relaxed);
+                            }
+                        }
+                        ignore::WalkState::Continue
+                    })
+                });
+                errors_found = atomic_errors.into_inner();
             }
         } else if args.identifiers {
             let checks = settings.build_identifier_parser();
-            let (cur_typos, cur_errors) =
+            let (cur_typos, cur_errors) = if single_threaded {
-                check_path(walk.build(), args.format, &checks, &parser, &dictionary)?;
+                check_path(walk.build(), args.format, &checks, &parser, &dictionary)
+            } else {
+                check_path_parallel(
+                    walk.build_parallel(),
+                    args.format,
+                    &checks,
+                    &parser,
+                    &dictionary,
+                )
+            };
             if cur_typos {
                 typos_found = true;
             }
@@ -500,8 +575,17 @@ fn run() -> Result<i32, anyhow::Error> {
             }
         } else if args.words {
             let checks = settings.build_word_parser();
-            let (cur_typos, cur_errors) =
+            let (cur_typos, cur_errors) = if single_threaded {
-                check_path(walk.build(), args.format, &checks, &parser, &dictionary)?;
+                check_path(walk.build(), args.format, &checks, &parser, &dictionary)
+            } else {
+                check_path_parallel(
+                    walk.build_parallel(),
+                    args.format,
+                    &checks,
+                    &parser,
+                    &dictionary,
+                )
+            };
             if cur_typos {
                 typos_found = true;
             }
@@ -510,8 +594,17 @@ fn run() -> Result<i32, anyhow::Error> {
             }
         } else {
             let checks = settings.build_checks();
-            let (cur_typos, cur_errors) =
+            let (cur_typos, cur_errors) = if single_threaded {
-                check_path(walk.build(), args.format, &checks, &parser, &dictionary)?;
+                check_path(walk.build(), args.format, &checks, &parser, &dictionary)
+            } else {
+                check_path_parallel(
+                    walk.build_parallel(),
+                    args.format,
+                    &checks,
+                    &parser,
+                    &dictionary,
+                )
+            };
             if cur_typos {
                 typos_found = true;
             }