From c3ac3aa5d9c14fd80edc78eb3c4af8bea0849fa6 Mon Sep 17 00:00:00 2001 From: Kin Fai Tse Date: Tue, 31 Oct 2023 08:13:33 +0800 Subject: [PATCH] Batched & parallel support for cfn-lint, eslint, gitleaks (#4088) * faster linter for cfn-lint and eslint * workaround shfmt error * fix xargs interleave large outputs * parallel gitleaks * fix exec bit, shfmt, bash linter * show parallel --citation * refactor a common interface using named pipe * add readme for the experimental impl * fix readme format * minimize change in worker.sh * will cite, showed once * remove junk comment * explicitly set EXPERIMENTAL_BATCH_WORKER=false * fix: errors from github/super-linter:v5 --- Dockerfile | 1 + README.md | 1 + .../experimental-batch-workers/README.md | 58 +++++++++ .../experimental-batch-workers/base.sh | 116 ++++++++++++++++++ .../experimental-batch-workers/cfn-lint.sh | 57 +++++++++ .../experimental-batch-workers/eslint.sh | 51 ++++++++ .../experimental-batch-workers/gitleaks.sh | 60 +++++++++ lib/functions/worker.sh | 18 +++ lib/linter.sh | 14 ++- 9 files changed, 375 insertions(+), 1 deletion(-) create mode 100644 lib/functions/experimental-batch-workers/README.md create mode 100755 lib/functions/experimental-batch-workers/base.sh create mode 100755 lib/functions/experimental-batch-workers/cfn-lint.sh create mode 100755 lib/functions/experimental-batch-workers/eslint.sh create mode 100755 lib/functions/experimental-batch-workers/gitleaks.sh diff --git a/Dockerfile b/Dockerfile index 1476ef88..bc99cde7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -81,6 +81,7 @@ RUN apk add --no-cache \ openjdk11-jre \ openssh-client \ openssl-dev \ + parallel \ perl perl-dev \ py3-setuptools python3-dev \ py3-pyflakes \ diff --git a/README.md b/README.md index b3e2b0f8..c38222ab 100644 --- a/README.md +++ b/README.md @@ -286,6 +286,7 @@ But if you wish to select or exclude specific linters, we give you full control | **DOCKERFILE_HADOLINT_FILE_NAME** | `.hadolint.yaml` | Filename for [hadolint configuration](https://github.com/hadolint/hadolint) (ex: `.hadolintlintrc.yaml`) | | **EDITORCONFIG_FILE_NAME** | `.ecrc` | Filename for [editorconfig-checker configuration](https://github.com/editorconfig-checker/editorconfig-checker) | | **ERROR_ON_MISSING_EXEC_BIT** | `false` | If set to `false`, the `bash-exec` linter will report a warning if a shell script is not executable. If set to `true`, the `bash-exec` linter will report an error instead. | +| **EXPERIMENTAL_BATCH_WORKER** | `false` | Flag to enable experimental parallel and batched worker. As of current only `eslint` and `cfn-lint` are supported, if there is no support, original version is used as fallback | | **FILTER_REGEX_EXCLUDE** | `none` | Regular expression defining which files will be excluded from linting (ex: `.*src/test.*`) | | **FILTER_REGEX_INCLUDE** | `all` | Regular expression defining which files will be processed by linters (ex: `.*src/.*`) | | **GITHUB_ACTIONS_CONFIG_FILE** | `actionlint.yml` | Filename for [Actionlint configuration](https://github.com/rhysd/actionlint/blob/main/docs/config.md) (ex: `actionlint.yml`) | diff --git a/lib/functions/experimental-batch-workers/README.md b/lib/functions/experimental-batch-workers/README.md new file mode 100644 index 00000000..b4b5e54e --- /dev/null +++ b/lib/functions/experimental-batch-workers/README.md @@ -0,0 +1,58 @@ +# Parallel / Batched Workers + +Running linters in parallel, and if possible in batch to speed up the linting process. + +This is an experimental feature, and is not enabled by default, but it is really fast if you enable it and have the linter support implemented. + +Since it is a parallel version, it might not be possible to reproduce a line-by-line match of serial output. + +In order to maximize compatibility to programs using output of super-linter, the following are guarenteed: +- Every linter error reported in serial version is reported by parallel version [^linter-error]; +- Super-linter log-level WARN and above that appears in serial version should appear in this version; +- Super-linter log-level INFO and above do not interleave between linter output; +- Failed file count logged at the end of super-linter matches serial version; + +[^linter-error]: Statistics are almost impossible to reproduce, e.g. was always `1 file linted, K errors` but now `M files linted, K errors`, I guess it is fine as the stat for linting 1 file produced by linter is not very useful. + +## Motivation + +Some linter might have a high startup cost, e.g. +- `eslint` with some popular frontend framework plugins requires reading thousands of js files to init +- `cfn-lint` which requires reading the whole cloudformation spec to run + +A lot of linter supports linting multiple files per invocation, i.e. ` file1 file2 file3 ...`, which can be leveraged to reduce the startup overhead. + +Modern CI/CD might be on a multi-core machine, so running multiple linters in parallel can also speed up linting process, shorten the time taken from push to deploy. + +Shift-left paradigm encourages running linters in the IDE, for example in `.githooks/pre-commit`, linting need to be fast for good Developer experience. + +## Supported linters + +| Linter | Batch | Parallel | Notes | +| -------- | ----- | -------- | --------------------------- | +| cfn-lint | o | o | | +| ESLint | o | o | | +| gitleaks | | o | Batch unsupported by linter | + +## Architecture + +By setting `EXPERIMENTAL_BATCH_WORKER=true`, the following code path will be enabled: + +```bash +# ../worker.sh +LintCodebase + # ./${LinterName}.sh + # TASK: Modify linter command for batch, parallelization and batching parameters suitable for the linter + ParallelLintCodebase${LinterName} + # ./base.sh + # gnu parallel run + ParallelLintCodebaseImpl + # ./${LinterName}.sh + # TASK: see ./base.sh + LintCodebase${LinterName}StdoutParser + # ./${LinterName}.sh + # TASK: see ./base.sh + LintCodebase${LinterName}StderrParser + # ./base.sh if the default works for you + LintCodebaseBaseStderrParser +``` diff --git a/lib/functions/experimental-batch-workers/base.sh b/lib/functions/experimental-batch-workers/base.sh new file mode 100755 index 00000000..6d2552ea --- /dev/null +++ b/lib/functions/experimental-batch-workers/base.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash + +# stderr contains `parallel` command trace (starting with $LINTER_COMMAND) and linter's stderr +# +# implement to report error count and traces correctly +# +# IN: pipe from ${STDERR_PIPENAME} +# - multiline text input +# OUT: pipe to ${STDERR_PIPENAME}.return number of file with linter error +# - int: number of file with linter error +function LintCodebaseBaseStderrParser() { + local STDERR_PIPENAME="${1}" && shift + local LINTER_NAME="${1}" && shift + local LINTER_COMMAND="${1}" && shift + + # usually linter reports failing linter rules to stdout + # stderr contains uncaught linter errors e.g. invalid parameter, which shall indicate a bug in the parallel implementation + # as the origin of error is unknown, we shall count each instance of linter error as 1 file to alert user of an error + local UNCAUGHT_LINTER_ERRORS=0 + local LINE + while IFS= read -r LINE; do + if [[ "${LINE}" == "${LINTER_COMMAND}"* ]]; then + trace "[parallel] ${LINE}" + continue + fi + error "[${LINTER_NAME}] ${LINE//\\/\\\\}" + UNCAUGHT_LINTER_ERRORS="$((UNCAUGHT_LINTER_ERRORS + 1))" + done <"${STDERR_PIPENAME}" + + echo "${UNCAUGHT_LINTER_ERRORS}" >"${STDERR_PIPENAME}.return" + + return 0 +} + +# stdout is piped from linter's stdout +# * this stream is already `tee`-ed to stdout by caller as in serial super-linter behavior +# +# implement to report error count correctly +# +# IN: pipe from ${STDERR_PIPENAME} +# - multiline text input +# OUT: pipe to ${STDERR_PIPENAME}.return +# - int: number of file with linter error +function LintCodebaseBaseStdoutParser() { + local STDOUT_PIPENAME="${1}" && shift + local LINTER_NAME="${1}" && shift + + # this function is an example only to illustrate the interface + # should be implemented for each linter, do not use this + + # * you can use any way to parse the linter output as you like + fatal "LintCodebaseBaseStdoutParser is not implemented" + + echo 0 >"${STDOUT_PIPENAME}.return" + return 0 +} + +# This function runs linter in parallel and batch# +# To reproduce serial behavior, ERRORS_FOUND_${FILE_TYPE} should be calculated from linter output +# The calculation should not affect, break or interleave linter output in any way +# logging level below info is allowed to interleave linter output +function ParallelLintCodebaseImpl() { + local FILE_TYPE="${1}" && shift # File type (Example: JSON) + local LINTER_NAME="${1}" && shift # Linter name (Example: jsonlint) + local LINTER_COMMAND="${1}" && shift # Full linter command including linter name (Example: jsonlint -c ConfigFile /path/to/file) + # shellcheck disable=SC2034 + local TEST_CASE_RUN="${1}" && shift # Flag for if running in test cases + local NUM_PROC="${1}" && shift # Number of processes to run in parallel + local FILES_PER_PROC="${1}" && shift # Max. number of file to pass into one linter process, still subject to maximum of 65536 characters per command line, which parallel will handle for us + local STDOUT_PARSER="${1}" && shift # Function to parse stdout to count number of files with linter error + local STDERR_PARSER="${1}" && shift # Function to parse stderr to count number of files with linter error + local FILE_ARRAY=("$@") # Array of files to validate (Example: ${FILE_ARRAY_JSON}) + + debug "Running ParallelLintCodebaseImpl on ${#FILE_ARRAY[@]} files. FILE_TYPE: ${FILE_TYPE}, LINTER_NAME: ${LINTER_NAME}, LINTER_COMMAND: ${LINTER_COMMAND}, TEST_CASE_RUN: ${TEST_CASE_RUN}, NUM_PROC: ${NUM_PROC}, FILES_PER_PROC: ${FILES_PER_PROC}, STDOUT_PARSER: ${STDOUT_PARSER}, STDERR_PARSER: ${STDERR_PARSER}" + + local PARALLEL_DEBUG_OPTS="" + if [ "${LOG_TRACE}" == "true" ]; then + PARALLEL_DEBUG_OPTS="--verbose" + fi + local PARALLEL_COMMAND="parallel --will-cite --keep-order --max-lines ${FILES_PER_PROC} --max-procs ${NUM_PROC} ${PARALLEL_DEBUG_OPTS} --xargs ${LINTER_COMMAND}" + info "Parallel command: ${PARALLEL_COMMAND}" + + # named pipes for routing linter outputs and return values + local STDOUT_PIPENAME="/tmp/parallel-${FILE_TYPE,,}.stdout" + local STDERR_PIPENAME="/tmp/parallel-${FILE_TYPE,,}.stderr" + trace "Stdout pipe: ${STDOUT_PIPENAME}" + trace "Stderr pipe: ${STDERR_PIPENAME}" + mkfifo "${STDOUT_PIPENAME}" "${STDOUT_PIPENAME}.return" "${STDERR_PIPENAME}" "${STDERR_PIPENAME}.return" + + # start all functions in bg + "${STDOUT_PARSER}" "${STDOUT_PIPENAME}" "${LINTER_NAME}" & + "${STDERR_PARSER}" "${STDERR_PIPENAME}" "${LINTER_NAME}" "${LINTER_COMMAND}" & + # start linter in parallel + printf "%s\n" "${FILE_ARRAY[@]}" | ${PARALLEL_COMMAND} 2>"${STDERR_PIPENAME}" | tee "${STDOUT_PIPENAME}" & + + local UNCAUGHT_LINTER_ERRORS + local ERRORS_FOUND + # wait for all parsers to finish, should read a number from each pipe + IFS= read -r UNCAUGHT_LINTER_ERRORS <"${STDERR_PIPENAME}.return" + trace "UNCAUGHT_LINTER_ERRORS: ${UNCAUGHT_LINTER_ERRORS}" + IFS= read -r ERRORS_FOUND <"${STDOUT_PIPENAME}.return" + trace "ERRORS_FOUND: ${ERRORS_FOUND}" + # assert return values are integers >= 0 just in case some implementation error + if ! [[ "${ERRORS_FOUND}" =~ ^[0-9]+$ ]]; then + fatal "ERRORS_FOUND is not a number: ${ERRORS_FOUND}" + exit 1 + fi + if ! [[ "${UNCAUGHT_LINTER_ERRORS}" =~ ^[0-9]+$ ]]; then + fatal "UNCAUGHT_LINTER_ERRORS is not a number: ${UNCAUGHT_LINTER_ERRORS}" + exit 1 + fi + ERRORS_FOUND=$((ERRORS_FOUND + UNCAUGHT_LINTER_ERRORS)) + printf -v "ERRORS_FOUND_${FILE_TYPE}" "%d" "${ERRORS_FOUND}" + + return 0 +} diff --git a/lib/functions/experimental-batch-workers/cfn-lint.sh b/lib/functions/experimental-batch-workers/cfn-lint.sh new file mode 100755 index 00000000..7e71436f --- /dev/null +++ b/lib/functions/experimental-batch-workers/cfn-lint.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Sample cfn-lint v0.x output: +# +# E3002 Invalid Property Resources/Whatever/Properties/Is/Wrong +# ./path/to/my-stack.yml:35:7 +# +function LintCodebaseCfnLintStdoutParser() { + local STDOUT_PIPENAME="${1}" && shift + local LINTER_NAME="${1}" && shift + + local ERRORS_FOUND=0 + local IS_ERROR + local CUR_FILENAME + local NEXT_FILENAME + local LINE + while IFS= read -r LINE; do + if grep "[EW][0-9]\+[[:space:]]" <<<"$LINE" >/dev/null; then + IS_ERROR="true" + continue + fi + if grep "$PWD" <<<"$LINE" >/dev/null; then + NEXT_FILENAME=$(cut -d: -f1 <<<"$LINE") + if [[ "$NEXT_FILENAME" != "$CUR_FILENAME" ]]; then + CUR_FILENAME=$NEXT_FILENAME + if [[ "$IS_ERROR" == "true" ]]; then + IS_ERROR="false" + ERRORS_FOUND=$((ERRORS_FOUND + 1)) + fi + fi + continue + fi + done <"${STDOUT_PIPENAME}" + + echo "${ERRORS_FOUND}" >"${STDOUT_PIPENAME}.return" + return 0 +} + +function ParallelLintCodebaseCfnLint() { + local FILE_TYPE="${1}" && shift + local LINTER_NAME="${1}" && shift + local LINTER_COMMAND="${1}" && shift + local TEST_CASE_RUN="${1}" && shift + local FILE_ARRAY=("$@") + local NUM_PROC="$(($(nproc) * 1))" + local FILES_PER_PROC="16" + local STDOUT_PARSER="LintCodebaseCfnLintStdoutParser" + local STDERR_PARSER="LintCodebaseBaseStderrParser" + + info "Running EXPERIMENTAL parallel ${FILE_TYPE} LintCodebase on ${#FILE_ARRAY[@]} files. LINTER_NAME: ${LINTER_NAME}, LINTER_COMMAND: ${LINTER_COMMAND}, TEST_CASE_RUN: ${TEST_CASE_RUN}" + + ParallelLintCodebaseImpl "${FILE_TYPE}" "${LINTER_NAME}" "${LINTER_COMMAND}" "${TEST_CASE_RUN}" "${NUM_PROC}" "${FILES_PER_PROC}" "${STDOUT_PARSER}" "${STDERR_PARSER}" "${FILE_ARRAY[@]}" + + info "Exiting EXPERIMENTAL parallel ${FILE_TYPE} LintCodebase on ${#FILE_ARRAY[@]} files. ERROR_FOUND: ${ERRORS_FOUND}. LINTER_NAME: ${LINTER_NAME}, LINTER_COMMAND: ${LINTER_COMMAND}" + + return 0 +} diff --git a/lib/functions/experimental-batch-workers/eslint.sh b/lib/functions/experimental-batch-workers/eslint.sh new file mode 100755 index 00000000..5ac830c0 --- /dev/null +++ b/lib/functions/experimental-batch-workers/eslint.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Sample eslint output: +# +# /path/to/failed.js +# 11:5 error 'a' is never reassigned. Use 'const' instead prefer-const +# 11:5 error 'a' is assigned a value but never used no-unused-vars +# +function LintCodebaseEslintStdoutParser() { + local STDOUT_PIPENAME="${1}" && shift + local LINTER_NAME="${1}" && shift + + local ERRORS_FOUND=0 + local CUR_FILE_COUNTED + local LINE + while IFS= read -r LINE; do + if grep "$PWD" <<<"$LINE" >/dev/null; then + CUR_FILE_COUNTED="false" + continue + fi + if grep "[[:space:]]\+[0-9]\+:[0-9]\+[[:space:]]\+error[[:space:]]\+" <<<"$LINE" >/dev/null; then + if [[ "$CUR_FILE_COUNTED" == "false" ]]; then + CUR_FILE_COUNTED="true" + ERRORS_FOUND=$((ERRORS_FOUND + 1)) + fi + fi + done <"${STDOUT_PIPENAME}" + + echo "${ERRORS_FOUND}" >"${STDOUT_PIPENAME}.return" + return 0 +} + +function ParallelLintCodebaseEslint() { + local FILE_TYPE="${1}" && shift + local LINTER_NAME="${1}" && shift + local LINTER_COMMAND="${1}" && shift + local TEST_CASE_RUN="${1}" && shift + local FILE_ARRAY=("$@") + local NUM_PROC="$(($(nproc) * 1))" + local FILES_PER_PROC="64" + local STDOUT_PARSER="LintCodebaseEslintStdoutParser" + local STDERR_PARSER="LintCodebaseBaseStderrParser" + + info "Running EXPERIMENTAL parallel ${FILE_TYPE} LintCodebase on ${#FILE_ARRAY[@]} files. LINTER_NAME: ${LINTER_NAME}, LINTER_COMMAND: ${LINTER_COMMAND}, TEST_CASE_RUN: ${TEST_CASE_RUN}" + + ParallelLintCodebaseImpl "${FILE_TYPE}" "${LINTER_NAME}" "${LINTER_COMMAND}" "${TEST_CASE_RUN}" "${NUM_PROC}" "${FILES_PER_PROC}" "${STDOUT_PARSER}" "${STDERR_PARSER}" "${FILE_ARRAY[@]}" + + info "Exiting EXPERIMENTAL parallel ${FILE_TYPE} LintCodebase on ${#FILE_ARRAY[@]} files. ERROR_FOUND: ${ERRORS_FOUND}. LINTER_NAME: ${LINTER_NAME}, LINTER_COMMAND: ${LINTER_COMMAND}" + + return 0 +} diff --git a/lib/functions/experimental-batch-workers/gitleaks.sh b/lib/functions/experimental-batch-workers/gitleaks.sh new file mode 100755 index 00000000..3504e9fc --- /dev/null +++ b/lib/functions/experimental-batch-workers/gitleaks.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +# gitleaks reports failing linter rules to stdout +# stderr contains uncaught linter errors e.g. invalid parameter, which shall indicate a bug in this script +# using default LintCodebaseBaseStderrParser + +# Sample gitleaks output: +# +# Finding: API_KEY=XXXXXXXXX +# Secret: XXXXXXXXX +# RuleID: generic-api-key +# Entropy: 1.000000 +# File: /tmp/lint/my-api-key.config +# Line: 1 +# Fingerprint: /tmp/lint/my-api-key.config:generic-api-key:1 +# +function LintCodebaseGitleaksStdoutParser() { + local STDOUT_PIPENAME="${1}" && shift + local LINTER_NAME="${1}" && shift + + # shellcheck disable=SC2155 + local ERRORS_FOUND=$( (grep "^File:[[:space:]]\+" | sort -u | wc -l) <"${STDOUT_PIPENAME}") + + echo "${ERRORS_FOUND}" >"${STDOUT_PIPENAME}.return" + return 0 +} + +function ParallelLintCodebaseGitleaks() { + local FILE_TYPE="${1}" && shift + local LINTER_NAME="${1}" && shift + local LINTER_COMMAND="${1}" && shift + local TEST_CASE_RUN="${1}" && shift + local FILE_ARRAY=("$@") + local NUM_PROC="$(($(nproc) * 1))" + local FILES_PER_PROC="1" # no file batching support for gitleaks + local STDOUT_PARSER="LintCodebaseGitleaksStdoutParser" + local STDERR_PARSER="LintCodebaseBaseStderrParser" + + info "Running EXPERIMENTAL parallel ${FILE_TYPE} LintCodebase on ${#FILE_ARRAY[@]} files. LINTER_NAME: ${LINTER_NAME}, LINTER_COMMAND: ${LINTER_COMMAND}, TEST_CASE_RUN: ${TEST_CASE_RUN}" + + local MODIFIED_LINTER_COMMAND="${LINTER_COMMAND}" + MODIFIED_LINTER_COMMAND=${MODIFIED_LINTER_COMMAND//--source/} + MODIFIED_LINTER_COMMAND=${MODIFIED_LINTER_COMMAND//-s/} + + warn "Gitleaks output \"WRN leaks found: \" is suppressed in parallel mode" + MODIFIED_LINTER_COMMAND=${MODIFIED_LINTER_COMMAND//--verbose/} + MODIFIED_LINTER_COMMAND=${MODIFIED_LINTER_COMMAND//-v/} + # shellcheck disable=SC2001 + MODIFIED_LINTER_COMMAND=$(sed "s/\-\(-log-level\|l\) \(info\|warn\)//g" <<<"${MODIFIED_LINTER_COMMAND}") + MODIFIED_LINTER_COMMAND="${MODIFIED_LINTER_COMMAND} -v -l error -s" + MODIFIED_LINTER_COMMAND=$(tr -s ' ' <<<"${MODIFIED_LINTER_COMMAND}" | xargs) + debug "Linter command updated from: ${LINTER_COMMAND}" + debug "to: ${MODIFIED_LINTER_COMMAND}" + + ParallelLintCodebaseImpl "${FILE_TYPE}" "${LINTER_NAME}" "${MODIFIED_LINTER_COMMAND}" "${TEST_CASE_RUN}" "${NUM_PROC}" "${FILES_PER_PROC}" "${STDOUT_PARSER}" "${STDERR_PARSER}" "${FILE_ARRAY[@]}" + + info "Exiting EXPERIMENTAL parallel ${FILE_TYPE} LintCodebase on ${#FILE_ARRAY[@]} files. ERROR_FOUND: ${ERRORS_FOUND}. LINTER_NAME: ${LINTER_NAME}, LINTER_COMMAND: ${LINTER_COMMAND}" + + return 0 +} diff --git a/lib/functions/worker.sh b/lib/functions/worker.sh index 0ff6357d..ff537c9b 100755 --- a/lib/functions/worker.sh +++ b/lib/functions/worker.sh @@ -21,6 +21,7 @@ function LintCodebase() { FILTER_REGEX_INCLUDE="${1}" && shift # Pull the variable and remove from array path (Example: */src/*,*/test/*) FILTER_REGEX_EXCLUDE="${1}" && shift # Pull the variable and remove from array path (Example: */examples/*,*/test/*.test) TEST_CASE_RUN="${1}" && shift # Flag for if running in test cases + EXPR_BATCH_WORKER="${1}" && shift # Flag for if running in experimental batch worker FILE_ARRAY=("$@") # Array of files to validate (Example: ${FILE_ARRAY_JSON}) ########################## @@ -84,6 +85,23 @@ function LintCodebase() { info "----------------------------------------------" info "----------------------------------------------" + # TODO: When testing in experimental batch mode, for implemented linters should filter out these files + # if [[ ${FILE} != *"${TEST_CASE_DIRECTORY}"* ]] && [ "${TEST_CASE_RUN}" == "true" ]; then + # debug "Skipping ${FILE} because it's not in the test case directory for ${FILE_TYPE}..." + # continue + # fi + # TODO: How to test $EXPR_BATCH_WORKER == true, now just skip it + if [ "$EXPR_BATCH_WORKER" == "true" ] && [ "${LINTER_NAME}" == "cfn-lint" ]; then + ParallelLintCodebaseCfnLint "${FILE_TYPE}" "${LINTER_NAME}" "${LINTER_COMMAND}" "${TEST_CASE_RUN}" "${FILE_ARRAY[@]}" + return 0 + elif [ "$EXPR_BATCH_WORKER" == "true" ] && [ "${LINTER_NAME}" == "eslint" ]; then + ParallelLintCodebaseEslint "${FILE_TYPE}" "${LINTER_NAME}" "${LINTER_COMMAND}" "${TEST_CASE_RUN}" "${FILE_ARRAY[@]}" + return 0 + elif [ "$EXPR_BATCH_WORKER" == "true" ] && [ "${LINTER_NAME}" == "gitleaks" ]; then + ParallelLintCodebaseGitleaks "${FILE_TYPE}" "${LINTER_NAME}" "${LINTER_COMMAND}" "${TEST_CASE_RUN}" "${FILE_ARRAY[@]}" + return 0 + fi + ################## # Lint the files # ################## diff --git a/lib/linter.sh b/lib/linter.sh index 3d9aaa00..4e1b171b 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -64,6 +64,11 @@ source /action/lib/functions/validation.sh # Source the function script(s) source /action/lib/functions/worker.sh # Source the function script(s) # shellcheck source=/dev/null source /action/lib/functions/setupSSH.sh # Source the function script(s) +# shellcheck source=/dev/null +for batch_worker_script in /action/lib/functions/experimental-batch-workers/*.sh; do + # shellcheck source=/dev/null + source "$batch_worker_script" +done ########### # GLOBALS # @@ -1046,6 +1051,13 @@ debug "ENV:" debug "${PRINTENV}" debug "------------------------------------" +if [ "${EXPERIMENTAL_BATCH_WORKER}" == "true" ]; then + # we have showed citation once, so every other parallel call will use --will-cite + info parallel --citation +else + EXPERIMENTAL_BATCH_WORKER="false" +fi + for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do debug "Running linter for the ${LANGUAGE} language..." VALIDATE_LANGUAGE_VARIABLE_NAME="VALIDATE_${LANGUAGE}" @@ -1093,7 +1105,7 @@ for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do debug "${FILE_ARRAY_VARIABLE_NAME} file array contents: ${!LANGUAGE_FILE_ARRAY}" debug "Invoking ${LINTER_NAME} linter. TEST_CASE_RUN: ${TEST_CASE_RUN}" - LintCodebase "${LANGUAGE}" "${LINTER_NAME}" "${LINTER_COMMAND}" "${FILTER_REGEX_INCLUDE}" "${FILTER_REGEX_EXCLUDE}" "${TEST_CASE_RUN}" "${!LANGUAGE_FILE_ARRAY}" + LintCodebase "${LANGUAGE}" "${LINTER_NAME}" "${LINTER_COMMAND}" "${FILTER_REGEX_INCLUDE}" "${FILTER_REGEX_EXCLUDE}" "${TEST_CASE_RUN}" "${EXPERIMENTAL_BATCH_WORKER}" "${!LANGUAGE_FILE_ARRAY}" fi done