feat: run linters in parallel (#5177)

.github/linters/.checkov-test-linters-failure.yaml

@@ -1,5 +1,9 @@
 ---
 # Options reference: https://www.checkov.io/2.Basics/CLI%20Command%20Reference.html
 
+directory:
+  - test/linters/checkov/bad
+
 quiet: false
+
 ...

.github/linters/.checkov-test-linters-success.yaml

@@ -0,0 +1,8 @@
+---
+# Options reference: https://www.checkov.io/2.Basics/CLI%20Command%20Reference.html
+
+directory:
+  - test/linters/checkov/good
+
+quiet: false
+...

.github/linters/.gitleaks-ignore-tests.toml

@@ -0,0 +1,13 @@
+
+title = "gitleaks config"
+
+[extend]
+# useDefault will extend the base configuration with the default gitleaks config:
+# https://github.com/zricethezav/gitleaks/blob/master/config/gitleaks.toml
+useDefault = true
+
+[allowlist]
+description = "Allow secrets in test files"
+paths = [
+  '''.*/test/linters/gitleaks/bad/.*'''
+]

.github/workflows/ci.yml

@@ -110,6 +110,7 @@ jobs:
           VALIDATE_ALL_CODEBASE: false
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           DEFAULT_BRANCH: main
+          GITLEAKS_CONFIG_FILE: .gitleaks-ignore-tests.toml
           RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES: "default.json,hoge.json"
           TYPESCRIPT_STANDARD_TSCONFIG_FILE: ".github/linters/tsconfig.json"
 

.gitignore

@@ -77,6 +77,9 @@ super-linter.report
 # Code coverage data for tests
 .coverage
 
+# Terraform workspace
+.terraform
+
 # Test reports
 test/reports
 

.gitleaksignore

@@ -1,9 +0,0 @@
-.github-personal-access-token:github-fine-grained-pat:1
-/github/workspace/.github-personal-access-token:github-fine-grained-pat:1
-/github/workspace/test/linters/gitleaks/bad/gitleaks_bad_01.txt:aws-access-token:1
-/github/workspace/test/linters/gitleaks/bad/gitleaks_bad_01.txt:generic-api-key:2
-/tmp/lint/.github-personal-access-token:github-fine-grained-pat:1
-/tmp/lint/test/linters/gitleaks/bad/gitleaks_bad_01.txt:aws-access-token:1
-/tmp/lint/test/linters/gitleaks/bad/gitleaks_bad_01.txt:generic-api-key:2
-test/linters/gitleaks/bad/gitleaks_bad_01.txt:aws-access-token:1
-test/linters/gitleaks/bad/gitleaks_bad_01.txt:generic-api-key:2

Dockerfile

@@ -134,6 +134,7 @@ RUN apk add --no-cache \
     nodejs-current \
     openjdk17-jre \
     openssh-client \
+    parallel \
     perl \
     php82 \
     php82-ctype \

Makefile

@@ -164,6 +164,7 @@ test-git-flags: ## Run super-linter with different git-related flags
 		-e ACTIONS_RUNNER_DEBUG=true \
 		-e ERROR_ON_MISSING_EXEC_BIT=true \
 		-e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true \
+		-e FILTER_REGEX_EXCLUDE=".*/test/linters/.*" \
 		-e DEFAULT_BRANCH=main \
 		-e IGNORE_GENERATED_FILES=true \
 		-e IGNORE_GITIGNORED_FILES=true \
@@ -178,6 +179,8 @@ lint-codebase: ## Lint the entire codebase
 		-e ACTIONS_RUNNER_DEBUG=true \
 		-e DEFAULT_BRANCH=main \
 		-e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true \
+		-e FILTER_REGEX_EXCLUDE=".*/test/linters/.*" \
+		-e GITLEAKS_CONFIG_FILE=".gitleaks-ignore-tests.toml" \
 		-e RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES="default.json,hoge.json" \
 		-e VALIDATE_ALL_CODEBASE=true \
 		-v "$(CURDIR):/tmp/lint" \
@@ -195,6 +198,7 @@ lint-subset-files-enable-only-one-type: ## Lint a small subset of files in the c
 		-e ACTIONS_RUNNER_DEBUG=true \
 		-e DEFAULT_BRANCH=main \
 		-e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true \
+		-e FILTER_REGEX_EXCLUDE=".*/test/linters/.*" \
 		-e VALIDATE_ALL_CODEBASE=true \
 		-e VALIDATE_MARKDOWN=true \
 		-v "$(CURDIR):/tmp/lint" \
@@ -207,6 +211,7 @@ lint-subset-files-enable-expensive-io-checks: ## Lint a small subset of files in
 		-e ACTIONS_RUNNER_DEBUG=true \
 		-e DEFAULT_BRANCH=main \
 		-e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true \
+		-e FILTER_REGEX_EXCLUDE=".*/test/linters/.*" \
 		-e VALIDATE_ALL_CODEBASE=true \
 		-e VALIDATE_ARM=true \
 		-e VALIDATE_CLOUDFORMATION=true \
@@ -272,19 +277,19 @@ test-custom-ssl-cert: ## Test the configuration of a custom SSL/TLS certificate
 		$(SUPER_LINTER_TEST_CONTAINER_URL)
 
 .phony: test-linters
-test-linters: ## Run the linters test suite
+test-linters: test-linters-expect-success test-linters-expect-failure ## Run the linters test suite
-	docker run \
+
-		-e ACTIONS_RUNNER_DEBUG=true \
+.phony: test-linters-expect-success
-		-e CHECKOV_FILE_NAME=".checkov-test-linters.yaml" \
+test-linters-expect-success: ## Run the linters test suite expecting successes
-		-e DEFAULT_BRANCH=main \
+	$(CURDIR)/test/run-super-linter-tests.sh \
-		-e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true \
+		$(SUPER_LINTER_TEST_CONTAINER_URL) \
-		-e JSCPD_CONFIG_FILE=".jscpd-test-linters.json" \
+		"run_test_cases_expect_success"
-		-e RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES="default.json,hoge.json" \
+
-		-e RUN_LOCAL=true \
+.phony: test-linters-expect-failure
-		-e TEST_CASE_RUN=true \
+test-linters-expect-failure: ## Run the linters test suite expecting failures
-		-e TYPESCRIPT_STANDARD_TSCONFIG_FILE=".github/linters/tsconfig.json" \
+	$(CURDIR)/test/run-super-linter-tests.sh \
-		-v "$(CURDIR):/tmp/lint" \
+		$(SUPER_LINTER_TEST_CONTAINER_URL) \
-		$(SUPER_LINTER_TEST_CONTAINER_URL)
+		"run_test_cases_expect_failure"
 
 .phony: build-dev-container-image
 build-dev-container-image: ## Build commit linter container image

docs/add-new-linter.md

@@ -90,6 +90,7 @@ new tool, it should include:
 - Update the orchestration scripts to run the new tool:
 
   - `lib/linter.sh`
+  - `lib/functions/linterCommands.sh`
   - Provide the logic to populate the list of files or directories to examine: `lib/buildFileList.sh`
   - If necessary, provide elaborate logic to detect if the tool should examine a file or a directory: `lib/detectFiles.sh`
   - If the tool needs to take into account special cases:

docs/upgrade-guide.md

@@ -52,10 +52,6 @@ This section helps you migrate from super-linter `v5` to `v6`.
 - If you defined secret patterns in `.gitleaks.toml`, Gitleaks may report errors
   about that file. If this happens, you can
   [configure Gitleaks to ignore that file](https://github.com/gitleaks/gitleaks/tree/master?tab=readme-ov-file#gitleaksignore).
-- Gitleaks doesn't consider the `FILTER_REGEX_EXCLUDE`, `FILTER_REGEX_INCLUDE`,
-  `IGNORE_GENERATED_FILES`, `IGNORE_GITIGNORED_FILES` variables. For more
-  information about how to ignore files with Gitleaks, see
-  [the Gitleaks documentation](https://github.com/gitleaks/gitleaks/tree/master?tab=readme-ov-file#gitleaksignore).
 
 ### Jscpd
 

lib/functions/buildFileList.sh

@@ -13,7 +13,6 @@ function GenerateFileDiff() {
   if [ "${GITHUB_EVENT_NAME:-}" == "push" ]; then
     RunFileDiffCommand "${DIFF_TREE_CMD}"
     if [ ${#RAW_FILE_ARRAY[@]} -eq 0 ]; then
-      debug "----------------------------------------------"
       debug "Generating the file array with diff-tree produced [0] items, trying with git diff against the default branch..."
       RunFileDiffCommand "${DIFF_GIT_DEFAULT_BRANCH_CMD}"
     fi
@@ -49,20 +48,13 @@ function BuildFileList() {
   debug "TEST_CASE_RUN: ${TEST_CASE_RUN}"
 
   if [ "${VALIDATE_ALL_CODEBASE}" == "false" ] && [ "${TEST_CASE_RUN}" != "true" ]; then
-    debug "----------------------------------------------"
     debug "Build the list of all changed files"
 
     GenerateFileDiff
   else
-    WORKSPACE_PATH="${GITHUB_WORKSPACE}"
-    if [ "${TEST_CASE_RUN}" == "true" ]; then
-      WORKSPACE_PATH="${GITHUB_WORKSPACE}/${TEST_CASE_FOLDER}"
-    fi
-
     if [ "${USE_FIND_ALGORITHM}" == 'true' ]; then
-      debug "----------------------------------------------"
+      debug "Populating the file list with all the files in the ${GITHUB_WORKSPACE} workspace using FIND algorithm"
-      debug "Populating the file list with all the files in the ${WORKSPACE_PATH} workspace using FIND algorithm"
+      if ! mapfile -t RAW_FILE_ARRAY < <(find "${GITHUB_WORKSPACE}" \
-      if ! mapfile -t RAW_FILE_ARRAY < <(find "${WORKSPACE_PATH}" \
         -not \( -path '*/\.git' -prune \) \
         -not \( -path '*/\.pytest_cache' -prune \) \
         -not \( -path '*/\.rbenv' -prune \) \
@@ -87,8 +79,7 @@ function BuildFileList() {
       fi
 
     else
-      debug "----------------------------------------------"
+      DIFF_GIT_VALIDATE_ALL_CODEBASE="git -C \"${GITHUB_WORKSPACE}\" ls-tree -r --name-only HEAD | xargs -I % sh -c \"echo ${GITHUB_WORKSPACE}/%\" 2>&1"
-      DIFF_GIT_VALIDATE_ALL_CODEBASE="git -C \"${WORKSPACE_PATH}\" ls-tree -r --name-only HEAD | xargs -I % sh -c \"echo ${WORKSPACE_PATH}/%\" 2>&1"
       debug "Populating the file list with: ${DIFF_GIT_VALIDATE_ALL_CODEBASE}"
       if ! mapfile -t RAW_FILE_ARRAY < <(eval "set -eo pipefail; ${DIFF_GIT_VALIDATE_ALL_CODEBASE}; set +eo pipefail"); then
         fatal "Failed to get a list of changed files. USE_FIND_ALGORITHM: ${USE_FIND_ALGORITHM}"
@@ -105,77 +96,108 @@ function BuildFileList() {
   ####################################################
   # Configure linters that scan the entire workspace #
   ####################################################
-  debug "Checking if we are in test mode before configuring the list of directories to lint"
+  debug "Checking if we are in test mode before configuring the list of directories to lint. TEST_CASE_RUN: ${TEST_CASE_RUN}"
   if [ "${TEST_CASE_RUN}" == "true" ]; then
     debug "We are running in test mode."
 
-    debug "Adding test case directories to the list of directories to analyze with ansible-lint."
+    debug "Adding test case directories to the list of directories to analyze with JSCPD."
-    DEFAULT_ANSIBLE_TEST_CASE_DIRECTORY="${GITHUB_WORKSPACE}/${TEST_CASE_FOLDER}/ansible"
-    debug "DEFAULT_ANSIBLE_TEST_CASE_DIRECTORY: ${DEFAULT_ANSIBLE_TEST_CASE_DIRECTORY}"
-    FILE_ARRAY_ANSIBLE+=("${DEFAULT_ANSIBLE_TEST_CASE_DIRECTORY}/bad")
-    FILE_ARRAY_ANSIBLE+=("${DEFAULT_ANSIBLE_TEST_CASE_DIRECTORY}/good")
-
-    debug "Adding test case directories to the list of directories to analyze with Checkov."
-    DEFAULT_CHECKOV_TEST_CASE_DIRECTORY="${GITHUB_WORKSPACE}/${TEST_CASE_FOLDER}/checkov"
-    debug "DEFAULT_CHECKOV_TEST_CASE_DIRECTORY: ${DEFAULT_CHECKOV_TEST_CASE_DIRECTORY}"
-    FILE_ARRAY_CHECKOV+=("${DEFAULT_CHECKOV_TEST_CASE_DIRECTORY}/bad")
-    FILE_ARRAY_CHECKOV+=("${DEFAULT_CHECKOV_TEST_CASE_DIRECTORY}/good")
-
-    debug "Adding test case directories to the list of directories to analyze with Gitleaks."
-    DEFAULT_GITLEAKS_TEST_CASE_DIRECTORY="${GITHUB_WORKSPACE}/${TEST_CASE_FOLDER}/gitleaks"
-    debug "DEFAULT_GITLEAKS_TEST_CASE_DIRECTORY: ${DEFAULT_GITLEAKS_TEST_CASE_DIRECTORY}"
-    FILE_ARRAY_GITLEAKS+=("${DEFAULT_GITLEAKS_TEST_CASE_DIRECTORY}/bad")
-    FILE_ARRAY_GITLEAKS+=("${DEFAULT_GITLEAKS_TEST_CASE_DIRECTORY}/good")
-
-    debug "Adding test case directories to the list of directories to analyze with Checkov."
     DEFAULT_JSCPD_TEST_CASE_DIRECTORY="${GITHUB_WORKSPACE}/${TEST_CASE_FOLDER}/jscpd"
+    # We need this for parallel
+    export DEFAULT_JSCPD_TEST_CASE_DIRECTORY
     debug "DEFAULT_JSCPD_TEST_CASE_DIRECTORY: ${DEFAULT_JSCPD_TEST_CASE_DIRECTORY}"
-    FILE_ARRAY_JSCPD+=("${DEFAULT_JSCPD_TEST_CASE_DIRECTORY}/bad")
+    RAW_FILE_ARRAY+=("${DEFAULT_JSCPD_TEST_CASE_DIRECTORY}/bad")
-    FILE_ARRAY_JSCPD+=("${DEFAULT_JSCPD_TEST_CASE_DIRECTORY}/good")
+    RAW_FILE_ARRAY+=("${DEFAULT_JSCPD_TEST_CASE_DIRECTORY}/good")
-  else
-    debug "We are not running in test mode (${TEST_CASE_RUN})."
-
-    if [ -d "${ANSIBLE_DIRECTORY}" ]; then
-      debug "Adding ANSIBLE_DIRECTORY (${ANSIBLE_DIRECTORY}) to the list of files and directories to lint."
-      FILE_ARRAY_ANSIBLE+=("${ANSIBLE_DIRECTORY}")
-    else
-      debug "ANSIBLE_DIRECTORY (${ANSIBLE_DIRECTORY}) does NOT exist."
-    fi
-
-    if CheckovConfigurationFileContainsDirectoryOption "${CHECKOV_LINTER_RULES}"; then
-      debug "No need to configure the directories to check for Checkov."
-    else
-      debug "Adding ${GITHUB_WORKSPACE} to the list of directories to analyze with Checkov."
-      FILE_ARRAY_CHECKOV+=("${GITHUB_WORKSPACE}")
-    fi
-
-    debug "Adding ${GITHUB_WORKSPACE} to the list of directories to analyze with Gitleaks."
-    FILE_ARRAY_GITLEAKS+=("${GITHUB_WORKSPACE}")
-
-    debug "Adding ${GITHUB_WORKSPACE} to the list of directories to analyze with JSCPD."
-    FILE_ARRAY_JSCPD+=("${GITHUB_WORKSPACE}")
   fi
 
-  if CheckovConfigurationFileContainsDirectoryOption "${CHECKOV_LINTER_RULES}"; then
+  debug "Add GITHUB_WORKSPACE (${GITHUB_WORKSPACE}) to the list of files to lint because we might need it for linters that lint the whole workspace"
-    debug "No need to configure the directories to check for Checkov."
+  RAW_FILE_ARRAY+=("${GITHUB_WORKSPACE}")
+
+  if [ -d "${ANSIBLE_DIRECTORY}" ]; then
+    debug "Adding ANSIBLE_DIRECTORY (${ANSIBLE_DIRECTORY}) to the list of files and directories to lint."
+    RAW_FILE_ARRAY+=("${ANSIBLE_DIRECTORY}")
   else
-    debug "Checking if we are in test mode before configuring the list of directories to lint with Checkov"
+    debug "ANSIBLE_DIRECTORY (${ANSIBLE_DIRECTORY}) does NOT exist."
-    if [ "${TEST_CASE_RUN}" == "true" ]; then
-      debug "We are running in test mode. Adding test case directories to the list of directories to analyze with Checkov."
-      FILE_ARRAY_CHECKOV+=("${DEFAULT_CHECKOV_TEST_CASE_DIRECTORY}/bad")
-      FILE_ARRAY_CHECKOV+=("${DEFAULT_CHECKOV_TEST_CASE_DIRECTORY}/good")
-    else
-      debug "We are not running in test mode (${TEST_CASE_RUN}). Adding ${GITHUB_WORKSPACE} to the list of directories to analyze with Checkov."
-      FILE_ARRAY_CHECKOV+=("${GITHUB_WORKSPACE}")
-    fi
   fi
 
-  ################################################
+  local PARALLEL_RESULTS_FILE_PATH
-  # Iterate through the array of all files found #
+  PARALLEL_RESULTS_FILE_PATH="/tmp/super-linter-parallel-results-build-file-list.json"
-  ################################################
+  debug "PARALLEL_RESULTS_FILE_PATH when building the file list: ${PARALLEL_RESULTS_FILE_PATH}"
-  info "---------------------------------"
+
-  info "------ File list to check: ------"
+  local -a PARALLEL_COMMAND
-  info "---------------------------------"
+  PARALLEL_COMMAND=(parallel --will-cite --keep-order --max-procs "$(($(nproc) * 1))" --results "${PARALLEL_RESULTS_FILE_PATH}" --xargs)
+
+  if [ "${LOG_DEBUG}" == "true" ]; then
+    debug "LOG_DEBUG is enabled. Enable verbose ouput for parallel"
+    PARALLEL_COMMAND+=(--verbose)
+  fi
+
+  # Max number of files to categorize per process
+  PARALLEL_COMMAND+=(--max-lines 10)
+
+  PARALLEL_COMMAND+=("BuildFileArrays")
+  debug "PARALLEL_COMMAND to build the list of files and directories to lint: ${PARALLEL_COMMAND[*]}"
+
+  FILE_ARRAYS_DIRECTORY_PATH="$(mktemp -d)"
+  export FILE_ARRAYS_DIRECTORY_PATH
+  debug "Created FILE_ARRAYS_DIRECTORY_PATH: ${FILE_ARRAYS_DIRECTORY_PATH}"
+
+  info "Building the list of files and directories to check"
+
+  PARALLEL_COMMAND_OUTPUT=$(printf "%s\n" "${RAW_FILE_ARRAY[@]}" | "${PARALLEL_COMMAND[@]}" 2>&1)
+  PARALLEL_COMMAND_RETURN_CODE=$?
+  debug "PARALLEL_COMMAND_OUTPUT to build the file list (exit code: ${PARALLEL_COMMAND_RETURN_CODE}):\n${PARALLEL_COMMAND_OUTPUT}"
+  debug "Parallel output file (${PARALLEL_RESULTS_FILE_PATH}) contents when building the file list:\n$(cat "${PARALLEL_RESULTS_FILE_PATH}")"
+
+  local RESULTS_OBJECT
+  RESULTS_OBJECT=
+  if ! RESULTS_OBJECT=$(jq -n '[inputs]' "${PARALLEL_RESULTS_FILE_PATH}"); then
+    fatal "Error loading results when building the file list: ${RESULTS_OBJECT}"
+  fi
+  debug "RESULTS_OBJECT for ${FILE_TYPE}:\n${RESULTS_OBJECT}"
+
+  local STDOUT_BUILD_FILE_LIST
+  # Get raw output so we can strip quotes from the data we load
+  if ! STDOUT_BUILD_FILE_LIST="$(jq --raw-output '.[].Stdout' <<<"${RESULTS_OBJECT}")"; then
+    fatal "Error when loading stdout when building the file list: ${STDOUT_BUILD_FILE_LIST}"
+  fi
+
+  if [ -n "${STDOUT_BUILD_FILE_LIST}" ]; then
+    info "Command output when building the file list:\n------\n${STDOUT_BUILD_FILE_LIST}\n------"
+  else
+    debug "Stdout when building the file list is empty"
+  fi
+
+  local STDERR_BUILD_FILE_LIST
+  if ! STDERR_BUILD_FILE_LIST="$(jq --raw-output '.[].Stderr' <<<"${RESULTS_OBJECT}")"; then
+    fatal "Error when loading stderr when building the file list:\n${STDERR_BUILD_FILE_LIST}"
+  fi
+
+  if [ -n "${STDERR_BUILD_FILE_LIST}" ]; then
+    info "Command output when building the file list:\n------\n${STDERR_BUILD_FILE_LIST}\n------"
+  else
+    debug "Stderr when building the file list is empty"
+  fi
+
+  if [[ ${PARALLEL_COMMAND_RETURN_CODE} -ne 0 ]]; then
+    fatal "Error when building the list of files and directories to lint."
+  fi
+
+  ################
+  # Footer print #
+  ################
+  info "Successfully gathered list of files..."
+}
+
+BuildFileArrays() {
+  local -a RAW_FILE_ARRAY
+  RAW_FILE_ARRAY=("$@")
+
+  debug "Categorizing the following files: ${RAW_FILE_ARRAY[*]}"
+  debug "FILTER_REGEX_INCLUDE: ${FILTER_REGEX_INCLUDE}, FILTER_REGEX_EXCLUDE: ${FILTER_REGEX_EXCLUDE}"
+
+  ValidateBooleanVariable "IGNORE_GENERATED_FILES" "${IGNORE_GENERATED_FILES}"
+  ValidateBooleanVariable "IGNORE_GITIGNORED_FILES" "${IGNORE_GITIGNORED_FILES}"
+
   for FILE in "${RAW_FILE_ARRAY[@]}"; do
     # Get the file extension
     FILE_TYPE="$(GetFileExtension "$FILE")"
@@ -187,23 +209,35 @@ function BuildFileList() {
 
     debug "FILE: ${FILE}, FILE_TYPE: ${FILE_TYPE}, BASE_FILE: ${BASE_FILE}, FILE_DIR_NAME: ${FILE_DIR_NAME}"
 
-    if [ ! -f "${FILE}" ]; then
+    if [ ! -e "${FILE}" ]; then
       # File not found in workspace
-      warn "File:{$FILE} existed in commit data, but not found on file system, skipping..."
+      warn "{$FILE} exists in commit data, but not found on file system, skipping..."
       continue
     fi
 
-    ########################################################
+    # Handle the corner cases of linters that are expected to lint the whole codebase,
-    # Don't include test cases if not running in test mode #
+    # but we don't have a variable to explicitly set the directory
-    ########################################################
+    # to lint.
-    if [[ ${FILE} == *"${TEST_CASE_FOLDER}"* ]] && [ "${TEST_CASE_RUN}" != "true" ]; then
+    if [[ "${FILE}" == "${GITHUB_WORKSPACE}" ]]; then
-      debug "TEST_CASE_RUN (${TEST_CASE_RUN}) is not true. Skipping ${FILE}..."
+      debug "${FILE} matches with ${GITHUB_WORKSPACE}. Adding it to the list of directories to lint for linters that are expected to lint the whole codebase"
+
+      if CheckovConfigurationFileContainsDirectoryOption "${CHECKOV_LINTER_RULES}"; then
+        debug "No need to configure the directories to check for Checkov because its configuration file contains the list of directories to analyze."
+        debug "Add the Checkov configuration file path to the list of items to check to consume as output later."
+        echo "${CHECKOV_LINTER_RULES}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-CHECKOV"
+      else
+        debug "Adding ${GITHUB_WORKSPACE} to the list of directories to analyze with Checkov."
+        echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-CHECKOV"
+      fi
+
+      # JSCPD test cases are handled below because we first need to exclude non-relevant test cases
+      if [[ "${TEST_CASE_RUN}" == "false" ]]; then
+        debug "Add ${FILE} to the list of items to lint with JSCPD"
+        echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-JSCPD"
+      fi
+
+      # No need to process this item furhter
       continue
-    ##################################################
-    # Include test cases if not running in test mode #
-    ##################################################
-    elif [[ ${FILE} != *"${TEST_CASE_FOLDER}"* ]] && [ "${TEST_CASE_RUN}" == "true" ]; then
-      debug "TEST_CASE_RUN (${TEST_CASE_RUN}) is true. Skipping ${FILE}..."
     fi
 
     ###############################################
@@ -238,20 +272,40 @@ function BuildFileList() {
       continue
     fi
 
-    # Editorconfig-checker should check every file
+    # These linters check every file
-    FILE_ARRAY_EDITORCONFIG+=("${FILE}")
+    local EDITORCONFIG_FILE_PATH
+    EDITORCONFIG_FILE_PATH="${GITHUB_WORKSPACE}/.editorconfig"
+    if [ -e "${EDITORCONFIG_FILE_PATH}" ]; then
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-EDITORCONFIG"
+    else
+      debug "Don't include ${FILE} in the list of files to lint with editorconfig-checker because the workspace doesn't contain an EditorConfig file: ${EDITORCONFIG_FILE_PATH}"
+    fi
+
+    echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-GITLEAKS"
+
+    if [[ ("${FILE}" =~ .*${ANSIBLE_DIRECTORY}.*) ]] && [[ -d "${FILE}" ]]; then
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-ANSIBLE"
+    fi
+
+    # Handle JSCPD test cases
+    # At this point, we already processed the options to include or exclude files, so we
+    # excluded test cases that are not relevant
+    if [[ "${TEST_CASE_RUN}" == "true" ]] && [[ "${FILE}" =~ .*${DEFAULT_JSCPD_TEST_CASE_DIRECTORY}.* ]] && [[ -d "${FILE}" ]]; then
+      debug "${FILE} is a test case for JSCPD. Adding it to the list of items to lint with JSCPD"
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-JSCPD"
+    fi
 
     # See https://docs.renovatebot.com/configuration-options/
     if [[ "${BASE_FILE}" =~ renovate.json5? ]] ||
       [ "${BASE_FILE}" == ".renovaterc" ] || [[ "${BASE_FILE}" =~ .renovaterc.json5? ]]; then
-      FILE_ARRAY_RENOVATE+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-RENOVATE"
     fi
 
     # See https://docs.renovatebot.com/config-presets/
     IFS="," read -r -a RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES_ARRAY <<<"${RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES}"
     for file_name in "${RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES_ARRAY[@]}"; do
       if [ "${BASE_FILE}" == "${file_name}" ]; then
-        FILE_ARRAY_RENOVATE+=("${FILE}")
+        echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-RENOVATE"
         break
       fi
     done
@@ -268,205 +322,99 @@ function BuildFileList() {
       else
         debug "Considering ${FILE_DIR_NAME} as a Go module."
       fi
-      FILE_ARRAY_GO_MODULES+=("${FILE_DIR_NAME}")
+      echo "${FILE_DIR_NAME}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-GO_MODULES"
     fi
 
-    #######################
-    # Get the shell files #
-    #######################
     if IsValidShellScript "${FILE}"; then
-      FILE_ARRAY_BASH+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-BASH"
-      FILE_ARRAY_BASH_EXEC+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-BASH_EXEC"
-      FILE_ARRAY_SHELL_SHFMT+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-SHELL_SHFMT"
-
-    #########################
-    # Get the CLOJURE files #
-    #########################
     elif [ "${FILE_TYPE}" == "clj" ] || [ "${FILE_TYPE}" == "cljs" ] ||
       [ "${FILE_TYPE}" == "cljc" ] || [ "${FILE_TYPE}" == "edn" ]; then
-      FILE_ARRAY_CLOJURE+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-CLOJURE"
-    #####################
-    # Get the C++ files #
-    #####################
     elif [ "${FILE_TYPE}" == "cpp" ] || [ "${FILE_TYPE}" == "h" ] ||
       [ "${FILE_TYPE}" == "cc" ] || [ "${FILE_TYPE}" == "hpp" ] ||
       [ "${FILE_TYPE}" == "cxx" ] || [ "${FILE_TYPE}" == "cu" ] ||
       [ "${FILE_TYPE}" == "hxx" ] || [ "${FILE_TYPE}" == "c++" ] ||
       [ "${FILE_TYPE}" == "hh" ] || [ "${FILE_TYPE}" == "h++" ] ||
       [ "${FILE_TYPE}" == "cuh" ] || [ "${FILE_TYPE}" == "c" ]; then
-      FILE_ARRAY_CPP+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-CPP"
-      FILE_ARRAY_CLANG_FORMAT+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-CLANG_FORMAT"
-
-    ########################
-    # Get the COFFEE files #
-    ########################
     elif [ "${FILE_TYPE}" == "coffee" ]; then
-      FILE_ARRAY_COFFEESCRIPT+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-COFFEESCRIPT"
-
-    ########################
-    # Get the CSHARP files #
-    ########################
     elif [ "${FILE_TYPE}" == "cs" ]; then
       FILE_ARRAY_CSHARP+=("${FILE}")
-
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-CSHARP"
-    #####################
-    # Get the CSS files #
-    #####################
     elif [ "${FILE_TYPE}" == "css" ] || [ "${FILE_TYPE}" == "scss" ] ||
       [ "${FILE_TYPE}" == "sass" ]; then
-      FILE_ARRAY_CSS+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-CSS"
-
-    ######################
-    # Get the DART files #
-    ######################
     elif [ "${FILE_TYPE}" == "dart" ]; then
-      FILE_ARRAY_DART+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-DART"
-
-    ########################
-    # Get the DOCKER files #
-    ########################
     # Use BASE_FILE here because FILE_TYPE is not reliable when there is no file extension
     elif [[ "${FILE_TYPE}" != "tap" ]] && [[ "${FILE_TYPE}" != "yml" ]] &&
       [[ "${FILE_TYPE}" != "yaml" ]] && [[ "${FILE_TYPE}" != "json" ]] &&
       [[ "${FILE_TYPE}" != "xml" ]] &&
       [[ "${BASE_FILE}" =~ ^(.+\.)?(contain|dock)erfile$ ]]; then
-      FILE_ARRAY_DOCKERFILE_HADOLINT+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-DOCKERFILE_HADOLINT"
-
-    #####################
-    # Get the ENV files #
-    #####################
     elif [ "${FILE_TYPE}" == "env" ] || [[ "${BASE_FILE}" == *".env."* ]]; then
-      FILE_ARRAY_ENV+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-ENV"
-
-    #########################
-    # Get the Gherkin files #
-    #########################
     elif [ "${FILE_TYPE}" == "feature" ]; then
-      FILE_ARRAY_GHERKIN+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-GHERKIN"
-
-    ########################
-    # Get the Golang files #
-    ########################
     elif [ "${FILE_TYPE}" == "go" ]; then
-      FILE_ARRAY_GO+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-GO"
-
-    ########################
-    # Get the GROOVY files #
-    ########################
     # Use BASE_FILE here because FILE_TYPE is not reliable when there is no file extension
     elif [ "$FILE_TYPE" == "groovy" ] || [ "$FILE_TYPE" == "jenkinsfile" ] ||
       [ "$FILE_TYPE" == "gradle" ] || [ "$FILE_TYPE" == "nf" ] ||
       [[ "$BASE_FILE" =~ .*jenkinsfile.* ]]; then
-      FILE_ARRAY_GROOVY+=("$FILE")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-GROOVY"
-
-    ######################
-    # Get the HTML files #
-    ######################
     elif [ "${FILE_TYPE}" == "html" ]; then
-      FILE_ARRAY_HTML+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-HTML"
-
-    ######################
-    # Get the Java files #
-    ######################
     elif [ "${FILE_TYPE}" == "java" ]; then
-      FILE_ARRAY_JAVA+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-JAVA"
-      FILE_ARRAY_GOOGLE_JAVA_FORMAT+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-GOOGLE_JAVA_FORMAT"
-
-    ############################
-    # Get the JavaScript files #
-    ############################
     elif [ "${FILE_TYPE}" == "js" ]; then
-      FILE_ARRAY_JAVASCRIPT_ES+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-JAVASCRIPT_ES"
-      FILE_ARRAY_JAVASCRIPT_STANDARD+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-JAVASCRIPT_STANDARD"
-      FILE_ARRAY_JAVASCRIPT_PRETTIER+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-JAVASCRIPT_PRETTIER"
-
-    #######################
-    # Get the JSONC files #
-    #######################
     elif [ "$FILE_TYPE" == "jsonc" ] || [ "$FILE_TYPE" == "json5" ]; then
-      FILE_ARRAY_JSONC+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-JSONC"
-
-    ######################
-    # Get the JSON files #
-    ######################
     elif [ "${FILE_TYPE}" == "json" ]; then
       FILE_ARRAY_JSON+=("${FILE}")
-
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-JSON"
-      ############################
-      # Check if file is OpenAPI #
-      ############################
       if DetectOpenAPIFile "${FILE}"; then
-        FILE_ARRAY_OPENAPI+=("${FILE}")
+        echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-OPENAPI"
       fi
-      ########################
+
-      # Check if file is ARM #
-      ########################
       if DetectARMFile "${FILE}"; then
-        FILE_ARRAY_ARM+=("${FILE}")
+        echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-ARM"
       fi
-      #####################################
+
-      # Check if the file is CFN template #
-      #####################################
       if DetectCloudFormationFile "${FILE}"; then
-        FILE_ARRAY_CLOUDFORMATION+=("${FILE}")
+        echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-CLOUDFORMATION"
       fi
-      ############################################
+
-      # Check if the file is AWS States Language #
-      ############################################
       if DetectAWSStatesFIle "${FILE}"; then
-        FILE_ARRAY_STATES+=("${FILE}")
+        echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-STATES"
       fi
-
-    #####################
-    # Get the JSX files #
-    #####################
     elif [ "${FILE_TYPE}" == "jsx" ]; then
-      FILE_ARRAY_JSX+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-JSX"
-
-    ########################
-    # Get the KOTLIN files #
-    ########################
     elif [ "${FILE_TYPE}" == "kt" ] || [ "${FILE_TYPE}" == "kts" ]; then
-      FILE_ARRAY_KOTLIN+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-KOTLIN"
-
-    #####################
-    # Get the LUA files #
-    #####################
     elif [ "$FILE_TYPE" == "lua" ]; then
-      FILE_ARRAY_LUA+=("$FILE")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-LUA"
-
-    #######################
-    # Get the LaTeX files #
-    #######################
     elif [ "${FILE_TYPE}" == "tex" ]; then
-      FILE_ARRAY_LATEX+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-LATEX"
-
-    ##########################
-    # Get the MARKDOWN files #
-    ##########################
     elif [ "${FILE_TYPE}" == "md" ]; then
-      FILE_ARRAY_MARKDOWN+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-MARKDOWN"
-      FILE_ARRAY_NATURAL_LANGUAGE+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-NATURAL_LANGUAGE"
-
-    ######################
-    # Get the PHP files #
-    ######################
     elif [ "${FILE_TYPE}" == "php" ]; then
-      FILE_ARRAY_PHP_BUILTIN+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PHP_BUILTIN"
-      FILE_ARRAY_PHP_PHPCS+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PHP_PHPCS"
-      FILE_ARRAY_PHP_PHPSTAN+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PHP_PHPSTAN"
-      FILE_ARRAY_PHP_PSALM+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PHP_PSALM"
-
-    ######################
-    # Get the PERL files #
-    ######################
     elif [ "${FILE_TYPE}" == "pl" ] || [ "${FILE_TYPE}" == "pm" ] ||
       [ "${FILE_TYPE}" == "t" ]; then
-      FILE_ARRAY_PERL+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PERL"
-
-    ############################
-    # Get the Powershell files #
-    ############################
     elif [ "${FILE_TYPE}" == "ps1" ] ||
       [ "${FILE_TYPE}" == "psm1" ] ||
       [ "${FILE_TYPE}" == "psd1" ] ||
@@ -474,179 +422,83 @@ function BuildFileList() {
       [ "${FILE_TYPE}" == "pssc" ] ||
       [ "${FILE_TYPE}" == "psrc" ] ||
       [ "${FILE_TYPE}" == "cdxml" ]; then
-      FILE_ARRAY_POWERSHELL+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-POWERSHELL"
-
-    #################################
-    # Get the PROTOCOL BUFFER files #
-    #################################
     elif [ "${FILE_TYPE}" == "proto" ]; then
-      FILE_ARRAY_PROTOBUF+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PROTOBUF"
-
-    ########################
-    # Get the PYTHON files #
-    ########################
     elif [ "${FILE_TYPE}" == "py" ]; then
-      FILE_ARRAY_PYTHON_BLACK+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PYTHON_BLACK"
-      FILE_ARRAY_PYTHON_FLAKE8+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PYTHON_FLAKE8"
-      FILE_ARRAY_PYTHON_ISORT+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PYTHON_ISORT"
-      FILE_ARRAY_PYTHON_PYLINT+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PYTHON_PYLINT"
-      FILE_ARRAY_PYTHON_MYPY+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PYTHON_MYPY"
-
-    ######################
-    # Get the RAKU files #
-    ######################
     elif [ "${FILE_TYPE}" == "raku" ] || [ "${FILE_TYPE}" == "rakumod" ] ||
       [ "${FILE_TYPE}" == "rakutest" ] || [ "${FILE_TYPE}" == "pm6" ] ||
       [ "${FILE_TYPE}" == "pl6" ] || [ "${FILE_TYPE}" == "p6" ]; then
-      FILE_ARRAY_RAKU+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-RAKU"
-
-    ####################
-    # Get the R files  #
-    ####################
     elif [ "${FILE_TYPE}" == "r" ] || [ "${FILE_TYPE}" == "rmd" ]; then
-      FILE_ARRAY_R+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-R"
-
-    ######################
-    # Get the RUBY files #
-    ######################
     elif [ "${FILE_TYPE}" == "rb" ]; then
-      FILE_ARRAY_RUBY+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-RUBY"
-
-    ######################
-    # Get the RUST files #
-    ######################
     elif [ "${FILE_TYPE}" == "rs" ]; then
-      FILE_ARRAY_RUST_2015+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-RUST_2015"
-      FILE_ARRAY_RUST_2018+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-RUST_2018"
-      FILE_ARRAY_RUST_2021+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-RUST_2021"
-
-    #######################
-    # Get the RUST crates #
-    #######################
     elif [ "${BASE_FILE}" == "cargo.toml" ]; then
-      ###############################################
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-RUST_CLIPPY"
-      # Append the crate manifest file to the array #
-      ###############################################
-      FILE_ARRAY_RUST_CLIPPY+=("${FILE}")
-
-    ###########################
-    # Get the SCALA files #
-    ###########################
     elif [ "${FILE_TYPE}" == "scala" ] || [ "${FILE_TYPE}" == "sc" ] || [ "${BASE_FILE}" == "??????" ]; then
-      FILE_ARRAY_SCALAFMT+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-SCALAFMT"
-
-    ###########################
-    # Get the SNAKEMAKE files #
-    ###########################
     elif [ "${FILE_TYPE}" == "smk" ] || [ "${BASE_FILE}" == "snakefile" ]; then
-      FILE_ARRAY_SNAKEMAKE_LINT+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-SNAKEMAKE_LINT"
-      FILE_ARRAY_SNAKEMAKE_SNAKEFMT+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-SNAKEMAKE_SNAKEFMT"
-
-    #####################
-    # Get the SQL files #
-    #####################
     elif [ "${FILE_TYPE}" == "sql" ]; then
-      FILE_ARRAY_SQL+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-SQL"
-      FILE_ARRAY_SQLFLUFF+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-SQLFLUFF"
-
-    ###########################
-    # Get the Terraform files #
-    ###########################
     elif [ "${FILE_TYPE}" == "tf" ]; then
-      FILE_ARRAY_TERRAFORM_TFLINT+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TERRAFORM_TFLINT"
-      FILE_ARRAY_TERRAFORM_TERRASCAN+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TERRAFORM_TERRASCAN"
-      FILE_ARRAY_TERRAFORM_FMT+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TERRAFORM_FMT"
-
-    ############################
-    # Get the Terragrunt files #
-    ############################
     elif [ "${FILE_TYPE}" == "hcl" ] &&
       [[ ${FILE} != *".tflint.hcl"* ]] &&
       [[ ${FILE} != *".pkr.hcl"* ]] &&
       [[ ${FILE} != *"docker-bake.hcl"* ]] &&
       [[ ${FILE} != *"docker-bake.override.hcl"* ]]; then
-      FILE_ARRAY_TERRAGRUNT+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TERRAGRUNT"
-
-    ############################
-    # Get the TypeScript files #
-    ############################
     elif [ "${FILE_TYPE}" == "ts" ]; then
-      FILE_ARRAY_TYPESCRIPT_ES+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TYPESCRIPT_ES"
-      FILE_ARRAY_TYPESCRIPT_STANDARD+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TYPESCRIPT_STANDARD"
-      FILE_ARRAY_TYPESCRIPT_PRETTIER+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TYPESCRIPT_PRETTIER"
-
-    #####################
-    # Get the TSX files #
-    #####################
     elif [ "${FILE_TYPE}" == "tsx" ]; then
-      FILE_ARRAY_TSX+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TSX"
     elif [ "${FILE_TYPE}" == "txt" ]; then
-      FILE_ARRAY_NATURAL_LANGUAGE+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TXT"
-
-    #####################
-    # Get the XML files #
-    #####################
     elif [ "${FILE_TYPE}" == "xml" ]; then
-      FILE_ARRAY_XML+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-XML"
-
-    ################################
-    # Get the CLOUDFORMATION files #
-    ################################
     elif [ "${FILE_TYPE}" == "yml" ] || [ "${FILE_TYPE}" == "yaml" ]; then
-      FILE_ARRAY_YAML+=("${FILE}")
+      echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-YAML"
-
-      ###################################
-      # Check if file is GitHub Actions #
-      ###################################
       if DetectActions "${FILE}"; then
-        FILE_ARRAY_GITHUB_ACTIONS+=("${FILE}")
+        echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-GITHUB_ACTIONS"
       fi
 
-      #####################################
-      # Check if the file is CFN template #
-      #####################################
       if DetectCloudFormationFile "${FILE}"; then
-        FILE_ARRAY_CLOUDFORMATION+=("${FILE}")
+        echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-CLOUDFORMATION"
       fi
 
-      ############################
-      # Check if file is OpenAPI #
-      ############################
       if DetectOpenAPIFile "${FILE}"; then
-        FILE_ARRAY_OPENAPI+=("${FILE}")
+        echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-OPENAPI"
       fi
 
-      ########################################
-      # Check if the file is Tekton template #
-      ########################################
       if DetectTektonFile "${FILE}"; then
-        FILE_ARRAY_TEKTON+=("${FILE}")
+        echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TEKTON"
       fi
 
-      ############################################
-      # Check if the file is Kubernetes template #
-      ############################################
       if DetectKubernetesFile "${FILE}"; then
-        FILE_ARRAY_KUBERNETES_KUBECONFORM+=("${FILE}")
+        echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-KUBERNETES_KUBECONFORM"
       fi
-    ########################################################################
-    # We have something that we need to try to check file type another way #
-    ########################################################################
     else
-      ##############################################
-      # Use file to see if we can parse what it is #
-      ##############################################
       CheckFileType "${FILE}"
     fi
-    ##########################################
-    # Print line break after each file debug #
-    ##########################################
-    debug ""
   done
-
-  ################
-  # Footer print #
-  ################
-  info "----------------------------------------------"
-  info "Successfully gathered list of files..."
 }
+
+# We need this for parallel
+export -f BuildFileArrays

lib/functions/detectFiles.sh

@@ -139,21 +139,6 @@ DetectAWSStatesFIle() {
   return 1
 }
 
-CheckInArray() {
-  NEEDLE="$1" # Language we need to match
-
-  ######################################
-  # Check if Language was in the array #
-  ######################################
-  for LANG in "${UNIQUE_LINTED_ARRAY[@]}"; do
-    if [[ "${LANG}" == "${NEEDLE}" ]]; then
-      return 0
-    fi
-  done
-
-  return 1
-}
-
 function GetFileType() {
   # Need to run the file through the 'file' exec to help determine
   # The type of file being parsed
@@ -168,21 +153,23 @@ function CheckFileType() {
   # Need to run the file through the 'file' exec to help determine
   # The type of file being parsed
 
+  local FILE
   FILE="$1"
 
+  local GET_FILE_TYPE_CMD
   GET_FILE_TYPE_CMD="$(GetFileType "$FILE")"
 
   local FILE_TYPE_MESSAGE
 
   if [[ ${GET_FILE_TYPE_CMD} == *"Ruby script"* ]]; then
     FILE_TYPE_MESSAGE="Found Ruby script without extension (${FILE}). Rename the file with proper extension for Ruby files."
-    FILE_ARRAY_RUBY+=("${FILE}")
+    echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-RUBY"
   elif [[ ${GET_FILE_TYPE_CMD} == *"Python script"* ]]; then
     FILE_TYPE_MESSAGE="Found Python script without extension (${FILE}). Rename the file with proper extension for Python files."
-    FILE_ARRAY_PYTHON+=("${FILE}")
+    echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PYTHON"
   elif [[ ${GET_FILE_TYPE_CMD} == *"Perl script"* ]]; then
     FILE_TYPE_MESSAGE="Found Perl script without extension (${FILE}). Rename the file with proper extension for Perl files."
-    FILE_ARRAY_PERL+=("${FILE}")
+    echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PERL"
   else
     FILE_TYPE_MESSAGE="Failed to get file type for: ${FILE}"
   fi
@@ -266,11 +253,30 @@ function IsGenerated() {
   fi
 }
 
+# We need these functions when building the file list with paralle
+export -f CheckFileType
+export -f DetectActions
+export -f DetectARMFile
+export -f DetectAWSStatesFIle
+export -f DetectCloudFormationFile
+export -f DetectKubernetesFile
+export -f DetectOpenAPIFile
+export -f DetectTektonFile
+export -f GetFileExtension
+export -f GetFileType
+export -f IsValidShellScript
+export -f IsGenerated
+
 function RunAdditionalInstalls() {
+
+  if [ -z "${FILE_ARRAYS_DIRECTORY_PATH}" ] || [ ! -d "${FILE_ARRAYS_DIRECTORY_PATH}" ]; then
+    fatal "FILE_ARRAYS_DIRECTORY_PATH (set to ${FILE_ARRAYS_DIRECTORY_PATH}) is empty or doesn't exist"
+  fi
+
   ##################################
   # Run installs for Psalm and PHP #
   ##################################
-  if [ "${VALIDATE_PHP_PSALM}" == "true" ] && [ "${#FILE_ARRAY_PHP_PSALM[@]}" -ne 0 ]; then
+  if [ "${VALIDATE_PHP_PSALM}" == "true" ] && [ -e "${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PHP_PSALM" ]; then
     # found PHP files and were validating it, need to composer install
     info "Found PHP files to validate, and [VALIDATE_PHP_PSALM] set to true, need to run composer install"
     info "looking for composer.json in the users repository..."
@@ -311,13 +317,13 @@ function RunAdditionalInstalls() {
   ###############################
   # Run installs for R language #
   ###############################
-  if [ "${VALIDATE_R}" == "true" ] && [ "${#FILE_ARRAY_R[@]}" -ne 0 ]; then
+  if [ "${VALIDATE_R}" == "true" ] && [ -e "${FILE_ARRAYS_DIRECTORY_PATH}/file-array-R" ]; then
     info "Detected R Language files to lint."
-    info "Trying to install the R package inside:[${WORKSPACE_PATH}]"
+    info "Trying to install the R package inside:[${GITHUB_WORKSPACE}]"
     #########################
     # Run the build command #
     #########################
-    BUILD_CMD=$(R CMD build "${WORKSPACE_PATH}" 2>&1)
+    BUILD_CMD=$(R CMD build "${GITHUB_WORKSPACE}" 2>&1)
 
     ##############
     # Error code #
@@ -329,19 +335,19 @@ function RunAdditionalInstalls() {
     ##############################
     if [ "${ERROR_CODE}" -ne 0 ]; then
       # Error
-      warn "ERROR! Failed to run:[R CMD build] at location:[${WORKSPACE_PATH}]"
+      warn "ERROR! Failed to run:[R CMD build] at location:[${GITHUB_WORKSPACE}]"
       warn "BUILD_CMD:[${BUILD_CMD}]"
     else
       # Get the build package
       BUILD_PKG=$(
-        cd "${WORKSPACE_PATH}" || exit 0
+        cd "${GITHUB_WORKSPACE}" || exit 0
         echo *.tar.gz 2>&1
       )
       ##############################
       # Install the build packages #
       ##############################
       INSTALL_CMD=$(
-        cd "${WORKSPACE_PATH}" || exit 0
+        cd "${GITHUB_WORKSPACE}" || exit 0
         R -e "remotes::install_local('.', dependencies=T)" 2>&1
       )
 
@@ -358,19 +364,26 @@ function RunAdditionalInstalls() {
         warn "ERROR: Failed to install the build package at:[${BUILD_PKG}]"
       fi
     fi
+
+    if [ ! -f "${R_RULES_FILE_PATH_IN_ROOT}" ]; then
+      info "No .lintr configuration file found, using defaults."
+      cp "$R_LINTER_RULES" "$GITHUB_WORKSPACE"
+      # shellcheck disable=SC2034
+      SUPER_LINTER_COPIED_R_LINTER_RULES_FILE="true"
+    fi
   fi
 
   ####################################
   # Run installs for TFLINT language #
   ####################################
-  if [ "${VALIDATE_TERRAFORM_TFLINT}" == "true" ] && [ "${#FILE_ARRAY_TERRAFORM_TFLINT[@]}" -ne 0 ]; then
+  if [ "${VALIDATE_TERRAFORM_TFLINT}" == "true" ] && [ -e "${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TERRAFORM_TFLINT" ]; then
     info "Detected TFLint Language files to lint."
-    info "Trying to install the TFLint init inside:[${WORKSPACE_PATH}]"
+    info "Trying to install the TFLint init inside:[${GITHUB_WORKSPACE}]"
     #########################
     # Run the build command #
     #########################
     BUILD_CMD=$(
-      cd "${WORKSPACE_PATH}" || exit 0
+      cd "${GITHUB_WORKSPACE}" || exit 0
       tflint --init -c "${TERRAFORM_TFLINT_LINTER_RULES}" 2>&1
     )
 
@@ -388,5 +401,44 @@ function RunAdditionalInstalls() {
       info "Successfully initialized tflint with the ${TERRAFORM_TFLINT_LINTER_RULES} config file"
       debug "Tflint output: ${BUILD_CMD}"
     fi
+
+    # Array to track directories where tflint was run
+    local -A TFLINT_SEEN_DIRS
+    TFLINT_SEEN_DIRS=()
+    for FILE in "${FILE_ARRAY_TERRAFORM_TFLINT[@]}"; do
+      local DIR_NAME
+      DIR_NAME=$(dirname "${FILE}" 2>&1)
+      debug "DIR_NAME for ${FILE}: ${DIR_NAME}"
+      # Check the cache to see if we've already prepped this directory for tflint
+      if [[ ! -v "TFLINT_SEEN_DIRS[${DIR_NAME}]" ]]; then
+        debug "Configuring Terraform data directory for ${DIR_NAME}"
+
+        # Define the path to an empty Terraform data directory
+        # (def: https://developer.hashicorp.com/terraform/cli/config/environment-variables#tf_data_dir)
+        # in case the user has a Terraform data directory already, and we don't
+        # want to modify it.
+        # TFlint considers this variable as well.
+        # Ref: https://github.com/terraform-linters/tflint/blob/master/docs/user-guide/compatibility.md#environment-variables
+        TF_DATA_DIR="/tmp/.terraform-${TERRAFORM_TFLINT}-${DIR_NAME}"
+
+        # Fetch Terraform modules
+        debug "Fetch Terraform modules for ${FILE} in ${DIR_NAME} in ${TF_DATA_DIR}"
+        local FETCH_TERRAFORM_MODULES_CMD
+        if ! FETCH_TERRAFORM_MODULES_CMD="$(terraform get)"; then
+          fatal "Error when fetching Terraform modules while linting ${FILE}. Command output: ${FETCH_TERRAFORM_MODULES_CMD}"
+        fi
+        debug "Fetch Terraform modules command for ${FILE} output: ${FETCH_TERRAFORM_MODULES_CMD}"
+        # Let the cache know we've seen this before
+        # Set the value to an arbitrary non-empty string.
+        TFLINT_SEEN_DIRS[${DIR_NAME}]="false"
+      else
+        debug "Skip fetching Terraform modules for ${FILE} because we already did that for ${DIR_NAME}"
+      fi
+    done
+  fi
+
+  # Check if there's local configuration for the Raku linter
+  if [ -e "${GITHUB_WORKSPACE}/META6.json" ]; then
+    cd "${GITHUB_WORKSPACE}" && zef install --deps-only --/test .
   fi
 }

lib/functions/linterCommands.sh

@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+
+##########################
+# Define linter commands #
+##########################
+
+# If there's no input argument, parallel adds a default {} at the end of the command.
+# In a few cases, such as ANSIBLE and GO_MODULES,
+# Consume the input before running the command because we need the input
+# to set the working directory, but we don't need it appended at the end of the command.
+# Setting -n 0 would not help in this case, because the input will not be passed
+# to the --workdir option as well.
+# shellcheck disable=SC2034 # Variable is referenced in other scripts
+LINTER_COMMANDS_ARRAY_ANSIBLE=(ansible-lint -c "${ANSIBLE_LINTER_RULES}" "&& echo \"Linted: {}\"")
+LINTER_COMMANDS_ARRAY_ARM=(pwsh -NoProfile -NoLogo -Command "\"Import-Module ${ARM_TTK_PSD1} ; \\\${config} = \\\$(Import-PowerShellDataFile -Path ${ARM_LINTER_RULES}) ; Test-AzTemplate @config -TemplatePath '{}'; if (\\\${Error}.Count) { exit 1 }\"")
+LINTER_COMMANDS_ARRAY_BASH=(shellcheck --color --external-sources)
+if [ -n "${BASH_SEVERITY}" ]; then
+  LINTER_COMMANDS_ARRAY_BASH+=(--severity="${BASH_SEVERITY}")
+fi
+LINTER_COMMANDS_ARRAY_BASH_EXEC=(bash-exec)
+LINTER_COMMANDS_ARRAY_CHECKOV=(checkov --config-file "${CHECKOV_LINTER_RULES}")
+if CheckovConfigurationFileContainsDirectoryOption "${CHECKOV_LINTER_RULES}"; then
+  # Consume the input as we do with ANSIBLE
+  debug "Consume the input of the Checkov command because we don't need to add it as an argument."
+  LINTER_COMMANDS_ARRAY_CHECKOV+=("&& echo \"Got the list of directories to lint from the configuration file: {}\"")
+else
+  debug "Adding the '--directory' option to the Checkov command."
+  LINTER_COMMANDS_ARRAY_CHECKOV+=(--directory)
+fi
+LINTER_COMMANDS_ARRAY_CLANG_FORMAT=(clang-format --Werror --dry-run)
+LINTER_COMMANDS_ARRAY_CLOJURE=(clj-kondo --config "${CLOJURE_LINTER_RULES}" --lint)
+LINTER_COMMANDS_ARRAY_CLOUDFORMATION=(cfn-lint --config-file "${CLOUDFORMATION_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_COFFEESCRIPT=(coffeelint -f "${COFFEESCRIPT_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_CPP=(cpplint)
+LINTER_COMMANDS_ARRAY_CSHARP=(dotnet format whitespace --folder --verify-no-changes --exclude / --include "{/}")
+LINTER_COMMANDS_ARRAY_CSS=(stylelint --config "${CSS_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_DART=(dart analyze --fatal-infos --fatal-warnings)
+LINTER_COMMANDS_ARRAY_DOCKERFILE_HADOLINT=(hadolint -c "${DOCKERFILE_HADOLINT_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_EDITORCONFIG=(editorconfig-checker -config "${EDITORCONFIG_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_ENV=(dotenv-linter)
+LINTER_COMMANDS_ARRAY_GITHUB_ACTIONS=(actionlint -config-file "${GITHUB_ACTIONS_LINTER_RULES}")
+if [ "${GITHUB_ACTIONS_COMMAND_ARGS}" != "null" ] && [ -n "${GITHUB_ACTIONS_COMMAND_ARGS}" ]; then
+  LINTER_COMMANDS_ARRAY_GITHUB_ACTIONS+=("${GITHUB_ACTIONS_COMMAND_ARGS}")
+fi
+LINTER_COMMANDS_ARRAY_GITLEAKS=(gitleaks detect --no-banner --no-git --redact --config "${GITLEAKS_LINTER_RULES}" --verbose --source)
+LINTER_COMMANDS_ARRAY_GHERKIN=(gherkin-lint -c "${GHERKIN_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_GO=(golangci-lint run -c "${GO_LINTER_RULES}" --fast)
+# Consume the input as we do with ANSIBLE
+LINTER_COMMANDS_ARRAY_GO_MODULES=(golangci-lint run --allow-parallel-runners -c "${GO_LINTER_RULES}" "&& echo \"Linted: {}\"")
+LINTER_COMMANDS_ARRAY_GOOGLE_JAVA_FORMAT=(java -jar /usr/bin/google-java-format --dry-run --set-exit-if-changed)
+LINTER_COMMANDS_ARRAY_GROOVY=(npm-groovy-lint -c "${GROOVY_LINTER_RULES}" --failon warning --no-insight)
+LINTER_COMMANDS_ARRAY_HTML=(htmlhint --config "${HTML_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_JAVA=(java -jar /usr/bin/checkstyle -c "${JAVA_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_JAVASCRIPT_ES=(eslint --no-eslintrc -c "${JAVASCRIPT_ES_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_JAVASCRIPT_STANDARD=(standard "${JAVASCRIPT_STANDARD_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_JAVASCRIPT_PRETTIER=(prettier --check)
+LINTER_COMMANDS_ARRAY_JSCPD=(jscpd --config "${JSCPD_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_JSON=(eslint --no-eslintrc -c "${JAVASCRIPT_ES_LINTER_RULES}" --ext '.json')
+LINTER_COMMANDS_ARRAY_JSONC=(eslint --no-eslintrc -c "${JAVASCRIPT_ES_LINTER_RULES}" --ext '.json5,.jsonc')
+LINTER_COMMANDS_ARRAY_JSX=(eslint --no-eslintrc -c "${JSX_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_KOTLIN=(ktlint "{/}")
+LINTER_COMMANDS_ARRAY_KUBERNETES_KUBECONFORM=(kubeconform -strict)
+if [ "${KUBERNETES_KUBECONFORM_OPTIONS}" != "null" ] && [ -n "${KUBERNETES_KUBECONFORM_OPTIONS}" ]; then
+  LINTER_COMMANDS_ARRAY_KUBERNETES_KUBECONFORM+=("${KUBERNETES_KUBECONFORM_OPTIONS}")
+fi
+LINTER_COMMANDS_ARRAY_LATEX=(chktex -q -l "${LATEX_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_LUA=(luacheck --config "${LUA_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_MARKDOWN=(markdownlint -c "${MARKDOWN_LINTER_RULES}")
+if [ -n "${MARKDOWN_CUSTOM_RULE_GLOBS}" ]; then
+  IFS="," read -r -a MARKDOWN_CUSTOM_RULE_GLOBS_ARRAY <<<"${MARKDOWN_CUSTOM_RULE_GLOBS}"
+  for glob in "${MARKDOWN_CUSTOM_RULE_GLOBS_ARRAY[@]}"; do
+    if [ -z "${LINTER_RULES_PATH}" ]; then
+      LINTER_COMMANDS_ARRAY_MARKDOWN+=(-r "${GITHUB_WORKSPACE}/${glob}")
+    else
+      LINTER_COMMANDS_ARRAY_MARKDOWN+=(-r "${GITHUB_WORKSPACE}/${LINTER_RULES_PATH}/${glob}")
+    fi
+  done
+fi
+LINTER_COMMANDS_ARRAY_NATURAL_LANGUAGE=(textlint -c "${NATURAL_LANGUAGE_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_OPENAPI=(spectral lint -r "${OPENAPI_LINTER_RULES}" -D)
+LINTER_COMMANDS_ARRAY_PERL=(perlcritic)
+if [ "${PERL_PERLCRITIC_OPTIONS}" != "null" ] && [ -n "${PERL_PERLCRITIC_OPTIONS}" ]; then
+  LINTER_COMMANDS_ARRAY_PERL+=("${PERL_PERLCRITIC_OPTIONS}")
+fi
+LINTER_COMMANDS_ARRAY_PHP_BUILTIN=(php -l -c "${PHP_BUILTIN_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_PHP_PHPCS=(phpcs --standard="${PHP_PHPCS_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_PHP_PHPSTAN=(phpstan analyse --no-progress --no-ansi --memory-limit 1G -c "${PHP_PHPSTAN_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_PHP_PSALM=(psalm --config="${PHP_PSALM_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_POWERSHELL=(pwsh -NoProfile -NoLogo -Command "\"Invoke-ScriptAnalyzer -EnableExit -Settings ${POWERSHELL_LINTER_RULES} -Path '{}'; if (\\\${Error}.Count) { exit 1 }\"")
+LINTER_COMMANDS_ARRAY_PROTOBUF=(protolint lint --config_path "${PROTOBUF_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_PYTHON_BLACK=(black --config "${PYTHON_BLACK_LINTER_RULES}" --diff --check)
+LINTER_COMMANDS_ARRAY_PYTHON_PYLINT=(pylint --rcfile "${PYTHON_PYLINT_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_PYTHON_FLAKE8=(flake8 --config="${PYTHON_FLAKE8_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_PYTHON_ISORT=(isort --check --diff --sp "${PYTHON_ISORT_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_PYTHON_MYPY=(mypy --config-file "${PYTHON_MYPY_LINTER_RULES}" --install-types --non-interactive)
+LINTER_COMMANDS_ARRAY_R=(R --slave -e "\"lints <- lintr::lint('{}');print(lints);errors <- purrr::keep(lints, ~ .\\\$type == 'error');quit(save = 'no', status = if (length(errors) > 0) 1 else 0)\"")
+LINTER_COMMANDS_ARRAY_RAKU=(raku)
+LINTER_COMMANDS_ARRAY_RENOVATE=(renovate-config-validator --strict)
+LINTER_COMMANDS_ARRAY_RUBY=(rubocop -c "${RUBY_LINTER_RULES}" --force-exclusion --ignore-unrecognized-cops)
+LINTER_COMMANDS_ARRAY_RUST_2015=(rustfmt --check --edition 2015)
+LINTER_COMMANDS_ARRAY_RUST_2018=(rustfmt --check --edition 2018)
+LINTER_COMMANDS_ARRAY_RUST_2021=(rustfmt --check --edition 2021)
+LINTER_COMMANDS_ARRAY_RUST_CLIPPY=(clippy)
+LINTER_COMMANDS_ARRAY_SCALAFMT=(scalafmt --config "${SCALAFMT_LINTER_RULES}" --test)
+LINTER_COMMANDS_ARRAY_SHELL_SHFMT=(shfmt -d)
+LINTER_COMMANDS_ARRAY_SNAKEMAKE_LINT=(snakemake --lint -s)
+LINTER_COMMANDS_ARRAY_SNAKEMAKE_SNAKEFMT=(snakefmt --config "${SNAKEMAKE_SNAKEFMT_LINTER_RULES}" --check --compact-diff)
+LINTER_COMMANDS_ARRAY_STATES=(asl-validator --json-path)
+LINTER_COMMANDS_ARRAY_SQL=(sql-lint --config "${SQL_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_SQLFLUFF=(sqlfluff lint --config "${SQLFLUFF_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_TEKTON=(tekton-lint)
+LINTER_COMMANDS_ARRAY_TERRAFORM_FMT=(terraform fmt -check -diff)
+LINTER_COMMANDS_ARRAY_TERRAFORM_TFLINT=("TF_DATA_DIR=\"/tmp/.terraform-TERRAFORM_TFLINT-{//}\"" tflint -c "${TERRAFORM_TFLINT_LINTER_RULES}" "--filter=\"{/}\"")
+LINTER_COMMANDS_ARRAY_TERRAFORM_TERRASCAN=(terrascan scan -i terraform -t all -c "${TERRAFORM_TERRASCAN_LINTER_RULES}" -f)
+LINTER_COMMANDS_ARRAY_TERRAGRUNT=(terragrunt hclfmt --terragrunt-check --terragrunt-log-level error --terragrunt-hclfmt-file)
+LINTER_COMMANDS_ARRAY_TSX=(eslint --no-eslintrc -c "${TSX_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_TYPESCRIPT_ES=(eslint --no-eslintrc -c "${TYPESCRIPT_ES_LINTER_RULES}")
+LINTER_COMMANDS_ARRAY_TYPESCRIPT_STANDARD=(ts-standard --parser @typescript-eslint/parser --plugin @typescript-eslint/eslint-plugin --project "${TYPESCRIPT_STANDARD_TSCONFIG_FILE}")
+LINTER_COMMANDS_ARRAY_TYPESCRIPT_PRETTIER=(prettier --check)
+LINTER_COMMANDS_ARRAY_XML=(xmllint)
+LINTER_COMMANDS_ARRAY_YAML=(yamllint -c "${YAML_LINTER_RULES}" -f parsable)
+if [ "${YAML_ERROR_ON_WARNING}" == 'true' ]; then
+  LINTER_COMMANDS_ARRAY_YAML+=(--strict)
+fi

lib/functions/log.sh

@@ -98,8 +98,11 @@ fatal() {
 
 # shellcheck disable=SC2034  # Variable is referenced in other files
 SUPER_LINTER_INITIALIZATION_LOG_GROUP_TITLE="Super-Linter initialization"
+export SUPER_LINTER_INITIALIZATION_LOG_GROUP_TITLE
 GITHUB_ACTIONS_LOG_GROUP_MARKER_START="start"
+export GITHUB_ACTIONS_LOG_GROUP_MARKER_START
 GITHUB_ACTIONS_LOG_GROUP_MARKER_END="end"
+export GITHUB_ACTIONS_LOG_GROUP_MARKER_END
 
 writeGitHubActionsLogGroupMarker() {
   local LOG_GROUP_MARKER_MODE="${1}"
@@ -139,3 +142,16 @@ startGitHubActionsLogGroup() {
 endGitHubActionsLogGroup() {
   writeGitHubActionsLogGroupMarker "${GITHUB_ACTIONS_LOG_GROUP_MARKER_END}" "${1}"
 }
+
+# We need these functions to be available when using parallel to run subprocesses
+export -f debug
+export -f endGitHubActionsLogGroup
+export -f error
+export -f fatal
+export -f info
+export -f log
+export -f notice
+export -f startGitHubActionsLogGroup
+export -f trace
+export -f warn
+export -f writeGitHubActionsLogGroupMarker

lib/functions/validation.sh

@@ -13,11 +13,11 @@ function ValidateBooleanConfigurationVariables() {
   ValidateBooleanVariable "SSH_SETUP_GITHUB" "${SSH_SETUP_GITHUB}"
   ValidateBooleanVariable "SUPPRESS_FILE_TYPE_WARN" "${SUPPRESS_FILE_TYPE_WARN}"
   ValidateBooleanVariable "SUPPRESS_POSSUM" "${SUPPRESS_POSSUM}"
-  ValidateBooleanVariable "USE_FIND_ALGORITHM" "${USE_FIND_ALGORITHM}"
   ValidateBooleanVariable "TEST_CASE_RUN" "${TEST_CASE_RUN}"
+  ValidateBooleanVariable "USE_FIND_ALGORITHM" "${USE_FIND_ALGORITHM}"
   ValidateBooleanVariable "VALIDATE_ALL_CODEBASE" "${VALIDATE_ALL_CODEBASE}"
-  ValidateBooleanVariable "YAML_ERROR_ON_WARNING" "${YAML_ERROR_ON_WARNING}"
   ValidateBooleanVariable "WRITE_LINTER_VERSIONS_FILE" "${WRITE_LINTER_VERSIONS_FILE}"
+  ValidateBooleanVariable "YAML_ERROR_ON_WARNING" "${YAML_ERROR_ON_WARNING}"
 }
 
 function ValidateGitHubWorkspace() {
@@ -100,14 +100,6 @@ function GetValidationInfo() {
     VALIDATE_LANGUAGE="VALIDATE_${LANGUAGE}"
     if [[ ${!VALIDATE_LANGUAGE} == "true" ]]; then
       debug "- Validating [${LANGUAGE}] files in code base..."
-
-      debug "Defining variables for ${LANGUAGE} linter..."
-
-      ERRORS_VARIABLE_NAME="ERRORS_FOUND_${LANGUAGE}"
-      debug "Setting ${ERRORS_VARIABLE_NAME} variable value to 0..."
-      eval "${ERRORS_VARIABLE_NAME}=0"
-      debug "Exporting ${ERRORS_VARIABLE_NAME} variable..."
-      eval "export ${ERRORS_VARIABLE_NAME}"
     else
       debug "- Excluding [$LANGUAGE] files in code base..."
     fi
@@ -139,10 +131,6 @@ function GetValidationInfo() {
     ANSIBLE_DIRECTORY="${TEMP_ANSIBLE_DIRECTORY}"
     debug "Setting Ansible directory to: ${ANSIBLE_DIRECTORY}"
   fi
-
-  debug "Runner: $(id -un 2>/dev/null)"
-  debug "ENV:"
-  debug "$(printenv | sort)"
 }
 
 function CheckIfGitBranchExists() {
@@ -170,6 +158,7 @@ function ValidateBooleanVariable() {
     debug "${VAR_NAME} has a valid boolean string value: ${VAR_VALUE}"
   fi
 }
+export -f ValidateBooleanVariable
 
 function ValidateLocalGitRepository() {
   debug "Check if ${GITHUB_WORKSPACE} is a Git repository"
@@ -251,6 +240,10 @@ function CheckovConfigurationFileContainsDirectoryOption() {
   local CONFIGURATION_OPTION_KEY="directory:"
   debug "Checking if ${CHECKOV_LINTER_RULES_PATH} contains a '${CONFIGURATION_OPTION_KEY}' configuration option"
 
+  if [ ! -e "${CHECKOV_LINTER_RULES_PATH}" ]; then
+    fatal "${CHECKOV_LINTER_RULES_PATH} doesn't exist. Cannot check if it contains a '${CONFIGURATION_OPTION_KEY}' configuration option"
+  fi
+
   if grep -q "${CONFIGURATION_OPTION_KEY}" "${CHECKOV_LINTER_RULES_PATH}"; then
     debug "${CHECKOV_LINTER_RULES_PATH} contains a '${CONFIGURATION_OPTION_KEY}' statement"
     return 0
@@ -259,6 +252,7 @@ function CheckovConfigurationFileContainsDirectoryOption() {
     return 1
   fi
 }
+export -f CheckovConfigurationFileContainsDirectoryOption
 
 function WarnIfVariableIsSet() {
   local INPUT_VARIABLE="${1}"

lib/functions/worker.sh

@@ -1,213 +1,218 @@
 #!/usr/bin/env bash
 
 function LintCodebase() {
+  local FILE_TYPE
   FILE_TYPE="${1}" && shift
-  LINTER_NAME="${1}" && shift
+  local TEST_CASE_RUN
-  LINTER_COMMAND="${1}" && shift
-  FILTER_REGEX_INCLUDE="${1}" && shift
-  FILTER_REGEX_EXCLUDE="${1}" && shift
   TEST_CASE_RUN="${1}" && shift
-  FILE_ARRAY=("$@")
 
-  # Array to track directories where tflint was run
+  declare -n VALIDATE_LANGUAGE
-  declare -A TFLINT_SEEN_DIRS
+  VALIDATE_LANGUAGE="VALIDATE_${FILE_TYPE}"
+
+  if [[ "${VALIDATE_LANGUAGE}" == "false" ]]; then
+    if [[ "${TEST_CASE_RUN}" == "false" ]]; then
+      debug "Skip validation of ${FILE_TYPE} because VALIDATE_LANGUAGE is ${VALIDATE_LANGUAGE}"
+      unset -n VALIDATE_LANGUAGE
+      return 0
+    else
+      fatal "Don't disable any validation when running in test mode. VALIDATE_${FILE_TYPE} is set to: ${VALIDATE_LANGUAGE}. Set it to: true"
+    fi
+  fi
+
+  debug "Running LintCodebase. FILE_TYPE: ${FILE_TYPE}. TEST_CASE_RUN: ${TEST_CASE_RUN}"
+
+  debug "VALIDATE_LANGUAGE for ${FILE_TYPE}: ${VALIDATE_LANGUAGE}..."
+
+  ValidateBooleanVariable "TEST_CASE_RUN" "${TEST_CASE_RUN}"
+  ValidateBooleanVariable "VALIDATE_${FILE_TYPE}" "${VALIDATE_LANGUAGE}"
+
+  unset -n VALIDATE_LANGUAGE
+
+  debug "Populating file array for ${FILE_TYPE}"
+  local -n FILE_ARRAY="FILE_ARRAY_${FILE_TYPE}"
+  local FILE_ARRAY_LANGUAGE_PATH="${FILE_ARRAYS_DIRECTORY_PATH}/file-array-${FILE_TYPE}"
+  if [[ -e "${FILE_ARRAY_LANGUAGE_PATH}" ]]; then
+    while read -r FILE; do
+      if [[ "${TEST_CASE_RUN}" == "true" ]]; then
+        debug "Ensure that the list files to check for ${FILE_TYPE} doesn't include test cases for other languages"
+        # Folder for specific tests. By convention, the last part of the path is the lowercased FILE_TYPE
+        local TEST_CASE_DIRECTORY
+        TEST_CASE_DIRECTORY="${FILE_TYPE,,}"
+
+        # We use configuration files to pass the list of files to lint to checkov
+        # Their name includes "checkov", which is equal to FILE_TYPE for Checkov.
+        # In this case, we don't add a trailing slash so we don't fail validation.
+        if [[ "${FILE_TYPE}" != "CHECKOV" ]]; then
+          TEST_CASE_DIRECTORY="${TEST_CASE_DIRECTORY}/"
+          debug "Adding a traling slash to the test case directory for ${FILE_TYPE}: ${TEST_CASE_DIRECTORY}"
+        fi
+
+        debug "TEST_CASE_DIRECTORY for ${FILE_TYPE}: ${TEST_CASE_DIRECTORY}"
+        if [[ ${FILE} != *"${TEST_CASE_DIRECTORY}"* ]]; then
+          debug "Excluding ${FILE} because it's not in the test case directory for ${FILE_TYPE}..."
+          continue
+        else
+          debug "Including ${FILE} because it's a test case for ${FILE_TYPE}"
+        fi
+      fi
+      FILE_ARRAY+=("${FILE}")
+    done <"${FILE_ARRAY_LANGUAGE_PATH}"
+  else
+    debug "${FILE_ARRAY_LANGUAGE_PATH} doesn't exist. Skip loading the list of files and directories to lint for ${FILE_TYPE}"
+  fi
+
+  if [[ "${#FILE_ARRAY[@]}" -eq 0 ]]; then
+    if [[ "${TEST_CASE_RUN}" == "false" ]]; then
+      debug "There are no items to lint for ${FILE_TYPE}"
+      unset -n FILE_ARRAY
+      return 0
+    else
+      fatal "Cannot find any tests for ${FILE_TYPE}"
+    fi
+  else
+    debug "There are ${#FILE_ARRAY[@]} items to lint for ${FILE_TYPE}: ${FILE_ARRAY[*]}"
+  fi
+
+  startGitHubActionsLogGroup "${FILE_TYPE}"
+
+  info "Linting ${FILE_TYPE} items..."
+
+  local PARALLEL_RESULTS_FILE_PATH
+  PARALLEL_RESULTS_FILE_PATH="/tmp/super-linter-worker-results-${FILE_TYPE}.json"
+  debug "PARALLEL_RESULTS_FILE_PATH for ${FILE_TYPE}: ${PARALLEL_RESULTS_FILE_PATH}"
+
+  local -a PARALLEL_COMMAND
+  PARALLEL_COMMAND=(parallel --will-cite --keep-order --max-procs "$(($(nproc) * 1))" --xargs --results "${PARALLEL_RESULTS_FILE_PATH}")
+
+  if [ "${LOG_DEBUG}" == "true" ]; then
+    debug "LOG_DEBUG is enabled. Enable verbose ouput for parallel"
+    PARALLEL_COMMAND+=(--verbose)
+  fi
+  debug "PARALLEL_COMMAND for ${FILE_TYPE}: ${PARALLEL_COMMAND[*]}"
+
+  # The following linters support linting one file at a time, and don't support linting a list of files,
+  # so we cannot pass more than one file per invocation
+  if [[ "${FILE_TYPE}" == "ANSIBLE" ]] ||
+    [[ "${FILE_TYPE}" == "ARM" ]] ||
+    [[ "${FILE_TYPE}" == "BASH_EXEC" ]] ||
+    [[ "${FILE_TYPE}" == "CLOJURE" ]] ||
+    [[ "${FILE_TYPE}" == "CSHARP" ]] ||
+    [[ "${FILE_TYPE}" == "GITLEAKS" ]] ||
+    [[ "${FILE_TYPE}" == "GO_MODULES" ]] ||
+    [[ "${FILE_TYPE}" == "JSCPD" ]] ||
+    [[ "${FILE_TYPE}" == "KOTLIN" ]] ||
+    [[ "${FILE_TYPE}" == "SQL" ]] ||
+    [[ "${FILE_TYPE}" == "SQLFLUFF" ]] ||
+    [[ "${FILE_TYPE}" == "CHECKOV" ]] ||
+    [[ "${FILE_TYPE}" == "POWERSHELL" ]] ||
+    [[ "${FILE_TYPE}" == "R" ]] ||
+    [[ "${FILE_TYPE}" == "RUST_CLIPPY" ]] ||
+    [[ "${FILE_TYPE}" == "SNAKEMAKE_LINT" ]] ||
+    [[ "${FILE_TYPE}" == "STATES" ]] ||
+    [[ "${FILE_TYPE}" == "TERRAFORM_TFLINT" ]] ||
+    [[ "${FILE_TYPE}" == "TERRAFORM_TERRASCAN" ]] ||
+    [[ "${FILE_TYPE}" == "TERRAGRUNT" ]]; then
+    debug "${FILE_TYPE} doesn't support linting files in batches. Configure the linter to run over the files to lint one by one"
+    PARALLEL_COMMAND+=(--max-lines 1)
+  fi
+  debug "PARALLEL_COMMAND for ${FILE_TYPE} after updating the number of files to lint per process: ${PARALLEL_COMMAND[*]}"
+
+  local LINTER_WORKING_DIRECTORY
+  LINTER_WORKING_DIRECTORY="${GITHUB_WORKSPACE}"
+
+  # GNU parallel parameter expansion:
+  # - {} input item
+  # - {/} basename of the input lint
+  # - {//} dirname of input line
+
+  if [[ ${FILE_TYPE} == "CSHARP" ]] ||
+    [[ (${FILE_TYPE} == "R" && -f "$(dirname "${FILE}")/.lintr") ]] ||
+    [[ ${FILE_TYPE} == "KOTLIN" ]] ||
+    [[ ${FILE_TYPE} == "TERRAFORM_TFLINT" ]]; then
+    LINTER_WORKING_DIRECTORY="{//}"
+  elif [[ ${FILE_TYPE} == "ANSIBLE" ]] ||
+    [[ ${FILE_TYPE} == "GO_MODULES" ]]; then
+    LINTER_WORKING_DIRECTORY="{}"
+  fi
+
+  debug "LINTER_WORKING_DIRECTORY for ${FILE_TYPE}: ${LINTER_WORKING_DIRECTORY}"
+  PARALLEL_COMMAND+=(--workdir "${LINTER_WORKING_DIRECTORY}")
+  debug "PARALLEL_COMMAND for ${FILE_TYPE} after updating the working directory: ${PARALLEL_COMMAND[*]}"
+
+  # shellcheck source=/dev/null
+  source /action/lib/functions/linterCommands.sh
+
+  local -n LINTER_COMMAND_ARRAY
+  LINTER_COMMAND_ARRAY="LINTER_COMMANDS_ARRAY_${FILE_TYPE}"
+  if [ ${#LINTER_COMMAND_ARRAY[@]} -eq 0 ]; then
+    fatal "LINTER_COMMAND_ARRAY for ${FILE_TYPE} is empty."
+  else
+    debug "LINTER_COMMAND_ARRAY for ${FILE_TYPE} has ${#LINTER_COMMAND_ARRAY[@]} elements: ${LINTER_COMMAND_ARRAY[*]}"
+  fi
+
+  PARALLEL_COMMAND+=("${LINTER_COMMAND_ARRAY[@]}")
+  debug "PARALLEL_COMMAND for ${FILE_TYPE} after LINTER_COMMAND_ARRAY concatenation: ${PARALLEL_COMMAND[*]}"
+
+  unset -n LINTER_COMMAND_ARRAY
+
+  local PARALLEL_COMMAND_OUTPUT
+  local PARALLEL_COMMAND_RETURN_CODE
+  PARALLEL_COMMAND_OUTPUT=$(printf "%s\n" "${FILE_ARRAY[@]}" | "${PARALLEL_COMMAND[@]}" 2>&1)
+  # Don't check for errors on this return code because commands can fail if linter report errors
+  PARALLEL_COMMAND_RETURN_CODE=$?
+  debug "PARALLEL_COMMAND_OUTPUT for ${FILE_TYPE} (exit code: ${PARALLEL_COMMAND_RETURN_CODE}): ${PARALLEL_COMMAND_OUTPUT}"
+  debug "Parallel output file (${PARALLEL_RESULTS_FILE_PATH}) contents for ${FILE_TYPE}:\n$(cat "${PARALLEL_RESULTS_FILE_PATH}")"
+
+  echo ${PARALLEL_COMMAND_RETURN_CODE} >"/tmp/super-linter-parallel-command-exit-code-${FILE_TYPE}"
+
+  if [ ${PARALLEL_COMMAND_RETURN_CODE} -ne 0 ]; then
+    error "Found errors when linting ${FILE_TYPE}. Exit code: ${PARALLEL_COMMAND_RETURN_CODE}."
+  else
+    notice "${FILE_TYPE} linted successfully"
+  fi
+
+  local RESULTS_OBJECT
+  RESULTS_OBJECT=
+  if ! RESULTS_OBJECT=$(jq -n '[inputs]' "${PARALLEL_RESULTS_FILE_PATH}"); then
+    fatal "Error loading results for ${FILE_TYPE}: ${RESULTS_OBJECT}"
+  fi
+  debug "RESULTS_OBJECT for ${FILE_TYPE}:\n${RESULTS_OBJECT}"
 
   # To count how many files were checked for a given FILE_TYPE
+  local INDEX
   INDEX=0
-
+  if ! ((INDEX = $(jq '[.[] | .V | length] | add' <<<"${RESULTS_OBJECT}"))); then
-  # To check how many "bad" and "good" test cases we ran
+    fatal "Error when setting INDEX for ${FILE_TYPE}: ${INDEX}"
-  BAD_TEST_CASES_COUNT=0
-  GOOD_TEST_CASES_COUNT=0
-
-  WORKSPACE_PATH="${GITHUB_WORKSPACE}"
-  if [ "${TEST_CASE_RUN}" == "true" ]; then
-    WORKSPACE_PATH="${GITHUB_WORKSPACE}/${TEST_CASE_FOLDER}"
   fi
-  debug "Workspace path: ${WORKSPACE_PATH}"
+  debug "Set INDEX for ${FILE_TYPE} to: ${INDEX}"
 
-  info ""
+  local STDOUT_LINTER
-  info "----------------------------------------------"
+  # Get raw output so we can strip quotes from the data we load
-  info "----------------------------------------------"
+  if ! STDOUT_LINTER="$(jq --raw-output '.[].Stdout' <<<"${RESULTS_OBJECT}")"; then
-  debug "Running LintCodebase. FILE_TYPE: ${FILE_TYPE}. Linter name: ${LINTER_NAME}, linter command: ${LINTER_COMMAND}, TEST_CASE_RUN: ${TEST_CASE_RUN}, FILTER_REGEX_INCLUDE: ${FILTER_REGEX_INCLUDE}, FILTER_REGEX_EXCLUDE: ${FILTER_REGEX_EXCLUDE}, files to lint: ${FILE_ARRAY[*]}"
+    fatal "Error when loading stdout for ${FILE_TYPE}:\n${STDOUT_LINTER}"
-  info "Linting ${FILE_TYPE} files..."
-  info "----------------------------------------------"
-  info "----------------------------------------------"
-
-  for FILE in "${FILE_ARRAY[@]}"; do
-    info "Checking file: ${FILE}"
-
-    if [[ "${TEST_CASE_RUN}" == "true" ]]; then
-      # Folder for specific tests. By convention, the last part of the path is the lowercased FILE_TYPE
-      local TEST_CASE_DIRECTORY
-      TEST_CASE_DIRECTORY="${TEST_CASE_FOLDER}/${FILE_TYPE,,}/"
-      debug "TEST_CASE_DIRECTORY for ${FILE}: ${TEST_CASE_DIRECTORY}"
-
-      if [[ ${FILE} != *"${TEST_CASE_DIRECTORY}"* ]]; then
-        debug "Skipping ${FILE} because it's not in the test case directory for ${FILE_TYPE}..."
-        continue
-      fi
-    fi
-
-    local FILE_NAME
-    FILE_NAME=$(basename "${FILE}" 2>&1)
-    debug "FILE_NAME for ${FILE}: ${FILE_NAME}"
-
-    local DIR_NAME
-    DIR_NAME=$(dirname "${FILE}" 2>&1)
-    debug "DIR_NAME for ${FILE}: ${DIR_NAME}"
-
-    (("INDEX++"))
-
-    LINTED_LANGUAGES_ARRAY+=("${FILE_TYPE}")
-    local LINT_CMD
-    LINT_CMD=''
-
-    if [[ ${FILE_TYPE} == "POWERSHELL" ]] || [[ ${FILE_TYPE} == "ARM" ]]; then
-      # Need to run PowerShell commands using pwsh -c, also exit with exit code from inner subshell
-      LINT_CMD=$(
-        cd "${WORKSPACE_PATH}" || exit
-        pwsh -NoProfile -NoLogo -Command "${LINTER_COMMAND} \"${FILE}\"; if (\${Error}.Count) { exit 1 }"
-        exit $? 2>&1
-      )
-    elif [[ ${FILE_TYPE} == "R" ]]; then
-      local r_dir
-      if [ ! -f "${DIR_NAME}/.lintr" ]; then
-        r_dir="${WORKSPACE_PATH}"
-      else
-        r_dir="${DIR_NAME}"
-      fi
-      LINT_CMD=$(
-        cd "$r_dir" || exit
-        R --slave -e "lints <- lintr::lint('$FILE');print(lints);errors <- purrr::keep(lints, ~ .\$type == 'error');quit(save = 'no', status = if (length(errors) > 0) 1 else 0)" 2>&1
-      )
-    elif [[ ${FILE_TYPE} == "CSHARP" ]]; then
-      # Because the C# linter writes to tty and not stdout
-      LINT_CMD=$(
-        cd "${DIR_NAME}" || exit
-        ${LINTER_COMMAND} "${FILE_NAME}" | tee /dev/tty2 2>&1
-        exit "${PIPESTATUS[0]}"
-      )
-    elif [[ ${FILE_TYPE} == "ANSIBLE" ]] ||
-      [[ ${FILE_TYPE} == "GO_MODULES" ]]; then
-      debug "Linting ${FILE_TYPE}. Changing the working directory to ${FILE} before running the linter."
-      # Because it expects that the working directory is a Go module (GO_MODULES) or
-      # because we want to enable ansible-lint autodetection mode.
-      # Ref: https://ansible-lint.readthedocs.io/usage
-      LINT_CMD=$(
-        cd "${FILE}" || exit 1
-        ${LINTER_COMMAND} 2>&1
-      )
-    elif [[ ${FILE_TYPE} == "KOTLIN" ]]; then
-      # Because it needs to change directory to where the file to lint is
-      LINT_CMD=$(
-        cd "${DIR_NAME}" || exit
-        ${LINTER_COMMAND} "${FILE_NAME}" 2>&1
-      )
-    elif [[ ${FILE_TYPE} == "TERRAFORM_TFLINT" ]]; then
-      # Check the cache to see if we've already prepped this directory for tflint
-      if [[ ! -v "TFLINT_SEEN_DIRS[${DIR_NAME}]" ]]; then
-        debug "Configuring Terraform data directory for ${DIR_NAME}"
-
-        # Define the path to an empty Terraform data directory
-        # (def: https://developer.hashicorp.com/terraform/cli/config/environment-variables#tf_data_dir)
-        # in case the user has a Terraform data directory already, and we don't
-        # want to modify it.
-        # TFlint considers this variable as well.
-        # Ref: https://github.com/terraform-linters/tflint/blob/master/docs/user-guide/compatibility.md#environment-variables
-        local TF_DATA_DIR
-        TF_DATA_DIR="/tmp/.terraform-${FILE_TYPE}-${DIR_NAME}"
-        export TF_DATA_DIR
-        # Let the cache know we've seen this before
-        # Set the value to an arbitrary non-empty string.
-
-        # Fetch Terraform modules
-        debug "Fetch Terraform modules for ${DIR_NAME} in ${TF_DATA_DIR}"
-        local FETCH_TERRAFORM_MODULES_CMD
-        FETCH_TERRAFORM_MODULES_CMD="$(terraform get)"
-        ERROR_CODE=$?
-        debug "Fetch Terraform modules. Exit code: ${ERROR_CODE}. Command output: ${FETCH_TERRAFORM_MODULES_CMD}"
-        if [ ${ERROR_CODE} -ne 0 ]; then
-          fatal "Error when fetching Terraform modules while linting ${FILE}"
-        fi
-        TFLINT_SEEN_DIRS[${DIR_NAME}]="false"
-      fi
-
-      # Because it needs to change the directory to where the file to lint is
-      LINT_CMD=$(
-        cd "${DIR_NAME}" || exit
-        ${LINTER_COMMAND} --filter="${FILE_NAME}" 2>&1
-      )
-    else
-      LINT_CMD=$(
-        cd "${WORKSPACE_PATH}" || exit
-        ${LINTER_COMMAND} "${FILE}" 2>&1
-      )
-    fi
-
-    ERROR_CODE=$?
-
-    local FILE_STATUS
-    # Assume that the file should pass linting checks
-    FILE_STATUS="good"
-
-    if [[ "${TEST_CASE_RUN}" == "true" ]] && [[ ${FILE} == *"bad"* ]]; then
-      FILE_STATUS="bad"
-      debug "We are running in test mode. Updating the expected FILE_STATUS for ${FILE} to: ${FILE_STATUS}"
-    fi
-
-    debug "Results for ${FILE}. Exit code: ${ERROR_CODE}. Command output:\n------\n${LINT_CMD}\n------"
-
-    ########################################
-    # File status = good, this should pass #
-    ########################################
-    if [[ ${FILE_STATUS} == "good" ]]; then
-      (("GOOD_TEST_CASES_COUNT++"))
-
-      if [ ${ERROR_CODE} -ne 0 ]; then
-        error "Found errors when linting ${FILE_NAME}. Exit code: ${ERROR_CODE}. Command output:\n------\n${LINT_CMD}\n------"
-        (("ERRORS_FOUND_${FILE_TYPE}++"))
-      else
-        notice "${FILE} was linted successfully"
-        if [ -n "${LINT_CMD}" ]; then
-          info "Command output for ${FILE_NAME}:\n------\n${LINT_CMD}\n------"
-        fi
-      fi
-    #######################################
-    # File status = bad, this should fail #
-    #######################################
-    else
-      if [[ "${TEST_CASE_RUN}" == "false" ]]; then
-        fatal "All files are supposed to pass linting checks when not running in test mode."
-      fi
-
-      (("BAD_TEST_CASES_COUNT++"))
-
-      if [ ${ERROR_CODE} -eq 0 ]; then
-        error "${FILE} should have failed test case."
-        (("ERRORS_FOUND_${FILE_TYPE}++"))
-      else
-        notice "${FILE} failed the test case as expected"
-      fi
-    fi
-  done
-
-  if [ "${TEST_CASE_RUN}" = "true" ]; then
-
-    debug "${LINTER_NAME} test suite has ${INDEX} test(s), of which ${BAD_TEST_CASES_COUNT} 'bad' (expected to fail), ${GOOD_TEST_CASES_COUNT} 'good' (expected to pass)."
-
-    # Check if we ran at least one test
-    if [ "${INDEX}" -eq 0 ]; then
-      fatal "Failed to find any tests ran for: ${LINTER_NAME}. Check that tests exist for linter: ${LINTER_NAME}"
-    fi
-
-    # Check if we ran at least one 'bad' test
-    if [ "${BAD_TEST_CASES_COUNT}" -eq 0 ]; then
-      fatal "Failed to find any tests that are expected to fail for: ${LINTER_NAME}. Check that tests that are expected to fail exist for linter: ${LINTER_NAME}"
-    fi
-
-    # Check if we ran at least one 'good' test
-    if [ "${GOOD_TEST_CASES_COUNT}" -eq 0 ]; then
-      fatal "Failed to find any tests that are expected to pass for: ${LINTER_NAME}. Check that tests that are expected to pass exist for linter: ${LINTER_NAME}"
-    fi
   fi
+
+  if [ -n "${STDOUT_LINTER}" ]; then
+    info "Command output for ${FILE_TYPE}:\n------\n${STDOUT_LINTER}\n------"
+  else
+    debug "Stdout for ${FILE_TYPE} is empty"
+  fi
+
+  local STDERR_LINTER
+  if ! STDERR_LINTER="$(jq --raw-output '.[].Stderr' <<<"${RESULTS_OBJECT}")"; then
+    fatal "Error when loading stderr for ${FILE_TYPE}:\n${STDERR_LINTER}"
+  fi
+
+  if [ -n "${STDERR_LINTER}" ]; then
+    info "Command output for ${FILE_TYPE}:\n------\n${STDERR_LINTER}\n------"
+  else
+    debug "Stderr for ${FILE_TYPE} is empty"
+  fi
+
+  unset -n FILE_ARRAY
+
+  endGitHubActionsLogGroup "${FILE_TYPE}"
 }
+
+# We need this for parallel
+export -f LintCodebase

lib/linter.sh

@@ -77,6 +77,7 @@ else
 fi
 # Let users configure GitHub Actions log markers regardless of running locally or not
 ENABLE_GITHUB_ACTIONS_GROUP_TITLE="${ENABLE_GITHUB_ACTIONS_GROUP_TITLE:-"${DEFAULT_ENABLE_GITHUB_ACTIONS_GROUP_TITLE}"}"
+export ENABLE_GITHUB_ACTIONS_GROUP_TITLE
 
 startGitHubActionsLogGroup "${SUPER_LINTER_INITIALIZATION_LOG_GROUP_TITLE}"
 
@@ -88,10 +89,12 @@ DISABLE_ERRORS="${DISABLE_ERRORS:-"false"}"
 declare -l IGNORE_GENERATED_FILES
 # Do not ignore generated files by default for backwards compatibility
 IGNORE_GENERATED_FILES="${IGNORE_GENERATED_FILES:-false}"
+export IGNORE_GENERATED_FILES
 
 # We want a lowercase value
 declare -l IGNORE_GITIGNORED_FILES
 IGNORE_GITIGNORED_FILES="${IGNORE_GITIGNORED_FILES:-false}"
+export IGNORE_GITIGNORED_FILES
 
 # We want a lowercase value
 declare -l MULTI_STATUS
@@ -151,7 +154,6 @@ GITHUB_API_URL="${GITHUB_API_URL%/}"
 GITHUB_SERVER_URL="${GITHUB_DOMAIN:-"https://github.com"}"
 # Extract domain name from URL
 GITHUB_SERVER_URL=$(echo "$GITHUB_SERVER_URL" | cut -d '/' -f 3)
-LINTED_LANGUAGES_ARRAY=()                                 # Will be filled at run time with all languages that were linted
 LINTER_RULES_PATH="${LINTER_RULES_PATH:-.github/linters}" # Linter rules directory
 # shellcheck disable=SC2034 # Variable is referenced in other scripts
 RAW_FILE_ARRAY=() # Array of all files that were changed
@@ -610,6 +612,9 @@ GetGitHubVars() {
   else
     debug "Skip GITHUB_TOKEN, GITHUB_REPOSITORY, and GITHUB_RUN_ID validation because we don't need these variables for GitHub Actions status reports. MULTI_STATUS: ${MULTI_STATUS}"
   fi
+
+  # We need this for parallel
+  export GITHUB_WORKSPACE
 }
 ################################################################################
 #### Function CallStatusAPI ####################################################
@@ -666,84 +671,62 @@ CallStatusAPI() {
     fi
   fi
 }
-################################################################################
+
-#### Function Footer ###########################################################
 Footer() {
   info "----------------------------------------------"
   info "----------------------------------------------"
-  info "The script has completed"
-  info "----------------------------------------------"
-  info "----------------------------------------------"
 
-  ####################################################
+  local ANY_LINTER_SUCCESS
-  # Need to clean up the lanuage array of duplicates #
+  ANY_LINTER_SUCCESS="false"
-  ####################################################
+
-  mapfile -t UNIQUE_LINTED_ARRAY < <(for LANG in "${LINTED_LANGUAGES_ARRAY[@]}"; do echo "${LANG}"; done | sort -u)
+  local SUPER_LINTER_EXIT_CODE
-  export UNIQUE_LINTED_ARRAY # Workaround SC2034
+  SUPER_LINTER_EXIT_CODE=0
 
-  ##############################
-  # Prints for errors if found #
-  ##############################
   for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do
-    ###########################
+    # This used to be the count of errors found for a given LANGUAGE, but since
-    # Build the error counter #
+    # after we switched to running linters against a batch of files, it may not
-    ###########################
+    # represent the actual number of files that didn't pass the validation,
-    ERROR_COUNTER="ERRORS_FOUND_${LANGUAGE}"
+    # but a number that's less than that because of how GNU parallel returns
+    # exit codes.
+    # Ref: https://www.gnu.org/software/parallel/parallel.html#exit-status
+    ERROR_COUNTER_FILE_PATH="/tmp/super-linter-parallel-command-exit-code-${LANGUAGE}"
+    if [ ! -f "${ERROR_COUNTER_FILE_PATH}" ]; then
+      debug "Error counter ${ERROR_COUNTER_FILE_PATH} doesn't exist"
+    else
+      ERROR_COUNTER=$(<"${ERROR_COUNTER_FILE_PATH}")
+      debug "ERROR_COUNTER for ${LANGUAGE}: ${ERROR_COUNTER}"
 
-    ##################
+      if [[ ${ERROR_COUNTER} -ne 0 ]]; then
-    # Print if not 0 #
+        error "Errors found in ${LANGUAGE}"
-    ##################
+        CallStatusAPI "${LANGUAGE}" "error"
-    if [[ ${!ERROR_COUNTER} -ne 0 ]]; then
+        SUPER_LINTER_EXIT_CODE=1
-      # We found errors in the language
+        debug "Setting super-linter exit code to ${SUPER_LINTER_EXIT_CODE} because there were errors for ${LANGUAGE}"
-      ###################
+      elif [[ ${ERROR_COUNTER} -eq 0 ]]; then
-      # Print the goods #
+        notice "Successfully linted ${LANGUAGE}"
-      ###################
-      error "ERRORS FOUND in ${LANGUAGE}:[${!ERROR_COUNTER}]"
-
-      #########################################
-      # Create status API for Failed language #
-      #########################################
-      CallStatusAPI "${LANGUAGE}" "error"
-      ######################################
-      # Check if we validated the language #
-      ######################################
-    elif [[ ${!ERROR_COUNTER} -eq 0 ]]; then
-      if CheckInArray "${LANGUAGE}"; then
-        # No errors found when linting the language
         CallStatusAPI "${LANGUAGE}" "success"
+        ANY_LINTER_SUCCESS="true"
+        debug "Set ANY_LINTER_SUCCESS to ${ANY_LINTER_SUCCESS} because ${LANGUAGE} reported a success"
       fi
     fi
   done
 
-  ##################################
+  if [[ "${ANY_LINTER_SUCCESS}" == "true" ]] && [[ ${SUPER_LINTER_EXIT_CODE} -ne 0 ]]; then
-  # Exit with 0 if errors disabled #
+    SUPER_LINTER_EXIT_CODE=2
-  ##################################
+    debug "There was at least one linter that reported a success. Setting the super-linter exit code to: ${SUPER_LINTER_EXIT_CODE}"
-  if [ "${DISABLE_ERRORS}" == "true" ]; then
-    warn "Exiting with exit code:[0] as:[DISABLE_ERRORS] was set to:[${DISABLE_ERRORS}]"
-    exit 0
   fi
 
-  ###############################
+  if [ "${DISABLE_ERRORS}" == "true" ]; then
-  # Exit with 1 if errors found #
+    warn "The super-linter exit code is ${SUPER_LINTER_EXIT_CODE}. Forcibly setting it to 0 because DISABLE_ERRORS is set to: ${DISABLE_ERRORS}"
-  ###############################
+    SUPER_LINTER_EXIT_CODE=0
-  # Loop through all languages
+  fi
-  for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do
-    # build the variable
-    ERRORS_FOUND_LANGUAGE="ERRORS_FOUND_${LANGUAGE}"
-    # Check if error was found
-    if [[ ${!ERRORS_FOUND_LANGUAGE} -ne 0 ]]; then
-      # Failed exit
-      fatal "Exiting with errors found!"
-    fi
-  done
 
-  ########################
+  if [[ ${SUPER_LINTER_EXIT_CODE} -eq 0 ]]; then
-  # Footer prints Exit 0 #
+    notice "All files and directories linted successfully"
-  ########################
+  else
-  notice "All file(s) linted successfully with no errors detected"
+    error "Super-linter detected linting errors"
-  info "----------------------------------------------"
+  fi
-  # Successful exit
+
-  exit 0
+  exit ${SUPER_LINTER_EXIT_CODE}
 }
 ################################################################################
 #### Function UpdateLoopsForImage ##############################################
@@ -892,133 +875,6 @@ done
 # Load rules for special cases
 GetStandardRules "javascript"
 
-##########################
-# Define linter commands #
-##########################
-declare -A LINTER_COMMANDS_ARRAY
-LINTER_COMMANDS_ARRAY['ANSIBLE']="ansible-lint -c ${ANSIBLE_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['ARM']="Import-Module ${ARM_TTK_PSD1} ; \${config} = \$(Import-PowerShellDataFile -Path ${ARM_LINTER_RULES}) ; Test-AzTemplate @config -TemplatePath"
-if [ -z "${BASH_SEVERITY}" ]; then
-  LINTER_COMMANDS_ARRAY['BASH']="shellcheck --color --external-sources"
-else
-  LINTER_COMMANDS_ARRAY['BASH']="shellcheck --color --external-sources --severity=${BASH_SEVERITY}"
-fi
-LINTER_COMMANDS_ARRAY['BASH_EXEC']="bash-exec"
-LINTER_COMMANDS_ARRAY['CHECKOV']="checkov --config-file ${CHECKOV_LINTER_RULES}"
-
-if CheckovConfigurationFileContainsDirectoryOption "${CHECKOV_LINTER_RULES}"; then
-  debug "No need to update the Checkov command."
-else
-  debug "Adding the '--directory' option to the Checkov command."
-  LINTER_COMMANDS_ARRAY['CHECKOV']="${LINTER_COMMANDS_ARRAY['CHECKOV']} --directory"
-fi
-
-LINTER_COMMANDS_ARRAY['CLANG_FORMAT']="clang-format --Werror --dry-run"
-LINTER_COMMANDS_ARRAY['CLOJURE']="clj-kondo --config ${CLOJURE_LINTER_RULES} --lint"
-LINTER_COMMANDS_ARRAY['CLOUDFORMATION']="cfn-lint --config-file ${CLOUDFORMATION_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['COFFEESCRIPT']="coffeelint -f ${COFFEESCRIPT_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['CPP']="cpplint"
-LINTER_COMMANDS_ARRAY['CSHARP']="dotnet format whitespace --folder --verify-no-changes --exclude / --include"
-LINTER_COMMANDS_ARRAY['CSS']="stylelint --config ${CSS_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['DART']="dart analyze --fatal-infos --fatal-warnings"
-LINTER_COMMANDS_ARRAY['DOCKERFILE_HADOLINT']="hadolint -c ${DOCKERFILE_HADOLINT_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['EDITORCONFIG']="editorconfig-checker -config ${EDITORCONFIG_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['ENV']="dotenv-linter"
-if [ "${GITHUB_ACTIONS_COMMAND_ARGS}" = "null" ]; then
-  LINTER_COMMANDS_ARRAY['GITHUB_ACTIONS']="actionlint -config-file ${GITHUB_ACTIONS_LINTER_RULES}"
-else
-  LINTER_COMMANDS_ARRAY['GITHUB_ACTIONS']="actionlint -config-file ${GITHUB_ACTIONS_LINTER_RULES} ${GITHUB_ACTIONS_COMMAND_ARGS}"
-fi
-LINTER_COMMANDS_ARRAY['GITLEAKS']="gitleaks detect --no-banner --no-git --redact --config ${GITLEAKS_LINTER_RULES} --verbose --source"
-LINTER_COMMANDS_ARRAY['GHERKIN']="gherkin-lint -c ${GHERKIN_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['GO_MODULES']="golangci-lint run -c ${GO_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['GO']="${LINTER_COMMANDS_ARRAY['GO_MODULES']} --fast"
-LINTER_COMMANDS_ARRAY['GOOGLE_JAVA_FORMAT']="java -jar /usr/bin/google-java-format --dry-run --set-exit-if-changed"
-LINTER_COMMANDS_ARRAY['GROOVY']="npm-groovy-lint -c ${GROOVY_LINTER_RULES} --failon warning --no-insight"
-LINTER_COMMANDS_ARRAY['HTML']="htmlhint --config ${HTML_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['JAVA']="java -jar /usr/bin/checkstyle -c ${JAVA_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['JAVASCRIPT_ES']="eslint --no-eslintrc -c ${JAVASCRIPT_ES_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['JAVASCRIPT_STANDARD']="standard ${JAVASCRIPT_STANDARD_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['JAVASCRIPT_PRETTIER']="prettier --check"
-LINTER_COMMANDS_ARRAY['JSCPD']="jscpd --config ${JSCPD_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['JSON']="eslint --no-eslintrc -c ${JAVASCRIPT_ES_LINTER_RULES} --ext .json"
-LINTER_COMMANDS_ARRAY['JSONC']="eslint --no-eslintrc -c ${JAVASCRIPT_ES_LINTER_RULES} --ext .json5,.jsonc"
-LINTER_COMMANDS_ARRAY['JSX']="eslint --no-eslintrc -c ${JSX_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['KOTLIN']="ktlint"
-if [ "${KUBERNETES_KUBECONFORM_OPTIONS}" == "null" ] || [ -z "${KUBERNETES_KUBECONFORM_OPTIONS}" ]; then
-  LINTER_COMMANDS_ARRAY['KUBERNETES_KUBECONFORM']="kubeconform -strict"
-else
-  LINTER_COMMANDS_ARRAY['KUBERNETES_KUBECONFORM']="kubeconform -strict ${KUBERNETES_KUBECONFORM_OPTIONS}"
-fi
-LINTER_COMMANDS_ARRAY['LATEX']="chktex -q -l ${LATEX_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['LUA']="luacheck --config ${LUA_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['MARKDOWN']="markdownlint -c ${MARKDOWN_LINTER_RULES}"
-if [ -n "${MARKDOWN_CUSTOM_RULE_GLOBS}" ]; then
-  IFS="," read -r -a MARKDOWN_CUSTOM_RULE_GLOBS_ARRAY <<<"${MARKDOWN_CUSTOM_RULE_GLOBS}"
-  for glob in "${MARKDOWN_CUSTOM_RULE_GLOBS_ARRAY[@]}"; do
-    if [ -z "${LINTER_RULES_PATH}" ]; then
-      LINTER_COMMANDS_ARRAY['MARKDOWN']="${LINTER_COMMANDS_ARRAY['MARKDOWN']} -r ${GITHUB_WORKSPACE}/${glob}"
-    else
-      LINTER_COMMANDS_ARRAY['MARKDOWN']="${LINTER_COMMANDS_ARRAY['MARKDOWN']} -r ${GITHUB_WORKSPACE}/${LINTER_RULES_PATH}/${glob}"
-    fi
-  done
-fi
-LINTER_COMMANDS_ARRAY['NATURAL_LANGUAGE']="textlint -c ${NATURAL_LANGUAGE_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['OPENAPI']="spectral lint -r ${OPENAPI_LINTER_RULES} -D"
-if [ "${PERL_PERLCRITIC_OPTIONS}" == "null" ] || [ -z "${PERL_PERLCRITIC_OPTIONS}" ]; then
-  LINTER_COMMANDS_ARRAY['PERL']="perlcritic"
-else
-  LINTER_COMMANDS_ARRAY['PERL']="perlcritic ${PERL_PERLCRITIC_OPTIONS}"
-fi
-LINTER_COMMANDS_ARRAY['PHP_BUILTIN']="php -l -c ${PHP_BUILTIN_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['PHP_PHPCS']="phpcs --standard=${PHP_PHPCS_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['PHP_PHPSTAN']="phpstan analyse --no-progress --no-ansi --memory-limit 1G -c ${PHP_PHPSTAN_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['PHP_PSALM']="psalm --config=${PHP_PSALM_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['POWERSHELL']="Invoke-ScriptAnalyzer -EnableExit -Settings ${POWERSHELL_LINTER_RULES} -Path"
-LINTER_COMMANDS_ARRAY['PROTOBUF']="protolint lint --config_path ${PROTOBUF_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['PYTHON_BLACK']="black --config ${PYTHON_BLACK_LINTER_RULES} --diff --check"
-LINTER_COMMANDS_ARRAY['PYTHON_PYLINT']="pylint --rcfile ${PYTHON_PYLINT_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['PYTHON_FLAKE8']="flake8 --config=${PYTHON_FLAKE8_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['PYTHON_ISORT']="isort --check --diff --sp ${PYTHON_ISORT_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['PYTHON_MYPY']="mypy --config-file ${PYTHON_MYPY_LINTER_RULES} --install-types --non-interactive"
-LINTER_COMMANDS_ARRAY['R']="lintr"
-LINTER_COMMANDS_ARRAY['RAKU']="raku"
-LINTER_COMMANDS_ARRAY['RENOVATE']="renovate-config-validator --strict"
-LINTER_COMMANDS_ARRAY['RUBY']="rubocop -c ${RUBY_LINTER_RULES} --force-exclusion --ignore-unrecognized-cops"
-LINTER_COMMANDS_ARRAY['RUST_2015']="rustfmt --check --edition 2015"
-LINTER_COMMANDS_ARRAY['RUST_2018']="rustfmt --check --edition 2018"
-LINTER_COMMANDS_ARRAY['RUST_2021']="rustfmt --check --edition 2021"
-LINTER_COMMANDS_ARRAY['RUST_CLIPPY']="clippy"
-LINTER_COMMANDS_ARRAY['SCALAFMT']="scalafmt --config ${SCALAFMT_LINTER_RULES} --test"
-LINTER_COMMANDS_ARRAY['SHELL_SHFMT']="shfmt -d"
-LINTER_COMMANDS_ARRAY['SNAKEMAKE_LINT']="snakemake --lint -s"
-LINTER_COMMANDS_ARRAY['SNAKEMAKE_SNAKEFMT']="snakefmt --config ${SNAKEMAKE_SNAKEFMT_LINTER_RULES} --check --compact-diff"
-LINTER_COMMANDS_ARRAY['STATES']="asl-validator --json-path"
-LINTER_COMMANDS_ARRAY['SQL']="sql-lint --config ${SQL_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['SQLFLUFF']="sqlfluff lint --config ${SQLFLUFF_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['TEKTON']="tekton-lint"
-LINTER_COMMANDS_ARRAY['TERRAFORM_FMT']="terraform fmt -check -diff"
-LINTER_COMMANDS_ARRAY['TERRAFORM_TFLINT']="tflint -c ${TERRAFORM_TFLINT_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['TERRAFORM_TERRASCAN']="terrascan scan -i terraform -t all -c ${TERRAFORM_TERRASCAN_LINTER_RULES} -f"
-LINTER_COMMANDS_ARRAY['TERRAGRUNT']="terragrunt hclfmt --terragrunt-check --terragrunt-log-level error --terragrunt-hclfmt-file"
-LINTER_COMMANDS_ARRAY['TSX']="eslint --no-eslintrc -c ${TSX_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['TYPESCRIPT_ES']="eslint --no-eslintrc -c ${TYPESCRIPT_ES_LINTER_RULES}"
-LINTER_COMMANDS_ARRAY['TYPESCRIPT_STANDARD']="ts-standard --parser @typescript-eslint/parser --plugin @typescript-eslint/eslint-plugin --project ${TYPESCRIPT_STANDARD_TSCONFIG_FILE}"
-LINTER_COMMANDS_ARRAY['TYPESCRIPT_PRETTIER']="prettier --check"
-LINTER_COMMANDS_ARRAY['XML']="xmllint"
-if [ "${YAML_ERROR_ON_WARNING}" == 'false' ]; then
-  LINTER_COMMANDS_ARRAY['YAML']="yamllint -c ${YAML_LINTER_RULES} -f parsable"
-else
-  LINTER_COMMANDS_ARRAY['YAML']="yamllint --strict -c ${YAML_LINTER_RULES} -f parsable"
-fi
-
-debug "--- Linter commands ---"
-debug "-----------------------"
-for i in "${!LINTER_COMMANDS_ARRAY[@]}"; do
-  debug "Linter key: $i, command: ${LINTER_COMMANDS_ARRAY[$i]}"
-done
-debug "---------------------------------------------"
-
 #################################
 # Check for SSL cert and update #
 #################################
@@ -1034,76 +890,73 @@ BuildFileList "${VALIDATE_ALL_CODEBASE}" "${TEST_CASE_RUN}"
 #####################################
 RunAdditionalInstalls
 
-###############
-# Run linters #
-###############
-EDITORCONFIG_FILE_PATH="${GITHUB_WORKSPACE}"/.editorconfig
-
 ####################################
 # Print ENV before running linters #
 ####################################
 debug "--- ENV (before running linters) ---"
 debug "------------------------------------"
 debug "ENV:"
-debug "$(printenv | sort)"
+debug "$(printenv)"
 debug "------------------------------------"
 
 endGitHubActionsLogGroup "${SUPER_LINTER_INITIALIZATION_LOG_GROUP_TITLE}"
 
-for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do
+###############
-  startGitHubActionsLogGroup "${LANGUAGE}"
+# Run linters #
-  debug "Running linter for the ${LANGUAGE} language..."
+###############
-  VALIDATE_LANGUAGE_VARIABLE_NAME="VALIDATE_${LANGUAGE}"
+declare PARALLEL_RESULTS_FILE_PATH
-  debug "Setting VALIDATE_LANGUAGE_VARIABLE_NAME to ${VALIDATE_LANGUAGE_VARIABLE_NAME}..."
+PARALLEL_RESULTS_FILE_PATH="/tmp/super-linter-results.json"
-  VALIDATE_LANGUAGE_VARIABLE_VALUE="${!VALIDATE_LANGUAGE_VARIABLE_NAME}"
+debug "PARALLEL_RESULTS_FILE_PATH: ${PARALLEL_RESULTS_FILE_PATH}"
-  debug "Setting VALIDATE_LANGUAGE_VARIABLE_VALUE to ${VALIDATE_LANGUAGE_VARIABLE_VALUE}..."
 
-  if [ "${VALIDATE_LANGUAGE_VARIABLE_VALUE}" = "true" ]; then
+declare -a PARALLEL_COMMAND
-    # Check if we need an .editorconfig file
+PARALLEL_COMMAND=(parallel --will-cite --keep-order --max-procs "$(($(nproc) * 1))" --xargs --results "${PARALLEL_RESULTS_FILE_PATH}")
-    # shellcheck disable=SC2153
-    if [ "${LANGUAGE}" = "EDITORCONFIG" ] || [ "${LANGUAGE}" = "SHELL_SHFMT" ]; then
-      if [ -e "${EDITORCONFIG_FILE_PATH}" ]; then
-        debug "Found an EditorConfig file at ${EDITORCONFIG_FILE_PATH}"
-      else
-        debug "No .editorconfig found at: $EDITORCONFIG_FILE_PATH. Skipping ${LANGUAGE} linting..."
-        continue
-      fi
-    elif [ "${LANGUAGE}" = "R" ] && [ ! -f "${R_RULES_FILE_PATH_IN_ROOT}" ] && ((${#FILE_ARRAY_R[@]})); then
-      info "No .lintr configuration file found, using defaults."
-      cp "$R_LINTER_RULES" "$GITHUB_WORKSPACE"
-      # shellcheck disable=SC2034
-      SUPER_LINTER_COPIED_R_LINTER_RULES_FILE="true"
-    # Check if there's local configuration for the Raku linter
-    elif [ "${LANGUAGE}" = "RAKU" ] && [ -e "${GITHUB_WORKSPACE}/META6.json" ]; then
-      cd "${GITHUB_WORKSPACE}" && zef install --deps-only --/test .
-    fi
 
-    LINTER_NAME="${LINTER_NAMES_ARRAY["${LANGUAGE}"]}"
+# Run one LANGUAGE per process. Each of these processes will run more processees in parellel if supported
-    if [ -z "${LINTER_NAME}" ]; then
+PARALLEL_COMMAND+=(--max-lines 1)
-      fatal "Cannot find the linter name for ${LANGUAGE} language."
-    else
-      debug "Setting LINTER_NAME to ${LINTER_NAME}..."
-    fi
 
-    LINTER_COMMAND="${LINTER_COMMANDS_ARRAY["${LANGUAGE}"]}"
+if [ "${LOG_DEBUG}" == "true" ]; then
-    if [ -z "${LINTER_COMMAND}" ]; then
+  debug "LOG_DEBUG is enabled. Enable verbose ouput for parallel"
-      fatal "Cannot find the linter command for ${LANGUAGE} language."
+  PARALLEL_COMMAND+=(--verbose)
-    else
+fi
-      debug "Setting LINTER_COMMAND to ${LINTER_COMMAND}..."
-    fi
 
-    FILE_ARRAY_VARIABLE_NAME="FILE_ARRAY_${LANGUAGE}"
+PARALLEL_COMMAND+=("LintCodebase" "{}" "\"${TEST_CASE_RUN}\"")
-    debug "Setting FILE_ARRAY_VARIABLE_NAME to ${FILE_ARRAY_VARIABLE_NAME}..."
+debug "PARALLEL_COMMAND: ${PARALLEL_COMMAND[*]}"
 
-    # shellcheck disable=SC2125
+PARALLEL_COMMAND_OUTPUT=$(printf "%s\n" "${LANGUAGE_ARRAY[@]}" | "${PARALLEL_COMMAND[@]}" 2>&1)
-    LANGUAGE_FILE_ARRAY="${FILE_ARRAY_VARIABLE_NAME}"[@]
+PARALLEL_COMMAND_RETURN_CODE=$?
-    debug "${FILE_ARRAY_VARIABLE_NAME} file array contents: ${!LANGUAGE_FILE_ARRAY}"
+debug "PARALLEL_COMMAND_OUTPUT when running linters (exit code: ${PARALLEL_COMMAND_RETURN_CODE}):\n${PARALLEL_COMMAND_OUTPUT}"
+debug "Parallel output file (${PARALLEL_RESULTS_FILE_PATH}) contents when running linters:\n$(cat "${PARALLEL_RESULTS_FILE_PATH}")"
 
-    debug "Invoking ${LINTER_NAME} linter. TEST_CASE_RUN: ${TEST_CASE_RUN}"
+RESULTS_OBJECT=
-    LintCodebase "${LANGUAGE}" "${LINTER_NAME}" "${LINTER_COMMAND}" "${FILTER_REGEX_INCLUDE}" "${FILTER_REGEX_EXCLUDE}" "${TEST_CASE_RUN}" "${!LANGUAGE_FILE_ARRAY}"
+if ! RESULTS_OBJECT=$(jq -n '[inputs]' "${PARALLEL_RESULTS_FILE_PATH}"); then
-  fi
+  fatal "Error loading results when building the file list: ${RESULTS_OBJECT}"
-  endGitHubActionsLogGroup "${LANGUAGE}"
+fi
-done
+debug "RESULTS_OBJECT when running linters:\n${RESULTS_OBJECT}"
+
+# Get raw output so we can strip quotes from the data we load
+if ! STDOUT_LINTERS="$(jq --raw-output '.[].Stdout' <<<"${RESULTS_OBJECT}")"; then
+  fatal "Error when loading stdout when running linters:\n${STDOUT_LINTERS}"
+fi
+
+if [ -n "${STDOUT_LINTERS}" ]; then
+  info "Command output when running linters:\n------\n${STDOUT_LINTERS}\n------"
+else
+  debug "Stdout when running linters is empty"
+fi
+
+if ! STDERR_LINTERS="$(jq --raw-output '.[].Stderr' <<<"${RESULTS_OBJECT}")"; then
+  fatal "Error when loading stderr for ${FILE_TYPE}:\n${STDERR_LINTERS}"
+fi
+
+if [ -n "${STDERR_LINTERS}" ]; then
+  info "Command output for ${FILE_TYPE}:\n------\n${STDERR_LINTERS}\n------"
+else
+  debug "Stderr when running linters is empty"
+fi
+
+if [[ ${PARALLEL_COMMAND_RETURN_CODE} -ne 0 ]]; then
+  fatal "Error when running linters. Exit code: ${PARALLEL_COMMAND_RETURN_CODE}"
+fi
 
 ##########
 # Footer #

test/inspec/super-linter/controls/super_linter.rb

@@ -436,6 +436,7 @@ control "super-linter-validate-files" do
     "/action/lib/functions/buildFileList.sh",
     "/action/lib/functions/detectFiles.sh",
     "/action/lib/functions/githubEvent.sh",
+    "/action/lib/functions/linterCommands.sh",
     "/action/lib/functions/linterRules.sh",
     "/action/lib/functions/linterVersions.sh",
     "/action/lib/functions/linterVersions.txt",

test/run-super-linter-tests.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+SUPER_LINTER_TEST_CONTAINER_URL="${1}"
+TEST_FUNCTION_NAME="${2}"
+
+COMMAND_TO_RUN=(docker run -e ACTIONS_RUNNER_DEBUG=true -e DEFAULT_BRANCH=main -e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true -e JSCPD_CONFIG_FILE=".jscpd-test-linters.json" -e RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES="default.json,hoge.json" -e RUN_LOCAL=true -e TEST_CASE_RUN=true -e TYPESCRIPT_STANDARD_TSCONFIG_FILE=".github/linters/tsconfig.json" -v "$(pwd):/tmp/lint")
+
+run_test_cases_expect_failure() {
+  COMMAND_TO_RUN+=(-e ANSIBLE_DIRECTORY="/test/linters/ansible/bad" -e CHECKOV_FILE_NAME=".checkov-test-linters-failure.yaml" -e FILTER_REGEX_INCLUDE=".*bad.*")
+  EXPECTED_EXIT_CODE=1
+}
+
+run_test_cases_expect_success() {
+  COMMAND_TO_RUN+=(-e ANSIBLE_DIRECTORY="/test/linters/ansible/good" -e CHECKOV_FILE_NAME=".checkov-test-linters-success.yaml" -e FILTER_REGEX_INCLUDE=".*good.*")
+}
+
+# Run the test setup function
+${TEST_FUNCTION_NAME}
+
+COMMAND_TO_RUN+=("${SUPER_LINTER_TEST_CONTAINER_URL}")
+
+declare -i EXPECTED_EXIT_CODE
+EXPECTED_EXIT_CODE=${EXPECTED_EXIT_CODE:-0}
+
+if [ ${EXPECTED_EXIT_CODE} -ne 0 ]; then
+  echo "Disable failures on error because the expected exit code is ${EXPECTED_EXIT_CODE}"
+  set +o errexit
+fi
+
+echo "Command to run: ${COMMAND_TO_RUN[*]}"
+
+"${COMMAND_TO_RUN[@]}"
+SUPER_LINTER_EXIT_CODE=$?
+# Enable the errexit option in case we disabled it
+set -o errexit
+
+echo "Super-linter exit code: ${SUPER_LINTER_EXIT_CODE}"
+
+if [ ${SUPER_LINTER_EXIT_CODE} -ne ${EXPECTED_EXIT_CODE} ]; then
+  echo "Super-linter exited with an unexpected code: ${SUPER_LINTER_EXIT_CODE}"
+  exit 1
+else
+  echo "Super-linter exited with the expected code: ${SUPER_LINTER_EXIT_CODE}"
+fi