Ignore files that are ignored by Git (#1185)

* Ignore files that are ignored by Git

* Add missing continue statement

* Fix linting errors

* Fix linting errors

* like it

* like it

* Add the IGNORE_GITIGNORED_FILES variable

* Fix setting the Ansible directory when running tests

* Fix var export

* Update linter.sh

Co-authored-by: Admiral Awkbar <admiralawkbar@github.com>

README.md

@@ -217,6 +217,7 @@ But if you wish to select or exclude specific linters, we give you full control
 | **ERROR_ON_MISSING_EXEC_BIT**      | `false`               | If set to `false`, the `bash-exec` linter will report a warning if a shell script is not executable. If set to `true`, the `bash-exec` linter will report an error instead.      |
 | **FILTER_REGEX_EXCLUDE**           | `none`                | Regular expression defining which files will be excluded from linting  (ex: `.*src/test.*`)                                                                                      |
 | **FILTER_REGEX_INCLUDE**           | `all`                 | Regular expression defining which files will be processed by linters (ex: `.*src/.*`)                                                                                            |
+| **IGNORE_GITIGNORED_FILES**        | `false`               | If set to `true`, super-linter will ignore all the files that are ignored by Git.                                                                                                |
 | **JAVASCRIPT_ES_CONFIG_FILE**      | `.eslintrc.yml`       | Filename for [eslint configuration](https://eslint.org/docs/user-guide/configuring#configuration-file-formats) (ex: `.eslintrc.yml`, `.eslintrc.json`)                           |
 | **JAVASCRIPT_DEFAULT_STYLE**       | `standard`            | Flag to set the default style of javascript. Available options: **standard**/**prettier**                                                                                        |
 | **JSCPD_CONFIG_FILE**              | `.jscpd.json`         | Filename for JSCPD configuration                                                                                                                                                 |
@@ -233,7 +234,7 @@ But if you wish to select or exclude specific linters, we give you full control
 | **PYTHON_ISORT_CONFIG_FILE**       | `.isort.cfg`          | Filename for [isort configuration](https://pycqa.github.io/isort/docs/configuration/config_files/) (ex: `.isort.cfg`, `pyproject.toml`)                                          |
 | **PYTHON_PYLINT_CONFIG_FILE**      | `.python-lint`        | Filename for [pylint configuration](https://pylint.pycqa.org/en/latest/user_guide/run.html?highlight=rcfile#command-line-options) (ex: `.python-lint`, `.pylintrc`)              |
 | **RUBY_CONFIG_FILE**               | `.ruby-lint.yml`      | Filename for [rubocop configuration](https://docs.rubocop.org/rubocop/configuration.html) (ex: `.ruby-lint.yml`, `.rubocop.yml`)                                                 |
-| **SUPPRESS_POSSUM**                | `false`               | If set to `true`, will hide the ASCII possum at top of log output. Default is `false`
+| **SUPPRESS_POSSUM**                | `false`               | If set to `true`, will hide the ASCII possum at top of log output. Default is `false`                                                                                            |
 | **SNAKEMAKE_SNAKEFMT_CONFIG_FILE** | `.snakefmt.toml`      | Filename for [Snakemake configuration](https://github.com/snakemake/snakefmt#configuration) (ex: `pyproject.toml`, `.snakefmt.toml`)                                             |
 | **TYPESCRIPT_ES_CONFIG_FILE**      | `.eslintrc.yml`       | Filename for [eslint configuration](https://eslint.org/docs/user-guide/configuring#configuration-file-formats) (ex: `.eslintrc.yml`, `.eslintrc.json`)                           |
 | **VALIDATE_ALL_CODEBASE**          | `true`                | Will parse the entire repository and find all files to validate across all types. **NOTE:** When set to `false`, only **new** or **edited** files will be parsed for validation. |

lib/functions/buildFileList.sh

@@ -24,6 +24,8 @@ function BuildFileList() {
   ANSIBLE_DIRECTORY="${3}"
   debug "ANSIBLE_DIRECTORY: ${ANSIBLE_DIRECTORY}..."
 
+  debug "IGNORE_GITIGNORED_FILES: ${IGNORE_GITIGNORED_FILES}..."
+
   if [ "${VALIDATE_ALL_CODEBASE}" == "false" ] && [ "${TEST_CASE_RUN}" != "true" ]; then
     # Need to build a list of all files changed
     # This can be pulled from the GITHUB_EVENT_PATH payload
@@ -182,6 +184,26 @@ function BuildFileList() {
     fi
   fi
 
+  ###############################
+  # Load the ignored files list #
+  ###############################
+  debug "Loading the files list that Git ignores..."
+  mapfile -t GIT_IGNORED_FILES < <(git -C "${GITHUB_WORKSPACE}" status --ignored --porcelain=v1 --short --untracked-files=normal | grep '!!' | awk -F ' ' '{print $2}' | sed -e 's#^#'"${GITHUB_WORKSPACE}"/'#' | sort)
+  debug "GIT_IGNORED_FILES contents: ${GIT_IGNORED_FILES[*]}"
+
+  # Build an associative array to avoid looping throug the ignored files list
+  local i
+  declare -g -A GIT_IGNORED_FILES_INDEX
+  for i in "${!GIT_IGNORED_FILES[@]}"; do
+    eval GIT_IGNORED_FILES_INDEX["${GIT_IGNORED_FILES[$i]}"]="$i"
+  done
+  debug "--- GIT_IGNORED_FILES_INDEX contents ---"
+  debug "-----------------------"
+  for i in "${!GIT_IGNORED_FILES_INDEX[@]}"; do
+    debug "key: $i, value: ${GIT_IGNORED_FILES_INDEX[$i]}"
+  done
+  debug "---------------------------------------------"
+
   ################################################
   # Iterate through the array of all files found #
   ################################################
@@ -237,6 +259,11 @@ function BuildFileList() {
       continue
     fi
 
+    if [ "${GIT_IGNORED_FILES_INDEX[$FILE]}" ] && [ "${IGNORE_GITIGNORED_FILES}" == "true" ]; then
+      debug "${FILE} is ignored by Git. Skipping ${FILE}"
+      continue
+    fi
+
     # Editorconfig-checker should check every file
     FILE_ARRAY_EDITORCONFIG+=("${FILE}")
     # jscpd also runs an all files

lib/functions/validation.sh

@@ -129,10 +129,17 @@ function GetValidationInfo() {
   ##############################
   # Validate Ansible Directory #
   ##############################
+  # No Value, need to default
   if [ -z "${ANSIBLE_DIRECTORY}" ]; then
-    # No Value, need to default
+
-    ANSIBLE_DIRECTORY="${DEFAULT_ANSIBLE_DIRECTORY}"
+    if [ "${TEST_CASE_RUN}" != "true" ]; then
-    debug "Setting Ansible directory to the default: ${DEFAULT_ANSIBLE_DIRECTORY}"
+      ANSIBLE_DIRECTORY="${DEFAULT_ANSIBLE_DIRECTORY}"
+      debug "Setting Ansible directory to the default: ${DEFAULT_ANSIBLE_DIRECTORY}"
+    else
+      ANSIBLE_DIRECTORY="${DEFAULT_TEST_CASE_ANSIBLE_DIRECTORY}"
+      debug "Setting Ansible directory to the default for test cases: ${DEFAULT_TEST_CASE_ANSIBLE_DIRECTORY}"
+    fi
+    debug "Setting Ansible directory to: ${ANSIBLE_DIRECTORY}"
   else
     # Check if first char is '/'
     if [[ ${ANSIBLE_DIRECTORY:0:1} == "/" ]]; then

lib/linter.sh

@@ -273,6 +273,8 @@ MULTI_STATUS="${MULTI_STATUS:-true}"             # Multiple status are created f
 TEST_CASE_RUN="${TEST_CASE_RUN}"                 # Boolean to validate only test cases
 VALIDATE_ALL_CODEBASE="${VALIDATE_ALL_CODEBASE}" # Boolean to validate all files
 
+IGNORE_GITIGNORED_FILES="${IGNORE_GITIGNORED_FILES:-false}"
+
 ################
 # Default Vars #
 ################
@@ -728,8 +730,11 @@ GetGitHubVars
 ########################################################
 # Initialize variables that depend on GitHub variables #
 ########################################################
-DEFAULT_ANSIBLE_DIRECTORY="${GITHUB_WORKSPACE}/ansible"     # Default Ansible Directory
+DEFAULT_ANSIBLE_DIRECTORY="${GITHUB_WORKSPACE}/ansible"                               # Default Ansible Directory
-export DEFAULT_ANSIBLE_DIRECTORY                            # Workaround SC2034
+export DEFAULT_ANSIBLE_DIRECTORY                                                      # Workaround SC2034
+DEFAULT_TEST_CASE_ANSIBLE_DIRECTORY="${GITHUB_WORKSPACE}/${TEST_CASE_FOLDER}/ansible" # Default Ansible directory when running test cases
+export DEFAULT_TEST_CASE_ANSIBLE_DIRECTORY                                            # Workaround SC2034
+
 REPORT_OUTPUT_FOLDER="${GITHUB_WORKSPACE}/${OUTPUT_FOLDER}" # Location for the report folder
 
 ############################