From 4faa6433abb21e99db0ea79f2ce3c3442b806a1d Mon Sep 17 00:00:00 2001 From: Lukas Gravley Date: Tue, 8 Jun 2021 10:40:59 -0500 Subject: [PATCH] adding find algorithm (#1640) * adding find * add default value --- README.md | 1 + lib/functions/buildFileList.sh | 52 ++++++++++++++++++++++++++++------ lib/linter.sh | 2 ++ 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c38e10ad..59495b17 100644 --- a/README.md +++ b/README.md @@ -300,6 +300,7 @@ But if you wish to select or exclude specific linters, we give you full control | **SSL_CERT_SECRET** | `none` | SSL cert to add to the **Super-Linter** trust store. This is needed for users on `self-hosted` runners or need to inject the cert for security standards (ex. ${{ secrets.SSL_CERT }}) | | **SQL_CONFIG_FILE** | `.sql-config.json` | Filename for [SQL-Lint configuration](https://sql-lint.readthedocs.io/en/latest/files/configuration.html) (ex: `sql-config.json` , `.config.json`) | | **TYPESCRIPT_ES_CONFIG_FILE** | `.eslintrc.yml` | Filename for [eslint configuration](https://eslint.org/docs/user-guide/configuring#configuration-file-formats) (ex: `.eslintrc.yml`, `.eslintrc.json`) | +| **USE_FIND_ALGORITHM** | `false` | By default, we use `git diff` to find all files in the workspace and what has been updated, this would enable the Linux `find` method instead to find all files to lint | | **VALIDATE_ALL_CODEBASE** | `true` | Will parse the entire repository and find all files to validate across all types. **NOTE:** When set to `false`, only **new** or **edited** files will be parsed for validation. | | **VALIDATE_ANSIBLE** | `true` | Flag to enable or disable the linting process of the Ansible language. | | **VALIDATE_ARM** | `true` | Flag to enable or disable the linting process of the ARM language. | diff --git a/lib/functions/buildFileList.sh b/lib/functions/buildFileList.sh index e518ee88..68b59e63 100755 --- a/lib/functions/buildFileList.sh +++ b/lib/functions/buildFileList.sh @@ -64,6 +64,8 @@ function BuildFileList() { debug "IGNORE_GITIGNORED_FILES: ${IGNORE_GITIGNORED_FILES}..." + debug "USE_FIND_ALGORITHM: ${USE_FIND_ALGORITHM}..." + if [ "${VALIDATE_ALL_CODEBASE}" == "false" ] && [ "${TEST_CASE_RUN}" != "true" ]; then # Need to build a list of all files changed # This can be pulled from the GITHUB_EVENT_PATH payload @@ -116,7 +118,6 @@ function BuildFileList() { DIFF_CMD="git -C ${GITHUB_WORKSPACE} diff --name-only ${DEFAULT_BRANCH}...${GITHUB_SHA} --diff-filter=d | xargs -I % sh -c 'echo \"${GITHUB_WORKSPACE}/%\"' 2>&1" GenerateFileDiff "$DIFF_CMD" - fi else ################ @@ -131,13 +132,48 @@ function BuildFileList() { WORKSPACE_PATH="${GITHUB_WORKSPACE}/${TEST_CASE_FOLDER}" fi - ################ - # print header # - ################ - debug "----------------------------------------------" - debug "Populating the file list with:[git -C \"${WORKSPACE_PATH}\" ls-tree -r --name-only HEAD | xargs -I % sh -c \"echo ${WORKSPACE_PATH}/%\"]" - mapfile -t RAW_FILE_ARRAY < <(git -C "${WORKSPACE_PATH}" ls-tree -r --name-only HEAD | xargs -I % sh -c "echo ${WORKSPACE_PATH}/%" 2>&1) - debug "RAW_FILE_ARRAY contents: ${RAW_FILE_ARRAY[*]}" + ############################# + # Use the find on all files # + ############################# + if [ "${USE_FIND_ALGORITHM}" == 'true' ]; then + ################ + # print header # + ################ + debug "----------------------------------------------" + debug "Populating the file list with all the files in the ${WORKSPACE_PATH} workspace using FIND algorithm" + mapfile -t RAW_FILE_ARRAY < <(find "${WORKSPACE_PATH}" \ + -not \( -path '*/\.git' -prune \) \ + -not \( -path '*/\.pytest_cache' -prune \) \ + -not \( -path '*/\.rbenv' -prune \) \ + -not \( -path '*/\.terragrunt-cache' -prune \) \ + -not \( -path '*/\.venv' -prune \) \ + -not \( -path '*/\__pycache__' -prune \) \ + -not \( -path '*/\node_modules' -prune \) \ + -not -name ".DS_Store" \ + -not -name "*.gif" \ + -not -name "*.ico" \ + -not -name "*.jpg" \ + -not -name "*.jpeg" \ + -not -name "*.pdf" \ + -not -name "*.png" \ + -not -name "*.webp" \ + -not -name "*.woff" \ + -not -name "*.woff2" \ + -not -name "*.zip" \ + -type f \ + 2>&1 | sort) + else + ############################## + # use the standard mechinism # + ############################## + ################ + # print header # + ################ + debug "----------------------------------------------" + debug "Populating the file list with:[git -C \"${WORKSPACE_PATH}\" ls-tree -r --name-only HEAD | xargs -I % sh -c \"echo ${WORKSPACE_PATH}/%\"]" + mapfile -t RAW_FILE_ARRAY < <(git -C "${WORKSPACE_PATH}" ls-tree -r --name-only HEAD | xargs -I % sh -c "echo ${WORKSPACE_PATH}/%" 2>&1) + debug "RAW_FILE_ARRAY contents: ${RAW_FILE_ARRAY[*]}" + fi fi ####################### diff --git a/lib/linter.sh b/lib/linter.sh index 675207ab..b11ab06d 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -164,6 +164,8 @@ TYPESCRIPT_ES_FILE_NAME="${TYPESCRIPT_ES_CONFIG_FILE:-.eslintrc.yml}" # shellcheck disable=SC2034 # Variable is referenced indirectly TYPESCRIPT_STANDARD_FILE_NAME="${TYPESCRIPT_ES_CONFIG_FILE:-.eslintrc.yml}" # shellcheck disable=SC2034 # Variable is referenced indirectly +USE_FIND_ALGORITHM="${USE_FIND_ALGORITHM:-false}" +# shellcheck disable=SC2034 # Variable is referenced indirectly YAML_FILE_NAME="${YAML_CONFIG_FILE:-.yaml-lint.yml}" #################################################