superlint/lib/functions/buildFileList.sh

864 lines
31 KiB
Bash
Raw Normal View History

2020-06-29 10:55:59 -04:00
#!/usr/bin/env bash
################################################################################
################################################################################
########### Super-Linter Build File List Functions @admiralawkbar ##############
################################################################################
################################################################################
########################## FUNCTION CALLS BELOW ################################
################################################################################
################################################################################
#### Function GenerateFileDiff #################################################
function GenerateFileDiff() {
CMD="${1}"
################
# print header #
################
debug "----------------------------------------------"
debug "Generating Diff with:[$CMD]"
#################################################
# Get the Array of files changed in the commits #
#################################################
CMD_OUTPUT=$(eval "$CMD")
#######################
# Load the error code #
#######################
ERROR_CODE=$?
##############################
# Check the shell for errors #
##############################
if [ ${ERROR_CODE} -ne 0 ]; then
# Error
info "Failed to get Diff with:[$CMD]"
info "Check that you have the full git history, the checkout is not shallow, etc"
info "See https://github.com/github/super-linter#example-connecting-github-action-workflow"
fatal "[${CMD_OUTPUT}]"
fi
###################################################
# Map command output to an array to proper handle #
###################################################
mapfile -t RAW_FILE_ARRAY < <(echo -n "$CMD_OUTPUT")
debug "RAW_FILE_ARRAY contents: ${RAW_FILE_ARRAY[*]}"
}
2020-06-29 10:55:59 -04:00
################################################################################
#### Function BuildFileList ####################################################
2020-07-01 17:40:40 -04:00
function BuildFileList() {
debug "Building file list..."
2020-06-29 10:55:59 -04:00
################
# Pull in vars #
2020-06-29 10:55:59 -04:00
################
VALIDATE_ALL_CODEBASE="${1}"
debug "VALIDATE_ALL_CODEBASE: ${VALIDATE_ALL_CODEBASE}"
TEST_CASE_RUN="${2}"
debug "TEST_CASE_RUN: ${TEST_CASE_RUN}"
ANSIBLE_DIRECTORY="${3}"
debug "ANSIBLE_DIRECTORY: ${ANSIBLE_DIRECTORY}"
debug "IGNORE_GITIGNORED_FILES: ${IGNORE_GITIGNORED_FILES}"
debug "IGNORE_GENERATED_FILES: ${IGNORE_GENERATED_FILES}"
Ignore files marked with @generated marker (#1689) * Ignore files marked with @generated marker `@generated` marker is used by certain tools to understand that the file is generated, so it should be treated differently than a file written by a human: * these files do not need to be reformatted, * diffs in these files are less important, * and linters should not be invoked on these files. This PR proposes builtin support for `@generated` marker (and `@not-generated` marker to mark file as not generated when it contains `@generated` marker, like `README.md`). I have not found a standard for a generated file marker, but: * Facebook [uses `@generated` marker](https://tinyurl.com/fb-generated) * Phabricator tool which was spawned from Facebook internal tool [also understands `@generated` marker](https://git.io/JnVHa) * Cargo inserts `@generated` marker into [generated Cargo.lock files](https://git.io/JnVHP) Super-linter supports regex includes and excludes, but they are harder to maintain (each repository needs to be configured) than patching the tools which generate the files. My personal story is that I maintain rust-protobuf crate, which started emitting `@generated` markers [six years ago](https://git.io/JnV5h) after a request of a Phabricator user. Test Plan: Create a test file `test.sh`: ``` echo $a ``` Run: ``` docker run -e RUN_LOCAL=true -v $HOME/tmp/g:/tmp/lint super-linter-test ``` Result is: ``` In /tmp/lint/test.sh line 1: echo $a ^-- SC2148: Tips depend on target shell and yours is unknown. Add a shebang or a 'shell' directive. ^-- SC2154: a is referenced but not assigned. ^-- SC2086: Double quote to prevent globbing and word splitting. ... 2021-06-22 23:46:16 [ERROR] ERRORS FOUND in BASH:[1] ``` Now add `@generated` to the file and run again: ``` 2021-06-22 23:47:13 [NOTICE] All file(s) linted successfully with no errors detected ``` Additionally, add `@not-generated` in addition to `@generated`, and linter error pops up again. * cleanup * remove space * fix non utf return * fix non utf return Co-authored-by: Lukas Gravley <admiralawkbar@github.com>
2021-06-28 08:59:11 -04:00
debug "USE_FIND_ALGORITHM: ${USE_FIND_ALGORITHM}"
debug "VALIDATE_JSCPD_ALL_CODEBASE: ${VALIDATE_JSCPD_ALL_CODEBASE}"
if [ "${VALIDATE_ALL_CODEBASE}" == "false" ] && [ "${TEST_CASE_RUN}" != "true" ]; then
# Need to build a list of all files changed
# This can be pulled from the GITHUB_EVENT_PATH payload
################
# print header #
################
debug "----------------------------------------------"
debug "Pulling in code history and branches..."
#################################################################################
# Switch codebase back to the default branch to get a list of all files changed #
#################################################################################
SWITCH_CMD=$(
git -C "${GITHUB_WORKSPACE}" pull --quiet
git -C "${GITHUB_WORKSPACE}" checkout "${DEFAULT_BRANCH}" 2>&1
)
#######################
# Load the error code #
#######################
ERROR_CODE=$?
##############################
# Check the shell for errors #
##############################
if [ ${ERROR_CODE} -ne 0 ]; then
# Error
info "Failed to switch to ${DEFAULT_BRANCH} branch to get files changed!"
fatal "[${SWITCH_CMD}]"
fi
2020-10-23 12:16:14 -04:00
if [ "${GITHUB_EVENT_NAME}" == "push" ]; then
################
# push event #
################
DIFF_TREE_CMD="git -C ${GITHUB_WORKSPACE} diff-tree --no-commit-id --name-only -r ${GITHUB_SHA} | xargs -I % sh -c 'echo \"${GITHUB_WORKSPACE}/%\"' 2>&1"
GenerateFileDiff "$DIFF_TREE_CMD"
###############################################################
# Need to see if the array is empty, if so, try the other way #
###############################################################
if [ ${#RAW_FILE_ARRAY[@]} -eq 0 ]; then
# Empty array, going to try to pull from main branch differences
################
# print header #
################
debug "----------------------------------------------"
debug "WARN: Generation of File array with diff-tree produced [0] items, trying with git diff..."
DIFF_CMD="git -C ${GITHUB_WORKSPACE} diff --name-only ${DEFAULT_BRANCH}...${GITHUB_SHA} --diff-filter=d | xargs -I % sh -c 'echo \"${GITHUB_WORKSPACE}/%\"' 2>&1"
GenerateFileDiff "$DIFF_CMD"
fi
2020-10-23 12:16:14 -04:00
else
################
# PR event #
################
DIFF_CMD="git -C ${GITHUB_WORKSPACE} diff --name-only ${DEFAULT_BRANCH}...${GITHUB_SHA} --diff-filter=d | xargs -I % sh -c 'echo \"${GITHUB_WORKSPACE}/%\"' 2>&1"
GenerateFileDiff "$DIFF_CMD"
2020-10-23 12:16:14 -04:00
fi
else
WORKSPACE_PATH="${GITHUB_WORKSPACE}"
if [ "${TEST_CASE_RUN}" == "true" ]; then
2020-11-06 17:10:09 -05:00
WORKSPACE_PATH="${GITHUB_WORKSPACE}/${TEST_CASE_FOLDER}"
fi
#############################
# Use the find on all files #
#############################
if [ "${USE_FIND_ALGORITHM}" == 'true' ]; then
################
# print header #
################
debug "----------------------------------------------"
debug "Populating the file list with all the files in the ${WORKSPACE_PATH} workspace using FIND algorithm"
mapfile -t RAW_FILE_ARRAY < <(find "${WORKSPACE_PATH}" \
-not \( -path '*/\.git' -prune \) \
-not \( -path '*/\.pytest_cache' -prune \) \
-not \( -path '*/\.rbenv' -prune \) \
-not \( -path '*/\.terragrunt-cache' -prune \) \
-not \( -path '*/\.venv' -prune \) \
-not \( -path '*/\__pycache__' -prune \) \
-not \( -path '*/\node_modules' -prune \) \
-not -name ".DS_Store" \
-not -name "*.gif" \
-not -name "*.ico" \
-not -name "*.jpg" \
-not -name "*.jpeg" \
-not -name "*.pdf" \
-not -name "*.png" \
-not -name "*.webp" \
-not -name "*.woff" \
-not -name "*.woff2" \
-not -name "*.zip" \
-type f \
2>&1 | sort)
else
##############################
# use the standard mechinism #
##############################
################
# print header #
################
debug "----------------------------------------------"
debug "Populating the file list with:[git -C \"${WORKSPACE_PATH}\" ls-tree -r --name-only HEAD | xargs -I % sh -c \"echo ${WORKSPACE_PATH}/%\"]"
mapfile -t RAW_FILE_ARRAY < <(git -C "${WORKSPACE_PATH}" ls-tree -r --name-only HEAD | xargs -I % sh -c "echo ${WORKSPACE_PATH}/%" 2>&1)
debug "RAW_FILE_ARRAY contents: ${RAW_FILE_ARRAY[*]}"
fi
2020-06-29 10:55:59 -04:00
fi
#######################
# Load the error code #
#######################
ERROR_CODE=$?
##############################
# Check the shell for errors #
##############################
2020-07-21 13:09:07 -04:00
if [ ${ERROR_CODE} -ne 0 ]; then
2020-11-05 10:10:49 -05:00
fatal "Failed to gain a list of all files changed! Error code: ${ERROR_CODE}"
2020-06-29 10:55:59 -04:00
fi
2020-10-14 11:31:55 -04:00
##########################################################################
# Check to make sure the raw file array is not empty or throw a warning! #
##########################################################################
if [ ${#RAW_FILE_ARRAY[@]} -eq 0 ]; then
###############################
# No files were found to lint #
###############################
warn "No files were found in the GITHUB_WORKSPACE:[${GITHUB_WORKSPACE}] to lint!"
fi
if [ "${VALIDATE_ALL_CODEBASE}" == "false" ]; then
#########################################
# Need to switch back to branch of code #
#########################################
SWITCH2_CMD=$(git -C "${GITHUB_WORKSPACE}" checkout --progress --force "${GITHUB_SHA}" 2>&1)
#######################
# Load the error code #
#######################
ERROR_CODE=$?
##############################
# Check the shell for errors #
##############################
if [ ${ERROR_CODE} -ne 0 ]; then
# Error
error "Failed to switch back to branch!"
fatal "[${SWITCH2_CMD}]"
fi
fi
###############################
# Load the ignored files list #
###############################
debug "Loading the files list that Git ignores..."
mapfile -t GIT_IGNORED_FILES < <(git -C "${GITHUB_WORKSPACE}" status --ignored --porcelain=v1 --short --untracked-files=normal | grep '!!' | awk -F ' ' '{print $2}' | sed -e 's#^#'"${GITHUB_WORKSPACE}"/'#' | sort)
debug "GIT_IGNORED_FILES contents: ${GIT_IGNORED_FILES[*]}"
# Build an associative array to avoid looping throug the ignored files list
local i
declare -g -A GIT_IGNORED_FILES_INDEX
for i in "${!GIT_IGNORED_FILES[@]}"; do
eval GIT_IGNORED_FILES_INDEX["${GIT_IGNORED_FILES[$i]}"]="$i"
done
debug "--- GIT_IGNORED_FILES_INDEX contents ---"
debug "-----------------------"
for i in "${!GIT_IGNORED_FILES_INDEX[@]}"; do
debug "key: $i, value: ${GIT_IGNORED_FILES_INDEX[$i]}"
done
debug "---------------------------------------------"
#########################################
# Check if the Ansible directory exists #
#########################################
if [ -d "${ANSIBLE_DIRECTORY}" ]; then
debug "Adding ANSIBLE_DIRECTORY (${ANSIBLE_DIRECTORY}) to the list of files and directories to lint."
FILE_ARRAY_ANSIBLE+=("${ANSIBLE_DIRECTORY}")
else
debug "ANSIBLE_DIRECTORY (${ANSIBLE_DIRECTORY}) does NOT exist."
fi
2020-06-29 10:55:59 -04:00
################################################
# Iterate through the array of all files found #
################################################
info "---------------------------------"
info "------ File list to check: ------"
info "---------------------------------"
2020-07-01 17:40:40 -04:00
for FILE in "${RAW_FILE_ARRAY[@]}"; do
2020-07-22 13:27:45 -04:00
# Extract just the file extension
2020-08-31 17:44:35 -04:00
FILE_TYPE="$(GetFileExtension "$FILE")"
# get the baseFile for additonal logic, lowercase
2020-08-13 13:40:21 -04:00
BASE_FILE=$(basename "${FILE,,}")
2020-06-29 10:55:59 -04:00
2020-07-06 09:29:36 -04:00
##############
# Print file #
##############
debug "File:[${FILE}], File_type:[${FILE_TYPE}], Base_file:[${BASE_FILE}]"
2020-06-29 10:55:59 -04:00
2020-11-10 11:02:03 -05:00
##########################################################
# Check if the file exists on the filesystem, or skip it #
##########################################################
if [ ! -f "${FILE}" ]; then
# File not found in workspace
warn "File:{$FILE} existed in commit data, but not found on file system, skipping..."
2020-11-10 11:02:03 -05:00
continue
fi
########################################################
# Don't include test cases if not running in test mode #
########################################################
if [[ ${FILE} == *"${TEST_CASE_FOLDER}"* ]] && [ "${TEST_CASE_RUN}" != "true" ]; then
debug "TEST_CASE_RUN (${TEST_CASE_RUN}) is not true. Skipping ${FILE}..."
continue
##################################################
# Include test cases if not running in test mode #
##################################################
elif [[ ${FILE} != *"${TEST_CASE_FOLDER}"* ]] && [ "${TEST_CASE_RUN}" == "true" ]; then
debug "TEST_CASE_RUN (${TEST_CASE_RUN}) is true. Skipping ${FILE}..."
fi
Ignore files marked with @generated marker (#1689) * Ignore files marked with @generated marker `@generated` marker is used by certain tools to understand that the file is generated, so it should be treated differently than a file written by a human: * these files do not need to be reformatted, * diffs in these files are less important, * and linters should not be invoked on these files. This PR proposes builtin support for `@generated` marker (and `@not-generated` marker to mark file as not generated when it contains `@generated` marker, like `README.md`). I have not found a standard for a generated file marker, but: * Facebook [uses `@generated` marker](https://tinyurl.com/fb-generated) * Phabricator tool which was spawned from Facebook internal tool [also understands `@generated` marker](https://git.io/JnVHa) * Cargo inserts `@generated` marker into [generated Cargo.lock files](https://git.io/JnVHP) Super-linter supports regex includes and excludes, but they are harder to maintain (each repository needs to be configured) than patching the tools which generate the files. My personal story is that I maintain rust-protobuf crate, which started emitting `@generated` markers [six years ago](https://git.io/JnV5h) after a request of a Phabricator user. Test Plan: Create a test file `test.sh`: ``` echo $a ``` Run: ``` docker run -e RUN_LOCAL=true -v $HOME/tmp/g:/tmp/lint super-linter-test ``` Result is: ``` In /tmp/lint/test.sh line 1: echo $a ^-- SC2148: Tips depend on target shell and yours is unknown. Add a shebang or a 'shell' directive. ^-- SC2154: a is referenced but not assigned. ^-- SC2086: Double quote to prevent globbing and word splitting. ... 2021-06-22 23:46:16 [ERROR] ERRORS FOUND in BASH:[1] ``` Now add `@generated` to the file and run again: ``` 2021-06-22 23:47:13 [NOTICE] All file(s) linted successfully with no errors detected ``` Additionally, add `@not-generated` in addition to `@generated`, and linter error pops up again. * cleanup * remove space * fix non utf return * fix non utf return Co-authored-by: Lukas Gravley <admiralawkbar@github.com>
2021-06-28 08:59:11 -04:00
###############################################
# Filter files if FILTER_REGEX_INCLUDE is set #
Ignore files marked with @generated marker (#1689) * Ignore files marked with @generated marker `@generated` marker is used by certain tools to understand that the file is generated, so it should be treated differently than a file written by a human: * these files do not need to be reformatted, * diffs in these files are less important, * and linters should not be invoked on these files. This PR proposes builtin support for `@generated` marker (and `@not-generated` marker to mark file as not generated when it contains `@generated` marker, like `README.md`). I have not found a standard for a generated file marker, but: * Facebook [uses `@generated` marker](https://tinyurl.com/fb-generated) * Phabricator tool which was spawned from Facebook internal tool [also understands `@generated` marker](https://git.io/JnVHa) * Cargo inserts `@generated` marker into [generated Cargo.lock files](https://git.io/JnVHP) Super-linter supports regex includes and excludes, but they are harder to maintain (each repository needs to be configured) than patching the tools which generate the files. My personal story is that I maintain rust-protobuf crate, which started emitting `@generated` markers [six years ago](https://git.io/JnV5h) after a request of a Phabricator user. Test Plan: Create a test file `test.sh`: ``` echo $a ``` Run: ``` docker run -e RUN_LOCAL=true -v $HOME/tmp/g:/tmp/lint super-linter-test ``` Result is: ``` In /tmp/lint/test.sh line 1: echo $a ^-- SC2148: Tips depend on target shell and yours is unknown. Add a shebang or a 'shell' directive. ^-- SC2154: a is referenced but not assigned. ^-- SC2086: Double quote to prevent globbing and word splitting. ... 2021-06-22 23:46:16 [ERROR] ERRORS FOUND in BASH:[1] ``` Now add `@generated` to the file and run again: ``` 2021-06-22 23:47:13 [NOTICE] All file(s) linted successfully with no errors detected ``` Additionally, add `@not-generated` in addition to `@generated`, and linter error pops up again. * cleanup * remove space * fix non utf return * fix non utf return Co-authored-by: Lukas Gravley <admiralawkbar@github.com>
2021-06-28 08:59:11 -04:00
###############################################
if [[ -n "$FILTER_REGEX_INCLUDE" ]] && [[ ! (${FILE} =~ $FILTER_REGEX_INCLUDE) ]]; then
debug "FILTER_REGEX_INCLUDE didn't match. Skipping ${FILE}"
continue
fi
Ignore files marked with @generated marker (#1689) * Ignore files marked with @generated marker `@generated` marker is used by certain tools to understand that the file is generated, so it should be treated differently than a file written by a human: * these files do not need to be reformatted, * diffs in these files are less important, * and linters should not be invoked on these files. This PR proposes builtin support for `@generated` marker (and `@not-generated` marker to mark file as not generated when it contains `@generated` marker, like `README.md`). I have not found a standard for a generated file marker, but: * Facebook [uses `@generated` marker](https://tinyurl.com/fb-generated) * Phabricator tool which was spawned from Facebook internal tool [also understands `@generated` marker](https://git.io/JnVHa) * Cargo inserts `@generated` marker into [generated Cargo.lock files](https://git.io/JnVHP) Super-linter supports regex includes and excludes, but they are harder to maintain (each repository needs to be configured) than patching the tools which generate the files. My personal story is that I maintain rust-protobuf crate, which started emitting `@generated` markers [six years ago](https://git.io/JnV5h) after a request of a Phabricator user. Test Plan: Create a test file `test.sh`: ``` echo $a ``` Run: ``` docker run -e RUN_LOCAL=true -v $HOME/tmp/g:/tmp/lint super-linter-test ``` Result is: ``` In /tmp/lint/test.sh line 1: echo $a ^-- SC2148: Tips depend on target shell and yours is unknown. Add a shebang or a 'shell' directive. ^-- SC2154: a is referenced but not assigned. ^-- SC2086: Double quote to prevent globbing and word splitting. ... 2021-06-22 23:46:16 [ERROR] ERRORS FOUND in BASH:[1] ``` Now add `@generated` to the file and run again: ``` 2021-06-22 23:47:13 [NOTICE] All file(s) linted successfully with no errors detected ``` Additionally, add `@not-generated` in addition to `@generated`, and linter error pops up again. * cleanup * remove space * fix non utf return * fix non utf return Co-authored-by: Lukas Gravley <admiralawkbar@github.com>
2021-06-28 08:59:11 -04:00
###############################################
# Filter files if FILTER_REGEX_EXCLUDE is set #
Ignore files marked with @generated marker (#1689) * Ignore files marked with @generated marker `@generated` marker is used by certain tools to understand that the file is generated, so it should be treated differently than a file written by a human: * these files do not need to be reformatted, * diffs in these files are less important, * and linters should not be invoked on these files. This PR proposes builtin support for `@generated` marker (and `@not-generated` marker to mark file as not generated when it contains `@generated` marker, like `README.md`). I have not found a standard for a generated file marker, but: * Facebook [uses `@generated` marker](https://tinyurl.com/fb-generated) * Phabricator tool which was spawned from Facebook internal tool [also understands `@generated` marker](https://git.io/JnVHa) * Cargo inserts `@generated` marker into [generated Cargo.lock files](https://git.io/JnVHP) Super-linter supports regex includes and excludes, but they are harder to maintain (each repository needs to be configured) than patching the tools which generate the files. My personal story is that I maintain rust-protobuf crate, which started emitting `@generated` markers [six years ago](https://git.io/JnV5h) after a request of a Phabricator user. Test Plan: Create a test file `test.sh`: ``` echo $a ``` Run: ``` docker run -e RUN_LOCAL=true -v $HOME/tmp/g:/tmp/lint super-linter-test ``` Result is: ``` In /tmp/lint/test.sh line 1: echo $a ^-- SC2148: Tips depend on target shell and yours is unknown. Add a shebang or a 'shell' directive. ^-- SC2154: a is referenced but not assigned. ^-- SC2086: Double quote to prevent globbing and word splitting. ... 2021-06-22 23:46:16 [ERROR] ERRORS FOUND in BASH:[1] ``` Now add `@generated` to the file and run again: ``` 2021-06-22 23:47:13 [NOTICE] All file(s) linted successfully with no errors detected ``` Additionally, add `@not-generated` in addition to `@generated`, and linter error pops up again. * cleanup * remove space * fix non utf return * fix non utf return Co-authored-by: Lukas Gravley <admiralawkbar@github.com>
2021-06-28 08:59:11 -04:00
###############################################
if [[ -n "$FILTER_REGEX_EXCLUDE" ]] && [[ ${FILE} =~ $FILTER_REGEX_EXCLUDE ]]; then
debug "FILTER_REGEX_EXCLUDE match. Skipping ${FILE}"
continue
fi
Ignore files marked with @generated marker (#1689) * Ignore files marked with @generated marker `@generated` marker is used by certain tools to understand that the file is generated, so it should be treated differently than a file written by a human: * these files do not need to be reformatted, * diffs in these files are less important, * and linters should not be invoked on these files. This PR proposes builtin support for `@generated` marker (and `@not-generated` marker to mark file as not generated when it contains `@generated` marker, like `README.md`). I have not found a standard for a generated file marker, but: * Facebook [uses `@generated` marker](https://tinyurl.com/fb-generated) * Phabricator tool which was spawned from Facebook internal tool [also understands `@generated` marker](https://git.io/JnVHa) * Cargo inserts `@generated` marker into [generated Cargo.lock files](https://git.io/JnVHP) Super-linter supports regex includes and excludes, but they are harder to maintain (each repository needs to be configured) than patching the tools which generate the files. My personal story is that I maintain rust-protobuf crate, which started emitting `@generated` markers [six years ago](https://git.io/JnV5h) after a request of a Phabricator user. Test Plan: Create a test file `test.sh`: ``` echo $a ``` Run: ``` docker run -e RUN_LOCAL=true -v $HOME/tmp/g:/tmp/lint super-linter-test ``` Result is: ``` In /tmp/lint/test.sh line 1: echo $a ^-- SC2148: Tips depend on target shell and yours is unknown. Add a shebang or a 'shell' directive. ^-- SC2154: a is referenced but not assigned. ^-- SC2086: Double quote to prevent globbing and word splitting. ... 2021-06-22 23:46:16 [ERROR] ERRORS FOUND in BASH:[1] ``` Now add `@generated` to the file and run again: ``` 2021-06-22 23:47:13 [NOTICE] All file(s) linted successfully with no errors detected ``` Additionally, add `@not-generated` in addition to `@generated`, and linter error pops up again. * cleanup * remove space * fix non utf return * fix non utf return Co-authored-by: Lukas Gravley <admiralawkbar@github.com>
2021-06-28 08:59:11 -04:00
###################################################
# Filter files if FILTER_REGEX_EXCLUDE is not set #
###################################################
if [ "${GIT_IGNORED_FILES_INDEX[$FILE]}" ] && [ "${IGNORE_GITIGNORED_FILES}" == "true" ]; then
debug "${FILE} is ignored by Git. Skipping ${FILE}"
continue
fi
Ignore files marked with @generated marker (#1689) * Ignore files marked with @generated marker `@generated` marker is used by certain tools to understand that the file is generated, so it should be treated differently than a file written by a human: * these files do not need to be reformatted, * diffs in these files are less important, * and linters should not be invoked on these files. This PR proposes builtin support for `@generated` marker (and `@not-generated` marker to mark file as not generated when it contains `@generated` marker, like `README.md`). I have not found a standard for a generated file marker, but: * Facebook [uses `@generated` marker](https://tinyurl.com/fb-generated) * Phabricator tool which was spawned from Facebook internal tool [also understands `@generated` marker](https://git.io/JnVHa) * Cargo inserts `@generated` marker into [generated Cargo.lock files](https://git.io/JnVHP) Super-linter supports regex includes and excludes, but they are harder to maintain (each repository needs to be configured) than patching the tools which generate the files. My personal story is that I maintain rust-protobuf crate, which started emitting `@generated` markers [six years ago](https://git.io/JnV5h) after a request of a Phabricator user. Test Plan: Create a test file `test.sh`: ``` echo $a ``` Run: ``` docker run -e RUN_LOCAL=true -v $HOME/tmp/g:/tmp/lint super-linter-test ``` Result is: ``` In /tmp/lint/test.sh line 1: echo $a ^-- SC2148: Tips depend on target shell and yours is unknown. Add a shebang or a 'shell' directive. ^-- SC2154: a is referenced but not assigned. ^-- SC2086: Double quote to prevent globbing and word splitting. ... 2021-06-22 23:46:16 [ERROR] ERRORS FOUND in BASH:[1] ``` Now add `@generated` to the file and run again: ``` 2021-06-22 23:47:13 [NOTICE] All file(s) linted successfully with no errors detected ``` Additionally, add `@not-generated` in addition to `@generated`, and linter error pops up again. * cleanup * remove space * fix non utf return * fix non utf return Co-authored-by: Lukas Gravley <admiralawkbar@github.com>
2021-06-28 08:59:11 -04:00
#########################################
# Filter files with at-generated marker #
#########################################
if [ "${IGNORE_GENERATED_FILES}" == "true" ] && IsGenerated "$FILE"; then
debug "${FILE} is generated. Skipping ${FILE}"
continue
fi
2020-10-02 18:33:01 -04:00
# Editorconfig-checker should check every file
FILE_ARRAY_EDITORCONFIG+=("${FILE}")
if [ "${VALIDATE_JSCPD_ALL_CODEBASE}" == "true" ]; then
debug "Not adding ${FILE} to FILE_ARRAY_JSCPD because we're going to lint the whole codebase anyway."
else
# jscpd also runs an all files
FILE_ARRAY_JSCPD+=("${FILE}")
fi
# Need to make sure we dont check the secrets paterns
# for secrets, as it will pop!
if [ "${BASE_FILE}" != ".gitleaks.toml" ]; then
# GitLeaks also runs an all files
FILE_ARRAY_GITLEAKS+=("${FILE}")
fi
2020-10-02 18:33:01 -04:00
2020-11-10 11:02:03 -05:00
#######################
2020-08-31 17:44:35 -04:00
# Get the shell files #
2020-11-10 11:02:03 -05:00
#######################
2020-08-31 17:44:35 -04:00
if IsValidShellScript "${FILE}"; then
################################
# Append the file to the array #
################################
FILE_ARRAY_BASH+=("${FILE}")
FILE_ARRAY_BASH_EXEC+=("${FILE}")
FILE_ARRAY_SHELL_SHFMT+=("${FILE}")
2020-08-31 12:51:03 -04:00
#########################
# Get the CLOJURE files #
#########################
elif [ "${FILE_TYPE}" == "clj" ] || [ "${FILE_TYPE}" == "cljs" ] ||
[ "${FILE_TYPE}" == "cljc" ] || [ "${FILE_TYPE}" == "edn" ]; then
2020-06-29 10:55:59 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_CLOJURE+=("${FILE}")
#####################
# Get the C++ files #
#####################
elif [ "${FILE_TYPE}" == "cpp" ] || [ "${FILE_TYPE}" == "h" ] ||
[ "${FILE_TYPE}" == "cc" ] || [ "${FILE_TYPE}" == "hpp" ] ||
[ "${FILE_TYPE}" == "cxx" ] || [ "${FILE_TYPE}" == "cu" ] ||
[ "${FILE_TYPE}" == "hxx" ] || [ "${FILE_TYPE}" == "c++" ] ||
[ "${FILE_TYPE}" == "hh" ] || [ "${FILE_TYPE}" == "h++" ] ||
[ "${FILE_TYPE}" == "cuh" ] || [ "${FILE_TYPE}" == "c" ]; then
################################
# Append the file to the array #
################################
FILE_ARRAY_CPP+=("${FILE}")
FILE_ARRAY_CLANG_FORMAT+=("${FILE}")
2020-08-31 12:51:03 -04:00
########################
# Get the COFFEE files #
########################
elif [ "${FILE_TYPE}" == "coffee" ]; then
2020-06-29 10:55:59 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_COFFEESCRIPT+=("${FILE}")
########################
# Get the CSHARP files #
########################
elif [ "${FILE_TYPE}" == "cs" ]; then
2020-06-29 10:55:59 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_CSHARP+=("${FILE}")
#####################
# Get the CSS files #
#####################
elif [ "${FILE_TYPE}" == "css" ] || [ "${FILE_TYPE}" == "scss" ] ||
[ "${FILE_TYPE}" == "sass" ]; then
2020-06-29 10:55:59 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_CSS+=("${FILE}")
2020-07-05 22:21:13 -04:00
######################
2020-08-31 12:51:03 -04:00
# Get the DART files #
2020-07-05 22:21:13 -04:00
######################
2020-08-31 12:51:03 -04:00
elif [ "${FILE_TYPE}" == "dart" ]; then
2020-07-05 22:21:13 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_DART+=("${FILE}")
########################
# Get the DOCKER files #
########################
# Use BASE_FILE here because FILE_TYPE is not reliable when there is no file extension
elif [[ "${FILE_TYPE}" != "tap" ]] && [[ "${FILE_TYPE}" != "yml" ]] &&
[[ "${FILE_TYPE}" != "yaml" ]] && [[ "${FILE_TYPE}" != "json" ]] && [[ "${FILE_TYPE}" != "xml" ]] && [[ "${BASE_FILE}" =~ .*(contain|dock)erfile.* ]]; then
2020-06-29 10:55:59 -04:00
################################
# Append the file to the array #
################################
FILE_ARRAY_DOCKERFILE_HADOLINT+=("${FILE}")
2020-08-31 12:51:03 -04:00
#####################
# Get the ENV files #
#####################
2021-03-10 09:38:31 -05:00
elif [ "${FILE_TYPE}" == "env" ] || [[ "${BASE_FILE}" == *".env."* ]]; then
2020-06-29 10:55:59 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_ENV+=("${FILE}")
#########################
# Get the Gherkin files #
#########################
elif [ "${FILE_TYPE}" == "feature" ]; then
################################
# Append the file to the array #
################################
FILE_ARRAY_GHERKIN+=("${FILE}")
2020-06-29 10:55:59 -04:00
########################
2020-08-31 12:51:03 -04:00
# Get the Golang files #
2020-06-29 10:55:59 -04:00
########################
2020-08-31 12:51:03 -04:00
elif [ "${FILE_TYPE}" == "go" ]; then
2020-06-29 10:55:59 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_GO+=("${FILE}")
2020-06-29 10:55:59 -04:00
########################
2020-08-31 12:51:03 -04:00
# Get the GROOVY files #
2020-06-29 10:55:59 -04:00
########################
2020-08-31 12:51:03 -04:00
elif [ "$FILE_TYPE" == "groovy" ] || [ "$FILE_TYPE" == "jenkinsfile" ] ||
[ "$FILE_TYPE" == "gradle" ] || [ "$FILE_TYPE" == "nf" ]; then
2020-06-29 10:55:59 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_GROOVY+=("$FILE")
######################
# Get the HTML files #
######################
elif [ "${FILE_TYPE}" == "html" ]; then
################################
# Append the file to the array #
##############################p##
FILE_ARRAY_HTML+=("${FILE}")
2020-08-05 14:35:14 -04:00
######################
# Get the Java files #
######################
elif [ "${FILE_TYPE}" == "java" ]; then
################################
# Append the file to the array #
################################
FILE_ARRAY_JAVA+=("${FILE}")
FILE_ARRAY_GOOGLE_JAVA_FORMAT+=("${FILE}")
2020-08-31 12:51:03 -04:00
2020-06-29 10:55:59 -04:00
############################
# Get the JavaScript files #
############################
2020-07-21 13:09:07 -04:00
elif [ "${FILE_TYPE}" == "js" ]; then
2020-06-29 10:55:59 -04:00
################################
# Append the file to the array #
################################
2020-07-21 13:09:07 -04:00
FILE_ARRAY_JAVASCRIPT_ES+=("${FILE}")
FILE_ARRAY_JAVASCRIPT_STANDARD+=("${FILE}")
FILE_ARRAY_JAVASCRIPT_PRETTIER+=("${FILE}")
2020-08-31 12:51:03 -04:00
#######################
# Get the JSONC files #
#######################
elif [ "$FILE_TYPE" == "jsonc" ] || [ "$FILE_TYPE" == "json5" ]; then
################################
# Append the file to the array #
################################
FILE_ARRAY_JSONC+=("${FILE}")
2020-08-31 12:51:03 -04:00
######################
# Get the JSON files #
######################
elif [ "${FILE_TYPE}" == "json" ]; then
2020-07-07 11:02:56 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_JSON+=("${FILE}")
2020-08-31 12:51:03 -04:00
############################
# Check if file is OpenAPI #
############################
if DetectOpenAPIFile "${FILE}"; then
################################
# Append the file to the array #
################################
FILE_ARRAY_OPENAPI+=("${FILE}")
fi
########################
2020-08-31 12:51:03 -04:00
# Check if file is ARM #
########################
2020-08-31 12:51:03 -04:00
if DetectARMFile "${FILE}"; then
################################
# Append the file to the array #
################################
FILE_ARRAY_ARM+=("${FILE}")
fi
#####################################
# Check if the file is CFN template #
#####################################
if DetectCloudFormationFile "${FILE}"; then
################################
# Append the file to the array #
################################
FILE_ARRAY_CLOUDFORMATION+=("${FILE}")
fi
############################################
# Check if the file is AWS States Language #
############################################
if DetectAWSStatesFIle "${FILE}"; then
################################
# Append the file to the array #
################################
FILE_ARRAY_STATES+=("${FILE}")
fi
#####################
# Get the JSX files #
#####################
2020-08-31 12:51:03 -04:00
elif [ "${FILE_TYPE}" == "jsx" ]; then
2020-07-07 11:02:56 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_JSX+=("${FILE}")
########################
# Get the KOTLIN files #
########################
elif [ "${FILE_TYPE}" == "kt" ] || [ "${FILE_TYPE}" == "kts" ]; then
2020-06-29 10:55:59 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_KOTLIN+=("${FILE}")
FILE_ARRAY_KOTLIN_ANDROID+=("${FILE}")
2020-08-31 12:51:03 -04:00
#####################
# Get the LUA files #
#####################
elif [ "$FILE_TYPE" == "lua" ]; then
2020-06-29 10:55:59 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_LUA+=("$FILE")
2020-08-18 18:54:32 -04:00
#######################
# Get the LaTeX files #
#######################
elif [ "${FILE_TYPE}" == "tex" ]; then
################################
# Append the file to the array #
################################
FILE_ARRAY_LATEX+=("${FILE}")
2020-08-31 12:51:03 -04:00
##########################
# Get the MARKDOWN files #
##########################
elif [ "${FILE_TYPE}" == "md" ]; then
2020-08-15 18:53:08 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_MARKDOWN+=("${FILE}")
FILE_ARRAY_NATURAL_LANGUAGE+=("${FILE}")
2020-08-31 12:51:03 -04:00
######################
# Get the PHP files #
######################
elif [ "${FILE_TYPE}" == "php" ]; then
################################
# Append the file to the array #
################################
FILE_ARRAY_PHP_BUILTIN+=("${FILE}")
FILE_ARRAY_PHP_PHPCS+=("${FILE}")
FILE_ARRAY_PHP_PHPSTAN+=("${FILE}")
FILE_ARRAY_PHP_PSALM+=("${FILE}")
######################
# Get the PERL files #
######################
elif [ "${FILE_TYPE}" == "pl" ] || [ "${FILE_TYPE}" == "pm" ] ||
[ "${FILE_TYPE}" == "t" ]; then
2020-06-29 10:55:59 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_PERL+=("${FILE}")
############################
2020-06-29 10:55:59 -04:00
# Get the Powershell files #
2020-08-31 12:51:03 -04:00
############################
elif [ "${FILE_TYPE}" == "ps1" ] ||
2020-11-06 17:10:09 -05:00
[ "${FILE_TYPE}" == "psm1" ] ||
[ "${FILE_TYPE}" == "psd1" ] ||
[ "${FILE_TYPE}" == "ps1xml" ] ||
[ "${FILE_TYPE}" == "pssc" ] ||
[ "${FILE_TYPE}" == "psrc" ] ||
[ "${FILE_TYPE}" == "cdxml" ]; then
2020-06-29 10:55:59 -04:00
################################
# Append the file to the array #
################################
2020-07-21 13:09:07 -04:00
FILE_ARRAY_POWERSHELL+=("${FILE}")
2020-08-31 12:51:03 -04:00
#################################
# Get the PROTOCOL BUFFER files #
#################################
elif [ "${FILE_TYPE}" == "proto" ]; then
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_PROTOBUF+=("${FILE}")
########################
# Get the PYTHON files #
########################
elif [ "${FILE_TYPE}" == "py" ]; then
2020-06-29 10:55:59 -04:00
################################
# Append the file to the array #
################################
FILE_ARRAY_PYTHON_BLACK+=("${FILE}")
2020-08-31 12:51:03 -04:00
FILE_ARRAY_PYTHON_FLAKE8+=("${FILE}")
2020-10-28 11:22:55 -04:00
FILE_ARRAY_PYTHON_ISORT+=("${FILE}")
FILE_ARRAY_PYTHON_PYLINT+=("${FILE}")
FILE_ARRAY_PYTHON_MYPY+=("${FILE}")
2020-08-31 12:51:03 -04:00
######################
# Get the RAKU files #
######################
elif [ "${FILE_TYPE}" == "raku" ] || [ "${FILE_TYPE}" == "rakumod" ] ||
[ "${FILE_TYPE}" == "rakutest" ] || [ "${FILE_TYPE}" == "pm6" ] ||
[ "${FILE_TYPE}" == "pl6" ] || [ "${FILE_TYPE}" == "p6" ]; then
2020-06-29 10:55:59 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_RAKU+=("${FILE}")
####################
# Get the R files #
####################
elif [ "${FILE_TYPE}" == "r" ] || [ "${FILE_TYPE}" == "rmd" ]; then
2020-06-29 10:55:59 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_R+=("${FILE}")
######################
# Get the RUBY files #
######################
elif [ "${FILE_TYPE}" == "rb" ]; then
2020-07-30 16:39:05 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_RUBY+=("${FILE}")
######################
# Get the RUST files #
######################
elif [ "${FILE_TYPE}" == "rs" ]; then
################################
# Append the file to the array #
################################
FILE_ARRAY_RUST_2015+=("${FILE}")
FILE_ARRAY_RUST_2018+=("${FILE}")
FILE_ARRAY_RUST_2021+=("${FILE}")
#######################
# Get the RUST crates #
#######################
elif [ "${BASE_FILE}" == "cargo.toml" ]; then
###############################################
# Append the crate manifest file to the array #
###############################################
FILE_ARRAY_RUST_CLIPPY+=("${FILE}")
###########################
# Get the SCALA files #
###########################
elif [ "${FILE_TYPE}" == "scala" ] || [ "${FILE_TYPE}" == "sc" ] || [ "${BASE_FILE}" == "??????" ]; then
################################
# Append the file to the array #
################################
FILE_ARRAY_SCALAFMT+=("${FILE}")
2020-09-07 10:49:07 -04:00
###########################
# Get the SNAKEMAKE files #
###########################
elif [ "${FILE_TYPE}" == "smk" ] || [ "${BASE_FILE}" == "snakefile" ]; then
2020-09-07 10:49:07 -04:00
################################
# Append the file to the array #
################################
FILE_ARRAY_SNAKEMAKE_LINT+=("${FILE}")
FILE_ARRAY_SNAKEMAKE_SNAKEFMT+=("${FILE}")
2020-09-07 10:49:07 -04:00
2020-08-31 12:51:03 -04:00
#####################
# Get the SQL files #
#####################
elif [ "${FILE_TYPE}" == "sql" ]; then
2020-07-06 04:17:20 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_SQL+=("${FILE}")
FILE_ARRAY_SQLFLUFF+=("${FILE}")
2020-08-31 12:51:03 -04:00
###########################
# Get the Terraform files #
###########################
elif [ "${FILE_TYPE}" == "tf" ]; then
2020-06-29 10:55:59 -04:00
################################
# Append the file to the array #
################################
FILE_ARRAY_TERRAFORM_TFLINT+=("${FILE}")
2020-08-31 12:51:03 -04:00
FILE_ARRAY_TERRAFORM_TERRASCAN+=("${FILE}")
2020-10-08 09:18:08 -04:00
############################
# Get the Terragrunt files #
############################
elif [ "${FILE_TYPE}" == "hcl" ] && [[ ${FILE} != *".tflint.hcl"* ]] && [[ ${FILE} != *".pkr.hcl"* ]]; then
2020-10-08 09:18:08 -04:00
################################
# Append the file to the array #
################################
FILE_ARRAY_TERRAGRUNT+=("${FILE}")
2020-08-31 12:51:03 -04:00
############################
# Get the TypeScript files #
############################
elif [ "${FILE_TYPE}" == "ts" ]; then
2020-06-29 10:55:59 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_TYPESCRIPT_ES+=("${FILE}")
FILE_ARRAY_TYPESCRIPT_STANDARD+=("${FILE}")
FILE_ARRAY_TYPESCRIPT_PRETTIER+=("${FILE}")
2020-08-31 12:51:03 -04:00
#####################
# Get the TSX files #
#####################
elif [ "${FILE_TYPE}" == "tsx" ]; then
2020-06-29 10:55:59 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_TSX+=("${FILE}")
#####################
# Get the XML files #
#####################
elif [ "${FILE_TYPE}" == "xml" ]; then
2020-07-04 18:14:27 -04:00
################################
# Append the file to the array #
2020-08-06 13:01:36 -04:00
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_XML+=("${FILE}")
################################
# Get the CLOUDFORMATION files #
################################
elif [ "${FILE_TYPE}" == "yml" ] || [ "${FILE_TYPE}" == "yaml" ]; then
2020-07-14 10:28:58 -04:00
################################
# Append the file to the array #
################################
2020-08-31 12:51:03 -04:00
FILE_ARRAY_YAML+=("${FILE}")
###################################
# Check if file is GitHub Actions #
###################################
if DetectActions "${FILE}"; then
################################
# Append the file to the array #
################################
FILE_ARRAY_GITHUB_ACTIONS+=("${FILE}")
fi
2020-08-31 12:51:03 -04:00
#####################################
# Check if the file is CFN template #
#####################################
if DetectCloudFormationFile "${FILE}"; then
################################
# Append the file to the array #
################################
FILE_ARRAY_CLOUDFORMATION+=("${FILE}")
fi
############################
# Check if file is OpenAPI #
############################
if DetectOpenAPIFile "${FILE}"; then
################################
# Append the file to the array #
################################
FILE_ARRAY_OPENAPI+=("${FILE}")
fi
2020-10-13 11:21:23 -04:00
########################################
# Check if the file is Tekton template #
########################################
if DetectTektonFile "${FILE}"; then
################################
# Append the file to the array #
################################
FILE_ARRAY_TEKTON+=("${FILE}")
fi
2020-09-21 18:53:30 -04:00
############################################
# Check if the file is Kubernetes template #
############################################
if DetectKubernetesFile "${FILE}"; then
################################
# Append the file to the array #
################################
FILE_ARRAY_KUBERNETES_KUBEVAL+=("${FILE}")
2020-09-21 18:53:30 -04:00
fi
2020-08-31 12:51:03 -04:00
########################################################################
# We have something that we need to try to check file type another way #
########################################################################
2020-06-29 10:55:59 -04:00
else
##############################################
# Use file to see if we can parse what it is #
##############################################
2020-08-24 12:11:10 -04:00
CheckFileType "${FILE}"
2020-08-24 12:16:22 -04:00
fi
2020-10-19 21:46:14 -04:00
##########################################
# Print line break after each file debug #
##########################################
debug ""
2020-06-29 10:55:59 -04:00
done
if [ "${VALIDATE_JSCPD_ALL_CODEBASE}" == "true" ]; then
debug "Adding the root of the workspaces to the list of files and directories to lint with JSCPD..."
FILE_ARRAY_JSCPD+=("${GITHUB_WORKSPACE}")
fi
2020-06-29 10:55:59 -04:00
################
# Footer print #
################
2020-07-30 16:39:05 -04:00
info "----------------------------------------------"
info "Successfully gathered list of files..."
2020-06-29 10:55:59 -04:00
}