feat: run linters in parallel (#5177)

This commit is contained in:
Marco Ferrari 2024-01-30 20:24:55 +01:00 committed by GitHub
parent 0578ab8daf
commit 99e41ce451
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 852 additions and 884 deletions

View file

@ -1,5 +1,9 @@
--- ---
# Options reference: https://www.checkov.io/2.Basics/CLI%20Command%20Reference.html # Options reference: https://www.checkov.io/2.Basics/CLI%20Command%20Reference.html
directory:
- test/linters/checkov/bad
quiet: false quiet: false
... ...

View file

@ -0,0 +1,8 @@
---
# Options reference: https://www.checkov.io/2.Basics/CLI%20Command%20Reference.html
directory:
- test/linters/checkov/good
quiet: false
...

View file

@ -0,0 +1,13 @@
title = "gitleaks config"
[extend]
# useDefault will extend the base configuration with the default gitleaks config:
# https://github.com/zricethezav/gitleaks/blob/master/config/gitleaks.toml
useDefault = true
[allowlist]
description = "Allow secrets in test files"
paths = [
'''.*/test/linters/gitleaks/bad/.*'''
]

View file

@ -110,6 +110,7 @@ jobs:
VALIDATE_ALL_CODEBASE: false VALIDATE_ALL_CODEBASE: false
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DEFAULT_BRANCH: main DEFAULT_BRANCH: main
GITLEAKS_CONFIG_FILE: .gitleaks-ignore-tests.toml
RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES: "default.json,hoge.json" RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES: "default.json,hoge.json"
TYPESCRIPT_STANDARD_TSCONFIG_FILE: ".github/linters/tsconfig.json" TYPESCRIPT_STANDARD_TSCONFIG_FILE: ".github/linters/tsconfig.json"

3
.gitignore vendored
View file

@ -77,6 +77,9 @@ super-linter.report
# Code coverage data for tests # Code coverage data for tests
.coverage .coverage
# Terraform workspace
.terraform
# Test reports # Test reports
test/reports test/reports

View file

@ -1,9 +0,0 @@
.github-personal-access-token:github-fine-grained-pat:1
/github/workspace/.github-personal-access-token:github-fine-grained-pat:1
/github/workspace/test/linters/gitleaks/bad/gitleaks_bad_01.txt:aws-access-token:1
/github/workspace/test/linters/gitleaks/bad/gitleaks_bad_01.txt:generic-api-key:2
/tmp/lint/.github-personal-access-token:github-fine-grained-pat:1
/tmp/lint/test/linters/gitleaks/bad/gitleaks_bad_01.txt:aws-access-token:1
/tmp/lint/test/linters/gitleaks/bad/gitleaks_bad_01.txt:generic-api-key:2
test/linters/gitleaks/bad/gitleaks_bad_01.txt:aws-access-token:1
test/linters/gitleaks/bad/gitleaks_bad_01.txt:generic-api-key:2

View file

@ -134,6 +134,7 @@ RUN apk add --no-cache \
nodejs-current \ nodejs-current \
openjdk17-jre \ openjdk17-jre \
openssh-client \ openssh-client \
parallel \
perl \ perl \
php82 \ php82 \
php82-ctype \ php82-ctype \

View file

@ -164,6 +164,7 @@ test-git-flags: ## Run super-linter with different git-related flags
-e ACTIONS_RUNNER_DEBUG=true \ -e ACTIONS_RUNNER_DEBUG=true \
-e ERROR_ON_MISSING_EXEC_BIT=true \ -e ERROR_ON_MISSING_EXEC_BIT=true \
-e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true \ -e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true \
-e FILTER_REGEX_EXCLUDE=".*/test/linters/.*" \
-e DEFAULT_BRANCH=main \ -e DEFAULT_BRANCH=main \
-e IGNORE_GENERATED_FILES=true \ -e IGNORE_GENERATED_FILES=true \
-e IGNORE_GITIGNORED_FILES=true \ -e IGNORE_GITIGNORED_FILES=true \
@ -178,6 +179,8 @@ lint-codebase: ## Lint the entire codebase
-e ACTIONS_RUNNER_DEBUG=true \ -e ACTIONS_RUNNER_DEBUG=true \
-e DEFAULT_BRANCH=main \ -e DEFAULT_BRANCH=main \
-e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true \ -e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true \
-e FILTER_REGEX_EXCLUDE=".*/test/linters/.*" \
-e GITLEAKS_CONFIG_FILE=".gitleaks-ignore-tests.toml" \
-e RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES="default.json,hoge.json" \ -e RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES="default.json,hoge.json" \
-e VALIDATE_ALL_CODEBASE=true \ -e VALIDATE_ALL_CODEBASE=true \
-v "$(CURDIR):/tmp/lint" \ -v "$(CURDIR):/tmp/lint" \
@ -195,6 +198,7 @@ lint-subset-files-enable-only-one-type: ## Lint a small subset of files in the c
-e ACTIONS_RUNNER_DEBUG=true \ -e ACTIONS_RUNNER_DEBUG=true \
-e DEFAULT_BRANCH=main \ -e DEFAULT_BRANCH=main \
-e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true \ -e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true \
-e FILTER_REGEX_EXCLUDE=".*/test/linters/.*" \
-e VALIDATE_ALL_CODEBASE=true \ -e VALIDATE_ALL_CODEBASE=true \
-e VALIDATE_MARKDOWN=true \ -e VALIDATE_MARKDOWN=true \
-v "$(CURDIR):/tmp/lint" \ -v "$(CURDIR):/tmp/lint" \
@ -207,6 +211,7 @@ lint-subset-files-enable-expensive-io-checks: ## Lint a small subset of files in
-e ACTIONS_RUNNER_DEBUG=true \ -e ACTIONS_RUNNER_DEBUG=true \
-e DEFAULT_BRANCH=main \ -e DEFAULT_BRANCH=main \
-e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true \ -e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true \
-e FILTER_REGEX_EXCLUDE=".*/test/linters/.*" \
-e VALIDATE_ALL_CODEBASE=true \ -e VALIDATE_ALL_CODEBASE=true \
-e VALIDATE_ARM=true \ -e VALIDATE_ARM=true \
-e VALIDATE_CLOUDFORMATION=true \ -e VALIDATE_CLOUDFORMATION=true \
@ -272,19 +277,19 @@ test-custom-ssl-cert: ## Test the configuration of a custom SSL/TLS certificate
$(SUPER_LINTER_TEST_CONTAINER_URL) $(SUPER_LINTER_TEST_CONTAINER_URL)
.phony: test-linters .phony: test-linters
test-linters: ## Run the linters test suite test-linters: test-linters-expect-success test-linters-expect-failure ## Run the linters test suite
docker run \
-e ACTIONS_RUNNER_DEBUG=true \ .phony: test-linters-expect-success
-e CHECKOV_FILE_NAME=".checkov-test-linters.yaml" \ test-linters-expect-success: ## Run the linters test suite expecting successes
-e DEFAULT_BRANCH=main \ $(CURDIR)/test/run-super-linter-tests.sh \
-e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true \ $(SUPER_LINTER_TEST_CONTAINER_URL) \
-e JSCPD_CONFIG_FILE=".jscpd-test-linters.json" \ "run_test_cases_expect_success"
-e RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES="default.json,hoge.json" \
-e RUN_LOCAL=true \ .phony: test-linters-expect-failure
-e TEST_CASE_RUN=true \ test-linters-expect-failure: ## Run the linters test suite expecting failures
-e TYPESCRIPT_STANDARD_TSCONFIG_FILE=".github/linters/tsconfig.json" \ $(CURDIR)/test/run-super-linter-tests.sh \
-v "$(CURDIR):/tmp/lint" \ $(SUPER_LINTER_TEST_CONTAINER_URL) \
$(SUPER_LINTER_TEST_CONTAINER_URL) "run_test_cases_expect_failure"
.phony: build-dev-container-image .phony: build-dev-container-image
build-dev-container-image: ## Build commit linter container image build-dev-container-image: ## Build commit linter container image

View file

@ -90,6 +90,7 @@ new tool, it should include:
- Update the orchestration scripts to run the new tool: - Update the orchestration scripts to run the new tool:
- `lib/linter.sh` - `lib/linter.sh`
- `lib/functions/linterCommands.sh`
- Provide the logic to populate the list of files or directories to examine: `lib/buildFileList.sh` - Provide the logic to populate the list of files or directories to examine: `lib/buildFileList.sh`
- If necessary, provide elaborate logic to detect if the tool should examine a file or a directory: `lib/detectFiles.sh` - If necessary, provide elaborate logic to detect if the tool should examine a file or a directory: `lib/detectFiles.sh`
- If the tool needs to take into account special cases: - If the tool needs to take into account special cases:

View file

@ -52,10 +52,6 @@ This section helps you migrate from super-linter `v5` to `v6`.
- If you defined secret patterns in `.gitleaks.toml`, Gitleaks may report errors - If you defined secret patterns in `.gitleaks.toml`, Gitleaks may report errors
about that file. If this happens, you can about that file. If this happens, you can
[configure Gitleaks to ignore that file](https://github.com/gitleaks/gitleaks/tree/master?tab=readme-ov-file#gitleaksignore). [configure Gitleaks to ignore that file](https://github.com/gitleaks/gitleaks/tree/master?tab=readme-ov-file#gitleaksignore).
- Gitleaks doesn't consider the `FILTER_REGEX_EXCLUDE`, `FILTER_REGEX_INCLUDE`,
`IGNORE_GENERATED_FILES`, `IGNORE_GITIGNORED_FILES` variables. For more
information about how to ignore files with Gitleaks, see
[the Gitleaks documentation](https://github.com/gitleaks/gitleaks/tree/master?tab=readme-ov-file#gitleaksignore).
### Jscpd ### Jscpd

View file

@ -13,7 +13,6 @@ function GenerateFileDiff() {
if [ "${GITHUB_EVENT_NAME:-}" == "push" ]; then if [ "${GITHUB_EVENT_NAME:-}" == "push" ]; then
RunFileDiffCommand "${DIFF_TREE_CMD}" RunFileDiffCommand "${DIFF_TREE_CMD}"
if [ ${#RAW_FILE_ARRAY[@]} -eq 0 ]; then if [ ${#RAW_FILE_ARRAY[@]} -eq 0 ]; then
debug "----------------------------------------------"
debug "Generating the file array with diff-tree produced [0] items, trying with git diff against the default branch..." debug "Generating the file array with diff-tree produced [0] items, trying with git diff against the default branch..."
RunFileDiffCommand "${DIFF_GIT_DEFAULT_BRANCH_CMD}" RunFileDiffCommand "${DIFF_GIT_DEFAULT_BRANCH_CMD}"
fi fi
@ -49,20 +48,13 @@ function BuildFileList() {
debug "TEST_CASE_RUN: ${TEST_CASE_RUN}" debug "TEST_CASE_RUN: ${TEST_CASE_RUN}"
if [ "${VALIDATE_ALL_CODEBASE}" == "false" ] && [ "${TEST_CASE_RUN}" != "true" ]; then if [ "${VALIDATE_ALL_CODEBASE}" == "false" ] && [ "${TEST_CASE_RUN}" != "true" ]; then
debug "----------------------------------------------"
debug "Build the list of all changed files" debug "Build the list of all changed files"
GenerateFileDiff GenerateFileDiff
else else
WORKSPACE_PATH="${GITHUB_WORKSPACE}"
if [ "${TEST_CASE_RUN}" == "true" ]; then
WORKSPACE_PATH="${GITHUB_WORKSPACE}/${TEST_CASE_FOLDER}"
fi
if [ "${USE_FIND_ALGORITHM}" == 'true' ]; then if [ "${USE_FIND_ALGORITHM}" == 'true' ]; then
debug "----------------------------------------------" debug "Populating the file list with all the files in the ${GITHUB_WORKSPACE} workspace using FIND algorithm"
debug "Populating the file list with all the files in the ${WORKSPACE_PATH} workspace using FIND algorithm" if ! mapfile -t RAW_FILE_ARRAY < <(find "${GITHUB_WORKSPACE}" \
if ! mapfile -t RAW_FILE_ARRAY < <(find "${WORKSPACE_PATH}" \
-not \( -path '*/\.git' -prune \) \ -not \( -path '*/\.git' -prune \) \
-not \( -path '*/\.pytest_cache' -prune \) \ -not \( -path '*/\.pytest_cache' -prune \) \
-not \( -path '*/\.rbenv' -prune \) \ -not \( -path '*/\.rbenv' -prune \) \
@ -87,8 +79,7 @@ function BuildFileList() {
fi fi
else else
debug "----------------------------------------------" DIFF_GIT_VALIDATE_ALL_CODEBASE="git -C \"${GITHUB_WORKSPACE}\" ls-tree -r --name-only HEAD | xargs -I % sh -c \"echo ${GITHUB_WORKSPACE}/%\" 2>&1"
DIFF_GIT_VALIDATE_ALL_CODEBASE="git -C \"${WORKSPACE_PATH}\" ls-tree -r --name-only HEAD | xargs -I % sh -c \"echo ${WORKSPACE_PATH}/%\" 2>&1"
debug "Populating the file list with: ${DIFF_GIT_VALIDATE_ALL_CODEBASE}" debug "Populating the file list with: ${DIFF_GIT_VALIDATE_ALL_CODEBASE}"
if ! mapfile -t RAW_FILE_ARRAY < <(eval "set -eo pipefail; ${DIFF_GIT_VALIDATE_ALL_CODEBASE}; set +eo pipefail"); then if ! mapfile -t RAW_FILE_ARRAY < <(eval "set -eo pipefail; ${DIFF_GIT_VALIDATE_ALL_CODEBASE}; set +eo pipefail"); then
fatal "Failed to get a list of changed files. USE_FIND_ALGORITHM: ${USE_FIND_ALGORITHM}" fatal "Failed to get a list of changed files. USE_FIND_ALGORITHM: ${USE_FIND_ALGORITHM}"
@ -105,77 +96,108 @@ function BuildFileList() {
#################################################### ####################################################
# Configure linters that scan the entire workspace # # Configure linters that scan the entire workspace #
#################################################### ####################################################
debug "Checking if we are in test mode before configuring the list of directories to lint" debug "Checking if we are in test mode before configuring the list of directories to lint. TEST_CASE_RUN: ${TEST_CASE_RUN}"
if [ "${TEST_CASE_RUN}" == "true" ]; then if [ "${TEST_CASE_RUN}" == "true" ]; then
debug "We are running in test mode." debug "We are running in test mode."
debug "Adding test case directories to the list of directories to analyze with ansible-lint." debug "Adding test case directories to the list of directories to analyze with JSCPD."
DEFAULT_ANSIBLE_TEST_CASE_DIRECTORY="${GITHUB_WORKSPACE}/${TEST_CASE_FOLDER}/ansible"
debug "DEFAULT_ANSIBLE_TEST_CASE_DIRECTORY: ${DEFAULT_ANSIBLE_TEST_CASE_DIRECTORY}"
FILE_ARRAY_ANSIBLE+=("${DEFAULT_ANSIBLE_TEST_CASE_DIRECTORY}/bad")
FILE_ARRAY_ANSIBLE+=("${DEFAULT_ANSIBLE_TEST_CASE_DIRECTORY}/good")
debug "Adding test case directories to the list of directories to analyze with Checkov."
DEFAULT_CHECKOV_TEST_CASE_DIRECTORY="${GITHUB_WORKSPACE}/${TEST_CASE_FOLDER}/checkov"
debug "DEFAULT_CHECKOV_TEST_CASE_DIRECTORY: ${DEFAULT_CHECKOV_TEST_CASE_DIRECTORY}"
FILE_ARRAY_CHECKOV+=("${DEFAULT_CHECKOV_TEST_CASE_DIRECTORY}/bad")
FILE_ARRAY_CHECKOV+=("${DEFAULT_CHECKOV_TEST_CASE_DIRECTORY}/good")
debug "Adding test case directories to the list of directories to analyze with Gitleaks."
DEFAULT_GITLEAKS_TEST_CASE_DIRECTORY="${GITHUB_WORKSPACE}/${TEST_CASE_FOLDER}/gitleaks"
debug "DEFAULT_GITLEAKS_TEST_CASE_DIRECTORY: ${DEFAULT_GITLEAKS_TEST_CASE_DIRECTORY}"
FILE_ARRAY_GITLEAKS+=("${DEFAULT_GITLEAKS_TEST_CASE_DIRECTORY}/bad")
FILE_ARRAY_GITLEAKS+=("${DEFAULT_GITLEAKS_TEST_CASE_DIRECTORY}/good")
debug "Adding test case directories to the list of directories to analyze with Checkov."
DEFAULT_JSCPD_TEST_CASE_DIRECTORY="${GITHUB_WORKSPACE}/${TEST_CASE_FOLDER}/jscpd" DEFAULT_JSCPD_TEST_CASE_DIRECTORY="${GITHUB_WORKSPACE}/${TEST_CASE_FOLDER}/jscpd"
# We need this for parallel
export DEFAULT_JSCPD_TEST_CASE_DIRECTORY
debug "DEFAULT_JSCPD_TEST_CASE_DIRECTORY: ${DEFAULT_JSCPD_TEST_CASE_DIRECTORY}" debug "DEFAULT_JSCPD_TEST_CASE_DIRECTORY: ${DEFAULT_JSCPD_TEST_CASE_DIRECTORY}"
FILE_ARRAY_JSCPD+=("${DEFAULT_JSCPD_TEST_CASE_DIRECTORY}/bad") RAW_FILE_ARRAY+=("${DEFAULT_JSCPD_TEST_CASE_DIRECTORY}/bad")
FILE_ARRAY_JSCPD+=("${DEFAULT_JSCPD_TEST_CASE_DIRECTORY}/good") RAW_FILE_ARRAY+=("${DEFAULT_JSCPD_TEST_CASE_DIRECTORY}/good")
else fi
debug "We are not running in test mode (${TEST_CASE_RUN})."
debug "Add GITHUB_WORKSPACE (${GITHUB_WORKSPACE}) to the list of files to lint because we might need it for linters that lint the whole workspace"
RAW_FILE_ARRAY+=("${GITHUB_WORKSPACE}")
if [ -d "${ANSIBLE_DIRECTORY}" ]; then if [ -d "${ANSIBLE_DIRECTORY}" ]; then
debug "Adding ANSIBLE_DIRECTORY (${ANSIBLE_DIRECTORY}) to the list of files and directories to lint." debug "Adding ANSIBLE_DIRECTORY (${ANSIBLE_DIRECTORY}) to the list of files and directories to lint."
FILE_ARRAY_ANSIBLE+=("${ANSIBLE_DIRECTORY}") RAW_FILE_ARRAY+=("${ANSIBLE_DIRECTORY}")
else else
debug "ANSIBLE_DIRECTORY (${ANSIBLE_DIRECTORY}) does NOT exist." debug "ANSIBLE_DIRECTORY (${ANSIBLE_DIRECTORY}) does NOT exist."
fi fi
if CheckovConfigurationFileContainsDirectoryOption "${CHECKOV_LINTER_RULES}"; then local PARALLEL_RESULTS_FILE_PATH
debug "No need to configure the directories to check for Checkov." PARALLEL_RESULTS_FILE_PATH="/tmp/super-linter-parallel-results-build-file-list.json"
debug "PARALLEL_RESULTS_FILE_PATH when building the file list: ${PARALLEL_RESULTS_FILE_PATH}"
local -a PARALLEL_COMMAND
PARALLEL_COMMAND=(parallel --will-cite --keep-order --max-procs "$(($(nproc) * 1))" --results "${PARALLEL_RESULTS_FILE_PATH}" --xargs)
if [ "${LOG_DEBUG}" == "true" ]; then
debug "LOG_DEBUG is enabled. Enable verbose ouput for parallel"
PARALLEL_COMMAND+=(--verbose)
fi
# Max number of files to categorize per process
PARALLEL_COMMAND+=(--max-lines 10)
PARALLEL_COMMAND+=("BuildFileArrays")
debug "PARALLEL_COMMAND to build the list of files and directories to lint: ${PARALLEL_COMMAND[*]}"
FILE_ARRAYS_DIRECTORY_PATH="$(mktemp -d)"
export FILE_ARRAYS_DIRECTORY_PATH
debug "Created FILE_ARRAYS_DIRECTORY_PATH: ${FILE_ARRAYS_DIRECTORY_PATH}"
info "Building the list of files and directories to check"
PARALLEL_COMMAND_OUTPUT=$(printf "%s\n" "${RAW_FILE_ARRAY[@]}" | "${PARALLEL_COMMAND[@]}" 2>&1)
PARALLEL_COMMAND_RETURN_CODE=$?
debug "PARALLEL_COMMAND_OUTPUT to build the file list (exit code: ${PARALLEL_COMMAND_RETURN_CODE}):\n${PARALLEL_COMMAND_OUTPUT}"
debug "Parallel output file (${PARALLEL_RESULTS_FILE_PATH}) contents when building the file list:\n$(cat "${PARALLEL_RESULTS_FILE_PATH}")"
local RESULTS_OBJECT
RESULTS_OBJECT=
if ! RESULTS_OBJECT=$(jq -n '[inputs]' "${PARALLEL_RESULTS_FILE_PATH}"); then
fatal "Error loading results when building the file list: ${RESULTS_OBJECT}"
fi
debug "RESULTS_OBJECT for ${FILE_TYPE}:\n${RESULTS_OBJECT}"
local STDOUT_BUILD_FILE_LIST
# Get raw output so we can strip quotes from the data we load
if ! STDOUT_BUILD_FILE_LIST="$(jq --raw-output '.[].Stdout' <<<"${RESULTS_OBJECT}")"; then
fatal "Error when loading stdout when building the file list: ${STDOUT_BUILD_FILE_LIST}"
fi
if [ -n "${STDOUT_BUILD_FILE_LIST}" ]; then
info "Command output when building the file list:\n------\n${STDOUT_BUILD_FILE_LIST}\n------"
else else
debug "Adding ${GITHUB_WORKSPACE} to the list of directories to analyze with Checkov." debug "Stdout when building the file list is empty"
FILE_ARRAY_CHECKOV+=("${GITHUB_WORKSPACE}")
fi fi
debug "Adding ${GITHUB_WORKSPACE} to the list of directories to analyze with Gitleaks." local STDERR_BUILD_FILE_LIST
FILE_ARRAY_GITLEAKS+=("${GITHUB_WORKSPACE}") if ! STDERR_BUILD_FILE_LIST="$(jq --raw-output '.[].Stderr' <<<"${RESULTS_OBJECT}")"; then
fatal "Error when loading stderr when building the file list:\n${STDERR_BUILD_FILE_LIST}"
debug "Adding ${GITHUB_WORKSPACE} to the list of directories to analyze with JSCPD."
FILE_ARRAY_JSCPD+=("${GITHUB_WORKSPACE}")
fi fi
if CheckovConfigurationFileContainsDirectoryOption "${CHECKOV_LINTER_RULES}"; then if [ -n "${STDERR_BUILD_FILE_LIST}" ]; then
debug "No need to configure the directories to check for Checkov." info "Command output when building the file list:\n------\n${STDERR_BUILD_FILE_LIST}\n------"
else else
debug "Checking if we are in test mode before configuring the list of directories to lint with Checkov" debug "Stderr when building the file list is empty"
if [ "${TEST_CASE_RUN}" == "true" ]; then
debug "We are running in test mode. Adding test case directories to the list of directories to analyze with Checkov."
FILE_ARRAY_CHECKOV+=("${DEFAULT_CHECKOV_TEST_CASE_DIRECTORY}/bad")
FILE_ARRAY_CHECKOV+=("${DEFAULT_CHECKOV_TEST_CASE_DIRECTORY}/good")
else
debug "We are not running in test mode (${TEST_CASE_RUN}). Adding ${GITHUB_WORKSPACE} to the list of directories to analyze with Checkov."
FILE_ARRAY_CHECKOV+=("${GITHUB_WORKSPACE}")
fi
fi fi
################################################ if [[ ${PARALLEL_COMMAND_RETURN_CODE} -ne 0 ]]; then
# Iterate through the array of all files found # fatal "Error when building the list of files and directories to lint."
################################################ fi
info "---------------------------------"
info "------ File list to check: ------" ################
info "---------------------------------" # Footer print #
################
info "Successfully gathered list of files..."
}
BuildFileArrays() {
local -a RAW_FILE_ARRAY
RAW_FILE_ARRAY=("$@")
debug "Categorizing the following files: ${RAW_FILE_ARRAY[*]}"
debug "FILTER_REGEX_INCLUDE: ${FILTER_REGEX_INCLUDE}, FILTER_REGEX_EXCLUDE: ${FILTER_REGEX_EXCLUDE}"
ValidateBooleanVariable "IGNORE_GENERATED_FILES" "${IGNORE_GENERATED_FILES}"
ValidateBooleanVariable "IGNORE_GITIGNORED_FILES" "${IGNORE_GITIGNORED_FILES}"
for FILE in "${RAW_FILE_ARRAY[@]}"; do for FILE in "${RAW_FILE_ARRAY[@]}"; do
# Get the file extension # Get the file extension
FILE_TYPE="$(GetFileExtension "$FILE")" FILE_TYPE="$(GetFileExtension "$FILE")"
@ -187,23 +209,35 @@ function BuildFileList() {
debug "FILE: ${FILE}, FILE_TYPE: ${FILE_TYPE}, BASE_FILE: ${BASE_FILE}, FILE_DIR_NAME: ${FILE_DIR_NAME}" debug "FILE: ${FILE}, FILE_TYPE: ${FILE_TYPE}, BASE_FILE: ${BASE_FILE}, FILE_DIR_NAME: ${FILE_DIR_NAME}"
if [ ! -f "${FILE}" ]; then if [ ! -e "${FILE}" ]; then
# File not found in workspace # File not found in workspace
warn "File:{$FILE} existed in commit data, but not found on file system, skipping..." warn "{$FILE} exists in commit data, but not found on file system, skipping..."
continue continue
fi fi
######################################################## # Handle the corner cases of linters that are expected to lint the whole codebase,
# Don't include test cases if not running in test mode # # but we don't have a variable to explicitly set the directory
######################################################## # to lint.
if [[ ${FILE} == *"${TEST_CASE_FOLDER}"* ]] && [ "${TEST_CASE_RUN}" != "true" ]; then if [[ "${FILE}" == "${GITHUB_WORKSPACE}" ]]; then
debug "TEST_CASE_RUN (${TEST_CASE_RUN}) is not true. Skipping ${FILE}..." debug "${FILE} matches with ${GITHUB_WORKSPACE}. Adding it to the list of directories to lint for linters that are expected to lint the whole codebase"
if CheckovConfigurationFileContainsDirectoryOption "${CHECKOV_LINTER_RULES}"; then
debug "No need to configure the directories to check for Checkov because its configuration file contains the list of directories to analyze."
debug "Add the Checkov configuration file path to the list of items to check to consume as output later."
echo "${CHECKOV_LINTER_RULES}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-CHECKOV"
else
debug "Adding ${GITHUB_WORKSPACE} to the list of directories to analyze with Checkov."
echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-CHECKOV"
fi
# JSCPD test cases are handled below because we first need to exclude non-relevant test cases
if [[ "${TEST_CASE_RUN}" == "false" ]]; then
debug "Add ${FILE} to the list of items to lint with JSCPD"
echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-JSCPD"
fi
# No need to process this item furhter
continue continue
##################################################
# Include test cases if not running in test mode #
##################################################
elif [[ ${FILE} != *"${TEST_CASE_FOLDER}"* ]] && [ "${TEST_CASE_RUN}" == "true" ]; then
debug "TEST_CASE_RUN (${TEST_CASE_RUN}) is true. Skipping ${FILE}..."
fi fi
############################################### ###############################################
@ -238,20 +272,40 @@ function BuildFileList() {
continue continue
fi fi
# Editorconfig-checker should check every file # These linters check every file
FILE_ARRAY_EDITORCONFIG+=("${FILE}") local EDITORCONFIG_FILE_PATH
EDITORCONFIG_FILE_PATH="${GITHUB_WORKSPACE}/.editorconfig"
if [ -e "${EDITORCONFIG_FILE_PATH}" ]; then
echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-EDITORCONFIG"
else
debug "Don't include ${FILE} in the list of files to lint with editorconfig-checker because the workspace doesn't contain an EditorConfig file: ${EDITORCONFIG_FILE_PATH}"
fi
echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-GITLEAKS"
if [[ ("${FILE}" =~ .*${ANSIBLE_DIRECTORY}.*) ]] && [[ -d "${FILE}" ]]; then
echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-ANSIBLE"
fi
# Handle JSCPD test cases
# At this point, we already processed the options to include or exclude files, so we
# excluded test cases that are not relevant
if [[ "${TEST_CASE_RUN}" == "true" ]] && [[ "${FILE}" =~ .*${DEFAULT_JSCPD_TEST_CASE_DIRECTORY}.* ]] && [[ -d "${FILE}" ]]; then
debug "${FILE} is a test case for JSCPD. Adding it to the list of items to lint with JSCPD"
echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-JSCPD"
fi
# See https://docs.renovatebot.com/configuration-options/ # See https://docs.renovatebot.com/configuration-options/
if [[ "${BASE_FILE}" =~ renovate.json5? ]] || if [[ "${BASE_FILE}" =~ renovate.json5? ]] ||
[ "${BASE_FILE}" == ".renovaterc" ] || [[ "${BASE_FILE}" =~ .renovaterc.json5? ]]; then [ "${BASE_FILE}" == ".renovaterc" ] || [[ "${BASE_FILE}" =~ .renovaterc.json5? ]]; then
FILE_ARRAY_RENOVATE+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-RENOVATE"
fi fi
# See https://docs.renovatebot.com/config-presets/ # See https://docs.renovatebot.com/config-presets/
IFS="," read -r -a RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES_ARRAY <<<"${RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES}" IFS="," read -r -a RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES_ARRAY <<<"${RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES}"
for file_name in "${RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES_ARRAY[@]}"; do for file_name in "${RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES_ARRAY[@]}"; do
if [ "${BASE_FILE}" == "${file_name}" ]; then if [ "${BASE_FILE}" == "${file_name}" ]; then
FILE_ARRAY_RENOVATE+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-RENOVATE"
break break
fi fi
done done
@ -268,205 +322,99 @@ function BuildFileList() {
else else
debug "Considering ${FILE_DIR_NAME} as a Go module." debug "Considering ${FILE_DIR_NAME} as a Go module."
fi fi
FILE_ARRAY_GO_MODULES+=("${FILE_DIR_NAME}") echo "${FILE_DIR_NAME}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-GO_MODULES"
fi fi
#######################
# Get the shell files #
#######################
if IsValidShellScript "${FILE}"; then if IsValidShellScript "${FILE}"; then
FILE_ARRAY_BASH+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-BASH"
FILE_ARRAY_BASH_EXEC+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-BASH_EXEC"
FILE_ARRAY_SHELL_SHFMT+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-SHELL_SHFMT"
#########################
# Get the CLOJURE files #
#########################
elif [ "${FILE_TYPE}" == "clj" ] || [ "${FILE_TYPE}" == "cljs" ] || elif [ "${FILE_TYPE}" == "clj" ] || [ "${FILE_TYPE}" == "cljs" ] ||
[ "${FILE_TYPE}" == "cljc" ] || [ "${FILE_TYPE}" == "edn" ]; then [ "${FILE_TYPE}" == "cljc" ] || [ "${FILE_TYPE}" == "edn" ]; then
FILE_ARRAY_CLOJURE+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-CLOJURE"
#####################
# Get the C++ files #
#####################
elif [ "${FILE_TYPE}" == "cpp" ] || [ "${FILE_TYPE}" == "h" ] || elif [ "${FILE_TYPE}" == "cpp" ] || [ "${FILE_TYPE}" == "h" ] ||
[ "${FILE_TYPE}" == "cc" ] || [ "${FILE_TYPE}" == "hpp" ] || [ "${FILE_TYPE}" == "cc" ] || [ "${FILE_TYPE}" == "hpp" ] ||
[ "${FILE_TYPE}" == "cxx" ] || [ "${FILE_TYPE}" == "cu" ] || [ "${FILE_TYPE}" == "cxx" ] || [ "${FILE_TYPE}" == "cu" ] ||
[ "${FILE_TYPE}" == "hxx" ] || [ "${FILE_TYPE}" == "c++" ] || [ "${FILE_TYPE}" == "hxx" ] || [ "${FILE_TYPE}" == "c++" ] ||
[ "${FILE_TYPE}" == "hh" ] || [ "${FILE_TYPE}" == "h++" ] || [ "${FILE_TYPE}" == "hh" ] || [ "${FILE_TYPE}" == "h++" ] ||
[ "${FILE_TYPE}" == "cuh" ] || [ "${FILE_TYPE}" == "c" ]; then [ "${FILE_TYPE}" == "cuh" ] || [ "${FILE_TYPE}" == "c" ]; then
FILE_ARRAY_CPP+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-CPP"
FILE_ARRAY_CLANG_FORMAT+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-CLANG_FORMAT"
########################
# Get the COFFEE files #
########################
elif [ "${FILE_TYPE}" == "coffee" ]; then elif [ "${FILE_TYPE}" == "coffee" ]; then
FILE_ARRAY_COFFEESCRIPT+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-COFFEESCRIPT"
########################
# Get the CSHARP files #
########################
elif [ "${FILE_TYPE}" == "cs" ]; then elif [ "${FILE_TYPE}" == "cs" ]; then
FILE_ARRAY_CSHARP+=("${FILE}") FILE_ARRAY_CSHARP+=("${FILE}")
echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-CSHARP"
#####################
# Get the CSS files #
#####################
elif [ "${FILE_TYPE}" == "css" ] || [ "${FILE_TYPE}" == "scss" ] || elif [ "${FILE_TYPE}" == "css" ] || [ "${FILE_TYPE}" == "scss" ] ||
[ "${FILE_TYPE}" == "sass" ]; then [ "${FILE_TYPE}" == "sass" ]; then
FILE_ARRAY_CSS+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-CSS"
######################
# Get the DART files #
######################
elif [ "${FILE_TYPE}" == "dart" ]; then elif [ "${FILE_TYPE}" == "dart" ]; then
FILE_ARRAY_DART+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-DART"
########################
# Get the DOCKER files #
########################
# Use BASE_FILE here because FILE_TYPE is not reliable when there is no file extension # Use BASE_FILE here because FILE_TYPE is not reliable when there is no file extension
elif [[ "${FILE_TYPE}" != "tap" ]] && [[ "${FILE_TYPE}" != "yml" ]] && elif [[ "${FILE_TYPE}" != "tap" ]] && [[ "${FILE_TYPE}" != "yml" ]] &&
[[ "${FILE_TYPE}" != "yaml" ]] && [[ "${FILE_TYPE}" != "json" ]] && [[ "${FILE_TYPE}" != "yaml" ]] && [[ "${FILE_TYPE}" != "json" ]] &&
[[ "${FILE_TYPE}" != "xml" ]] && [[ "${FILE_TYPE}" != "xml" ]] &&
[[ "${BASE_FILE}" =~ ^(.+\.)?(contain|dock)erfile$ ]]; then [[ "${BASE_FILE}" =~ ^(.+\.)?(contain|dock)erfile$ ]]; then
FILE_ARRAY_DOCKERFILE_HADOLINT+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-DOCKERFILE_HADOLINT"
#####################
# Get the ENV files #
#####################
elif [ "${FILE_TYPE}" == "env" ] || [[ "${BASE_FILE}" == *".env."* ]]; then elif [ "${FILE_TYPE}" == "env" ] || [[ "${BASE_FILE}" == *".env."* ]]; then
FILE_ARRAY_ENV+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-ENV"
#########################
# Get the Gherkin files #
#########################
elif [ "${FILE_TYPE}" == "feature" ]; then elif [ "${FILE_TYPE}" == "feature" ]; then
FILE_ARRAY_GHERKIN+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-GHERKIN"
########################
# Get the Golang files #
########################
elif [ "${FILE_TYPE}" == "go" ]; then elif [ "${FILE_TYPE}" == "go" ]; then
FILE_ARRAY_GO+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-GO"
########################
# Get the GROOVY files #
########################
# Use BASE_FILE here because FILE_TYPE is not reliable when there is no file extension # Use BASE_FILE here because FILE_TYPE is not reliable when there is no file extension
elif [ "$FILE_TYPE" == "groovy" ] || [ "$FILE_TYPE" == "jenkinsfile" ] || elif [ "$FILE_TYPE" == "groovy" ] || [ "$FILE_TYPE" == "jenkinsfile" ] ||
[ "$FILE_TYPE" == "gradle" ] || [ "$FILE_TYPE" == "nf" ] || [ "$FILE_TYPE" == "gradle" ] || [ "$FILE_TYPE" == "nf" ] ||
[[ "$BASE_FILE" =~ .*jenkinsfile.* ]]; then [[ "$BASE_FILE" =~ .*jenkinsfile.* ]]; then
FILE_ARRAY_GROOVY+=("$FILE") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-GROOVY"
######################
# Get the HTML files #
######################
elif [ "${FILE_TYPE}" == "html" ]; then elif [ "${FILE_TYPE}" == "html" ]; then
FILE_ARRAY_HTML+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-HTML"
######################
# Get the Java files #
######################
elif [ "${FILE_TYPE}" == "java" ]; then elif [ "${FILE_TYPE}" == "java" ]; then
FILE_ARRAY_JAVA+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-JAVA"
FILE_ARRAY_GOOGLE_JAVA_FORMAT+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-GOOGLE_JAVA_FORMAT"
############################
# Get the JavaScript files #
############################
elif [ "${FILE_TYPE}" == "js" ]; then elif [ "${FILE_TYPE}" == "js" ]; then
FILE_ARRAY_JAVASCRIPT_ES+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-JAVASCRIPT_ES"
FILE_ARRAY_JAVASCRIPT_STANDARD+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-JAVASCRIPT_STANDARD"
FILE_ARRAY_JAVASCRIPT_PRETTIER+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-JAVASCRIPT_PRETTIER"
#######################
# Get the JSONC files #
#######################
elif [ "$FILE_TYPE" == "jsonc" ] || [ "$FILE_TYPE" == "json5" ]; then elif [ "$FILE_TYPE" == "jsonc" ] || [ "$FILE_TYPE" == "json5" ]; then
FILE_ARRAY_JSONC+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-JSONC"
######################
# Get the JSON files #
######################
elif [ "${FILE_TYPE}" == "json" ]; then elif [ "${FILE_TYPE}" == "json" ]; then
FILE_ARRAY_JSON+=("${FILE}") FILE_ARRAY_JSON+=("${FILE}")
echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-JSON"
############################
# Check if file is OpenAPI #
############################
if DetectOpenAPIFile "${FILE}"; then if DetectOpenAPIFile "${FILE}"; then
FILE_ARRAY_OPENAPI+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-OPENAPI"
fi fi
########################
# Check if file is ARM #
########################
if DetectARMFile "${FILE}"; then if DetectARMFile "${FILE}"; then
FILE_ARRAY_ARM+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-ARM"
fi fi
#####################################
# Check if the file is CFN template #
#####################################
if DetectCloudFormationFile "${FILE}"; then if DetectCloudFormationFile "${FILE}"; then
FILE_ARRAY_CLOUDFORMATION+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-CLOUDFORMATION"
fi fi
############################################
# Check if the file is AWS States Language #
############################################
if DetectAWSStatesFIle "${FILE}"; then if DetectAWSStatesFIle "${FILE}"; then
FILE_ARRAY_STATES+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-STATES"
fi fi
#####################
# Get the JSX files #
#####################
elif [ "${FILE_TYPE}" == "jsx" ]; then elif [ "${FILE_TYPE}" == "jsx" ]; then
FILE_ARRAY_JSX+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-JSX"
########################
# Get the KOTLIN files #
########################
elif [ "${FILE_TYPE}" == "kt" ] || [ "${FILE_TYPE}" == "kts" ]; then elif [ "${FILE_TYPE}" == "kt" ] || [ "${FILE_TYPE}" == "kts" ]; then
FILE_ARRAY_KOTLIN+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-KOTLIN"
#####################
# Get the LUA files #
#####################
elif [ "$FILE_TYPE" == "lua" ]; then elif [ "$FILE_TYPE" == "lua" ]; then
FILE_ARRAY_LUA+=("$FILE") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-LUA"
#######################
# Get the LaTeX files #
#######################
elif [ "${FILE_TYPE}" == "tex" ]; then elif [ "${FILE_TYPE}" == "tex" ]; then
FILE_ARRAY_LATEX+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-LATEX"
##########################
# Get the MARKDOWN files #
##########################
elif [ "${FILE_TYPE}" == "md" ]; then elif [ "${FILE_TYPE}" == "md" ]; then
FILE_ARRAY_MARKDOWN+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-MARKDOWN"
FILE_ARRAY_NATURAL_LANGUAGE+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-NATURAL_LANGUAGE"
######################
# Get the PHP files #
######################
elif [ "${FILE_TYPE}" == "php" ]; then elif [ "${FILE_TYPE}" == "php" ]; then
FILE_ARRAY_PHP_BUILTIN+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PHP_BUILTIN"
FILE_ARRAY_PHP_PHPCS+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PHP_PHPCS"
FILE_ARRAY_PHP_PHPSTAN+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PHP_PHPSTAN"
FILE_ARRAY_PHP_PSALM+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PHP_PSALM"
######################
# Get the PERL files #
######################
elif [ "${FILE_TYPE}" == "pl" ] || [ "${FILE_TYPE}" == "pm" ] || elif [ "${FILE_TYPE}" == "pl" ] || [ "${FILE_TYPE}" == "pm" ] ||
[ "${FILE_TYPE}" == "t" ]; then [ "${FILE_TYPE}" == "t" ]; then
FILE_ARRAY_PERL+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PERL"
############################
# Get the Powershell files #
############################
elif [ "${FILE_TYPE}" == "ps1" ] || elif [ "${FILE_TYPE}" == "ps1" ] ||
[ "${FILE_TYPE}" == "psm1" ] || [ "${FILE_TYPE}" == "psm1" ] ||
[ "${FILE_TYPE}" == "psd1" ] || [ "${FILE_TYPE}" == "psd1" ] ||
@ -474,179 +422,83 @@ function BuildFileList() {
[ "${FILE_TYPE}" == "pssc" ] || [ "${FILE_TYPE}" == "pssc" ] ||
[ "${FILE_TYPE}" == "psrc" ] || [ "${FILE_TYPE}" == "psrc" ] ||
[ "${FILE_TYPE}" == "cdxml" ]; then [ "${FILE_TYPE}" == "cdxml" ]; then
FILE_ARRAY_POWERSHELL+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-POWERSHELL"
#################################
# Get the PROTOCOL BUFFER files #
#################################
elif [ "${FILE_TYPE}" == "proto" ]; then elif [ "${FILE_TYPE}" == "proto" ]; then
FILE_ARRAY_PROTOBUF+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PROTOBUF"
########################
# Get the PYTHON files #
########################
elif [ "${FILE_TYPE}" == "py" ]; then elif [ "${FILE_TYPE}" == "py" ]; then
FILE_ARRAY_PYTHON_BLACK+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PYTHON_BLACK"
FILE_ARRAY_PYTHON_FLAKE8+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PYTHON_FLAKE8"
FILE_ARRAY_PYTHON_ISORT+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PYTHON_ISORT"
FILE_ARRAY_PYTHON_PYLINT+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PYTHON_PYLINT"
FILE_ARRAY_PYTHON_MYPY+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PYTHON_MYPY"
######################
# Get the RAKU files #
######################
elif [ "${FILE_TYPE}" == "raku" ] || [ "${FILE_TYPE}" == "rakumod" ] || elif [ "${FILE_TYPE}" == "raku" ] || [ "${FILE_TYPE}" == "rakumod" ] ||
[ "${FILE_TYPE}" == "rakutest" ] || [ "${FILE_TYPE}" == "pm6" ] || [ "${FILE_TYPE}" == "rakutest" ] || [ "${FILE_TYPE}" == "pm6" ] ||
[ "${FILE_TYPE}" == "pl6" ] || [ "${FILE_TYPE}" == "p6" ]; then [ "${FILE_TYPE}" == "pl6" ] || [ "${FILE_TYPE}" == "p6" ]; then
FILE_ARRAY_RAKU+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-RAKU"
####################
# Get the R files #
####################
elif [ "${FILE_TYPE}" == "r" ] || [ "${FILE_TYPE}" == "rmd" ]; then elif [ "${FILE_TYPE}" == "r" ] || [ "${FILE_TYPE}" == "rmd" ]; then
FILE_ARRAY_R+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-R"
######################
# Get the RUBY files #
######################
elif [ "${FILE_TYPE}" == "rb" ]; then elif [ "${FILE_TYPE}" == "rb" ]; then
FILE_ARRAY_RUBY+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-RUBY"
######################
# Get the RUST files #
######################
elif [ "${FILE_TYPE}" == "rs" ]; then elif [ "${FILE_TYPE}" == "rs" ]; then
FILE_ARRAY_RUST_2015+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-RUST_2015"
FILE_ARRAY_RUST_2018+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-RUST_2018"
FILE_ARRAY_RUST_2021+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-RUST_2021"
#######################
# Get the RUST crates #
#######################
elif [ "${BASE_FILE}" == "cargo.toml" ]; then elif [ "${BASE_FILE}" == "cargo.toml" ]; then
############################################### echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-RUST_CLIPPY"
# Append the crate manifest file to the array #
###############################################
FILE_ARRAY_RUST_CLIPPY+=("${FILE}")
###########################
# Get the SCALA files #
###########################
elif [ "${FILE_TYPE}" == "scala" ] || [ "${FILE_TYPE}" == "sc" ] || [ "${BASE_FILE}" == "??????" ]; then elif [ "${FILE_TYPE}" == "scala" ] || [ "${FILE_TYPE}" == "sc" ] || [ "${BASE_FILE}" == "??????" ]; then
FILE_ARRAY_SCALAFMT+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-SCALAFMT"
###########################
# Get the SNAKEMAKE files #
###########################
elif [ "${FILE_TYPE}" == "smk" ] || [ "${BASE_FILE}" == "snakefile" ]; then elif [ "${FILE_TYPE}" == "smk" ] || [ "${BASE_FILE}" == "snakefile" ]; then
FILE_ARRAY_SNAKEMAKE_LINT+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-SNAKEMAKE_LINT"
FILE_ARRAY_SNAKEMAKE_SNAKEFMT+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-SNAKEMAKE_SNAKEFMT"
#####################
# Get the SQL files #
#####################
elif [ "${FILE_TYPE}" == "sql" ]; then elif [ "${FILE_TYPE}" == "sql" ]; then
FILE_ARRAY_SQL+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-SQL"
FILE_ARRAY_SQLFLUFF+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-SQLFLUFF"
###########################
# Get the Terraform files #
###########################
elif [ "${FILE_TYPE}" == "tf" ]; then elif [ "${FILE_TYPE}" == "tf" ]; then
FILE_ARRAY_TERRAFORM_TFLINT+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TERRAFORM_TFLINT"
FILE_ARRAY_TERRAFORM_TERRASCAN+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TERRAFORM_TERRASCAN"
FILE_ARRAY_TERRAFORM_FMT+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TERRAFORM_FMT"
############################
# Get the Terragrunt files #
############################
elif [ "${FILE_TYPE}" == "hcl" ] && elif [ "${FILE_TYPE}" == "hcl" ] &&
[[ ${FILE} != *".tflint.hcl"* ]] && [[ ${FILE} != *".tflint.hcl"* ]] &&
[[ ${FILE} != *".pkr.hcl"* ]] && [[ ${FILE} != *".pkr.hcl"* ]] &&
[[ ${FILE} != *"docker-bake.hcl"* ]] && [[ ${FILE} != *"docker-bake.hcl"* ]] &&
[[ ${FILE} != *"docker-bake.override.hcl"* ]]; then [[ ${FILE} != *"docker-bake.override.hcl"* ]]; then
FILE_ARRAY_TERRAGRUNT+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TERRAGRUNT"
############################
# Get the TypeScript files #
############################
elif [ "${FILE_TYPE}" == "ts" ]; then elif [ "${FILE_TYPE}" == "ts" ]; then
FILE_ARRAY_TYPESCRIPT_ES+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TYPESCRIPT_ES"
FILE_ARRAY_TYPESCRIPT_STANDARD+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TYPESCRIPT_STANDARD"
FILE_ARRAY_TYPESCRIPT_PRETTIER+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TYPESCRIPT_PRETTIER"
#####################
# Get the TSX files #
#####################
elif [ "${FILE_TYPE}" == "tsx" ]; then elif [ "${FILE_TYPE}" == "tsx" ]; then
FILE_ARRAY_TSX+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TSX"
elif [ "${FILE_TYPE}" == "txt" ]; then elif [ "${FILE_TYPE}" == "txt" ]; then
FILE_ARRAY_NATURAL_LANGUAGE+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TXT"
#####################
# Get the XML files #
#####################
elif [ "${FILE_TYPE}" == "xml" ]; then elif [ "${FILE_TYPE}" == "xml" ]; then
FILE_ARRAY_XML+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-XML"
################################
# Get the CLOUDFORMATION files #
################################
elif [ "${FILE_TYPE}" == "yml" ] || [ "${FILE_TYPE}" == "yaml" ]; then elif [ "${FILE_TYPE}" == "yml" ] || [ "${FILE_TYPE}" == "yaml" ]; then
FILE_ARRAY_YAML+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-YAML"
###################################
# Check if file is GitHub Actions #
###################################
if DetectActions "${FILE}"; then if DetectActions "${FILE}"; then
FILE_ARRAY_GITHUB_ACTIONS+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-GITHUB_ACTIONS"
fi fi
#####################################
# Check if the file is CFN template #
#####################################
if DetectCloudFormationFile "${FILE}"; then if DetectCloudFormationFile "${FILE}"; then
FILE_ARRAY_CLOUDFORMATION+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-CLOUDFORMATION"
fi fi
############################
# Check if file is OpenAPI #
############################
if DetectOpenAPIFile "${FILE}"; then if DetectOpenAPIFile "${FILE}"; then
FILE_ARRAY_OPENAPI+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-OPENAPI"
fi fi
########################################
# Check if the file is Tekton template #
########################################
if DetectTektonFile "${FILE}"; then if DetectTektonFile "${FILE}"; then
FILE_ARRAY_TEKTON+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TEKTON"
fi fi
############################################
# Check if the file is Kubernetes template #
############################################
if DetectKubernetesFile "${FILE}"; then if DetectKubernetesFile "${FILE}"; then
FILE_ARRAY_KUBERNETES_KUBECONFORM+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-KUBERNETES_KUBECONFORM"
fi fi
########################################################################
# We have something that we need to try to check file type another way #
########################################################################
else else
##############################################
# Use file to see if we can parse what it is #
##############################################
CheckFileType "${FILE}" CheckFileType "${FILE}"
fi fi
##########################################
# Print line break after each file debug #
##########################################
debug ""
done done
################
# Footer print #
################
info "----------------------------------------------"
info "Successfully gathered list of files..."
} }
# We need this for parallel
export -f BuildFileArrays

View file

@ -139,21 +139,6 @@ DetectAWSStatesFIle() {
return 1 return 1
} }
CheckInArray() {
NEEDLE="$1" # Language we need to match
######################################
# Check if Language was in the array #
######################################
for LANG in "${UNIQUE_LINTED_ARRAY[@]}"; do
if [[ "${LANG}" == "${NEEDLE}" ]]; then
return 0
fi
done
return 1
}
function GetFileType() { function GetFileType() {
# Need to run the file through the 'file' exec to help determine # Need to run the file through the 'file' exec to help determine
# The type of file being parsed # The type of file being parsed
@ -168,21 +153,23 @@ function CheckFileType() {
# Need to run the file through the 'file' exec to help determine # Need to run the file through the 'file' exec to help determine
# The type of file being parsed # The type of file being parsed
local FILE
FILE="$1" FILE="$1"
local GET_FILE_TYPE_CMD
GET_FILE_TYPE_CMD="$(GetFileType "$FILE")" GET_FILE_TYPE_CMD="$(GetFileType "$FILE")"
local FILE_TYPE_MESSAGE local FILE_TYPE_MESSAGE
if [[ ${GET_FILE_TYPE_CMD} == *"Ruby script"* ]]; then if [[ ${GET_FILE_TYPE_CMD} == *"Ruby script"* ]]; then
FILE_TYPE_MESSAGE="Found Ruby script without extension (${FILE}). Rename the file with proper extension for Ruby files." FILE_TYPE_MESSAGE="Found Ruby script without extension (${FILE}). Rename the file with proper extension for Ruby files."
FILE_ARRAY_RUBY+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-RUBY"
elif [[ ${GET_FILE_TYPE_CMD} == *"Python script"* ]]; then elif [[ ${GET_FILE_TYPE_CMD} == *"Python script"* ]]; then
FILE_TYPE_MESSAGE="Found Python script without extension (${FILE}). Rename the file with proper extension for Python files." FILE_TYPE_MESSAGE="Found Python script without extension (${FILE}). Rename the file with proper extension for Python files."
FILE_ARRAY_PYTHON+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PYTHON"
elif [[ ${GET_FILE_TYPE_CMD} == *"Perl script"* ]]; then elif [[ ${GET_FILE_TYPE_CMD} == *"Perl script"* ]]; then
FILE_TYPE_MESSAGE="Found Perl script without extension (${FILE}). Rename the file with proper extension for Perl files." FILE_TYPE_MESSAGE="Found Perl script without extension (${FILE}). Rename the file with proper extension for Perl files."
FILE_ARRAY_PERL+=("${FILE}") echo "${FILE}" >>"${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PERL"
else else
FILE_TYPE_MESSAGE="Failed to get file type for: ${FILE}" FILE_TYPE_MESSAGE="Failed to get file type for: ${FILE}"
fi fi
@ -266,11 +253,30 @@ function IsGenerated() {
fi fi
} }
# We need these functions when building the file list with paralle
export -f CheckFileType
export -f DetectActions
export -f DetectARMFile
export -f DetectAWSStatesFIle
export -f DetectCloudFormationFile
export -f DetectKubernetesFile
export -f DetectOpenAPIFile
export -f DetectTektonFile
export -f GetFileExtension
export -f GetFileType
export -f IsValidShellScript
export -f IsGenerated
function RunAdditionalInstalls() { function RunAdditionalInstalls() {
if [ -z "${FILE_ARRAYS_DIRECTORY_PATH}" ] || [ ! -d "${FILE_ARRAYS_DIRECTORY_PATH}" ]; then
fatal "FILE_ARRAYS_DIRECTORY_PATH (set to ${FILE_ARRAYS_DIRECTORY_PATH}) is empty or doesn't exist"
fi
################################## ##################################
# Run installs for Psalm and PHP # # Run installs for Psalm and PHP #
################################## ##################################
if [ "${VALIDATE_PHP_PSALM}" == "true" ] && [ "${#FILE_ARRAY_PHP_PSALM[@]}" -ne 0 ]; then if [ "${VALIDATE_PHP_PSALM}" == "true" ] && [ -e "${FILE_ARRAYS_DIRECTORY_PATH}/file-array-PHP_PSALM" ]; then
# found PHP files and were validating it, need to composer install # found PHP files and were validating it, need to composer install
info "Found PHP files to validate, and [VALIDATE_PHP_PSALM] set to true, need to run composer install" info "Found PHP files to validate, and [VALIDATE_PHP_PSALM] set to true, need to run composer install"
info "looking for composer.json in the users repository..." info "looking for composer.json in the users repository..."
@ -311,13 +317,13 @@ function RunAdditionalInstalls() {
############################### ###############################
# Run installs for R language # # Run installs for R language #
############################### ###############################
if [ "${VALIDATE_R}" == "true" ] && [ "${#FILE_ARRAY_R[@]}" -ne 0 ]; then if [ "${VALIDATE_R}" == "true" ] && [ -e "${FILE_ARRAYS_DIRECTORY_PATH}/file-array-R" ]; then
info "Detected R Language files to lint." info "Detected R Language files to lint."
info "Trying to install the R package inside:[${WORKSPACE_PATH}]" info "Trying to install the R package inside:[${GITHUB_WORKSPACE}]"
######################### #########################
# Run the build command # # Run the build command #
######################### #########################
BUILD_CMD=$(R CMD build "${WORKSPACE_PATH}" 2>&1) BUILD_CMD=$(R CMD build "${GITHUB_WORKSPACE}" 2>&1)
############## ##############
# Error code # # Error code #
@ -329,19 +335,19 @@ function RunAdditionalInstalls() {
############################## ##############################
if [ "${ERROR_CODE}" -ne 0 ]; then if [ "${ERROR_CODE}" -ne 0 ]; then
# Error # Error
warn "ERROR! Failed to run:[R CMD build] at location:[${WORKSPACE_PATH}]" warn "ERROR! Failed to run:[R CMD build] at location:[${GITHUB_WORKSPACE}]"
warn "BUILD_CMD:[${BUILD_CMD}]" warn "BUILD_CMD:[${BUILD_CMD}]"
else else
# Get the build package # Get the build package
BUILD_PKG=$( BUILD_PKG=$(
cd "${WORKSPACE_PATH}" || exit 0 cd "${GITHUB_WORKSPACE}" || exit 0
echo *.tar.gz 2>&1 echo *.tar.gz 2>&1
) )
############################## ##############################
# Install the build packages # # Install the build packages #
############################## ##############################
INSTALL_CMD=$( INSTALL_CMD=$(
cd "${WORKSPACE_PATH}" || exit 0 cd "${GITHUB_WORKSPACE}" || exit 0
R -e "remotes::install_local('.', dependencies=T)" 2>&1 R -e "remotes::install_local('.', dependencies=T)" 2>&1
) )
@ -358,19 +364,26 @@ function RunAdditionalInstalls() {
warn "ERROR: Failed to install the build package at:[${BUILD_PKG}]" warn "ERROR: Failed to install the build package at:[${BUILD_PKG}]"
fi fi
fi fi
if [ ! -f "${R_RULES_FILE_PATH_IN_ROOT}" ]; then
info "No .lintr configuration file found, using defaults."
cp "$R_LINTER_RULES" "$GITHUB_WORKSPACE"
# shellcheck disable=SC2034
SUPER_LINTER_COPIED_R_LINTER_RULES_FILE="true"
fi
fi fi
#################################### ####################################
# Run installs for TFLINT language # # Run installs for TFLINT language #
#################################### ####################################
if [ "${VALIDATE_TERRAFORM_TFLINT}" == "true" ] && [ "${#FILE_ARRAY_TERRAFORM_TFLINT[@]}" -ne 0 ]; then if [ "${VALIDATE_TERRAFORM_TFLINT}" == "true" ] && [ -e "${FILE_ARRAYS_DIRECTORY_PATH}/file-array-TERRAFORM_TFLINT" ]; then
info "Detected TFLint Language files to lint." info "Detected TFLint Language files to lint."
info "Trying to install the TFLint init inside:[${WORKSPACE_PATH}]" info "Trying to install the TFLint init inside:[${GITHUB_WORKSPACE}]"
######################### #########################
# Run the build command # # Run the build command #
######################### #########################
BUILD_CMD=$( BUILD_CMD=$(
cd "${WORKSPACE_PATH}" || exit 0 cd "${GITHUB_WORKSPACE}" || exit 0
tflint --init -c "${TERRAFORM_TFLINT_LINTER_RULES}" 2>&1 tflint --init -c "${TERRAFORM_TFLINT_LINTER_RULES}" 2>&1
) )
@ -388,5 +401,44 @@ function RunAdditionalInstalls() {
info "Successfully initialized tflint with the ${TERRAFORM_TFLINT_LINTER_RULES} config file" info "Successfully initialized tflint with the ${TERRAFORM_TFLINT_LINTER_RULES} config file"
debug "Tflint output: ${BUILD_CMD}" debug "Tflint output: ${BUILD_CMD}"
fi fi
# Array to track directories where tflint was run
local -A TFLINT_SEEN_DIRS
TFLINT_SEEN_DIRS=()
for FILE in "${FILE_ARRAY_TERRAFORM_TFLINT[@]}"; do
local DIR_NAME
DIR_NAME=$(dirname "${FILE}" 2>&1)
debug "DIR_NAME for ${FILE}: ${DIR_NAME}"
# Check the cache to see if we've already prepped this directory for tflint
if [[ ! -v "TFLINT_SEEN_DIRS[${DIR_NAME}]" ]]; then
debug "Configuring Terraform data directory for ${DIR_NAME}"
# Define the path to an empty Terraform data directory
# (def: https://developer.hashicorp.com/terraform/cli/config/environment-variables#tf_data_dir)
# in case the user has a Terraform data directory already, and we don't
# want to modify it.
# TFlint considers this variable as well.
# Ref: https://github.com/terraform-linters/tflint/blob/master/docs/user-guide/compatibility.md#environment-variables
TF_DATA_DIR="/tmp/.terraform-${TERRAFORM_TFLINT}-${DIR_NAME}"
# Fetch Terraform modules
debug "Fetch Terraform modules for ${FILE} in ${DIR_NAME} in ${TF_DATA_DIR}"
local FETCH_TERRAFORM_MODULES_CMD
if ! FETCH_TERRAFORM_MODULES_CMD="$(terraform get)"; then
fatal "Error when fetching Terraform modules while linting ${FILE}. Command output: ${FETCH_TERRAFORM_MODULES_CMD}"
fi
debug "Fetch Terraform modules command for ${FILE} output: ${FETCH_TERRAFORM_MODULES_CMD}"
# Let the cache know we've seen this before
# Set the value to an arbitrary non-empty string.
TFLINT_SEEN_DIRS[${DIR_NAME}]="false"
else
debug "Skip fetching Terraform modules for ${FILE} because we already did that for ${DIR_NAME}"
fi
done
fi
# Check if there's local configuration for the Raku linter
if [ -e "${GITHUB_WORKSPACE}/META6.json" ]; then
cd "${GITHUB_WORKSPACE}" && zef install --deps-only --/test .
fi fi
} }

124
lib/functions/linterCommands.sh Executable file
View file

@ -0,0 +1,124 @@
#!/usr/bin/env bash
##########################
# Define linter commands #
##########################
# If there's no input argument, parallel adds a default {} at the end of the command.
# In a few cases, such as ANSIBLE and GO_MODULES,
# Consume the input before running the command because we need the input
# to set the working directory, but we don't need it appended at the end of the command.
# Setting -n 0 would not help in this case, because the input will not be passed
# to the --workdir option as well.
# shellcheck disable=SC2034 # Variable is referenced in other scripts
LINTER_COMMANDS_ARRAY_ANSIBLE=(ansible-lint -c "${ANSIBLE_LINTER_RULES}" "&& echo \"Linted: {}\"")
LINTER_COMMANDS_ARRAY_ARM=(pwsh -NoProfile -NoLogo -Command "\"Import-Module ${ARM_TTK_PSD1} ; \\\${config} = \\\$(Import-PowerShellDataFile -Path ${ARM_LINTER_RULES}) ; Test-AzTemplate @config -TemplatePath '{}'; if (\\\${Error}.Count) { exit 1 }\"")
LINTER_COMMANDS_ARRAY_BASH=(shellcheck --color --external-sources)
if [ -n "${BASH_SEVERITY}" ]; then
LINTER_COMMANDS_ARRAY_BASH+=(--severity="${BASH_SEVERITY}")
fi
LINTER_COMMANDS_ARRAY_BASH_EXEC=(bash-exec)
LINTER_COMMANDS_ARRAY_CHECKOV=(checkov --config-file "${CHECKOV_LINTER_RULES}")
if CheckovConfigurationFileContainsDirectoryOption "${CHECKOV_LINTER_RULES}"; then
# Consume the input as we do with ANSIBLE
debug "Consume the input of the Checkov command because we don't need to add it as an argument."
LINTER_COMMANDS_ARRAY_CHECKOV+=("&& echo \"Got the list of directories to lint from the configuration file: {}\"")
else
debug "Adding the '--directory' option to the Checkov command."
LINTER_COMMANDS_ARRAY_CHECKOV+=(--directory)
fi
LINTER_COMMANDS_ARRAY_CLANG_FORMAT=(clang-format --Werror --dry-run)
LINTER_COMMANDS_ARRAY_CLOJURE=(clj-kondo --config "${CLOJURE_LINTER_RULES}" --lint)
LINTER_COMMANDS_ARRAY_CLOUDFORMATION=(cfn-lint --config-file "${CLOUDFORMATION_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_COFFEESCRIPT=(coffeelint -f "${COFFEESCRIPT_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_CPP=(cpplint)
LINTER_COMMANDS_ARRAY_CSHARP=(dotnet format whitespace --folder --verify-no-changes --exclude / --include "{/}")
LINTER_COMMANDS_ARRAY_CSS=(stylelint --config "${CSS_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_DART=(dart analyze --fatal-infos --fatal-warnings)
LINTER_COMMANDS_ARRAY_DOCKERFILE_HADOLINT=(hadolint -c "${DOCKERFILE_HADOLINT_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_EDITORCONFIG=(editorconfig-checker -config "${EDITORCONFIG_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_ENV=(dotenv-linter)
LINTER_COMMANDS_ARRAY_GITHUB_ACTIONS=(actionlint -config-file "${GITHUB_ACTIONS_LINTER_RULES}")
if [ "${GITHUB_ACTIONS_COMMAND_ARGS}" != "null" ] && [ -n "${GITHUB_ACTIONS_COMMAND_ARGS}" ]; then
LINTER_COMMANDS_ARRAY_GITHUB_ACTIONS+=("${GITHUB_ACTIONS_COMMAND_ARGS}")
fi
LINTER_COMMANDS_ARRAY_GITLEAKS=(gitleaks detect --no-banner --no-git --redact --config "${GITLEAKS_LINTER_RULES}" --verbose --source)
LINTER_COMMANDS_ARRAY_GHERKIN=(gherkin-lint -c "${GHERKIN_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_GO=(golangci-lint run -c "${GO_LINTER_RULES}" --fast)
# Consume the input as we do with ANSIBLE
LINTER_COMMANDS_ARRAY_GO_MODULES=(golangci-lint run --allow-parallel-runners -c "${GO_LINTER_RULES}" "&& echo \"Linted: {}\"")
LINTER_COMMANDS_ARRAY_GOOGLE_JAVA_FORMAT=(java -jar /usr/bin/google-java-format --dry-run --set-exit-if-changed)
LINTER_COMMANDS_ARRAY_GROOVY=(npm-groovy-lint -c "${GROOVY_LINTER_RULES}" --failon warning --no-insight)
LINTER_COMMANDS_ARRAY_HTML=(htmlhint --config "${HTML_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_JAVA=(java -jar /usr/bin/checkstyle -c "${JAVA_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_JAVASCRIPT_ES=(eslint --no-eslintrc -c "${JAVASCRIPT_ES_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_JAVASCRIPT_STANDARD=(standard "${JAVASCRIPT_STANDARD_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_JAVASCRIPT_PRETTIER=(prettier --check)
LINTER_COMMANDS_ARRAY_JSCPD=(jscpd --config "${JSCPD_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_JSON=(eslint --no-eslintrc -c "${JAVASCRIPT_ES_LINTER_RULES}" --ext '.json')
LINTER_COMMANDS_ARRAY_JSONC=(eslint --no-eslintrc -c "${JAVASCRIPT_ES_LINTER_RULES}" --ext '.json5,.jsonc')
LINTER_COMMANDS_ARRAY_JSX=(eslint --no-eslintrc -c "${JSX_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_KOTLIN=(ktlint "{/}")
LINTER_COMMANDS_ARRAY_KUBERNETES_KUBECONFORM=(kubeconform -strict)
if [ "${KUBERNETES_KUBECONFORM_OPTIONS}" != "null" ] && [ -n "${KUBERNETES_KUBECONFORM_OPTIONS}" ]; then
LINTER_COMMANDS_ARRAY_KUBERNETES_KUBECONFORM+=("${KUBERNETES_KUBECONFORM_OPTIONS}")
fi
LINTER_COMMANDS_ARRAY_LATEX=(chktex -q -l "${LATEX_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_LUA=(luacheck --config "${LUA_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_MARKDOWN=(markdownlint -c "${MARKDOWN_LINTER_RULES}")
if [ -n "${MARKDOWN_CUSTOM_RULE_GLOBS}" ]; then
IFS="," read -r -a MARKDOWN_CUSTOM_RULE_GLOBS_ARRAY <<<"${MARKDOWN_CUSTOM_RULE_GLOBS}"
for glob in "${MARKDOWN_CUSTOM_RULE_GLOBS_ARRAY[@]}"; do
if [ -z "${LINTER_RULES_PATH}" ]; then
LINTER_COMMANDS_ARRAY_MARKDOWN+=(-r "${GITHUB_WORKSPACE}/${glob}")
else
LINTER_COMMANDS_ARRAY_MARKDOWN+=(-r "${GITHUB_WORKSPACE}/${LINTER_RULES_PATH}/${glob}")
fi
done
fi
LINTER_COMMANDS_ARRAY_NATURAL_LANGUAGE=(textlint -c "${NATURAL_LANGUAGE_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_OPENAPI=(spectral lint -r "${OPENAPI_LINTER_RULES}" -D)
LINTER_COMMANDS_ARRAY_PERL=(perlcritic)
if [ "${PERL_PERLCRITIC_OPTIONS}" != "null" ] && [ -n "${PERL_PERLCRITIC_OPTIONS}" ]; then
LINTER_COMMANDS_ARRAY_PERL+=("${PERL_PERLCRITIC_OPTIONS}")
fi
LINTER_COMMANDS_ARRAY_PHP_BUILTIN=(php -l -c "${PHP_BUILTIN_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_PHP_PHPCS=(phpcs --standard="${PHP_PHPCS_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_PHP_PHPSTAN=(phpstan analyse --no-progress --no-ansi --memory-limit 1G -c "${PHP_PHPSTAN_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_PHP_PSALM=(psalm --config="${PHP_PSALM_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_POWERSHELL=(pwsh -NoProfile -NoLogo -Command "\"Invoke-ScriptAnalyzer -EnableExit -Settings ${POWERSHELL_LINTER_RULES} -Path '{}'; if (\\\${Error}.Count) { exit 1 }\"")
LINTER_COMMANDS_ARRAY_PROTOBUF=(protolint lint --config_path "${PROTOBUF_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_PYTHON_BLACK=(black --config "${PYTHON_BLACK_LINTER_RULES}" --diff --check)
LINTER_COMMANDS_ARRAY_PYTHON_PYLINT=(pylint --rcfile "${PYTHON_PYLINT_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_PYTHON_FLAKE8=(flake8 --config="${PYTHON_FLAKE8_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_PYTHON_ISORT=(isort --check --diff --sp "${PYTHON_ISORT_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_PYTHON_MYPY=(mypy --config-file "${PYTHON_MYPY_LINTER_RULES}" --install-types --non-interactive)
LINTER_COMMANDS_ARRAY_R=(R --slave -e "\"lints <- lintr::lint('{}');print(lints);errors <- purrr::keep(lints, ~ .\\\$type == 'error');quit(save = 'no', status = if (length(errors) > 0) 1 else 0)\"")
LINTER_COMMANDS_ARRAY_RAKU=(raku)
LINTER_COMMANDS_ARRAY_RENOVATE=(renovate-config-validator --strict)
LINTER_COMMANDS_ARRAY_RUBY=(rubocop -c "${RUBY_LINTER_RULES}" --force-exclusion --ignore-unrecognized-cops)
LINTER_COMMANDS_ARRAY_RUST_2015=(rustfmt --check --edition 2015)
LINTER_COMMANDS_ARRAY_RUST_2018=(rustfmt --check --edition 2018)
LINTER_COMMANDS_ARRAY_RUST_2021=(rustfmt --check --edition 2021)
LINTER_COMMANDS_ARRAY_RUST_CLIPPY=(clippy)
LINTER_COMMANDS_ARRAY_SCALAFMT=(scalafmt --config "${SCALAFMT_LINTER_RULES}" --test)
LINTER_COMMANDS_ARRAY_SHELL_SHFMT=(shfmt -d)
LINTER_COMMANDS_ARRAY_SNAKEMAKE_LINT=(snakemake --lint -s)
LINTER_COMMANDS_ARRAY_SNAKEMAKE_SNAKEFMT=(snakefmt --config "${SNAKEMAKE_SNAKEFMT_LINTER_RULES}" --check --compact-diff)
LINTER_COMMANDS_ARRAY_STATES=(asl-validator --json-path)
LINTER_COMMANDS_ARRAY_SQL=(sql-lint --config "${SQL_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_SQLFLUFF=(sqlfluff lint --config "${SQLFLUFF_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_TEKTON=(tekton-lint)
LINTER_COMMANDS_ARRAY_TERRAFORM_FMT=(terraform fmt -check -diff)
LINTER_COMMANDS_ARRAY_TERRAFORM_TFLINT=("TF_DATA_DIR=\"/tmp/.terraform-TERRAFORM_TFLINT-{//}\"" tflint -c "${TERRAFORM_TFLINT_LINTER_RULES}" "--filter=\"{/}\"")
LINTER_COMMANDS_ARRAY_TERRAFORM_TERRASCAN=(terrascan scan -i terraform -t all -c "${TERRAFORM_TERRASCAN_LINTER_RULES}" -f)
LINTER_COMMANDS_ARRAY_TERRAGRUNT=(terragrunt hclfmt --terragrunt-check --terragrunt-log-level error --terragrunt-hclfmt-file)
LINTER_COMMANDS_ARRAY_TSX=(eslint --no-eslintrc -c "${TSX_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_TYPESCRIPT_ES=(eslint --no-eslintrc -c "${TYPESCRIPT_ES_LINTER_RULES}")
LINTER_COMMANDS_ARRAY_TYPESCRIPT_STANDARD=(ts-standard --parser @typescript-eslint/parser --plugin @typescript-eslint/eslint-plugin --project "${TYPESCRIPT_STANDARD_TSCONFIG_FILE}")
LINTER_COMMANDS_ARRAY_TYPESCRIPT_PRETTIER=(prettier --check)
LINTER_COMMANDS_ARRAY_XML=(xmllint)
LINTER_COMMANDS_ARRAY_YAML=(yamllint -c "${YAML_LINTER_RULES}" -f parsable)
if [ "${YAML_ERROR_ON_WARNING}" == 'true' ]; then
LINTER_COMMANDS_ARRAY_YAML+=(--strict)
fi

View file

@ -98,8 +98,11 @@ fatal() {
# shellcheck disable=SC2034 # Variable is referenced in other files # shellcheck disable=SC2034 # Variable is referenced in other files
SUPER_LINTER_INITIALIZATION_LOG_GROUP_TITLE="Super-Linter initialization" SUPER_LINTER_INITIALIZATION_LOG_GROUP_TITLE="Super-Linter initialization"
export SUPER_LINTER_INITIALIZATION_LOG_GROUP_TITLE
GITHUB_ACTIONS_LOG_GROUP_MARKER_START="start" GITHUB_ACTIONS_LOG_GROUP_MARKER_START="start"
export GITHUB_ACTIONS_LOG_GROUP_MARKER_START
GITHUB_ACTIONS_LOG_GROUP_MARKER_END="end" GITHUB_ACTIONS_LOG_GROUP_MARKER_END="end"
export GITHUB_ACTIONS_LOG_GROUP_MARKER_END
writeGitHubActionsLogGroupMarker() { writeGitHubActionsLogGroupMarker() {
local LOG_GROUP_MARKER_MODE="${1}" local LOG_GROUP_MARKER_MODE="${1}"
@ -139,3 +142,16 @@ startGitHubActionsLogGroup() {
endGitHubActionsLogGroup() { endGitHubActionsLogGroup() {
writeGitHubActionsLogGroupMarker "${GITHUB_ACTIONS_LOG_GROUP_MARKER_END}" "${1}" writeGitHubActionsLogGroupMarker "${GITHUB_ACTIONS_LOG_GROUP_MARKER_END}" "${1}"
} }
# We need these functions to be available when using parallel to run subprocesses
export -f debug
export -f endGitHubActionsLogGroup
export -f error
export -f fatal
export -f info
export -f log
export -f notice
export -f startGitHubActionsLogGroup
export -f trace
export -f warn
export -f writeGitHubActionsLogGroupMarker

View file

@ -13,11 +13,11 @@ function ValidateBooleanConfigurationVariables() {
ValidateBooleanVariable "SSH_SETUP_GITHUB" "${SSH_SETUP_GITHUB}" ValidateBooleanVariable "SSH_SETUP_GITHUB" "${SSH_SETUP_GITHUB}"
ValidateBooleanVariable "SUPPRESS_FILE_TYPE_WARN" "${SUPPRESS_FILE_TYPE_WARN}" ValidateBooleanVariable "SUPPRESS_FILE_TYPE_WARN" "${SUPPRESS_FILE_TYPE_WARN}"
ValidateBooleanVariable "SUPPRESS_POSSUM" "${SUPPRESS_POSSUM}" ValidateBooleanVariable "SUPPRESS_POSSUM" "${SUPPRESS_POSSUM}"
ValidateBooleanVariable "USE_FIND_ALGORITHM" "${USE_FIND_ALGORITHM}"
ValidateBooleanVariable "TEST_CASE_RUN" "${TEST_CASE_RUN}" ValidateBooleanVariable "TEST_CASE_RUN" "${TEST_CASE_RUN}"
ValidateBooleanVariable "USE_FIND_ALGORITHM" "${USE_FIND_ALGORITHM}"
ValidateBooleanVariable "VALIDATE_ALL_CODEBASE" "${VALIDATE_ALL_CODEBASE}" ValidateBooleanVariable "VALIDATE_ALL_CODEBASE" "${VALIDATE_ALL_CODEBASE}"
ValidateBooleanVariable "YAML_ERROR_ON_WARNING" "${YAML_ERROR_ON_WARNING}"
ValidateBooleanVariable "WRITE_LINTER_VERSIONS_FILE" "${WRITE_LINTER_VERSIONS_FILE}" ValidateBooleanVariable "WRITE_LINTER_VERSIONS_FILE" "${WRITE_LINTER_VERSIONS_FILE}"
ValidateBooleanVariable "YAML_ERROR_ON_WARNING" "${YAML_ERROR_ON_WARNING}"
} }
function ValidateGitHubWorkspace() { function ValidateGitHubWorkspace() {
@ -100,14 +100,6 @@ function GetValidationInfo() {
VALIDATE_LANGUAGE="VALIDATE_${LANGUAGE}" VALIDATE_LANGUAGE="VALIDATE_${LANGUAGE}"
if [[ ${!VALIDATE_LANGUAGE} == "true" ]]; then if [[ ${!VALIDATE_LANGUAGE} == "true" ]]; then
debug "- Validating [${LANGUAGE}] files in code base..." debug "- Validating [${LANGUAGE}] files in code base..."
debug "Defining variables for ${LANGUAGE} linter..."
ERRORS_VARIABLE_NAME="ERRORS_FOUND_${LANGUAGE}"
debug "Setting ${ERRORS_VARIABLE_NAME} variable value to 0..."
eval "${ERRORS_VARIABLE_NAME}=0"
debug "Exporting ${ERRORS_VARIABLE_NAME} variable..."
eval "export ${ERRORS_VARIABLE_NAME}"
else else
debug "- Excluding [$LANGUAGE] files in code base..." debug "- Excluding [$LANGUAGE] files in code base..."
fi fi
@ -139,10 +131,6 @@ function GetValidationInfo() {
ANSIBLE_DIRECTORY="${TEMP_ANSIBLE_DIRECTORY}" ANSIBLE_DIRECTORY="${TEMP_ANSIBLE_DIRECTORY}"
debug "Setting Ansible directory to: ${ANSIBLE_DIRECTORY}" debug "Setting Ansible directory to: ${ANSIBLE_DIRECTORY}"
fi fi
debug "Runner: $(id -un 2>/dev/null)"
debug "ENV:"
debug "$(printenv | sort)"
} }
function CheckIfGitBranchExists() { function CheckIfGitBranchExists() {
@ -170,6 +158,7 @@ function ValidateBooleanVariable() {
debug "${VAR_NAME} has a valid boolean string value: ${VAR_VALUE}" debug "${VAR_NAME} has a valid boolean string value: ${VAR_VALUE}"
fi fi
} }
export -f ValidateBooleanVariable
function ValidateLocalGitRepository() { function ValidateLocalGitRepository() {
debug "Check if ${GITHUB_WORKSPACE} is a Git repository" debug "Check if ${GITHUB_WORKSPACE} is a Git repository"
@ -251,6 +240,10 @@ function CheckovConfigurationFileContainsDirectoryOption() {
local CONFIGURATION_OPTION_KEY="directory:" local CONFIGURATION_OPTION_KEY="directory:"
debug "Checking if ${CHECKOV_LINTER_RULES_PATH} contains a '${CONFIGURATION_OPTION_KEY}' configuration option" debug "Checking if ${CHECKOV_LINTER_RULES_PATH} contains a '${CONFIGURATION_OPTION_KEY}' configuration option"
if [ ! -e "${CHECKOV_LINTER_RULES_PATH}" ]; then
fatal "${CHECKOV_LINTER_RULES_PATH} doesn't exist. Cannot check if it contains a '${CONFIGURATION_OPTION_KEY}' configuration option"
fi
if grep -q "${CONFIGURATION_OPTION_KEY}" "${CHECKOV_LINTER_RULES_PATH}"; then if grep -q "${CONFIGURATION_OPTION_KEY}" "${CHECKOV_LINTER_RULES_PATH}"; then
debug "${CHECKOV_LINTER_RULES_PATH} contains a '${CONFIGURATION_OPTION_KEY}' statement" debug "${CHECKOV_LINTER_RULES_PATH} contains a '${CONFIGURATION_OPTION_KEY}' statement"
return 0 return 0
@ -259,6 +252,7 @@ function CheckovConfigurationFileContainsDirectoryOption() {
return 1 return 1
fi fi
} }
export -f CheckovConfigurationFileContainsDirectoryOption
function WarnIfVariableIsSet() { function WarnIfVariableIsSet() {
local INPUT_VARIABLE="${1}" local INPUT_VARIABLE="${1}"

View file

@ -1,213 +1,218 @@
#!/usr/bin/env bash #!/usr/bin/env bash
function LintCodebase() { function LintCodebase() {
local FILE_TYPE
FILE_TYPE="${1}" && shift FILE_TYPE="${1}" && shift
LINTER_NAME="${1}" && shift local TEST_CASE_RUN
LINTER_COMMAND="${1}" && shift
FILTER_REGEX_INCLUDE="${1}" && shift
FILTER_REGEX_EXCLUDE="${1}" && shift
TEST_CASE_RUN="${1}" && shift TEST_CASE_RUN="${1}" && shift
FILE_ARRAY=("$@")
# Array to track directories where tflint was run declare -n VALIDATE_LANGUAGE
declare -A TFLINT_SEEN_DIRS VALIDATE_LANGUAGE="VALIDATE_${FILE_TYPE}"
# To count how many files were checked for a given FILE_TYPE if [[ "${VALIDATE_LANGUAGE}" == "false" ]]; then
INDEX=0 if [[ "${TEST_CASE_RUN}" == "false" ]]; then
debug "Skip validation of ${FILE_TYPE} because VALIDATE_LANGUAGE is ${VALIDATE_LANGUAGE}"
# To check how many "bad" and "good" test cases we ran unset -n VALIDATE_LANGUAGE
BAD_TEST_CASES_COUNT=0 return 0
GOOD_TEST_CASES_COUNT=0 else
fatal "Don't disable any validation when running in test mode. VALIDATE_${FILE_TYPE} is set to: ${VALIDATE_LANGUAGE}. Set it to: true"
WORKSPACE_PATH="${GITHUB_WORKSPACE}" fi
if [ "${TEST_CASE_RUN}" == "true" ]; then
WORKSPACE_PATH="${GITHUB_WORKSPACE}/${TEST_CASE_FOLDER}"
fi fi
debug "Workspace path: ${WORKSPACE_PATH}"
info "" debug "Running LintCodebase. FILE_TYPE: ${FILE_TYPE}. TEST_CASE_RUN: ${TEST_CASE_RUN}"
info "----------------------------------------------"
info "----------------------------------------------"
debug "Running LintCodebase. FILE_TYPE: ${FILE_TYPE}. Linter name: ${LINTER_NAME}, linter command: ${LINTER_COMMAND}, TEST_CASE_RUN: ${TEST_CASE_RUN}, FILTER_REGEX_INCLUDE: ${FILTER_REGEX_INCLUDE}, FILTER_REGEX_EXCLUDE: ${FILTER_REGEX_EXCLUDE}, files to lint: ${FILE_ARRAY[*]}"
info "Linting ${FILE_TYPE} files..."
info "----------------------------------------------"
info "----------------------------------------------"
for FILE in "${FILE_ARRAY[@]}"; do debug "VALIDATE_LANGUAGE for ${FILE_TYPE}: ${VALIDATE_LANGUAGE}..."
info "Checking file: ${FILE}"
ValidateBooleanVariable "TEST_CASE_RUN" "${TEST_CASE_RUN}"
ValidateBooleanVariable "VALIDATE_${FILE_TYPE}" "${VALIDATE_LANGUAGE}"
unset -n VALIDATE_LANGUAGE
debug "Populating file array for ${FILE_TYPE}"
local -n FILE_ARRAY="FILE_ARRAY_${FILE_TYPE}"
local FILE_ARRAY_LANGUAGE_PATH="${FILE_ARRAYS_DIRECTORY_PATH}/file-array-${FILE_TYPE}"
if [[ -e "${FILE_ARRAY_LANGUAGE_PATH}" ]]; then
while read -r FILE; do
if [[ "${TEST_CASE_RUN}" == "true" ]]; then if [[ "${TEST_CASE_RUN}" == "true" ]]; then
debug "Ensure that the list files to check for ${FILE_TYPE} doesn't include test cases for other languages"
# Folder for specific tests. By convention, the last part of the path is the lowercased FILE_TYPE # Folder for specific tests. By convention, the last part of the path is the lowercased FILE_TYPE
local TEST_CASE_DIRECTORY local TEST_CASE_DIRECTORY
TEST_CASE_DIRECTORY="${TEST_CASE_FOLDER}/${FILE_TYPE,,}/" TEST_CASE_DIRECTORY="${FILE_TYPE,,}"
debug "TEST_CASE_DIRECTORY for ${FILE}: ${TEST_CASE_DIRECTORY}"
# We use configuration files to pass the list of files to lint to checkov
# Their name includes "checkov", which is equal to FILE_TYPE for Checkov.
# In this case, we don't add a trailing slash so we don't fail validation.
if [[ "${FILE_TYPE}" != "CHECKOV" ]]; then
TEST_CASE_DIRECTORY="${TEST_CASE_DIRECTORY}/"
debug "Adding a traling slash to the test case directory for ${FILE_TYPE}: ${TEST_CASE_DIRECTORY}"
fi
debug "TEST_CASE_DIRECTORY for ${FILE_TYPE}: ${TEST_CASE_DIRECTORY}"
if [[ ${FILE} != *"${TEST_CASE_DIRECTORY}"* ]]; then if [[ ${FILE} != *"${TEST_CASE_DIRECTORY}"* ]]; then
debug "Skipping ${FILE} because it's not in the test case directory for ${FILE_TYPE}..." debug "Excluding ${FILE} because it's not in the test case directory for ${FILE_TYPE}..."
continue continue
fi
fi
local FILE_NAME
FILE_NAME=$(basename "${FILE}" 2>&1)
debug "FILE_NAME for ${FILE}: ${FILE_NAME}"
local DIR_NAME
DIR_NAME=$(dirname "${FILE}" 2>&1)
debug "DIR_NAME for ${FILE}: ${DIR_NAME}"
(("INDEX++"))
LINTED_LANGUAGES_ARRAY+=("${FILE_TYPE}")
local LINT_CMD
LINT_CMD=''
if [[ ${FILE_TYPE} == "POWERSHELL" ]] || [[ ${FILE_TYPE} == "ARM" ]]; then
# Need to run PowerShell commands using pwsh -c, also exit with exit code from inner subshell
LINT_CMD=$(
cd "${WORKSPACE_PATH}" || exit
pwsh -NoProfile -NoLogo -Command "${LINTER_COMMAND} \"${FILE}\"; if (\${Error}.Count) { exit 1 }"
exit $? 2>&1
)
elif [[ ${FILE_TYPE} == "R" ]]; then
local r_dir
if [ ! -f "${DIR_NAME}/.lintr" ]; then
r_dir="${WORKSPACE_PATH}"
else else
r_dir="${DIR_NAME}" debug "Including ${FILE} because it's a test case for ${FILE_TYPE}"
fi fi
LINT_CMD=$( fi
cd "$r_dir" || exit FILE_ARRAY+=("${FILE}")
R --slave -e "lints <- lintr::lint('$FILE');print(lints);errors <- purrr::keep(lints, ~ .\$type == 'error');quit(save = 'no', status = if (length(errors) > 0) 1 else 0)" 2>&1 done <"${FILE_ARRAY_LANGUAGE_PATH}"
) else
elif [[ ${FILE_TYPE} == "CSHARP" ]]; then debug "${FILE_ARRAY_LANGUAGE_PATH} doesn't exist. Skip loading the list of files and directories to lint for ${FILE_TYPE}"
# Because the C# linter writes to tty and not stdout fi
LINT_CMD=$(
cd "${DIR_NAME}" || exit if [[ "${#FILE_ARRAY[@]}" -eq 0 ]]; then
${LINTER_COMMAND} "${FILE_NAME}" | tee /dev/tty2 2>&1 if [[ "${TEST_CASE_RUN}" == "false" ]]; then
exit "${PIPESTATUS[0]}" debug "There are no items to lint for ${FILE_TYPE}"
) unset -n FILE_ARRAY
return 0
else
fatal "Cannot find any tests for ${FILE_TYPE}"
fi
else
debug "There are ${#FILE_ARRAY[@]} items to lint for ${FILE_TYPE}: ${FILE_ARRAY[*]}"
fi
startGitHubActionsLogGroup "${FILE_TYPE}"
info "Linting ${FILE_TYPE} items..."
local PARALLEL_RESULTS_FILE_PATH
PARALLEL_RESULTS_FILE_PATH="/tmp/super-linter-worker-results-${FILE_TYPE}.json"
debug "PARALLEL_RESULTS_FILE_PATH for ${FILE_TYPE}: ${PARALLEL_RESULTS_FILE_PATH}"
local -a PARALLEL_COMMAND
PARALLEL_COMMAND=(parallel --will-cite --keep-order --max-procs "$(($(nproc) * 1))" --xargs --results "${PARALLEL_RESULTS_FILE_PATH}")
if [ "${LOG_DEBUG}" == "true" ]; then
debug "LOG_DEBUG is enabled. Enable verbose ouput for parallel"
PARALLEL_COMMAND+=(--verbose)
fi
debug "PARALLEL_COMMAND for ${FILE_TYPE}: ${PARALLEL_COMMAND[*]}"
# The following linters support linting one file at a time, and don't support linting a list of files,
# so we cannot pass more than one file per invocation
if [[ "${FILE_TYPE}" == "ANSIBLE" ]] ||
[[ "${FILE_TYPE}" == "ARM" ]] ||
[[ "${FILE_TYPE}" == "BASH_EXEC" ]] ||
[[ "${FILE_TYPE}" == "CLOJURE" ]] ||
[[ "${FILE_TYPE}" == "CSHARP" ]] ||
[[ "${FILE_TYPE}" == "GITLEAKS" ]] ||
[[ "${FILE_TYPE}" == "GO_MODULES" ]] ||
[[ "${FILE_TYPE}" == "JSCPD" ]] ||
[[ "${FILE_TYPE}" == "KOTLIN" ]] ||
[[ "${FILE_TYPE}" == "SQL" ]] ||
[[ "${FILE_TYPE}" == "SQLFLUFF" ]] ||
[[ "${FILE_TYPE}" == "CHECKOV" ]] ||
[[ "${FILE_TYPE}" == "POWERSHELL" ]] ||
[[ "${FILE_TYPE}" == "R" ]] ||
[[ "${FILE_TYPE}" == "RUST_CLIPPY" ]] ||
[[ "${FILE_TYPE}" == "SNAKEMAKE_LINT" ]] ||
[[ "${FILE_TYPE}" == "STATES" ]] ||
[[ "${FILE_TYPE}" == "TERRAFORM_TFLINT" ]] ||
[[ "${FILE_TYPE}" == "TERRAFORM_TERRASCAN" ]] ||
[[ "${FILE_TYPE}" == "TERRAGRUNT" ]]; then
debug "${FILE_TYPE} doesn't support linting files in batches. Configure the linter to run over the files to lint one by one"
PARALLEL_COMMAND+=(--max-lines 1)
fi
debug "PARALLEL_COMMAND for ${FILE_TYPE} after updating the number of files to lint per process: ${PARALLEL_COMMAND[*]}"
local LINTER_WORKING_DIRECTORY
LINTER_WORKING_DIRECTORY="${GITHUB_WORKSPACE}"
# GNU parallel parameter expansion:
# - {} input item
# - {/} basename of the input lint
# - {//} dirname of input line
if [[ ${FILE_TYPE} == "CSHARP" ]] ||
[[ (${FILE_TYPE} == "R" && -f "$(dirname "${FILE}")/.lintr") ]] ||
[[ ${FILE_TYPE} == "KOTLIN" ]] ||
[[ ${FILE_TYPE} == "TERRAFORM_TFLINT" ]]; then
LINTER_WORKING_DIRECTORY="{//}"
elif [[ ${FILE_TYPE} == "ANSIBLE" ]] || elif [[ ${FILE_TYPE} == "ANSIBLE" ]] ||
[[ ${FILE_TYPE} == "GO_MODULES" ]]; then [[ ${FILE_TYPE} == "GO_MODULES" ]]; then
debug "Linting ${FILE_TYPE}. Changing the working directory to ${FILE} before running the linter." LINTER_WORKING_DIRECTORY="{}"
# Because it expects that the working directory is a Go module (GO_MODULES) or
# because we want to enable ansible-lint autodetection mode.
# Ref: https://ansible-lint.readthedocs.io/usage
LINT_CMD=$(
cd "${FILE}" || exit 1
${LINTER_COMMAND} 2>&1
)
elif [[ ${FILE_TYPE} == "KOTLIN" ]]; then
# Because it needs to change directory to where the file to lint is
LINT_CMD=$(
cd "${DIR_NAME}" || exit
${LINTER_COMMAND} "${FILE_NAME}" 2>&1
)
elif [[ ${FILE_TYPE} == "TERRAFORM_TFLINT" ]]; then
# Check the cache to see if we've already prepped this directory for tflint
if [[ ! -v "TFLINT_SEEN_DIRS[${DIR_NAME}]" ]]; then
debug "Configuring Terraform data directory for ${DIR_NAME}"
# Define the path to an empty Terraform data directory
# (def: https://developer.hashicorp.com/terraform/cli/config/environment-variables#tf_data_dir)
# in case the user has a Terraform data directory already, and we don't
# want to modify it.
# TFlint considers this variable as well.
# Ref: https://github.com/terraform-linters/tflint/blob/master/docs/user-guide/compatibility.md#environment-variables
local TF_DATA_DIR
TF_DATA_DIR="/tmp/.terraform-${FILE_TYPE}-${DIR_NAME}"
export TF_DATA_DIR
# Let the cache know we've seen this before
# Set the value to an arbitrary non-empty string.
# Fetch Terraform modules
debug "Fetch Terraform modules for ${DIR_NAME} in ${TF_DATA_DIR}"
local FETCH_TERRAFORM_MODULES_CMD
FETCH_TERRAFORM_MODULES_CMD="$(terraform get)"
ERROR_CODE=$?
debug "Fetch Terraform modules. Exit code: ${ERROR_CODE}. Command output: ${FETCH_TERRAFORM_MODULES_CMD}"
if [ ${ERROR_CODE} -ne 0 ]; then
fatal "Error when fetching Terraform modules while linting ${FILE}"
fi
TFLINT_SEEN_DIRS[${DIR_NAME}]="false"
fi fi
# Because it needs to change the directory to where the file to lint is debug "LINTER_WORKING_DIRECTORY for ${FILE_TYPE}: ${LINTER_WORKING_DIRECTORY}"
LINT_CMD=$( PARALLEL_COMMAND+=(--workdir "${LINTER_WORKING_DIRECTORY}")
cd "${DIR_NAME}" || exit debug "PARALLEL_COMMAND for ${FILE_TYPE} after updating the working directory: ${PARALLEL_COMMAND[*]}"
${LINTER_COMMAND} --filter="${FILE_NAME}" 2>&1
) # shellcheck source=/dev/null
source /action/lib/functions/linterCommands.sh
local -n LINTER_COMMAND_ARRAY
LINTER_COMMAND_ARRAY="LINTER_COMMANDS_ARRAY_${FILE_TYPE}"
if [ ${#LINTER_COMMAND_ARRAY[@]} -eq 0 ]; then
fatal "LINTER_COMMAND_ARRAY for ${FILE_TYPE} is empty."
else else
LINT_CMD=$( debug "LINTER_COMMAND_ARRAY for ${FILE_TYPE} has ${#LINTER_COMMAND_ARRAY[@]} elements: ${LINTER_COMMAND_ARRAY[*]}"
cd "${WORKSPACE_PATH}" || exit
${LINTER_COMMAND} "${FILE}" 2>&1
)
fi fi
ERROR_CODE=$? PARALLEL_COMMAND+=("${LINTER_COMMAND_ARRAY[@]}")
debug "PARALLEL_COMMAND for ${FILE_TYPE} after LINTER_COMMAND_ARRAY concatenation: ${PARALLEL_COMMAND[*]}"
local FILE_STATUS unset -n LINTER_COMMAND_ARRAY
# Assume that the file should pass linting checks
FILE_STATUS="good"
if [[ "${TEST_CASE_RUN}" == "true" ]] && [[ ${FILE} == *"bad"* ]]; then local PARALLEL_COMMAND_OUTPUT
FILE_STATUS="bad" local PARALLEL_COMMAND_RETURN_CODE
debug "We are running in test mode. Updating the expected FILE_STATUS for ${FILE} to: ${FILE_STATUS}" PARALLEL_COMMAND_OUTPUT=$(printf "%s\n" "${FILE_ARRAY[@]}" | "${PARALLEL_COMMAND[@]}" 2>&1)
fi # Don't check for errors on this return code because commands can fail if linter report errors
PARALLEL_COMMAND_RETURN_CODE=$?
debug "PARALLEL_COMMAND_OUTPUT for ${FILE_TYPE} (exit code: ${PARALLEL_COMMAND_RETURN_CODE}): ${PARALLEL_COMMAND_OUTPUT}"
debug "Parallel output file (${PARALLEL_RESULTS_FILE_PATH}) contents for ${FILE_TYPE}:\n$(cat "${PARALLEL_RESULTS_FILE_PATH}")"
debug "Results for ${FILE}. Exit code: ${ERROR_CODE}. Command output:\n------\n${LINT_CMD}\n------" echo ${PARALLEL_COMMAND_RETURN_CODE} >"/tmp/super-linter-parallel-command-exit-code-${FILE_TYPE}"
######################################## if [ ${PARALLEL_COMMAND_RETURN_CODE} -ne 0 ]; then
# File status = good, this should pass # error "Found errors when linting ${FILE_TYPE}. Exit code: ${PARALLEL_COMMAND_RETURN_CODE}."
########################################
if [[ ${FILE_STATUS} == "good" ]]; then
(("GOOD_TEST_CASES_COUNT++"))
if [ ${ERROR_CODE} -ne 0 ]; then
error "Found errors when linting ${FILE_NAME}. Exit code: ${ERROR_CODE}. Command output:\n------\n${LINT_CMD}\n------"
(("ERRORS_FOUND_${FILE_TYPE}++"))
else else
notice "${FILE} was linted successfully" notice "${FILE_TYPE} linted successfully"
if [ -n "${LINT_CMD}" ]; then
info "Command output for ${FILE_NAME}:\n------\n${LINT_CMD}\n------"
fi fi
local RESULTS_OBJECT
RESULTS_OBJECT=
if ! RESULTS_OBJECT=$(jq -n '[inputs]' "${PARALLEL_RESULTS_FILE_PATH}"); then
fatal "Error loading results for ${FILE_TYPE}: ${RESULTS_OBJECT}"
fi fi
####################################### debug "RESULTS_OBJECT for ${FILE_TYPE}:\n${RESULTS_OBJECT}"
# File status = bad, this should fail #
####################################### # To count how many files were checked for a given FILE_TYPE
local INDEX
INDEX=0
if ! ((INDEX = $(jq '[.[] | .V | length] | add' <<<"${RESULTS_OBJECT}"))); then
fatal "Error when setting INDEX for ${FILE_TYPE}: ${INDEX}"
fi
debug "Set INDEX for ${FILE_TYPE} to: ${INDEX}"
local STDOUT_LINTER
# Get raw output so we can strip quotes from the data we load
if ! STDOUT_LINTER="$(jq --raw-output '.[].Stdout' <<<"${RESULTS_OBJECT}")"; then
fatal "Error when loading stdout for ${FILE_TYPE}:\n${STDOUT_LINTER}"
fi
if [ -n "${STDOUT_LINTER}" ]; then
info "Command output for ${FILE_TYPE}:\n------\n${STDOUT_LINTER}\n------"
else else
if [[ "${TEST_CASE_RUN}" == "false" ]]; then debug "Stdout for ${FILE_TYPE} is empty"
fatal "All files are supposed to pass linting checks when not running in test mode."
fi fi
(("BAD_TEST_CASES_COUNT++")) local STDERR_LINTER
if ! STDERR_LINTER="$(jq --raw-output '.[].Stderr' <<<"${RESULTS_OBJECT}")"; then
fatal "Error when loading stderr for ${FILE_TYPE}:\n${STDERR_LINTER}"
fi
if [ ${ERROR_CODE} -eq 0 ]; then if [ -n "${STDERR_LINTER}" ]; then
error "${FILE} should have failed test case." info "Command output for ${FILE_TYPE}:\n------\n${STDERR_LINTER}\n------"
(("ERRORS_FOUND_${FILE_TYPE}++"))
else else
notice "${FILE} failed the test case as expected" debug "Stderr for ${FILE_TYPE} is empty"
fi
fi
done
if [ "${TEST_CASE_RUN}" = "true" ]; then
debug "${LINTER_NAME} test suite has ${INDEX} test(s), of which ${BAD_TEST_CASES_COUNT} 'bad' (expected to fail), ${GOOD_TEST_CASES_COUNT} 'good' (expected to pass)."
# Check if we ran at least one test
if [ "${INDEX}" -eq 0 ]; then
fatal "Failed to find any tests ran for: ${LINTER_NAME}. Check that tests exist for linter: ${LINTER_NAME}"
fi fi
# Check if we ran at least one 'bad' test unset -n FILE_ARRAY
if [ "${BAD_TEST_CASES_COUNT}" -eq 0 ]; then
fatal "Failed to find any tests that are expected to fail for: ${LINTER_NAME}. Check that tests that are expected to fail exist for linter: ${LINTER_NAME}"
fi
# Check if we ran at least one 'good' test endGitHubActionsLogGroup "${FILE_TYPE}"
if [ "${GOOD_TEST_CASES_COUNT}" -eq 0 ]; then
fatal "Failed to find any tests that are expected to pass for: ${LINTER_NAME}. Check that tests that are expected to pass exist for linter: ${LINTER_NAME}"
fi
fi
} }
# We need this for parallel
export -f LintCodebase

View file

@ -77,6 +77,7 @@ else
fi fi
# Let users configure GitHub Actions log markers regardless of running locally or not # Let users configure GitHub Actions log markers regardless of running locally or not
ENABLE_GITHUB_ACTIONS_GROUP_TITLE="${ENABLE_GITHUB_ACTIONS_GROUP_TITLE:-"${DEFAULT_ENABLE_GITHUB_ACTIONS_GROUP_TITLE}"}" ENABLE_GITHUB_ACTIONS_GROUP_TITLE="${ENABLE_GITHUB_ACTIONS_GROUP_TITLE:-"${DEFAULT_ENABLE_GITHUB_ACTIONS_GROUP_TITLE}"}"
export ENABLE_GITHUB_ACTIONS_GROUP_TITLE
startGitHubActionsLogGroup "${SUPER_LINTER_INITIALIZATION_LOG_GROUP_TITLE}" startGitHubActionsLogGroup "${SUPER_LINTER_INITIALIZATION_LOG_GROUP_TITLE}"
@ -88,10 +89,12 @@ DISABLE_ERRORS="${DISABLE_ERRORS:-"false"}"
declare -l IGNORE_GENERATED_FILES declare -l IGNORE_GENERATED_FILES
# Do not ignore generated files by default for backwards compatibility # Do not ignore generated files by default for backwards compatibility
IGNORE_GENERATED_FILES="${IGNORE_GENERATED_FILES:-false}" IGNORE_GENERATED_FILES="${IGNORE_GENERATED_FILES:-false}"
export IGNORE_GENERATED_FILES
# We want a lowercase value # We want a lowercase value
declare -l IGNORE_GITIGNORED_FILES declare -l IGNORE_GITIGNORED_FILES
IGNORE_GITIGNORED_FILES="${IGNORE_GITIGNORED_FILES:-false}" IGNORE_GITIGNORED_FILES="${IGNORE_GITIGNORED_FILES:-false}"
export IGNORE_GITIGNORED_FILES
# We want a lowercase value # We want a lowercase value
declare -l MULTI_STATUS declare -l MULTI_STATUS
@ -151,7 +154,6 @@ GITHUB_API_URL="${GITHUB_API_URL%/}"
GITHUB_SERVER_URL="${GITHUB_DOMAIN:-"https://github.com"}" GITHUB_SERVER_URL="${GITHUB_DOMAIN:-"https://github.com"}"
# Extract domain name from URL # Extract domain name from URL
GITHUB_SERVER_URL=$(echo "$GITHUB_SERVER_URL" | cut -d '/' -f 3) GITHUB_SERVER_URL=$(echo "$GITHUB_SERVER_URL" | cut -d '/' -f 3)
LINTED_LANGUAGES_ARRAY=() # Will be filled at run time with all languages that were linted
LINTER_RULES_PATH="${LINTER_RULES_PATH:-.github/linters}" # Linter rules directory LINTER_RULES_PATH="${LINTER_RULES_PATH:-.github/linters}" # Linter rules directory
# shellcheck disable=SC2034 # Variable is referenced in other scripts # shellcheck disable=SC2034 # Variable is referenced in other scripts
RAW_FILE_ARRAY=() # Array of all files that were changed RAW_FILE_ARRAY=() # Array of all files that were changed
@ -610,6 +612,9 @@ GetGitHubVars() {
else else
debug "Skip GITHUB_TOKEN, GITHUB_REPOSITORY, and GITHUB_RUN_ID validation because we don't need these variables for GitHub Actions status reports. MULTI_STATUS: ${MULTI_STATUS}" debug "Skip GITHUB_TOKEN, GITHUB_REPOSITORY, and GITHUB_RUN_ID validation because we don't need these variables for GitHub Actions status reports. MULTI_STATUS: ${MULTI_STATUS}"
fi fi
# We need this for parallel
export GITHUB_WORKSPACE
} }
################################################################################ ################################################################################
#### Function CallStatusAPI #################################################### #### Function CallStatusAPI ####################################################
@ -666,84 +671,62 @@ CallStatusAPI() {
fi fi
fi fi
} }
################################################################################
#### Function Footer ###########################################################
Footer() { Footer() {
info "----------------------------------------------" info "----------------------------------------------"
info "----------------------------------------------" info "----------------------------------------------"
info "The script has completed"
info "----------------------------------------------"
info "----------------------------------------------"
#################################################### local ANY_LINTER_SUCCESS
# Need to clean up the lanuage array of duplicates # ANY_LINTER_SUCCESS="false"
####################################################
mapfile -t UNIQUE_LINTED_ARRAY < <(for LANG in "${LINTED_LANGUAGES_ARRAY[@]}"; do echo "${LANG}"; done | sort -u) local SUPER_LINTER_EXIT_CODE
export UNIQUE_LINTED_ARRAY # Workaround SC2034 SUPER_LINTER_EXIT_CODE=0
##############################
# Prints for errors if found #
##############################
for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do
########################### # This used to be the count of errors found for a given LANGUAGE, but since
# Build the error counter # # after we switched to running linters against a batch of files, it may not
########################### # represent the actual number of files that didn't pass the validation,
ERROR_COUNTER="ERRORS_FOUND_${LANGUAGE}" # but a number that's less than that because of how GNU parallel returns
# exit codes.
# Ref: https://www.gnu.org/software/parallel/parallel.html#exit-status
ERROR_COUNTER_FILE_PATH="/tmp/super-linter-parallel-command-exit-code-${LANGUAGE}"
if [ ! -f "${ERROR_COUNTER_FILE_PATH}" ]; then
debug "Error counter ${ERROR_COUNTER_FILE_PATH} doesn't exist"
else
ERROR_COUNTER=$(<"${ERROR_COUNTER_FILE_PATH}")
debug "ERROR_COUNTER for ${LANGUAGE}: ${ERROR_COUNTER}"
################## if [[ ${ERROR_COUNTER} -ne 0 ]]; then
# Print if not 0 # error "Errors found in ${LANGUAGE}"
##################
if [[ ${!ERROR_COUNTER} -ne 0 ]]; then
# We found errors in the language
###################
# Print the goods #
###################
error "ERRORS FOUND in ${LANGUAGE}:[${!ERROR_COUNTER}]"
#########################################
# Create status API for Failed language #
#########################################
CallStatusAPI "${LANGUAGE}" "error" CallStatusAPI "${LANGUAGE}" "error"
###################################### SUPER_LINTER_EXIT_CODE=1
# Check if we validated the language # debug "Setting super-linter exit code to ${SUPER_LINTER_EXIT_CODE} because there were errors for ${LANGUAGE}"
###################################### elif [[ ${ERROR_COUNTER} -eq 0 ]]; then
elif [[ ${!ERROR_COUNTER} -eq 0 ]]; then notice "Successfully linted ${LANGUAGE}"
if CheckInArray "${LANGUAGE}"; then
# No errors found when linting the language
CallStatusAPI "${LANGUAGE}" "success" CallStatusAPI "${LANGUAGE}" "success"
ANY_LINTER_SUCCESS="true"
debug "Set ANY_LINTER_SUCCESS to ${ANY_LINTER_SUCCESS} because ${LANGUAGE} reported a success"
fi fi
fi fi
done done
################################## if [[ "${ANY_LINTER_SUCCESS}" == "true" ]] && [[ ${SUPER_LINTER_EXIT_CODE} -ne 0 ]]; then
# Exit with 0 if errors disabled # SUPER_LINTER_EXIT_CODE=2
################################## debug "There was at least one linter that reported a success. Setting the super-linter exit code to: ${SUPER_LINTER_EXIT_CODE}"
fi
if [ "${DISABLE_ERRORS}" == "true" ]; then if [ "${DISABLE_ERRORS}" == "true" ]; then
warn "Exiting with exit code:[0] as:[DISABLE_ERRORS] was set to:[${DISABLE_ERRORS}]" warn "The super-linter exit code is ${SUPER_LINTER_EXIT_CODE}. Forcibly setting it to 0 because DISABLE_ERRORS is set to: ${DISABLE_ERRORS}"
exit 0 SUPER_LINTER_EXIT_CODE=0
fi fi
############################### if [[ ${SUPER_LINTER_EXIT_CODE} -eq 0 ]]; then
# Exit with 1 if errors found # notice "All files and directories linted successfully"
############################### else
# Loop through all languages error "Super-linter detected linting errors"
for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do
# build the variable
ERRORS_FOUND_LANGUAGE="ERRORS_FOUND_${LANGUAGE}"
# Check if error was found
if [[ ${!ERRORS_FOUND_LANGUAGE} -ne 0 ]]; then
# Failed exit
fatal "Exiting with errors found!"
fi fi
done
######################## exit ${SUPER_LINTER_EXIT_CODE}
# Footer prints Exit 0 #
########################
notice "All file(s) linted successfully with no errors detected"
info "----------------------------------------------"
# Successful exit
exit 0
} }
################################################################################ ################################################################################
#### Function UpdateLoopsForImage ############################################## #### Function UpdateLoopsForImage ##############################################
@ -892,133 +875,6 @@ done
# Load rules for special cases # Load rules for special cases
GetStandardRules "javascript" GetStandardRules "javascript"
##########################
# Define linter commands #
##########################
declare -A LINTER_COMMANDS_ARRAY
LINTER_COMMANDS_ARRAY['ANSIBLE']="ansible-lint -c ${ANSIBLE_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['ARM']="Import-Module ${ARM_TTK_PSD1} ; \${config} = \$(Import-PowerShellDataFile -Path ${ARM_LINTER_RULES}) ; Test-AzTemplate @config -TemplatePath"
if [ -z "${BASH_SEVERITY}" ]; then
LINTER_COMMANDS_ARRAY['BASH']="shellcheck --color --external-sources"
else
LINTER_COMMANDS_ARRAY['BASH']="shellcheck --color --external-sources --severity=${BASH_SEVERITY}"
fi
LINTER_COMMANDS_ARRAY['BASH_EXEC']="bash-exec"
LINTER_COMMANDS_ARRAY['CHECKOV']="checkov --config-file ${CHECKOV_LINTER_RULES}"
if CheckovConfigurationFileContainsDirectoryOption "${CHECKOV_LINTER_RULES}"; then
debug "No need to update the Checkov command."
else
debug "Adding the '--directory' option to the Checkov command."
LINTER_COMMANDS_ARRAY['CHECKOV']="${LINTER_COMMANDS_ARRAY['CHECKOV']} --directory"
fi
LINTER_COMMANDS_ARRAY['CLANG_FORMAT']="clang-format --Werror --dry-run"
LINTER_COMMANDS_ARRAY['CLOJURE']="clj-kondo --config ${CLOJURE_LINTER_RULES} --lint"
LINTER_COMMANDS_ARRAY['CLOUDFORMATION']="cfn-lint --config-file ${CLOUDFORMATION_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['COFFEESCRIPT']="coffeelint -f ${COFFEESCRIPT_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['CPP']="cpplint"
LINTER_COMMANDS_ARRAY['CSHARP']="dotnet format whitespace --folder --verify-no-changes --exclude / --include"
LINTER_COMMANDS_ARRAY['CSS']="stylelint --config ${CSS_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['DART']="dart analyze --fatal-infos --fatal-warnings"
LINTER_COMMANDS_ARRAY['DOCKERFILE_HADOLINT']="hadolint -c ${DOCKERFILE_HADOLINT_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['EDITORCONFIG']="editorconfig-checker -config ${EDITORCONFIG_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['ENV']="dotenv-linter"
if [ "${GITHUB_ACTIONS_COMMAND_ARGS}" = "null" ]; then
LINTER_COMMANDS_ARRAY['GITHUB_ACTIONS']="actionlint -config-file ${GITHUB_ACTIONS_LINTER_RULES}"
else
LINTER_COMMANDS_ARRAY['GITHUB_ACTIONS']="actionlint -config-file ${GITHUB_ACTIONS_LINTER_RULES} ${GITHUB_ACTIONS_COMMAND_ARGS}"
fi
LINTER_COMMANDS_ARRAY['GITLEAKS']="gitleaks detect --no-banner --no-git --redact --config ${GITLEAKS_LINTER_RULES} --verbose --source"
LINTER_COMMANDS_ARRAY['GHERKIN']="gherkin-lint -c ${GHERKIN_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['GO_MODULES']="golangci-lint run -c ${GO_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['GO']="${LINTER_COMMANDS_ARRAY['GO_MODULES']} --fast"
LINTER_COMMANDS_ARRAY['GOOGLE_JAVA_FORMAT']="java -jar /usr/bin/google-java-format --dry-run --set-exit-if-changed"
LINTER_COMMANDS_ARRAY['GROOVY']="npm-groovy-lint -c ${GROOVY_LINTER_RULES} --failon warning --no-insight"
LINTER_COMMANDS_ARRAY['HTML']="htmlhint --config ${HTML_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['JAVA']="java -jar /usr/bin/checkstyle -c ${JAVA_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['JAVASCRIPT_ES']="eslint --no-eslintrc -c ${JAVASCRIPT_ES_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['JAVASCRIPT_STANDARD']="standard ${JAVASCRIPT_STANDARD_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['JAVASCRIPT_PRETTIER']="prettier --check"
LINTER_COMMANDS_ARRAY['JSCPD']="jscpd --config ${JSCPD_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['JSON']="eslint --no-eslintrc -c ${JAVASCRIPT_ES_LINTER_RULES} --ext .json"
LINTER_COMMANDS_ARRAY['JSONC']="eslint --no-eslintrc -c ${JAVASCRIPT_ES_LINTER_RULES} --ext .json5,.jsonc"
LINTER_COMMANDS_ARRAY['JSX']="eslint --no-eslintrc -c ${JSX_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['KOTLIN']="ktlint"
if [ "${KUBERNETES_KUBECONFORM_OPTIONS}" == "null" ] || [ -z "${KUBERNETES_KUBECONFORM_OPTIONS}" ]; then
LINTER_COMMANDS_ARRAY['KUBERNETES_KUBECONFORM']="kubeconform -strict"
else
LINTER_COMMANDS_ARRAY['KUBERNETES_KUBECONFORM']="kubeconform -strict ${KUBERNETES_KUBECONFORM_OPTIONS}"
fi
LINTER_COMMANDS_ARRAY['LATEX']="chktex -q -l ${LATEX_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['LUA']="luacheck --config ${LUA_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['MARKDOWN']="markdownlint -c ${MARKDOWN_LINTER_RULES}"
if [ -n "${MARKDOWN_CUSTOM_RULE_GLOBS}" ]; then
IFS="," read -r -a MARKDOWN_CUSTOM_RULE_GLOBS_ARRAY <<<"${MARKDOWN_CUSTOM_RULE_GLOBS}"
for glob in "${MARKDOWN_CUSTOM_RULE_GLOBS_ARRAY[@]}"; do
if [ -z "${LINTER_RULES_PATH}" ]; then
LINTER_COMMANDS_ARRAY['MARKDOWN']="${LINTER_COMMANDS_ARRAY['MARKDOWN']} -r ${GITHUB_WORKSPACE}/${glob}"
else
LINTER_COMMANDS_ARRAY['MARKDOWN']="${LINTER_COMMANDS_ARRAY['MARKDOWN']} -r ${GITHUB_WORKSPACE}/${LINTER_RULES_PATH}/${glob}"
fi
done
fi
LINTER_COMMANDS_ARRAY['NATURAL_LANGUAGE']="textlint -c ${NATURAL_LANGUAGE_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['OPENAPI']="spectral lint -r ${OPENAPI_LINTER_RULES} -D"
if [ "${PERL_PERLCRITIC_OPTIONS}" == "null" ] || [ -z "${PERL_PERLCRITIC_OPTIONS}" ]; then
LINTER_COMMANDS_ARRAY['PERL']="perlcritic"
else
LINTER_COMMANDS_ARRAY['PERL']="perlcritic ${PERL_PERLCRITIC_OPTIONS}"
fi
LINTER_COMMANDS_ARRAY['PHP_BUILTIN']="php -l -c ${PHP_BUILTIN_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['PHP_PHPCS']="phpcs --standard=${PHP_PHPCS_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['PHP_PHPSTAN']="phpstan analyse --no-progress --no-ansi --memory-limit 1G -c ${PHP_PHPSTAN_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['PHP_PSALM']="psalm --config=${PHP_PSALM_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['POWERSHELL']="Invoke-ScriptAnalyzer -EnableExit -Settings ${POWERSHELL_LINTER_RULES} -Path"
LINTER_COMMANDS_ARRAY['PROTOBUF']="protolint lint --config_path ${PROTOBUF_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['PYTHON_BLACK']="black --config ${PYTHON_BLACK_LINTER_RULES} --diff --check"
LINTER_COMMANDS_ARRAY['PYTHON_PYLINT']="pylint --rcfile ${PYTHON_PYLINT_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['PYTHON_FLAKE8']="flake8 --config=${PYTHON_FLAKE8_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['PYTHON_ISORT']="isort --check --diff --sp ${PYTHON_ISORT_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['PYTHON_MYPY']="mypy --config-file ${PYTHON_MYPY_LINTER_RULES} --install-types --non-interactive"
LINTER_COMMANDS_ARRAY['R']="lintr"
LINTER_COMMANDS_ARRAY['RAKU']="raku"
LINTER_COMMANDS_ARRAY['RENOVATE']="renovate-config-validator --strict"
LINTER_COMMANDS_ARRAY['RUBY']="rubocop -c ${RUBY_LINTER_RULES} --force-exclusion --ignore-unrecognized-cops"
LINTER_COMMANDS_ARRAY['RUST_2015']="rustfmt --check --edition 2015"
LINTER_COMMANDS_ARRAY['RUST_2018']="rustfmt --check --edition 2018"
LINTER_COMMANDS_ARRAY['RUST_2021']="rustfmt --check --edition 2021"
LINTER_COMMANDS_ARRAY['RUST_CLIPPY']="clippy"
LINTER_COMMANDS_ARRAY['SCALAFMT']="scalafmt --config ${SCALAFMT_LINTER_RULES} --test"
LINTER_COMMANDS_ARRAY['SHELL_SHFMT']="shfmt -d"
LINTER_COMMANDS_ARRAY['SNAKEMAKE_LINT']="snakemake --lint -s"
LINTER_COMMANDS_ARRAY['SNAKEMAKE_SNAKEFMT']="snakefmt --config ${SNAKEMAKE_SNAKEFMT_LINTER_RULES} --check --compact-diff"
LINTER_COMMANDS_ARRAY['STATES']="asl-validator --json-path"
LINTER_COMMANDS_ARRAY['SQL']="sql-lint --config ${SQL_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['SQLFLUFF']="sqlfluff lint --config ${SQLFLUFF_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['TEKTON']="tekton-lint"
LINTER_COMMANDS_ARRAY['TERRAFORM_FMT']="terraform fmt -check -diff"
LINTER_COMMANDS_ARRAY['TERRAFORM_TFLINT']="tflint -c ${TERRAFORM_TFLINT_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['TERRAFORM_TERRASCAN']="terrascan scan -i terraform -t all -c ${TERRAFORM_TERRASCAN_LINTER_RULES} -f"
LINTER_COMMANDS_ARRAY['TERRAGRUNT']="terragrunt hclfmt --terragrunt-check --terragrunt-log-level error --terragrunt-hclfmt-file"
LINTER_COMMANDS_ARRAY['TSX']="eslint --no-eslintrc -c ${TSX_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['TYPESCRIPT_ES']="eslint --no-eslintrc -c ${TYPESCRIPT_ES_LINTER_RULES}"
LINTER_COMMANDS_ARRAY['TYPESCRIPT_STANDARD']="ts-standard --parser @typescript-eslint/parser --plugin @typescript-eslint/eslint-plugin --project ${TYPESCRIPT_STANDARD_TSCONFIG_FILE}"
LINTER_COMMANDS_ARRAY['TYPESCRIPT_PRETTIER']="prettier --check"
LINTER_COMMANDS_ARRAY['XML']="xmllint"
if [ "${YAML_ERROR_ON_WARNING}" == 'false' ]; then
LINTER_COMMANDS_ARRAY['YAML']="yamllint -c ${YAML_LINTER_RULES} -f parsable"
else
LINTER_COMMANDS_ARRAY['YAML']="yamllint --strict -c ${YAML_LINTER_RULES} -f parsable"
fi
debug "--- Linter commands ---"
debug "-----------------------"
for i in "${!LINTER_COMMANDS_ARRAY[@]}"; do
debug "Linter key: $i, command: ${LINTER_COMMANDS_ARRAY[$i]}"
done
debug "---------------------------------------------"
################################# #################################
# Check for SSL cert and update # # Check for SSL cert and update #
################################# #################################
@ -1034,77 +890,74 @@ BuildFileList "${VALIDATE_ALL_CODEBASE}" "${TEST_CASE_RUN}"
##################################### #####################################
RunAdditionalInstalls RunAdditionalInstalls
###############
# Run linters #
###############
EDITORCONFIG_FILE_PATH="${GITHUB_WORKSPACE}"/.editorconfig
#################################### ####################################
# Print ENV before running linters # # Print ENV before running linters #
#################################### ####################################
debug "--- ENV (before running linters) ---" debug "--- ENV (before running linters) ---"
debug "------------------------------------" debug "------------------------------------"
debug "ENV:" debug "ENV:"
debug "$(printenv | sort)" debug "$(printenv)"
debug "------------------------------------" debug "------------------------------------"
endGitHubActionsLogGroup "${SUPER_LINTER_INITIALIZATION_LOG_GROUP_TITLE}" endGitHubActionsLogGroup "${SUPER_LINTER_INITIALIZATION_LOG_GROUP_TITLE}"
for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do ###############
startGitHubActionsLogGroup "${LANGUAGE}" # Run linters #
debug "Running linter for the ${LANGUAGE} language..." ###############
VALIDATE_LANGUAGE_VARIABLE_NAME="VALIDATE_${LANGUAGE}" declare PARALLEL_RESULTS_FILE_PATH
debug "Setting VALIDATE_LANGUAGE_VARIABLE_NAME to ${VALIDATE_LANGUAGE_VARIABLE_NAME}..." PARALLEL_RESULTS_FILE_PATH="/tmp/super-linter-results.json"
VALIDATE_LANGUAGE_VARIABLE_VALUE="${!VALIDATE_LANGUAGE_VARIABLE_NAME}" debug "PARALLEL_RESULTS_FILE_PATH: ${PARALLEL_RESULTS_FILE_PATH}"
debug "Setting VALIDATE_LANGUAGE_VARIABLE_VALUE to ${VALIDATE_LANGUAGE_VARIABLE_VALUE}..."
if [ "${VALIDATE_LANGUAGE_VARIABLE_VALUE}" = "true" ]; then declare -a PARALLEL_COMMAND
# Check if we need an .editorconfig file PARALLEL_COMMAND=(parallel --will-cite --keep-order --max-procs "$(($(nproc) * 1))" --xargs --results "${PARALLEL_RESULTS_FILE_PATH}")
# shellcheck disable=SC2153
if [ "${LANGUAGE}" = "EDITORCONFIG" ] || [ "${LANGUAGE}" = "SHELL_SHFMT" ]; then # Run one LANGUAGE per process. Each of these processes will run more processees in parellel if supported
if [ -e "${EDITORCONFIG_FILE_PATH}" ]; then PARALLEL_COMMAND+=(--max-lines 1)
debug "Found an EditorConfig file at ${EDITORCONFIG_FILE_PATH}"
if [ "${LOG_DEBUG}" == "true" ]; then
debug "LOG_DEBUG is enabled. Enable verbose ouput for parallel"
PARALLEL_COMMAND+=(--verbose)
fi
PARALLEL_COMMAND+=("LintCodebase" "{}" "\"${TEST_CASE_RUN}\"")
debug "PARALLEL_COMMAND: ${PARALLEL_COMMAND[*]}"
PARALLEL_COMMAND_OUTPUT=$(printf "%s\n" "${LANGUAGE_ARRAY[@]}" | "${PARALLEL_COMMAND[@]}" 2>&1)
PARALLEL_COMMAND_RETURN_CODE=$?
debug "PARALLEL_COMMAND_OUTPUT when running linters (exit code: ${PARALLEL_COMMAND_RETURN_CODE}):\n${PARALLEL_COMMAND_OUTPUT}"
debug "Parallel output file (${PARALLEL_RESULTS_FILE_PATH}) contents when running linters:\n$(cat "${PARALLEL_RESULTS_FILE_PATH}")"
RESULTS_OBJECT=
if ! RESULTS_OBJECT=$(jq -n '[inputs]' "${PARALLEL_RESULTS_FILE_PATH}"); then
fatal "Error loading results when building the file list: ${RESULTS_OBJECT}"
fi
debug "RESULTS_OBJECT when running linters:\n${RESULTS_OBJECT}"
# Get raw output so we can strip quotes from the data we load
if ! STDOUT_LINTERS="$(jq --raw-output '.[].Stdout' <<<"${RESULTS_OBJECT}")"; then
fatal "Error when loading stdout when running linters:\n${STDOUT_LINTERS}"
fi
if [ -n "${STDOUT_LINTERS}" ]; then
info "Command output when running linters:\n------\n${STDOUT_LINTERS}\n------"
else else
debug "No .editorconfig found at: $EDITORCONFIG_FILE_PATH. Skipping ${LANGUAGE} linting..." debug "Stdout when running linters is empty"
continue
fi
elif [ "${LANGUAGE}" = "R" ] && [ ! -f "${R_RULES_FILE_PATH_IN_ROOT}" ] && ((${#FILE_ARRAY_R[@]})); then
info "No .lintr configuration file found, using defaults."
cp "$R_LINTER_RULES" "$GITHUB_WORKSPACE"
# shellcheck disable=SC2034
SUPER_LINTER_COPIED_R_LINTER_RULES_FILE="true"
# Check if there's local configuration for the Raku linter
elif [ "${LANGUAGE}" = "RAKU" ] && [ -e "${GITHUB_WORKSPACE}/META6.json" ]; then
cd "${GITHUB_WORKSPACE}" && zef install --deps-only --/test .
fi fi
LINTER_NAME="${LINTER_NAMES_ARRAY["${LANGUAGE}"]}" if ! STDERR_LINTERS="$(jq --raw-output '.[].Stderr' <<<"${RESULTS_OBJECT}")"; then
if [ -z "${LINTER_NAME}" ]; then fatal "Error when loading stderr for ${FILE_TYPE}:\n${STDERR_LINTERS}"
fatal "Cannot find the linter name for ${LANGUAGE} language." fi
if [ -n "${STDERR_LINTERS}" ]; then
info "Command output for ${FILE_TYPE}:\n------\n${STDERR_LINTERS}\n------"
else else
debug "Setting LINTER_NAME to ${LINTER_NAME}..." debug "Stderr when running linters is empty"
fi fi
LINTER_COMMAND="${LINTER_COMMANDS_ARRAY["${LANGUAGE}"]}" if [[ ${PARALLEL_COMMAND_RETURN_CODE} -ne 0 ]]; then
if [ -z "${LINTER_COMMAND}" ]; then fatal "Error when running linters. Exit code: ${PARALLEL_COMMAND_RETURN_CODE}"
fatal "Cannot find the linter command for ${LANGUAGE} language."
else
debug "Setting LINTER_COMMAND to ${LINTER_COMMAND}..."
fi fi
FILE_ARRAY_VARIABLE_NAME="FILE_ARRAY_${LANGUAGE}"
debug "Setting FILE_ARRAY_VARIABLE_NAME to ${FILE_ARRAY_VARIABLE_NAME}..."
# shellcheck disable=SC2125
LANGUAGE_FILE_ARRAY="${FILE_ARRAY_VARIABLE_NAME}"[@]
debug "${FILE_ARRAY_VARIABLE_NAME} file array contents: ${!LANGUAGE_FILE_ARRAY}"
debug "Invoking ${LINTER_NAME} linter. TEST_CASE_RUN: ${TEST_CASE_RUN}"
LintCodebase "${LANGUAGE}" "${LINTER_NAME}" "${LINTER_COMMAND}" "${FILTER_REGEX_INCLUDE}" "${FILTER_REGEX_EXCLUDE}" "${TEST_CASE_RUN}" "${!LANGUAGE_FILE_ARRAY}"
fi
endGitHubActionsLogGroup "${LANGUAGE}"
done
########## ##########
# Footer # # Footer #
########## ##########

View file

@ -436,6 +436,7 @@ control "super-linter-validate-files" do
"/action/lib/functions/buildFileList.sh", "/action/lib/functions/buildFileList.sh",
"/action/lib/functions/detectFiles.sh", "/action/lib/functions/detectFiles.sh",
"/action/lib/functions/githubEvent.sh", "/action/lib/functions/githubEvent.sh",
"/action/lib/functions/linterCommands.sh",
"/action/lib/functions/linterRules.sh", "/action/lib/functions/linterRules.sh",
"/action/lib/functions/linterVersions.sh", "/action/lib/functions/linterVersions.sh",
"/action/lib/functions/linterVersions.txt", "/action/lib/functions/linterVersions.txt",

48
test/run-super-linter-tests.sh Executable file
View file

@ -0,0 +1,48 @@
#!/usr/bin/env bash
set -o errexit
set -o nounset
set -o pipefail
SUPER_LINTER_TEST_CONTAINER_URL="${1}"
TEST_FUNCTION_NAME="${2}"
COMMAND_TO_RUN=(docker run -e ACTIONS_RUNNER_DEBUG=true -e DEFAULT_BRANCH=main -e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true -e JSCPD_CONFIG_FILE=".jscpd-test-linters.json" -e RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES="default.json,hoge.json" -e RUN_LOCAL=true -e TEST_CASE_RUN=true -e TYPESCRIPT_STANDARD_TSCONFIG_FILE=".github/linters/tsconfig.json" -v "$(pwd):/tmp/lint")
run_test_cases_expect_failure() {
COMMAND_TO_RUN+=(-e ANSIBLE_DIRECTORY="/test/linters/ansible/bad" -e CHECKOV_FILE_NAME=".checkov-test-linters-failure.yaml" -e FILTER_REGEX_INCLUDE=".*bad.*")
EXPECTED_EXIT_CODE=1
}
run_test_cases_expect_success() {
COMMAND_TO_RUN+=(-e ANSIBLE_DIRECTORY="/test/linters/ansible/good" -e CHECKOV_FILE_NAME=".checkov-test-linters-success.yaml" -e FILTER_REGEX_INCLUDE=".*good.*")
}
# Run the test setup function
${TEST_FUNCTION_NAME}
COMMAND_TO_RUN+=("${SUPER_LINTER_TEST_CONTAINER_URL}")
declare -i EXPECTED_EXIT_CODE
EXPECTED_EXIT_CODE=${EXPECTED_EXIT_CODE:-0}
if [ ${EXPECTED_EXIT_CODE} -ne 0 ]; then
echo "Disable failures on error because the expected exit code is ${EXPECTED_EXIT_CODE}"
set +o errexit
fi
echo "Command to run: ${COMMAND_TO_RUN[*]}"
"${COMMAND_TO_RUN[@]}"
SUPER_LINTER_EXIT_CODE=$?
# Enable the errexit option in case we disabled it
set -o errexit
echo "Super-linter exit code: ${SUPER_LINTER_EXIT_CODE}"
if [ ${SUPER_LINTER_EXIT_CODE} -ne ${EXPECTED_EXIT_CODE} ]; then
echo "Super-linter exited with an unexpected code: ${SUPER_LINTER_EXIT_CODE}"
exit 1
else
echo "Super-linter exited with the expected code: ${SUPER_LINTER_EXIT_CODE}"
fi