diff --git a/.devcontainer/devcontainer.env b/.devcontainer/devcontainer.env index dc9a5cba..2ccd0512 100644 --- a/.devcontainer/devcontainer.env +++ b/.devcontainer/devcontainer.env @@ -2,4 +2,5 @@ DEFAULT_BRANCH=main DEFAULT_WORKSPACE=/workspaces/super-linter ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true LOG_LEVEL=DEBUG +REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT=true RUN_LOCAL=true diff --git a/README.md b/README.md index f47750f9..93ca98f6 100644 --- a/README.md +++ b/README.md @@ -296,6 +296,7 @@ You can configure Super-linter using the following environment variables: | **PYTHON_PYLINT_CONFIG_FILE** | `.python-lint` | Filename for [pylint configuration](https://pylint.pycqa.org/en/latest/user_guide/run.html?highlight=rcfile#command-line-options) (ex: `.python-lint`, `.pylintrc`) | | **PYTHON_RUFF_CONFIG_FILE** | `.ruff.toml` | Filename for [ruff configuration](https://docs.astral.sh/ruff/configuration/) | | **RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES** | not set | Comma-separated filenames for [renovate shareable config preset](https://docs.renovatebot.com/config-presets/) (ex: `default.json`) | +| **REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT** | `false` | If set to `true`, Super-linter removes ANSI color codes from linters stdout and stderr files, and from the Super-linter log file. | | **RUBY_CONFIG_FILE** | `.ruby-lint.yml` | Filename for [rubocop configuration](https://docs.rubocop.org/rubocop/configuration.html) (ex: `.ruby-lint.yml`, `.rubocop.yml`) | | **RUST_CLIPPY_COMMAND_OPTIONS** | not set | Additional options and arguments to pass to the command when running Clippy. | | **SAVE_SUPER_LINTER_OUTPUT** | `false` | If set to `true`, Super-linter will save its output in the workspace. For more information, see [Super-linter outputs](#super-linter-outputs). | diff --git a/lib/functions/output.sh b/lib/functions/output.sh index 1d27405e..838331d1 100755 --- a/lib/functions/output.sh +++ b/lib/functions/output.sh @@ -55,3 +55,16 @@ FormatSuperLinterSummaryFile() { return 1 fi } + +# 0x1B (= ^[) is the control code that starts all ANSI color codes escape sequences +# Ref: https://en.wikipedia.org/wiki/ANSI_escape_code#C0_control_codes +ANSI_COLOR_CODES_SEARCH_PATTERN='\x1b\[[0-9;]*m' +export ANSI_COLOR_CODES_SEARCH_PATTERN +RemoveAnsiColorCodesFromFile() { + local FILE_PATH="${1}" + debug "Removing ANSI color codes from ${FILE_PATH}" + if ! sed -i "s/${ANSI_COLOR_CODES_SEARCH_PATTERN}//g" "${FILE_PATH}"; then + error "Error while removing ANSI color codes from ${FILE_PATH}" + return 1 + fi +} diff --git a/lib/functions/validation.sh b/lib/functions/validation.sh index 12845580..113ba088 100755 --- a/lib/functions/validation.sh +++ b/lib/functions/validation.sh @@ -17,6 +17,7 @@ function ValidateBooleanConfigurationVariables() { ValidateBooleanVariable "LOG_VERBOSE" "${LOG_VERBOSE}" ValidateBooleanVariable "LOG_WARN" "${LOG_WARN}" ValidateBooleanVariable "MULTI_STATUS" "${MULTI_STATUS}" + ValidateBooleanVariable "REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT" "${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}" ValidateBooleanVariable "RUN_LOCAL" "${RUN_LOCAL}" ValidateBooleanVariable "SAVE_SUPER_LINTER_OUTPUT" "${SAVE_SUPER_LINTER_OUTPUT}" ValidateBooleanVariable "SAVE_SUPER_LINTER_SUMMARY" "${SAVE_SUPER_LINTER_SUMMARY}" diff --git a/lib/functions/worker.sh b/lib/functions/worker.sh index c3e31807..defd5859 100755 --- a/lib/functions/worker.sh +++ b/lib/functions/worker.sh @@ -211,6 +211,10 @@ function LintCodebase() { fatal "Error when loading stdout for ${FILE_TYPE}:\n${STDOUT_LINTER}" fi + # Load output functions because we might need to process stdout and stderr + # shellcheck source=/dev/null + source /action/lib/functions/output.sh + if [ -n "${STDOUT_LINTER}" ]; then info "Command output for ${FILE_TYPE}:\n------\n${STDOUT_LINTER}\n------" @@ -218,6 +222,10 @@ function LintCodebase() { STDOUT_LINTER_FILE_PATH="${SUPER_LINTER_PRIVATE_OUTPUT_DIRECTORY_PATH}/super-linter-parallel-stdout-${FILE_TYPE}" debug "Saving stdout for ${FILE_TYPE} to ${STDOUT_LINTER_FILE_PATH} in case we need it later" printf '%s\n' "${STDOUT_LINTER}" >"${STDOUT_LINTER_FILE_PATH}" + if [[ "${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}" == "true" ]] && + ! RemoveAnsiColorCodesFromFile "${STDOUT_LINTER_FILE_PATH}"; then + fatal "Error while removing ANSI color codes from ${STDOUT_LINTER_FILE_PATH}" + fi else debug "Stdout for ${FILE_TYPE} is empty" fi @@ -234,6 +242,10 @@ function LintCodebase() { STDERR_LINTER_FILE_PATH="${SUPER_LINTER_PRIVATE_OUTPUT_DIRECTORY_PATH}/super-linter-parallel-stderr-${FILE_TYPE}" debug "Saving stderr for ${FILE_TYPE} to ${STDERR_LINTER_FILE_PATH} in case we need it later" printf '%s\n' "${STDERR_LINTER}" >"${STDERR_LINTER_FILE_PATH}" + if [[ "${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}" == "true" ]] && + ! RemoveAnsiColorCodesFromFile "${STDERR_LINTER_FILE_PATH}"; then + fatal "Error while removing ANSI color codes from ${STDERR_LINTER_FILE_PATH}" + fi else debug "Stderr for ${FILE_TYPE} is empty" fi diff --git a/lib/linter.sh b/lib/linter.sh index 96b4e6c2..9080edc2 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -133,6 +133,10 @@ YAML_ERROR_ON_WARNING="${YAML_ERROR_ON_WARNING:-false}" declare -l SAVE_SUPER_LINTER_SUMMARY SAVE_SUPER_LINTER_SUMMARY="${SAVE_SUPER_LINTER_SUMMARY:-false}" +declare -l REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT +REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT="${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT:-"false"}" +export REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT + # Define private output paths early because cleanup depends on those being defined DEFAULT_SUPER_LINTER_OUTPUT_DIRECTORY_NAME="super-linter-output" SUPER_LINTER_OUTPUT_DIRECTORY_NAME="${SUPER_LINTER_OUTPUT_DIRECTORY_NAME:-${DEFAULT_SUPER_LINTER_OUTPUT_DIRECTORY_NAME}}" @@ -631,6 +635,10 @@ cleanup() { local LOG_FILE_PATH="${GITHUB_WORKSPACE}/${LOG_FILE}" debug "LOG_FILE_PATH: ${LOG_FILE_PATH}" if [ "${CREATE_LOG_FILE}" = "true" ]; then + if [[ "${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}" == "true" ]] && + ! RemoveAnsiColorCodesFromFile "${LOG_TEMP}"; then + fatal "Error while removing ANSI color codes from ${LOG_TEMP}" + fi debug "Moving log file from ${LOG_TEMP} to ${LOG_FILE_PATH}" mv \ --force \ diff --git a/test/data/output/ansi-color-codes/super-linter-parallel-stdout-ARM b/test/data/output/ansi-color-codes/super-linter-parallel-stdout-ARM new file mode 100644 index 00000000..e259d39e --- /dev/null +++ b/test/data/output/ansi-color-codes/super-linter-parallel-stdout-ARM @@ -0,0 +1,72 @@ + +Validating arm\arm_good_1.json + JSONFiles Should Be Valid + [+] JSONFiles Should Be Valid (30 ms) +Pass : 1 +Fail : 0 +Total : 1 + + + adminUsername Should Not Be A Literal + [+] adminUsername Should Not Be A Literal (42 ms) + apiVersions Should Be Recent In Reference Functions + [+] apiVersions Should Be Recent In Reference Functions (23 ms) + apiVersions Should Be Recent + [+] apiVersions Should Be Recent (133 ms) + artifacts parameter + [+] artifacts parameter (12 ms) + CommandToExecute Must Use ProtectedSettings For Secrets + [+] CommandToExecute Must Use ProtectedSettings For Secrets (17 ms) + DependsOn Best Practices + [+] DependsOn Best Practices (8 ms) + Deployment Resources Must Not Be Debug + [+] Deployment Resources Must Not Be Debug (11 ms) + DeploymentTemplate Must Not Contain Hardcoded Uri + [+] DeploymentTemplate Must Not Contain Hardcoded Uri (72 ms) + DeploymentTemplate Schema Is Correct + [+] DeploymentTemplate Schema Is Correct (4 ms) + Dynamic Variable References Should Not Use Concat + [+] Dynamic Variable References Should Not Use Concat (8 ms) + IDs Should Be Derived From ResourceIDs + [+] IDs Should Be Derived From ResourceIDs (20 ms) + Location Should Not Be Hardcoded + [+] Location Should Not Be Hardcoded (157 ms) + ManagedIdentityExtension must not be used + [+] ManagedIdentityExtension must not be used (5 ms) + Min And Max Value Are Numbers + [+] Min And Max Value Are Numbers (13 ms) + Outputs Must Not Contain Secrets + [+] Outputs Must Not Contain Secrets (21 ms) + Parameter Types Should Be Consistent + [+] Parameter Types Should Be Consistent (23 ms) + Parameters Must Be Referenced + [+] Parameters Must Be Referenced (23 ms) + Password params must be secure + [+] Password params must be secure (7 ms) + providers apiVersions Is Not Permitted + [+] providers apiVersions Is Not Permitted (3 ms) + ResourceIds should not contain + [+] ResourceIds should not contain (23 ms) + Resources Should Have Location + [+] Resources Should Have Location (11 ms) + Resources Should Not Be Ambiguous + [+] Resources Should Not Be Ambiguous (13 ms) + Secure Params In Nested Deployments + [+] Secure Params In Nested Deployments (9 ms) + Secure String Parameters Cannot Have Default + [+] Secure String Parameters Cannot Have Default (5 ms) + Template Should Not Contain Blanks + [+] Template Should Not Contain Blanks (15 ms) + URIs Should Be Properly Constructed + [+] URIs Should Be Properly Constructed (82 ms) + Variables Must Be Referenced + [+] Variables Must Be Referenced (10 ms) + Virtual Machines Should Not Be Preview + [+] Virtual Machines Should Not Be Preview (12 ms) + VM Images Should Use Latest Version + [+] VM Images Should Use Latest Version (2 ms) + VM Size Should Be A Parameter + [+] VM Size Should Be A Parameter (12 ms) +Pass : 31 +Fail : 0 +Total : 31 diff --git a/test/data/output/ansi-color-codes/super-linter-parallel-stdout-ARM-no-ANSI-color-codes b/test/data/output/ansi-color-codes/super-linter-parallel-stdout-ARM-no-ANSI-color-codes new file mode 100644 index 00000000..1bb4ace8 --- /dev/null +++ b/test/data/output/ansi-color-codes/super-linter-parallel-stdout-ARM-no-ANSI-color-codes @@ -0,0 +1,72 @@ + +Validating arm\arm_good_1.json + JSONFiles Should Be Valid + [+] JSONFiles Should Be Valid (30 ms) +Pass : 1 +Fail : 0 +Total : 1 + + + adminUsername Should Not Be A Literal + [+] adminUsername Should Not Be A Literal (42 ms) + apiVersions Should Be Recent In Reference Functions + [+] apiVersions Should Be Recent In Reference Functions (23 ms) + apiVersions Should Be Recent + [+] apiVersions Should Be Recent (133 ms) + artifacts parameter + [+] artifacts parameter (12 ms) + CommandToExecute Must Use ProtectedSettings For Secrets + [+] CommandToExecute Must Use ProtectedSettings For Secrets (17 ms) + DependsOn Best Practices + [+] DependsOn Best Practices (8 ms) + Deployment Resources Must Not Be Debug + [+] Deployment Resources Must Not Be Debug (11 ms) + DeploymentTemplate Must Not Contain Hardcoded Uri + [+] DeploymentTemplate Must Not Contain Hardcoded Uri (72 ms) + DeploymentTemplate Schema Is Correct + [+] DeploymentTemplate Schema Is Correct (4 ms) + Dynamic Variable References Should Not Use Concat + [+] Dynamic Variable References Should Not Use Concat (8 ms) + IDs Should Be Derived From ResourceIDs + [+] IDs Should Be Derived From ResourceIDs (20 ms) + Location Should Not Be Hardcoded + [+] Location Should Not Be Hardcoded (157 ms) + ManagedIdentityExtension must not be used + [+] ManagedIdentityExtension must not be used (5 ms) + Min And Max Value Are Numbers + [+] Min And Max Value Are Numbers (13 ms) + Outputs Must Not Contain Secrets + [+] Outputs Must Not Contain Secrets (21 ms) + Parameter Types Should Be Consistent + [+] Parameter Types Should Be Consistent (23 ms) + Parameters Must Be Referenced + [+] Parameters Must Be Referenced (23 ms) + Password params must be secure + [+] Password params must be secure (7 ms) + providers apiVersions Is Not Permitted + [+] providers apiVersions Is Not Permitted (3 ms) + ResourceIds should not contain + [+] ResourceIds should not contain (23 ms) + Resources Should Have Location + [+] Resources Should Have Location (11 ms) + Resources Should Not Be Ambiguous + [+] Resources Should Not Be Ambiguous (13 ms) + Secure Params In Nested Deployments + [+] Secure Params In Nested Deployments (9 ms) + Secure String Parameters Cannot Have Default + [+] Secure String Parameters Cannot Have Default (5 ms) + Template Should Not Contain Blanks + [+] Template Should Not Contain Blanks (15 ms) + URIs Should Be Properly Constructed + [+] URIs Should Be Properly Constructed (82 ms) + Variables Must Be Referenced + [+] Variables Must Be Referenced (10 ms) + Virtual Machines Should Not Be Preview + [+] Virtual Machines Should Not Be Preview (12 ms) + VM Images Should Use Latest Version + [+] VM Images Should Use Latest Version (2 ms) + VM Size Should Be A Parameter + [+] VM Size Should Be A Parameter (12 ms) +Pass : 31 +Fail : 0 +Total : 31 diff --git a/test/lib/outputTest.sh b/test/lib/outputTest.sh index efb6a50e..eee13353 100755 --- a/test/lib/outputTest.sh +++ b/test/lib/outputTest.sh @@ -131,8 +131,29 @@ EOF notice "${FUNCTION_NAME} PASS" } +RemoveAnsiColorCodesFromFileTest() { + local FUNCTION_NAME + FUNCTION_NAME="${FUNCNAME[0]}" + info "${FUNCTION_NAME} start" + + InitWorkspace + + local TEST_CASE_FILE_WITH_ANSI_COLOR_CODES="test/data/output/ansi-color-codes/super-linter-parallel-stdout-ARM" + local EXPECTED_TEST_CASE_FILE_WITHOUT_ANSI_COLOR_CODES="test/data/output/ansi-color-codes/super-linter-parallel-stdout-ARM-no-ANSI-color-codes" + local INPUT_FILE + INPUT_FILE="${TEMP_WORKSPACE}/$(basename "${TEST_CASE_FILE_WITH_ANSI_COLOR_CODES}")" + cp "${TEST_CASE_FILE_WITH_ANSI_COLOR_CODES}" "${INPUT_FILE}" + RemoveAnsiColorCodesFromFile "${INPUT_FILE}" + AssertFileAndDirContentsMatch "${INPUT_FILE}" "${EXPECTED_TEST_CASE_FILE_WITHOUT_ANSI_COLOR_CODES}" + + CleanupWorkspace + + notice "${FUNCTION_NAME} PASS" +} + WriteSummaryMarkdownTableHeaderTest WriteSummaryMarkdownTableLineSuccessTest WriteSummaryMarkdownTableLineFailureTest WriteSummaryMarkdownTableFooterSuccessTest WriteSummaryMarkdownTableFooterFailureTest +RemoveAnsiColorCodesFromFileTest diff --git a/test/run-super-linter-tests.sh b/test/run-super-linter-tests.sh index 25fdd9fc..49d80e57 100755 --- a/test/run-super-linter-tests.sh +++ b/test/run-super-linter-tests.sh @@ -246,6 +246,10 @@ debug "Super-linter main output path: ${SUPER_LINTER_MAIN_OUTPUT_PATH}" SUPER_LINTER_OUTPUT_PATH="${SUPER_LINTER_MAIN_OUTPUT_PATH}/super-linter" debug "Super-linter output path: ${SUPER_LINTER_OUTPUT_PATH}" +# Remove color codes from output by default +REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT="${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT:-"true"}" +COMMAND_TO_RUN+=(--env REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT="${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}") + COMMAND_TO_RUN+=(-e CREATE_LOG_FILE="${CREATE_LOG_FILE}") COMMAND_TO_RUN+=(-e LOG_LEVEL="${LOG_LEVEL:-"DEBUG"}") COMMAND_TO_RUN+=(-e RUN_LOCAL="${RUN_LOCAL:-true}") @@ -324,6 +328,12 @@ if [[ "${CREATE_LOG_FILE}" == true ]]; then debug "Copying Super-linter log from the workspace (${SUPER_LINTER_WORKSPACE}) to the current working directory for easier inspection" cp -v "${LOG_FILE_PATH}" "$(pwd)/" fi + + if [[ "${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}" == "true" ]]; then + if AreAnsiColorCodesInFile "${LOG_FILE_PATH}"; then + fatal "${LOG_FILE_PATH} contains unexpected ANSI color codes" + fi + fi fi else debug "Log file was not requested. CREATE_LOG_FILE: ${CREATE_LOG_FILE}" @@ -344,10 +354,33 @@ if [[ "${SAVE_SUPER_LINTER_OUTPUT}" == true ]]; then if [[ "${SUPER_LINTER_WORKSPACE}" != "$(pwd)" ]]; then debug "Copying Super-linter output from the workspace (${SUPER_LINTER_WORKSPACE}) to the current working directory for easier inspection" - SUPER_LINTER_OUTPUT_PATH_PWD="$(pwd)/super-linter-output/" - mkdir -p "${SUPER_LINTER_OUTPUT_PATH_PWD}" - cp -r "${SUPER_LINTER_OUTPUT_PATH}" "${SUPER_LINTER_OUTPUT_PATH_PWD}" + SUPER_LINTER_MAIN_OUTPUT_PATH_PWD="$(pwd)/${SUPER_LINTER_OUTPUT_DIRECTORY_NAME}" + SUPER_LINTER_OUTPUT_PATH_PWD="${SUPER_LINTER_MAIN_OUTPUT_PATH_PWD}/super-linter" + mkdir -p "${SUPER_LINTER_MAIN_OUTPUT_PATH_PWD}" + cp -r "${SUPER_LINTER_OUTPUT_PATH}" "${SUPER_LINTER_MAIN_OUTPUT_PATH_PWD}/" fi + + for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do + LANGUAGE_STDERR_FILE_PATH="${SUPER_LINTER_OUTPUT_PATH_PWD:-"${SUPER_LINTER_OUTPUT_PATH}"}/super-linter-parallel-stderr-${LANGUAGE}" + LANGUAGE_STDOUT_FILE_PATH="${SUPER_LINTER_OUTPUT_PATH_PWD:-"${SUPER_LINTER_OUTPUT_PATH}"}/super-linter-parallel-stdout-${LANGUAGE}" + + if [[ "${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}" == "true" ]]; then + if [[ -e "${LANGUAGE_STDERR_FILE_PATH}" ]]; then + if AreAnsiColorCodesInFile "${LANGUAGE_STDERR_FILE_PATH}"; then + fatal "${LANGUAGE_STDERR_FILE_PATH} contains unexpected ANSI color codes" + fi + fi + + if [[ -e "${LANGUAGE_STDOUT_FILE_PATH}" ]]; then + if AreAnsiColorCodesInFile "${LANGUAGE_STDOUT_FILE_PATH}"; then + fatal "${LANGUAGE_STDOUT_FILE_PATH} contains unexpected ANSI color codes" + fi + fi + fi + + unset LANGUAGE_STDERR_FILE_PATH + unset LANGUAGE_STDOUT_FILE_PATH + done fi else debug "Super-linter output was not requested. SAVE_SUPER_LINTER_OUTPUT: ${SAVE_SUPER_LINTER_OUTPUT}" diff --git a/test/testUtils.sh b/test/testUtils.sh index a7f317e5..661e02d5 100755 --- a/test/testUtils.sh +++ b/test/testUtils.sh @@ -15,6 +15,13 @@ CREATE_LOG_FILE="true" # shellcheck source=/dev/null source "lib/functions/log.sh" +# shellcheck source=/dev/null +source "lib/globals/languages.sh" + +# Because we need variables defined there +# shellcheck source=/dev/null +source "lib/functions/output.sh" + # TODO: use TEST_CASE_FOLDER instead of redefining this after we extract the # initialization of TEST_CASE_FOLDER from linter.sh # shellcheck disable=SC2034 @@ -168,6 +175,17 @@ IsStandardImage() { fi } +AreAnsiColorCodesInFile() { + local FILE_TO_SEARCH_IN="${1}" + if grep --color=never --quiet --perl-regexp "${ANSI_COLOR_CODES_SEARCH_PATTERN}" "${FILE_TO_SEARCH_IN}"; then + debug "Found at least one ANSI color code in ${FILE_TO_SEARCH_IN}" + return 0 + else + debug "Found no ANSI color codes in ${FILE_TO_SEARCH_IN}" + return 1 + fi +} + RemoveTestLeftovers() { local LEFTOVERS_TO_CLEAN=() LEFTOVERS_TO_CLEAN+=("${SUPER_LINTER_WORKSPACE}/${LINTERS_TEST_CASE_DIRECTORY}/rust_clippy/bad/target")