feat: optionally remove color codes from output (#6095)

If REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT is set to true, remove ANSI color
codes from linters stdout and stderr files, and from the Super-linter
log file.

Close #5540
This commit is contained in:
Marco Ferrari 2024-08-31 19:50:00 +02:00 committed by GitHub
parent 05d4d4e128
commit 94920ffcc7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 255 additions and 3 deletions

View file

@ -2,4 +2,5 @@ DEFAULT_BRANCH=main
DEFAULT_WORKSPACE=/workspaces/super-linter
ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true
LOG_LEVEL=DEBUG
REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT=true
RUN_LOCAL=true

View file

@ -296,6 +296,7 @@ You can configure Super-linter using the following environment variables:
| **PYTHON_PYLINT_CONFIG_FILE** | `.python-lint` | Filename for [pylint configuration](https://pylint.pycqa.org/en/latest/user_guide/run.html?highlight=rcfile#command-line-options) (ex: `.python-lint`, `.pylintrc`) |
| **PYTHON_RUFF_CONFIG_FILE** | `.ruff.toml` | Filename for [ruff configuration](https://docs.astral.sh/ruff/configuration/) |
| **RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES** | not set | Comma-separated filenames for [renovate shareable config preset](https://docs.renovatebot.com/config-presets/) (ex: `default.json`) |
| **REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT** | `false` | If set to `true`, Super-linter removes ANSI color codes from linters stdout and stderr files, and from the Super-linter log file. |
| **RUBY_CONFIG_FILE** | `.ruby-lint.yml` | Filename for [rubocop configuration](https://docs.rubocop.org/rubocop/configuration.html) (ex: `.ruby-lint.yml`, `.rubocop.yml`) |
| **RUST_CLIPPY_COMMAND_OPTIONS** | not set | Additional options and arguments to pass to the command when running Clippy. |
| **SAVE_SUPER_LINTER_OUTPUT** | `false` | If set to `true`, Super-linter will save its output in the workspace. For more information, see [Super-linter outputs](#super-linter-outputs). |

View file

@ -55,3 +55,16 @@ FormatSuperLinterSummaryFile() {
return 1
fi
}
# 0x1B (= ^[) is the control code that starts all ANSI color codes escape sequences
# Ref: https://en.wikipedia.org/wiki/ANSI_escape_code#C0_control_codes
ANSI_COLOR_CODES_SEARCH_PATTERN='\x1b\[[0-9;]*m'
export ANSI_COLOR_CODES_SEARCH_PATTERN
RemoveAnsiColorCodesFromFile() {
local FILE_PATH="${1}"
debug "Removing ANSI color codes from ${FILE_PATH}"
if ! sed -i "s/${ANSI_COLOR_CODES_SEARCH_PATTERN}//g" "${FILE_PATH}"; then
error "Error while removing ANSI color codes from ${FILE_PATH}"
return 1
fi
}

View file

@ -17,6 +17,7 @@ function ValidateBooleanConfigurationVariables() {
ValidateBooleanVariable "LOG_VERBOSE" "${LOG_VERBOSE}"
ValidateBooleanVariable "LOG_WARN" "${LOG_WARN}"
ValidateBooleanVariable "MULTI_STATUS" "${MULTI_STATUS}"
ValidateBooleanVariable "REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT" "${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}"
ValidateBooleanVariable "RUN_LOCAL" "${RUN_LOCAL}"
ValidateBooleanVariable "SAVE_SUPER_LINTER_OUTPUT" "${SAVE_SUPER_LINTER_OUTPUT}"
ValidateBooleanVariable "SAVE_SUPER_LINTER_SUMMARY" "${SAVE_SUPER_LINTER_SUMMARY}"

View file

@ -211,6 +211,10 @@ function LintCodebase() {
fatal "Error when loading stdout for ${FILE_TYPE}:\n${STDOUT_LINTER}"
fi
# Load output functions because we might need to process stdout and stderr
# shellcheck source=/dev/null
source /action/lib/functions/output.sh
if [ -n "${STDOUT_LINTER}" ]; then
info "Command output for ${FILE_TYPE}:\n------\n${STDOUT_LINTER}\n------"
@ -218,6 +222,10 @@ function LintCodebase() {
STDOUT_LINTER_FILE_PATH="${SUPER_LINTER_PRIVATE_OUTPUT_DIRECTORY_PATH}/super-linter-parallel-stdout-${FILE_TYPE}"
debug "Saving stdout for ${FILE_TYPE} to ${STDOUT_LINTER_FILE_PATH} in case we need it later"
printf '%s\n' "${STDOUT_LINTER}" >"${STDOUT_LINTER_FILE_PATH}"
if [[ "${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}" == "true" ]] &&
! RemoveAnsiColorCodesFromFile "${STDOUT_LINTER_FILE_PATH}"; then
fatal "Error while removing ANSI color codes from ${STDOUT_LINTER_FILE_PATH}"
fi
else
debug "Stdout for ${FILE_TYPE} is empty"
fi
@ -234,6 +242,10 @@ function LintCodebase() {
STDERR_LINTER_FILE_PATH="${SUPER_LINTER_PRIVATE_OUTPUT_DIRECTORY_PATH}/super-linter-parallel-stderr-${FILE_TYPE}"
debug "Saving stderr for ${FILE_TYPE} to ${STDERR_LINTER_FILE_PATH} in case we need it later"
printf '%s\n' "${STDERR_LINTER}" >"${STDERR_LINTER_FILE_PATH}"
if [[ "${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}" == "true" ]] &&
! RemoveAnsiColorCodesFromFile "${STDERR_LINTER_FILE_PATH}"; then
fatal "Error while removing ANSI color codes from ${STDERR_LINTER_FILE_PATH}"
fi
else
debug "Stderr for ${FILE_TYPE} is empty"
fi

View file

@ -133,6 +133,10 @@ YAML_ERROR_ON_WARNING="${YAML_ERROR_ON_WARNING:-false}"
declare -l SAVE_SUPER_LINTER_SUMMARY
SAVE_SUPER_LINTER_SUMMARY="${SAVE_SUPER_LINTER_SUMMARY:-false}"
declare -l REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT
REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT="${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT:-"false"}"
export REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT
# Define private output paths early because cleanup depends on those being defined
DEFAULT_SUPER_LINTER_OUTPUT_DIRECTORY_NAME="super-linter-output"
SUPER_LINTER_OUTPUT_DIRECTORY_NAME="${SUPER_LINTER_OUTPUT_DIRECTORY_NAME:-${DEFAULT_SUPER_LINTER_OUTPUT_DIRECTORY_NAME}}"
@ -631,6 +635,10 @@ cleanup() {
local LOG_FILE_PATH="${GITHUB_WORKSPACE}/${LOG_FILE}"
debug "LOG_FILE_PATH: ${LOG_FILE_PATH}"
if [ "${CREATE_LOG_FILE}" = "true" ]; then
if [[ "${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}" == "true" ]] &&
! RemoveAnsiColorCodesFromFile "${LOG_TEMP}"; then
fatal "Error while removing ANSI color codes from ${LOG_TEMP}"
fi
debug "Moving log file from ${LOG_TEMP} to ${LOG_FILE_PATH}"
mv \
--force \

View file

@ -0,0 +1,72 @@
Validating arm\arm_good_1.json
 JSONFiles Should Be Valid
 [+] JSONFiles Should Be Valid (30 ms)
Pass : 1
Fail : 0
Total : 1

 adminUsername Should Not Be A Literal
 [+] adminUsername Should Not Be A Literal (42 ms)
 apiVersions Should Be Recent In Reference Functions
 [+] apiVersions Should Be Recent In Reference Functions (23 ms)
 apiVersions Should Be Recent
 [+] apiVersions Should Be Recent (133 ms)
 artifacts parameter
 [+] artifacts parameter (12 ms)
 CommandToExecute Must Use ProtectedSettings For Secrets
 [+] CommandToExecute Must Use ProtectedSettings For Secrets (17 ms)
 DependsOn Best Practices
 [+] DependsOn Best Practices (8 ms)
 Deployment Resources Must Not Be Debug
 [+] Deployment Resources Must Not Be Debug (11 ms)
 DeploymentTemplate Must Not Contain Hardcoded Uri
 [+] DeploymentTemplate Must Not Contain Hardcoded Uri (72 ms)
 DeploymentTemplate Schema Is Correct
 [+] DeploymentTemplate Schema Is Correct (4 ms)
 Dynamic Variable References Should Not Use Concat
 [+] Dynamic Variable References Should Not Use Concat (8 ms)
 IDs Should Be Derived From ResourceIDs
 [+] IDs Should Be Derived From ResourceIDs (20 ms)
 Location Should Not Be Hardcoded
 [+] Location Should Not Be Hardcoded (157 ms)
 ManagedIdentityExtension must not be used
 [+] ManagedIdentityExtension must not be used (5 ms)
 Min And Max Value Are Numbers
 [+] Min And Max Value Are Numbers (13 ms)
 Outputs Must Not Contain Secrets
 [+] Outputs Must Not Contain Secrets (21 ms)
 Parameter Types Should Be Consistent
 [+] Parameter Types Should Be Consistent (23 ms)
 Parameters Must Be Referenced
 [+] Parameters Must Be Referenced (23 ms)
 Password params must be secure
 [+] Password params must be secure (7 ms)
 providers apiVersions Is Not Permitted
 [+] providers apiVersions Is Not Permitted (3 ms)
 ResourceIds should not contain
 [+] ResourceIds should not contain (23 ms)
 Resources Should Have Location
 [+] Resources Should Have Location (11 ms)
 Resources Should Not Be Ambiguous
 [+] Resources Should Not Be Ambiguous (13 ms)
 Secure Params In Nested Deployments
 [+] Secure Params In Nested Deployments (9 ms)
 Secure String Parameters Cannot Have Default
 [+] Secure String Parameters Cannot Have Default (5 ms)
 Template Should Not Contain Blanks
 [+] Template Should Not Contain Blanks (15 ms)
 URIs Should Be Properly Constructed
 [+] URIs Should Be Properly Constructed (82 ms)
 Variables Must Be Referenced
 [+] Variables Must Be Referenced (10 ms)
 Virtual Machines Should Not Be Preview
 [+] Virtual Machines Should Not Be Preview (12 ms)
 VM Images Should Use Latest Version
 [+] VM Images Should Use Latest Version (2 ms)
 VM Size Should Be A Parameter
 [+] VM Size Should Be A Parameter (12 ms)
Pass : 31
Fail : 0
Total : 31

View file

@ -0,0 +1,72 @@
Validating arm\arm_good_1.json
JSONFiles Should Be Valid
[+] JSONFiles Should Be Valid (30 ms)
Pass : 1
Fail : 0
Total : 1
adminUsername Should Not Be A Literal
[+] adminUsername Should Not Be A Literal (42 ms)
apiVersions Should Be Recent In Reference Functions
[+] apiVersions Should Be Recent In Reference Functions (23 ms)
apiVersions Should Be Recent
[+] apiVersions Should Be Recent (133 ms)
artifacts parameter
[+] artifacts parameter (12 ms)
CommandToExecute Must Use ProtectedSettings For Secrets
[+] CommandToExecute Must Use ProtectedSettings For Secrets (17 ms)
DependsOn Best Practices
[+] DependsOn Best Practices (8 ms)
Deployment Resources Must Not Be Debug
[+] Deployment Resources Must Not Be Debug (11 ms)
DeploymentTemplate Must Not Contain Hardcoded Uri
[+] DeploymentTemplate Must Not Contain Hardcoded Uri (72 ms)
DeploymentTemplate Schema Is Correct
[+] DeploymentTemplate Schema Is Correct (4 ms)
Dynamic Variable References Should Not Use Concat
[+] Dynamic Variable References Should Not Use Concat (8 ms)
IDs Should Be Derived From ResourceIDs
[+] IDs Should Be Derived From ResourceIDs (20 ms)
Location Should Not Be Hardcoded
[+] Location Should Not Be Hardcoded (157 ms)
ManagedIdentityExtension must not be used
[+] ManagedIdentityExtension must not be used (5 ms)
Min And Max Value Are Numbers
[+] Min And Max Value Are Numbers (13 ms)
Outputs Must Not Contain Secrets
[+] Outputs Must Not Contain Secrets (21 ms)
Parameter Types Should Be Consistent
[+] Parameter Types Should Be Consistent (23 ms)
Parameters Must Be Referenced
[+] Parameters Must Be Referenced (23 ms)
Password params must be secure
[+] Password params must be secure (7 ms)
providers apiVersions Is Not Permitted
[+] providers apiVersions Is Not Permitted (3 ms)
ResourceIds should not contain
[+] ResourceIds should not contain (23 ms)
Resources Should Have Location
[+] Resources Should Have Location (11 ms)
Resources Should Not Be Ambiguous
[+] Resources Should Not Be Ambiguous (13 ms)
Secure Params In Nested Deployments
[+] Secure Params In Nested Deployments (9 ms)
Secure String Parameters Cannot Have Default
[+] Secure String Parameters Cannot Have Default (5 ms)
Template Should Not Contain Blanks
[+] Template Should Not Contain Blanks (15 ms)
URIs Should Be Properly Constructed
[+] URIs Should Be Properly Constructed (82 ms)
Variables Must Be Referenced
[+] Variables Must Be Referenced (10 ms)
Virtual Machines Should Not Be Preview
[+] Virtual Machines Should Not Be Preview (12 ms)
VM Images Should Use Latest Version
[+] VM Images Should Use Latest Version (2 ms)
VM Size Should Be A Parameter
[+] VM Size Should Be A Parameter (12 ms)
Pass : 31
Fail : 0
Total : 31

View file

@ -131,8 +131,29 @@ EOF
notice "${FUNCTION_NAME} PASS"
}
RemoveAnsiColorCodesFromFileTest() {
local FUNCTION_NAME
FUNCTION_NAME="${FUNCNAME[0]}"
info "${FUNCTION_NAME} start"
InitWorkspace
local TEST_CASE_FILE_WITH_ANSI_COLOR_CODES="test/data/output/ansi-color-codes/super-linter-parallel-stdout-ARM"
local EXPECTED_TEST_CASE_FILE_WITHOUT_ANSI_COLOR_CODES="test/data/output/ansi-color-codes/super-linter-parallel-stdout-ARM-no-ANSI-color-codes"
local INPUT_FILE
INPUT_FILE="${TEMP_WORKSPACE}/$(basename "${TEST_CASE_FILE_WITH_ANSI_COLOR_CODES}")"
cp "${TEST_CASE_FILE_WITH_ANSI_COLOR_CODES}" "${INPUT_FILE}"
RemoveAnsiColorCodesFromFile "${INPUT_FILE}"
AssertFileAndDirContentsMatch "${INPUT_FILE}" "${EXPECTED_TEST_CASE_FILE_WITHOUT_ANSI_COLOR_CODES}"
CleanupWorkspace
notice "${FUNCTION_NAME} PASS"
}
WriteSummaryMarkdownTableHeaderTest
WriteSummaryMarkdownTableLineSuccessTest
WriteSummaryMarkdownTableLineFailureTest
WriteSummaryMarkdownTableFooterSuccessTest
WriteSummaryMarkdownTableFooterFailureTest
RemoveAnsiColorCodesFromFileTest

View file

@ -246,6 +246,10 @@ debug "Super-linter main output path: ${SUPER_LINTER_MAIN_OUTPUT_PATH}"
SUPER_LINTER_OUTPUT_PATH="${SUPER_LINTER_MAIN_OUTPUT_PATH}/super-linter"
debug "Super-linter output path: ${SUPER_LINTER_OUTPUT_PATH}"
# Remove color codes from output by default
REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT="${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT:-"true"}"
COMMAND_TO_RUN+=(--env REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT="${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}")
COMMAND_TO_RUN+=(-e CREATE_LOG_FILE="${CREATE_LOG_FILE}")
COMMAND_TO_RUN+=(-e LOG_LEVEL="${LOG_LEVEL:-"DEBUG"}")
COMMAND_TO_RUN+=(-e RUN_LOCAL="${RUN_LOCAL:-true}")
@ -324,6 +328,12 @@ if [[ "${CREATE_LOG_FILE}" == true ]]; then
debug "Copying Super-linter log from the workspace (${SUPER_LINTER_WORKSPACE}) to the current working directory for easier inspection"
cp -v "${LOG_FILE_PATH}" "$(pwd)/"
fi
if [[ "${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}" == "true" ]]; then
if AreAnsiColorCodesInFile "${LOG_FILE_PATH}"; then
fatal "${LOG_FILE_PATH} contains unexpected ANSI color codes"
fi
fi
fi
else
debug "Log file was not requested. CREATE_LOG_FILE: ${CREATE_LOG_FILE}"
@ -344,10 +354,33 @@ if [[ "${SAVE_SUPER_LINTER_OUTPUT}" == true ]]; then
if [[ "${SUPER_LINTER_WORKSPACE}" != "$(pwd)" ]]; then
debug "Copying Super-linter output from the workspace (${SUPER_LINTER_WORKSPACE}) to the current working directory for easier inspection"
SUPER_LINTER_OUTPUT_PATH_PWD="$(pwd)/super-linter-output/"
mkdir -p "${SUPER_LINTER_OUTPUT_PATH_PWD}"
cp -r "${SUPER_LINTER_OUTPUT_PATH}" "${SUPER_LINTER_OUTPUT_PATH_PWD}"
SUPER_LINTER_MAIN_OUTPUT_PATH_PWD="$(pwd)/${SUPER_LINTER_OUTPUT_DIRECTORY_NAME}"
SUPER_LINTER_OUTPUT_PATH_PWD="${SUPER_LINTER_MAIN_OUTPUT_PATH_PWD}/super-linter"
mkdir -p "${SUPER_LINTER_MAIN_OUTPUT_PATH_PWD}"
cp -r "${SUPER_LINTER_OUTPUT_PATH}" "${SUPER_LINTER_MAIN_OUTPUT_PATH_PWD}/"
fi
for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do
LANGUAGE_STDERR_FILE_PATH="${SUPER_LINTER_OUTPUT_PATH_PWD:-"${SUPER_LINTER_OUTPUT_PATH}"}/super-linter-parallel-stderr-${LANGUAGE}"
LANGUAGE_STDOUT_FILE_PATH="${SUPER_LINTER_OUTPUT_PATH_PWD:-"${SUPER_LINTER_OUTPUT_PATH}"}/super-linter-parallel-stdout-${LANGUAGE}"
if [[ "${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}" == "true" ]]; then
if [[ -e "${LANGUAGE_STDERR_FILE_PATH}" ]]; then
if AreAnsiColorCodesInFile "${LANGUAGE_STDERR_FILE_PATH}"; then
fatal "${LANGUAGE_STDERR_FILE_PATH} contains unexpected ANSI color codes"
fi
fi
if [[ -e "${LANGUAGE_STDOUT_FILE_PATH}" ]]; then
if AreAnsiColorCodesInFile "${LANGUAGE_STDOUT_FILE_PATH}"; then
fatal "${LANGUAGE_STDOUT_FILE_PATH} contains unexpected ANSI color codes"
fi
fi
fi
unset LANGUAGE_STDERR_FILE_PATH
unset LANGUAGE_STDOUT_FILE_PATH
done
fi
else
debug "Super-linter output was not requested. SAVE_SUPER_LINTER_OUTPUT: ${SAVE_SUPER_LINTER_OUTPUT}"

View file

@ -15,6 +15,13 @@ CREATE_LOG_FILE="true"
# shellcheck source=/dev/null
source "lib/functions/log.sh"
# shellcheck source=/dev/null
source "lib/globals/languages.sh"
# Because we need variables defined there
# shellcheck source=/dev/null
source "lib/functions/output.sh"
# TODO: use TEST_CASE_FOLDER instead of redefining this after we extract the
# initialization of TEST_CASE_FOLDER from linter.sh
# shellcheck disable=SC2034
@ -168,6 +175,17 @@ IsStandardImage() {
fi
}
AreAnsiColorCodesInFile() {
local FILE_TO_SEARCH_IN="${1}"
if grep --color=never --quiet --perl-regexp "${ANSI_COLOR_CODES_SEARCH_PATTERN}" "${FILE_TO_SEARCH_IN}"; then
debug "Found at least one ANSI color code in ${FILE_TO_SEARCH_IN}"
return 0
else
debug "Found no ANSI color codes in ${FILE_TO_SEARCH_IN}"
return 1
fi
}
RemoveTestLeftovers() {
local LEFTOVERS_TO_CLEAN=()
LEFTOVERS_TO_CLEAN+=("${SUPER_LINTER_WORKSPACE}/${LINTERS_TEST_CASE_DIRECTORY}/rust_clippy/bad/target")