feat: optionally remove color codes from output (#6095)

If REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT is set to true, remove ANSI color
codes from linters stdout and stderr files, and from the Super-linter
log file.

Close #5540

.devcontainer/devcontainer.env

@@ -2,4 +2,5 @@ DEFAULT_BRANCH=main
 DEFAULT_WORKSPACE=/workspaces/super-linter
 ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true
 LOG_LEVEL=DEBUG
+REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT=true
 RUN_LOCAL=true

README.md

@@ -296,6 +296,7 @@ You can configure Super-linter using the following environment variables:
 | **PYTHON_PYLINT_CONFIG_FILE**                   | `.python-lint`                                                               | Filename for [pylint configuration](https://pylint.pycqa.org/en/latest/user_guide/run.html?highlight=rcfile#command-line-options) (ex: `.python-lint`, `.pylintrc`)                                                                                                                                                                  |
 | **PYTHON_RUFF_CONFIG_FILE**                     | `.ruff.toml`                                                                 | Filename for [ruff configuration](https://docs.astral.sh/ruff/configuration/)                                                                                                                                                                                                                                                        |
 | **RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES** | not set                                                                      | Comma-separated filenames for [renovate shareable config preset](https://docs.renovatebot.com/config-presets/) (ex: `default.json`)                                                                                                                                                                                                  |
+| **REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT**         | `false`                                                                      | If set to `true`, Super-linter removes ANSI color codes from linters stdout and stderr files, and from the Super-linter log file.                                                                                                                                                                                                    |
 | **RUBY_CONFIG_FILE**                            | `.ruby-lint.yml`                                                             | Filename for [rubocop configuration](https://docs.rubocop.org/rubocop/configuration.html) (ex: `.ruby-lint.yml`, `.rubocop.yml`)                                                                                                                                                                                                     |
 | **RUST_CLIPPY_COMMAND_OPTIONS**                 | not set                                                                      | Additional options and arguments to pass to the command when running Clippy.                                                                                                                                                                                                                                                         |
 | **SAVE_SUPER_LINTER_OUTPUT**                    | `false`                                                                      | If set to `true`, Super-linter will save its output in the workspace. For more information, see [Super-linter outputs](#super-linter-outputs).                                                                                                                                                                                       |

lib/functions/output.sh

@@ -55,3 +55,16 @@ FormatSuperLinterSummaryFile() {
     return 1
   fi
 }
+
+# 0x1B (= ^[) is the control code that starts all ANSI color codes escape sequences
+# Ref: https://en.wikipedia.org/wiki/ANSI_escape_code#C0_control_codes
+ANSI_COLOR_CODES_SEARCH_PATTERN='\x1b\[[0-9;]*m'
+export ANSI_COLOR_CODES_SEARCH_PATTERN
+RemoveAnsiColorCodesFromFile() {
+  local FILE_PATH="${1}"
+  debug "Removing ANSI color codes from ${FILE_PATH}"
+  if ! sed -i "s/${ANSI_COLOR_CODES_SEARCH_PATTERN}//g" "${FILE_PATH}"; then
+    error "Error while removing ANSI color codes from ${FILE_PATH}"
+    return 1
+  fi
+}

lib/functions/validation.sh

@@ -17,6 +17,7 @@ function ValidateBooleanConfigurationVariables() {
   ValidateBooleanVariable "LOG_VERBOSE" "${LOG_VERBOSE}"
   ValidateBooleanVariable "LOG_WARN" "${LOG_WARN}"
   ValidateBooleanVariable "MULTI_STATUS" "${MULTI_STATUS}"
+  ValidateBooleanVariable "REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT" "${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}"
   ValidateBooleanVariable "RUN_LOCAL" "${RUN_LOCAL}"
   ValidateBooleanVariable "SAVE_SUPER_LINTER_OUTPUT" "${SAVE_SUPER_LINTER_OUTPUT}"
   ValidateBooleanVariable "SAVE_SUPER_LINTER_SUMMARY" "${SAVE_SUPER_LINTER_SUMMARY}"

lib/functions/worker.sh

@@ -211,6 +211,10 @@ function LintCodebase() {
     fatal "Error when loading stdout for ${FILE_TYPE}:\n${STDOUT_LINTER}"
   fi
 
+  # Load output functions because we might need to process stdout and stderr
+  # shellcheck source=/dev/null
+  source /action/lib/functions/output.sh
+
   if [ -n "${STDOUT_LINTER}" ]; then
     info "Command output for ${FILE_TYPE}:\n------\n${STDOUT_LINTER}\n------"
 
@@ -218,6 +222,10 @@ function LintCodebase() {
     STDOUT_LINTER_FILE_PATH="${SUPER_LINTER_PRIVATE_OUTPUT_DIRECTORY_PATH}/super-linter-parallel-stdout-${FILE_TYPE}"
     debug "Saving stdout for ${FILE_TYPE} to ${STDOUT_LINTER_FILE_PATH} in case we need it later"
     printf '%s\n' "${STDOUT_LINTER}" >"${STDOUT_LINTER_FILE_PATH}"
+    if [[ "${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}" == "true" ]] &&
+      ! RemoveAnsiColorCodesFromFile "${STDOUT_LINTER_FILE_PATH}"; then
+      fatal "Error while removing ANSI color codes from ${STDOUT_LINTER_FILE_PATH}"
+    fi
   else
     debug "Stdout for ${FILE_TYPE} is empty"
   fi
@@ -234,6 +242,10 @@ function LintCodebase() {
     STDERR_LINTER_FILE_PATH="${SUPER_LINTER_PRIVATE_OUTPUT_DIRECTORY_PATH}/super-linter-parallel-stderr-${FILE_TYPE}"
     debug "Saving stderr for ${FILE_TYPE} to ${STDERR_LINTER_FILE_PATH} in case we need it later"
     printf '%s\n' "${STDERR_LINTER}" >"${STDERR_LINTER_FILE_PATH}"
+    if [[ "${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}" == "true" ]] &&
+      ! RemoveAnsiColorCodesFromFile "${STDERR_LINTER_FILE_PATH}"; then
+      fatal "Error while removing ANSI color codes from ${STDERR_LINTER_FILE_PATH}"
+    fi
   else
     debug "Stderr for ${FILE_TYPE} is empty"
   fi

lib/linter.sh

@@ -133,6 +133,10 @@ YAML_ERROR_ON_WARNING="${YAML_ERROR_ON_WARNING:-false}"
 declare -l SAVE_SUPER_LINTER_SUMMARY
 SAVE_SUPER_LINTER_SUMMARY="${SAVE_SUPER_LINTER_SUMMARY:-false}"
 
+declare -l REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT
+REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT="${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT:-"false"}"
+export REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT
+
 # Define private output paths early because cleanup depends on those being defined
 DEFAULT_SUPER_LINTER_OUTPUT_DIRECTORY_NAME="super-linter-output"
 SUPER_LINTER_OUTPUT_DIRECTORY_NAME="${SUPER_LINTER_OUTPUT_DIRECTORY_NAME:-${DEFAULT_SUPER_LINTER_OUTPUT_DIRECTORY_NAME}}"
@@ -631,6 +635,10 @@ cleanup() {
     local LOG_FILE_PATH="${GITHUB_WORKSPACE}/${LOG_FILE}"
     debug "LOG_FILE_PATH: ${LOG_FILE_PATH}"
     if [ "${CREATE_LOG_FILE}" = "true" ]; then
+      if [[ "${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}" == "true" ]] &&
+        ! RemoveAnsiColorCodesFromFile "${LOG_TEMP}"; then
+        fatal "Error while removing ANSI color codes from ${LOG_TEMP}"
+      fi
       debug "Moving log file from ${LOG_TEMP} to ${LOG_FILE_PATH}"
       mv \
         --force \

test/data/output/ansi-color-codes/super-linter-parallel-stdout-ARM

@@ -0,0 +1,72 @@
+
+Validating arm\arm_good_1.json
+  JSONFiles Should Be Valid
+    [+] JSONFiles Should Be Valid (30 ms)
+Pass  : 1
+Fail  : 0
+Total : 1
+
+
+  adminUsername Should Not Be A Literal
+    [+] adminUsername Should Not Be A Literal (42 ms)
+  apiVersions Should Be Recent In Reference Functions
+    [+] apiVersions Should Be Recent In Reference Functions (23 ms)
+  apiVersions Should Be Recent
+    [+] apiVersions Should Be Recent (133 ms)
+  artifacts parameter
+    [+] artifacts parameter (12 ms)
+  CommandToExecute Must Use ProtectedSettings For Secrets
+    [+] CommandToExecute Must Use ProtectedSettings For Secrets (17 ms)
+  DependsOn Best Practices
+    [+] DependsOn Best Practices (8 ms)
+  Deployment Resources Must Not Be Debug
+    [+] Deployment Resources Must Not Be Debug (11 ms)
+  DeploymentTemplate Must Not Contain Hardcoded Uri
+    [+] DeploymentTemplate Must Not Contain Hardcoded Uri (72 ms)
+  DeploymentTemplate Schema Is Correct
+    [+] DeploymentTemplate Schema Is Correct (4 ms)
+  Dynamic Variable References Should Not Use Concat
+    [+] Dynamic Variable References Should Not Use Concat (8 ms)
+  IDs Should Be Derived From ResourceIDs
+    [+] IDs Should Be Derived From ResourceIDs (20 ms)
+  Location Should Not Be Hardcoded
+    [+] Location Should Not Be Hardcoded (157 ms)
+  ManagedIdentityExtension must not be used
+    [+] ManagedIdentityExtension must not be used (5 ms)
+  Min And Max Value Are Numbers
+    [+] Min And Max Value Are Numbers (13 ms)
+  Outputs Must Not Contain Secrets
+    [+] Outputs Must Not Contain Secrets (21 ms)
+  Parameter Types Should Be Consistent
+    [+] Parameter Types Should Be Consistent (23 ms)
+  Parameters Must Be Referenced
+    [+] Parameters Must Be Referenced (23 ms)
+  Password params must be secure
+    [+] Password params must be secure (7 ms)
+  providers apiVersions Is Not Permitted
+    [+] providers apiVersions Is Not Permitted (3 ms)
+  ResourceIds should not contain
+    [+] ResourceIds should not contain (23 ms)
+  Resources Should Have Location
+    [+] Resources Should Have Location (11 ms)
+  Resources Should Not Be Ambiguous
+    [+] Resources Should Not Be Ambiguous (13 ms)
+  Secure Params In Nested Deployments
+    [+] Secure Params In Nested Deployments (9 ms)
+  Secure String Parameters Cannot Have Default
+    [+] Secure String Parameters Cannot Have Default (5 ms)
+  Template Should Not Contain Blanks
+    [+] Template Should Not Contain Blanks (15 ms)
+  URIs Should Be Properly Constructed
+    [+] URIs Should Be Properly Constructed (82 ms)
+  Variables Must Be Referenced
+    [+] Variables Must Be Referenced (10 ms)
+  Virtual Machines Should Not Be Preview
+    [+] Virtual Machines Should Not Be Preview (12 ms)
+  VM Images Should Use Latest Version
+    [+] VM Images Should Use Latest Version (2 ms)
+  VM Size Should Be A Parameter
+    [+] VM Size Should Be A Parameter (12 ms)
+Pass  : 31
+Fail  : 0
+Total : 31

test/data/output/ansi-color-codes/super-linter-parallel-stdout-ARM-no-ANSI-color-codes

@@ -0,0 +1,72 @@
+
+Validating arm\arm_good_1.json
+  JSONFiles Should Be Valid
+    [+] JSONFiles Should Be Valid (30 ms)
+Pass  : 1
+Fail  : 0
+Total : 1
+
+
+  adminUsername Should Not Be A Literal
+    [+] adminUsername Should Not Be A Literal (42 ms)
+  apiVersions Should Be Recent In Reference Functions
+    [+] apiVersions Should Be Recent In Reference Functions (23 ms)
+  apiVersions Should Be Recent
+    [+] apiVersions Should Be Recent (133 ms)
+  artifacts parameter
+    [+] artifacts parameter (12 ms)
+  CommandToExecute Must Use ProtectedSettings For Secrets
+    [+] CommandToExecute Must Use ProtectedSettings For Secrets (17 ms)
+  DependsOn Best Practices
+    [+] DependsOn Best Practices (8 ms)
+  Deployment Resources Must Not Be Debug
+    [+] Deployment Resources Must Not Be Debug (11 ms)
+  DeploymentTemplate Must Not Contain Hardcoded Uri
+    [+] DeploymentTemplate Must Not Contain Hardcoded Uri (72 ms)
+  DeploymentTemplate Schema Is Correct
+    [+] DeploymentTemplate Schema Is Correct (4 ms)
+  Dynamic Variable References Should Not Use Concat
+    [+] Dynamic Variable References Should Not Use Concat (8 ms)
+  IDs Should Be Derived From ResourceIDs
+    [+] IDs Should Be Derived From ResourceIDs (20 ms)
+  Location Should Not Be Hardcoded
+    [+] Location Should Not Be Hardcoded (157 ms)
+  ManagedIdentityExtension must not be used
+    [+] ManagedIdentityExtension must not be used (5 ms)
+  Min And Max Value Are Numbers
+    [+] Min And Max Value Are Numbers (13 ms)
+  Outputs Must Not Contain Secrets
+    [+] Outputs Must Not Contain Secrets (21 ms)
+  Parameter Types Should Be Consistent
+    [+] Parameter Types Should Be Consistent (23 ms)
+  Parameters Must Be Referenced
+    [+] Parameters Must Be Referenced (23 ms)
+  Password params must be secure
+    [+] Password params must be secure (7 ms)
+  providers apiVersions Is Not Permitted
+    [+] providers apiVersions Is Not Permitted (3 ms)
+  ResourceIds should not contain
+    [+] ResourceIds should not contain (23 ms)
+  Resources Should Have Location
+    [+] Resources Should Have Location (11 ms)
+  Resources Should Not Be Ambiguous
+    [+] Resources Should Not Be Ambiguous (13 ms)
+  Secure Params In Nested Deployments
+    [+] Secure Params In Nested Deployments (9 ms)
+  Secure String Parameters Cannot Have Default
+    [+] Secure String Parameters Cannot Have Default (5 ms)
+  Template Should Not Contain Blanks
+    [+] Template Should Not Contain Blanks (15 ms)
+  URIs Should Be Properly Constructed
+    [+] URIs Should Be Properly Constructed (82 ms)
+  Variables Must Be Referenced
+    [+] Variables Must Be Referenced (10 ms)
+  Virtual Machines Should Not Be Preview
+    [+] Virtual Machines Should Not Be Preview (12 ms)
+  VM Images Should Use Latest Version
+    [+] VM Images Should Use Latest Version (2 ms)
+  VM Size Should Be A Parameter
+    [+] VM Size Should Be A Parameter (12 ms)
+Pass  : 31
+Fail  : 0
+Total : 31

test/lib/outputTest.sh

@@ -131,8 +131,29 @@ EOF
   notice "${FUNCTION_NAME} PASS"
 }
 
+RemoveAnsiColorCodesFromFileTest() {
+  local FUNCTION_NAME
+  FUNCTION_NAME="${FUNCNAME[0]}"
+  info "${FUNCTION_NAME} start"
+
+  InitWorkspace
+
+  local TEST_CASE_FILE_WITH_ANSI_COLOR_CODES="test/data/output/ansi-color-codes/super-linter-parallel-stdout-ARM"
+  local EXPECTED_TEST_CASE_FILE_WITHOUT_ANSI_COLOR_CODES="test/data/output/ansi-color-codes/super-linter-parallel-stdout-ARM-no-ANSI-color-codes"
+  local INPUT_FILE
+  INPUT_FILE="${TEMP_WORKSPACE}/$(basename "${TEST_CASE_FILE_WITH_ANSI_COLOR_CODES}")"
+  cp "${TEST_CASE_FILE_WITH_ANSI_COLOR_CODES}" "${INPUT_FILE}"
+  RemoveAnsiColorCodesFromFile "${INPUT_FILE}"
+  AssertFileAndDirContentsMatch "${INPUT_FILE}" "${EXPECTED_TEST_CASE_FILE_WITHOUT_ANSI_COLOR_CODES}"
+
+  CleanupWorkspace
+
+  notice "${FUNCTION_NAME} PASS"
+}
+
 WriteSummaryMarkdownTableHeaderTest
 WriteSummaryMarkdownTableLineSuccessTest
 WriteSummaryMarkdownTableLineFailureTest
 WriteSummaryMarkdownTableFooterSuccessTest
 WriteSummaryMarkdownTableFooterFailureTest
+RemoveAnsiColorCodesFromFileTest

test/run-super-linter-tests.sh

@@ -246,6 +246,10 @@ debug "Super-linter main output path: ${SUPER_LINTER_MAIN_OUTPUT_PATH}"
 SUPER_LINTER_OUTPUT_PATH="${SUPER_LINTER_MAIN_OUTPUT_PATH}/super-linter"
 debug "Super-linter output path: ${SUPER_LINTER_OUTPUT_PATH}"
 
+# Remove color codes from output by default
+REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT="${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT:-"true"}"
+COMMAND_TO_RUN+=(--env REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT="${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}")
+
 COMMAND_TO_RUN+=(-e CREATE_LOG_FILE="${CREATE_LOG_FILE}")
 COMMAND_TO_RUN+=(-e LOG_LEVEL="${LOG_LEVEL:-"DEBUG"}")
 COMMAND_TO_RUN+=(-e RUN_LOCAL="${RUN_LOCAL:-true}")
@@ -324,6 +328,12 @@ if [[ "${CREATE_LOG_FILE}" == true ]]; then
       debug "Copying Super-linter log from the workspace (${SUPER_LINTER_WORKSPACE}) to the current working directory for easier inspection"
       cp -v "${LOG_FILE_PATH}" "$(pwd)/"
     fi
+
+    if [[ "${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}" == "true" ]]; then
+      if AreAnsiColorCodesInFile "${LOG_FILE_PATH}"; then
+        fatal "${LOG_FILE_PATH} contains unexpected ANSI color codes"
+      fi
+    fi
   fi
 else
   debug "Log file was not requested. CREATE_LOG_FILE: ${CREATE_LOG_FILE}"
@@ -344,10 +354,33 @@ if [[ "${SAVE_SUPER_LINTER_OUTPUT}" == true ]]; then
 
     if [[ "${SUPER_LINTER_WORKSPACE}" != "$(pwd)" ]]; then
       debug "Copying Super-linter output from the workspace (${SUPER_LINTER_WORKSPACE}) to the current working directory for easier inspection"
-      SUPER_LINTER_OUTPUT_PATH_PWD="$(pwd)/super-linter-output/"
+      SUPER_LINTER_MAIN_OUTPUT_PATH_PWD="$(pwd)/${SUPER_LINTER_OUTPUT_DIRECTORY_NAME}"
-      mkdir -p "${SUPER_LINTER_OUTPUT_PATH_PWD}"
+      SUPER_LINTER_OUTPUT_PATH_PWD="${SUPER_LINTER_MAIN_OUTPUT_PATH_PWD}/super-linter"
-      cp -r "${SUPER_LINTER_OUTPUT_PATH}" "${SUPER_LINTER_OUTPUT_PATH_PWD}"
+      mkdir -p "${SUPER_LINTER_MAIN_OUTPUT_PATH_PWD}"
+      cp -r "${SUPER_LINTER_OUTPUT_PATH}" "${SUPER_LINTER_MAIN_OUTPUT_PATH_PWD}/"
     fi
+
+    for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do
+      LANGUAGE_STDERR_FILE_PATH="${SUPER_LINTER_OUTPUT_PATH_PWD:-"${SUPER_LINTER_OUTPUT_PATH}"}/super-linter-parallel-stderr-${LANGUAGE}"
+      LANGUAGE_STDOUT_FILE_PATH="${SUPER_LINTER_OUTPUT_PATH_PWD:-"${SUPER_LINTER_OUTPUT_PATH}"}/super-linter-parallel-stdout-${LANGUAGE}"
+
+      if [[ "${REMOVE_ANSI_COLOR_CODES_FROM_OUTPUT}" == "true" ]]; then
+        if [[ -e "${LANGUAGE_STDERR_FILE_PATH}" ]]; then
+          if AreAnsiColorCodesInFile "${LANGUAGE_STDERR_FILE_PATH}"; then
+            fatal "${LANGUAGE_STDERR_FILE_PATH} contains unexpected ANSI color codes"
+          fi
+        fi
+
+        if [[ -e "${LANGUAGE_STDOUT_FILE_PATH}" ]]; then
+          if AreAnsiColorCodesInFile "${LANGUAGE_STDOUT_FILE_PATH}"; then
+            fatal "${LANGUAGE_STDOUT_FILE_PATH} contains unexpected ANSI color codes"
+          fi
+        fi
+      fi
+
+      unset LANGUAGE_STDERR_FILE_PATH
+      unset LANGUAGE_STDOUT_FILE_PATH
+    done
   fi
 else
   debug "Super-linter output was not requested. SAVE_SUPER_LINTER_OUTPUT: ${SAVE_SUPER_LINTER_OUTPUT}"

test/testUtils.sh

@@ -15,6 +15,13 @@ CREATE_LOG_FILE="true"
 # shellcheck source=/dev/null
 source "lib/functions/log.sh"
 
+# shellcheck source=/dev/null
+source "lib/globals/languages.sh"
+
+# Because we need variables defined there
+# shellcheck source=/dev/null
+source "lib/functions/output.sh"
+
 # TODO: use TEST_CASE_FOLDER instead of redefining this after we extract the
 # initialization of TEST_CASE_FOLDER from linter.sh
 # shellcheck disable=SC2034
@@ -168,6 +175,17 @@ IsStandardImage() {
   fi
 }
 
+AreAnsiColorCodesInFile() {
+  local FILE_TO_SEARCH_IN="${1}"
+  if grep --color=never --quiet --perl-regexp "${ANSI_COLOR_CODES_SEARCH_PATTERN}" "${FILE_TO_SEARCH_IN}"; then
+    debug "Found at least one ANSI color code in ${FILE_TO_SEARCH_IN}"
+    return 0
+  else
+    debug "Found no ANSI color codes in ${FILE_TO_SEARCH_IN}"
+    return 1
+  fi
+}
+
 RemoveTestLeftovers() {
   local LEFTOVERS_TO_CLEAN=()
   LEFTOVERS_TO_CLEAN+=("${SUPER_LINTER_WORKSPACE}/${LINTERS_TEST_CASE_DIRECTORY}/rust_clippy/bad/target")