chore: shorten linter versions (#6176)

Shorten the linter versions file by keeping only the version string for
each linter, instead of the entire output of the version command

Makefile

@@ -342,7 +342,7 @@ lint-subset-files-enable-expensive-io-checks: ## Lint a small subset of files in
 		$(SUPER_LINTER_TEST_CONTAINER_URL)
 
 .PHONY: test-lib
-test-lib: test-globals-languages test-linter-rules test-build-file-list test-detect-files test-github-event test-setup-ssh test-validation test-output test-linter-commands ## Test super-linter libs and globals
+test-lib: test-globals-languages test-linter-rules test-build-file-list test-detect-files test-github-event test-setup-ssh test-validation test-output test-linter-commands test-linter-versions ## Test super-linter libs and globals
 
 .PHONY: test-globals-languages
 test-globals-languages: ## Test globals/languages.sh
@@ -435,6 +435,15 @@ test-linter-commands: ## Test linterCommands
 		--rm \
 		$(SUPER_LINTER_TEST_CONTAINER_URL)
 
+.PHONY: test-linter-versions
+test-linter-versions: ## Test linterVersions
+	docker run \
+		-v "$(CURDIR):/tmp/lint" \
+		-w /tmp/lint \
+		--entrypoint /tmp/lint/test/lib/linterVersionsTest.sh \
+		--rm \
+		$(SUPER_LINTER_TEST_CONTAINER_URL)
+
 # Run this test against a small directory because we're only interested in
 # loading default configuration files. The directory that we run super-linter
 # against should not be .github because that includes default linter rules.

lib/linter.sh

@@ -686,7 +686,7 @@ Header
 UpdateLoopsForImage
 
 # Print linter versions
-info "$(cat "${VERSION_FILE}")"
+info "This version of Super-linter includes the following tools:\n$(cat "${VERSION_FILE}")"
 
 #######################
 # Get GitHub Env Vars #

scripts/linterVersions.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+set -o errexit
 set -o nounset
 set -o pipefail
 
@@ -17,6 +18,7 @@ LINTER_NAMES_ARRAY['CLOUDFORMATION']="cfn-lint"
 LINTER_NAMES_ARRAY['COFFEESCRIPT']="coffeelint"
 LINTER_NAMES_ARRAY['CPP']="cpplint"
 LINTER_NAMES_ARRAY['CSS']="stylelint"
+LINTER_NAMES_ARRAY['CSS_PRETTIER']="prettier"
 LINTER_NAMES_ARRAY['DART']="dart"
 LINTER_NAMES_ARRAY['DOCKERFILE_HADOLINT']="hadolint"
 LINTER_NAMES_ARRAY['EDITORCONFIG']="editorconfig-checker"
@@ -30,21 +32,27 @@ LINTER_NAMES_ARRAY['GO']="golangci-lint"
 LINTER_NAMES_ARRAY['GO_MODULES']="${LINTER_NAMES_ARRAY['GO']}"
 LINTER_NAMES_ARRAY['GO_RELEASER']="goreleaser"
 LINTER_NAMES_ARRAY['GOOGLE_JAVA_FORMAT']="google-java-format"
+LINTER_NAMES_ARRAY['GRAPHQL_PRETTIER']="prettier"
 LINTER_NAMES_ARRAY['GROOVY']="npm-groovy-lint"
 LINTER_NAMES_ARRAY['HTML']="htmlhint"
+LINTER_NAMES_ARRAY['HTML_PRETTIER']="prettier"
 LINTER_NAMES_ARRAY['JAVA']="checkstyle"
 LINTER_NAMES_ARRAY['JAVASCRIPT_ES']="eslint"
 LINTER_NAMES_ARRAY['JAVASCRIPT_PRETTIER']="prettier"
 LINTER_NAMES_ARRAY['JAVASCRIPT_STANDARD']="standard"
 LINTER_NAMES_ARRAY['JSCPD']="jscpd"
 LINTER_NAMES_ARRAY['JSON']="eslint"
+LINTER_NAMES_ARRAY['JSON_PRETTIER']="prettier"
 LINTER_NAMES_ARRAY['JSONC']="eslint"
+LINTER_NAMES_ARRAY['JSONC_PRETTIER']="prettier"
 LINTER_NAMES_ARRAY['JSX']="eslint"
+LINTER_NAMES_ARRAY['JSX_PRETTIER']="prettier"
 LINTER_NAMES_ARRAY['KOTLIN']="ktlint"
 LINTER_NAMES_ARRAY['KUBERNETES_KUBECONFORM']="kubeconform"
 LINTER_NAMES_ARRAY['LATEX']="chktex"
 LINTER_NAMES_ARRAY['LUA']="lua"
 LINTER_NAMES_ARRAY['MARKDOWN']="markdownlint"
+LINTER_NAMES_ARRAY['MARKDOWN_PRETTIER']="prettier"
 LINTER_NAMES_ARRAY['NATURAL_LANGUAGE']="textlint"
 LINTER_NAMES_ARRAY['OPENAPI']="spectral"
 LINTER_NAMES_ARRAY['PERL']="perl"
@@ -79,8 +87,10 @@ LINTER_NAMES_ARRAY['TSX']="eslint"
 LINTER_NAMES_ARRAY['TYPESCRIPT_ES']="eslint"
 LINTER_NAMES_ARRAY['TYPESCRIPT_PRETTIER']="prettier"
 LINTER_NAMES_ARRAY['TYPESCRIPT_STANDARD']="ts-standard"
+LINTER_NAMES_ARRAY['VUE_PRETTIER']="prettier"
 LINTER_NAMES_ARRAY['XML']="xmllint"
 LINTER_NAMES_ARRAY['YAML']="yamllint"
+LINTER_NAMES_ARRAY['YAML_PRETTIER']="prettier"
 
 if [[ "${IMAGE}" == "standard" ]]; then
   LINTER_NAMES_ARRAY['ARM']="arm-ttk"
@@ -103,20 +113,55 @@ rm -rfv "${VERSION_FILE}"
 
 echo "Building linter version file ${VERSION_FILE} for the following linters: ${LINTER_NAMES_ARRAY[*]}..."
 
-for LINTER in "${LINTER_NAMES_ARRAY[@]}"; do
+for LANGUAGE in "${!LINTER_NAMES_ARRAY[@]}"; do
-  # Some linters need to account for special commands to get their version
+  LINTER="${LINTER_NAMES_ARRAY[${LANGUAGE}]}"
+  echo "Get version for ${LINTER}"
 
-  if [[ ${LINTER} == "arm-ttk" ]]; then
+  # Some linters need to account for special commands to get their version instead
-    GET_VERSION_CMD="$(grep -iE 'version' "/usr/bin/arm-ttk" | xargs 2>&1)"
+  # of the default --version option
-  # Some linters don't support a "get version" command
+
-  elif [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]]; then
+  if [[ "${LINTER}" == "actionlint" ]]; then
-    GET_VERSION_CMD="Version command not supported"
+    GET_VERSION_CMD="$("${LINTER}" --version | head -n 1)"
+  elif [[ "${LINTER}" == "ansible-lint" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | grep -v 'available' | awk '{ print $2 }')"
+  elif [[ ${LINTER} == "arm-ttk" ]]; then
+    GET_VERSION_CMD="$(grep -iE 'version' "/usr/bin/arm-ttk" | xargs 2>&1 | awk '{ print $3 }')"
+  elif [[ "${LINTER}" == "black" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | grep 'black' | awk '{ print $2 }')"
+  elif [[ "${LINTER}" == "cfn-lint" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $2 }')"
+  elif [[ "${LINTER}" == "chktex" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version 2>/dev/null | grep 'ChkTeX' | awk '{ print $2 }')"
   elif [[ ${LINTER} == "checkstyle" ]] || [[ ${LINTER} == "google-java-format" ]]; then
-    GET_VERSION_CMD="$(java -jar "/usr/bin/${LINTER}" --version 2>&1)"
+    GET_VERSION_CMD="$(java -jar "/usr/bin/${LINTER}" --version 2>&1 | awk '{ print $3 }')"
+  elif [[ "${LINTER}" == "clang-format" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $3 }')"
+  elif [[ "${LINTER}" == "clj-kondo" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $2 }')"
   elif [[ ${LINTER} == "clippy" ]]; then
-    GET_VERSION_CMD="$(cargo clippy --version 2>&1)"
+    GET_VERSION_CMD="$(cargo clippy --version 2>&1 | awk '{ print $2 }')"
+  elif [[ "${LINTER}" == "cpplint" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | grep 'cpplint' | grep -v 'github' | awk '{ print $2 }')"
+  elif [[ "${LINTER}" == "dart" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $4 }')"
+  elif [[ "${LINTER}" == "dotenv-linter" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $2 }')"
   elif [[ ${LINTER} == "editorconfig-checker" ]]; then
     GET_VERSION_CMD="$(${LINTER} -version)"
+  elif [[ "${LINTER}" == "flake8" ]]; then
+    GET_VERSION_CMD="$(${LINTER} --version | grep 'mccabe' | awk '{ print $1 }')"
+  elif [[ ${LINTER} == "gitleaks" ]]; then
+    GET_VERSION_CMD="$(${LINTER} version)"
+  elif [[ "${LINTER}" == "golangci-lint" ]]; then
+    GET_VERSION_CMD="$(${LINTER} --version | awk '{ print $4 }')"
+  elif [[ "${LINTER}" == "goreleaser" ]]; then
+    GET_VERSION_CMD="$(${LINTER} --version | grep 'GitVersion' | awk '{ print $2 }')"
+  elif [[ "${LINTER}" == "hadolint" ]]; then
+    GET_VERSION_CMD="$(${LINTER} --version | awk '{ print $4 }')"
+  elif [[ "${LINTER}" == "isort" ]]; then
+    GET_VERSION_CMD="$(${LINTER} --version | grep 'VERSION' | awk '{ print $2 }')"
+  elif [[ "${LINTER}" == "ktlint" ]]; then
+    GET_VERSION_CMD="$(${LINTER} --version | awk '{ print $3 }')"
   elif [[ ${LINTER} == "kubeconform" ]]; then
     GET_VERSION_CMD="$(${LINTER} -v)"
   elif [[ ${LINTER} == "lintr" ]]; then
@@ -124,23 +169,72 @@ for LINTER in "${LINTER_NAMES_ARRAY[@]}"; do
     GET_VERSION_CMD="$(R --slave -e "r_ver <- R.Version()\$version.string; \
                 lintr_ver <- packageVersion('lintr'); \
                 glue::glue('lintr { lintr_ver } on { r_ver }')")"
-  elif [[ ${LINTER} == "protolint" ]] || [[ ${LINTER} == "gitleaks" ]]; then
+  elif [[ "${LINTER}" == "perl" ]]; then
-    GET_VERSION_CMD="$(${LINTER} version)"
+    GET_VERSION_CMD="$("${LINTER}" --version | grep 'This' | awk '{ print $9 }')"
+  elif [[ "${LINTER}" == "php" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | grep 'cli' | awk '{ print $2 }')"
+  elif [[ "${LINTER}" == "phpcs" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $3 }')"
+  elif [[ "${LINTER}" == "phpstan" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $7 }')"
+  elif [[ ${LINTER} == "protolint" ]]; then
+    GET_VERSION_CMD="$(${LINTER} version | awk '{ print $3 }')"
+  elif [[ ${LINTER} == "psalm" ]]; then
+    GET_VERSION_CMD="$(${LINTER} --version | awk '{ print $2 }')"
+  elif [[ "${LINTER}" == "pyink" ]]; then
+    GET_VERSION_CMD="$(${LINTER} --version | grep 'pyink' | awk '{ print $2 }')"
+  elif [[ ${LINTER} == "pylint" ]]; then
+    GET_VERSION_CMD="$(${LINTER} --version | grep 'pylint' | awk '{ print $2 }')"
   elif [[ ${LINTER} == "lua" ]]; then
-    GET_VERSION_CMD="$("${LINTER}" -v 2>&1)"
+    GET_VERSION_CMD="$("${LINTER}" -v 2>&1 | awk '{ print $2 }')"
+  elif [[ ${LINTER} == "mypy" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $2 }')"
+  elif [[ "${LINTER}" == "npm-groovy-lint" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | grep 'npm-groovy-lint' | awk '{ print $3 }')"
+  elif [[ "${LINTER}" == "pwsh" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $2 }')"
+  elif [[ "${LINTER}" == "R" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | head -n 1 | awk '{ print $3 }')"
+  elif [[ "${LINTER}" == "raku" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | grep 'Rakudo' | awk '{ print $4 }' | sed 's/\.$//')"
   elif [[ ${LINTER} == "renovate-config-validator" ]]; then
-    GET_VERSION_CMD="$(renovate --version 2>&1)"
+    GET_VERSION_CMD="$(renovate --version 2>/dev/null)"
+  elif [[ "${LINTER}" == "ruff" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $2 }')"
+  elif [[ "${LINTER}" == "rustfmt" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $2 }')"
+  elif [[ "${LINTER}" == "scalafmt" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $2 }')"
+  elif [[ "${LINTER}" == "shellcheck" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | grep 'version:' | awk '{ print $2 }')"
+  elif [[ "${LINTER}" == "snakefmt" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $3 }')"
+  elif [[ "${LINTER}" == "sqlfluff" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $3 }')"
+  elif [[ ${LINTER} == "terraform" ]]; then
+    GET_VERSION_CMD="$(CHECKPOINT_DISABLE="not needed for version checks" "${LINTER}" --version | head -n 1 | awk '{ print $2 }')"
+  elif [[ "${LINTER}" == "terragrunt" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $2 }')"
   elif [[ ${LINTER} == "terrascan" ]]; then
-    GET_VERSION_CMD="$("${LINTER}" version 2>&1)"
+    GET_VERSION_CMD="$("${LINTER}" version 2>&1 | awk '{ print $2 }')"
-  else
+  elif [[ ${LINTER} == "tflint" ]]; then
     # Unset TF_LOG_LEVEL so that the version file doesn't contain debug log when running
     # commands that read TF_LOG_LEVEL or TFLINT_LOG, which are likely set to DEBUG when
     # building the versions file
     GET_VERSION_CMD="$(
       unset TF_LOG_LEVEL
       unset TFLINT_LOG
-      "${LINTER}" --version 2>&1
+      "${LINTER}" --version | grep 'version' | awk '{ print $3 }'
     )"
+  elif [[ ${LINTER} == "xmllint" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version 2>&1 | grep 'xmllint' | awk '{ print $5 }')"
+  elif [[ "${LINTER}" == "yamllint" ]]; then
+    GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $2 }')"
+  # Some linters don't support a "get version" command
+  elif [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]]; then
+    GET_VERSION_CMD="Version command not supported"
+  else
+    GET_VERSION_CMD="$("${LINTER}" --version 2>&1)"
   fi
 
   ERROR_CODE=$?
@@ -149,9 +243,16 @@ for LINTER in "${LINTER_NAMES_ARRAY[@]}"; do
     exit 1
   else
     echo "Successfully found version for ${LINTER}: ${GET_VERSION_CMD}"
-    if ! echo "${LINTER}: ${GET_VERSION_CMD}" >>"${VERSION_FILE}" 2>&1; then
+    if ! echo "[${LANGUAGE}] ${LINTER}: ${GET_VERSION_CMD}" >>"${VERSION_FILE}" 2>&1; then
-      echo "[ERROR] Failed to write data to file!"
+      echo "[ERROR]: Failed to write data to file!"
       exit 1
     fi
   fi
 done
+
+if ! sort --ignore-case --unique --output="${VERSION_FILE}" "${VERSION_FILE}"; then
+  echo "[ERROR]:Failed to sort file!"
+  exit 1
+fi
+
+echo -e "Versions file contents:\n$(cat "${VERSION_FILE}")"

test/lib/linterVersionsTest.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# shellcheck source=/dev/null
+source "test/testUtils.sh"
+
+echo -e "Versions file (${VERSION_FILE}) contents:\n$(cat "${VERSION_FILE}")"
+
+VersionsFileSortTest() {
+  local FUNCTION_NAME
+  FUNCTION_NAME="${FUNCNAME[0]}"
+  info "${FUNCTION_NAME} start"
+
+  if ! sort --check "${VERSION_FILE}"; then
+    fatal "Linters version file (${VERSION_FILE}) is not sorted"
+  fi
+
+  notice "${FUNCTION_NAME} PASS"
+}
+
+VersionsFileCompletenessTest() {
+  local FUNCTION_NAME
+  FUNCTION_NAME="${FUNCNAME[0]}"
+  info "${FUNCTION_NAME} start"
+
+  local LINTERS_VERSION_FILE_LINES_COUNT
+  LINTERS_VERSION_FILE_LINES_COUNT=$(wc --lines "${VERSION_FILE}" | awk '{print $1}')
+  debug "Linters version file lines count: ${LINTERS_VERSION_FILE_LINES_COUNT}"
+
+  local EXPECTED_LANGUAGE_COUNT=${#LANGUAGE_ARRAY[@]}
+
+  if ! IsStandardImage; then
+    EXPECTED_LANGUAGE_COUNT=$((EXPECTED_LANGUAGE_COUNT - ${#LANGUAGES_NOT_IN_SLIM_IMAGE[@]}))
+  fi
+
+  if [[ ${LINTERS_VERSION_FILE_LINES_COUNT} -ne ${EXPECTED_LANGUAGE_COUNT} ]]; then
+    fatal "Linters version file lines count (${LINTERS_VERSION_FILE_LINES_COUNT}) doesn't match the length of the languages array (${EXPECTED_LANGUAGE_COUNT}). Is a version descriptor missing in the versions file? Is the version descriptor spanning multiple lines?"
+  else
+    debug "The versions file lines count (${LINTERS_VERSION_FILE_LINES_COUNT}) matches the expected value (${EXPECTED_LANGUAGE_COUNT})"
+  fi
+
+  for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do
+    if ! IsStandardImage && ! IsLanguageInSlimImage "${LANGUAGE}"; then
+      debug "Skip checking if ${LANGUAGE} is in the versions file because ${LANGUAGE} is not included in the slim image"
+      continue
+    fi
+
+    if ! grep -q "${LANGUAGE}" "${VERSION_FILE}"; then
+      fatal "${LANGUAGE} is absent from the versions file"
+    else
+      debug "${LANGUAGE} present in the versions file"
+    fi
+  done
+
+  notice "${FUNCTION_NAME} PASS"
+}
+
+VersionsFileSortTest
+VersionsFileCompletenessTest