From fc34cb53d9e6a866f9fbeec5c6abd638c3ac46da Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Mon, 14 Oct 2024 18:43:10 +0200 Subject: [PATCH] chore: shorten linter versions (#6176) Shorten the linter versions file by keeping only the version string for each linter, instead of the entire output of the version command --- Makefile | 11 ++- lib/linter.sh | 2 +- scripts/linterVersions.sh | 137 ++++++++++++++++++++++++++++----- test/lib/linterVersionsTest.sh | 62 +++++++++++++++ 4 files changed, 192 insertions(+), 20 deletions(-) create mode 100755 test/lib/linterVersionsTest.sh diff --git a/Makefile b/Makefile index 3f1d8ca1..b0246bf1 100644 --- a/Makefile +++ b/Makefile @@ -342,7 +342,7 @@ lint-subset-files-enable-expensive-io-checks: ## Lint a small subset of files in $(SUPER_LINTER_TEST_CONTAINER_URL) .PHONY: test-lib -test-lib: test-globals-languages test-linter-rules test-build-file-list test-detect-files test-github-event test-setup-ssh test-validation test-output test-linter-commands ## Test super-linter libs and globals +test-lib: test-globals-languages test-linter-rules test-build-file-list test-detect-files test-github-event test-setup-ssh test-validation test-output test-linter-commands test-linter-versions ## Test super-linter libs and globals .PHONY: test-globals-languages test-globals-languages: ## Test globals/languages.sh @@ -435,6 +435,15 @@ test-linter-commands: ## Test linterCommands --rm \ $(SUPER_LINTER_TEST_CONTAINER_URL) +.PHONY: test-linter-versions +test-linter-versions: ## Test linterVersions + docker run \ + -v "$(CURDIR):/tmp/lint" \ + -w /tmp/lint \ + --entrypoint /tmp/lint/test/lib/linterVersionsTest.sh \ + --rm \ + $(SUPER_LINTER_TEST_CONTAINER_URL) + # Run this test against a small directory because we're only interested in # loading default configuration files. The directory that we run super-linter # against should not be .github because that includes default linter rules. diff --git a/lib/linter.sh b/lib/linter.sh index 1b457fa4..b1cb9516 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -686,7 +686,7 @@ Header UpdateLoopsForImage # Print linter versions -info "$(cat "${VERSION_FILE}")" +info "This version of Super-linter includes the following tools:\n$(cat "${VERSION_FILE}")" ####################### # Get GitHub Env Vars # diff --git a/scripts/linterVersions.sh b/scripts/linterVersions.sh index 4bb1f62f..db599f9d 100755 --- a/scripts/linterVersions.sh +++ b/scripts/linterVersions.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +set -o errexit set -o nounset set -o pipefail @@ -17,6 +18,7 @@ LINTER_NAMES_ARRAY['CLOUDFORMATION']="cfn-lint" LINTER_NAMES_ARRAY['COFFEESCRIPT']="coffeelint" LINTER_NAMES_ARRAY['CPP']="cpplint" LINTER_NAMES_ARRAY['CSS']="stylelint" +LINTER_NAMES_ARRAY['CSS_PRETTIER']="prettier" LINTER_NAMES_ARRAY['DART']="dart" LINTER_NAMES_ARRAY['DOCKERFILE_HADOLINT']="hadolint" LINTER_NAMES_ARRAY['EDITORCONFIG']="editorconfig-checker" @@ -30,21 +32,27 @@ LINTER_NAMES_ARRAY['GO']="golangci-lint" LINTER_NAMES_ARRAY['GO_MODULES']="${LINTER_NAMES_ARRAY['GO']}" LINTER_NAMES_ARRAY['GO_RELEASER']="goreleaser" LINTER_NAMES_ARRAY['GOOGLE_JAVA_FORMAT']="google-java-format" +LINTER_NAMES_ARRAY['GRAPHQL_PRETTIER']="prettier" LINTER_NAMES_ARRAY['GROOVY']="npm-groovy-lint" LINTER_NAMES_ARRAY['HTML']="htmlhint" +LINTER_NAMES_ARRAY['HTML_PRETTIER']="prettier" LINTER_NAMES_ARRAY['JAVA']="checkstyle" LINTER_NAMES_ARRAY['JAVASCRIPT_ES']="eslint" LINTER_NAMES_ARRAY['JAVASCRIPT_PRETTIER']="prettier" LINTER_NAMES_ARRAY['JAVASCRIPT_STANDARD']="standard" LINTER_NAMES_ARRAY['JSCPD']="jscpd" LINTER_NAMES_ARRAY['JSON']="eslint" +LINTER_NAMES_ARRAY['JSON_PRETTIER']="prettier" LINTER_NAMES_ARRAY['JSONC']="eslint" +LINTER_NAMES_ARRAY['JSONC_PRETTIER']="prettier" LINTER_NAMES_ARRAY['JSX']="eslint" +LINTER_NAMES_ARRAY['JSX_PRETTIER']="prettier" LINTER_NAMES_ARRAY['KOTLIN']="ktlint" LINTER_NAMES_ARRAY['KUBERNETES_KUBECONFORM']="kubeconform" LINTER_NAMES_ARRAY['LATEX']="chktex" LINTER_NAMES_ARRAY['LUA']="lua" LINTER_NAMES_ARRAY['MARKDOWN']="markdownlint" +LINTER_NAMES_ARRAY['MARKDOWN_PRETTIER']="prettier" LINTER_NAMES_ARRAY['NATURAL_LANGUAGE']="textlint" LINTER_NAMES_ARRAY['OPENAPI']="spectral" LINTER_NAMES_ARRAY['PERL']="perl" @@ -79,8 +87,10 @@ LINTER_NAMES_ARRAY['TSX']="eslint" LINTER_NAMES_ARRAY['TYPESCRIPT_ES']="eslint" LINTER_NAMES_ARRAY['TYPESCRIPT_PRETTIER']="prettier" LINTER_NAMES_ARRAY['TYPESCRIPT_STANDARD']="ts-standard" +LINTER_NAMES_ARRAY['VUE_PRETTIER']="prettier" LINTER_NAMES_ARRAY['XML']="xmllint" LINTER_NAMES_ARRAY['YAML']="yamllint" +LINTER_NAMES_ARRAY['YAML_PRETTIER']="prettier" if [[ "${IMAGE}" == "standard" ]]; then LINTER_NAMES_ARRAY['ARM']="arm-ttk" @@ -103,20 +113,55 @@ rm -rfv "${VERSION_FILE}" echo "Building linter version file ${VERSION_FILE} for the following linters: ${LINTER_NAMES_ARRAY[*]}..." -for LINTER in "${LINTER_NAMES_ARRAY[@]}"; do - # Some linters need to account for special commands to get their version +for LANGUAGE in "${!LINTER_NAMES_ARRAY[@]}"; do + LINTER="${LINTER_NAMES_ARRAY[${LANGUAGE}]}" + echo "Get version for ${LINTER}" - if [[ ${LINTER} == "arm-ttk" ]]; then - GET_VERSION_CMD="$(grep -iE 'version' "/usr/bin/arm-ttk" | xargs 2>&1)" - # Some linters don't support a "get version" command - elif [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]]; then - GET_VERSION_CMD="Version command not supported" + # Some linters need to account for special commands to get their version instead + # of the default --version option + + if [[ "${LINTER}" == "actionlint" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | head -n 1)" + elif [[ "${LINTER}" == "ansible-lint" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | grep -v 'available' | awk '{ print $2 }')" + elif [[ ${LINTER} == "arm-ttk" ]]; then + GET_VERSION_CMD="$(grep -iE 'version' "/usr/bin/arm-ttk" | xargs 2>&1 | awk '{ print $3 }')" + elif [[ "${LINTER}" == "black" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | grep 'black' | awk '{ print $2 }')" + elif [[ "${LINTER}" == "cfn-lint" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $2 }')" + elif [[ "${LINTER}" == "chktex" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version 2>/dev/null | grep 'ChkTeX' | awk '{ print $2 }')" elif [[ ${LINTER} == "checkstyle" ]] || [[ ${LINTER} == "google-java-format" ]]; then - GET_VERSION_CMD="$(java -jar "/usr/bin/${LINTER}" --version 2>&1)" + GET_VERSION_CMD="$(java -jar "/usr/bin/${LINTER}" --version 2>&1 | awk '{ print $3 }')" + elif [[ "${LINTER}" == "clang-format" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $3 }')" + elif [[ "${LINTER}" == "clj-kondo" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $2 }')" elif [[ ${LINTER} == "clippy" ]]; then - GET_VERSION_CMD="$(cargo clippy --version 2>&1)" + GET_VERSION_CMD="$(cargo clippy --version 2>&1 | awk '{ print $2 }')" + elif [[ "${LINTER}" == "cpplint" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | grep 'cpplint' | grep -v 'github' | awk '{ print $2 }')" + elif [[ "${LINTER}" == "dart" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $4 }')" + elif [[ "${LINTER}" == "dotenv-linter" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $2 }')" elif [[ ${LINTER} == "editorconfig-checker" ]]; then GET_VERSION_CMD="$(${LINTER} -version)" + elif [[ "${LINTER}" == "flake8" ]]; then + GET_VERSION_CMD="$(${LINTER} --version | grep 'mccabe' | awk '{ print $1 }')" + elif [[ ${LINTER} == "gitleaks" ]]; then + GET_VERSION_CMD="$(${LINTER} version)" + elif [[ "${LINTER}" == "golangci-lint" ]]; then + GET_VERSION_CMD="$(${LINTER} --version | awk '{ print $4 }')" + elif [[ "${LINTER}" == "goreleaser" ]]; then + GET_VERSION_CMD="$(${LINTER} --version | grep 'GitVersion' | awk '{ print $2 }')" + elif [[ "${LINTER}" == "hadolint" ]]; then + GET_VERSION_CMD="$(${LINTER} --version | awk '{ print $4 }')" + elif [[ "${LINTER}" == "isort" ]]; then + GET_VERSION_CMD="$(${LINTER} --version | grep 'VERSION' | awk '{ print $2 }')" + elif [[ "${LINTER}" == "ktlint" ]]; then + GET_VERSION_CMD="$(${LINTER} --version | awk '{ print $3 }')" elif [[ ${LINTER} == "kubeconform" ]]; then GET_VERSION_CMD="$(${LINTER} -v)" elif [[ ${LINTER} == "lintr" ]]; then @@ -124,23 +169,72 @@ for LINTER in "${LINTER_NAMES_ARRAY[@]}"; do GET_VERSION_CMD="$(R --slave -e "r_ver <- R.Version()\$version.string; \ lintr_ver <- packageVersion('lintr'); \ glue::glue('lintr { lintr_ver } on { r_ver }')")" - elif [[ ${LINTER} == "protolint" ]] || [[ ${LINTER} == "gitleaks" ]]; then - GET_VERSION_CMD="$(${LINTER} version)" + elif [[ "${LINTER}" == "perl" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | grep 'This' | awk '{ print $9 }')" + elif [[ "${LINTER}" == "php" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | grep 'cli' | awk '{ print $2 }')" + elif [[ "${LINTER}" == "phpcs" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $3 }')" + elif [[ "${LINTER}" == "phpstan" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $7 }')" + elif [[ ${LINTER} == "protolint" ]]; then + GET_VERSION_CMD="$(${LINTER} version | awk '{ print $3 }')" + elif [[ ${LINTER} == "psalm" ]]; then + GET_VERSION_CMD="$(${LINTER} --version | awk '{ print $2 }')" + elif [[ "${LINTER}" == "pyink" ]]; then + GET_VERSION_CMD="$(${LINTER} --version | grep 'pyink' | awk '{ print $2 }')" + elif [[ ${LINTER} == "pylint" ]]; then + GET_VERSION_CMD="$(${LINTER} --version | grep 'pylint' | awk '{ print $2 }')" elif [[ ${LINTER} == "lua" ]]; then - GET_VERSION_CMD="$("${LINTER}" -v 2>&1)" + GET_VERSION_CMD="$("${LINTER}" -v 2>&1 | awk '{ print $2 }')" + elif [[ ${LINTER} == "mypy" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $2 }')" + elif [[ "${LINTER}" == "npm-groovy-lint" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | grep 'npm-groovy-lint' | awk '{ print $3 }')" + elif [[ "${LINTER}" == "pwsh" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $2 }')" + elif [[ "${LINTER}" == "R" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | head -n 1 | awk '{ print $3 }')" + elif [[ "${LINTER}" == "raku" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | grep 'Rakudo' | awk '{ print $4 }' | sed 's/\.$//')" elif [[ ${LINTER} == "renovate-config-validator" ]]; then - GET_VERSION_CMD="$(renovate --version 2>&1)" + GET_VERSION_CMD="$(renovate --version 2>/dev/null)" + elif [[ "${LINTER}" == "ruff" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $2 }')" + elif [[ "${LINTER}" == "rustfmt" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $2 }')" + elif [[ "${LINTER}" == "scalafmt" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $2 }')" + elif [[ "${LINTER}" == "shellcheck" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | grep 'version:' | awk '{ print $2 }')" + elif [[ "${LINTER}" == "snakefmt" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $3 }')" + elif [[ "${LINTER}" == "sqlfluff" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $3 }')" + elif [[ ${LINTER} == "terraform" ]]; then + GET_VERSION_CMD="$(CHECKPOINT_DISABLE="not needed for version checks" "${LINTER}" --version | head -n 1 | awk '{ print $2 }')" + elif [[ "${LINTER}" == "terragrunt" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $2 }')" elif [[ ${LINTER} == "terrascan" ]]; then - GET_VERSION_CMD="$("${LINTER}" version 2>&1)" - else + GET_VERSION_CMD="$("${LINTER}" version 2>&1 | awk '{ print $2 }')" + elif [[ ${LINTER} == "tflint" ]]; then # Unset TF_LOG_LEVEL so that the version file doesn't contain debug log when running # commands that read TF_LOG_LEVEL or TFLINT_LOG, which are likely set to DEBUG when # building the versions file GET_VERSION_CMD="$( unset TF_LOG_LEVEL unset TFLINT_LOG - "${LINTER}" --version 2>&1 + "${LINTER}" --version | grep 'version' | awk '{ print $3 }' )" + elif [[ ${LINTER} == "xmllint" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version 2>&1 | grep 'xmllint' | awk '{ print $5 }')" + elif [[ "${LINTER}" == "yamllint" ]]; then + GET_VERSION_CMD="$("${LINTER}" --version | awk '{ print $2 }')" + # Some linters don't support a "get version" command + elif [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]]; then + GET_VERSION_CMD="Version command not supported" + else + GET_VERSION_CMD="$("${LINTER}" --version 2>&1)" fi ERROR_CODE=$? @@ -149,9 +243,16 @@ for LINTER in "${LINTER_NAMES_ARRAY[@]}"; do exit 1 else echo "Successfully found version for ${LINTER}: ${GET_VERSION_CMD}" - if ! echo "${LINTER}: ${GET_VERSION_CMD}" >>"${VERSION_FILE}" 2>&1; then - echo "[ERROR] Failed to write data to file!" + if ! echo "[${LANGUAGE}] ${LINTER}: ${GET_VERSION_CMD}" >>"${VERSION_FILE}" 2>&1; then + echo "[ERROR]: Failed to write data to file!" exit 1 fi fi done + +if ! sort --ignore-case --unique --output="${VERSION_FILE}" "${VERSION_FILE}"; then + echo "[ERROR]:Failed to sort file!" + exit 1 +fi + +echo -e "Versions file contents:\n$(cat "${VERSION_FILE}")" diff --git a/test/lib/linterVersionsTest.sh b/test/lib/linterVersionsTest.sh new file mode 100755 index 00000000..96f83293 --- /dev/null +++ b/test/lib/linterVersionsTest.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +# shellcheck source=/dev/null +source "test/testUtils.sh" + +echo -e "Versions file (${VERSION_FILE}) contents:\n$(cat "${VERSION_FILE}")" + +VersionsFileSortTest() { + local FUNCTION_NAME + FUNCTION_NAME="${FUNCNAME[0]}" + info "${FUNCTION_NAME} start" + + if ! sort --check "${VERSION_FILE}"; then + fatal "Linters version file (${VERSION_FILE}) is not sorted" + fi + + notice "${FUNCTION_NAME} PASS" +} + +VersionsFileCompletenessTest() { + local FUNCTION_NAME + FUNCTION_NAME="${FUNCNAME[0]}" + info "${FUNCTION_NAME} start" + + local LINTERS_VERSION_FILE_LINES_COUNT + LINTERS_VERSION_FILE_LINES_COUNT=$(wc --lines "${VERSION_FILE}" | awk '{print $1}') + debug "Linters version file lines count: ${LINTERS_VERSION_FILE_LINES_COUNT}" + + local EXPECTED_LANGUAGE_COUNT=${#LANGUAGE_ARRAY[@]} + + if ! IsStandardImage; then + EXPECTED_LANGUAGE_COUNT=$((EXPECTED_LANGUAGE_COUNT - ${#LANGUAGES_NOT_IN_SLIM_IMAGE[@]})) + fi + + if [[ ${LINTERS_VERSION_FILE_LINES_COUNT} -ne ${EXPECTED_LANGUAGE_COUNT} ]]; then + fatal "Linters version file lines count (${LINTERS_VERSION_FILE_LINES_COUNT}) doesn't match the length of the languages array (${EXPECTED_LANGUAGE_COUNT}). Is a version descriptor missing in the versions file? Is the version descriptor spanning multiple lines?" + else + debug "The versions file lines count (${LINTERS_VERSION_FILE_LINES_COUNT}) matches the expected value (${EXPECTED_LANGUAGE_COUNT})" + fi + + for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do + if ! IsStandardImage && ! IsLanguageInSlimImage "${LANGUAGE}"; then + debug "Skip checking if ${LANGUAGE} is in the versions file because ${LANGUAGE} is not included in the slim image" + continue + fi + + if ! grep -q "${LANGUAGE}" "${VERSION_FILE}"; then + fatal "${LANGUAGE} is absent from the versions file" + else + debug "${LANGUAGE} present in the versions file" + fi + done + + notice "${FUNCTION_NAME} PASS" +} + +VersionsFileSortTest +VersionsFileCompletenessTest