From 78ed3ef5fc8fe45290b530f94077281d2ac8e634 Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Sun, 28 Jul 2024 21:34:40 +0200 Subject: [PATCH] fix: store outputs in the main output directory (#5899) - Store Super-linter outputs in the main output directory. - Update README with a few fixes and details about outputs. - Add missing test cases for when the configuration didn't enable any output. --- Makefile | 9 ++- README.md | 38 +++++++++--- lib/functions/validation.sh | 34 +++++------ lib/linter.sh | 84 ++++++++++++++++++++------ test/lib/validationTest.sh | 55 ++++++----------- test/run-super-linter-tests.sh | 105 +++++++++++++++++++++++++-------- 6 files changed, 218 insertions(+), 107 deletions(-) diff --git a/Makefile b/Makefile index 9866ccaf..cefdedfb 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ all: info docker test ## Run all targets. .PHONY: test -test: info validate-container-image-labels docker-build-check docker-dev-container-build-check test-lib inspec lint-codebase test-default-config-files test-actions-runner-debug test-actions-steps-debug test-runner-debug test-find lint-subset-files test-custom-ssl-cert test-non-default-workdir test-git-flags test-non-default-home-directory test-git-initial-commit test-log-level test-use-find-and-ignore-gitignored-files test-linters-expect-failure-log-level-notice test-bash-exec-library-expect-success test-bash-exec-library-expect-failure test-save-super-linter-output test-save-super-linter-output-custom-path test-linters ## Run the test suite +test: info validate-container-image-labels docker-build-check docker-dev-container-build-check test-lib inspec lint-codebase test-default-config-files test-actions-runner-debug test-actions-steps-debug test-runner-debug test-find lint-subset-files test-custom-ssl-cert test-non-default-workdir test-git-flags test-non-default-home-directory test-git-initial-commit test-log-level test-use-find-and-ignore-gitignored-files test-linters-expect-failure-log-level-notice test-bash-exec-library-expect-success test-bash-exec-library-expect-failure test-save-super-linter-output test-save-super-linter-output-custom-path test-save-super-linter-custom-summary test-linters ## Run the test suite # if this session isn't interactive, then we don't want to allocate a # TTY, which would fail, but if it is interactive, we do want to attach @@ -429,6 +429,13 @@ test-save-super-linter-output-custom-path: ## Run super-linter with SAVE_SUPER_L "run_test_cases_save_super_linter_output_custom_path" \ "$(IMAGE)" +.PHONY: test-save-super-linter-custom-summary +test-save-super-linter-custom-summary: ## Run super-linter with a custom SUPER_LINTER_SUMMARY_FILE_NAME + $(CURDIR)/test/run-super-linter-tests.sh \ + $(SUPER_LINTER_TEST_CONTAINER_URL) \ + "run_test_case_custom_summary" \ + "$(IMAGE)" + .PHONY: docker-dev-container-build-check ## Run Docker build checks against the dev-container image docker-dev-container-build-check: DOCKER_BUILDKIT=1 docker buildx build --check \ diff --git a/README.md b/README.md index c05226fa..dbf3226e 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,7 @@ You can configure super-linter using the following environment variables: | **DOCKERFILE_HADOLINT_FILE_NAME** | `.hadolint.yaml` | Filename for [hadolint configuration](https://github.com/hadolint/hadolint) (ex: `.hadolintlintrc.yaml`) | | **EDITORCONFIG_FILE_NAME** | `.ecrc` | Filename for [editorconfig-checker configuration](https://github.com/editorconfig-checker/editorconfig-checker) | | **ENABLE_GITHUB_ACTIONS_GROUP_TITLE** | `false` if `RUN_LOCAL=true`, `true` otherwise | Flag to enable [GitHub Actions log grouping](https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines). | -| **ENABLE_GITHUB_ACTIONS_STEP_SUMMARY** | `false` if `RUN_LOCAL=true`, `true` otherwise | Flag to enable [GitHub Actions job summary](https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary) for the Super-linter action. | +| **ENABLE_GITHUB_ACTIONS_STEP_SUMMARY** | `false` if `RUN_LOCAL=true`, `true` otherwise | Flag to enable [GitHub Actions job summary](https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary) for the Super-linter action. For more information, see [Summary outputs](#summary-outputs). | | **FILTER_REGEX_EXCLUDE** | not set | Regular expression defining which files will be excluded from linting (ex: `.*src/test.*`). Not setting this variable means to process all files. | | **FILTER_REGEX_INCLUDE** | not set | Regular expression defining which files will be processed by linters (ex: `.*src/.*`). Not setting this variable means to process all files. `FILTER_REGEX_INCLUDE` is evaluated before `FILTER_REGEX_EXCLUDE`. | | **GITHUB_ACTIONS_CONFIG_FILE** | `actionlint.yml` | Filename for [Actionlint configuration](https://github.com/rhysd/actionlint/blob/main/docs/config.md) (ex: `actionlint.yml`) | @@ -220,14 +220,14 @@ You can configure super-linter using the following environment variables: | **JSCPD_CONFIG_FILE** | `.jscpd.json` | Filename for JSCPD configuration | | **KUBERNETES_KUBECONFORM_OPTIONS** | `null` | Additional arguments to pass to the command-line when running **Kubernetes Kubeconform** (Example: --ignore-missing-schemas) | | **LINTER_RULES_PATH** | `.github/linters` | Directory for all linter configuration rules. | -| **LOG_FILE** | `super-linter.log` | The filename for outputting logs. All output is sent to the log file regardless of `LOG_LEVEL`. | +| **LOG_FILE** | `super-linter.log` | The filename for outputting logs. Super-linter saves the log file to `${DEFAULT_WORKSPACE}/${LOG_FILE}`. | | **LOG_LEVEL** | `INFO` | How much output the script will generate to the console. One of `ERROR`, `WARN`, `NOTICE`, `INFO`, or `DEBUG`. | | **MARKDOWN_CONFIG_FILE** | `.markdown-lint.yml` | Filename for [Markdownlint configuration](https://github.com/DavidAnson/markdownlint#optionsconfig) (ex: `.markdown-lint.yml`, `.markdownlint.json`, `.markdownlint.yaml`) | | **MARKDOWN_CUSTOM_RULE_GLOBS** | `.markdown-lint/rules,rules/**` | Comma-separated list of [file globs](https://github.com/igorshubovych/markdownlint-cli#globbing) matching [custom Markdownlint rule files](https://github.com/DavidAnson/markdownlint/blob/main/doc/CustomRules.md). | | **MULTI_STATUS** | `true` | A status API is made for each language that is linted to make visual parsing easier. | | **NATURAL_LANGUAGE_CONFIG_FILE** | `.textlintrc` | Filename for [textlint configuration](https://textlint.github.io/docs/getting-started.html#configuration) (ex: `.textlintrc`) | | **PERL_PERLCRITIC_OPTIONS** | `null` | Additional arguments to pass to the command-line when running **perlcritic** (Example: --theme community) | -| **POWERSHELL_CONFIG_FILE** | `.powershell-psscriptanalyzer.psd1` | Filename for [PSScriptAnalyzer configuration](https://learn.microsoft.com/en-gb/powershell/utility-modules/psscriptanalyzer/using-scriptanalyzer) (ex: `.powershell-psscriptanalyzer.psd1`, `PSScriptAnalyzerSettings.psd1`) | +| **POWERSHELL_CONFIG_FILE** | `.powershell-psscriptanalyzer.psd1` | Filename for [PSScriptAnalyzer configuration](https://learn.microsoft.com/en-gb/powershell/utility-modules/psscriptanalyzer/using-scriptanalyzer) | | **PHP_CONFIG_FILE** | `php.ini` | Filename for [PHP Configuration](https://www.php.net/manual/en/configuration.file.php) (ex: `php.ini`) | | **PHP_PHPCS_FILE_NAME** | `phpcs.xml` | Filename for [PHP CodeSniffer](https://github.com/squizlabs/PHP_CodeSniffer) (ex: `.phpcs.xml`, `.phpcs.xml.dist`) | | **PROTOBUF_CONFIG_FILE** | `.protolintrc.yml` | Filename for [protolint configuration](https://github.com/yoheimuta/protolint/blob/master/_example/config/.protolint.yaml) (ex: `.protolintrc.yml`) | @@ -239,7 +239,8 @@ You can configure super-linter using the following environment variables: | **PYTHON_RUFF_CONFIG_FILE** | `.ruff.toml` | Filename for [ruff configuration](https://docs.astral.sh/ruff/configuration/) | | **RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES** | not set | Comma-separated filenames for [renovate shareable config preset](https://docs.renovatebot.com/config-presets/) (ex: `default.json`) | | **RUBY_CONFIG_FILE** | `.ruby-lint.yml` | Filename for [rubocop configuration](https://docs.rubocop.org/rubocop/configuration.html) (ex: `.ruby-lint.yml`, `.rubocop.yml`) | -| **SAVE_SUPER_LINTER_OUTPUT** | `false` | If set to `true`, super-linter will save its output to `${DEFAULT_WORKSPACE}/${SUPER_LINTER_OUTPUT_DIRECTORY_NAME}` | +| **SAVE_SUPER_LINTER_OUTPUT** | `false` | If set to `true`, Super-linter will save its output in the workspace. For more information, see [Super-linter outputs](#super-linter-outputs). | +| **SAVE_SUPER_LINTER_SUMMARY** | `false` | If set to `true`, Super-linter will save a summary. For more information, see [Summary outputs](#summary-outputs). | | **SCALAFMT_CONFIG_FILE** | `.scalafmt.conf` | Filename for [scalafmt configuration](https://scalameta.org/scalafmt/docs/configuration.html) (ex: `.scalafmt.conf`) | | **SNAKEMAKE_SNAKEFMT_CONFIG_FILE** | `.snakefmt.toml` | Filename for [Snakemake configuration](https://github.com/snakemake/snakefmt#configuration) (ex: `pyproject.toml`, `.snakefmt.toml`) | | **SSL_CERT_SECRET** | `none` | SSL cert to add to the **Super-Linter** trust store. This is needed for users on `self-hosted` runners or need to inject the cert for security standards (ex. ${{ secrets.SSL_CERT }}) | @@ -249,6 +250,7 @@ You can configure super-linter using the following environment variables: | **SQL_CONFIG_FILE** | `.sql-config.json` | Filename for [SQL-Lint configuration](https://sql-lint.readthedocs.io/en/latest/files/configuration.html) (ex: `sql-config.json` , `.config.json`) | | **SQLFLUFF_CONFIG_FILE** | `/.sqlfluff` | Filename for [SQLFLUFF configuration](https://docs.sqlfluff.com/en/stable/configuration.html) (ex: `/.sqlfluff`, `pyproject.toml`) | | **SUPER_LINTER_OUTPUT_DIRECTORY_NAME** | `super-linter-output` | Name of the directory where super-linter saves its output. | +| **SUPER_LINTER_SUMMARY_FILE_NAME** | `super-linter-summary.md` | Name of the file where to save the summary output. For more information, see [Summary outputs](#summary-outputs). | | **SUPPRESS_FILE_TYPE_WARN** | `false` | If set to `true`, will hide warning messages about files without their proper extensions. Default is `false` | | **SUPPRESS_POSSUM** | `false` | If set to `true`, will hide the ASCII possum at top of log output. Default is `false` | | **TERRAFORM_TERRASCAN_CONFIG_FILE** | `terrascan.toml` | Filename for [terrascan configuration](https://github.com/accurics/terrascan) (ex: `terrascan.toml`) | @@ -454,17 +456,37 @@ path to the files that contains a CA that can be used to valide the certificate: SSL_CERT_SECRET: ${{ secrets.ROOT_CA }} ``` -## Super-linter outputs +## Outputs + +Super-linter supports generating several outputs, and also supports exposing +the output of individual linters. + +### Summary outputs + +Super-linter writes a summary of all the checks: + +- If `SAVE_SUPER_LINTER_SUMMARY` is set to `true`, Super-linter writes + a summary to + `${DEFAULT_WORKSPACE}/${SUPER_LINTER_OUTPUT_DIRECTORY_NAME}/${SUPER_LINTER_SUMMARY_FILE_NAME}`. +- If `ENABLE_GITHUB_ACTIONS_STEP_SUMMARY` is set to `true`, Super-linter writes + a GitHub Actions job summary. Setting `ENABLE_GITHUB_ACTIONS_STEP_SUMMARY` to + `true`, implies setting `SAVE_SUPER_LINTER_SUMMARY` to `true`. + +The summary is in Markdown format. Super-linter supports the following formats: + +- Table (default) + +### Super-linter outputs If you set `SAVE_SUPER_LINTER_OUTPUT` to `true`, Super-linter saves its output -to `${DEFAULT_WORKSPACE}/${DEFAULT_SUPER_LINTER_OUTPUT_DIRECTORY_NAME}`, so you +to `${DEFAULT_WORKSPACE}/${SUPER_LINTER_OUTPUT_DIRECTORY_NAME}/super-linter`, so you can further process it, if needed. Most outputs are in JSON format. -The output of previous Super-linter runs is not preserved when running locally. +The output of previous Super-linter runs is not preserved. -## Linter reports and outputs +### Linter reports and outputs Some linters support configuring the format of their outputs for further processing. To get access to that output, enable it using the respective linter diff --git a/lib/functions/validation.sh b/lib/functions/validation.sh index 87bd1fee..93448537 100755 --- a/lib/functions/validation.sh +++ b/lib/functions/validation.sh @@ -17,6 +17,7 @@ function ValidateBooleanConfigurationVariables() { ValidateBooleanVariable "MULTI_STATUS" "${MULTI_STATUS}" ValidateBooleanVariable "RUN_LOCAL" "${RUN_LOCAL}" ValidateBooleanVariable "SAVE_SUPER_LINTER_OUTPUT" "${SAVE_SUPER_LINTER_OUTPUT}" + ValidateBooleanVariable "SAVE_SUPER_LINTER_SUMMARY" "${SAVE_SUPER_LINTER_SUMMARY}" ValidateBooleanVariable "SSH_INSECURE_NO_VERIFY_GITHUB_KEY" "${SSH_INSECURE_NO_VERIFY_GITHUB_KEY}" ValidateBooleanVariable "SSH_SETUP_GITHUB" "${SSH_SETUP_GITHUB}" ValidateBooleanVariable "SUPPRESS_FILE_TYPE_WARN" "${SUPPRESS_FILE_TYPE_WARN}" @@ -292,25 +293,22 @@ function ValidateGitHubUrls() { fi } -function ValidateGitHubActionsStepSummary() { - if [[ "${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY:-}" == "true" ]]; then - debug "GitHub Actions step summary is enabled. ENABLE_GITHUB_ACTIONS_STEP_SUMMARY: ${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY}" - if [[ -z "${GITHUB_STEP_SUMMARY:-}" ]]; then - error "GITHUB_STEP_SUMMARY is not set." - return 1 - fi - debug "GITHUB_STEP_SUMMARY is set to: ${GITHUB_STEP_SUMMARY}" - if [[ ! -e "${GITHUB_STEP_SUMMARY}" ]]; then - error "GITHUB_STEP_SUMMARY (${GITHUB_STEP_SUMMARY}) doesn't exist." - return 1 - fi - if [[ ! -f "${GITHUB_STEP_SUMMARY}" ]]; then - error "GITHUB_STEP_SUMMARY (${GITHUB_STEP_SUMMARY}) is not a file." - return 1 - fi - else - debug "GitHub Actions step summary is disabled because ENABLE_GITHUB_ACTIONS_STEP_SUMMARY is set to: ${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY}). No need to validate its configuration." +function ValidateSuperLinterSummaryOutputPath() { + debug "Validating SUPER_LINTER_SUMMARY_OUTPUT_PATH" + if [[ -z "${SUPER_LINTER_SUMMARY_OUTPUT_PATH:-}" ]]; then + error "SUPER_LINTER_SUMMARY_OUTPUT_PATH is not set." + return 1 fi + debug "SUPER_LINTER_SUMMARY_OUTPUT_PATH is set to: ${SUPER_LINTER_SUMMARY_OUTPUT_PATH}" + if [[ ! -e "${SUPER_LINTER_SUMMARY_OUTPUT_PATH}" ]]; then + error "SUPER_LINTER_SUMMARY_OUTPUT_PATH (${SUPER_LINTER_SUMMARY_OUTPUT_PATH}) doesn't exist." + return 1 + fi + if [[ ! -f "${SUPER_LINTER_SUMMARY_OUTPUT_PATH}" ]]; then + error "SUPER_LINTER_SUMMARY_OUTPUT_PATH (${SUPER_LINTER_SUMMARY_OUTPUT_PATH}) is not a file." + return 1 + fi + debug "Super-linter summary ouput path passed validation" } function WarnIfVariableIsSet() { diff --git a/lib/linter.sh b/lib/linter.sh index 2f639785..a9d67000 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -61,9 +61,6 @@ startGitHubActionsLogGroup "${SUPER_LINTER_INITIALIZATION_LOG_GROUP_TITLE}" # Let users configure GitHub Actions step summary regardless of running locally or not ENABLE_GITHUB_ACTIONS_STEP_SUMMARY="${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY:-"${DEFAULT_ENABLE_GITHUB_ACTIONS_STEP_SUMMARY}"}" export ENABLE_GITHUB_ACTIONS_STEP_SUMMARY -if ! ValidateGitHubActionsStepSummary; then - fatal "GitHub Actions job summary configuration failed validation" -fi # We want a lowercase value declare -l BASH_EXEC_IGNORE_LIBRARIES @@ -126,6 +123,10 @@ VALIDATE_ALL_CODEBASE="${VALIDATE_ALL_CODEBASE:-"true"}" declare -l YAML_ERROR_ON_WARNING YAML_ERROR_ON_WARNING="${YAML_ERROR_ON_WARNING:-false}" +# We want a lowercase value +declare -l SAVE_SUPER_LINTER_SUMMARY +SAVE_SUPER_LINTER_SUMMARY="${SAVE_SUPER_LINTER_SUMMARY:-false}" + ValidateBooleanConfigurationVariables ########### @@ -562,14 +563,14 @@ Footer() { local SUPER_LINTER_EXIT_CODE SUPER_LINTER_EXIT_CODE=0 - if [[ "${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY}" == "true" ]]; then - debug "Saving GitHub Actions step summary to ${GITHUB_STEP_SUMMARY}" + if [[ "${SAVE_SUPER_LINTER_SUMMARY}" == "true" ]]; then + debug "Saving Super-linter summary to ${SUPER_LINTER_SUMMARY_OUTPUT_PATH}" { echo "# Super-linter summary" echo "" echo "| Language | Validation result |" echo "| -----------------------|-------------------|" - } >>"${GITHUB_STEP_SUMMARY}" + } >>"${SUPER_LINTER_SUMMARY_OUTPUT_PATH}" fi for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do @@ -589,8 +590,8 @@ Footer() { if [[ ${ERROR_COUNTER} -ne 0 ]]; then error "Errors found in ${LANGUAGE}" - if [[ "${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY}" == "true" ]]; then - echo "| ${LANGUAGE} | Fail ❌ |" >>"${GITHUB_STEP_SUMMARY}" + if [[ "${SAVE_SUPER_LINTER_SUMMARY}" == "true" ]]; then + echo "| ${LANGUAGE} | Fail ❌ |" >>"${SUPER_LINTER_SUMMARY_OUTPUT_PATH}" fi # Print output as error in case users disabled the INFO level so they @@ -617,8 +618,8 @@ Footer() { debug "Setting super-linter exit code to ${SUPER_LINTER_EXIT_CODE} because there were errors for ${LANGUAGE}" elif [[ ${ERROR_COUNTER} -eq 0 ]]; then notice "Successfully linted ${LANGUAGE}" - if [[ "${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY}" == "true" ]]; then - echo "| ${LANGUAGE} | Pass ✅ |" >>"${GITHUB_STEP_SUMMARY}" + if [[ "${SAVE_SUPER_LINTER_SUMMARY}" == "true" ]]; then + echo "| ${LANGUAGE} | Pass ✅ |" >>"${SUPER_LINTER_SUMMARY_OUTPUT_PATH}" fi CallStatusAPI "${LANGUAGE}" "success" ANY_LINTER_SUCCESS="true" @@ -639,24 +640,31 @@ Footer() { if [[ ${SUPER_LINTER_EXIT_CODE} -eq 0 ]]; then notice "All files and directories linted successfully" - if [[ "${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY}" == "true" ]]; then + if [[ "${SAVE_SUPER_LINTER_SUMMARY}" == "true" ]]; then { echo "" echo "All files and directories linted successfully" - } >>"${GITHUB_STEP_SUMMARY}" + } >>"${SUPER_LINTER_SUMMARY_OUTPUT_PATH}" fi else error "Super-linter detected linting errors" - if [[ "${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY}" == "true" ]]; then + if [[ "${SAVE_SUPER_LINTER_SUMMARY}" == "true" ]]; then { echo "" echo "Super-linter detected linting errors" - } >>"${GITHUB_STEP_SUMMARY}" + } >>"${SUPER_LINTER_SUMMARY_OUTPUT_PATH}" fi fi + if [[ "${SAVE_SUPER_LINTER_SUMMARY}" == "true" ]]; then + debug "Super-linter summary file (${SUPER_LINTER_SUMMARY_OUTPUT_PATH}) contents:\n$(cat "${SUPER_LINTER_SUMMARY_OUTPUT_PATH}")" + fi + if [[ "${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY}" == "true" ]]; then - debug "GitHub Actions step summary file (${GITHUB_STEP_SUMMARY}) contents:\n$(cat "${GITHUB_STEP_SUMMARY}")" + debug "Appending Super-linter summary to ${GITHUB_STEP_SUMMARY}" + if ! cat "${SUPER_LINTER_SUMMARY_OUTPUT_PATH}" >>"${GITHUB_STEP_SUMMARY}"; then + fatal "Error while appending the content of ${SUPER_LINTER_SUMMARY_OUTPUT_PATH} to ${GITHUB_STEP_SUMMARY}" + fi fi exit ${SUPER_LINTER_EXIT_CODE} @@ -776,9 +784,13 @@ debug "R_RULES_FILE_PATH_IN_ROOT: ${R_RULES_FILE_PATH_IN_ROOT}" DEFAULT_SUPER_LINTER_OUTPUT_DIRECTORY_NAME="super-linter-output" SUPER_LINTER_OUTPUT_DIRECTORY_NAME="${SUPER_LINTER_OUTPUT_DIRECTORY_NAME:-${DEFAULT_SUPER_LINTER_OUTPUT_DIRECTORY_NAME}}" export SUPER_LINTER_OUTPUT_DIRECTORY_NAME -debug "Super-linter output directory name: ${SUPER_LINTER_OUTPUT_DIRECTORY_NAME}" +debug "Super-linter main output directory name: ${SUPER_LINTER_OUTPUT_DIRECTORY_NAME}" -SUPER_LINTER_OUTPUT_DIRECTORY_PATH="${GITHUB_WORKSPACE}/${SUPER_LINTER_OUTPUT_DIRECTORY_NAME}" +SUPER_LINTER_MAIN_OUTPUT_DIRECTORY_PATH="${GITHUB_WORKSPACE}/${SUPER_LINTER_OUTPUT_DIRECTORY_NAME}" +export SUPER_LINTER_MAIN_OUTPUT_DIRECTORY_PATH +debug "Super-linter main output directory path: ${SUPER_LINTER_MAIN_OUTPUT_DIRECTORY_PATH}" + +SUPER_LINTER_OUTPUT_DIRECTORY_PATH="${SUPER_LINTER_MAIN_OUTPUT_DIRECTORY_PATH}/super-linter" export SUPER_LINTER_OUTPUT_DIRECTORY_PATH debug "Super-linter output directory path: ${SUPER_LINTER_OUTPUT_DIRECTORY_PATH}" @@ -787,6 +799,33 @@ export SUPER_LINTER_PRIVATE_OUTPUT_DIRECTORY_PATH debug "Super-linter private output directory path: ${SUPER_LINTER_PRIVATE_OUTPUT_DIRECTORY_PATH}" mkdir -p "${SUPER_LINTER_PRIVATE_OUTPUT_DIRECTORY_PATH}" +SUPER_LINTER_SUMMARY_OUTPUT_PATH="${SUPER_LINTER_MAIN_OUTPUT_DIRECTORY_PATH}/${SUPER_LINTER_SUMMARY_FILE_NAME:-"super-linter-summary.md"}" +export SUPER_LINTER_SUMMARY_OUTPUT_PATH +debug "Super-linter summary output path: ${SUPER_LINTER_SUMMARY_OUTPUT_PATH}" + +if [[ "${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY}" == "true" ]] && [[ "${SAVE_SUPER_LINTER_SUMMARY}" == "false" ]]; then + debug "ENABLE_GITHUB_ACTIONS_STEP_SUMMARY is set to ${SAVE_SUPER_LINTER_SUMMARY}, but SAVE_SUPER_LINTER_SUMMARY is set to ${SAVE_SUPER_LINTER_SUMMARY}" + SAVE_SUPER_LINTER_SUMMARY="true" + debug "Set SAVE_SUPER_LINTER_SUMMARY to ${SAVE_SUPER_LINTER_SUMMARY} because we need to append its contents to ${GITHUB_STEP_SUMMARY} later" +fi + +# Ensure that the main output directory and files exist because the user might not have created them +# before running Super-linter. These conditions list all the cases that require an output +# directory to be there. +if [[ "${SAVE_SUPER_LINTER_OUTPUT}" = "true" ]] || + [[ "${SAVE_SUPER_LINTER_SUMMARY}" == "true" ]] || + [[ "${CREATE_LOG_FILE}" = "true" ]]; then + debug "Ensure that ${SUPER_LINTER_MAIN_OUTPUT_DIRECTORY_PATH} exists" + mkdir -p "${SUPER_LINTER_MAIN_OUTPUT_DIRECTORY_PATH}" +fi + +if [[ "${SAVE_SUPER_LINTER_SUMMARY}" == "true" ]]; then + debug "Ensuring that ${SUPER_LINTER_SUMMARY_OUTPUT_PATH} exists." + if ! touch "${SUPER_LINTER_SUMMARY_OUTPUT_PATH}"; then + fatal "Cannot create Super-linter summary file: ${SUPER_LINTER_SUMMARY_OUTPUT_PATH}" + fi +fi + ############################ # Validate the environment # ############################ @@ -799,7 +838,16 @@ if ! ValidateValidationVariables; then fatal "Error while validating the configuration of enabled linters" fi if ! ValidateAnsibleDirectory; then - fatal "Error while validating the configuration of enabled linters" + fatal "Error while validating the configuration of the Ansible directory" +fi + +if [[ "${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY}" == "true" ]] || + [[ "${SAVE_SUPER_LINTER_SUMMARY}" == "true" ]]; then + if ! ValidateSuperLinterSummaryOutputPath; then + fatal "Super-linter summary configuration failed validation" + fi +else + debug "Super-linter summary is disabled. No need to validate its configuration." fi if [[ "${USE_FIND_ALGORITHM}" == "false" ]] || [[ "${IGNORE_GITIGNORED_FILES}" == "true" ]]; then diff --git a/test/lib/validationTest.sh b/test/lib/validationTest.sh index 2bf3ff55..c2a9c86b 100755 --- a/test/lib/validationTest.sh +++ b/test/lib/validationTest.sh @@ -156,55 +156,34 @@ function ValidateGitHubUrlsTest() { notice "${FUNCTION_NAME} PASS" } -function ValidateGitHubActionsStepSummaryTest() { +function ValidateSuperLinterSummaryOutputPathTest() { local FUNCTION_NAME FUNCTION_NAME="${FUNCNAME[0]}" info "${FUNCTION_NAME} start" - ENABLE_GITHUB_ACTIONS_STEP_SUMMARY="false" - if ! ValidateGitHubActionsStepSummary; then - fatal "ValidateGitHubActionsStepSummary shouldn't fail when ENABLE_GITHUB_ACTIONS_STEP_SUMMARY is ${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY}" + SUPER_LINTER_SUMMARY_OUTPUT_PATH="/non/existing/file" + if ValidateSuperLinterSummaryOutputPath; then + fatal "SUPER_LINTER_SUMMARY_OUTPUT_PATH=${SUPER_LINTER_SUMMARY_OUTPUT_PATH} should have failed validation when SUPER_LINTER_SUMMARY_OUTPUT_PATH is set to a non-existing file" else - info "ENABLE_GITHUB_ACTIONS_STEP_SUMMARY=${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY} passed validation as expected" + info "SUPER_LINTER_SUMMARY_OUTPUT_PATH=${SUPER_LINTER_SUMMARY_OUTPUT_PATH} failed validation as expected" fi + unset SUPER_LINTER_SUMMARY_OUTPUT_PATH - ENABLE_GITHUB_ACTIONS_STEP_SUMMARY="true" - if ValidateGitHubActionsStepSummary; then - fatal "ENABLE_GITHUB_ACTIONS_STEP_SUMMARY=${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY} should have failed validation when GITHUB_STEP_SUMMARY is not set" + SUPER_LINTER_SUMMARY_OUTPUT_PATH="$(pwd)" + if ValidateSuperLinterSummaryOutputPath; then + fatal "SUPER_LINTER_SUMMARY_OUTPUT_PATH=${SUPER_LINTER_SUMMARY_OUTPUT_PATH} should have failed validation when SUPER_LINTER_SUMMARY_OUTPUT_PATH is set to a directory" else - info "ENABLE_GITHUB_ACTIONS_STEP_SUMMARY=${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY} failed validation as expected" + info "SUPER_LINTER_SUMMARY_OUTPUT_PATH=${SUPER_LINTER_SUMMARY_OUTPUT_PATH} failed validation as expected" fi - unset ENABLE_GITHUB_ACTIONS_STEP_SUMMARY + unset SUPER_LINTER_SUMMARY_OUTPUT_PATH - ENABLE_GITHUB_ACTIONS_STEP_SUMMARY="true" - GITHUB_STEP_SUMMARY="/non/existing/file" - if ValidateGitHubActionsStepSummary; then - fatal "ENABLE_GITHUB_ACTIONS_STEP_SUMMARY=${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY}, GITHUB_STEP_SUMMARY=${GITHUB_STEP_SUMMARY} should have failed validation when GITHUB_STEP_SUMMARY is set to a non-existing file" + SUPER_LINTER_SUMMARY_OUTPUT_PATH="${0}" + if ! ValidateSuperLinterSummaryOutputPath; then + fatal "SUPER_LINTER_SUMMARY_OUTPUT_PATH=${SUPER_LINTER_SUMMARY_OUTPUT_PATH} should have passed validation when SUPER_LINTER_SUMMARY_OUTPUT_PATH is set to a file" else - info "ENABLE_GITHUB_ACTIONS_STEP_SUMMARY=${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY}, GITHUB_STEP_SUMMARY=${GITHUB_STEP_SUMMARY} failed validation as expected" + info "SUPER_LINTER_SUMMARY_OUTPUT_PATH=${SUPER_LINTER_SUMMARY_OUTPUT_PATH} passed validation as expected" fi - unset ENABLE_GITHUB_ACTIONS_STEP_SUMMARY - unset GITHUB_STEP_SUMMARY - - ENABLE_GITHUB_ACTIONS_STEP_SUMMARY="true" - GITHUB_STEP_SUMMARY="$(pwd)" - if ValidateGitHubActionsStepSummary; then - fatal "ENABLE_GITHUB_ACTIONS_STEP_SUMMARY=${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY}, GITHUB_STEP_SUMMARY=${GITHUB_STEP_SUMMARY} should have failed validation when GITHUB_STEP_SUMMARY is set to a directory" - else - info "ENABLE_GITHUB_ACTIONS_STEP_SUMMARY=${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY}, GITHUB_STEP_SUMMARY=${GITHUB_STEP_SUMMARY} failed validation as expected" - fi - unset ENABLE_GITHUB_ACTIONS_STEP_SUMMARY - unset GITHUB_STEP_SUMMARY - - ENABLE_GITHUB_ACTIONS_STEP_SUMMARY="true" - GITHUB_STEP_SUMMARY="${0}" - if ! ValidateGitHubActionsStepSummary; then - fatal "ENABLE_GITHUB_ACTIONS_STEP_SUMMARY=${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY}, GITHUB_STEP_SUMMARY=${GITHUB_STEP_SUMMARY} should have passed validation when GITHUB_STEP_SUMMARY is set to a file" - else - info "ENABLE_GITHUB_ACTIONS_STEP_SUMMARY=${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY}, GITHUB_STEP_SUMMARY=${GITHUB_STEP_SUMMARY} passed validation as expected" - fi - unset ENABLE_GITHUB_ACTIONS_STEP_SUMMARY - unset GITHUB_STEP_SUMMARY + unset SUPER_LINTER_SUMMARY_OUTPUT_PATH notice "${FUNCTION_NAME} PASS" } @@ -402,7 +381,7 @@ IsUnsignedIntegerSuccessTest IsUnsignedIntegerFailureTest ValidateDeprecatedVariablesTest ValidateGitHubUrlsTest -ValidateGitHubActionsStepSummaryTest +ValidateSuperLinterSummaryOutputPathTest ValidateFindModeTest ValidateAnsibleDirectoryTest ValidateValidationVariablesTest diff --git a/test/run-super-linter-tests.sh b/test/run-super-linter-tests.sh index b274a99d..046cda39 100755 --- a/test/run-super-linter-tests.sh +++ b/test/run-super-linter-tests.sh @@ -13,6 +13,8 @@ DEFAULT_BRANCH="main" COMMAND_TO_RUN=(docker run -t -e DEFAULT_BRANCH="${DEFAULT_BRANCH}" -e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true) +LEFTOVERS_TO_CLEAN=() + ignore_test_cases() { COMMAND_TO_RUN+=(-e FILTER_REGEX_EXCLUDE=".*(/test/linters/|CHANGELOG.md).*") } @@ -25,13 +27,13 @@ run_test_cases_expect_failure() { configure_linters_for_test_cases COMMAND_TO_RUN+=(-e ANSIBLE_DIRECTORY="/test/linters/ansible/bad" -e CHECKOV_FILE_NAME=".checkov-test-linters-failure.yaml" -e FILTER_REGEX_INCLUDE=".*bad.*") EXPECTED_EXIT_CODE=1 - EXPECTED_SUPER_LINTER_STEP_SUMMARY_FILE_PATH="test/data/github-actions-step-summary/expected-step-summary-test-linters-expect-failure-${SUPER_LINTER_CONTAINER_IMAGE_TYPE}.md" + EXPECTED_SUPER_LINTER_SUMMARY_FILE_PATH="test/data/github-actions-step-summary/expected-step-summary-test-linters-expect-failure-${SUPER_LINTER_CONTAINER_IMAGE_TYPE}.md" } run_test_cases_expect_success() { configure_linters_for_test_cases COMMAND_TO_RUN+=(-e ANSIBLE_DIRECTORY="/test/linters/ansible/good" -e CHECKOV_FILE_NAME=".checkov-test-linters-success.yaml" -e FILTER_REGEX_INCLUDE=".*good.*") - EXPECTED_SUPER_LINTER_STEP_SUMMARY_FILE_PATH="test/data/github-actions-step-summary/expected-step-summary-test-linters-expect-success-${SUPER_LINTER_CONTAINER_IMAGE_TYPE}.md" + EXPECTED_SUPER_LINTER_SUMMARY_FILE_PATH="test/data/github-actions-step-summary/expected-step-summary-test-linters-expect-success-${SUPER_LINTER_CONTAINER_IMAGE_TYPE}.md" } run_test_cases_log_level() { @@ -103,6 +105,11 @@ run_test_cases_save_super_linter_output_custom_path() { SUPER_LINTER_OUTPUT_DIRECTORY_NAME="custom-super-linter-output-directory-name" } +run_test_case_custom_summary() { + run_test_cases_expect_success + SUPER_LINTER_SUMMARY_FILE_NAME="custom-github-step-summary.md" +} + # Run the test setup function ${TEST_FUNCTION_NAME} @@ -112,39 +119,72 @@ SAVE_SUPER_LINTER_OUTPUT="${SAVE_SUPER_LINTER_OUTPUT:-false}" if [ -n "${SUPER_LINTER_OUTPUT_DIRECTORY_NAME:-}" ]; then COMMAND_TO_RUN+=(-e SUPER_LINTER_OUTPUT_DIRECTORY_NAME="${SUPER_LINTER_OUTPUT_DIRECTORY_NAME}") fi +SUPER_LINTER_OUTPUT_DIRECTORY_NAME="${SUPER_LINTER_OUTPUT_DIRECTORY_NAME:-"super-linter-output"}" +SUPER_LINTER_MAIN_OUTPUT_PATH="$(pwd)/${SUPER_LINTER_OUTPUT_DIRECTORY_NAME}" +echo "Super-linter main output path: ${SUPER_LINTER_MAIN_OUTPUT_PATH}" +SUPER_LINTER_OUTPUT_PATH="${SUPER_LINTER_MAIN_OUTPUT_PATH}/super-linter" +echo "Super-linter output path: ${SUPER_LINTER_OUTPUT_PATH}" COMMAND_TO_RUN+=(-e CREATE_LOG_FILE="${CREATE_LOG_FILE}") COMMAND_TO_RUN+=(-e LOG_LEVEL="${LOG_LEVEL:-"DEBUG"}") COMMAND_TO_RUN+=(-e RUN_LOCAL="${RUN_LOCAL:-true}") COMMAND_TO_RUN+=(-e SAVE_SUPER_LINTER_OUTPUT="${SAVE_SUPER_LINTER_OUTPUT}") -COMMAND_TO_RUN+=(-v "${SUPER_LINTER_WORKSPACE:-$(pwd)}:/tmp/lint") +COMMAND_TO_RUN+=(-v "${SUPER_LINTER_WORKSPACE:-$(pwd)}":"/tmp/lint") -if [ -n "${EXPECTED_SUPER_LINTER_STEP_SUMMARY_FILE_PATH:-}" ]; then - echo "Expected Super-linter step summary file path: ${EXPECTED_SUPER_LINTER_STEP_SUMMARY_FILE_PATH}" - SUPER_LINTER_STEP_SUMMARY_FILE="$(pwd)/super-linter-github-actions-step-summary-output.md" - echo "Create Super-linter step summary file: ${SUPER_LINTER_STEP_SUMMARY_FILE}" - # Remove eventual leftovers from previous tests - rm --force "${SUPER_LINTER_STEP_SUMMARY_FILE}" - touch "${SUPER_LINTER_STEP_SUMMARY_FILE}" - SUPER_LINTER_STEP_SUMMARY_FILE_INSIDE_CONTAINER="/tmp/lint/$(basename "${SUPER_LINTER_STEP_SUMMARY_FILE}")" - COMMAND_TO_RUN+=(-e GITHUB_STEP_SUMMARY="${SUPER_LINTER_STEP_SUMMARY_FILE_INSIDE_CONTAINER}") +SUPER_LINTER_GITHUB_STEP_SUMMARY_FILE_PATH="$(pwd)/github-step-summary.md" +# We can't put this inside SUPER_LINTER_MAIN_OUTPUT_PATH because it doesn't exist +# before Super-linter creates it, and we want to verify that as well. +echo "SUPER_LINTER_GITHUB_STEP_SUMMARY_FILE_PATH: ${SUPER_LINTER_GITHUB_STEP_SUMMARY_FILE_PATH}" + +if [ -n "${EXPECTED_SUPER_LINTER_SUMMARY_FILE_PATH:-}" ]; then + echo "Expected Super-linter step summary file path: ${EXPECTED_SUPER_LINTER_SUMMARY_FILE_PATH}" ENABLE_GITHUB_ACTIONS_STEP_SUMMARY="true" + SAVE_SUPER_LINTER_SUMMARY="true" + + COMMAND_TO_RUN+=(-e GITHUB_STEP_SUMMARY="${SUPER_LINTER_GITHUB_STEP_SUMMARY_FILE_PATH}") + COMMAND_TO_RUN+=(-v "${SUPER_LINTER_GITHUB_STEP_SUMMARY_FILE_PATH}":"${SUPER_LINTER_GITHUB_STEP_SUMMARY_FILE_PATH}") fi -COMMAND_TO_RUN+=(-e ENABLE_GITHUB_ACTIONS_STEP_SUMMARY="${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY:-"false"}") + +ENABLE_GITHUB_ACTIONS_STEP_SUMMARY="${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY:-"false"}" +COMMAND_TO_RUN+=(-e ENABLE_GITHUB_ACTIONS_STEP_SUMMARY="${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY}") +COMMAND_TO_RUN+=(-e SAVE_SUPER_LINTER_SUMMARY="${SAVE_SUPER_LINTER_SUMMARY:-"false"}") + +if [ -n "${SUPER_LINTER_SUMMARY_FILE_NAME:-}" ]; then + COMMAND_TO_RUN+=(-e SUPER_LINTER_SUMMARY_FILE_NAME="${SUPER_LINTER_SUMMARY_FILE_NAME}") +fi +SUPER_LINTER_SUMMARY_FILE_NAME="${SUPER_LINTER_SUMMARY_FILE_NAME:-"super-linter-summary.md"}" +echo "SUPER_LINTER_SUMMARY_FILE_NAME: ${SUPER_LINTER_SUMMARY_FILE_NAME}" + +SUPER_LINTER_SUMMARY_FILE_PATH="${SUPER_LINTER_MAIN_OUTPUT_PATH}/${SUPER_LINTER_SUMMARY_FILE_NAME}" +echo "Super-linter summary output path: ${SUPER_LINTER_SUMMARY_FILE_PATH}" + +LOG_FILE_PATH="$(pwd)/super-linter.log" COMMAND_TO_RUN+=("${SUPER_LINTER_TEST_CONTAINER_URL}") declare -i EXPECTED_EXIT_CODE EXPECTED_EXIT_CODE=${EXPECTED_EXIT_CODE:-0} +echo "Cleaning eventual leftovers before running tests: ${LEFTOVERS_TO_CLEAN[*]}" +LEFTOVERS_TO_CLEAN+=("${LOG_FILE_PATH}") +LEFTOVERS_TO_CLEAN+=("${SUPER_LINTER_GITHUB_STEP_SUMMARY_FILE_PATH}") +LEFTOVERS_TO_CLEAN+=("${SUPER_LINTER_MAIN_OUTPUT_PATH}") +LEFTOVERS_TO_CLEAN+=("${SUPER_LINTER_SUMMARY_FILE_PATH}") +sudo rm -rfv "${LEFTOVERS_TO_CLEAN[@]}" + +if [[ "${ENABLE_GITHUB_ACTIONS_STEP_SUMMARY}" == "true" ]]; then + echo "Creating GitHub Actions step summary file: ${SUPER_LINTER_GITHUB_STEP_SUMMARY_FILE_PATH}" + touch "${SUPER_LINTER_GITHUB_STEP_SUMMARY_FILE_PATH}" +fi + if [ ${EXPECTED_EXIT_CODE} -ne 0 ]; then echo "Disable failures on error because the expected exit code is ${EXPECTED_EXIT_CODE}" set +o errexit fi echo "Command to run: ${COMMAND_TO_RUN[*]}" - "${COMMAND_TO_RUN[@]}" + SUPER_LINTER_EXIT_CODE=$? # Enable the errexit option in case we disabled it set -o errexit @@ -152,7 +192,6 @@ set -o errexit echo "Super-linter exit code: ${SUPER_LINTER_EXIT_CODE}" if [[ "${CREATE_LOG_FILE}" == true ]]; then - LOG_FILE_PATH="$(pwd)/super-linter.log" if [ ! -e "${LOG_FILE_PATH}" ]; then echo "Log file was requested but it's not available at ${LOG_FILE_PATH}" exit 1 @@ -166,10 +205,6 @@ else fi if [[ "${SAVE_SUPER_LINTER_OUTPUT}" == true ]]; then - - SUPER_LINTER_OUTPUT_DIRECTORY_NAME="${SUPER_LINTER_OUTPUT_DIRECTORY_NAME:-"super-linter-output"}" - - SUPER_LINTER_OUTPUT_PATH="$(pwd)/${SUPER_LINTER_OUTPUT_DIRECTORY_NAME}" if [ ! -d "${SUPER_LINTER_OUTPUT_PATH}" ]; then echo "Super-linter output was requested but it's not available at ${SUPER_LINTER_OUTPUT_PATH}" exit 1 @@ -180,18 +215,40 @@ if [[ "${SAVE_SUPER_LINTER_OUTPUT}" == true ]]; then fi else echo "Super-linter output was not requested. SAVE_SUPER_LINTER_OUTPUT: ${SAVE_SUPER_LINTER_OUTPUT}" + + if [ -e "${SUPER_LINTER_OUTPUT_PATH}" ]; then + echo "Super-linter output was not requested but it's available at ${SUPER_LINTER_OUTPUT_PATH}" + exit 1 + fi fi -if [ -n "${EXPECTED_SUPER_LINTER_STEP_SUMMARY_FILE_PATH:-}" ]; then +if [ -n "${EXPECTED_SUPER_LINTER_SUMMARY_FILE_PATH:-}" ]; then # Remove eventual HTML comments from the expected file because we use them to disable certain linter rules - if ! diff "${SUPER_LINTER_STEP_SUMMARY_FILE}" <(grep -vE '^\s*