From 8dc1475a3abfd302a74754e0414914525224e30f Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Wed, 14 Aug 2024 14:54:16 +0200 Subject: [PATCH] feat: allow customizing gitleaks log level (#5993) --- Makefile | 9 +++- README.md | 1 + lib/functions/linterCommands.sh | 8 +++- lib/globals/linterCommandsOptions.sh | 2 + test/lib/linterCommandsTest.sh | 62 ++++++++++++++++++++++++++++ test/run-super-linter-tests.sh | 7 ++++ 6 files changed, 87 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 3bca7fdd..55dd6ef6 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ all: info docker test ## Run all targets. .PHONY: test -test: info validate-container-image-labels docker-build-check docker-dev-container-build-check test-lib inspec lint-codebase fix-codebase test-default-config-files test-actions-runner-debug test-actions-steps-debug test-runner-debug test-find lint-subset-files test-custom-ssl-cert test-non-default-workdir test-git-flags test-non-default-home-directory test-git-initial-commit test-git-merge-commit-push test-log-level test-use-find-and-ignore-gitignored-files test-linters-expect-failure-log-level-notice test-bash-exec-library-expect-success test-bash-exec-library-expect-failure test-save-super-linter-output test-save-super-linter-output-custom-path test-save-super-linter-custom-summary test-linters test-linters-fix-mode ## Run the test suite +test: info validate-container-image-labels docker-build-check docker-dev-container-build-check test-lib inspec lint-codebase fix-codebase test-default-config-files test-actions-runner-debug test-actions-steps-debug test-runner-debug test-find lint-subset-files test-custom-ssl-cert test-non-default-workdir test-git-flags test-non-default-home-directory test-git-initial-commit test-git-merge-commit-push test-log-level test-use-find-and-ignore-gitignored-files test-linters-expect-failure-log-level-notice test-bash-exec-library-expect-success test-bash-exec-library-expect-failure test-save-super-linter-output test-save-super-linter-output-custom-path test-save-super-linter-custom-summary test-custom-gitleaks-log-level test-linters test-linters-fix-mode ## Run the test suite # if this session isn't interactive, then we don't want to allocate a # TTY, which would fail, but if it is interactive, we do want to attach @@ -521,6 +521,13 @@ test-save-super-linter-custom-summary: ## Run super-linter with a custom SUPER_L "run_test_case_custom_summary" \ "$(IMAGE)" +.PHONY: test-custom-gitleaks-log-level +test-custom-gitleaks-log-level: ## Run super-linter with a custom Gitleaks log level + $(CURDIR)/test/run-super-linter-tests.sh \ + $(SUPER_LINTER_TEST_CONTAINER_URL) \ + "run_test_case_gitleaks_custom_log_level" \ + "$(IMAGE)" + .PHONY: docker-dev-container-build-check ## Run Docker build checks against the dev-container image docker-dev-container-build-check: DOCKER_BUILDKIT=1 docker buildx build --check \ diff --git a/README.md b/README.md index a37b8181..3c7f0a38 100644 --- a/README.md +++ b/README.md @@ -250,6 +250,7 @@ You can configure Super-linter using the following environment variables: | **GITHUB_CUSTOM_SERVER_URL** | `https://${GITHUB_DOMAIN}"` | Specify a custom GitHub server URL. Useful for GitHub Enterprise instances. | | **GITHUB_DOMAIN** | `github.com` | Specify a custom GitHub domain in case GitHub Enterprise is used: e.g. `github.myenterprise.com`. `GITHUB_DOMAIN` is a convenience configuration variable to automatically build `GITHUB_CUSTOM_API_URL` and `GITHUB_CUSTOM_SERVER_URL`. | | **GITLEAKS_CONFIG_FILE** | `.gitleaks.toml` | Filename for [GitLeaks configuration](https://github.com/zricethezav/gitleaks#configuration) (ex: `.gitleaks.toml`) | +| **GITLEAKS_LOG_LEVEL** | Gitleaks default log level | Gitleaks log level. Defaults to the Gitleaks default log level. | | **IGNORE_GENERATED_FILES** | `false` | If set to `true`, super-linter will ignore all the files with `@generated` marker but without `@not-generated` marker. | | **IGNORE_GITIGNORED_FILES** | `false` | If set to `true`, super-linter will ignore all the files that are ignored by Git. | | **JAVA_FILE_NAME** | `sun_checks.xml` | Filename for [Checkstyle configuration](https://checkstyle.sourceforge.io/config.html). Checkstyle embeds several configuration files, such as `sun_checks.xml`, `google_checks.xml` that you can use without providing your own configuration file. | diff --git a/lib/functions/linterCommands.sh b/lib/functions/linterCommands.sh index 155a1d62..a9ba45e7 100755 --- a/lib/functions/linterCommands.sh +++ b/lib/functions/linterCommands.sh @@ -52,7 +52,13 @@ if [ "${GITHUB_ACTIONS_COMMAND_ARGS}" != "null" ] && [ -n "${GITHUB_ACTIONS_COMM export GITHUB_ACTIONS_COMMAND_ARGS LINTER_COMMANDS_ARRAY_GITHUB_ACTIONS+=("${GITHUB_ACTIONS_COMMAND_ARGS}") fi -LINTER_COMMANDS_ARRAY_GITLEAKS=(gitleaks detect --no-banner --no-git --redact --config "${GITLEAKS_LINTER_RULES}" --verbose --source) +LINTER_COMMANDS_ARRAY_GITLEAKS=(gitleaks detect --no-banner --no-git --redact --config "${GITLEAKS_LINTER_RULES}" --verbose) +if [ -n "${GITLEAKS_LOG_LEVEL:-}" ]; then + export GITLEAKS_LOG_LEVEL + LINTER_COMMANDS_ARRAY_GITLEAKS+=("${GITLEAKS_LOG_LEVEL_OPTIONS[@]}" "${GITLEAKS_LOG_LEVEL}") + debug "Add log options to the Gitleaks command: ${LINTER_COMMANDS_ARRAY_GITLEAKS[*]}" +fi +LINTER_COMMANDS_ARRAY_GITLEAKS+=(--source) LINTER_COMMANDS_ARRAY_GHERKIN=(gherkin-lint -c "${GHERKIN_LINTER_RULES}") LINTER_COMMANDS_ARRAY_GO=(golangci-lint run -c "${GO_LINTER_RULES}" --fast) LINTER_COMMANDS_ARRAY_GO_MODULES=(golangci-lint run --allow-parallel-runners -c "${GO_LINTER_RULES}") diff --git a/lib/globals/linterCommandsOptions.sh b/lib/globals/linterCommandsOptions.sh index d92204b3..ab6b35d9 100755 --- a/lib/globals/linterCommandsOptions.sh +++ b/lib/globals/linterCommandsOptions.sh @@ -77,3 +77,5 @@ SQLFLUFF_FIX_MODE_OPTIONS+=("${SQLFLUFF_SHARED_SUBCOMMAND_OPTIONS[@]}") # Setting the -n 0 GNU Parallel would not help in this case, because the input # will not be passed to the --workdir option as well. INPUT_CONSUME_COMMAND=("&& echo \"Linted: {}\"") + +GITLEAKS_LOG_LEVEL_OPTIONS=("--log-level") diff --git a/test/lib/linterCommandsTest.sh b/test/lib/linterCommandsTest.sh index 22807af6..b7cf0c25 100755 --- a/test/lib/linterCommandsTest.sh +++ b/test/lib/linterCommandsTest.sh @@ -61,6 +61,7 @@ source "lib/functions/linterCommands.sh" # Initialize the variables we're going to use to verify tests before running tests # because some tests modify LINTER_COMMANDS_xxx variables BASE_LINTER_COMMANDS_ARRAY_ANSIBLE=("${LINTER_COMMANDS_ARRAY_ANSIBLE[@]}") +BASE_LINTER_COMMANDS_ARRAY_GITLEAKS=("${LINTER_COMMANDS_ARRAY_GITLEAKS[@]}") BASE_LINTER_COMMANDS_ARRAY_GO_MODULES=("${LINTER_COMMANDS_ARRAY_GO_MODULES[@]}") BASE_LINTER_COMMANDS_ARRAY_JSCPD=("${LINTER_COMMANDS_ARRAY_JSCPD[@]}") BASE_LINTER_COMMANDS_ARRAY_RUST_CLIPPY=("${LINTER_COMMANDS_ARRAY_RUST_CLIPPY[@]}") @@ -128,6 +129,59 @@ function JscpdCommandTest() { notice "${FUNCTION_NAME} PASS" } +function GitleaksCommandTest() { + local FUNCTION_NAME + FUNCTION_NAME="${FUNCNAME[0]}" + info "${FUNCTION_NAME} start" + + # shellcheck disable=SC2034 + EXPECTED_COMMAND=("${BASE_LINTER_COMMANDS_ARRAY_GITLEAKS[@]}") + + if [[ "${EXPECTED_GITLEAKS_LOG_LEVEL:-}" ]]; then + # The gitleaks command ends with an option to specify the path + # to the file to check, so we need to append the log option before that. + local GITLEAKS_FILE_PATH_OPTION="${EXPECTED_COMMAND[-1]}" + + # Remove the file path option so we can append the log option + unset 'EXPECTED_COMMAND[-1]' + # shellcheck disable=SC2034 + GITLEAKS_LOG_LEVEL="${EXPECTED_GITLEAKS_LOG_LEVEL}" + EXPECTED_COMMAND+=("${GITLEAKS_LOG_LEVEL_OPTIONS[@]}" "${EXPECTED_GITLEAKS_LOG_LEVEL}") + + # Add the file path option back + EXPECTED_COMMAND+=("${GITLEAKS_FILE_PATH_OPTION}") + fi + + # Source the file again so it accounts for modifications + # shellcheck source=/dev/null + source "lib/functions/linterCommands.sh" + + if [[ ! -v GITLEAKS_LOG_LEVEL_OPTIONS ]]; then + fatal "GITLEAKS_LOG_LEVEL_OPTIONS is not defined" + fi + + if [[ "${#GITLEAKS_LOG_LEVEL_OPTIONS[@]}" -eq 0 ]]; then + fatal "GITLEAKS_LOG_LEVEL_OPTIONS is empty" + fi + + if ! AssertArraysElementsContentMatch "LINTER_COMMANDS_ARRAY_GITLEAKS" "EXPECTED_COMMAND"; then + fatal "${FUNCTION_NAME} test failed" + fi + + notice "${FUNCTION_NAME} PASS" +} + +function GitleaksCommandCustomLogLevelTest() { + local FUNCTION_NAME + FUNCTION_NAME="${FUNCNAME[0]}" + info "${FUNCTION_NAME} start" + + EXPECTED_GITLEAKS_LOG_LEVEL="debug" + GitleaksCommandTest + + notice "${FUNCTION_NAME} PASS" +} + function InitInputConsumeCommandsTest() { local FUNCTION_NAME FUNCTION_NAME="${FUNCNAME[0]}" @@ -231,6 +285,10 @@ function InitFixModeOptionsAndCommandsTest() { } function InitPowerShellCommandTest() { + local FUNCTION_NAME + FUNCTION_NAME="${FUNCNAME[0]}" + info "${FUNCTION_NAME} start" + # shellcheck disable=SC2034 EXPECTED_LINTER_COMMANDS_ARRAY_POWERSHELL=(pwsh -NoProfile -NoLogo -Command "\"${LINTER_COMMANDS_ARRAY_POWERSHELL[*]}; if (\\\${Error}.Count) { exit 1 }\"") InitPowerShellCommand @@ -238,11 +296,15 @@ function InitPowerShellCommandTest() { if ! AssertArraysElementsContentMatch "LINTER_COMMANDS_ARRAY_POWERSHELL" "EXPECTED_LINTER_COMMANDS_ARRAY_POWERSHELL"; then fatal "${FUNCTION_NAME} test failed" fi + + notice "${FUNCTION_NAME} PASS" } LinterCommandPresenceTest IgnoreGitIgnoredFilesJscpdCommandTest JscpdCommandTest +GitleaksCommandTest +GitleaksCommandCustomLogLevelTest InitInputConsumeCommandsTest InitFixModeOptionsAndCommandsTest InitPowerShellCommandTest diff --git a/test/run-super-linter-tests.sh b/test/run-super-linter-tests.sh index 90b01d66..54227874 100755 --- a/test/run-super-linter-tests.sh +++ b/test/run-super-linter-tests.sh @@ -162,6 +162,13 @@ run_test_case_custom_summary() { SUPER_LINTER_SUMMARY_FILE_NAME="custom-github-step-summary.md" } +run_test_case_gitleaks_custom_log_level() { + run_test_cases_expect_success + CREATE_LOG_FILE="true" + SAVE_SUPER_LINTER_OUTPUT="true" + COMMAND_TO_RUN+=(--env GITLEAKS_LOG_LEVEL="warn") +} + run_test_case_fix_mode() { CREATE_LOG_FILE="true" SAVE_SUPER_LINTER_OUTPUT="true"