feat(bash-exec): add option to ignore shell library files (#5254)

Introduce a new configuration variable, BASH_EXEC_IGNORE_LIBRARIES. If
set to true, the behaviour of bash-exec is modified: if a shell file has
a file extension and no shebang line, it is ignored, i.e., allowed to be
non-executable. This allows files that are only every sourced from other
shell files, acting as libraries and not executables, to have no
executable bit set without failing the bash-exec linter.
This commit is contained in:
Benjamin Wuethrich 2024-02-27 13:17:22 -05:00 committed by GitHub
parent 4a05d78ed4
commit 95aabd4cfa
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 233 additions and 33 deletions

View file

@ -4,7 +4,7 @@
all: info docker test ## Run all targets.
.PHONY: test
test: info validate-container-image-labels test-lib inspec lint-codebase test-default-config-files test-actions-runner-debug test-actions-steps-debug test-runner-debug test-find lint-subset-files test-custom-ssl-cert test-non-default-workdir test-git-flags test-non-default-home-directory test-log-level test-linters-expect-failure-log-level-notice test-linters ## Run the test suite
test: info validate-container-image-labels test-lib inspec lint-codebase test-default-config-files test-actions-runner-debug test-actions-steps-debug test-runner-debug test-find lint-subset-files test-custom-ssl-cert test-non-default-workdir test-git-flags test-non-default-home-directory test-log-level test-linters-expect-failure-log-level-notice test-bash-exec-library-expect-success test-bash-exec-library-expect-failure test-linters ## Run the test suite
# if this session isn't interactive, then we don't want to allocate a
# TTY, which would fail, but if it is interactive, we do want to attach
@ -86,11 +86,11 @@ else
RELEASE_PLEASE_TARGET_BRANCH := "${GITHUB_HEAD_REF}"
endif
.phony: check-github-token
.PHONY: check-github-token
check-github-token:
@if [ ! -f "${GITHUB_TOKEN_PATH}" ]; then echo "Cannot find the file to load the GitHub access token: $(GITHUB_TOKEN_PATH). Create a readable file there, and populate it with a GitHub personal access token."; exit 1; fi
.phony: inspec
.PHONY: inspec
inspec: inspec-check ## Run InSpec tests
DOCKER_CONTAINER_STATE="$$(docker inspect --format "{{.State.Running}}" $(SUPER_LINTER_TEST_CONTAINER_NAME) 2>/dev/null || echo "")"; \
if [ "$$DOCKER_CONTAINER_STATE" = "true" ]; then docker kill $(SUPER_LINTER_TEST_CONTAINER_NAME); fi && \
@ -110,7 +110,7 @@ inspec: inspec-check ## Run InSpec tests
&& docker ps \
&& docker kill $(SUPER_LINTER_TEST_CONTAINER_NAME)
.phony: docker
.PHONY: docker
docker: check-github-token ## Build the container image
DOCKER_BUILDKIT=1 docker buildx build --load \
--build-arg BUILD_DATE=$(BUILD_DATE) \
@ -120,11 +120,11 @@ docker: check-github-token ## Build the container image
--target $(IMAGE) \
-t $(SUPER_LINTER_TEST_CONTAINER_URL) .
.phony: docker-pull
.PHONY: docker-pull
docker-pull: ## Pull the container image from registry
docker pull $(SUPER_LINTER_TEST_CONTAINER_URL)
.phony: validate-container-image-labels
.PHONY: validate-container-image-labels
validate-container-image-labels: ## Validate container image labels
$(CURDIR)/test/validate-docker-labels.sh \
$(SUPER_LINTER_TEST_CONTAINER_URL) \
@ -167,7 +167,7 @@ test-runner-debug: ## Run super-linter with RUNNER_DEBUG=1
-v "$(CURDIR)/.github":/tmp/lint/.github \
$(SUPER_LINTER_TEST_CONTAINER_URL)
.phony: test-find
.PHONY: test-find
test-find: ## Run super-linter on a subdirectory with USE_FIND_ALGORITHM=true
docker run \
-e RUN_LOCAL=true \
@ -180,7 +180,7 @@ test-find: ## Run super-linter on a subdirectory with USE_FIND_ALGORITHM=true
# We need to set USE_FIND_ALGORITHM=true because the DEFALUT_WORKSPACE is not
# a Git directory in this test case
.phony: test-non-default-workdir
.PHONY: test-non-default-workdir
test-non-default-workdir: ## Run super-linter with DEFAULT_WORKSPACE set
docker run \
-e RUN_LOCAL=true \
@ -193,7 +193,7 @@ test-non-default-workdir: ## Run super-linter with DEFAULT_WORKSPACE set
-v $(CURDIR)/.github:/tmp/not-default-workspace/.github \
$(SUPER_LINTER_TEST_CONTAINER_URL)
.phony: test-git-flags
.PHONY: test-git-flags
test-git-flags: ## Run super-linter with different git-related flags
docker run \
-e RUN_LOCAL=true \
@ -207,7 +207,7 @@ test-git-flags: ## Run super-linter with different git-related flags
-v "$(CURDIR)":/tmp/lint \
$(SUPER_LINTER_TEST_CONTAINER_URL)
.phony: lint-codebase
.PHONY: lint-codebase
lint-codebase: ## Lint the entire codebase
docker run \
-e RUN_LOCAL=true \
@ -223,10 +223,10 @@ lint-codebase: ## Lint the entire codebase
# This is a smoke test to check how much time it takes to lint only a small
# subset of files, compared to linting the whole codebase.
.phony: lint-subset-files
.PHONY: lint-subset-files
lint-subset-files: lint-subset-files-enable-only-one-type lint-subset-files-enable-expensive-io-checks
.phony: lint-subset-files-enable-only-one-type
.PHONY: lint-subset-files-enable-only-one-type
lint-subset-files-enable-only-one-type: ## Lint a small subset of files in the codebase by enabling only one linter
time docker run \
-e RUN_LOCAL=true \
@ -239,7 +239,7 @@ lint-subset-files-enable-only-one-type: ## Lint a small subset of files in the c
-v "$(CURDIR):/tmp/lint" \
$(SUPER_LINTER_TEST_CONTAINER_URL)
.phony: lint-subset-files-enable-expensive-io-checks
.PHONY: lint-subset-files-enable-expensive-io-checks
lint-subset-files-enable-expensive-io-checks: ## Lint a small subset of files in the codebase and keep expensive I/O operations to check file types enabled
time docker run \
-e RUN_LOCAL=true \
@ -258,10 +258,10 @@ lint-subset-files-enable-expensive-io-checks: ## Lint a small subset of files in
-v "$(CURDIR):/tmp/lint" \
$(SUPER_LINTER_TEST_CONTAINER_URL)
.phony: test-lib
test-lib: test-build-file-list test-github-event test-setup-ssh test-validation ## Test super-linter
.PHONY: test-lib
test-lib: test-build-file-list test-detect-files test-github-event test-setup-ssh test-validation ## Test super-linter
.phony: test-build-file-list
.PHONY: test-build-file-list
test-build-file-list: ## Test buildFileList
docker run \
-v "$(CURDIR):/tmp/lint" \
@ -269,7 +269,15 @@ test-build-file-list: ## Test buildFileList
--entrypoint /tmp/lint/test/lib/buildFileListTest.sh \
$(SUPER_LINTER_TEST_CONTAINER_URL)
.phony: test-github-event
.PHONY: test-detect-files
test-detect-files: ## Test detectFiles
docker run \
-v "$(CURDIR):/tmp/lint" \
-w /tmp/lint \
--entrypoint /tmp/lint/test/lib/detectFilesTest.sh \
$(SUPER_LINTER_TEST_CONTAINER_URL)
.PHONY: test-github-event
test-github-event: ## Test githubEvent
docker run \
-v "$(CURDIR):/tmp/lint" \
@ -277,7 +285,7 @@ test-github-event: ## Test githubEvent
--entrypoint /tmp/lint/test/lib/githubEventTest.sh \
$(SUPER_LINTER_TEST_CONTAINER_URL)
.phony: test-setup-ssh
.PHONY: test-setup-ssh
test-setup-ssh: ## Test setupSSH
@docker run \
-e GITHUB_TOKEN=${GITHUB_TOKEN} \
@ -286,7 +294,7 @@ test-setup-ssh: ## Test setupSSH
--entrypoint /tmp/lint/test/lib/setupSSHTest.sh \
$(SUPER_LINTER_TEST_CONTAINER_URL)
.phony: test-validation
.PHONY: test-validation
test-validation: ## Test validation
docker run \
-v "$(CURDIR):/tmp/lint" \
@ -297,7 +305,7 @@ test-validation: ## Test validation
# Run this test against a small directory because we're only interested in
# loading default configuration files. The directory that we run super-linter
# against should not be .github because that includes default linter rules.
.phony: test-default-config-files
.PHONY: test-default-config-files
test-default-config-files: ## Test default configuration files loading
docker run \
-e RUN_LOCAL=true \
@ -308,7 +316,7 @@ test-default-config-files: ## Test default configuration files loading
-v "$(CURDIR)/docs":/tmp/lint \
$(SUPER_LINTER_TEST_CONTAINER_URL)
.phony: test-custom-ssl-cert
.PHONY: test-custom-ssl-cert
test-custom-ssl-cert: ## Test the configuration of a custom SSL/TLS certificate
docker run \
-e RUN_LOCAL=true \
@ -320,47 +328,59 @@ test-custom-ssl-cert: ## Test the configuration of a custom SSL/TLS certificate
-v "$(CURDIR)/docs":/tmp/lint \
$(SUPER_LINTER_TEST_CONTAINER_URL)
.phony: test-non-default-home-directory
.PHONY: test-non-default-home-directory
test-non-default-home-directory: ## Test a non-default HOME directory
$(CURDIR)/test/run-super-linter-tests.sh \
$(SUPER_LINTER_TEST_CONTAINER_URL) \
"run_test_cases_non_default_home"
.phony: test-linters
.PHONY: test-linters
test-linters: test-linters-expect-success test-linters-expect-failure ## Run the linters test suite
.phony: test-linters-expect-success
.PHONY: test-linters-expect-success
test-linters-expect-success: ## Run the linters test suite expecting successes
$(CURDIR)/test/run-super-linter-tests.sh \
$(SUPER_LINTER_TEST_CONTAINER_URL) \
"run_test_cases_expect_success"
.phony: test-linters-expect-failure
.PHONY: test-linters-expect-failure
test-linters-expect-failure: ## Run the linters test suite expecting failures
$(CURDIR)/test/run-super-linter-tests.sh \
$(SUPER_LINTER_TEST_CONTAINER_URL) \
"run_test_cases_expect_failure"
.phony: test-log-level
.PHONY: test-log-level
test-log-level: ## Run a test to check if there are conflicts with the LOG_LEVEL variable
$(CURDIR)/test/run-super-linter-tests.sh \
$(SUPER_LINTER_TEST_CONTAINER_URL) \
"run_test_cases_log_level"
.phony: test-linters-expect-failure-log-level-notice
.PHONY: test-linters-expect-failure-log-level-notice
test-linters-expect-failure-log-level-notice: ## Run the linters test suite expecting failures with a LOG_LEVEL set to NOTICE
$(CURDIR)/test/run-super-linter-tests.sh \
$(SUPER_LINTER_TEST_CONTAINER_URL) \
"run_test_cases_expect_failure_notice_log"
.phony: build-dev-container-image
.PHONY: test-bash-exec-library-expect-success
test-bash-exec-library-expect-success: ## Run the linters test cases for BASH_EXEC expecting successes with BASH_EXEC_IGNORE_LIBRARIES set to true
$(CURDIR)/test/run-super-linter-tests.sh \
$(SUPER_LINTER_TEST_CONTAINER_URL) \
"run_test_case_bash_exec_library_expect_success"
.PHONY: test-bash-exec-library-expect-failure
test-bash-exec-library-expect-failure: ## Run the linters test cases for BASH_EXEC expecting failures with BASH_EXEC_IGNORE_LIBRARIES set to true
$(CURDIR)/test/run-super-linter-tests.sh \
$(SUPER_LINTER_TEST_CONTAINER_URL) \
"run_test_case_bash_exec_library_expect_failure"
.PHONY: build-dev-container-image
build-dev-container-image: ## Build commit linter container image
DOCKER_BUILDKIT=1 docker buildx build --load \
--build-arg GID=$(shell id -g) \
--build-arg UID=$(shell id -u) \
-t ${DEV_CONTAINER_URL} "${CURDIR}/dev-dependencies"
.phony: lint-commits
.PHONY: lint-commits
lint-commits: build-dev-container-image ## Lint commits
docker run \
-v "$(CURDIR):/source-repository" \
@ -372,7 +392,7 @@ lint-commits: build-dev-container-image ## Lint commits
--to ${TO_INTERVAL_COMMITLINT} \
--verbose
.phony: release-please-dry-run
.PHONY: release-please-dry-run
release-please-dry-run: build-dev-container-image check-github-token ## Run release-please in dry-run mode to preview the release pull request
@echo "Running release-please against branch: ${RELEASE_PLEASE_TARGET_BRANCH}"; \
docker run \

View file

@ -189,6 +189,7 @@ You can configure super-linter using the following environment variables:
|-------------------------------------------------|---------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| **ANSIBLE_CONFIG_FILE** | `.ansible-lint.yml` | Filename for [Ansible-lint configuration](https://ansible.readthedocs.io/projects/lint/configuring/) (ex: `.ansible-lint`, `.ansible-lint.yml`) |
| **ANSIBLE_DIRECTORY** | `/ansible` | Flag to set the root directory for Ansible file location(s), relative to `DEFAULT_WORKSPACE`. Set to `.` to use the top-level of the `DEFAULT_WORKSPACE`. |
| **BASH_EXEC_IGNORE_LIBRARIES** | `false | If set to `true`, shell files with a file extension and no shebang line are ignored when checking if the executable bit is set. |
| **BASH_SEVERITY** | `style` | Specify the minimum severity of errors to consider in shellcheck. Valid values in order of severity are error, warning, info and style. |
| **CHECKOV_FILE_NAME** | `.checkov.yaml` | Configuration filename for Checkov. |
| **CREATE_LOG_FILE** | `false` | If set to `true`, it creates the log file. You can set the log filename using the `LOG_FILE` environment variable. This overrides any existing log files. |

View file

@ -231,6 +231,22 @@ function IsValidShellScript() {
return 1
}
# HasNoShebang returns true if a file has no shebang line
function HasNoShebang() {
local FILE SHEBANG FIRST_TWO
FILE="${1}"
SHEBANG='#!'
IFS= read -rn2 FIRST_TWO <"${FILE}"
if [[ ${FIRST_TWO} == "${SHEBANG}" ]]; then
return 1
fi
debug "${FILE} doesn't contain a shebang"
return 0
}
function IsGenerated() {
FILE="$1"
@ -265,6 +281,7 @@ export -f DetectTektonFile
export -f GetFileExtension
export -f GetFileType
export -f IsValidShellScript
export -f HasNoShebang
export -f IsGenerated
function RunAdditionalInstalls() {

View file

@ -17,7 +17,11 @@ LINTER_COMMANDS_ARRAY_BASH=(shellcheck --color --external-sources)
if [ -n "${BASH_SEVERITY}" ]; then
LINTER_COMMANDS_ARRAY_BASH+=(--severity="${BASH_SEVERITY}")
fi
LINTER_COMMANDS_ARRAY_BASH_EXEC=(bash-exec)
LINTER_COMMANDS_ARRAY_BASH_EXEC=(bash-exec '{}')
if [ "${BASH_EXEC_IGNORE_LIBRARIES}" == 'true' ]; then
debug "Enabling bash-exec option to ignore shell library files."
LINTER_COMMANDS_ARRAY_BASH_EXEC+=('true')
fi
LINTER_COMMANDS_ARRAY_CHECKOV=(checkov --config-file "${CHECKOV_LINTER_RULES}")
if CheckovConfigurationFileContainsDirectoryOption "${CHECKOV_LINTER_RULES}"; then
# Consume the input as we do with ANSIBLE

View file

@ -15,6 +15,7 @@ LOG_FILE="${LOG_FILE:-"super-linter.log"}"
LOG_LEVEL="${LOG_LEVEL:-"INFO"}"
declare -l CREATE_LOG_FILE
CREATE_LOG_FILE="${CREATE_LOG_FILE:-"false"}"
export CREATE_LOG_FILE
if [[ ${ACTIONS_RUNNER_DEBUG} == true ]] ||
[[ ${ACTIONS_STEPS_DEBUG} == true ]] ||

View file

@ -2,6 +2,7 @@
function ValidateBooleanConfigurationVariables() {
ValidateBooleanVariable "ACTIONS_RUNNER_DEBUG" "${ACTIONS_RUNNER_DEBUG}"
ValidateBooleanVariable "BASH_EXEC_IGNORE_LIBRARIES" "${BASH_EXEC_IGNORE_LIBRARIES}"
ValidateBooleanVariable "CREATE_LOG_FILE" "${CREATE_LOG_FILE}"
ValidateBooleanVariable "DISABLE_ERRORS" "${DISABLE_ERRORS}"
ValidateBooleanVariable "ENABLE_GITHUB_ACTIONS_GROUP_TITLE" "${ENABLE_GITHUB_ACTIONS_GROUP_TITLE}"

View file

@ -50,6 +50,10 @@ export ENABLE_GITHUB_ACTIONS_GROUP_TITLE
startGitHubActionsLogGroup "${SUPER_LINTER_INITIALIZATION_LOG_GROUP_TITLE}"
# We want a lowercase value
declare -l BASH_EXEC_IGNORE_LIBRARIES
BASH_EXEC_IGNORE_LIBRARIES="${BASH_EXEC_IGNORE_LIBRARIES:-false}"
# We want a lowercase value
declare -l DISABLE_ERRORS
DISABLE_ERRORS="${DISABLE_ERRORS:-"false"}"

View file

@ -2,7 +2,15 @@
set -euo pipefail
if ! [[ -x "$1" ]]; then
echo "Error: File:[$1] is not executable"
FILE="${1}"
IGNORE_LIBRARY="${2:-false}"
if [[ "${IGNORE_LIBRARY}" == "true" ]] && HasNoShebang "${FILE}"; then
echo "${FILE} is being ignored because IGNORE_LIBRARY is set to ${IGNORE_LIBRARY}"
exit 0
fi
if ! [[ -x "${FILE}" ]]; then
echo "Error: File:[${FILE}] is not executable"
exit 1
fi

110
test/lib/detectFilesTest.sh Executable file
View file

@ -0,0 +1,110 @@
#!/usr/bin/env bash
set -o errexit
set -o nounset
set -o pipefail
# shellcheck disable=SC2034
LOG_DEBUG="true"
# shellcheck disable=SC2034
LOG_VERBOSE="true"
# shellcheck disable=SC2034
LOG_NOTICE="true"
# shellcheck disable=SC2034
LOG_WARN="true"
# shellcheck disable=SC2034
LOG_ERROR="true"
# shellcheck source=/dev/null
source "lib/functions/log.sh"
# shellcheck disable=SC2034
CREATE_LOG_FILE=false
# shellcheck source=/dev/null
source "lib/functions/detectFiles.sh"
function RecognizeNoShebangTest() {
local FILE="test/linters/bash_exec/libraries/noShebang_bad.sh"
debug "Confirming ${FILE} has no shebang"
if ! HasNoShebang "${FILE}"; then
fatal "${FILE} is mis-classified as having a shebang"
fi
FUNCTION_NAME="${FUNCNAME[0]}"
notice "${FUNCTION_NAME} PASS"
}
RecognizeCommentIsNotShebangTest() {
local FILE="test/linters/bash_exec/libraries/comment_bad.sh"
debug "Confirming ${FILE} starting with a comment has no shebang"
if ! HasNoShebang "${FILE}"; then
fatal "${FILE} with a comment is mis-classified as having a shebang"
fi
FUNCTION_NAME="${FUNCNAME[0]}"
notice "${FUNCTION_NAME} PASS"
}
RecognizeIndentedShebangAsCommentTest() {
local FILE="test/linters/bash_exec/libraries/indentedShebang_bad.sh"
debug "Confirming indented shebang in ${FILE} is considered a comment"
if ! HasNoShebang "${FILE}"; then
fatal "${FILE} with a comment is mis-classified as having a shebang"
fi
FUNCTION_NAME="${FUNCNAME[0]}"
notice "${FUNCTION_NAME} PASS"
}
RecognizeSecondLineShebangAsCommentTest() {
local FILE="test/linters/bash_exec/libraries/secondLineShebang_bad.sh"
debug "Confirming shebang on second line in ${FILE} is considered a comment"
if ! HasNoShebang "${FILE}"; then
fatal "${FILE} with a comment is mis-classified as having a shebang"
fi
FUNCTION_NAME="${FUNCNAME[0]}"
notice "${FUNCTION_NAME} PASS"
}
function RecognizeShebangTest() {
local FILE="test/linters/bash_exec/libraries/shebang_bad.sh"
debug "Confirming ${FILE} has a shebang"
if HasNoShebang "${FILE}"; then
fatal "${FILE} is mis-classified as not having a shebang"
fi
FUNCTION_NAME="${FUNCNAME[0]}"
notice "${FUNCTION_NAME} PASS"
}
function RecognizeShebangWithBlankTest() {
local FILE="test/linters/bash_exec/libraries/shebangWithBlank_bad.sh"
debug "Confirming shebang with blank in ${FILE} is recognized"
if HasNoShebang "${FILE}"; then
fatal "${FILE} is mis-classified as not having a shebang"
fi
FUNCTION_NAME="${FUNCNAME[0]}"
notice "${FUNCTION_NAME} PASS"
}
RecognizeNoShebangTest
RecognizeCommentIsNotShebangTest
RecognizeIndentedShebangAsCommentTest
RecognizeSecondLineShebangAsCommentTest
RecognizeShebangTest
RecognizeShebangWithBlankTest

View file

@ -0,0 +1,4 @@
# A comment is not a shebang
function hello() {
echo "Hello, world!"
}

View file

@ -0,0 +1,4 @@
#!/usr/bin/env whatever
function hello() {
echo "Hello, world!"
}

View file

@ -0,0 +1,3 @@
function hello() {
echo "Hello, world!"
}

View file

@ -0,0 +1,5 @@
#!/usr/bin/env bash
function hello() {
echo "Hello, world!"
}

View file

@ -0,0 +1,4 @@
#! /usr/bin/env bash
function hello() {
echo "Hello, world!"
}

View file

@ -0,0 +1,4 @@
#!/usr/bin/env bash
function hello() {
echo "Hello, world!"
}

View file

@ -33,6 +33,16 @@ run_test_cases_non_default_home() {
COMMAND_TO_RUN+=(-e HOME=/tmp)
}
run_test_case_bash_exec_library_expect_failure() {
run_test_cases_expect_failure
COMMAND_TO_RUN+=(-e BASH_EXEC_IGNORE_LIBRARIES="true")
}
run_test_case_bash_exec_library_expect_success() {
run_test_cases_expect_success
COMMAND_TO_RUN+=(-e BASH_EXEC_IGNORE_LIBRARIES="true")
}
# Run the test setup function
${TEST_FUNCTION_NAME}