diff --git a/.automation/validate-docker-labels.sh b/.automation/validate-docker-labels.sh index 4d25ed0f..745583bf 100755 --- a/.automation/validate-docker-labels.sh +++ b/.automation/validate-docker-labels.sh @@ -7,9 +7,6 @@ ########### # Globals # ########### -# GITHUB_WORKSPACE="${GITHUB_WORKSPACE}" # GitHub Workspace -# GITHUB_SHA="${GITHUB_SHA}" # Sha used to create this branch -# BUILD_DATE="${BUILD_DATE}" # Date the container was built IMAGE="${1}" # Image of the super-linter we build BUILD_REVISION="${GITHUB_SHA}" # GitHub Sha BUILD_VERSION="${GITHUB_SHA}" # Version of the container diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c899021..0aed2774 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,7 +73,6 @@ jobs: VALIDATE_ALL_CODEBASE: false GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} DEFAULT_BRANCH: main - LOCAL_UPDATES: true RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES: "default.json,hoge.json" TYPESCRIPT_STANDARD_TSCONFIG_FILE: ".github/linters/tsconfig.json" diff --git a/docs/run-linter-locally.md b/docs/run-linter-locally.md index 5402ff98..c8050e3f 100644 --- a/docs/run-linter-locally.md +++ b/docs/run-linter-locally.md @@ -109,7 +109,7 @@ jobs: uses: super-linter/super-linter@v5 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DEFAULT_BRANCH: develop + DEFAULT_BRANCH: main ``` ## Build the container image and run the test suite locally diff --git a/lib/README.md b/lib/README.md deleted file mode 100644 index 76496541..00000000 --- a/lib/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Super-Linter Library - -## Main script - -The file `linter.sh` is the main script that is called for the process and loads all other scripts as functions. - -## Functions - -The additional files in the folder are functions to help streamline the main build process and allow for easier maintenance. - -- `possum.sh` - - Official mascot of the **Super-Linter** -- `buildFileList.sh` - - Functions to help find files that were modified, or need to be scanned -- `validation.sh` - - Logic to see what linters are enabled -- `worker.sh` - - Calls to the various linters and the test cases for each diff --git a/lib/functions/buildFileList.sh b/lib/functions/buildFileList.sh index e7ba1361..252f349e 100755 --- a/lib/functions/buildFileList.sh +++ b/lib/functions/buildFileList.sh @@ -12,6 +12,7 @@ #### Function BuildFileList #################################################### function IssueHintForFullGitHistory() { info "Check that you have the full git history, the checkout is not shallow, etc" + info "Is shallow repository: $(git -C "${GITHUB_WORKSPACE}" rev-parse --is-shallow-repository)" info "See https://github.com/super-linter/super-linter#example-connecting-github-action-workflow" } @@ -19,30 +20,26 @@ function IssueHintForFullGitHistory() { #### Function GenerateFileDiff ################################################# function GenerateFileDiff() { CMD="${1}" - ################ - # print header # - ################ - debug "----------------------------------------------" debug "Generating Diff with:[$CMD]" ################################################# # Get the Array of files changed in the commits # ################################################# - CMD_OUTPUT=$(eval "$CMD") + CMD_OUTPUT=$(eval "set -eo pipefail; $CMD; set +eo pipefail") ####################### # Load the error code # ####################### ERROR_CODE=$? + debug "Diff command return code: ${ERROR_CODE}" ############################## # Check the shell for errors # ############################## if [ ${ERROR_CODE} -ne 0 ]; then - # Error - info "Failed to get Diff with:[$CMD]" + error "Failed to get Diff with:[$CMD]" IssueHintForFullGitHistory - fatal "[${CMD_OUTPUT}]" + fatal "[Diff command output: ${CMD_OUTPUT}]" fi ################################################### @@ -69,78 +66,32 @@ function BuildFileList() { ANSIBLE_DIRECTORY="${3}" debug "ANSIBLE_DIRECTORY: ${ANSIBLE_DIRECTORY}" - debug "IGNORE_GITIGNORED_FILES: ${IGNORE_GITIGNORED_FILES}" - - debug "IGNORE_GENERATED_FILES: ${IGNORE_GENERATED_FILES}" - - debug "USE_FIND_ALGORITHM: ${USE_FIND_ALGORITHM}" - - debug "VALIDATE_JSCPD_ALL_CODEBASE: ${VALIDATE_JSCPD_ALL_CODEBASE}" - - # Solve issue with git unsafe dirs - debug "Running git config for safe dirs" - git config --global --add safe.directory "${GITHUB_WORKSPACE}" 2>&1 - git config --global --add safe.directory "/tmp/lint" 2>&1 - if [ "${VALIDATE_ALL_CODEBASE}" == "false" ] && [ "${TEST_CASE_RUN}" != "true" ]; then - # Need to build a list of all files changed - # This can be pulled from the GITHUB_EVENT_PATH payload - - ################ - # print header # - ################ debug "----------------------------------------------" - debug "Pulling in code history and branches..." + debug "Build the list of all changed files" - ################################################################################# - # Switch codebase back to the default branch to get a list of all files changed # - ################################################################################# - SWITCH_CMD=$( - git -C "${GITHUB_WORKSPACE}" pull --quiet - git -C "${GITHUB_WORKSPACE}" checkout "${DEFAULT_BRANCH}" 2>&1 - ) - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ ${ERROR_CODE} -ne 0 ] && [ "${LOCAL_UPDATES}" == "false" ]; then - # Error - info "Failed to switch to ${DEFAULT_BRANCH} branch to get files changed!" - fatal "[${SWITCH_CMD}]" - fi + DIFF_GIT_DEFAULT_BRANCH_CMD="git -C ${GITHUB_WORKSPACE} diff --diff-filter=d --name-only ${DEFAULT_BRANCH}...${GITHUB_SHA} | xargs -I % sh -c 'echo \"${GITHUB_WORKSPACE}/%\"' 2>&1" if [ "${GITHUB_EVENT_NAME}" == "push" ]; then ################ # push event # ################ DIFF_TREE_CMD="git -C ${GITHUB_WORKSPACE} diff-tree --no-commit-id --name-only -r ${GITHUB_SHA} | xargs -I % sh -c 'echo \"${GITHUB_WORKSPACE}/%\"' 2>&1" - GenerateFileDiff "$DIFF_TREE_CMD" + GenerateFileDiff "${DIFF_TREE_CMD}" ############################################################### # Need to see if the array is empty, if so, try the other way # ############################################################### if [ ${#RAW_FILE_ARRAY[@]} -eq 0 ]; then - # Empty array, going to try to pull from main branch differences - ################ - # print header # - ################ debug "----------------------------------------------" - debug "WARN: Generation of File array with diff-tree produced [0] items, trying with git diff..." - - DIFF_CMD="git -C ${GITHUB_WORKSPACE} diff --name-only ${DEFAULT_BRANCH}...${GITHUB_SHA} --diff-filter=d | xargs -I % sh -c 'echo \"${GITHUB_WORKSPACE}/%\"' 2>&1" - GenerateFileDiff "$DIFF_CMD" + debug "Generating the file array with diff-tree produced [0] items, trying with git diff against the default branch..." + GenerateFileDiff "${DIFF_GIT_DEFAULT_BRANCH_CMD}" fi else ################ # PR event # ################ - DIFF_CMD="git -C ${GITHUB_WORKSPACE} diff --name-only ${DEFAULT_BRANCH}...${GITHUB_SHA} --diff-filter=d | xargs -I % sh -c 'echo \"${GITHUB_WORKSPACE}/%\"' 2>&1" - GenerateFileDiff "$DIFF_CMD" + GenerateFileDiff "${DIFF_GIT_DEFAULT_BRANCH_CMD}" fi else WORKSPACE_PATH="${GITHUB_WORKSPACE}" @@ -179,12 +130,6 @@ function BuildFileList() { -type f \ 2>&1 | sort) else - ############################## - # use the standard mechinism # - ############################## - ################ - # print header # - ################ debug "----------------------------------------------" debug "Populating the file list with:[git -C \"${WORKSPACE_PATH}\" ls-tree -r --name-only HEAD | xargs -I % sh -c \"echo ${WORKSPACE_PATH}/%\"]" mapfile -t RAW_FILE_ARRAY < <(git -C "${WORKSPACE_PATH}" ls-tree -r --name-only HEAD | xargs -I % sh -c "echo ${WORKSPACE_PATH}/%" 2>&1) @@ -214,28 +159,6 @@ function BuildFileList() { warn "No files were found in the GITHUB_WORKSPACE:[${GITHUB_WORKSPACE}] to lint!" fi - if [ "${VALIDATE_ALL_CODEBASE}" == "false" ]; then - ######################################### - # Need to switch back to branch of code # - ######################################### - SWITCH2_CMD=$(git -C "${GITHUB_WORKSPACE}" checkout --progress --force "${GITHUB_SHA}" 2>&1) - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ ${ERROR_CODE} -ne 0 ]; then - # Error - error "Failed to switch back to branch!" - IssueHintForFullGitHistory - fatal "[${SWITCH2_CMD}]" - fi - fi - ######################################### # Check if the Ansible directory exists # ######################################### diff --git a/lib/functions/validation.sh b/lib/functions/validation.sh index 24113f91..764efd0e 100755 --- a/lib/functions/validation.sh +++ b/lib/functions/validation.sh @@ -1,14 +1,5 @@ #!/usr/bin/env bash -################################################################################ -################################################################################ -########### Super-Linter Validation Functions @admiralawkbar ################### -################################################################################ -################################################################################ -########################## FUNCTION CALLS BELOW ################################ -################################################################################ -################################################################################ -#### Function GetValidationInfo ################################################ function GetValidationInfo() { ############################################ # Print headers for user provided env vars # @@ -32,7 +23,6 @@ function GetValidationInfo() { VALIDATE_ALL_CODEBASE="${DEFAULT_VALIDATE_ALL_CODEBASE}" info "- Validating ALL files in code base..." else - # Its false info "- Only validating [new], or [edited] files in code base..." fi fi @@ -220,3 +210,52 @@ function GetValidationInfo() { debug "${PRINTENV}" debug "---------------------------------------------" } + +function CheckIfGitBranchExists() { + local BRANCH_NAME="${1}" + debug "Check if the ${BRANCH_NAME} branch exists in ${GITHUB_WORKSPACE}" + if ! git -C "${GITHUB_WORKSPACE}" rev-parse --quiet --verify "${BRANCH_NAME}"; then + info "The ${BRANCH_NAME} branch doesn't exist in ${GITHUB_WORKSPACE}" + return 1 + else + debug "The ${BRANCH_NAME} branch exists in ${GITHUB_WORKSPACE}" + return 0 + fi +} + +function ValidateLocalGitRepository() { + debug "Check if ${GITHUB_WORKSPACE} is a Git repository" + if ! git -C "${GITHUB_WORKSPACE}" rev-parse --git-dir; then + fatal "${GITHUB_WORKSPACE} is not a Git repository." + else + debug "${GITHUB_WORKSPACE} is a Git repository" + fi + + debug "Git branches: $(git -C "${GITHUB_WORKSPACE}" branch -a)" +} + +function ValidateGitShaReference() { + debug "Validate that the GITHUB_SHA reference (${GITHUB_SHA}) exists in this Git repository." + if ! git -C "${GITHUB_WORKSPACE}" cat-file -e "${GITHUB_SHA}"; then + fatal "The GITHUB_SHA reference (${GITHUB_SHA}) doesn't exist in this Git repository" + else + debug "The GITHUB_SHA reference (${GITHUB_SHA}) exists in this repository" + fi +} + +function ValidateDefaultGitBranch() { + debug "Check if the default branch (${DEFAULT_BRANCH}) exists" + if ! CheckIfGitBranchExists "${DEFAULT_BRANCH}"; then + REMOTE_DEFAULT_BRANCH="origin/${DEFAULT_BRANCH}" + debug "The default branch (${DEFAULT_BRANCH}) doesn't exist in this Git repository. Trying with ${REMOTE_DEFAULT_BRANCH}" + if ! CheckIfGitBranchExists "${REMOTE_DEFAULT_BRANCH}"; then + fatal "Neither ${DEFAULT_BRANCH}, nor ${REMOTE_DEFAULT_BRANCH} exist in ${GITHUB_WORKSPACE}" + else + info "${DEFAULT_BRANCH} doesn't exist, however ${REMOTE_DEFAULT_BRANCH} exists. Setting DEFAULT_BRANCH to: ${REMOTE_DEFAULT_BRANCH}" + DEFAULT_BRANCH="${REMOTE_DEFAULT_BRANCH}" + debug "Updated DEFAULT_BRANCH: ${DEFAULT_BRANCH}" + fi + else + debug "The default branch (${DEFAULT_BRANCH}) exists in this repository" + fi +} diff --git a/lib/linter.sh b/lib/linter.sh index 85b4b9f4..394fe6f0 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -10,7 +10,6 @@ # Debug Vars # # Define these early, so we can use debug logging ASAP if needed # ################################################################## -# RUN_LOCAL="${RUN_LOCAL}" # Boolean to see if we are running locally ACTIONS_RUNNER_DEBUG="${ACTIONS_RUNNER_DEBUG:-false}" # Boolean to see even more info (debug) IMAGE="${IMAGE:-standard}" # Version of the Super-linter (standard,slim,etc) @@ -150,8 +149,6 @@ LATEX_FILE_NAME=".chktexrc" # shellcheck disable=SC2034 # Variable is referenced indirectly LUA_FILE_NAME=".luacheckrc" # shellcheck disable=SC2034 # Variable is referenced indirectly -LOCAL_UPDATES="${LOCAL_UPDATES:-false}" -# shellcheck disable=SC2034 # Variable is referenced indirectly MARKDOWN_FILE_NAME="${MARKDOWN_CONFIG_FILE:-.markdown-lint.yml}" # shellcheck disable=SC2034 # Variable is referenced indirectly OPENAPI_FILE_NAME=".openapirc.yml" @@ -216,6 +213,8 @@ TYPESCRIPT_STYLE='' # Variable for the style TYPESCRIPT_ES_FILE_NAME="${TYPESCRIPT_ES_CONFIG_FILE:-.eslintrc.yml}" # shellcheck disable=SC2034 # Variable is referenced indirectly USE_FIND_ALGORITHM="${USE_FIND_ALGORITHM:-false}" +debug "USE_FIND_ALGORITHM: ${USE_FIND_ALGORITHM}" + # shellcheck disable=SC2034 # Variable is referenced indirectly YAML_FILE_NAME="${YAML_CONFIG_FILE:-.yaml-lint.yml}" # shellcheck disable=SC2034 # Variable is referenced indirectly @@ -361,24 +360,14 @@ LINTED_LANGUAGES_ARRAY=() # Will be filled at run time with all languages that w ################### # GitHub ENV Vars # ################### -# ANSIBLE_DIRECTORY="${ANSIBLE_DIRECTORY}" # Ansible Directory MULTI_STATUS="${MULTI_STATUS:-true}" # Multiple status are created for each check ran DEFAULT_BRANCH="${DEFAULT_BRANCH:-master}" # Default Git Branch to use (master by default) -# DISABLE_ERRORS="${DISABLE_ERRORS}" # Boolean to enable warning-only output without throwing errors -# FILTER_REGEX_INCLUDE="${FILTER_REGEX_INCLUDE}" # RegExp defining which files will be processed by linters (all by default) -# FILTER_REGEX_EXCLUDE="${FILTER_REGEX_EXCLUDE}" # RegExp defining which files will be excluded from linting (none by default) -# GITHUB_EVENT_PATH="${GITHUB_EVENT_PATH}" # Github Event Path -# GITHUB_REPOSITORY="${GITHUB_REPOSITORY}" # GitHub Org/Repo passed from system -# GITHUB_RUN_ID="${GITHUB_RUN_ID}" # GitHub RUn ID to point to logs -# GITHUB_SHA="${GITHUB_SHA}" # GitHub sha from the commit -# GITHUB_WORKSPACE="${GITHUB_WORKSPACE}" # Github Workspace -# TEST_CASE_RUN="${TEST_CASE_RUN}" # Boolean to validate only test cases -# VALIDATE_ALL_CODEBASE="${VALIDATE_ALL_CODEBASE}" # Boolean to validate all files - IGNORE_GITIGNORED_FILES="${IGNORE_GITIGNORED_FILES:-false}" +debug "IGNORE_GITIGNORED_FILES: ${IGNORE_GITIGNORED_FILES}" # Do not ignore generated files by default for backwards compatibility IGNORE_GENERATED_FILES="${IGNORE_GENERATED_FILES:-false}" +debug "IGNORE_GENERATED_FILES: ${IGNORE_GENERATED_FILES}" ################ # Default Vars # @@ -388,6 +377,22 @@ DEFAULT_WORKSPACE="${DEFAULT_WORKSPACE:-/tmp/lint}" # Default workspace if runni DEFAULT_RUN_LOCAL='false' # Default value for debugging locally DEFAULT_TEST_CASE_RUN='false' # Flag to tell code to run only test cases +if [ -z "${TEST_CASE_RUN}" ]; then + TEST_CASE_RUN="${DEFAULT_TEST_CASE_RUN}" +fi + +# Convert string to lowercase +TEST_CASE_RUN="${TEST_CASE_RUN,,}" +debug "TEST_CASE_RUN: ${TEST_CASE_RUN}" + +if [ -z "${RUN_LOCAL}" ]; then + RUN_LOCAL="${DEFAULT_RUN_LOCAL}" +fi + +# Convert string to lowercase +RUN_LOCAL="${RUN_LOCAL,,}" +debug "RUN_LOCAL: ${RUN_LOCAL}" + ############################################################### # Default Vars that are called in Subs and need to be ignored # ############################################################### @@ -445,49 +450,9 @@ GetGitHubVars() { info "--------------------------------------------" info "Gathering GitHub information..." - ############################### - # Get the Run test cases flag # - ############################### - if [ -z "${TEST_CASE_RUN}" ]; then - ################################## - # No flag passed, set to default # - ################################## - TEST_CASE_RUN="${DEFAULT_TEST_CASE_RUN}" - fi - - ############################### - # Convert string to lowercase # - ############################### - TEST_CASE_RUN="${TEST_CASE_RUN,,}" - - ########################## - # Get the run local flag # - ########################## - if [ -z "${RUN_LOCAL}" ]; then - ################################## - # No flag passed, set to default # - ################################## - RUN_LOCAL="${DEFAULT_RUN_LOCAL}" - fi - - ############################### - # Convert string to lowercase # - ############################### - RUN_LOCAL="${RUN_LOCAL,,}" - - ################################# - # Check if were running locally # - ################################# if [[ ${RUN_LOCAL} != "false" ]]; then - ########################################## - # We are running locally for a debug run # - ########################################## - info "NOTE: ENV VAR [RUN_LOCAL] has been set to:[true]" - info "bypassing GitHub Actions variables..." + info "RUN_LOCAL has been set to:[${RUN_LOCAL}]. Bypassing GitHub Actions variables..." - ############################ - # Set the GITHUB_WORKSPACE # - ############################ if [ -z "${GITHUB_WORKSPACE}" ]; then GITHUB_WORKSPACE="${DEFAULT_WORKSPACE}" fi @@ -500,29 +465,8 @@ GetGitHubVars() { pushd "${GITHUB_WORKSPACE}" >/dev/null || exit 1 - # No need to touch or set the GITHUB_SHA - # No need to touch or set the GITHUB_EVENT_PATH - # No need to touch or set the GITHUB_ORG - # No need to touch or set the GITHUB_REPO - - ################################# - # Set the VALIDATE_ALL_CODEBASE # - ################################# VALIDATE_ALL_CODEBASE="${DEFAULT_VALIDATE_ALL_CODEBASE}" else - ############################ - # Validate we have a value # - ############################ - if [ -z "${GITHUB_SHA}" ]; then - error "Failed to get [GITHUB_SHA]!" - fatal "[${GITHUB_SHA}]" - else - info "Successfully found:${F[W]}[GITHUB_SHA]${F[B]}, value:${F[W]}[${GITHUB_SHA}]" - fi - - ############################ - # Validate we have a value # - ############################ if [ -z "${GITHUB_WORKSPACE}" ]; then error "Failed to get [GITHUB_WORKSPACE]!" fatal "[${GITHUB_WORKSPACE}]" @@ -530,9 +474,6 @@ GetGitHubVars() { info "Successfully found:${F[W]}[GITHUB_WORKSPACE]${F[B]}, value:${F[W]}[${GITHUB_WORKSPACE}]" fi - ############################ - # Validate we have a value # - ############################ if [ -z "${GITHUB_EVENT_PATH}" ]; then error "Failed to get [GITHUB_EVENT_PATH]!" fatal "[${GITHUB_EVENT_PATH}]" @@ -540,6 +481,12 @@ GetGitHubVars() { info "Successfully found:${F[W]}[GITHUB_EVENT_PATH]${F[B]}, value:${F[W]}[${GITHUB_EVENT_PATH}]${F[B]}" fi + if [ -z "${GITHUB_SHA}" ]; then + fatal "Failed to get the value for the GITHUB_SHA variable [${GITHUB_SHA}]" + else + info "Successfully found:${F[W]}[GITHUB_SHA]${F[B]}, value:${F[W]}[${GITHUB_SHA}]" + fi + ################################################## # Need to pull the GitHub Vars from the env file # ################################################## @@ -555,7 +502,16 @@ GetGitHubVars() { # Github sha on PR events is not the latest commit. # https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then + debug "This is a GitHub pull request. Updating the current GITHUB_SHA (${GITHUB_SHA}) to the pull request HEAD sha" GITHUB_SHA=$(jq -r .pull_request.head.sha <"$GITHUB_EVENT_PATH") + ERROR_CODE=$? + debug "GITHUB_SHA update error code: ${ERROR_CODE}" + + if [ ${ERROR_CODE} -ne 0 ]; then + error "Failed to update GITHUB_SHA for pull request event." + fatal "[Output: ${GITHUB_SHA}]" + fi + debug "Updated GITHUB_SHA: ${GITHUB_SHA}" fi ############################ @@ -665,7 +621,7 @@ CallStatusAPI() { STATUS="success" fi - debug "URL: ${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/statuses/${GITHUB_SHA}" + debug "Status URL: ${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/statuses/${GITHUB_SHA}" ############################################## # Call the status API to create status check # @@ -858,6 +814,10 @@ UpdateLoopsForImage ################################## GetLinterVersions +debug "Allow Git to work on ${GITHUB_WORKSPACE}" +git config --global --add safe.directory "${GITHUB_WORKSPACE}" 2>&1 +git config --global --add safe.directory "/tmp/lint" 2>&1 + ####################### # Get GitHub Env Vars # ####################### @@ -879,6 +839,15 @@ debug "TYPESCRIPT_STANDARD_TSCONFIG_FILE: ${TYPESCRIPT_STANDARD_TSCONFIG_FILE}" ############################ # Validate the environment # ############################ +if [[ "${USE_FIND_ALGORITHM}" != "false" ]] || [[ "${VALIDATE_ALL_CODEBASE}" != "true" ]]; then + debug "Validate the local Git environment" + ValidateLocalGitRepository + ValidateGitShaReference + ValidateDefaultGitBranch +else + debug "Skipped the validation of the local Git environment because we don't depend on it." +fi + GetValidationInfo ################################# @@ -1025,6 +994,7 @@ CheckSSLCert # Check if we need to lint the whole codebase with JSCPD VALIDATE_JSCPD_ALL_CODEBASE="${VALIDATE_JSCPD_ALL_CODEBASE:-"false"}" export VALIDATE_JSCPD_ALL_CODEBASE +debug "VALIDATE_JSCPD_ALL_CODEBASE: ${VALIDATE_JSCPD_ALL_CODEBASE}" ########################################### # Build the list of files for each linter #