Exit on errors when running Git (#4889)

* Exit on errors when running Git

* Skip pulling changes entirely

* Enable pipefail when generating diffs

* Cleanup

* Shallow repo check

* Echo GITHUB_SHA update

* Check if GITHUB_SHA exists before using it

* Move GITHUB_SHA validation to validation script

* Rely on cat-file return code

* Check if DEFAULT_BRANCH exists

* Change dir when checking DEFAULT_BRANCH

* Show git branches

* Don't switch branches

* Check GITHUB_SHA only when needed

* Ensure we have permissions before interacting with the repo

* Remove the DIFF_CMD variable

* Move TEST_CASE_RUN and RUN_LOCAL init up

* Validate if Git repo and if SHA exists

* Move validation function

* Change dir when getting branch names

* Move debug messages up to be less verbose

* Move branch validation in a function

* Fix linting errors
This commit is contained in:
Marco Ferrari 2023-12-04 10:47:49 +01:00 committed by GitHub
parent a8150b40c8
commit 5a8805dc4f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 112 additions and 202 deletions

View file

@ -7,9 +7,6 @@
###########
# Globals #
###########
# GITHUB_WORKSPACE="${GITHUB_WORKSPACE}" # GitHub Workspace
# GITHUB_SHA="${GITHUB_SHA}" # Sha used to create this branch
# BUILD_DATE="${BUILD_DATE}" # Date the container was built
IMAGE="${1}" # Image of the super-linter we build
BUILD_REVISION="${GITHUB_SHA}" # GitHub Sha
BUILD_VERSION="${GITHUB_SHA}" # Version of the container

View file

@ -73,7 +73,6 @@ jobs:
VALIDATE_ALL_CODEBASE: false
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DEFAULT_BRANCH: main
LOCAL_UPDATES: true
RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES: "default.json,hoge.json"
TYPESCRIPT_STANDARD_TSCONFIG_FILE: ".github/linters/tsconfig.json"

View file

@ -109,7 +109,7 @@ jobs:
uses: super-linter/super-linter@v5
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DEFAULT_BRANCH: develop
DEFAULT_BRANCH: main
```
## Build the container image and run the test suite locally

View file

@ -1,18 +0,0 @@
# Super-Linter Library
## Main script
The file `linter.sh` is the main script that is called for the process and loads all other scripts as functions.
## Functions
The additional files in the folder are functions to help streamline the main build process and allow for easier maintenance.
- `possum.sh`
- Official mascot of the **Super-Linter**
- `buildFileList.sh`
- Functions to help find files that were modified, or need to be scanned
- `validation.sh`
- Logic to see what linters are enabled
- `worker.sh`
- Calls to the various linters and the test cases for each

View file

@ -12,6 +12,7 @@
#### Function BuildFileList ####################################################
function IssueHintForFullGitHistory() {
info "Check that you have the full git history, the checkout is not shallow, etc"
info "Is shallow repository: $(git -C "${GITHUB_WORKSPACE}" rev-parse --is-shallow-repository)"
info "See https://github.com/super-linter/super-linter#example-connecting-github-action-workflow"
}
@ -19,30 +20,26 @@ function IssueHintForFullGitHistory() {
#### Function GenerateFileDiff #################################################
function GenerateFileDiff() {
CMD="${1}"
################
# print header #
################
debug "----------------------------------------------"
debug "Generating Diff with:[$CMD]"
#################################################
# Get the Array of files changed in the commits #
#################################################
CMD_OUTPUT=$(eval "$CMD")
CMD_OUTPUT=$(eval "set -eo pipefail; $CMD; set +eo pipefail")
#######################
# Load the error code #
#######################
ERROR_CODE=$?
debug "Diff command return code: ${ERROR_CODE}"
##############################
# Check the shell for errors #
##############################
if [ ${ERROR_CODE} -ne 0 ]; then
# Error
info "Failed to get Diff with:[$CMD]"
error "Failed to get Diff with:[$CMD]"
IssueHintForFullGitHistory
fatal "[${CMD_OUTPUT}]"
fatal "[Diff command output: ${CMD_OUTPUT}]"
fi
###################################################
@ -69,78 +66,32 @@ function BuildFileList() {
ANSIBLE_DIRECTORY="${3}"
debug "ANSIBLE_DIRECTORY: ${ANSIBLE_DIRECTORY}"
debug "IGNORE_GITIGNORED_FILES: ${IGNORE_GITIGNORED_FILES}"
debug "IGNORE_GENERATED_FILES: ${IGNORE_GENERATED_FILES}"
debug "USE_FIND_ALGORITHM: ${USE_FIND_ALGORITHM}"
debug "VALIDATE_JSCPD_ALL_CODEBASE: ${VALIDATE_JSCPD_ALL_CODEBASE}"
# Solve issue with git unsafe dirs
debug "Running git config for safe dirs"
git config --global --add safe.directory "${GITHUB_WORKSPACE}" 2>&1
git config --global --add safe.directory "/tmp/lint" 2>&1
if [ "${VALIDATE_ALL_CODEBASE}" == "false" ] && [ "${TEST_CASE_RUN}" != "true" ]; then
# Need to build a list of all files changed
# This can be pulled from the GITHUB_EVENT_PATH payload
################
# print header #
################
debug "----------------------------------------------"
debug "Pulling in code history and branches..."
debug "Build the list of all changed files"
#################################################################################
# Switch codebase back to the default branch to get a list of all files changed #
#################################################################################
SWITCH_CMD=$(
git -C "${GITHUB_WORKSPACE}" pull --quiet
git -C "${GITHUB_WORKSPACE}" checkout "${DEFAULT_BRANCH}" 2>&1
)
#######################
# Load the error code #
#######################
ERROR_CODE=$?
##############################
# Check the shell for errors #
##############################
if [ ${ERROR_CODE} -ne 0 ] && [ "${LOCAL_UPDATES}" == "false" ]; then
# Error
info "Failed to switch to ${DEFAULT_BRANCH} branch to get files changed!"
fatal "[${SWITCH_CMD}]"
fi
DIFF_GIT_DEFAULT_BRANCH_CMD="git -C ${GITHUB_WORKSPACE} diff --diff-filter=d --name-only ${DEFAULT_BRANCH}...${GITHUB_SHA} | xargs -I % sh -c 'echo \"${GITHUB_WORKSPACE}/%\"' 2>&1"
if [ "${GITHUB_EVENT_NAME}" == "push" ]; then
################
# push event #
################
DIFF_TREE_CMD="git -C ${GITHUB_WORKSPACE} diff-tree --no-commit-id --name-only -r ${GITHUB_SHA} | xargs -I % sh -c 'echo \"${GITHUB_WORKSPACE}/%\"' 2>&1"
GenerateFileDiff "$DIFF_TREE_CMD"
GenerateFileDiff "${DIFF_TREE_CMD}"
###############################################################
# Need to see if the array is empty, if so, try the other way #
###############################################################
if [ ${#RAW_FILE_ARRAY[@]} -eq 0 ]; then
# Empty array, going to try to pull from main branch differences
################
# print header #
################
debug "----------------------------------------------"
debug "WARN: Generation of File array with diff-tree produced [0] items, trying with git diff..."
DIFF_CMD="git -C ${GITHUB_WORKSPACE} diff --name-only ${DEFAULT_BRANCH}...${GITHUB_SHA} --diff-filter=d | xargs -I % sh -c 'echo \"${GITHUB_WORKSPACE}/%\"' 2>&1"
GenerateFileDiff "$DIFF_CMD"
debug "Generating the file array with diff-tree produced [0] items, trying with git diff against the default branch..."
GenerateFileDiff "${DIFF_GIT_DEFAULT_BRANCH_CMD}"
fi
else
################
# PR event #
################
DIFF_CMD="git -C ${GITHUB_WORKSPACE} diff --name-only ${DEFAULT_BRANCH}...${GITHUB_SHA} --diff-filter=d | xargs -I % sh -c 'echo \"${GITHUB_WORKSPACE}/%\"' 2>&1"
GenerateFileDiff "$DIFF_CMD"
GenerateFileDiff "${DIFF_GIT_DEFAULT_BRANCH_CMD}"
fi
else
WORKSPACE_PATH="${GITHUB_WORKSPACE}"
@ -179,12 +130,6 @@ function BuildFileList() {
-type f \
2>&1 | sort)
else
##############################
# use the standard mechinism #
##############################
################
# print header #
################
debug "----------------------------------------------"
debug "Populating the file list with:[git -C \"${WORKSPACE_PATH}\" ls-tree -r --name-only HEAD | xargs -I % sh -c \"echo ${WORKSPACE_PATH}/%\"]"
mapfile -t RAW_FILE_ARRAY < <(git -C "${WORKSPACE_PATH}" ls-tree -r --name-only HEAD | xargs -I % sh -c "echo ${WORKSPACE_PATH}/%" 2>&1)
@ -214,28 +159,6 @@ function BuildFileList() {
warn "No files were found in the GITHUB_WORKSPACE:[${GITHUB_WORKSPACE}] to lint!"
fi
if [ "${VALIDATE_ALL_CODEBASE}" == "false" ]; then
#########################################
# Need to switch back to branch of code #
#########################################
SWITCH2_CMD=$(git -C "${GITHUB_WORKSPACE}" checkout --progress --force "${GITHUB_SHA}" 2>&1)
#######################
# Load the error code #
#######################
ERROR_CODE=$?
##############################
# Check the shell for errors #
##############################
if [ ${ERROR_CODE} -ne 0 ]; then
# Error
error "Failed to switch back to branch!"
IssueHintForFullGitHistory
fatal "[${SWITCH2_CMD}]"
fi
fi
#########################################
# Check if the Ansible directory exists #
#########################################

View file

@ -1,14 +1,5 @@
#!/usr/bin/env bash
################################################################################
################################################################################
########### Super-Linter Validation Functions @admiralawkbar ###################
################################################################################
################################################################################
########################## FUNCTION CALLS BELOW ################################
################################################################################
################################################################################
#### Function GetValidationInfo ################################################
function GetValidationInfo() {
############################################
# Print headers for user provided env vars #
@ -32,7 +23,6 @@ function GetValidationInfo() {
VALIDATE_ALL_CODEBASE="${DEFAULT_VALIDATE_ALL_CODEBASE}"
info "- Validating ALL files in code base..."
else
# Its false
info "- Only validating [new], or [edited] files in code base..."
fi
fi
@ -220,3 +210,52 @@ function GetValidationInfo() {
debug "${PRINTENV}"
debug "---------------------------------------------"
}
function CheckIfGitBranchExists() {
local BRANCH_NAME="${1}"
debug "Check if the ${BRANCH_NAME} branch exists in ${GITHUB_WORKSPACE}"
if ! git -C "${GITHUB_WORKSPACE}" rev-parse --quiet --verify "${BRANCH_NAME}"; then
info "The ${BRANCH_NAME} branch doesn't exist in ${GITHUB_WORKSPACE}"
return 1
else
debug "The ${BRANCH_NAME} branch exists in ${GITHUB_WORKSPACE}"
return 0
fi
}
function ValidateLocalGitRepository() {
debug "Check if ${GITHUB_WORKSPACE} is a Git repository"
if ! git -C "${GITHUB_WORKSPACE}" rev-parse --git-dir; then
fatal "${GITHUB_WORKSPACE} is not a Git repository."
else
debug "${GITHUB_WORKSPACE} is a Git repository"
fi
debug "Git branches: $(git -C "${GITHUB_WORKSPACE}" branch -a)"
}
function ValidateGitShaReference() {
debug "Validate that the GITHUB_SHA reference (${GITHUB_SHA}) exists in this Git repository."
if ! git -C "${GITHUB_WORKSPACE}" cat-file -e "${GITHUB_SHA}"; then
fatal "The GITHUB_SHA reference (${GITHUB_SHA}) doesn't exist in this Git repository"
else
debug "The GITHUB_SHA reference (${GITHUB_SHA}) exists in this repository"
fi
}
function ValidateDefaultGitBranch() {
debug "Check if the default branch (${DEFAULT_BRANCH}) exists"
if ! CheckIfGitBranchExists "${DEFAULT_BRANCH}"; then
REMOTE_DEFAULT_BRANCH="origin/${DEFAULT_BRANCH}"
debug "The default branch (${DEFAULT_BRANCH}) doesn't exist in this Git repository. Trying with ${REMOTE_DEFAULT_BRANCH}"
if ! CheckIfGitBranchExists "${REMOTE_DEFAULT_BRANCH}"; then
fatal "Neither ${DEFAULT_BRANCH}, nor ${REMOTE_DEFAULT_BRANCH} exist in ${GITHUB_WORKSPACE}"
else
info "${DEFAULT_BRANCH} doesn't exist, however ${REMOTE_DEFAULT_BRANCH} exists. Setting DEFAULT_BRANCH to: ${REMOTE_DEFAULT_BRANCH}"
DEFAULT_BRANCH="${REMOTE_DEFAULT_BRANCH}"
debug "Updated DEFAULT_BRANCH: ${DEFAULT_BRANCH}"
fi
else
debug "The default branch (${DEFAULT_BRANCH}) exists in this repository"
fi
}

View file

@ -10,7 +10,6 @@
# Debug Vars #
# Define these early, so we can use debug logging ASAP if needed #
##################################################################
# RUN_LOCAL="${RUN_LOCAL}" # Boolean to see if we are running locally
ACTIONS_RUNNER_DEBUG="${ACTIONS_RUNNER_DEBUG:-false}" # Boolean to see even more info (debug)
IMAGE="${IMAGE:-standard}" # Version of the Super-linter (standard,slim,etc)
@ -150,8 +149,6 @@ LATEX_FILE_NAME=".chktexrc"
# shellcheck disable=SC2034 # Variable is referenced indirectly
LUA_FILE_NAME=".luacheckrc"
# shellcheck disable=SC2034 # Variable is referenced indirectly
LOCAL_UPDATES="${LOCAL_UPDATES:-false}"
# shellcheck disable=SC2034 # Variable is referenced indirectly
MARKDOWN_FILE_NAME="${MARKDOWN_CONFIG_FILE:-.markdown-lint.yml}"
# shellcheck disable=SC2034 # Variable is referenced indirectly
OPENAPI_FILE_NAME=".openapirc.yml"
@ -216,6 +213,8 @@ TYPESCRIPT_STYLE='' # Variable for the style
TYPESCRIPT_ES_FILE_NAME="${TYPESCRIPT_ES_CONFIG_FILE:-.eslintrc.yml}"
# shellcheck disable=SC2034 # Variable is referenced indirectly
USE_FIND_ALGORITHM="${USE_FIND_ALGORITHM:-false}"
debug "USE_FIND_ALGORITHM: ${USE_FIND_ALGORITHM}"
# shellcheck disable=SC2034 # Variable is referenced indirectly
YAML_FILE_NAME="${YAML_CONFIG_FILE:-.yaml-lint.yml}"
# shellcheck disable=SC2034 # Variable is referenced indirectly
@ -361,24 +360,14 @@ LINTED_LANGUAGES_ARRAY=() # Will be filled at run time with all languages that w
###################
# GitHub ENV Vars #
###################
# ANSIBLE_DIRECTORY="${ANSIBLE_DIRECTORY}" # Ansible Directory
MULTI_STATUS="${MULTI_STATUS:-true}" # Multiple status are created for each check ran
DEFAULT_BRANCH="${DEFAULT_BRANCH:-master}" # Default Git Branch to use (master by default)
# DISABLE_ERRORS="${DISABLE_ERRORS}" # Boolean to enable warning-only output without throwing errors
# FILTER_REGEX_INCLUDE="${FILTER_REGEX_INCLUDE}" # RegExp defining which files will be processed by linters (all by default)
# FILTER_REGEX_EXCLUDE="${FILTER_REGEX_EXCLUDE}" # RegExp defining which files will be excluded from linting (none by default)
# GITHUB_EVENT_PATH="${GITHUB_EVENT_PATH}" # Github Event Path
# GITHUB_REPOSITORY="${GITHUB_REPOSITORY}" # GitHub Org/Repo passed from system
# GITHUB_RUN_ID="${GITHUB_RUN_ID}" # GitHub RUn ID to point to logs
# GITHUB_SHA="${GITHUB_SHA}" # GitHub sha from the commit
# GITHUB_WORKSPACE="${GITHUB_WORKSPACE}" # Github Workspace
# TEST_CASE_RUN="${TEST_CASE_RUN}" # Boolean to validate only test cases
# VALIDATE_ALL_CODEBASE="${VALIDATE_ALL_CODEBASE}" # Boolean to validate all files
IGNORE_GITIGNORED_FILES="${IGNORE_GITIGNORED_FILES:-false}"
debug "IGNORE_GITIGNORED_FILES: ${IGNORE_GITIGNORED_FILES}"
# Do not ignore generated files by default for backwards compatibility
IGNORE_GENERATED_FILES="${IGNORE_GENERATED_FILES:-false}"
debug "IGNORE_GENERATED_FILES: ${IGNORE_GENERATED_FILES}"
################
# Default Vars #
@ -388,6 +377,22 @@ DEFAULT_WORKSPACE="${DEFAULT_WORKSPACE:-/tmp/lint}" # Default workspace if runni
DEFAULT_RUN_LOCAL='false' # Default value for debugging locally
DEFAULT_TEST_CASE_RUN='false' # Flag to tell code to run only test cases
if [ -z "${TEST_CASE_RUN}" ]; then
TEST_CASE_RUN="${DEFAULT_TEST_CASE_RUN}"
fi
# Convert string to lowercase
TEST_CASE_RUN="${TEST_CASE_RUN,,}"
debug "TEST_CASE_RUN: ${TEST_CASE_RUN}"
if [ -z "${RUN_LOCAL}" ]; then
RUN_LOCAL="${DEFAULT_RUN_LOCAL}"
fi
# Convert string to lowercase
RUN_LOCAL="${RUN_LOCAL,,}"
debug "RUN_LOCAL: ${RUN_LOCAL}"
###############################################################
# Default Vars that are called in Subs and need to be ignored #
###############################################################
@ -445,49 +450,9 @@ GetGitHubVars() {
info "--------------------------------------------"
info "Gathering GitHub information..."
###############################
# Get the Run test cases flag #
###############################
if [ -z "${TEST_CASE_RUN}" ]; then
##################################
# No flag passed, set to default #
##################################
TEST_CASE_RUN="${DEFAULT_TEST_CASE_RUN}"
fi
###############################
# Convert string to lowercase #
###############################
TEST_CASE_RUN="${TEST_CASE_RUN,,}"
##########################
# Get the run local flag #
##########################
if [ -z "${RUN_LOCAL}" ]; then
##################################
# No flag passed, set to default #
##################################
RUN_LOCAL="${DEFAULT_RUN_LOCAL}"
fi
###############################
# Convert string to lowercase #
###############################
RUN_LOCAL="${RUN_LOCAL,,}"
#################################
# Check if were running locally #
#################################
if [[ ${RUN_LOCAL} != "false" ]]; then
##########################################
# We are running locally for a debug run #
##########################################
info "NOTE: ENV VAR [RUN_LOCAL] has been set to:[true]"
info "bypassing GitHub Actions variables..."
info "RUN_LOCAL has been set to:[${RUN_LOCAL}]. Bypassing GitHub Actions variables..."
############################
# Set the GITHUB_WORKSPACE #
############################
if [ -z "${GITHUB_WORKSPACE}" ]; then
GITHUB_WORKSPACE="${DEFAULT_WORKSPACE}"
fi
@ -500,29 +465,8 @@ GetGitHubVars() {
pushd "${GITHUB_WORKSPACE}" >/dev/null || exit 1
# No need to touch or set the GITHUB_SHA
# No need to touch or set the GITHUB_EVENT_PATH
# No need to touch or set the GITHUB_ORG
# No need to touch or set the GITHUB_REPO
#################################
# Set the VALIDATE_ALL_CODEBASE #
#################################
VALIDATE_ALL_CODEBASE="${DEFAULT_VALIDATE_ALL_CODEBASE}"
else
############################
# Validate we have a value #
############################
if [ -z "${GITHUB_SHA}" ]; then
error "Failed to get [GITHUB_SHA]!"
fatal "[${GITHUB_SHA}]"
else
info "Successfully found:${F[W]}[GITHUB_SHA]${F[B]}, value:${F[W]}[${GITHUB_SHA}]"
fi
############################
# Validate we have a value #
############################
if [ -z "${GITHUB_WORKSPACE}" ]; then
error "Failed to get [GITHUB_WORKSPACE]!"
fatal "[${GITHUB_WORKSPACE}]"
@ -530,9 +474,6 @@ GetGitHubVars() {
info "Successfully found:${F[W]}[GITHUB_WORKSPACE]${F[B]}, value:${F[W]}[${GITHUB_WORKSPACE}]"
fi
############################
# Validate we have a value #
############################
if [ -z "${GITHUB_EVENT_PATH}" ]; then
error "Failed to get [GITHUB_EVENT_PATH]!"
fatal "[${GITHUB_EVENT_PATH}]"
@ -540,6 +481,12 @@ GetGitHubVars() {
info "Successfully found:${F[W]}[GITHUB_EVENT_PATH]${F[B]}, value:${F[W]}[${GITHUB_EVENT_PATH}]${F[B]}"
fi
if [ -z "${GITHUB_SHA}" ]; then
fatal "Failed to get the value for the GITHUB_SHA variable [${GITHUB_SHA}]"
else
info "Successfully found:${F[W]}[GITHUB_SHA]${F[B]}, value:${F[W]}[${GITHUB_SHA}]"
fi
##################################################
# Need to pull the GitHub Vars from the env file #
##################################################
@ -555,7 +502,16 @@ GetGitHubVars() {
# Github sha on PR events is not the latest commit.
# https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request
if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then
debug "This is a GitHub pull request. Updating the current GITHUB_SHA (${GITHUB_SHA}) to the pull request HEAD sha"
GITHUB_SHA=$(jq -r .pull_request.head.sha <"$GITHUB_EVENT_PATH")
ERROR_CODE=$?
debug "GITHUB_SHA update error code: ${ERROR_CODE}"
if [ ${ERROR_CODE} -ne 0 ]; then
error "Failed to update GITHUB_SHA for pull request event."
fatal "[Output: ${GITHUB_SHA}]"
fi
debug "Updated GITHUB_SHA: ${GITHUB_SHA}"
fi
############################
@ -665,7 +621,7 @@ CallStatusAPI() {
STATUS="success"
fi
debug "URL: ${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/statuses/${GITHUB_SHA}"
debug "Status URL: ${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/statuses/${GITHUB_SHA}"
##############################################
# Call the status API to create status check #
@ -858,6 +814,10 @@ UpdateLoopsForImage
##################################
GetLinterVersions
debug "Allow Git to work on ${GITHUB_WORKSPACE}"
git config --global --add safe.directory "${GITHUB_WORKSPACE}" 2>&1
git config --global --add safe.directory "/tmp/lint" 2>&1
#######################
# Get GitHub Env Vars #
#######################
@ -879,6 +839,15 @@ debug "TYPESCRIPT_STANDARD_TSCONFIG_FILE: ${TYPESCRIPT_STANDARD_TSCONFIG_FILE}"
############################
# Validate the environment #
############################
if [[ "${USE_FIND_ALGORITHM}" != "false" ]] || [[ "${VALIDATE_ALL_CODEBASE}" != "true" ]]; then
debug "Validate the local Git environment"
ValidateLocalGitRepository
ValidateGitShaReference
ValidateDefaultGitBranch
else
debug "Skipped the validation of the local Git environment because we don't depend on it."
fi
GetValidationInfo
#################################
@ -1025,6 +994,7 @@ CheckSSLCert
# Check if we need to lint the whole codebase with JSCPD
VALIDATE_JSCPD_ALL_CODEBASE="${VALIDATE_JSCPD_ALL_CODEBASE:-"false"}"
export VALIDATE_JSCPD_ALL_CODEBASE
debug "VALIDATE_JSCPD_ALL_CODEBASE: ${VALIDATE_JSCPD_ALL_CODEBASE}"
###########################################
# Build the list of files for each linter #