Streamline shell file list building

This commit is contained in:
Marco Ferrari 2020-08-31 23:44:35 +02:00
parent 7ecf9a132c
commit 30763f2575
3 changed files with 160 additions and 77 deletions

View file

@ -73,13 +73,8 @@ function BuildFileList() {
info "------ Files modified in the commit(s): ------" info "------ Files modified in the commit(s): ------"
info "----------------------------------------------" info "----------------------------------------------"
for FILE in "${RAW_FILE_ARRAY[@]}"; do for FILE in "${RAW_FILE_ARRAY[@]}"; do
###########################
# Get the files extension #
###########################
# Extract just the file extension # Extract just the file extension
FILE_TYPE=${FILE##*.} FILE_TYPE="$(GetFileExtension "$FILE")"
# To lowercase
FILE_TYPE=${FILE_TYPE,,}
# get the baseFile for additonal logic # get the baseFile for additonal logic
BASE_FILE=$(basename "${FILE,,}") BASE_FILE=$(basename "${FILE,,}")
@ -94,14 +89,9 @@ function BuildFileList() {
debug "FILE_TYPE:[${FILE_TYPE}]" debug "FILE_TYPE:[${FILE_TYPE}]"
###################### ######################
# Get the BASH files # # Get the shell files #
###################### ######################
if [ "${FILE_TYPE}" == "sh" ] || [ "${FILE_TYPE}" == "bash" ] || if IsValidShellScript "${FILE}"; then
[ "${FILE_TYPE}" == "dash" ] || [ "${FILE_TYPE}" == "ksh" ]; then
# Need to check if its a zsh file as we cannot parse it
if CheckZsh "${FILE}"; then
warn "ShellCheck and shfmt do NOT currently support zsh, skipping file"
else
################################ ################################
# Append the file to the array # # Append the file to the array #
################################ ################################
@ -110,7 +100,6 @@ function BuildFileList() {
# Set the READ_ONLY_CHANGE_FLAG since this could be exec # # Set the READ_ONLY_CHANGE_FLAG since this could be exec #
########################################################## ##########################################################
READ_ONLY_CHANGE_FLAG=1 READ_ONLY_CHANGE_FLAG=1
fi
######################### #########################
# Get the CLOJURE files # # Get the CLOJURE files #
@ -628,8 +617,8 @@ function BuildFileList() {
info "Successfully gathered list of files..." info "Successfully gathered list of files..."
} }
################################################################################ ################################################################################
#### Function CheckFileType #################################################### #### Function GetFileType ######################################################
function CheckFileType() { function GetFileType() {
# Need to run the file through the 'file' exec to help determine # Need to run the file through the 'file' exec to help determine
# The type of file being parsed # The type of file being parsed
@ -643,15 +632,28 @@ function CheckFileType() {
################## ##################
GET_FILE_TYPE_CMD=$(file "${FILE}" 2>&1) GET_FILE_TYPE_CMD=$(file "${FILE}" 2>&1)
echo "${GET_FILE_TYPE_CMD}"
}
################################################################################
#### Function CheckFileType ####################################################
function CheckFileType() {
# Need to run the file through the 'file' exec to help determine
# The type of file being parsed
################
# Pull in Vars #
################
FILE="$1"
#################
# Get file type #
#################
GET_FILE_TYPE_CMD="$(GetFileType "$FILE")"
################# #################
# Check if bash # # Check if bash #
################# #################
if [[ ${GET_FILE_TYPE_CMD} == *"Bourne-Again shell script"* ]]; then if IsValidShellScript "$FILE"; then
#######################
# It is a bash script #
#######################
warn "Found bash script without extension:[.sh]"
info "Please update file with proper extensions."
################################ ################################
# Append the file to the array # # Append the file to the array #
################################ ################################
@ -674,12 +676,6 @@ function CheckFileType() {
# Set the READ_ONLY_CHANGE_FLAG since this could be exec # # Set the READ_ONLY_CHANGE_FLAG since this could be exec #
########################################################## ##########################################################
READ_ONLY_CHANGE_FLAG=1 READ_ONLY_CHANGE_FLAG=1
elif [[ ${GET_FILE_TYPE_CMD} == *"zsh script"* ]]; then
######################
# It is a ZSH script #
######################
warn "Found [zsh] script: ${FILE}"
info "ShellCheck does NOT currently support zsh, skipping file"
else else
############################ ############################
# Extension was not found! # # Extension was not found! #
@ -692,34 +688,90 @@ function CheckFileType() {
fi fi
} }
################################################################################ ################################################################################
#### Function CheckZsh ######################################################### #### Function GetFileExtension ###############################################
function CheckZsh() { function GetFileExtension() {
# Spagetti code to make sure were properly excluding zsh
# until we get a proper linter
################ ################
# Pull in Vars # # Pull in Vars #
################ ################
FILE="$1" FILE="$1"
################## ###########################
# Check the file # # Get the files extension #
################## ###########################
GET_FILE_TYPE_CMD=$(file "${FILE}" 2>&1) # Extract just the file extension
FILE_TYPE=${FILE##*.}
# To lowercase
FILE_TYPE=${FILE_TYPE,,}
if [[ ${GET_FILE_TYPE_CMD} == *"zsh script"* ]]; then echo "$FILE_TYPE"
###################### }
# It is a ZSH script # ################################################################################
###################### #### Function IsValidShellScript ###############################################
debug "Found [zsh] script: ${FILE}" function IsValidShellScript() {
################################################### ################
# We found zsh file and need to return with a hit # # Pull in Vars #
################################################### ################
return 0 FILE="$1"
else
################## #################
# Not a zsh file # # Get file type #
################## #################
FILE_EXTENSION="$(GetFileExtension "$FILE")"
GET_FILE_TYPE_CMD="$(GetFileType "$FILE")"
trace "File:[${FILE}], File extension:[${FILE_EXTENSION}], File type: [${GET_FILE_TYPE_CMD}]"
if [[ "${FILE_EXTENSION}" == "zsh" ]] ||
[[ ${GET_FILE_TYPE_CMD} == *"zsh script"* ]]; then
warn "$FILE is a ZSH script. Skipping..."
return 1 return 1
fi fi
if [ "${FILE_EXTENSION}" == "sh" ] ||
[ "${FILE_EXTENSION}" == "bash" ] ||
[ "${FILE_EXTENSION}" == "dash" ] ||
[ "${FILE_EXTENSION}" == "ksh" ]; then
debug "$FILE is a valid shell script (has a valid extension: ${FILE_EXTENSION})"
return 0
fi
if [[ "${GET_FILE_TYPE_CMD}" == *"POSIX shell script"* ]] ||
[[ ${GET_FILE_TYPE_CMD} == *"Bourne-Again shell script"* ]] ||
[[ ${GET_FILE_TYPE_CMD} == *"dash script"* ]] ||
[[ ${GET_FILE_TYPE_CMD} == *"ksh script"* ]] ||
[[ ${GET_FILE_TYPE_CMD} == *"/usr/bin/env sh script"* ]]; then
debug "$FILE is a valid shell script (has a valid file type: ${GET_FILE_TYPE_CMD})"
return 0
fi
trace "$FILE is NOT a supported shell script. Skipping"
return 1
}
################################################################################
#### Function IsValidShellScript ###############################################
function PopulateShellScriptsList() {
debug "Populating shell script file list. Source: ${GITHUB_WORKSPACE}"
###############################################################################
# Set the file seperator to newline to allow for grabbing objects with spaces #
###############################################################################
IFS=$'\n'
mapfile -t LIST_FILES < <(find "${GITHUB_WORKSPACE}" -path "*/node_modules" -prune -o -path "*/\.git" -prune -o -type f 2>&1)
for FILE in "${LIST_FILES[@]}"; do
if IsValidShellScript "${FILE}"; then
debug "Adding ${FILE} to shell script files list"
FILE_ARRAY_BASH+=("${FILE}")
##########################################################
# Set the READ_ONLY_CHANGE_FLAG since this could be exec #
##########################################################
READ_ONLY_CHANGE_FLAG=1
fi
done
###########################
# Set IFS back to default #
###########################
IFS="${DEFAULT_IFS}"
} }

View file

@ -1366,7 +1366,7 @@ if [ "${VALIDATE_BASH}" == "true" ]; then
# Lint the bash files # # Lint the bash files #
####################### #######################
# LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY" # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY"
LintCodebase "BASH" "shellcheck" "shellcheck --color --external-sources" ".*\.\(sh\|bash\|dash\|ksh\)\$" "${FILE_ARRAY_BASH[@]}" LintCodebase "BASH" "shellcheck" "shellcheck --color --external-sources" "disabledfileext" "${FILE_ARRAY_BASH[@]}"
fi fi
##################### #####################
@ -1377,7 +1377,7 @@ if [ "${VALIDATE_BASH_EXEC}" == "true" ]; then
# Lint the bash files # # Lint the bash files #
####################### #######################
# LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY" # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY"
LintCodebase "BASH_EXEC" "bash-exec" "bash-exec" ".*\.\(sh\|bash\|dash\|ksh\)\$" "${FILE_ARRAY_BASH[@]}" LintCodebase "BASH_EXEC" "bash-exec" "bash-exec" "disabledfileext" "${FILE_ARRAY_BASH[@]}"
fi fi
########################## ##########################
@ -1836,6 +1836,26 @@ if [ "${VALIDATE_RUBY}" == "true" ]; then
LintCodebase "RUBY" "rubocop" "rubocop -c ${RUBY_LINTER_RULES} --force-exclusion" ".*\.\(rb\)\$" "${FILE_ARRAY_RUBY[@]}" LintCodebase "RUBY" "rubocop" "rubocop -c ${RUBY_LINTER_RULES} --force-exclusion" ".*\.\(rb\)\$" "${FILE_ARRAY_RUBY[@]}"
fi fi
#################
# SHFMT LINTING #
#################
if [ "${VALIDATE_SHELL_SHFMT}" == "true" ]; then
####################################
# Lint the files with shfmt #
####################################
EDITORCONFIG_FILE_PATH="${GITHUB_WORKSPACE}"/.editorconfig
if [ -e "$EDITORCONFIG_FILE_PATH" ]; then
# LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY"
LintCodebase "SHELL_SHFMT" "shfmt" "shfmt -d" "disabledfileext" "${FILE_ARRAY_BASH[@]}"
else
###############################
# No .editorconfig file found #
###############################
warn "No .editorconfig found at:[$EDITORCONFIG_FILE_PATH]"
debug "skipping shfmt"
fi
fi
###################### ######################
# AWS STATES LINTING # # AWS STATES LINTING #
###################### ######################

View file

@ -80,6 +80,12 @@ function LintCodebase() {
elif [ ${#FILE_ARRAY[@]} -ne 0 ]; then elif [ ${#FILE_ARRAY[@]} -ne 0 ]; then
# We have files added to array of files to check # We have files added to array of files to check
LIST_FILES=("${FILE_ARRAY[@]}") # Copy the array into list LIST_FILES=("${FILE_ARRAY[@]}") # Copy the array into list
else
if [[ ${FILE_TYPE} == "BASH" ]] ||
[[ ${FILE_TYPE} == "BASH_EXEC" ]] ||
[[ ${FILE_TYPE} == "SHELL_SHFMT" ]]; then
# Populate a list of valid shell scripts.
PopulateShellScriptsList
else else
############################################################################### ###############################################################################
# Set the file separator to newline to allow for grabbing objects with spaces # # Set the file separator to newline to allow for grabbing objects with spaces #
@ -95,6 +101,7 @@ function LintCodebase() {
# Set IFS back to default # # Set IFS back to default #
########################### ###########################
IFS="${DEFAULT_IFS}" IFS="${DEFAULT_IFS}"
fi
############################################################ ############################################################
# Set it back to empty if loaded with blanks from scanning # # Set it back to empty if loaded with blanks from scanning #
@ -163,10 +170,14 @@ function LintCodebase() {
elif [[ ${FILE} == *".rbenv"* ]]; then elif [[ ${FILE} == *".rbenv"* ]]; then
# This is likely the ruby environment folder and shouldn't be parsed # This is likely the ruby environment folder and shouldn't be parsed
continue continue
elif [[ ${FILE_TYPE} == "BASH" ]] || [[ ${FILE_TYPE} == "SHELL_SHFMT" ]]; then elif [[ ${FILE_TYPE} == "BASH" ]] && ! IsValidShellScript "${FILE}"; then
if CheckZsh "${FILE}"; then # not a valid script and we need to skip
# ZSH file and we need to skip continue
warn "$LINTER_NAME does NOT currently support zsh, skipping file" elif [[ ${FILE_TYPE} == "BASH_EXEC" ]] && ! IsValidShellScript "${FILE}"; then
# not a valid script and we need to skip
continue
elif [[ ${FILE_TYPE} == "SHELL_SHFMT" ]] && ! IsValidShellScript "${FILE}"; then
# not a valid script and we need to skip
continue continue
fi fi