From 30763f2575d4a0396a913da1735860ca092d0b6a Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Mon, 31 Aug 2020 23:44:35 +0200 Subject: [PATCH] Streamline shell file list building --- lib/buildFileList.sh | 168 ++++++++++++++++++++++++++++--------------- lib/linter.sh | 24 ++++++- lib/worker.sh | 45 +++++++----- 3 files changed, 160 insertions(+), 77 deletions(-) diff --git a/lib/buildFileList.sh b/lib/buildFileList.sh index bbb31e8c..ff03f197 100755 --- a/lib/buildFileList.sh +++ b/lib/buildFileList.sh @@ -73,13 +73,8 @@ function BuildFileList() { info "------ Files modified in the commit(s): ------" info "----------------------------------------------" for FILE in "${RAW_FILE_ARRAY[@]}"; do - ########################### - # Get the files extension # - ########################### # Extract just the file extension - FILE_TYPE=${FILE##*.} - # To lowercase - FILE_TYPE=${FILE_TYPE,,} + FILE_TYPE="$(GetFileExtension "$FILE")" # get the baseFile for additonal logic BASE_FILE=$(basename "${FILE,,}") @@ -94,23 +89,17 @@ function BuildFileList() { debug "FILE_TYPE:[${FILE_TYPE}]" ###################### - # Get the BASH files # + # Get the shell files # ###################### - if [ "${FILE_TYPE}" == "sh" ] || [ "${FILE_TYPE}" == "bash" ] || - [ "${FILE_TYPE}" == "dash" ] || [ "${FILE_TYPE}" == "ksh" ]; then - # Need to check if its a zsh file as we cannot parse it - if CheckZsh "${FILE}"; then - warn "ShellCheck and shfmt do NOT currently support zsh, skipping file" - else - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_BASH+=("${FILE}") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - fi + if IsValidShellScript "${FILE}"; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_BASH+=("${FILE}") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 ######################### # Get the CLOJURE files # @@ -628,8 +617,8 @@ function BuildFileList() { info "Successfully gathered list of files..." } ################################################################################ -#### Function CheckFileType #################################################### -function CheckFileType() { +#### Function GetFileType ###################################################### +function GetFileType() { # Need to run the file through the 'file' exec to help determine # The type of file being parsed @@ -643,15 +632,28 @@ function CheckFileType() { ################## GET_FILE_TYPE_CMD=$(file "${FILE}" 2>&1) + echo "${GET_FILE_TYPE_CMD}" +} +################################################################################ +#### Function CheckFileType #################################################### +function CheckFileType() { + # Need to run the file through the 'file' exec to help determine + # The type of file being parsed + + ################ + # Pull in Vars # + ################ + FILE="$1" + + ################# + # Get file type # + ################# + GET_FILE_TYPE_CMD="$(GetFileType "$FILE")" + ################# # Check if bash # ################# - if [[ ${GET_FILE_TYPE_CMD} == *"Bourne-Again shell script"* ]]; then - ####################### - # It is a bash script # - ####################### - warn "Found bash script without extension:[.sh]" - info "Please update file with proper extensions." + if IsValidShellScript "$FILE"; then ################################ # Append the file to the array # ################################ @@ -674,12 +676,6 @@ function CheckFileType() { # Set the READ_ONLY_CHANGE_FLAG since this could be exec # ########################################################## READ_ONLY_CHANGE_FLAG=1 - elif [[ ${GET_FILE_TYPE_CMD} == *"zsh script"* ]]; then - ###################### - # It is a ZSH script # - ###################### - warn "Found [zsh] script: ${FILE}" - info "ShellCheck does NOT currently support zsh, skipping file" else ############################ # Extension was not found! # @@ -692,34 +688,90 @@ function CheckFileType() { fi } ################################################################################ -#### Function CheckZsh ######################################################### -function CheckZsh() { - # Spagetti code to make sure were properly excluding zsh - # until we get a proper linter - +#### Function GetFileExtension ############################################### +function GetFileExtension() { ################ # Pull in Vars # ################ FILE="$1" - ################## - # Check the file # - ################## - GET_FILE_TYPE_CMD=$(file "${FILE}" 2>&1) + ########################### + # Get the files extension # + ########################### + # Extract just the file extension + FILE_TYPE=${FILE##*.} + # To lowercase + FILE_TYPE=${FILE_TYPE,,} - if [[ ${GET_FILE_TYPE_CMD} == *"zsh script"* ]]; then - ###################### - # It is a ZSH script # - ###################### - debug "Found [zsh] script: ${FILE}" - ################################################### - # We found zsh file and need to return with a hit # - ################################################### - return 0 - else - ################## - # Not a zsh file # - ################## + echo "$FILE_TYPE" +} +################################################################################ +#### Function IsValidShellScript ############################################### +function IsValidShellScript() { + ################ + # Pull in Vars # + ################ + FILE="$1" + + ################# + # Get file type # + ################# + FILE_EXTENSION="$(GetFileExtension "$FILE")" + GET_FILE_TYPE_CMD="$(GetFileType "$FILE")" + + trace "File:[${FILE}], File extension:[${FILE_EXTENSION}], File type: [${GET_FILE_TYPE_CMD}]" + + if [[ "${FILE_EXTENSION}" == "zsh" ]] || + [[ ${GET_FILE_TYPE_CMD} == *"zsh script"* ]]; then + warn "$FILE is a ZSH script. Skipping..." return 1 fi + + if [ "${FILE_EXTENSION}" == "sh" ] || + [ "${FILE_EXTENSION}" == "bash" ] || + [ "${FILE_EXTENSION}" == "dash" ] || + [ "${FILE_EXTENSION}" == "ksh" ]; then + debug "$FILE is a valid shell script (has a valid extension: ${FILE_EXTENSION})" + return 0 + fi + + if [[ "${GET_FILE_TYPE_CMD}" == *"POSIX shell script"* ]] || + [[ ${GET_FILE_TYPE_CMD} == *"Bourne-Again shell script"* ]] || + [[ ${GET_FILE_TYPE_CMD} == *"dash script"* ]] || + [[ ${GET_FILE_TYPE_CMD} == *"ksh script"* ]] || + [[ ${GET_FILE_TYPE_CMD} == *"/usr/bin/env sh script"* ]]; then + debug "$FILE is a valid shell script (has a valid file type: ${GET_FILE_TYPE_CMD})" + return 0 + fi + + trace "$FILE is NOT a supported shell script. Skipping" + return 1 +} +################################################################################ +#### Function IsValidShellScript ############################################### +function PopulateShellScriptsList() { + debug "Populating shell script file list. Source: ${GITHUB_WORKSPACE}" + + ############################################################################### + # Set the file seperator to newline to allow for grabbing objects with spaces # + ############################################################################### + IFS=$'\n' + + mapfile -t LIST_FILES < <(find "${GITHUB_WORKSPACE}" -path "*/node_modules" -prune -o -path "*/\.git" -prune -o -type f 2>&1) + for FILE in "${LIST_FILES[@]}"; do + if IsValidShellScript "${FILE}"; then + debug "Adding ${FILE} to shell script files list" + FILE_ARRAY_BASH+=("${FILE}") + + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + fi + done + + ########################### + # Set IFS back to default # + ########################### + IFS="${DEFAULT_IFS}" } diff --git a/lib/linter.sh b/lib/linter.sh index c2e54bd7..077e35b2 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -1366,7 +1366,7 @@ if [ "${VALIDATE_BASH}" == "true" ]; then # Lint the bash files # ####################### # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY" - LintCodebase "BASH" "shellcheck" "shellcheck --color --external-sources" ".*\.\(sh\|bash\|dash\|ksh\)\$" "${FILE_ARRAY_BASH[@]}" + LintCodebase "BASH" "shellcheck" "shellcheck --color --external-sources" "disabledfileext" "${FILE_ARRAY_BASH[@]}" fi ##################### @@ -1377,7 +1377,7 @@ if [ "${VALIDATE_BASH_EXEC}" == "true" ]; then # Lint the bash files # ####################### # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY" - LintCodebase "BASH_EXEC" "bash-exec" "bash-exec" ".*\.\(sh\|bash\|dash\|ksh\)\$" "${FILE_ARRAY_BASH[@]}" + LintCodebase "BASH_EXEC" "bash-exec" "bash-exec" "disabledfileext" "${FILE_ARRAY_BASH[@]}" fi ########################## @@ -1836,6 +1836,26 @@ if [ "${VALIDATE_RUBY}" == "true" ]; then LintCodebase "RUBY" "rubocop" "rubocop -c ${RUBY_LINTER_RULES} --force-exclusion" ".*\.\(rb\)\$" "${FILE_ARRAY_RUBY[@]}" fi +################# +# SHFMT LINTING # +################# +if [ "${VALIDATE_SHELL_SHFMT}" == "true" ]; then + #################################### + # Lint the files with shfmt # + #################################### + EDITORCONFIG_FILE_PATH="${GITHUB_WORKSPACE}"/.editorconfig + if [ -e "$EDITORCONFIG_FILE_PATH" ]; then + # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY" + LintCodebase "SHELL_SHFMT" "shfmt" "shfmt -d" "disabledfileext" "${FILE_ARRAY_BASH[@]}" + else + ############################### + # No .editorconfig file found # + ############################### + warn "No .editorconfig found at:[$EDITORCONFIG_FILE_PATH]" + debug "skipping shfmt" + fi +fi + ###################### # AWS STATES LINTING # ###################### diff --git a/lib/worker.sh b/lib/worker.sh index 8cfed3ca..35247b75 100755 --- a/lib/worker.sh +++ b/lib/worker.sh @@ -81,20 +81,27 @@ function LintCodebase() { # We have files added to array of files to check LIST_FILES=("${FILE_ARRAY[@]}") # Copy the array into list else - ############################################################################### - # Set the file separator to newline to allow for grabbing objects with spaces # - ############################################################################### - IFS=$'\n' + if [[ ${FILE_TYPE} == "BASH" ]] || + [[ ${FILE_TYPE} == "BASH_EXEC" ]] || + [[ ${FILE_TYPE} == "SHELL_SHFMT" ]]; then + # Populate a list of valid shell scripts. + PopulateShellScriptsList + else + ############################################################################### + # Set the file separator to newline to allow for grabbing objects with spaces # + ############################################################################### + IFS=$'\n' - ################################# - # Get list of all files to lint # - ################################# - mapfile -t LIST_FILES < <(find "${GITHUB_WORKSPACE}" -path "*/node_modules" -prune -o -type f -regex "${FILE_EXTENSIONS}" 2>&1) + ################################# + # Get list of all files to lint # + ################################# + mapfile -t LIST_FILES < <(find "${GITHUB_WORKSPACE}" -path "*/node_modules" -prune -o -type f -regex "${FILE_EXTENSIONS}" 2>&1) - ########################### - # Set IFS back to default # - ########################### - IFS="${DEFAULT_IFS}" + ########################### + # Set IFS back to default # + ########################### + IFS="${DEFAULT_IFS}" + fi ############################################################ # Set it back to empty if loaded with blanks from scanning # @@ -163,11 +170,15 @@ function LintCodebase() { elif [[ ${FILE} == *".rbenv"* ]]; then # This is likely the ruby environment folder and shouldn't be parsed continue - elif [[ ${FILE_TYPE} == "BASH" ]] || [[ ${FILE_TYPE} == "SHELL_SHFMT" ]]; then - if CheckZsh "${FILE}"; then - # ZSH file and we need to skip - warn "$LINTER_NAME does NOT currently support zsh, skipping file" - continue + elif [[ ${FILE_TYPE} == "BASH" ]] && ! IsValidShellScript "${FILE}"; then + # not a valid script and we need to skip + continue + elif [[ ${FILE_TYPE} == "BASH_EXEC" ]] && ! IsValidShellScript "${FILE}"; then + # not a valid script and we need to skip + continue + elif [[ ${FILE_TYPE} == "SHELL_SHFMT" ]] && ! IsValidShellScript "${FILE}"; then + # not a valid script and we need to skip + continue fi ##################################