Merge pull request #649 from ferrarimarco/streamline-file-list

Streamline shell scripts file list building and bootstrap npm-groovy-lint

Dockerfile

@@ -293,6 +293,11 @@ ENV PATH="$PATH":"$GOROOT"/bin:"$GOPATH"/bin
 RUN mkdir -p ${GOPATH}/src ${GOPATH}/bin
 RUN go get mvdan.cc/sh/v3/cmd/shfmt
 
+#############################
+# Bootstrap npm-groovy-lint #
+#############################
+RUN npm-groovy-lint --version
+
 ###########################################
 # Load GitHub Env Vars for GitHub Actions #
 ###########################################

README.md

@@ -73,7 +73,7 @@ Developers on **GitHub** can call the **GitHub Action** to lint their code base
 | **R**                            | [lintr](https://github.com/jimhester/lintr)                                                                                                                                    |
 | **Raku**                         | [Raku](https://raku.org)                                                                                                                                                       |
 | **Ruby**                         | [RuboCop](https://github.com/rubocop-hq/rubocop)                                                                                                                               |
-| **Shell**                        | [Shellcheck](https://github.com/koalaman/shellcheck) / [Bash-exec] / [shfmt](https://github.com/mvdan/sh)                                                                      |
+| **Shell**                        | [Shellcheck](https://github.com/koalaman/shellcheck) / [executable bit check] / [shfmt](https://github.com/mvdan/sh)                                                           |
 | **SQL**                          | [sql-lint](https://github.com/joereynolds/sql-lint)                                                                                                                            |
 | **Terraform**                    | [tflint](https://github.com/terraform-linters/tflint) / [terrascan](https://github.com/accurics/terrascan)                                                                     |
 | **TypeScript**                   | [eslint](https://eslint.org/) / [standard js](https://standardjs.com/)                                                                                                         |

lib/buildFileList.sh

@@ -73,13 +73,8 @@ function BuildFileList() {
   info "------ Files modified in the commit(s): ------"
   info "----------------------------------------------"
   for FILE in "${RAW_FILE_ARRAY[@]}"; do
-    ###########################
-    # Get the files extension #
-    ###########################
     # Extract just the file extension
-    FILE_TYPE=${FILE##*.}
+    FILE_TYPE="$(GetFileExtension "$FILE")"
-    # To lowercase
-    FILE_TYPE=${FILE_TYPE,,}
     # get the baseFile for additonal logic
     BASE_FILE=$(basename "${FILE,,}")
 
@@ -94,23 +89,17 @@ function BuildFileList() {
     debug "FILE_TYPE:[${FILE_TYPE}]"
 
     ######################
-    # Get the BASH files #
+    # Get the shell files #
     ######################
-    if [ "${FILE_TYPE}" == "sh" ] || [ "${FILE_TYPE}" == "bash" ] ||
+    if IsValidShellScript "${FILE}"; then
-      [ "${FILE_TYPE}" == "dash" ] || [ "${FILE_TYPE}" == "ksh" ]; then
+      ################################
-      # Need to check if its a zsh file as we cannot parse it
+      # Append the file to the array #
-      if CheckZsh "${FILE}"; then
+      ################################
-        warn "ShellCheck and shfmt do NOT currently support zsh, skipping file"
+      FILE_ARRAY_BASH+=("${FILE}")
-      else
+      ##########################################################
-        ################################
+      # Set the READ_ONLY_CHANGE_FLAG since this could be exec #
-        # Append the file to the array #
+      ##########################################################
-        ################################
+      READ_ONLY_CHANGE_FLAG=1
-        FILE_ARRAY_BASH+=("${FILE}")
-        ##########################################################
-        # Set the READ_ONLY_CHANGE_FLAG since this could be exec #
-        ##########################################################
-        READ_ONLY_CHANGE_FLAG=1
-      fi
 
     #########################
     # Get the CLOJURE files #
@@ -628,8 +617,8 @@ function BuildFileList() {
   info "Successfully gathered list of files..."
 }
 ################################################################################
-#### Function CheckFileType ####################################################
+#### Function GetFileType ######################################################
-function CheckFileType() {
+function GetFileType() {
   # Need to run the file through the 'file' exec to help determine
   # The type of file being parsed
 
@@ -643,15 +632,28 @@ function CheckFileType() {
   ##################
   GET_FILE_TYPE_CMD=$(file "${FILE}" 2>&1)
 
+  echo "${GET_FILE_TYPE_CMD}"
+}
+################################################################################
+#### Function CheckFileType ####################################################
+function CheckFileType() {
+  # Need to run the file through the 'file' exec to help determine
+  # The type of file being parsed
+
+  ################
+  # Pull in Vars #
+  ################
+  FILE="$1"
+
+  #################
+  # Get file type #
+  #################
+  GET_FILE_TYPE_CMD="$(GetFileType "$FILE")"
+
   #################
   # Check if bash #
   #################
-  if [[ ${GET_FILE_TYPE_CMD} == *"Bourne-Again shell script"* ]]; then
+  if IsValidShellScript "$FILE"; then
-    #######################
-    # It is a bash script #
-    #######################
-    warn "Found bash script without extension:[.sh]"
-    info "Please update file with proper extensions."
     ################################
     # Append the file to the array #
     ################################
@@ -674,12 +676,6 @@ function CheckFileType() {
     # Set the READ_ONLY_CHANGE_FLAG since this could be exec #
     ##########################################################
     READ_ONLY_CHANGE_FLAG=1
-  elif [[ ${GET_FILE_TYPE_CMD} == *"zsh script"* ]]; then
-    ######################
-    # It is a ZSH script #
-    ######################
-    warn "Found [zsh] script: ${FILE}"
-    info "ShellCheck does NOT currently support zsh, skipping file"
   else
     ############################
     # Extension was not found! #
@@ -692,34 +688,90 @@ function CheckFileType() {
   fi
 }
 ################################################################################
-#### Function CheckZsh #########################################################
+#### Function GetFileExtension ###############################################
-function CheckZsh() {
+function GetFileExtension() {
-  # Spagetti code to make sure were properly excluding zsh
-  # until we get a proper linter
-
   ################
   # Pull in Vars #
   ################
   FILE="$1"
 
-  ##################
+  ###########################
-  # Check the file #
+  # Get the files extension #
-  ##################
+  ###########################
-  GET_FILE_TYPE_CMD=$(file "${FILE}" 2>&1)
+  # Extract just the file extension
+  FILE_TYPE=${FILE##*.}
+  # To lowercase
+  FILE_TYPE=${FILE_TYPE,,}
 
-  if [[ ${GET_FILE_TYPE_CMD} == *"zsh script"* ]]; then
+  echo "$FILE_TYPE"
-    ######################
+}
-    # It is a ZSH script #
+################################################################################
-    ######################
+#### Function IsValidShellScript ###############################################
-    debug "Found [zsh] script: ${FILE}"
+function IsValidShellScript() {
-    ###################################################
+  ################
-    # We found zsh file and need to return with a hit #
+  # Pull in Vars #
-    ###################################################
+  ################
-    return 0
+  FILE="$1"
-  else
+
-    ##################
+  #################
-    # Not a zsh file #
+  # Get file type #
-    ##################
+  #################
+  FILE_EXTENSION="$(GetFileExtension "$FILE")"
+  GET_FILE_TYPE_CMD="$(GetFileType "$FILE")"
+
+  trace "File:[${FILE}], File extension:[${FILE_EXTENSION}], File type: [${GET_FILE_TYPE_CMD}]"
+
+  if [[ "${FILE_EXTENSION}" == "zsh" ]] ||
+    [[ ${GET_FILE_TYPE_CMD} == *"zsh script"* ]]; then
+    warn "$FILE is a ZSH script. Skipping..."
     return 1
   fi
+
+  if [ "${FILE_EXTENSION}" == "sh" ] ||
+    [ "${FILE_EXTENSION}" == "bash" ] ||
+    [ "${FILE_EXTENSION}" == "dash" ] ||
+    [ "${FILE_EXTENSION}" == "ksh" ]; then
+      debug "$FILE is a valid shell script (has a valid extension: ${FILE_EXTENSION})"
+      return 0
+  fi
+
+  if [[ "${GET_FILE_TYPE_CMD}" == *"POSIX shell script"* ]] ||
+    [[ ${GET_FILE_TYPE_CMD} == *"Bourne-Again shell script"* ]] ||
+    [[ ${GET_FILE_TYPE_CMD} == *"dash script"* ]] ||
+    [[ ${GET_FILE_TYPE_CMD} == *"ksh script"* ]] ||
+    [[ ${GET_FILE_TYPE_CMD} == *"/usr/bin/env sh script"* ]]; then
+      debug "$FILE is a valid shell script (has a valid file type: ${GET_FILE_TYPE_CMD})"
+      return 0
+  fi
+
+  trace "$FILE is NOT a supported shell script. Skipping"
+  return 1
+}
+################################################################################
+#### Function IsValidShellScript ###############################################
+function PopulateShellScriptsList() {
+  debug "Populating shell script file list. Source: ${GITHUB_WORKSPACE}"
+
+  ###############################################################################
+  # Set the file seperator to newline to allow for grabbing objects with spaces #
+  ###############################################################################
+  IFS=$'\n'
+
+  mapfile -t LIST_FILES < <(find "${GITHUB_WORKSPACE}" -path "*/node_modules" -prune -o -path "*/\.git" -prune -o -type f 2>&1)
+  for FILE in "${LIST_FILES[@]}"; do
+    if IsValidShellScript "${FILE}"; then
+      debug "Adding ${FILE} to shell script files list"
+      FILE_ARRAY_BASH+=("${FILE}")
+
+      ##########################################################
+      # Set the READ_ONLY_CHANGE_FLAG since this could be exec #
+      ##########################################################
+      READ_ONLY_CHANGE_FLAG=1
+    fi
+  done
+
+  ###########################
+  # Set IFS back to default #
+  ###########################
+  IFS="${DEFAULT_IFS}"
 }

lib/linter.sh

@@ -136,7 +136,7 @@ YAML_LINTER_RULES="${DEFAULT_RULES_LOCATION}/${YAML_FILE_NAME}" # Path to the ya
 LINTER_ARRAY=('ansible-lint' 'arm-ttk' 'asl-validator' 'bash-exec' 'black' 'cfn-lint' 'checkstyle' 'chktex' 'clj-kondo' 'coffeelint'
   'dotnet-format' 'dart' 'dockerfilelint' 'dotenv-linter' 'editorconfig-checker' 'eslint' 'flake8' 'golangci-lint'
   'hadolint' 'htmlhint' 'jsonlint' 'ktlint' 'lintr' 'lua' 'markdownlint' 'npm-groovy-lint' 'perl' 'protolint'
-  'pwsh' 'pylint' 'raku' 'rubocop' 'shellcheck' 'spectral' 'standard' 'stylelint' 'sql-lint'
+  'pwsh' 'pylint' 'raku' 'rubocop' 'shellcheck' 'shfmt' 'spectral' 'standard' 'stylelint' 'sql-lint'
   'terrascan' 'tflint' 'xmllint' 'yamllint')
 
 #############################
@@ -146,7 +146,7 @@ LANGUAGE_ARRAY=('ANSIBLE' 'ARM' 'BASH' 'BASH_EXEC' 'CLOUDFORMATION' 'CLOJURE' 'C
   'DART' 'DOCKERFILE' 'DOCKERFILE_HADOLINT' 'EDITORCONFIG' 'ENV' 'GO' 'GROOVY' 'HTML'
   'JAVA' 'JAVASCRIPT_ES' 'JAVASCRIPT_STANDARD' 'JSON' 'JSX' 'KOTLIN' 'LATEX' 'LUA' 'MARKDOWN'
   'OPENAPI' 'PERL' 'PHP_BUILTIN' 'PHP_PHPCS' 'PHP_PHPSTAN' 'PHP_PSALM' 'POWERSHELL'
-  'PROTOBUF' 'PYTHON_BLACK' 'PYTHON_PYLINT' 'PYTHON_FLAKE8' 'R' 'RAKU' 'RUBY' 'STATES' 'SQL' 'TERRAFORM'
+  'PROTOBUF' 'PYTHON_BLACK' 'PYTHON_PYLINT' 'PYTHON_FLAKE8' 'R' 'RAKU' 'RUBY' 'SHELL_SHFMT' 'STATES' 'SQL' 'TERRAFORM'
   'TERRAFORM_TERRASCAN' 'TSX' 'TYPESCRIPT_ES' 'TYPESCRIPT_STANDARD' 'XML' 'YAML')
 
 ############################################
@@ -211,6 +211,7 @@ VALIDATE_R="${VALIDATE_R}"                                           # Boolean t
 VALIDATE_RAKU="${VALIDATE_RAKU}"                                     # Boolean to validate language
 VALIDATE_RUBY="${VALIDATE_RUBY}"                                     # Boolean to validate language
 VALIDATE_STATES="${VALIDATE_STATES}"                                 # Boolean to validate language
+VALIDATE_SHELL_SHFMT="${VALIDATE_SHELL_SHFMT}"                       # Boolean to check Shell files against editorconfig
 VALIDATE_SQL="${VALIDATE_SQL}"                                       # Boolean to validate language
 VALIDATE_TERRAFORM="${VALIDATE_TERRAFORM}"                           # Boolean to validate language
 VALIDATE_TERRAFORM_TERRASCAN="${VALIDATE_TERRAFORM_TERRASCAN}"       # Boolean to validate language
@@ -404,6 +405,8 @@ ERRORS_FOUND_RAKU=0                     # Count of errors found
 export ERRORS_FOUND_RAKU                # Workaround SC2034
 ERRORS_FOUND_RUBY=0                     # Count of errors found
 export ERRORS_FOUND_RUBY                # Workaround SC2034
+ERRORS_FOUND_SHELL_SHFMT=0              # Count of errors found
+export ERRORS_FOUND_SHELL_SHFMT
 ERRORS_FOUND_STATES=0                   # Count of errors found
 export ERRORS_FOUND_STATES              # Workaround SC2034
 ERRORS_FOUND_SQL=0                      # Count of errors found
@@ -464,7 +467,7 @@ GetLinterVersions() {
     if [[ ${LINTER} == "arm-ttk" ]]; then
       # Need specific command for ARM
       mapfile -t GET_VERSION_CMD < <(grep -iE 'version' "${ARM_TTK_PSD1}" | xargs 2>&1)
-    elif [[ ${LINTER} == "protolint" ]] || [[ ${LINTER} == "editorconfig-checker" ]]; then
+    elif [[ ${LINTER} == "protolint" ]] || [[ ${LINTER} == "editorconfig-checker" ]] || [[ ${LINTER} == "bash-exec" ]]; then
       # Need specific command for Protolint and editorconfig-checker
       mapfile -t GET_VERSION_CMD < <(echo "--version not supported")
     elif [[ ${LINTER} == "lintr" ]]; then
@@ -475,6 +478,8 @@ GetLinterVersions() {
     elif [[ ${LINTER} == "lua" ]]; then
       # Semi standardversion command
       mapfile -t GET_VERSION_CMD < <("${LINTER}" -v 2>&1)
+    elif [[ ${LINTER} == "terrascan" ]]; then
+      mapfile -t GET_VERSION_CMD < <("${LINTER}" version 2>&1)
     else
       # Standard version command
       mapfile -t GET_VERSION_CMD < <("${LINTER}" --version 2>&1)
@@ -1065,9 +1070,13 @@ Reports() {
     #############################################
     # Print info on reports that were generated #
     #############################################
-    info "Contents of report folder:"
+    if [ -d "${REPORT_OUTPUT_FOLDER}" ]; then
-    OUTPUT_CONTENTS_CMD=$(ls "${REPORT_OUTPUT_FOLDER}")
+      info "Contents of report folder:"
-    info "$OUTPUT_CONTENTS_CMD"
+      OUTPUT_CONTENTS_CMD=$(ls "${REPORT_OUTPUT_FOLDER}")
+      info "$OUTPUT_CONTENTS_CMD"
+    else
+      warn "Report output folder (${REPORT_OUTPUT_FOLDER}) does NOT exist."
+    fi
   fi
 
   ################################
@@ -1366,7 +1375,7 @@ if [ "${VALIDATE_BASH}" == "true" ]; then
   # Lint the bash files #
   #######################
   # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY"
-  LintCodebase "BASH" "shellcheck" "shellcheck --color --external-sources" ".*\.\(sh\|bash\|dash\|ksh\)\$" "${FILE_ARRAY_BASH[@]}"
+  LintCodebase "BASH" "shellcheck" "shellcheck --color --external-sources" "disabledfileext" "${FILE_ARRAY_BASH[@]}"
 fi
 
 #####################
@@ -1377,7 +1386,7 @@ if [ "${VALIDATE_BASH_EXEC}" == "true" ]; then
   # Lint the bash files #
   #######################
   # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY"
-  LintCodebase "BASH_EXEC" "bash-exec" "bash-exec" ".*\.\(sh\|bash\|dash\|ksh\)\$" "${FILE_ARRAY_BASH[@]}"
+  LintCodebase "BASH_EXEC" "bash-exec" "bash-exec" "disabledfileext" "${FILE_ARRAY_BASH[@]}"
 fi
 
 ##########################
@@ -1836,6 +1845,26 @@ if [ "${VALIDATE_RUBY}" == "true" ]; then
   LintCodebase "RUBY" "rubocop" "rubocop -c ${RUBY_LINTER_RULES} --force-exclusion" ".*\.\(rb\)\$" "${FILE_ARRAY_RUBY[@]}"
 fi
 
+#################
+# SHFMT LINTING #
+#################
+if [ "${VALIDATE_SHELL_SHFMT}" == "true" ]; then
+  ####################################
+  # Lint the files with shfmt #
+  ####################################
+  EDITORCONFIG_FILE_PATH="${GITHUB_WORKSPACE}"/.editorconfig
+  if [ -e "$EDITORCONFIG_FILE_PATH" ]; then
+    # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY"
+    LintCodebase "SHELL_SHFMT" "shfmt" "shfmt -d" ".*\.\(sh\|bash\|dash\|ksh\)\$" "${FILE_ARRAY_BASH[@]}"
+  else
+    ###############################
+    # No .editorconfig file found #
+    ###############################
+    warn "No .editorconfig found at:[$EDITORCONFIG_FILE_PATH]"
+    debug "skipping shfmt"
+  fi
+fi
+
 ######################
 # AWS STATES LINTING #
 ######################

lib/worker.sh

@@ -81,20 +81,27 @@ function LintCodebase() {
     # We have files added to array of files to check
     LIST_FILES=("${FILE_ARRAY[@]}") # Copy the array into list
   else
-    ###############################################################################
+    if [[ ${FILE_TYPE} == "BASH" ]] ||
-    # Set the file separator to newline to allow for grabbing objects with spaces #
+      [[ ${FILE_TYPE} == "BASH_EXEC" ]] ||
-    ###############################################################################
+      [[ ${FILE_TYPE} == "SHELL_SHFMT" ]]; then
-    IFS=$'\n'
+        # Populate a list of valid shell scripts.
+        PopulateShellScriptsList
+    else
+      ###############################################################################
+      # Set the file separator to newline to allow for grabbing objects with spaces #
+      ###############################################################################
+      IFS=$'\n'
 
-    #################################
+      #################################
-    # Get list of all files to lint #
+      # Get list of all files to lint #
-    #################################
+      #################################
-    mapfile -t LIST_FILES < <(find "${GITHUB_WORKSPACE}" -path "*/node_modules" -prune -o -type f -regex "${FILE_EXTENSIONS}" 2>&1)
+      mapfile -t LIST_FILES < <(find "${GITHUB_WORKSPACE}" -path "*/node_modules" -prune -o -type f -regex "${FILE_EXTENSIONS}" 2>&1)
 
-    ###########################
+      ###########################
-    # Set IFS back to default #
+      # Set IFS back to default #
-    ###########################
+      ###########################
-    IFS="${DEFAULT_IFS}"
+      IFS="${DEFAULT_IFS}"
+    fi
 
     ############################################################
     # Set it back to empty if loaded with blanks from scanning #
@@ -163,11 +170,15 @@ function LintCodebase() {
       elif [[ ${FILE} == *".rbenv"* ]]; then
         # This is likely the ruby environment folder and shouldn't be parsed
         continue
-      elif [[ ${FILE_TYPE} == "BASH" ]] || [[ ${FILE_TYPE} == "SHELL_SHFMT" ]]; then
+      elif [[ ${FILE_TYPE} == "BASH" ]] && ! IsValidShellScript "${FILE}"; then
-        if CheckZsh "${FILE}"; then
+        # not a valid script and we need to skip
-          # ZSH file and we need to skip
+        continue
-          warn "$LINTER_NAME does NOT currently support zsh, skipping file"
+      elif [[ ${FILE_TYPE} == "BASH_EXEC" ]] && ! IsValidShellScript "${FILE}"; then
-          continue
+        # not a valid script and we need to skip
+        continue
+      elif [[ ${FILE_TYPE} == "SHELL_SHFMT" ]] && ! IsValidShellScript "${FILE}"; then
+        # not a valid script and we need to skip
+        continue
       fi
 
       ##################################