Merge pull request #642 from ferrarimarco/master

Lint shell files with shfmt, fix bash-exec version

.automation/test/shell_shfmt/.editorconfig

@@ -0,0 +1,6 @@
+# top-most EditorConfig file
+root = true
+
+[*.ext]
+indent_style = space
+indent_size = 4

.automation/test/shell_shfmt/README.md

@@ -0,0 +1,19 @@
+# SHFMT Test Cases
+
+This folder holds the test cases for **SHFMT**.
+
+## Additional Docs
+
+No Additional information is needed for this test case.
+
+## Good Test Cases
+
+The test cases denoted: `LANGUAGE_good_FILE.EXTENSION` are all valid, and should pass successfully when linted.
+
+- **Note:** They are linted utilizing the default linter rules.
+
+## Bad Test Cases
+
+The test cases denoted: `LANGUAGE_bad_FILE.EXTENSION` are **NOT** valid, and should trigger errors when linted.
+
+- **Note:** They are linted utilizing the default linter rules.

.automation/test/shell_shfmt/shell_shfmt_bad_1.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# CMD
+HELLO_WORLD=$(echo "Hello World" | cut -f1 -d' ' 2>&1)
+
+# Load the error code
+ERROR_CODE=$?
+
+# Check the shell
+if [ ${ERROR_CODE} -ne 0 ]; then
+  echo "We did it!"
+  exit 0
+else
+  echo "We done goofed it..."
+  echo "${HELLO_WORLD}"
+  exit 1
+fi

.automation/test/shell_shfmt/shell_shfmt_good_1.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# CMD
+HELLO_WORLD=$(echo "Hello World" | cut -f1 -d' ' 2>&1)
+
+# Load the error code
+ERROR_CODE=$?
+
+# Check the shell
+if [ ${ERROR_CODE} -ne 0 ]; then
+    echo "We did it!"
+    exit 0
+else
+    echo "We done goofed it..."
+    echo "${HELLO_WORLD}"
+    exit 1
+fi

Dockerfile

@@ -281,6 +281,18 @@ RUN R -e "install.packages(list.dirs('/home/r-library',recursive = FALSE), repos
 COPY --from=chktex /usr/bin/chktex /usr/bin/
 RUN cd ~ && touch .chktexrc
 
+#################
+# Install shfmt #
+#################
+ENV GO111MODULE=on \
+    GOROOT=/usr/lib/go \
+    GOPATH=/go
+
+ENV PATH="$PATH":"$GOROOT"/bin:"$GOPATH"/bin
+
+RUN mkdir -p ${GOPATH}/src ${GOPATH}/bin
+RUN go get mvdan.cc/sh/v3/cmd/shfmt
+
 ###########################################
 # Load GitHub Env Vars for GitHub Actions #
 ###########################################
@@ -345,6 +357,7 @@ ENV ACTIONS_RUNNER_DEBUG=${ACTIONS_RUNNER_DEBUG} \
     VALIDATE_R=${VALIDATE_R} \
     VALIDATE_RAKU=${VALIDATE_RAKU} \
     VALIDATE_RUBY=${VALIDATE_RUBY} \
+    VALIDATE_SHELL_SHFMT=${VALIDATE_SHELL_SHFMT} \
     VALIDATE_STATES=${VALIDATE_STATES} \
     VALIDATE_SQL=${VALIDATE_SQL} \
     VALIDATE_TERRAFORM=${VALIDATE_TERRAFORM} \

README.md

@@ -73,7 +73,7 @@ Developers on **GitHub** can call the **GitHub Action** to lint their code base
 | **R**                            | [lintr](https://github.com/jimhester/lintr)                                                                                                                                    |
 | **Raku**                         | [Raku](https://raku.org)                                                                                                                                                       |
 | **Ruby**                         | [RuboCop](https://github.com/rubocop-hq/rubocop)                                                                                                                               |
-| **Shell**                        | [Shellcheck](https://github.com/koalaman/shellcheck) / [Bash-exec]                                                                                                             |
+| **Shell**                        | [Shellcheck](https://github.com/koalaman/shellcheck) / [Bash-exec] / [shfmt](https://github.com/mvdan/sh)                                                                      |
 | **SQL**                          | [sql-lint](https://github.com/joereynolds/sql-lint)                                                                                                                            |
 | **Terraform**                    | [tflint](https://github.com/terraform-linters/tflint) / [terrascan](https://github.com/accurics/terrascan)                                                                     |
 | **TypeScript**                   | [eslint](https://eslint.org/) / [standard js](https://standardjs.com/)                                                                                                         |
@@ -236,6 +236,7 @@ But if you wish to select or exclude specific linters, we give you full control
 | **VALIDATE_R**                    | `true`                | Flag to enable or disable the linting process of the R language.                                                                                                                 |
 | **VALIDATE_RAKU**                 | `true`                | Flag to enable or disable the linting process of the Raku language.                                                                                                              |
 | **VALIDATE_RUBY**                 | `true`                | Flag to enable or disable the linting process of the Ruby language.                                                                                                              |
+| **VALIDATE_SHELL_SHFMT**          | `true`                | Flag to enable or disable the linting process of Shell scripts. (Utilizing: shfmt)                                                                                               |
 | **VALIDATE_STATES**               | `true`                | Flag to enable or disable the linting process for AWS States Language.                                                                                                           |
 | **VALIDATE_SQL**                  | `true`                | Flag to enable or disable the linting process of the SQL language.                                                                                                               |
 | **VALIDATE_TERRAFORM**            | `true`                | Flag to enable or disable the linting process of the Terraform language.                                                                                                         |

docs/disabling-linters.md

@@ -1079,6 +1079,7 @@ AllCops:
 ## Shell
 
 - [Shellcheck](https://github.com/koalaman/shellcheck)
+- [shfmt](https://github.com/mvdan/sh)
 
 ### Shellcheck Config file
 
@@ -1110,6 +1111,10 @@ echo "stuff"
 moreThings()
 ```
 
+### shfmt Config file
+
+shfmt [supports EditorConfig files for configuration](https://github.com/mvdan/sh#shfmt), if available.
+
 ---
 
 ## SQL

lib/buildFileList.sh

@@ -100,8 +100,7 @@ function BuildFileList() {
       [ "${FILE_TYPE}" == "dash" ] || [ "${FILE_TYPE}" == "ksh" ]; then
       # Need to check if its a zsh file as we cannot parse it
       if CheckZsh "${FILE}"; then
-        warn "Found [zsh] script"
+        warn "ShellCheck and shfmt do NOT currently support zsh, skipping file"
-        info "ShellCheck does NOT currently support zsh, skipping file"
       else
         ################################
         # Append the file to the array #
@@ -679,7 +678,7 @@ function CheckFileType() {
     ######################
     # It is a ZSH script #
     ######################
-    warn "Found [zsh] script"
+    warn "Found [zsh] script: ${FILE}"
     info "ShellCheck does NOT currently support zsh, skipping file"
   else
     ############################
@@ -712,8 +711,7 @@ function CheckZsh() {
     ######################
     # It is a ZSH script #
     ######################
-    warn "Found [zsh] script"
+    debug "Found [zsh] script: ${FILE}"
-    info "ShellCheck does NOT currently support zsh, skipping file"
     ###################################################
     # We found zsh file and need to return with a hit #
     ###################################################

lib/linter.sh

@@ -136,7 +136,7 @@ YAML_LINTER_RULES="${DEFAULT_RULES_LOCATION}/${YAML_FILE_NAME}" # Path to the ya
 LINTER_ARRAY=('ansible-lint' 'arm-ttk' 'asl-validator' 'bash-exec' 'black' 'cfn-lint' 'checkstyle' 'chktex' 'clj-kondo' 'coffeelint'
   'dotnet-format' 'dart' 'dockerfilelint' 'dotenv-linter' 'editorconfig-checker' 'eslint' 'flake8' 'golangci-lint'
   'hadolint' 'htmlhint' 'jsonlint' 'ktlint' 'lintr' 'lua' 'markdownlint' 'npm-groovy-lint' 'perl' 'protolint'
-  'pwsh' 'pylint' 'raku' 'rubocop' 'shellcheck' 'spectral' 'standard' 'stylelint' 'sql-lint'
+  'pwsh' 'pylint' 'raku' 'rubocop' 'shellcheck' 'shfmt' 'spectral' 'standard' 'stylelint' 'sql-lint'
   'terrascan' 'tflint' 'xmllint' 'yamllint')
 
 #############################
@@ -146,7 +146,7 @@ LANGUAGE_ARRAY=('ANSIBLE' 'ARM' 'BASH' 'BASH_EXEC' 'CLOUDFORMATION' 'CLOJURE' 'C
   'DART' 'DOCKERFILE' 'DOCKERFILE_HADOLINT' 'EDITORCONFIG' 'ENV' 'GO' 'GROOVY' 'HTML'
   'JAVA' 'JAVASCRIPT_ES' 'JAVASCRIPT_STANDARD' 'JSON' 'JSX' 'KOTLIN' 'LATEX' 'LUA' 'MARKDOWN'
   'OPENAPI' 'PERL' 'PHP_BUILTIN' 'PHP_PHPCS' 'PHP_PHPSTAN' 'PHP_PSALM' 'POWERSHELL'
-  'PROTOBUF' 'PYTHON_BLACK' 'PYTHON_PYLINT' 'PYTHON_FLAKE8' 'R' 'RAKU' 'RUBY' 'STATES' 'SQL' 'TERRAFORM'
+  'PROTOBUF' 'PYTHON_BLACK' 'PYTHON_PYLINT' 'PYTHON_FLAKE8' 'R' 'RAKU' 'RUBY' 'SHELL_SHFMT' 'STATES' 'SQL' 'TERRAFORM'
   'TERRAFORM_TERRASCAN' 'TSX' 'TYPESCRIPT_ES' 'TYPESCRIPT_STANDARD' 'XML' 'YAML')
 
 ############################################
@@ -211,6 +211,7 @@ VALIDATE_R="${VALIDATE_R}"                                           # Boolean t
 VALIDATE_RAKU="${VALIDATE_RAKU}"                                     # Boolean to validate language
 VALIDATE_RUBY="${VALIDATE_RUBY}"                                     # Boolean to validate language
 VALIDATE_STATES="${VALIDATE_STATES}"                                 # Boolean to validate language
+VALIDATE_SHELL_SHFMT="${VALIDATE_SHELL_SHFMT}"                       # Boolean to check Shell files against editorconfig
 VALIDATE_SQL="${VALIDATE_SQL}"                                       # Boolean to validate language
 VALIDATE_TERRAFORM="${VALIDATE_TERRAFORM}"                           # Boolean to validate language
 VALIDATE_TERRAFORM_TERRASCAN="${VALIDATE_TERRAFORM_TERRASCAN}"       # Boolean to validate language
@@ -404,6 +405,8 @@ ERRORS_FOUND_RAKU=0                     # Count of errors found
 export ERRORS_FOUND_RAKU                # Workaround SC2034
 ERRORS_FOUND_RUBY=0                     # Count of errors found
 export ERRORS_FOUND_RUBY                # Workaround SC2034
+ERRORS_FOUND_SHELL_SHFMT=0              # Count of errors found
+export ERRORS_FOUND_SHELL_SHFMT
 ERRORS_FOUND_STATES=0                   # Count of errors found
 export ERRORS_FOUND_STATES              # Workaround SC2034
 ERRORS_FOUND_SQL=0                      # Count of errors found
@@ -464,7 +467,7 @@ GetLinterVersions() {
     if [[ ${LINTER} == "arm-ttk" ]]; then
       # Need specific command for ARM
       mapfile -t GET_VERSION_CMD < <(grep -iE 'version' "${ARM_TTK_PSD1}" | xargs 2>&1)
-    elif [[ ${LINTER} == "protolint" ]] || [[ ${LINTER} == "editorconfig-checker" ]]; then
+    elif [[ ${LINTER} == "protolint" ]] || [[ ${LINTER} == "editorconfig-checker" ]] || [[ ${LINTER} == "bash-exec" ]]; then
       # Need specific command for Protolint and editorconfig-checker
       mapfile -t GET_VERSION_CMD < <(echo "--version not supported")
     elif [[ ${LINTER} == "lintr" ]]; then
@@ -475,6 +478,8 @@ GetLinterVersions() {
     elif [[ ${LINTER} == "lua" ]]; then
       # Semi standardversion command
       mapfile -t GET_VERSION_CMD < <("${LINTER}" -v 2>&1)
+    elif [[ ${LINTER} == "terrascan" ]]; then
+      mapfile -t GET_VERSION_CMD < <("${LINTER}" version 2>&1)
     else
       # Standard version command
       mapfile -t GET_VERSION_CMD < <("${LINTER}" --version 2>&1)
@@ -1065,9 +1070,13 @@ Reports() {
     #############################################
     # Print info on reports that were generated #
     #############################################
+    if [ -d "${REPORT_OUTPUT_FOLDER}" ]; then
       info "Contents of report folder:"
       OUTPUT_CONTENTS_CMD=$(ls "${REPORT_OUTPUT_FOLDER}")
       info "$OUTPUT_CONTENTS_CMD"
+    else
+      warn "Report output folder (${REPORT_OUTPUT_FOLDER}) does NOT exist."
+    fi
   fi
 
   ################################
@@ -1836,6 +1845,26 @@ if [ "${VALIDATE_RUBY}" == "true" ]; then
   LintCodebase "RUBY" "rubocop" "rubocop -c ${RUBY_LINTER_RULES} --force-exclusion" ".*\.\(rb\)\$" "${FILE_ARRAY_RUBY[@]}"
 fi
 
+#################
+# SHFMT LINTING #
+#################
+if [ "${VALIDATE_SHELL_SHFMT}" == "true" ]; then
+  ####################################
+  # Lint the files with shfmt #
+  ####################################
+  EDITORCONFIG_FILE_PATH="${GITHUB_WORKSPACE}"/.editorconfig
+  if [ -e "$EDITORCONFIG_FILE_PATH" ]; then
+    # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY"
+    LintCodebase "SHELL_SHFMT" "shfmt" "shfmt -d" ".*\.\(sh\|bash\|dash\|ksh\)\$" "${FILE_ARRAY_BASH[@]}"
+  else
+    ###############################
+    # No .editorconfig file found #
+    ###############################
+    warn "No .editorconfig found at:[$EDITORCONFIG_FILE_PATH]"
+    debug "skipping shfmt"
+  fi
+fi
+
 ######################
 # AWS STATES LINTING #
 ######################

lib/worker.sh

@@ -165,8 +165,11 @@ function LintCodebase() {
         continue
       elif [[ ${FILE_TYPE} == "BASH" ]] && CheckZsh "${FILE}"; then
         # ZSH file and we need to skip
-        warn "Found [zsh] script"
+        warn "$LINTER_NAME does NOT currently support zsh, skipping file"
-        info "ShellCheck does NOT currently support zsh, skipping file"
+        continue
+      elif [[ ${FILE_TYPE} == "SHELL_SHFMT" ]] && CheckZsh "${FILE}"; then
+        # ZSH file and we need to skip
+        warn "$LINTER_NAME does NOT currently support zsh, skipping file"
         continue
       fi
 
@@ -679,6 +682,7 @@ function RunTestCases() {
   TestCodebase "R" "lintr" "lintr::lint()" ".*\.\(r\|rmd\)\$" "r"
   TestCodebase "RAKU" "raku" "raku -c" ".*\.\(raku\|rakumod\|rakutest\|pm6\|pl6\|p6\)\$" "raku"
   TestCodebase "RUBY" "rubocop" "rubocop -c ${RUBY_LINTER_RULES}" ".*\.\(rb\)\$" "ruby"
+  TestCodebase "SHELL_SHFMT" "shfmt" "shfmt -d" ".*\.\(sh\|bash\|dash\|ksh\)\$" "shell_shfmt"
   TestCodebase "STATES" "asl-validator" "asl-validator --json-path" ".*\.\(json\)\$" "states"
   TestCodebase "SQL" "sql-lint" "sql-lint --config ${SQL_LINTER_RULES}" ".*\.\(sql\)\$" "sql"
   TestCodebase "TERRAFORM" "tflint" "tflint -c ${TERRAFORM_LINTER_RULES}" ".*\.\(tf\)\$" "terraform"