From 3a7717b9392718824c3aaa70f1a231b136381466 Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Mon, 31 Aug 2020 23:42:51 +0200 Subject: [PATCH 1/4] Bootstrap npm-groovy-lint --- Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index d10d260b..a2930b2d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -293,6 +293,11 @@ ENV PATH="$PATH":"$GOROOT"/bin:"$GOPATH"/bin RUN mkdir -p ${GOPATH}/src ${GOPATH}/bin RUN go get mvdan.cc/sh/v3/cmd/shfmt +############################# +# Bootstrap npm-groovy-lint # +############################# +RUN npm-groovy-lint --version + ########################################### # Load GitHub Env Vars for GitHub Actions # ########################################### From 7ecf9a132cbafd94a4a5d066998c5297801cb050 Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Mon, 31 Aug 2020 23:43:26 +0200 Subject: [PATCH 2/4] Add details about bash-exec in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 22085a4f..549657d8 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Developers on **GitHub** can call the **GitHub Action** to lint their code base | **R** | [lintr](https://github.com/jimhester/lintr) | | **Raku** | [Raku](https://raku.org) | | **Ruby** | [RuboCop](https://github.com/rubocop-hq/rubocop) | -| **Shell** | [Shellcheck](https://github.com/koalaman/shellcheck) / [Bash-exec] / [shfmt](https://github.com/mvdan/sh) | +| **Shell** | [Shellcheck](https://github.com/koalaman/shellcheck) / [executable bit check] / [shfmt](https://github.com/mvdan/sh) | | **SQL** | [sql-lint](https://github.com/joereynolds/sql-lint) | | **Terraform** | [tflint](https://github.com/terraform-linters/tflint) / [terrascan](https://github.com/accurics/terrascan) | | **TypeScript** | [eslint](https://eslint.org/) / [standard js](https://standardjs.com/) | From 30763f2575d4a0396a913da1735860ca092d0b6a Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Mon, 31 Aug 2020 23:44:35 +0200 Subject: [PATCH 3/4] Streamline shell file list building --- lib/buildFileList.sh | 168 ++++++++++++++++++++++++++++--------------- lib/linter.sh | 24 ++++++- lib/worker.sh | 45 +++++++----- 3 files changed, 160 insertions(+), 77 deletions(-) diff --git a/lib/buildFileList.sh b/lib/buildFileList.sh index bbb31e8c..ff03f197 100755 --- a/lib/buildFileList.sh +++ b/lib/buildFileList.sh @@ -73,13 +73,8 @@ function BuildFileList() { info "------ Files modified in the commit(s): ------" info "----------------------------------------------" for FILE in "${RAW_FILE_ARRAY[@]}"; do - ########################### - # Get the files extension # - ########################### # Extract just the file extension - FILE_TYPE=${FILE##*.} - # To lowercase - FILE_TYPE=${FILE_TYPE,,} + FILE_TYPE="$(GetFileExtension "$FILE")" # get the baseFile for additonal logic BASE_FILE=$(basename "${FILE,,}") @@ -94,23 +89,17 @@ function BuildFileList() { debug "FILE_TYPE:[${FILE_TYPE}]" ###################### - # Get the BASH files # + # Get the shell files # ###################### - if [ "${FILE_TYPE}" == "sh" ] || [ "${FILE_TYPE}" == "bash" ] || - [ "${FILE_TYPE}" == "dash" ] || [ "${FILE_TYPE}" == "ksh" ]; then - # Need to check if its a zsh file as we cannot parse it - if CheckZsh "${FILE}"; then - warn "ShellCheck and shfmt do NOT currently support zsh, skipping file" - else - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_BASH+=("${FILE}") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - fi + if IsValidShellScript "${FILE}"; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_BASH+=("${FILE}") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 ######################### # Get the CLOJURE files # @@ -628,8 +617,8 @@ function BuildFileList() { info "Successfully gathered list of files..." } ################################################################################ -#### Function CheckFileType #################################################### -function CheckFileType() { +#### Function GetFileType ###################################################### +function GetFileType() { # Need to run the file through the 'file' exec to help determine # The type of file being parsed @@ -643,15 +632,28 @@ function CheckFileType() { ################## GET_FILE_TYPE_CMD=$(file "${FILE}" 2>&1) + echo "${GET_FILE_TYPE_CMD}" +} +################################################################################ +#### Function CheckFileType #################################################### +function CheckFileType() { + # Need to run the file through the 'file' exec to help determine + # The type of file being parsed + + ################ + # Pull in Vars # + ################ + FILE="$1" + + ################# + # Get file type # + ################# + GET_FILE_TYPE_CMD="$(GetFileType "$FILE")" + ################# # Check if bash # ################# - if [[ ${GET_FILE_TYPE_CMD} == *"Bourne-Again shell script"* ]]; then - ####################### - # It is a bash script # - ####################### - warn "Found bash script without extension:[.sh]" - info "Please update file with proper extensions." + if IsValidShellScript "$FILE"; then ################################ # Append the file to the array # ################################ @@ -674,12 +676,6 @@ function CheckFileType() { # Set the READ_ONLY_CHANGE_FLAG since this could be exec # ########################################################## READ_ONLY_CHANGE_FLAG=1 - elif [[ ${GET_FILE_TYPE_CMD} == *"zsh script"* ]]; then - ###################### - # It is a ZSH script # - ###################### - warn "Found [zsh] script: ${FILE}" - info "ShellCheck does NOT currently support zsh, skipping file" else ############################ # Extension was not found! # @@ -692,34 +688,90 @@ function CheckFileType() { fi } ################################################################################ -#### Function CheckZsh ######################################################### -function CheckZsh() { - # Spagetti code to make sure were properly excluding zsh - # until we get a proper linter - +#### Function GetFileExtension ############################################### +function GetFileExtension() { ################ # Pull in Vars # ################ FILE="$1" - ################## - # Check the file # - ################## - GET_FILE_TYPE_CMD=$(file "${FILE}" 2>&1) + ########################### + # Get the files extension # + ########################### + # Extract just the file extension + FILE_TYPE=${FILE##*.} + # To lowercase + FILE_TYPE=${FILE_TYPE,,} - if [[ ${GET_FILE_TYPE_CMD} == *"zsh script"* ]]; then - ###################### - # It is a ZSH script # - ###################### - debug "Found [zsh] script: ${FILE}" - ################################################### - # We found zsh file and need to return with a hit # - ################################################### - return 0 - else - ################## - # Not a zsh file # - ################## + echo "$FILE_TYPE" +} +################################################################################ +#### Function IsValidShellScript ############################################### +function IsValidShellScript() { + ################ + # Pull in Vars # + ################ + FILE="$1" + + ################# + # Get file type # + ################# + FILE_EXTENSION="$(GetFileExtension "$FILE")" + GET_FILE_TYPE_CMD="$(GetFileType "$FILE")" + + trace "File:[${FILE}], File extension:[${FILE_EXTENSION}], File type: [${GET_FILE_TYPE_CMD}]" + + if [[ "${FILE_EXTENSION}" == "zsh" ]] || + [[ ${GET_FILE_TYPE_CMD} == *"zsh script"* ]]; then + warn "$FILE is a ZSH script. Skipping..." return 1 fi + + if [ "${FILE_EXTENSION}" == "sh" ] || + [ "${FILE_EXTENSION}" == "bash" ] || + [ "${FILE_EXTENSION}" == "dash" ] || + [ "${FILE_EXTENSION}" == "ksh" ]; then + debug "$FILE is a valid shell script (has a valid extension: ${FILE_EXTENSION})" + return 0 + fi + + if [[ "${GET_FILE_TYPE_CMD}" == *"POSIX shell script"* ]] || + [[ ${GET_FILE_TYPE_CMD} == *"Bourne-Again shell script"* ]] || + [[ ${GET_FILE_TYPE_CMD} == *"dash script"* ]] || + [[ ${GET_FILE_TYPE_CMD} == *"ksh script"* ]] || + [[ ${GET_FILE_TYPE_CMD} == *"/usr/bin/env sh script"* ]]; then + debug "$FILE is a valid shell script (has a valid file type: ${GET_FILE_TYPE_CMD})" + return 0 + fi + + trace "$FILE is NOT a supported shell script. Skipping" + return 1 +} +################################################################################ +#### Function IsValidShellScript ############################################### +function PopulateShellScriptsList() { + debug "Populating shell script file list. Source: ${GITHUB_WORKSPACE}" + + ############################################################################### + # Set the file seperator to newline to allow for grabbing objects with spaces # + ############################################################################### + IFS=$'\n' + + mapfile -t LIST_FILES < <(find "${GITHUB_WORKSPACE}" -path "*/node_modules" -prune -o -path "*/\.git" -prune -o -type f 2>&1) + for FILE in "${LIST_FILES[@]}"; do + if IsValidShellScript "${FILE}"; then + debug "Adding ${FILE} to shell script files list" + FILE_ARRAY_BASH+=("${FILE}") + + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + fi + done + + ########################### + # Set IFS back to default # + ########################### + IFS="${DEFAULT_IFS}" } diff --git a/lib/linter.sh b/lib/linter.sh index c2e54bd7..077e35b2 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -1366,7 +1366,7 @@ if [ "${VALIDATE_BASH}" == "true" ]; then # Lint the bash files # ####################### # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY" - LintCodebase "BASH" "shellcheck" "shellcheck --color --external-sources" ".*\.\(sh\|bash\|dash\|ksh\)\$" "${FILE_ARRAY_BASH[@]}" + LintCodebase "BASH" "shellcheck" "shellcheck --color --external-sources" "disabledfileext" "${FILE_ARRAY_BASH[@]}" fi ##################### @@ -1377,7 +1377,7 @@ if [ "${VALIDATE_BASH_EXEC}" == "true" ]; then # Lint the bash files # ####################### # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY" - LintCodebase "BASH_EXEC" "bash-exec" "bash-exec" ".*\.\(sh\|bash\|dash\|ksh\)\$" "${FILE_ARRAY_BASH[@]}" + LintCodebase "BASH_EXEC" "bash-exec" "bash-exec" "disabledfileext" "${FILE_ARRAY_BASH[@]}" fi ########################## @@ -1836,6 +1836,26 @@ if [ "${VALIDATE_RUBY}" == "true" ]; then LintCodebase "RUBY" "rubocop" "rubocop -c ${RUBY_LINTER_RULES} --force-exclusion" ".*\.\(rb\)\$" "${FILE_ARRAY_RUBY[@]}" fi +################# +# SHFMT LINTING # +################# +if [ "${VALIDATE_SHELL_SHFMT}" == "true" ]; then + #################################### + # Lint the files with shfmt # + #################################### + EDITORCONFIG_FILE_PATH="${GITHUB_WORKSPACE}"/.editorconfig + if [ -e "$EDITORCONFIG_FILE_PATH" ]; then + # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY" + LintCodebase "SHELL_SHFMT" "shfmt" "shfmt -d" "disabledfileext" "${FILE_ARRAY_BASH[@]}" + else + ############################### + # No .editorconfig file found # + ############################### + warn "No .editorconfig found at:[$EDITORCONFIG_FILE_PATH]" + debug "skipping shfmt" + fi +fi + ###################### # AWS STATES LINTING # ###################### diff --git a/lib/worker.sh b/lib/worker.sh index 8cfed3ca..35247b75 100755 --- a/lib/worker.sh +++ b/lib/worker.sh @@ -81,20 +81,27 @@ function LintCodebase() { # We have files added to array of files to check LIST_FILES=("${FILE_ARRAY[@]}") # Copy the array into list else - ############################################################################### - # Set the file separator to newline to allow for grabbing objects with spaces # - ############################################################################### - IFS=$'\n' + if [[ ${FILE_TYPE} == "BASH" ]] || + [[ ${FILE_TYPE} == "BASH_EXEC" ]] || + [[ ${FILE_TYPE} == "SHELL_SHFMT" ]]; then + # Populate a list of valid shell scripts. + PopulateShellScriptsList + else + ############################################################################### + # Set the file separator to newline to allow for grabbing objects with spaces # + ############################################################################### + IFS=$'\n' - ################################# - # Get list of all files to lint # - ################################# - mapfile -t LIST_FILES < <(find "${GITHUB_WORKSPACE}" -path "*/node_modules" -prune -o -type f -regex "${FILE_EXTENSIONS}" 2>&1) + ################################# + # Get list of all files to lint # + ################################# + mapfile -t LIST_FILES < <(find "${GITHUB_WORKSPACE}" -path "*/node_modules" -prune -o -type f -regex "${FILE_EXTENSIONS}" 2>&1) - ########################### - # Set IFS back to default # - ########################### - IFS="${DEFAULT_IFS}" + ########################### + # Set IFS back to default # + ########################### + IFS="${DEFAULT_IFS}" + fi ############################################################ # Set it back to empty if loaded with blanks from scanning # @@ -163,11 +170,15 @@ function LintCodebase() { elif [[ ${FILE} == *".rbenv"* ]]; then # This is likely the ruby environment folder and shouldn't be parsed continue - elif [[ ${FILE_TYPE} == "BASH" ]] || [[ ${FILE_TYPE} == "SHELL_SHFMT" ]]; then - if CheckZsh "${FILE}"; then - # ZSH file and we need to skip - warn "$LINTER_NAME does NOT currently support zsh, skipping file" - continue + elif [[ ${FILE_TYPE} == "BASH" ]] && ! IsValidShellScript "${FILE}"; then + # not a valid script and we need to skip + continue + elif [[ ${FILE_TYPE} == "BASH_EXEC" ]] && ! IsValidShellScript "${FILE}"; then + # not a valid script and we need to skip + continue + elif [[ ${FILE_TYPE} == "SHELL_SHFMT" ]] && ! IsValidShellScript "${FILE}"; then + # not a valid script and we need to skip + continue fi ################################## From de4b8a4fa0a9132d682792238127382f39937318 Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Tue, 1 Sep 2020 00:06:15 +0200 Subject: [PATCH 4/4] Add shfmt and fixes back --- lib/linter.sh | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/lib/linter.sh b/lib/linter.sh index 077e35b2..2b0030b3 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -136,7 +136,7 @@ YAML_LINTER_RULES="${DEFAULT_RULES_LOCATION}/${YAML_FILE_NAME}" # Path to the ya LINTER_ARRAY=('ansible-lint' 'arm-ttk' 'asl-validator' 'bash-exec' 'black' 'cfn-lint' 'checkstyle' 'chktex' 'clj-kondo' 'coffeelint' 'dotnet-format' 'dart' 'dockerfilelint' 'dotenv-linter' 'editorconfig-checker' 'eslint' 'flake8' 'golangci-lint' 'hadolint' 'htmlhint' 'jsonlint' 'ktlint' 'lintr' 'lua' 'markdownlint' 'npm-groovy-lint' 'perl' 'protolint' - 'pwsh' 'pylint' 'raku' 'rubocop' 'shellcheck' 'spectral' 'standard' 'stylelint' 'sql-lint' + 'pwsh' 'pylint' 'raku' 'rubocop' 'shellcheck' 'shfmt' 'spectral' 'standard' 'stylelint' 'sql-lint' 'terrascan' 'tflint' 'xmllint' 'yamllint') ############################# @@ -146,7 +146,7 @@ LANGUAGE_ARRAY=('ANSIBLE' 'ARM' 'BASH' 'BASH_EXEC' 'CLOUDFORMATION' 'CLOJURE' 'C 'DART' 'DOCKERFILE' 'DOCKERFILE_HADOLINT' 'EDITORCONFIG' 'ENV' 'GO' 'GROOVY' 'HTML' 'JAVA' 'JAVASCRIPT_ES' 'JAVASCRIPT_STANDARD' 'JSON' 'JSX' 'KOTLIN' 'LATEX' 'LUA' 'MARKDOWN' 'OPENAPI' 'PERL' 'PHP_BUILTIN' 'PHP_PHPCS' 'PHP_PHPSTAN' 'PHP_PSALM' 'POWERSHELL' - 'PROTOBUF' 'PYTHON_BLACK' 'PYTHON_PYLINT' 'PYTHON_FLAKE8' 'R' 'RAKU' 'RUBY' 'STATES' 'SQL' 'TERRAFORM' + 'PROTOBUF' 'PYTHON_BLACK' 'PYTHON_PYLINT' 'PYTHON_FLAKE8' 'R' 'RAKU' 'RUBY' 'SHELL_SHFMT' 'STATES' 'SQL' 'TERRAFORM' 'TERRAFORM_TERRASCAN' 'TSX' 'TYPESCRIPT_ES' 'TYPESCRIPT_STANDARD' 'XML' 'YAML') ############################################ @@ -211,6 +211,7 @@ VALIDATE_R="${VALIDATE_R}" # Boolean t VALIDATE_RAKU="${VALIDATE_RAKU}" # Boolean to validate language VALIDATE_RUBY="${VALIDATE_RUBY}" # Boolean to validate language VALIDATE_STATES="${VALIDATE_STATES}" # Boolean to validate language +VALIDATE_SHELL_SHFMT="${VALIDATE_SHELL_SHFMT}" # Boolean to check Shell files against editorconfig VALIDATE_SQL="${VALIDATE_SQL}" # Boolean to validate language VALIDATE_TERRAFORM="${VALIDATE_TERRAFORM}" # Boolean to validate language VALIDATE_TERRAFORM_TERRASCAN="${VALIDATE_TERRAFORM_TERRASCAN}" # Boolean to validate language @@ -404,6 +405,8 @@ ERRORS_FOUND_RAKU=0 # Count of errors found export ERRORS_FOUND_RAKU # Workaround SC2034 ERRORS_FOUND_RUBY=0 # Count of errors found export ERRORS_FOUND_RUBY # Workaround SC2034 +ERRORS_FOUND_SHELL_SHFMT=0 # Count of errors found +export ERRORS_FOUND_SHELL_SHFMT ERRORS_FOUND_STATES=0 # Count of errors found export ERRORS_FOUND_STATES # Workaround SC2034 ERRORS_FOUND_SQL=0 # Count of errors found @@ -464,7 +467,7 @@ GetLinterVersions() { if [[ ${LINTER} == "arm-ttk" ]]; then # Need specific command for ARM mapfile -t GET_VERSION_CMD < <(grep -iE 'version' "${ARM_TTK_PSD1}" | xargs 2>&1) - elif [[ ${LINTER} == "protolint" ]] || [[ ${LINTER} == "editorconfig-checker" ]]; then + elif [[ ${LINTER} == "protolint" ]] || [[ ${LINTER} == "editorconfig-checker" ]] || [[ ${LINTER} == "bash-exec" ]]; then # Need specific command for Protolint and editorconfig-checker mapfile -t GET_VERSION_CMD < <(echo "--version not supported") elif [[ ${LINTER} == "lintr" ]]; then @@ -475,6 +478,8 @@ GetLinterVersions() { elif [[ ${LINTER} == "lua" ]]; then # Semi standardversion command mapfile -t GET_VERSION_CMD < <("${LINTER}" -v 2>&1) + elif [[ ${LINTER} == "terrascan" ]]; then + mapfile -t GET_VERSION_CMD < <("${LINTER}" version 2>&1) else # Standard version command mapfile -t GET_VERSION_CMD < <("${LINTER}" --version 2>&1) @@ -1065,9 +1070,13 @@ Reports() { ############################################# # Print info on reports that were generated # ############################################# - info "Contents of report folder:" - OUTPUT_CONTENTS_CMD=$(ls "${REPORT_OUTPUT_FOLDER}") - info "$OUTPUT_CONTENTS_CMD" + if [ -d "${REPORT_OUTPUT_FOLDER}" ]; then + info "Contents of report folder:" + OUTPUT_CONTENTS_CMD=$(ls "${REPORT_OUTPUT_FOLDER}") + info "$OUTPUT_CONTENTS_CMD" + else + warn "Report output folder (${REPORT_OUTPUT_FOLDER}) does NOT exist." + fi fi ################################ @@ -1846,7 +1855,7 @@ if [ "${VALIDATE_SHELL_SHFMT}" == "true" ]; then EDITORCONFIG_FILE_PATH="${GITHUB_WORKSPACE}"/.editorconfig if [ -e "$EDITORCONFIG_FILE_PATH" ]; then # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY" - LintCodebase "SHELL_SHFMT" "shfmt" "shfmt -d" "disabledfileext" "${FILE_ARRAY_BASH[@]}" + LintCodebase "SHELL_SHFMT" "shfmt" "shfmt -d" ".*\.\(sh\|bash\|dash\|ksh\)\$" "${FILE_ARRAY_BASH[@]}" else ############################### # No .editorconfig file found #