Merge pull request #649 from ferrarimarco/streamline-file-list

Streamline shell scripts file list building and bootstrap npm-groovy-lint
This commit is contained in:
Lukas Gravley 2020-09-01 08:06:10 -05:00 committed by GitHub
commit c2224ac075
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 181 additions and 84 deletions

View file

@ -293,6 +293,11 @@ ENV PATH="$PATH":"$GOROOT"/bin:"$GOPATH"/bin
RUN mkdir -p ${GOPATH}/src ${GOPATH}/bin
RUN go get mvdan.cc/sh/v3/cmd/shfmt
#############################
# Bootstrap npm-groovy-lint #
#############################
RUN npm-groovy-lint --version
###########################################
# Load GitHub Env Vars for GitHub Actions #
###########################################

View file

@ -73,7 +73,7 @@ Developers on **GitHub** can call the **GitHub Action** to lint their code base
| **R** | [lintr](https://github.com/jimhester/lintr) |
| **Raku** | [Raku](https://raku.org) |
| **Ruby** | [RuboCop](https://github.com/rubocop-hq/rubocop) |
| **Shell** | [Shellcheck](https://github.com/koalaman/shellcheck) / [Bash-exec] / [shfmt](https://github.com/mvdan/sh) |
| **Shell** | [Shellcheck](https://github.com/koalaman/shellcheck) / [executable bit check] / [shfmt](https://github.com/mvdan/sh) |
| **SQL** | [sql-lint](https://github.com/joereynolds/sql-lint) |
| **Terraform** | [tflint](https://github.com/terraform-linters/tflint) / [terrascan](https://github.com/accurics/terrascan) |
| **TypeScript** | [eslint](https://eslint.org/) / [standard js](https://standardjs.com/) |

View file

@ -73,13 +73,8 @@ function BuildFileList() {
info "------ Files modified in the commit(s): ------"
info "----------------------------------------------"
for FILE in "${RAW_FILE_ARRAY[@]}"; do
###########################
# Get the files extension #
###########################
# Extract just the file extension
FILE_TYPE=${FILE##*.}
# To lowercase
FILE_TYPE=${FILE_TYPE,,}
FILE_TYPE="$(GetFileExtension "$FILE")"
# get the baseFile for additonal logic
BASE_FILE=$(basename "${FILE,,}")
@ -94,23 +89,17 @@ function BuildFileList() {
debug "FILE_TYPE:[${FILE_TYPE}]"
######################
# Get the BASH files #
# Get the shell files #
######################
if [ "${FILE_TYPE}" == "sh" ] || [ "${FILE_TYPE}" == "bash" ] ||
[ "${FILE_TYPE}" == "dash" ] || [ "${FILE_TYPE}" == "ksh" ]; then
# Need to check if its a zsh file as we cannot parse it
if CheckZsh "${FILE}"; then
warn "ShellCheck and shfmt do NOT currently support zsh, skipping file"
else
################################
# Append the file to the array #
################################
FILE_ARRAY_BASH+=("${FILE}")
##########################################################
# Set the READ_ONLY_CHANGE_FLAG since this could be exec #
##########################################################
READ_ONLY_CHANGE_FLAG=1
fi
if IsValidShellScript "${FILE}"; then
################################
# Append the file to the array #
################################
FILE_ARRAY_BASH+=("${FILE}")
##########################################################
# Set the READ_ONLY_CHANGE_FLAG since this could be exec #
##########################################################
READ_ONLY_CHANGE_FLAG=1
#########################
# Get the CLOJURE files #
@ -628,8 +617,8 @@ function BuildFileList() {
info "Successfully gathered list of files..."
}
################################################################################
#### Function CheckFileType ####################################################
function CheckFileType() {
#### Function GetFileType ######################################################
function GetFileType() {
# Need to run the file through the 'file' exec to help determine
# The type of file being parsed
@ -643,15 +632,28 @@ function CheckFileType() {
##################
GET_FILE_TYPE_CMD=$(file "${FILE}" 2>&1)
echo "${GET_FILE_TYPE_CMD}"
}
################################################################################
#### Function CheckFileType ####################################################
function CheckFileType() {
# Need to run the file through the 'file' exec to help determine
# The type of file being parsed
################
# Pull in Vars #
################
FILE="$1"
#################
# Get file type #
#################
GET_FILE_TYPE_CMD="$(GetFileType "$FILE")"
#################
# Check if bash #
#################
if [[ ${GET_FILE_TYPE_CMD} == *"Bourne-Again shell script"* ]]; then
#######################
# It is a bash script #
#######################
warn "Found bash script without extension:[.sh]"
info "Please update file with proper extensions."
if IsValidShellScript "$FILE"; then
################################
# Append the file to the array #
################################
@ -674,12 +676,6 @@ function CheckFileType() {
# Set the READ_ONLY_CHANGE_FLAG since this could be exec #
##########################################################
READ_ONLY_CHANGE_FLAG=1
elif [[ ${GET_FILE_TYPE_CMD} == *"zsh script"* ]]; then
######################
# It is a ZSH script #
######################
warn "Found [zsh] script: ${FILE}"
info "ShellCheck does NOT currently support zsh, skipping file"
else
############################
# Extension was not found! #
@ -692,34 +688,90 @@ function CheckFileType() {
fi
}
################################################################################
#### Function CheckZsh #########################################################
function CheckZsh() {
# Spagetti code to make sure were properly excluding zsh
# until we get a proper linter
#### Function GetFileExtension ###############################################
function GetFileExtension() {
################
# Pull in Vars #
################
FILE="$1"
##################
# Check the file #
##################
GET_FILE_TYPE_CMD=$(file "${FILE}" 2>&1)
###########################
# Get the files extension #
###########################
# Extract just the file extension
FILE_TYPE=${FILE##*.}
# To lowercase
FILE_TYPE=${FILE_TYPE,,}
if [[ ${GET_FILE_TYPE_CMD} == *"zsh script"* ]]; then
######################
# It is a ZSH script #
######################
debug "Found [zsh] script: ${FILE}"
###################################################
# We found zsh file and need to return with a hit #
###################################################
return 0
else
##################
# Not a zsh file #
##################
echo "$FILE_TYPE"
}
################################################################################
#### Function IsValidShellScript ###############################################
function IsValidShellScript() {
################
# Pull in Vars #
################
FILE="$1"
#################
# Get file type #
#################
FILE_EXTENSION="$(GetFileExtension "$FILE")"
GET_FILE_TYPE_CMD="$(GetFileType "$FILE")"
trace "File:[${FILE}], File extension:[${FILE_EXTENSION}], File type: [${GET_FILE_TYPE_CMD}]"
if [[ "${FILE_EXTENSION}" == "zsh" ]] ||
[[ ${GET_FILE_TYPE_CMD} == *"zsh script"* ]]; then
warn "$FILE is a ZSH script. Skipping..."
return 1
fi
if [ "${FILE_EXTENSION}" == "sh" ] ||
[ "${FILE_EXTENSION}" == "bash" ] ||
[ "${FILE_EXTENSION}" == "dash" ] ||
[ "${FILE_EXTENSION}" == "ksh" ]; then
debug "$FILE is a valid shell script (has a valid extension: ${FILE_EXTENSION})"
return 0
fi
if [[ "${GET_FILE_TYPE_CMD}" == *"POSIX shell script"* ]] ||
[[ ${GET_FILE_TYPE_CMD} == *"Bourne-Again shell script"* ]] ||
[[ ${GET_FILE_TYPE_CMD} == *"dash script"* ]] ||
[[ ${GET_FILE_TYPE_CMD} == *"ksh script"* ]] ||
[[ ${GET_FILE_TYPE_CMD} == *"/usr/bin/env sh script"* ]]; then
debug "$FILE is a valid shell script (has a valid file type: ${GET_FILE_TYPE_CMD})"
return 0
fi
trace "$FILE is NOT a supported shell script. Skipping"
return 1
}
################################################################################
#### Function IsValidShellScript ###############################################
function PopulateShellScriptsList() {
debug "Populating shell script file list. Source: ${GITHUB_WORKSPACE}"
###############################################################################
# Set the file seperator to newline to allow for grabbing objects with spaces #
###############################################################################
IFS=$'\n'
mapfile -t LIST_FILES < <(find "${GITHUB_WORKSPACE}" -path "*/node_modules" -prune -o -path "*/\.git" -prune -o -type f 2>&1)
for FILE in "${LIST_FILES[@]}"; do
if IsValidShellScript "${FILE}"; then
debug "Adding ${FILE} to shell script files list"
FILE_ARRAY_BASH+=("${FILE}")
##########################################################
# Set the READ_ONLY_CHANGE_FLAG since this could be exec #
##########################################################
READ_ONLY_CHANGE_FLAG=1
fi
done
###########################
# Set IFS back to default #
###########################
IFS="${DEFAULT_IFS}"
}

View file

@ -136,7 +136,7 @@ YAML_LINTER_RULES="${DEFAULT_RULES_LOCATION}/${YAML_FILE_NAME}" # Path to the ya
LINTER_ARRAY=('ansible-lint' 'arm-ttk' 'asl-validator' 'bash-exec' 'black' 'cfn-lint' 'checkstyle' 'chktex' 'clj-kondo' 'coffeelint'
'dotnet-format' 'dart' 'dockerfilelint' 'dotenv-linter' 'editorconfig-checker' 'eslint' 'flake8' 'golangci-lint'
'hadolint' 'htmlhint' 'jsonlint' 'ktlint' 'lintr' 'lua' 'markdownlint' 'npm-groovy-lint' 'perl' 'protolint'
'pwsh' 'pylint' 'raku' 'rubocop' 'shellcheck' 'spectral' 'standard' 'stylelint' 'sql-lint'
'pwsh' 'pylint' 'raku' 'rubocop' 'shellcheck' 'shfmt' 'spectral' 'standard' 'stylelint' 'sql-lint'
'terrascan' 'tflint' 'xmllint' 'yamllint')
#############################
@ -146,7 +146,7 @@ LANGUAGE_ARRAY=('ANSIBLE' 'ARM' 'BASH' 'BASH_EXEC' 'CLOUDFORMATION' 'CLOJURE' 'C
'DART' 'DOCKERFILE' 'DOCKERFILE_HADOLINT' 'EDITORCONFIG' 'ENV' 'GO' 'GROOVY' 'HTML'
'JAVA' 'JAVASCRIPT_ES' 'JAVASCRIPT_STANDARD' 'JSON' 'JSX' 'KOTLIN' 'LATEX' 'LUA' 'MARKDOWN'
'OPENAPI' 'PERL' 'PHP_BUILTIN' 'PHP_PHPCS' 'PHP_PHPSTAN' 'PHP_PSALM' 'POWERSHELL'
'PROTOBUF' 'PYTHON_BLACK' 'PYTHON_PYLINT' 'PYTHON_FLAKE8' 'R' 'RAKU' 'RUBY' 'STATES' 'SQL' 'TERRAFORM'
'PROTOBUF' 'PYTHON_BLACK' 'PYTHON_PYLINT' 'PYTHON_FLAKE8' 'R' 'RAKU' 'RUBY' 'SHELL_SHFMT' 'STATES' 'SQL' 'TERRAFORM'
'TERRAFORM_TERRASCAN' 'TSX' 'TYPESCRIPT_ES' 'TYPESCRIPT_STANDARD' 'XML' 'YAML')
############################################
@ -211,6 +211,7 @@ VALIDATE_R="${VALIDATE_R}" # Boolean t
VALIDATE_RAKU="${VALIDATE_RAKU}" # Boolean to validate language
VALIDATE_RUBY="${VALIDATE_RUBY}" # Boolean to validate language
VALIDATE_STATES="${VALIDATE_STATES}" # Boolean to validate language
VALIDATE_SHELL_SHFMT="${VALIDATE_SHELL_SHFMT}" # Boolean to check Shell files against editorconfig
VALIDATE_SQL="${VALIDATE_SQL}" # Boolean to validate language
VALIDATE_TERRAFORM="${VALIDATE_TERRAFORM}" # Boolean to validate language
VALIDATE_TERRAFORM_TERRASCAN="${VALIDATE_TERRAFORM_TERRASCAN}" # Boolean to validate language
@ -404,6 +405,8 @@ ERRORS_FOUND_RAKU=0 # Count of errors found
export ERRORS_FOUND_RAKU # Workaround SC2034
ERRORS_FOUND_RUBY=0 # Count of errors found
export ERRORS_FOUND_RUBY # Workaround SC2034
ERRORS_FOUND_SHELL_SHFMT=0 # Count of errors found
export ERRORS_FOUND_SHELL_SHFMT
ERRORS_FOUND_STATES=0 # Count of errors found
export ERRORS_FOUND_STATES # Workaround SC2034
ERRORS_FOUND_SQL=0 # Count of errors found
@ -464,7 +467,7 @@ GetLinterVersions() {
if [[ ${LINTER} == "arm-ttk" ]]; then
# Need specific command for ARM
mapfile -t GET_VERSION_CMD < <(grep -iE 'version' "${ARM_TTK_PSD1}" | xargs 2>&1)
elif [[ ${LINTER} == "protolint" ]] || [[ ${LINTER} == "editorconfig-checker" ]]; then
elif [[ ${LINTER} == "protolint" ]] || [[ ${LINTER} == "editorconfig-checker" ]] || [[ ${LINTER} == "bash-exec" ]]; then
# Need specific command for Protolint and editorconfig-checker
mapfile -t GET_VERSION_CMD < <(echo "--version not supported")
elif [[ ${LINTER} == "lintr" ]]; then
@ -475,6 +478,8 @@ GetLinterVersions() {
elif [[ ${LINTER} == "lua" ]]; then
# Semi standardversion command
mapfile -t GET_VERSION_CMD < <("${LINTER}" -v 2>&1)
elif [[ ${LINTER} == "terrascan" ]]; then
mapfile -t GET_VERSION_CMD < <("${LINTER}" version 2>&1)
else
# Standard version command
mapfile -t GET_VERSION_CMD < <("${LINTER}" --version 2>&1)
@ -1065,9 +1070,13 @@ Reports() {
#############################################
# Print info on reports that were generated #
#############################################
info "Contents of report folder:"
OUTPUT_CONTENTS_CMD=$(ls "${REPORT_OUTPUT_FOLDER}")
info "$OUTPUT_CONTENTS_CMD"
if [ -d "${REPORT_OUTPUT_FOLDER}" ]; then
info "Contents of report folder:"
OUTPUT_CONTENTS_CMD=$(ls "${REPORT_OUTPUT_FOLDER}")
info "$OUTPUT_CONTENTS_CMD"
else
warn "Report output folder (${REPORT_OUTPUT_FOLDER}) does NOT exist."
fi
fi
################################
@ -1366,7 +1375,7 @@ if [ "${VALIDATE_BASH}" == "true" ]; then
# Lint the bash files #
#######################
# LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY"
LintCodebase "BASH" "shellcheck" "shellcheck --color --external-sources" ".*\.\(sh\|bash\|dash\|ksh\)\$" "${FILE_ARRAY_BASH[@]}"
LintCodebase "BASH" "shellcheck" "shellcheck --color --external-sources" "disabledfileext" "${FILE_ARRAY_BASH[@]}"
fi
#####################
@ -1377,7 +1386,7 @@ if [ "${VALIDATE_BASH_EXEC}" == "true" ]; then
# Lint the bash files #
#######################
# LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY"
LintCodebase "BASH_EXEC" "bash-exec" "bash-exec" ".*\.\(sh\|bash\|dash\|ksh\)\$" "${FILE_ARRAY_BASH[@]}"
LintCodebase "BASH_EXEC" "bash-exec" "bash-exec" "disabledfileext" "${FILE_ARRAY_BASH[@]}"
fi
##########################
@ -1836,6 +1845,26 @@ if [ "${VALIDATE_RUBY}" == "true" ]; then
LintCodebase "RUBY" "rubocop" "rubocop -c ${RUBY_LINTER_RULES} --force-exclusion" ".*\.\(rb\)\$" "${FILE_ARRAY_RUBY[@]}"
fi
#################
# SHFMT LINTING #
#################
if [ "${VALIDATE_SHELL_SHFMT}" == "true" ]; then
####################################
# Lint the files with shfmt #
####################################
EDITORCONFIG_FILE_PATH="${GITHUB_WORKSPACE}"/.editorconfig
if [ -e "$EDITORCONFIG_FILE_PATH" ]; then
# LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY"
LintCodebase "SHELL_SHFMT" "shfmt" "shfmt -d" ".*\.\(sh\|bash\|dash\|ksh\)\$" "${FILE_ARRAY_BASH[@]}"
else
###############################
# No .editorconfig file found #
###############################
warn "No .editorconfig found at:[$EDITORCONFIG_FILE_PATH]"
debug "skipping shfmt"
fi
fi
######################
# AWS STATES LINTING #
######################

View file

@ -81,20 +81,27 @@ function LintCodebase() {
# We have files added to array of files to check
LIST_FILES=("${FILE_ARRAY[@]}") # Copy the array into list
else
###############################################################################
# Set the file separator to newline to allow for grabbing objects with spaces #
###############################################################################
IFS=$'\n'
if [[ ${FILE_TYPE} == "BASH" ]] ||
[[ ${FILE_TYPE} == "BASH_EXEC" ]] ||
[[ ${FILE_TYPE} == "SHELL_SHFMT" ]]; then
# Populate a list of valid shell scripts.
PopulateShellScriptsList
else
###############################################################################
# Set the file separator to newline to allow for grabbing objects with spaces #
###############################################################################
IFS=$'\n'
#################################
# Get list of all files to lint #
#################################
mapfile -t LIST_FILES < <(find "${GITHUB_WORKSPACE}" -path "*/node_modules" -prune -o -type f -regex "${FILE_EXTENSIONS}" 2>&1)
#################################
# Get list of all files to lint #
#################################
mapfile -t LIST_FILES < <(find "${GITHUB_WORKSPACE}" -path "*/node_modules" -prune -o -type f -regex "${FILE_EXTENSIONS}" 2>&1)
###########################
# Set IFS back to default #
###########################
IFS="${DEFAULT_IFS}"
###########################
# Set IFS back to default #
###########################
IFS="${DEFAULT_IFS}"
fi
############################################################
# Set it back to empty if loaded with blanks from scanning #
@ -163,11 +170,15 @@ function LintCodebase() {
elif [[ ${FILE} == *".rbenv"* ]]; then
# This is likely the ruby environment folder and shouldn't be parsed
continue
elif [[ ${FILE_TYPE} == "BASH" ]] || [[ ${FILE_TYPE} == "SHELL_SHFMT" ]]; then
if CheckZsh "${FILE}"; then
# ZSH file and we need to skip
warn "$LINTER_NAME does NOT currently support zsh, skipping file"
continue
elif [[ ${FILE_TYPE} == "BASH" ]] && ! IsValidShellScript "${FILE}"; then
# not a valid script and we need to skip
continue
elif [[ ${FILE_TYPE} == "BASH_EXEC" ]] && ! IsValidShellScript "${FILE}"; then
# not a valid script and we need to skip
continue
elif [[ ${FILE_TYPE} == "SHELL_SHFMT" ]] && ! IsValidShellScript "${FILE}"; then
# not a valid script and we need to skip
continue
fi
##################################