mirror of
https://github.com/super-linter/super-linter.git
synced 2024-12-22 12:42:09 -05:00
feat: don't inspect files if not needed (#5094)
Don't run potentially expensive I/O operations to check file types if we're not going to analyze them anyway.
This commit is contained in:
Marco Ferrari
GitHub
parent
cbbf484361
commit
e62b382bf0
2 changed files with 113 additions and 192 deletions
39
Makefile
39
Makefile
|
|
@ -4,7 +4,7 @@
|
|||
all: info docker test ## Run all targets.
|
||||
|
||||
.PHONY: test
|
||||
test: info validate-container-image-labels test-lib inspec lint-codebase test-default-config-files test-find test-linters ## Run the test suite
|
||||
test: info validate-container-image-labels test-lib inspec lint-codebase test-default-config-files test-find lint-subset-files test-linters ## Run the test suite
|
||||
|
||||
# if this session isn't interactive, then we don't want to allocate a
|
||||
# TTY, which would fail, but if it is interactive, we do want to attach
|
||||
|
|
@ -153,6 +153,43 @@ lint-codebase: ## Lint the entire codebase
|
|||
-v "$(CURDIR):/tmp/lint" \
|
||||
$(SUPER_LINTER_TEST_CONTAINER_URL)
|
||||
|
||||
# This is a smoke test to check how much time it takes to lint only a small
|
||||
# subset of files, compared to linting the whole codebase.
|
||||
.phony: lint-subset-files
|
||||
lint-subset-files: lint-subset-files-enable-only-one-type lint-subset-files-enable-expensive-io-checks
|
||||
|
||||
.phony: lint-subset-files-enable-only-one-type
|
||||
lint-subset-files-enable-only-one-type: ## Lint a small subset of files in the codebase by enabling only one linter
|
||||
time docker run \
|
||||
-e RUN_LOCAL=true \
|
||||
-e ACTIONS_RUNNER_DEBUG=true \
|
||||
-e DEFAULT_BRANCH=main \
|
||||
-e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true \
|
||||
-e ERROR_ON_MISSING_EXEC_BIT=true \
|
||||
-e VALIDATE_ALL_CODEBASE=true \
|
||||
-e VALIDATE_MARKDOWN=true \
|
||||
-v "$(CURDIR):/tmp/lint" \
|
||||
$(SUPER_LINTER_TEST_CONTAINER_URL)
|
||||
|
||||
.phony: lint-subset-files-enable-expensive-io-checks
|
||||
lint-subset-files-enable-expensive-io-checks: ## Lint a small subset of files in the codebase and keep expensive I/O operations to check file types enabled
|
||||
time docker run \
|
||||
-e RUN_LOCAL=true \
|
||||
-e ACTIONS_RUNNER_DEBUG=true \
|
||||
-e DEFAULT_BRANCH=main \
|
||||
-e ENABLE_GITHUB_ACTIONS_GROUP_TITLE=true \
|
||||
-e ERROR_ON_MISSING_EXEC_BIT=true \
|
||||
-e VALIDATE_ALL_CODEBASE=true \
|
||||
-e VALIDATE_ARM=true \
|
||||
-e VALIDATE_CLOUDFORMATION=true \
|
||||
-e VALIDATE_KUBERNETES_KUBECONFORM=true \
|
||||
-e VALIDATE_MARKDOWN=true \
|
||||
-e VALIDATE_OPENAPI=true \
|
||||
-e VALIDATE_STATES=true \
|
||||
-e VALIDATE_TEKTON=true \
|
||||
-v "$(CURDIR):/tmp/lint" \
|
||||
$(SUPER_LINTER_TEST_CONTAINER_URL)
|
||||
|
||||
.phony: test-lib
|
||||
test-lib: test-build-file-list test-github-event test-validation ## Test super-linter
|
||||
|
||||
|
|
|
|||
|
|
@ -1,16 +1,13 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
################################################################################
|
||||
################################################################################
|
||||
########### Super-Linter linting Functions @admiralawkbar ######################
|
||||
################################################################################
|
||||
################################################################################
|
||||
########################## FUNCTION CALLS BELOW ################################
|
||||
################################################################################
|
||||
#### Function DetectActions ####################################################
|
||||
DetectActions() {
|
||||
FILE="${1}"
|
||||
|
||||
if [ "${VALIDATE_GITHUB_ACTIONS}" == "false" ]; then
|
||||
debug "Don't check if ${FILE} is a GitHub Actions file because VALIDATE_GITHUB_ACTIONS is: ${VALIDATE_GITHUB_ACTIONS}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
debug "Checking if ${FILE} is a GitHub Actions file..."
|
||||
|
||||
# Check if in the users .github, or the super linter test suite
|
||||
|
|
@ -22,29 +19,18 @@ DetectActions() {
|
|||
return 1
|
||||
fi
|
||||
}
|
||||
################################################################################
|
||||
#### Function DetectOpenAPIFile ################################################
|
||||
|
||||
DetectOpenAPIFile() {
|
||||
################
|
||||
# Pull in vars #
|
||||
################
|
||||
FILE="${1}"
|
||||
|
||||
if [ "${VALIDATE_OPENAPI}" == "false" ]; then
|
||||
debug "Don't check if ${FILE} is an OpenAPI file because VALIDATE_OPENAPI is: ${VALIDATE_OPENAPI}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
debug "Checking if ${FILE} is an OpenAPI file..."
|
||||
|
||||
###############################
|
||||
# Check the file for keywords #
|
||||
###############################
|
||||
grep -E '"openapi":|"swagger":|^openapi:|^swagger:' "${FILE}" >/dev/null
|
||||
|
||||
#######################
|
||||
# Load the error code #
|
||||
#######################
|
||||
ERROR_CODE=$?
|
||||
|
||||
##############################
|
||||
# Check the shell for errors #
|
||||
##############################
|
||||
if [ ${ERROR_CODE} -eq 0 ]; then
|
||||
if grep -E '"openapi":|"swagger":|^openapi:|^swagger:' "${FILE}" >/dev/null; then
|
||||
debug "${FILE} is an OpenAPI descriptor"
|
||||
return 0
|
||||
else
|
||||
|
|
@ -52,113 +38,75 @@ DetectOpenAPIFile() {
|
|||
return 1
|
||||
fi
|
||||
}
|
||||
################################################################################
|
||||
#### Function DetectTektonFile #################################################
|
||||
|
||||
DetectTektonFile() {
|
||||
################
|
||||
# Pull in vars #
|
||||
################
|
||||
FILE="${1}"
|
||||
|
||||
if [ "${VALIDATE_TEKTON}" == "false" ]; then
|
||||
debug "Don't check if ${FILE} is a Tekton file because VALIDATE_TEKTON is: ${VALIDATE_TEKTON}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
debug "Checking if ${FILE} is a Tekton file..."
|
||||
|
||||
###############################
|
||||
# Check the file for keywords #
|
||||
###############################
|
||||
grep -q -E 'apiVersion: tekton' "${FILE}" >/dev/null
|
||||
|
||||
#######################
|
||||
# Load the error code #
|
||||
#######################
|
||||
ERROR_CODE=$?
|
||||
|
||||
##############################
|
||||
# Check the shell for errors #
|
||||
##############################
|
||||
if [ ${ERROR_CODE} -eq 0 ]; then
|
||||
########################
|
||||
# Found string in file #
|
||||
########################
|
||||
if grep -q -E 'apiVersion: tekton' "${FILE}" >/dev/null; then
|
||||
return 0
|
||||
else
|
||||
###################
|
||||
# No string match #
|
||||
###################
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
################################################################################
|
||||
#### Function DetectARMFile ####################################################
|
||||
|
||||
DetectARMFile() {
|
||||
################
|
||||
# Pull in vars #
|
||||
################
|
||||
FILE="${1}" # Name of the file/path we are validating
|
||||
FILE="${1}"
|
||||
|
||||
if [ "${VALIDATE_ARM}" == "false" ]; then
|
||||
debug "Don't check if ${FILE} is an ARM file because VALIDATE_ARM is: ${VALIDATE_ARM}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
debug "Checking if ${FILE} is an ARM file..."
|
||||
|
||||
###############################
|
||||
# Check the file for keywords #
|
||||
###############################
|
||||
grep -E 'schema.management.azure.com' "${FILE}" >/dev/null
|
||||
|
||||
#######################
|
||||
# Load the error code #
|
||||
#######################
|
||||
ERROR_CODE=$?
|
||||
|
||||
##############################
|
||||
# Check the shell for errors #
|
||||
##############################
|
||||
if [ ${ERROR_CODE} -eq 0 ]; then
|
||||
########################
|
||||
# Found string in file #
|
||||
########################
|
||||
if grep -E 'schema.management.azure.com' "${FILE}" >/dev/null; then
|
||||
return 0
|
||||
else
|
||||
###################
|
||||
# No string match #
|
||||
###################
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
################################################################################
|
||||
#### Function DetectCloudFormationFile #########################################
|
||||
|
||||
DetectCloudFormationFile() {
|
||||
################
|
||||
# Pull in Vars #
|
||||
################
|
||||
FILE="${1}" # File that we need to validate
|
||||
FILE="${1}"
|
||||
|
||||
if [ "${VALIDATE_CLOUDFORMATION}" == "false" ]; then
|
||||
debug "Don't check if ${FILE} is a CloudFormation file because VALIDATE_CLOUDFORMATION is: ${VALIDATE_CLOUDFORMATION}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
debug "Checking if ${FILE} is a Cloud Formation file..."
|
||||
|
||||
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/template-formats.html
|
||||
# AWSTemplateFormatVersion is optional
|
||||
#######################################
|
||||
# Check if file has AWS Template info #
|
||||
#######################################
|
||||
|
||||
# Check if file has AWS Template info
|
||||
if grep -q 'AWSTemplateFormatVersion' "${FILE}" >/dev/null; then
|
||||
# Found it
|
||||
return 0
|
||||
fi
|
||||
|
||||
#####################################
|
||||
# See if it contains AWS References #
|
||||
#####################################
|
||||
# See if it contains AWS References
|
||||
if grep -q -E '(AWS|Alexa|Custom)::' "${FILE}" >/dev/null; then
|
||||
# Found it
|
||||
return 0
|
||||
fi
|
||||
|
||||
#####################################################
|
||||
# No identifiers of a CLOUDFORMATION template found #
|
||||
#####################################################
|
||||
return 1
|
||||
}
|
||||
################################################################################
|
||||
#### Function DetectKubernetesFile #########################################
|
||||
|
||||
DetectKubernetesFile() {
|
||||
################
|
||||
# Pull in Vars #
|
||||
################
|
||||
FILE="${1}" # File that we need to validate
|
||||
FILE="${1}"
|
||||
|
||||
if [ "${VALIDATE_KUBERNETES_KUBECONFORM}" == "false" ]; then
|
||||
debug "Don't check if ${FILE} is a Kubernetes file because VALIDATE_KUBERNETES_KUBECONFORM is: ${VALIDATE_KUBERNETES_KUBECONFORM}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
debug "Checking if ${FILE} is a Kubernetes descriptor..."
|
||||
if grep -q -v 'kustomize.config.k8s.io' "${FILE}" &&
|
||||
grep -q -v "tekton" "${FILE}" &&
|
||||
|
|
@ -171,36 +119,27 @@ DetectKubernetesFile() {
|
|||
debug "${FILE} is NOT a Kubernetes descriptor"
|
||||
return 1
|
||||
}
|
||||
################################################################################
|
||||
#### Function DetectAWSStatesFIle ##############################################
|
||||
|
||||
DetectAWSStatesFIle() {
|
||||
################
|
||||
# Pull in Vars #
|
||||
################
|
||||
FILE="${1}" # File that we need to validate
|
||||
FILE="${1}"
|
||||
|
||||
if [ "${VALIDATE_STATES}" == "false" ]; then
|
||||
debug "Don't check if ${FILE} is an AWS states file because VALIDATE_STATES is: ${VALIDATE_STATES}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
debug "Checking if ${FILE} is a AWS states descriptor..."
|
||||
|
||||
# https://states-language.net/spec.html#example
|
||||
###############################
|
||||
# check if file has resources #
|
||||
###############################
|
||||
if grep -q '"Resource": *"arn' "${FILE}" &&
|
||||
grep -q '"States"' "${FILE}"; then
|
||||
# Found it
|
||||
return 0
|
||||
fi
|
||||
|
||||
#################################################
|
||||
# No identifiers of a AWS States Language found #
|
||||
#################################################
|
||||
return 1
|
||||
}
|
||||
################################################################################
|
||||
#### Function CheckInArray #####################################################
|
||||
|
||||
CheckInArray() {
|
||||
###############
|
||||
# Pull in Var #
|
||||
###############
|
||||
NEEDLE="$1" # Language we need to match
|
||||
|
||||
######################################
|
||||
|
|
@ -208,57 +147,33 @@ CheckInArray() {
|
|||
######################################
|
||||
for LANG in "${UNIQUE_LINTED_ARRAY[@]}"; do
|
||||
if [[ "${LANG}" == "${NEEDLE}" ]]; then
|
||||
############
|
||||
# Found it #
|
||||
############
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
###################
|
||||
# Did not find it #
|
||||
###################
|
||||
return 1
|
||||
}
|
||||
################################################################################
|
||||
#### Function GetFileType ######################################################
|
||||
|
||||
function GetFileType() {
|
||||
# Need to run the file through the 'file' exec to help determine
|
||||
# The type of file being parsed
|
||||
|
||||
################
|
||||
# Pull in Vars #
|
||||
################
|
||||
FILE="$1"
|
||||
|
||||
##################
|
||||
# Check the file #
|
||||
##################
|
||||
GET_FILE_TYPE_CMD=$(file "${FILE}" 2>&1)
|
||||
|
||||
echo "${GET_FILE_TYPE_CMD}"
|
||||
}
|
||||
################################################################################
|
||||
#### Function CheckFileType ####################################################
|
||||
|
||||
function CheckFileType() {
|
||||
# Need to run the file through the 'file' exec to help determine
|
||||
# The type of file being parsed
|
||||
|
||||
################
|
||||
# Pull in Vars #
|
||||
################
|
||||
FILE="$1"
|
||||
|
||||
#################
|
||||
# Get file type #
|
||||
#################
|
||||
GET_FILE_TYPE_CMD="$(GetFileType "$FILE")"
|
||||
|
||||
if [[ ${GET_FILE_TYPE_CMD} == *"Ruby script"* ]]; then
|
||||
if [ "${SUPPRESS_FILE_TYPE_WARN}" == "false" ]; then
|
||||
#######################
|
||||
# It is a Ruby script #
|
||||
#######################
|
||||
warn "Found ruby script without extension:[.rb]"
|
||||
info "Please update file with proper extensions."
|
||||
fi
|
||||
|
|
@ -269,9 +184,6 @@ function CheckFileType() {
|
|||
FILE_ARRAY_RUBY+=("${FILE}")
|
||||
elif [[ ${GET_FILE_TYPE_CMD} == *"Python script"* ]]; then
|
||||
if [ "${SUPPRESS_FILE_TYPE_WARN}" == "false" ]; then
|
||||
#########################
|
||||
# It is a Python script #
|
||||
#########################
|
||||
warn "Found Python script without extension:[.py]"
|
||||
info "Please update file with proper extensions."
|
||||
fi
|
||||
|
|
@ -282,9 +194,6 @@ function CheckFileType() {
|
|||
FILE_ARRAY_PYTHON+=("${FILE}")
|
||||
elif [[ ${GET_FILE_TYPE_CMD} == *"Perl script"* ]]; then
|
||||
if [ "${SUPPRESS_FILE_TYPE_WARN}" == "false" ]; then
|
||||
#######################
|
||||
# It is a Perl script #
|
||||
#######################
|
||||
warn "Found Perl script without extension:[.pl]"
|
||||
info "Please update file with proper extensions."
|
||||
fi
|
||||
|
|
@ -300,12 +209,8 @@ function CheckFileType() {
|
|||
debug "Failed to get filetype for:[${FILE}]!"
|
||||
fi
|
||||
}
|
||||
################################################################################
|
||||
#### Function GetFileExtension ###############################################
|
||||
|
||||
function GetFileExtension() {
|
||||
################
|
||||
# Pull in Vars #
|
||||
################
|
||||
FILE="$1"
|
||||
|
||||
###########################
|
||||
|
|
@ -318,17 +223,15 @@ function GetFileExtension() {
|
|||
|
||||
echo "$FILE_TYPE"
|
||||
}
|
||||
################################################################################
|
||||
#### Function IsValidShellScript ###############################################
|
||||
|
||||
function IsValidShellScript() {
|
||||
################
|
||||
# Pull in Vars #
|
||||
################
|
||||
FILE="$1"
|
||||
|
||||
#################
|
||||
# Get file type #
|
||||
#################
|
||||
if [ "${VALIDATE_BASH}" == "false" ] && [ "${VALIDATE_BASH_EXEC}" == "false" ] && [ "${VALIDATE_SHELL_SHFMT}" == "false" ]; then
|
||||
debug "Don't check if ${FILE} is a shell script because VALIDATE_BASH, VALIDATE_BASH_EXEC, and VALIDATE_SHELL_SHFMT are set to: ${VALIDATE_BASH}, ${VALIDATE_BASH_EXEC}, ${VALIDATE_SHELL_SHFMT}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
FILE_EXTENSION="$(GetFileExtension "$FILE")"
|
||||
GET_FILE_TYPE_CMD="$(GetFileType "$FILE")"
|
||||
|
||||
|
|
@ -361,39 +264,21 @@ function IsValidShellScript() {
|
|||
trace "$FILE is NOT a supported shell script. Skipping"
|
||||
return 1
|
||||
}
|
||||
################################################################################
|
||||
#### Function IsGenerated ######################################################
|
||||
|
||||
function IsGenerated() {
|
||||
# Pull in Vars #
|
||||
################
|
||||
FILE="$1"
|
||||
|
||||
##############################
|
||||
# Check the file for keyword #
|
||||
##############################
|
||||
grep -q "@generated" "$FILE"
|
||||
if [ "${IGNORE_GENERATED_FILES}" == "false" ]; then
|
||||
debug "Don't check if ${FILE} is generated because IGNORE_GENERATED_FILES is: ${IGNORE_GENERATED_FILES}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
#######################
|
||||
# Load the error code #
|
||||
#######################
|
||||
ERROR_CODE=$?
|
||||
|
||||
if [ ${ERROR_CODE} -ne 0 ]; then
|
||||
if ! grep -q "@generated" "$FILE"; then
|
||||
trace "File:[${FILE}] is not generated, because it doesn't have @generated marker"
|
||||
return 1
|
||||
fi
|
||||
|
||||
##############################
|
||||
# Check the file for keyword #
|
||||
##############################
|
||||
grep -q "@not-generated" "$FILE"
|
||||
|
||||
#######################
|
||||
# Load the error code #
|
||||
#######################
|
||||
ERROR_CODE=$?
|
||||
|
||||
if [ ${ERROR_CODE} -eq 0 ]; then
|
||||
if grep -q "@not-generated" "$FILE"; then
|
||||
trace "File:[${FILE}] is not-generated because it has @not-generated marker"
|
||||
return 1
|
||||
else
|
||||
|
|
@ -401,8 +286,7 @@ function IsGenerated() {
|
|||
return 0
|
||||
fi
|
||||
}
|
||||
################################################################################
|
||||
#### Function RunAdditionalInstalls ############################################
|
||||
|
||||
function RunAdditionalInstalls() {
|
||||
##################################
|
||||
# Run installs for Psalm and PHP #
|
||||
|
|
|
|||
Loading…
Reference in a new issue