Simplify linter and worker by generalizing ansible-lint (#1035)

Co-authored-by: Lukas Gravley <admiralawkbar@github.com>

lib/functions/buildFileList.sh

@@ -21,6 +21,9 @@ function BuildFileList() {
   TEST_CASE_RUN="${2}"
   debug "TEST_CASE_RUN: ${TEST_CASE_RUN}..."
 
+  ANSIBLE_DIRECTORY="${3}"
+  debug "ANSIBLE_DIRECTORY: ${ANSIBLE_DIRECTORY}..."
+
   if [ "${VALIDATE_ALL_CODEBASE}" == "false" ] && [ "${TEST_CASE_RUN}" != "true" ]; then
     # Need to build a list of all files changed
     # This can be pulled from the GITHUB_EVENT_PATH payload
@@ -346,6 +349,16 @@ function BuildFileList() {
       # Append the file to the array #
       ################################
       FILE_ARRAY_JSON+=("${FILE}")
+
+      ############################
+      # Check if file is Ansible #
+      ############################
+      if DetectAnsibleFile "${ANSIBLE_DIRECTORY}" "${FILE}"; then
+        ################################
+        # Append the file to the array #
+        ################################
+        FILE_ARRAY_ANSIBLE+=("${FILE}")
+      fi
       ############################
       # Check if file is OpenAPI #
       ############################
@@ -587,6 +600,20 @@ function BuildFileList() {
       ################################
       FILE_ARRAY_YAML+=("${FILE}")
 
+      ############################
+      # Check if file is Ansible #
+      ############################
+      if [ -d "${ANSIBLE_DIRECTORY}" ]; then
+        if DetectAnsibleFile "${ANSIBLE_DIRECTORY}" "${FILE}"; then
+          ################################
+          # Append the file to the array #
+          ################################
+          FILE_ARRAY_ANSIBLE+=("${FILE}")
+        fi
+      else
+        debug "ANSIBLE_DIRECTORY (${ANSIBLE_DIRECTORY}) does NOT exist."
+      fi
+
       #####################################
       # Check if the file is CFN template #
       #####################################

lib/functions/detectFiles.sh

@@ -8,6 +8,25 @@
 ########################## FUNCTION CALLS BELOW ################################
 ################################################################################
 ################################################################################
+
+DetectAnsibleFile() {
+  ANSIBLE_DIRECTORY="${1}"
+  FILE="${2}"
+
+  debug "Checking if ${FILE} is an Ansible file. Ansible directory: ${ANSIBLE_DIRECTORY}..."
+
+  if [[ ${FILE} == *"vault.yml" ]] || [[ ${FILE} == *"galaxy.yml" ]] || [[ ${FILE} == *"vault.yaml" ]] || [[ ${FILE} == *"galaxy.yaml" ]]; then
+    debug "${FILE} is a file that super-linter ignores. Ignoring it..."
+    return 1
+  elif [[ "$(dirname "${FILE}")" == "${ANSIBLE_DIRECTORY}" ]]; then
+    debug "${FILE} is an Ansible-related file."
+    return 0
+  else
+    debug "${FILE} is NOT an Ansible-related file."
+    return 1
+  fi
+}
+
 #### Function DetectOpenAPIFile ################################################
 DetectOpenAPIFile() {
   ################

lib/functions/worker.sh

@@ -384,157 +384,3 @@ function LintCodebase() {
     fi
   fi
 }
-################################################################################
-#### Function LintAnsibleFiles #################################################
-function LintAnsibleFiles() {
-  ######################
-  # Create Print Array #
-  ######################
-  PRINT_ARRAY=()
-
-  ################
-  # print header #
-  ################
-  PRINT_ARRAY+=("")
-  PRINT_ARRAY+=("----------------------------------------------")
-  PRINT_ARRAY+=("----------------------------------------------")
-  PRINT_ARRAY+=("Linting [Ansible] files...")
-  PRINT_ARRAY+=("----------------------------------------------")
-  PRINT_ARRAY+=("----------------------------------------------")
-
-  ######################
-  # Name of the linter #
-  ######################
-  LINTER_NAME="ansible-lint"
-
-  ##########################
-  # Initialize empty Array #
-  ##########################
-  LIST_FILES=()
-
-  #######################
-  # Create flag to skip #
-  #######################
-  SKIP_FLAG=0
-
-  ######################################################
-  # Only go into ansible linter if we have base folder #
-  ######################################################
-  if [ -d "${ANSIBLE_DIRECTORY}" ]; then
-
-    #################################
-    # Get list of all files to lint #
-    #################################
-    mapfile -t LIST_FILES < <(find "${ANSIBLE_DIRECTORY}" -path "*/node_modules" -prune -o -type f -regex ".*\.\(yml\|yaml\|json\)\$" 2>&1)
-
-    ####################################
-    # Check if we have data to look at #
-    ####################################
-    if [ ${SKIP_FLAG} -eq 0 ]; then
-      for LINE in "${PRINT_ARRAY[@]}"; do
-        #########################
-        # Print the header line #
-        #########################
-        info "${LINE}"
-      done
-    fi
-
-    ########################################
-    # Prepare context if TAP output format #
-    ########################################
-    if IsTAP; then
-      TMPFILE=$(mktemp -q "/tmp/super-linter-${FILE_TYPE}.XXXXXX")
-      INDEX=0
-      mkdir -p "${REPORT_OUTPUT_FOLDER}"
-      REPORT_OUTPUT_FILE="${REPORT_OUTPUT_FOLDER}/super-linter-${FILE_TYPE}.${OUTPUT_FORMAT}"
-    fi
-
-    ##################
-    # Lint the files #
-    ##################
-    for FILE in "${LIST_FILES[@]}"; do
-
-      ########################################
-      # Make sure we dont lint certain files #
-      ########################################
-      if [[ ${FILE} == *"vault.yml"* ]] || [[ ${FILE} == *"galaxy.yml"* ]] || [[ ${FILE} == *"vault.yaml"* ]] || [[ ${FILE} == *"galaxy.yaml"* ]]; then
-        # This is a file we dont look at
-        continue
-      fi
-
-      ##################################
-      # Increase the linted file index #
-      ##################################
-      (("INDEX++"))
-
-      ####################
-      # Get the filename #
-      ####################
-      FILE_NAME=$(basename "${ANSIBLE_DIRECTORY}/${FILE}" 2>&1)
-
-      ##############
-      # File print #
-      ##############
-      info "---------------------------"
-      info "File:[${FILE}]"
-
-      ################################
-      # Lint the file with the rules #
-      ################################
-      LINT_CMD=$("${LINTER_NAME}" -v -c "${ANSIBLE_LINTER_RULES}" "${ANSIBLE_DIRECTORY}/${FILE}" 2>&1)
-
-      #######################
-      # Load the error code #
-      #######################
-      ERROR_CODE=$?
-
-      ##############################
-      # Check the shell for errors #
-      ##############################
-      if [ ${ERROR_CODE} -ne 0 ]; then
-        #########
-        # Error #
-        #########
-        error "Found errors in [${LINTER_NAME}] linter!"
-        error "[${LINT_CMD}]"
-        # Increment error count
-        ((ERRORS_FOUND_ANSIBLE++))
-
-        #######################################################
-        # Store the linting as a temporary file in TAP format #
-        #######################################################
-        if IsTAP; then
-          NotOkTap "${INDEX}" "${FILE_NAME}" "${TMPFILE}"
-          AddDetailedMessageIfEnabled "${LINT_CMD}" "${TMPFILE}"
-        fi
-
-      else
-        ###########
-        # Success #
-        ###########
-        info " - File:${F[W]}[${FILE_NAME}]${F[B]} was linted with ${F[W]}[${LINTER_NAME}]${F[B]} successfully"
-
-        #######################################################
-        # Store the linting as a temporary file in TAP format #
-        #######################################################
-        if IsTAP; then
-          OkTap "${INDEX}" "${FILE_NAME}" "${TMPFILE}"
-        fi
-      fi
-    done
-
-    #################################
-    # Generate report in TAP format #
-    #################################
-    if IsTAP && [ ${INDEX} -gt 0 ]; then
-      HeaderTap "${INDEX}" "${REPORT_OUTPUT_FILE}"
-      cat "${TMPFILE}" >>"${REPORT_OUTPUT_FILE}"
-    fi
-  else
-    ########################
-    # No Ansible dir found #
-    ########################
-    warn "No Ansible base directory found at:[${ANSIBLE_DIRECTORY}]"
-    debug "skipping ansible lint"
-  fi
-}

lib/linter.sh

@@ -724,6 +724,7 @@ GetStandardRules "typescript"
 # Define linter commands #
 ##########################
 declare -A LINTER_COMMANDS_ARRAY
+LINTER_COMMANDS_ARRAY['ANSIBLE']="ansible-lint -v -c ${ANSIBLE_LINTER_RULES}"
 LINTER_COMMANDS_ARRAY['ARM']="Import-Module ${ARM_TTK_PSD1} ; \${config} = \$(Import-PowerShellDataFile -Path ${ARM_LINTER_RULES}) ; Test-AzTemplate @config -TemplatePath"
 LINTER_COMMANDS_ARRAY['BASH']="shellcheck --color --external-sources"
 LINTER_COMMANDS_ARRAY['BASH_EXEC']="bash-exec"
@@ -792,7 +793,7 @@ debug "---------------------------------------------"
 ###########################################
 # Build the list of files for each linter #
 ###########################################
-BuildFileList "${VALIDATE_ALL_CODEBASE}" "${TEST_CASE_RUN}"
+BuildFileList "${VALIDATE_ALL_CODEBASE}" "${TEST_CASE_RUN}" "${ANSIBLE_DIRECTORY}"
 
 ###############
 # Run linters #
@@ -834,36 +835,29 @@ for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do
       cd "${GITHUB_WORKSPACE}" && zef install --deps-only --/test .
     fi
 
-    if [ "${LANGUAGE}" = "ANSIBLE" ]; then
+    LINTER_NAME="${LINTER_NAMES_ARRAY["${LANGUAGE}"]}"
-      # Due to the nature of how we want to validate Ansible, we cannot use the
+    if [ -z "${LINTER_NAME}" ]; then
-      # standard loop, since it looks for an ansible folder, excludes certain
+      fatal "Cannot find the linter name for ${LANGUAGE} language."
-      # files, and looks for additional changes, it should be an outlier
-      LintAnsibleFiles "${ANSIBLE_LINTER_RULES}" # Passing rules but not needed, dont want to exclude unused var
     else
-      LINTER_NAME="${LINTER_NAMES_ARRAY["${LANGUAGE}"]}"
+      debug "Setting LINTER_NAME to ${LINTER_NAME}..."
-      if [ -z "${LINTER_NAME}" ]; then
-        fatal "Cannot find the linter name for ${LANGUAGE} language."
-      else
-        debug "Setting LINTER_NAME to ${LINTER_NAME}..."
-      fi
-
-      LINTER_COMMAND="${LINTER_COMMANDS_ARRAY["${LANGUAGE}"]}"
-      if [ -z "${LINTER_COMMAND}" ]; then
-        fatal "Cannot find the linter command for ${LANGUAGE} language."
-      else
-        debug "Setting LINTER_COMMAND to ${LINTER_COMMAND}..."
-      fi
-
-      FILE_ARRAY_VARIABLE_NAME="FILE_ARRAY_${LANGUAGE}"
-      debug "Setting FILE_ARRAY_VARIABLE_NAME to ${FILE_ARRAY_VARIABLE_NAME}..."
-
-      # shellcheck disable=SC2125
-      LANGUAGE_FILE_ARRAY="${FILE_ARRAY_VARIABLE_NAME}"[@]
-      debug "${FILE_ARRAY_VARIABLE_NAME} file array contents: ${!LANGUAGE_FILE_ARRAY}"
-
-      debug "Invoking ${LINTER_NAME} linter. TEST_CASE_RUN: ${TEST_CASE_RUN}"
-      LintCodebase "${LANGUAGE}" "${LINTER_NAME}" "${LINTER_COMMAND}" "${FILTER_REGEX_INCLUDE}" "${FILTER_REGEX_EXCLUDE}" "${TEST_CASE_RUN}" "${!LANGUAGE_FILE_ARRAY}"
     fi
+
+    LINTER_COMMAND="${LINTER_COMMANDS_ARRAY["${LANGUAGE}"]}"
+    if [ -z "${LINTER_COMMAND}" ]; then
+      fatal "Cannot find the linter command for ${LANGUAGE} language."
+    else
+      debug "Setting LINTER_COMMAND to ${LINTER_COMMAND}..."
+    fi
+
+    FILE_ARRAY_VARIABLE_NAME="FILE_ARRAY_${LANGUAGE}"
+    debug "Setting FILE_ARRAY_VARIABLE_NAME to ${FILE_ARRAY_VARIABLE_NAME}..."
+
+    # shellcheck disable=SC2125
+    LANGUAGE_FILE_ARRAY="${FILE_ARRAY_VARIABLE_NAME}"[@]
+    debug "${FILE_ARRAY_VARIABLE_NAME} file array contents: ${!LANGUAGE_FILE_ARRAY}"
+
+    debug "Invoking ${LINTER_NAME} linter. TEST_CASE_RUN: ${TEST_CASE_RUN}"
+    LintCodebase "${LANGUAGE}" "${LINTER_NAME}" "${LINTER_COMMAND}" "${FILTER_REGEX_INCLUDE}" "${FILTER_REGEX_EXCLUDE}" "${TEST_CASE_RUN}" "${!LANGUAGE_FILE_ARRAY}"
   fi
 done