fix: fix file list when looking for changes (#5044)

- Fix the file diff function on push events.
- Implement a test for the file diff function

Makefile

@@ -4,7 +4,7 @@
 all: info docker test ## Run all targets.
 
 .PHONY: test
-test: info validate-container-image-labels inspec lint-codebase test-find test-linters ## Run the test suite
+test: info validate-container-image-labels test-lib inspec lint-codebase test-find test-linters ## Run the test suite
 
 # if this session isn't interactive, then we don't want to allocate a
 # TTY, which would fail, but if it is interactive, we do want to attach
@@ -152,6 +152,17 @@ lint-codebase: ## Lint the entire codebase
 		-v "$(CURDIR):/tmp/lint" \
 		$(SUPER_LINTER_TEST_CONTAINER_URL)
 
+.phony: test-lib
+test-lib: test-build-file-list ## Test super-linter
+
+.phony: test-build-file-list
+test-build-file-list: ## Test buildFileList
+	docker run \
+		-v "$(CURDIR):/tmp/lint" \
+		-w /tmp/lint \
+		--entrypoint /tmp/lint/test/lib/buildFileListTest.sh \
+		$(SUPER_LINTER_TEST_CONTAINER_URL)
+
 .phony: test-linters
 test-linters: ## Run the linters test suite
 	docker run \

lib/functions/buildFileList.sh

@@ -1,24 +1,28 @@
 #!/usr/bin/env bash
 
-################################################################################
-################################################################################
-########### Super-Linter Build File List Functions @admiralawkbar ##############
-################################################################################
-################################################################################
-########################## FUNCTION CALLS BELOW ################################
-################################################################################
-
-################################################################################
-#### Function BuildFileList ####################################################
 function IssueHintForFullGitHistory() {
   info "Check that you have the full git history, the checkout is not shallow, etc"
   info "Is shallow repository: $(git -C "${GITHUB_WORKSPACE}" rev-parse --is-shallow-repository)"
   info "See https://github.com/super-linter/super-linter#example-connecting-github-action-workflow"
 }
 
-################################################################################
-#### Function GenerateFileDiff #################################################
 function GenerateFileDiff() {
+  DIFF_GIT_DEFAULT_BRANCH_CMD="git -C \"${GITHUB_WORKSPACE}\" diff --diff-filter=d --name-only ${DEFAULT_BRANCH}...${GITHUB_SHA} | xargs -I % sh -c 'echo \"${GITHUB_WORKSPACE}/%\"' 2>&1"
+  DIFF_TREE_CMD="git -C \"${GITHUB_WORKSPACE}\" diff-tree --no-commit-id --name-only -r ${GITHUB_SHA} ${GITHUB_BEFORE_SHA} | xargs -I % sh -c 'echo \"${GITHUB_WORKSPACE}/%\"' 2>&1"
+
+  if [ "${GITHUB_EVENT_NAME:-}" == "push" ]; then
+    RunFileDiffCommand "${DIFF_TREE_CMD}"
+    if [ ${#RAW_FILE_ARRAY[@]} -eq 0 ]; then
+      debug "----------------------------------------------"
+      debug "Generating the file array with diff-tree produced [0] items, trying with git diff against the default branch..."
+      RunFileDiffCommand "${DIFF_GIT_DEFAULT_BRANCH_CMD}"
+    fi
+  else
+    RunFileDiffCommand "${DIFF_GIT_DEFAULT_BRANCH_CMD}"
+  fi
+}
+
+function RunFileDiffCommand() {
   CMD="${1}"
   debug "Generating Diff with:[$CMD]"
 
@@ -49,8 +53,6 @@ function GenerateFileDiff() {
   debug "RAW_FILE_ARRAY contents: ${RAW_FILE_ARRAY[*]}"
 }
 
-################################################################################
-#### Function BuildFileList ####################################################
 function BuildFileList() {
   debug "Building file list..."
 
@@ -70,29 +72,7 @@ function BuildFileList() {
     debug "----------------------------------------------"
     debug "Build the list of all changed files"
 
-    DIFF_GIT_DEFAULT_BRANCH_CMD="git -C ${GITHUB_WORKSPACE} diff --diff-filter=d --name-only ${DEFAULT_BRANCH}...${GITHUB_SHA} | xargs -I % sh -c 'echo \"${GITHUB_WORKSPACE}/%\"' 2>&1"
+    GenerateFileDiff
-
-    if [ "${GITHUB_EVENT_NAME}" == "push" ]; then
-      ################
-      # push event   #
-      ################
-      DIFF_TREE_CMD="git -C ${GITHUB_WORKSPACE} diff-tree --no-commit-id --name-only -r ${GITHUB_SHA} | xargs -I % sh -c 'echo \"${GITHUB_WORKSPACE}/%\"' 2>&1"
-      GenerateFileDiff "${DIFF_TREE_CMD}"
-
-      ###############################################################
-      # Need to see if the array is empty, if so, try the other way #
-      ###############################################################
-      if [ ${#RAW_FILE_ARRAY[@]} -eq 0 ]; then
-        debug "----------------------------------------------"
-        debug "Generating the file array with diff-tree produced [0] items, trying with git diff against the default branch..."
-        GenerateFileDiff "${DIFF_GIT_DEFAULT_BRANCH_CMD}"
-      fi
-    else
-      ################
-      # PR event     #
-      ################
-      GenerateFileDiff "${DIFF_GIT_DEFAULT_BRANCH_CMD}"
-    fi
   else
     WORKSPACE_PATH="${GITHUB_WORKSPACE}"
     if [ "${TEST_CASE_RUN}" == "true" ]; then
@@ -131,8 +111,9 @@ function BuildFileList() {
         2>&1 | sort)
     else
       debug "----------------------------------------------"
-      debug "Populating the file list with:[git -C \"${WORKSPACE_PATH}\" ls-tree -r --name-only HEAD | xargs -I % sh -c \"echo ${WORKSPACE_PATH}/%\"]"
+      DIFF_GIT_VALIDATE_ALL_CODEBASE="git -C \"${WORKSPACE_PATH}\" ls-tree -r --name-only HEAD | xargs -I % sh -c \"echo ${WORKSPACE_PATH}/%\" 2>&1"
-      mapfile -t RAW_FILE_ARRAY < <(git -C "${WORKSPACE_PATH}" ls-tree -r --name-only HEAD | xargs -I % sh -c "echo ${WORKSPACE_PATH}/%" 2>&1)
+      debug "Populating the file list with: ${DIFF_GIT_VALIDATE_ALL_CODEBASE}"
+      mapfile -t RAW_FILE_ARRAY < <(eval "set -eo pipefail; ${DIFF_GIT_VALIDATE_ALL_CODEBASE}; set +eo pipefail")
       debug "RAW_FILE_ARRAY contents: ${RAW_FILE_ARRAY[*]}"
     fi
   fi

lib/linter.sh

@@ -571,6 +571,21 @@ GetGitHubVars() {
         fatal "[Output: ${GITHUB_SHA}]"
       fi
       debug "Updated GITHUB_SHA: ${GITHUB_SHA}"
+    elif [ "${GITHUB_EVENT_NAME}" == "push" ]; then
+      debug "This is a GitHub push event."
+
+      GITHUB_BEFORE_SHA=$(jq -r .push.before <"$GITHUB_EVENT_PATH")
+      ERROR_CODE=$?
+      debug "GITHUB_BEFORE_SHA initialization error code: ${ERROR_CODE}"
+      if [ ${ERROR_CODE} -ne 0 ]; then
+        fatal "Failed to initialize GITHUB_BEFORE_SHA for a push event. Output: ${GITHUB_BEFORE_SHA}"
+      fi
+
+      if [ -z "${GITHUB_BEFORE_SHA}" ]; then
+        fatal "Failed to get GITHUB_BEFORE_SHA: [${GITHUB_BEFORE_SHA}]"
+      else
+        info "Successfully found:${F[W]}[GITHUB_BEFORE_SHA]${F[B]}, value:${F[W]}[${GITHUB_BEFORE_SHA}]"
+      fi
     fi
 
     ############################

test/lib/buildFileListTest.sh

@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# shellcheck disable=SC2034
+LOG_TRACE="true"
+# shellcheck disable=SC2034
+LOG_DEBUG="true"
+# shellcheck disable=SC2034
+LOG_VERBOSE="true"
+# shellcheck disable=SC2034
+LOG_NOTICE="true"
+# shellcheck disable=SC2034
+LOG_WARN="true"
+# shellcheck disable=SC2034
+LOG_ERROR="true"
+
+# shellcheck source=/dev/null
+source "lib/functions/log.sh"
+
+# shellcheck disable=SC2034
+DEFAULT_BRANCH=main
+
+git config --global init.defaultBranch "${DEFAULT_BRANCH}"
+git config --global user.email "super-linter@example.com"
+git config --global user.name "Super-linter"
+
+# shellcheck disable=SC2034
+CREATE_LOG_FILE=false
+
+function InitGitRepositoryAndCommitFiles() {
+  local REPOSITORY_PATH="${1}" && shift
+  local FILES_TO_COMMIT="${1}"
+
+  git -C "${REPOSITORY_PATH}" init
+  git -C "${REPOSITORY_PATH}" commit --allow-empty -m "Initial commit"
+  GITHUB_BEFORE_SHA=$(git -C "${REPOSITORY_PATH}" rev-parse HEAD)
+  debug "GITHUB_BEFORE_SHA: ${GITHUB_BEFORE_SHA}"
+
+  git -C "${REPOSITORY_PATH}" checkout -b test-branch
+
+  for ((i = 1; i <= FILES_TO_COMMIT; i++)); do
+    local TEST_FILE_PATH="${REPOSITORY_PATH}/test${i}.txt"
+    touch "${TEST_FILE_PATH}"
+    git -C "${REPOSITORY_PATH}" add .
+    git -C "${REPOSITORY_PATH}" commit -m "Add ${TEST_FILE_PATH}"
+  done
+
+  GITHUB_SHA=$(git -C "${REPOSITORY_PATH}" rev-parse HEAD)
+  debug "GITHUB_SHA: ${GITHUB_SHA}"
+  git -C "${REPOSITORY_PATH}" log --oneline "${DEFAULT_BRANCH}...${GITHUB_SHA}"
+}
+
+function GenerateFileDiffOneFileTest() {
+  local GITHUB_WORKSPACE
+  GITHUB_WORKSPACE="$(mktemp -d)"
+  echo "GITHUB_WORKSPACE: ${GITHUB_WORKSPACE}"
+
+  InitGitRepositoryAndCommitFiles "${GITHUB_WORKSPACE}" 1
+
+  # shellcheck source=/dev/null
+  source "lib/functions/buildFileList.sh"
+
+  GenerateFileDiff
+
+  RAW_FILE_ARRAY_SIZE=${#RAW_FILE_ARRAY[@]}
+  if [ "${RAW_FILE_ARRAY_SIZE}" -ne 1 ]; then
+    fatal "RAW_FILE_ARRAY does not have exactly one element: ${RAW_FILE_ARRAY_SIZE}"
+  fi
+
+  local FUNCTION_NAME
+  FUNCTION_NAME="${1:-${FUNCNAME[0]}}"
+  notice "${FUNCTION_NAME} PASS"
+}
+
+function GenerateFileDiffOneFilePushEventTest() {
+  # shellcheck disable=SC2034
+  local GITHUB_EVENT_NAME="push"
+  GenerateFileDiffOneFileTest "${FUNCNAME[0]}"
+}
+
+function GenerateFileDiffTwoFilesTest() {
+  local GITHUB_WORKSPACE
+  GITHUB_WORKSPACE="$(mktemp -d)"
+  debug "GITHUB_WORKSPACE: ${GITHUB_WORKSPACE}"
+  local FILES_TO_COMMIT=2
+
+  InitGitRepositoryAndCommitFiles "${GITHUB_WORKSPACE}" ${FILES_TO_COMMIT}
+
+  # shellcheck source=/dev/null
+  source "lib/functions/buildFileList.sh"
+
+  GenerateFileDiff
+
+  RAW_FILE_ARRAY_SIZE=${#RAW_FILE_ARRAY[@]}
+  if [ "${RAW_FILE_ARRAY_SIZE}" -ne 2 ]; then
+    fatal "RAW_FILE_ARRAY does not have exactly ${FILES_TO_COMMIT} elements: ${RAW_FILE_ARRAY_SIZE}"
+  fi
+
+  local FUNCTION_NAME
+  FUNCTION_NAME="${1:-${FUNCNAME[0]}}"
+  notice "${FUNCTION_NAME} PASS"
+}
+
+function GenerateFileDiffTwoFilesPushEventTest() {
+  # shellcheck disable=SC2034
+  local GITHUB_EVENT_NAME="push"
+  GenerateFileDiffTwoFilesTest "${FUNCNAME[0]}"
+}
+
+GenerateFileDiffOneFileTest
+GenerateFileDiffOneFilePushEventTest
+GenerateFileDiffTwoFilesTest
+GenerateFileDiffTwoFilesPushEventTest