Reduce duplication in CI and CD workflows (#4982)

* Reduce duplication in CI and CD workflows

* Fix indentation in README

* Load token from file

* Fix instructions

* Ignore test leftovers

.github/workflows/cd.yml

@@ -93,32 +93,6 @@ jobs:
       - name: Run Test Suite
         run: make test
 
-      - name: Run Super-Linter Tests
-        run: |
-          docker run \
-          -e RUN_LOCAL=true \
-          -e TEST_CASE_RUN=true \
-          -e ANSIBLE_DIRECTORY=.automation/test/ansible \
-          -e ACTIONS_RUNNER_DEBUG=true \
-          -e DEFAULT_BRANCH=main \
-          -e RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES="default.json,hoge.json" \
-          -e ERROR_ON_MISSING_EXEC_BIT=true \
-          -e TYPESCRIPT_STANDARD_TSCONFIG_FILE=".github/linters/tsconfig.json" \
-          -v "${GITHUB_WORKSPACE}:/tmp/lint" \
-          "${CONTAINER_IMAGE_ID}"
-
-      - name: Lint Entire Codebase
-        run: |
-          docker run \
-          -e RUN_LOCAL=true \
-          -e OUTPUT_DETAILS=detailed \
-          -e ACTIONS_RUNNER_DEBUG=true \
-          -e DEFAULT_BRANCH=main \
-          -e RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES="default.json,hoge.json" \
-          -e ERROR_ON_MISSING_EXEC_BIT=true \
-          -v "${GITHUB_WORKSPACE}:/tmp/lint" \
-          "${CONTAINER_IMAGE_ID}"
-
       - name: Login to GHCR
         uses: docker/login-action@v3.0.0
         with:

.github/workflows/ci.yml

@@ -113,28 +113,3 @@ jobs:
 
       - name: Run Test Suite
         run: make test
-
-      - name: Run Super-Linter Tests
-        run: |
-          docker run \
-          -e RUN_LOCAL=true \
-          -e TEST_CASE_RUN=true \
-          -e ANSIBLE_DIRECTORY=.automation/test/ansible \
-          -e ACTIONS_RUNNER_DEBUG=true \
-          -e DEFAULT_BRANCH=main \
-          -e RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES="default.json,hoge.json" \
-          -e ERROR_ON_MISSING_EXEC_BIT=true \
-          -e TYPESCRIPT_STANDARD_TSCONFIG_FILE=".github/linters/tsconfig.json" \
-          -v "${GITHUB_WORKSPACE}:/tmp/lint" \
-          "${CONTAINER_IMAGE_ID}"
-
-      - name: Lint Entire Codebase
-        run: |
-          docker run \
-          -e RUN_LOCAL=true \
-          -e ACTIONS_RUNNER_DEBUG=true \
-          -e DEFAULT_BRANCH=main \
-          -e RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES="default.json,hoge.json" \
-          -e ERROR_ON_MISSING_EXEC_BIT=true \
-          -v "${GITHUB_WORKSPACE}:/tmp/lint" \
-          "${CONTAINER_IMAGE_ID}"

.gitignore

@@ -79,3 +79,11 @@ super-linter.report
 
 # Test reports
 test/reports
+
+# Developer credentials
+.github-personal-access-token
+
+# Test leftovers
+.lintr
+.automation/test/rust_clippy/**/Cargo.lock
+.automation/test/rust_clippy/**/target/**

Makefile

@@ -4,7 +4,7 @@
 all: info docker test ## Run all targets.
 
 .PHONY: test
-test: info validate-container-image-labels inspec test-find ## Run tests
+test: info validate-container-image-labels inspec lint-codebase test-find test-linters ## Run the test suite
 
 # if this session isn't interactive, then we don't want to allocate a
 # TTY, which would fail, but if it is interactive, we do want to attach
@@ -63,6 +63,8 @@ ifeq ($(BUILD_VERSION),)
 BUILD_VERSION := $(shell git rev-parse HEAD)
 endif
 
+GITHUB_TOKEN_PATH := "$(CURDIR)/.github-personal-access-token"
+
 .PHONY: inspec
 inspec: inspec-check ## Run InSpec tests
 	DOCKER_CONTAINER_STATE="$$(docker inspect --format "{{.State.Running}}" $(SUPER_LINTER_TEST_CONTAINER_NAME) 2>/dev/null || echo "")"; \
@@ -85,12 +87,12 @@ inspec: inspec-check ## Run InSpec tests
 
 .phony: docker
 docker: ## Build the container image
-	@if [ -z "${GITHUB_TOKEN}" ]; then echo "GITHUB_TOKEN environment variable not set. Please set your GitHub Personal Access Token."; exit 1; fi
+	@if [ ! -f "${GITHUB_TOKEN_PATH}" ]; then echo "Cannot find the file to load the GitHub access token: $(GITHUB_TOKEN_PATH). Create a readable file there, and populate it with a GitHub personal access token."; exit 1; fi
 	DOCKER_BUILDKIT=1 docker buildx build --load \
 		--build-arg BUILD_DATE=$(BUILD_DATE) \
 		--build-arg BUILD_REVISION=$(BUILD_REVISION) \
 		--build-arg BUILD_VERSION=$(BUILD_VERSION) \
-		--secret id=GITHUB_TOKEN,env=GITHUB_TOKEN \
+		--secret id=GITHUB_TOKEN,src=$(GITHUB_TOKEN_PATH) \
 		-t $(SUPER_LINTER_TEST_CONTAINER_URL) .
 
 .phony: docker-pull
@@ -110,8 +112,33 @@ test-find: ## Run super-linter on a subdirectory with USE_FIND_ALGORITHM=true
 	docker run \
 		-e RUN_LOCAL=true \
 		-e ACTIONS_RUNNER_DEBUG=true \
-		-e ERROR_ON_MISSING_EXEC_BIT=true \
 		-e DEFAULT_BRANCH=main \
+		-e ERROR_ON_MISSING_EXEC_BIT=true \
 		-e USE_FIND_ALGORITHM=true \
 		-v "$(CURDIR)/.github":/tmp/lint \
 		$(SUPER_LINTER_TEST_CONTAINER_URL)
+
+.phony: lint-codebase
+lint-codebase: ## Lint the entire codebase
+	docker run \
+		-e RUN_LOCAL=true \
+		-e ACTIONS_RUNNER_DEBUG=true \
+		-e DEFAULT_BRANCH=main \
+		-e ERROR_ON_MISSING_EXEC_BIT=true \
+		-e RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES="default.json,hoge.json" \
+		-v "$(CURDIR):/tmp/lint" \
+		$(SUPER_LINTER_TEST_CONTAINER_URL)
+
+.phony: test-linters
+test-linters: ## Run the linters test suite
+	docker run \
+		-e ACTIONS_RUNNER_DEBUG=true \
+		-e ANSIBLE_DIRECTORY=.automation/test/ansible \
+		-e DEFAULT_BRANCH=main \
+		-e ERROR_ON_MISSING_EXEC_BIT=true \
+		-e RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES="default.json,hoge.json" \
+		-e RUN_LOCAL=true \
+		-e TEST_CASE_RUN=true \
+		-e TYPESCRIPT_STANDARD_TSCONFIG_FILE=".github/linters/tsconfig.json" \
+		-v "$(CURDIR):/tmp/lint" \
+		$(SUPER_LINTER_TEST_CONTAINER_URL)

README.md

@@ -407,4 +407,4 @@ path to the files that contains a CA that can be used to valide the certificate:
 ## How to contribute
 
 If you would like to help contribute to super-linter, see
-[CONTRIBUTING](https://github.com/super-linter/super-linter/blob/main/.github/CONTRIBUTING.md)
+[CONTRIBUTING](https://github.com/super-linter/super-linter/blob/main/.github/CONTRIBUTING.md).

docs/run-linter-locally.md

@@ -73,9 +73,30 @@ them accordingly:
 
 ## Build the container image and run the test suite locally
 
-You can run the build and test process locally with the following command:
+To run the build and test process locally, do the following:
 
-```shell
+1. [Create a fine-grained GitHub personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token).
+1. Create a file to store the personal access token on your machine:
+
+    ```bash
+    touch .github-personal-access-token
+    ```
+
+    The file to store the personal access token is ignored by Git.
+
+1. Run the build process:
+
+    ```bash
+    make
+    ```
+
+To avoid invalidating the build cache, and reuse it, you can set build metadata
+to arbitrary values before running `make`:
+
+```bash
+BUILD_DATE=2023-12-12T09:32:05Z \
+BUILD_REVISION=83c16f63caa9d432df4519efb4c58a56e2190bd6 \
+BUILD_VERSION=83c16f63caa9d432df4519efb4c58a56e2190bd6 \
 make
 ```
 
@@ -88,7 +109,7 @@ image version.
 
 ```shell
 CONTAINER_IMAGE_ID="ghcr.io/super-linter/super-linter:v5.4.3" \
-BUILD_DATE="2023-10-17T16:19:11Z" \
+BUILD_DATE="2023-10-17T17:00:53Z" \
 BUILD_REVISION=b0d1acee1f8050d1684a28ddbf8315f81d084fe9 \
 BUILD_VERSION=b0d1acee1f8050d1684a28ddbf8315f81d084fe9 \
 make docker-pull test

lib/linter.sh

@@ -473,7 +473,7 @@ GetGitHubVars() {
     fi
 
     if [ ! -d "${GITHUB_WORKSPACE}" ]; then
-      fatal "Provided volume is not a directory!"
+      fatal "The workspace (${GITHUB_WORKSPACE}) is not a directory!"
     fi
 
     pushd "${GITHUB_WORKSPACE}" >/dev/null || exit 1