From ac4b767bd7f94b948412c0fab2a5cec504039b21 Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Tue, 12 Dec 2023 19:53:48 +0100 Subject: [PATCH] Reduce duplication in CI and CD workflows (#4982) * Reduce duplication in CI and CD workflows * Fix indentation in README * Load token from file * Fix instructions * Ignore test leftovers --- .github/workflows/cd.yml | 26 ------------------ .github/workflows/ci.yml | 25 ------------------ .gitignore | 8 ++++++ Makefile | 35 +++++++++++++++++++++--- README.md | 54 +++++++++++++++++++------------------- docs/run-linter-locally.md | 27 ++++++++++++++++--- lib/linter.sh | 2 +- 7 files changed, 91 insertions(+), 86 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 8e4d5504..274f427e 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -93,32 +93,6 @@ jobs: - name: Run Test Suite run: make test - - name: Run Super-Linter Tests - run: | - docker run \ - -e RUN_LOCAL=true \ - -e TEST_CASE_RUN=true \ - -e ANSIBLE_DIRECTORY=.automation/test/ansible \ - -e ACTIONS_RUNNER_DEBUG=true \ - -e DEFAULT_BRANCH=main \ - -e RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES="default.json,hoge.json" \ - -e ERROR_ON_MISSING_EXEC_BIT=true \ - -e TYPESCRIPT_STANDARD_TSCONFIG_FILE=".github/linters/tsconfig.json" \ - -v "${GITHUB_WORKSPACE}:/tmp/lint" \ - "${CONTAINER_IMAGE_ID}" - - - name: Lint Entire Codebase - run: | - docker run \ - -e RUN_LOCAL=true \ - -e OUTPUT_DETAILS=detailed \ - -e ACTIONS_RUNNER_DEBUG=true \ - -e DEFAULT_BRANCH=main \ - -e RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES="default.json,hoge.json" \ - -e ERROR_ON_MISSING_EXEC_BIT=true \ - -v "${GITHUB_WORKSPACE}:/tmp/lint" \ - "${CONTAINER_IMAGE_ID}" - - name: Login to GHCR uses: docker/login-action@v3.0.0 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71c0117a..66670a83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -113,28 +113,3 @@ jobs: - name: Run Test Suite run: make test - - - name: Run Super-Linter Tests - run: | - docker run \ - -e RUN_LOCAL=true \ - -e TEST_CASE_RUN=true \ - -e ANSIBLE_DIRECTORY=.automation/test/ansible \ - -e ACTIONS_RUNNER_DEBUG=true \ - -e DEFAULT_BRANCH=main \ - -e RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES="default.json,hoge.json" \ - -e ERROR_ON_MISSING_EXEC_BIT=true \ - -e TYPESCRIPT_STANDARD_TSCONFIG_FILE=".github/linters/tsconfig.json" \ - -v "${GITHUB_WORKSPACE}:/tmp/lint" \ - "${CONTAINER_IMAGE_ID}" - - - name: Lint Entire Codebase - run: | - docker run \ - -e RUN_LOCAL=true \ - -e ACTIONS_RUNNER_DEBUG=true \ - -e DEFAULT_BRANCH=main \ - -e RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES="default.json,hoge.json" \ - -e ERROR_ON_MISSING_EXEC_BIT=true \ - -v "${GITHUB_WORKSPACE}:/tmp/lint" \ - "${CONTAINER_IMAGE_ID}" diff --git a/.gitignore b/.gitignore index a1afcd54..141fa543 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,11 @@ super-linter.report # Test reports test/reports + +# Developer credentials +.github-personal-access-token + +# Test leftovers +.lintr +.automation/test/rust_clippy/**/Cargo.lock +.automation/test/rust_clippy/**/target/** diff --git a/Makefile b/Makefile index 6cef671e..9617c705 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ all: info docker test ## Run all targets. .PHONY: test -test: info validate-container-image-labels inspec test-find ## Run tests +test: info validate-container-image-labels inspec lint-codebase test-find test-linters ## Run the test suite # if this session isn't interactive, then we don't want to allocate a # TTY, which would fail, but if it is interactive, we do want to attach @@ -63,6 +63,8 @@ ifeq ($(BUILD_VERSION),) BUILD_VERSION := $(shell git rev-parse HEAD) endif +GITHUB_TOKEN_PATH := "$(CURDIR)/.github-personal-access-token" + .PHONY: inspec inspec: inspec-check ## Run InSpec tests DOCKER_CONTAINER_STATE="$$(docker inspect --format "{{.State.Running}}" $(SUPER_LINTER_TEST_CONTAINER_NAME) 2>/dev/null || echo "")"; \ @@ -85,12 +87,12 @@ inspec: inspec-check ## Run InSpec tests .phony: docker docker: ## Build the container image - @if [ -z "${GITHUB_TOKEN}" ]; then echo "GITHUB_TOKEN environment variable not set. Please set your GitHub Personal Access Token."; exit 1; fi + @if [ ! -f "${GITHUB_TOKEN_PATH}" ]; then echo "Cannot find the file to load the GitHub access token: $(GITHUB_TOKEN_PATH). Create a readable file there, and populate it with a GitHub personal access token."; exit 1; fi DOCKER_BUILDKIT=1 docker buildx build --load \ --build-arg BUILD_DATE=$(BUILD_DATE) \ --build-arg BUILD_REVISION=$(BUILD_REVISION) \ --build-arg BUILD_VERSION=$(BUILD_VERSION) \ - --secret id=GITHUB_TOKEN,env=GITHUB_TOKEN \ + --secret id=GITHUB_TOKEN,src=$(GITHUB_TOKEN_PATH) \ -t $(SUPER_LINTER_TEST_CONTAINER_URL) . .phony: docker-pull @@ -110,8 +112,33 @@ test-find: ## Run super-linter on a subdirectory with USE_FIND_ALGORITHM=true docker run \ -e RUN_LOCAL=true \ -e ACTIONS_RUNNER_DEBUG=true \ - -e ERROR_ON_MISSING_EXEC_BIT=true \ -e DEFAULT_BRANCH=main \ + -e ERROR_ON_MISSING_EXEC_BIT=true \ -e USE_FIND_ALGORITHM=true \ -v "$(CURDIR)/.github":/tmp/lint \ $(SUPER_LINTER_TEST_CONTAINER_URL) + +.phony: lint-codebase +lint-codebase: ## Lint the entire codebase + docker run \ + -e RUN_LOCAL=true \ + -e ACTIONS_RUNNER_DEBUG=true \ + -e DEFAULT_BRANCH=main \ + -e ERROR_ON_MISSING_EXEC_BIT=true \ + -e RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES="default.json,hoge.json" \ + -v "$(CURDIR):/tmp/lint" \ + $(SUPER_LINTER_TEST_CONTAINER_URL) + +.phony: test-linters +test-linters: ## Run the linters test suite + docker run \ + -e ACTIONS_RUNNER_DEBUG=true \ + -e ANSIBLE_DIRECTORY=.automation/test/ansible \ + -e DEFAULT_BRANCH=main \ + -e ERROR_ON_MISSING_EXEC_BIT=true \ + -e RENOVATE_SHAREABLE_CONFIG_PRESET_FILE_NAMES="default.json,hoge.json" \ + -e RUN_LOCAL=true \ + -e TEST_CASE_RUN=true \ + -e TYPESCRIPT_STANDARD_TSCONFIG_FILE=".github/linters/tsconfig.json" \ + -v "$(CURDIR):/tmp/lint" \ + $(SUPER_LINTER_TEST_CONTAINER_URL) diff --git a/README.md b/README.md index 18e7f9ac..c8676bb0 100644 --- a/README.md +++ b/README.md @@ -82,37 +82,37 @@ To run super-linter as a GitHub Action, you do the following: 1. Create a new [GitHub Actions workflow](https://docs.github.com/en/actions/using-workflows/about-workflows#about-workflows) in your repository with the following content: - ```yaml - --- - name: Lint + ```yaml + --- + name: Lint - on: # yamllint disable-line rule:truthy - push: null - pull_request: null + on: # yamllint disable-line rule:truthy + push: null + pull_request: null - jobs: - build: - name: Lint - runs-on: ubuntu-latest + jobs: + build: + name: Lint + runs-on: ubuntu-latest - permissions: - contents: read - packages: read - # To report GitHub Actions status checks - statuses: write + permissions: + contents: read + packages: read + # To report GitHub Actions status checks + statuses: write - steps: - - name: Checkout code - uses: actions/checkout@v4 + steps: + - name: Checkout code + uses: actions/checkout@v4 - - name: Super-linter - uses: super-linter/super-linter@v5 - env: - DEFAULT_BRANCH: main - # To report GitHub Actions status checks - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ... - ``` + - name: Super-linter + uses: super-linter/super-linter@v5 + env: + DEFAULT_BRANCH: main + # To report GitHub Actions status checks + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ... + ``` 1. Commit that file to a new branch. 1. Push the new commit to the remote repository. @@ -407,4 +407,4 @@ path to the files that contains a CA that can be used to valide the certificate: ## How to contribute If you would like to help contribute to super-linter, see -[CONTRIBUTING](https://github.com/super-linter/super-linter/blob/main/.github/CONTRIBUTING.md) +[CONTRIBUTING](https://github.com/super-linter/super-linter/blob/main/.github/CONTRIBUTING.md). diff --git a/docs/run-linter-locally.md b/docs/run-linter-locally.md index bb3306a5..fafed3a9 100644 --- a/docs/run-linter-locally.md +++ b/docs/run-linter-locally.md @@ -73,9 +73,30 @@ them accordingly: ## Build the container image and run the test suite locally -You can run the build and test process locally with the following command: +To run the build and test process locally, do the following: -```shell +1. [Create a fine-grained GitHub personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token). +1. Create a file to store the personal access token on your machine: + + ```bash + touch .github-personal-access-token + ``` + + The file to store the personal access token is ignored by Git. + +1. Run the build process: + + ```bash + make + ``` + +To avoid invalidating the build cache, and reuse it, you can set build metadata +to arbitrary values before running `make`: + +```bash +BUILD_DATE=2023-12-12T09:32:05Z \ +BUILD_REVISION=83c16f63caa9d432df4519efb4c58a56e2190bd6 \ +BUILD_VERSION=83c16f63caa9d432df4519efb4c58a56e2190bd6 \ make ``` @@ -88,7 +109,7 @@ image version. ```shell CONTAINER_IMAGE_ID="ghcr.io/super-linter/super-linter:v5.4.3" \ -BUILD_DATE="2023-10-17T16:19:11Z" \ +BUILD_DATE="2023-10-17T17:00:53Z" \ BUILD_REVISION=b0d1acee1f8050d1684a28ddbf8315f81d084fe9 \ BUILD_VERSION=b0d1acee1f8050d1684a28ddbf8315f81d084fe9 \ make docker-pull test diff --git a/lib/linter.sh b/lib/linter.sh index db8fabb7..55356061 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -473,7 +473,7 @@ GetGitHubVars() { fi if [ ! -d "${GITHUB_WORKSPACE}" ]; then - fatal "Provided volume is not a directory!" + fatal "The workspace (${GITHUB_WORKSPACE}) is not a directory!" fi pushd "${GITHUB_WORKSPACE}" >/dev/null || exit 1