From 23e8ced872469d168d00da1f887ed7dbe1b87727 Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Tue, 27 Sep 2022 16:30:54 +0200 Subject: [PATCH] Fix issues in the build workflow (#3357) * Fix issues in the build workflow * Don't build the image when running trivy * Move the alpine glibc package key in the repo * Move the alpine glibc package key in the repo * Update labels * Pull and tag --- .github/workflows/deploy-production.yml | 138 ++++------------------- .github/workflows/deploy-release.yml | 108 +++--------------- .github/workflows/repo-visualization.yml | 71 ------------ .github/workflows/trivy.yml | 8 -- Dockerfile | 18 ++- README.md | 16 +-- dependencies/sgerrand.rsa.pub | 9 ++ 7 files changed, 62 insertions(+), 306 deletions(-) delete mode 100644 .github/workflows/repo-visualization.yml create mode 100644 dependencies/sgerrand.rsa.pub diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 4bf06f42..5d0c38e5 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -1,31 +1,13 @@ --- -############################################# -############################################# -## Deploy Docker Image test and Production ## -############################################# -############################################# +name: Build, test, and eventually deploy super-linter container images -# -# Documentation: -# https://help.github.com/en/articles/workflow-syntax-for-github-actions -# - -name: Deploy Production -########################################### -# Start the job on all push or PR to main # -########################################### on: pull_request: push: -############### -# Set the Job # -############### jobs: build: - # Name the Job - name: Deploy Docker Image - DEV - # Set the agent to run on + name: Build, test, and eventually push the container image runs-on: ubuntu-latest strategy: fail-fast: false @@ -41,13 +23,7 @@ jobs: image-id: standard timeout-minutes: 60 - ############### - # Steps below # - ############### steps: - ############################ - # Checkout the source code # - ############################ - name: Checkout Code uses: actions/checkout@v3 with: @@ -55,23 +31,14 @@ jobs: # of changed files within `super-linter` fetch-depth: 0 - ########################### - # Set current date to ENV # - ########################### - - name: Get current date + - name: Get current date and make it available as an environment variable run: | echo "Appending the build date contents to GITHUB_ENV..." echo "BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> "${GITHUB_ENV}" - ####################### - # Setup Docker BuildX # - ####################### - - name: Setup BuildX + - name: Setup Docker BuildX uses: docker/setup-buildx-action@v2.0.0 - ####################################### - # Build local docker images for tests # - ####################################### - name: Build Docker image - ${{ matrix.images.image-id }} if: ${{ github.repository == 'github/super-linter' && github.ref != 'refs/heads/main' }} uses: docker/build-push-action@v3.1.1 @@ -89,25 +56,16 @@ jobs: ghcr.io/github/super-linter:${{ matrix.images.container-image-id-prefix }}test target: "${{ matrix.images.container-build-target }}" - ################################ - # Run local docker labels test # - ################################ - - name: Run Docker label test cases + - name: Run container image label test cases if: ${{ github.repository == 'github/super-linter' && github.ref != 'refs/heads/main' }} shell: bash run: .automation/validate-docker-labels.sh "${{ matrix.images.image-id }}" - ####################################### - # Edit the action.yml for local tests # - ####################################### - name: Edit an action.yml file for test local build if: ${{ github.repository == 'github/super-linter' && github.ref != 'refs/heads/main' }} run: | sed -i "s/super-linter:.*/super-linter:${{ matrix.images.container-image-id-prefix }}${GITHUB_SHA}'/g" action.yml - ###################### - # Gather information # - ###################### - name: Gather information about the runtime environment if: ${{ github.repository == 'github/super-linter' && github.ref != 'refs/heads/main' }} shell: bash @@ -129,29 +87,19 @@ jobs: DEFAULT_BRANCH: main LOCAL_UPDATES: true - - ############################################################### - # Fix file and dir ownership. # - # Workaround for https://github.com/actions/runner/issues/434 # - ############################################################### + # Workaround for https://github.com/actions/runner/issues/434 - name: Fix file and directory ownership if: ${{ github.repository == 'github/super-linter' && github.ref != 'refs/heads/main' }} shell: bash run: | sudo chown -R "$(id -u)":"$(id -g)" "$(pwd)" - ######################## - # Run local make tests # - ######################## - name: Run the test suite if: ${{ github.repository == 'github/super-linter' && github.ref != 'refs/heads/main' }} shell: bash run: | make IMAGE=${{ matrix.images.image-id }} test - ########################## - # Codacy Coverage Report # - ########################## - name: Upload the code coverage report if: ${{ github.repository == 'github/super-linter' && github.ref != 'refs/heads/main' }} uses: codacy/codacy-coverage-reporter-action@v1.1 @@ -161,10 +109,7 @@ jobs: project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} coverage-reports: test/reports/cobertura/runTests.sh/cobertura.xml - ##################################### - # Run Linter against Test code base # - ##################################### - - name: Run Test Cases - ${{ matrix.images.image-id }} + - name: Run test cases - ${{ matrix.images.image-id }} if: ${{ github.repository == 'github/super-linter' && github.ref != 'refs/heads/main' }} shell: bash run: | @@ -177,17 +122,11 @@ jobs: -v "${GITHUB_WORKSPACE}:/tmp/lint" \ "ghcr.io/github/super-linter:${{ matrix.images.container-image-id-prefix }}${GITHUB_SHA}" - ######################################### - # Clean code base to run against it all # - ######################################### - - name: Clean Test code base for additional testing + - name: Clean the working directory for additional testing if: ${{ github.repository == 'github/super-linter' && github.ref != 'refs/heads/main' }} shell: bash run: .automation/clean-code-base-for-tests.sh - ############################################ - # Run Linter against ALL cleaned code base # - ############################################ - name: Run against all code base - ${{ matrix.images.image-id }} if: ${{ github.repository == 'github/super-linter' && github.ref != 'refs/heads/main' }} shell: bash @@ -200,26 +139,13 @@ jobs: -v "${GITHUB_WORKSPACE}:/tmp/lint" \ "ghcr.io/github/super-linter:${{ matrix.images.container-image-id-prefix }}${GITHUB_SHA}" - ############################################################# - ############################################################# - ## The following steps are only run if the PR is merges ## - ## into the 'main' branch and push the image to registries ## - ############################################################# - ############################################################# - - ###################### - # Login to DockerHub # - ###################### - - name: Login to DockerHub + - name: Login to Docker Hub if: ${{ github.repository == 'github/super-linter' && github.ref == 'refs/heads/main' }} uses: docker/login-action@v2.0.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - #################### - # Login to GHCR.io # - #################### - name: Login to GitHub Container Registry if: ${{ github.repository == 'github/super-linter' && github.ref == 'refs/heads/main' }} uses: docker/login-action@v2.0.0 @@ -228,10 +154,7 @@ jobs: username: ${{ secrets.GCR_USERNAME }} password: ${{ secrets.GCR_TOKEN }} - ######################### - # Update deployment API # - ######################### - - name: Start deployment + - name: Start the deployment for the ${{ matrix.images.deployment-environment-identifier }} environment if: ${{ github.repository == 'github/super-linter' && github.ref == 'refs/heads/main' }} uses: bobheadxi/deployments@v1.3.0 id: deployment @@ -240,10 +163,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} env: ${{ matrix.images.deployment-environment-identifier }} - ###################################### - # Build the docker image and push it # - ###################################### - - name: Build Docker image - ${{ matrix.images.image-id }} + - name: Build the container image and push it - ${{ matrix.images.image-id }} if: ${{ github.repository == 'github/super-linter' && github.ref == 'refs/heads/main' }} uses: docker/build-push-action@v3.1.1 with: @@ -260,10 +180,18 @@ jobs: ghcr.io/github/super-linter:${{ matrix.images.container-image-id-prefix }}latest target: "${{ matrix.images.container-build-target }}" - ####################################################### - # Create a GitHub Issue with the info from this build # - ####################################################### - - name: Create GitHub Issue for failure + - name: Update the deployment status for the ${{ matrix.images.deployment-environment-identifier }} environment + if: ${{ github.repository == 'github/super-linter' && github.ref == 'refs/heads/main' }} + uses: bobheadxi/deployments@v1.3.0 + with: + step: finish + token: ${{ secrets.GITHUB_TOKEN }} + status: ${{ job.status }} + deployment_id: ${{ steps.deployment.outputs.deployment_id }} + env: ${{ steps.deployment.outputs.env }} + env_url: https://github.com/github/super-linter + + - name: Create a GitHub issue on failure if: ${{ github.repository == 'github/super-linter' && github.ref == 'refs/heads/main' && failure() }} uses: actions/github-script@v6 id: create-issue @@ -280,14 +208,10 @@ jobs: console.log('create', create) return create.data.number - #################################### - # Deploy was failure, alert admins # - #################################### - - name: Assign Admins on failure + - name: Assign admins to the issue created on failure uses: actions/github-script@v6 if: ${{ github.repository == 'github/super-linter' && github.ref == 'refs/heads/main' && failure() }} with: - # https://octokit.github.io/rest.js/v18#issues-create github-token: ${{secrets.GITHUB_TOKEN}} script: | github.rest.issues.addAssignees({ @@ -304,17 +228,3 @@ jobs: 'ferrarimarco' ] }) - - ######################### - # Update Deployment API # - ######################### - - name: Update deployment status - if: ${{ github.repository == 'github/super-linter' && github.ref == 'refs/heads/main' }} - uses: bobheadxi/deployments@v1.3.0 - with: - step: finish - token: ${{ secrets.GITHUB_TOKEN }} - status: ${{ job.status }} - deployment_id: ${{ steps.deployment.outputs.deployment_id }} - env: ${{ steps.deployment.outputs.env }} - env_url: https://github.com/github/super-linter diff --git a/.github/workflows/deploy-release.yml b/.github/workflows/deploy-release.yml index 19a80a15..323f5b84 100644 --- a/.github/workflows/deploy-release.yml +++ b/.github/workflows/deploy-release.yml @@ -1,89 +1,40 @@ --- -################################# -################################# -## Deploy Docker Image Release ## -################################# -################################# +name: Deploy super-linter release container images -# -# Documentation: -# https://help.github.com/en/articles/workflow-syntax-for-github-actions -# -name: Deploy Release -########################################### -# Start the job upon publishing a release # -########################################### on: - # Start when a release is published release: types: [published] workflow_dispatch: - # Set the input variables you want to pull in inputs: release_version: description: 'version to release. Ex: v4.3.2' required: true default: 'v' -############### -# Set the Job # -############### jobs: build: - # Name the Job - name: Deploy Docker Image - Release - # Set the agent to run on + name: Deploy container image - Release runs-on: ubuntu-latest strategy: fail-fast: false matrix: images: - - container-build-target: final_slim - container-image-id-prefix: slim- + - container-image-id-prefix: slim- deployment-environment-identifier: Release-SLIM - image-id: slim - - container-build-target: final_standard - container-image-id-prefix: "" + - container-image-id-prefix: "" deployment-environment-identifier: Release - image-id: standard timeout-minutes: 60 - ############### - # Steps below # - ############### steps: - ############################ - # Checkout the source code # - ############################ - - name: Checkout Code - uses: actions/checkout@v3 - - ########################### - # Set current date to ENV # - ########################### - - name: Get current date - run: | - echo "Appending the build date contents to GITHUB_ENV..." - echo "BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> "${GITHUB_ENV}" - - ####################### - # Setup Docker BuildX # - ####################### - - name: Setup BuildX + - name: Setup Docker BuildX uses: docker/setup-buildx-action@v2.0.0 - ###################### - # Login to DockerHub # - ###################### - name: Login to DockerHub uses: docker/login-action@v2.0.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - #################### - # Login to GHCR.io # - #################### - name: Login to GitHub Container Registry uses: docker/login-action@v2.0.0 with: @@ -91,10 +42,7 @@ jobs: username: ${{ secrets.GCR_USERNAME }} password: ${{ secrets.GCR_TOKEN }} - ########################### - # Get the current release # - ########################### - - name: Get current Release number + - name: Get current release identifier # shellcheck disable=SC2062 run: | echo "RELEASE_VERSION=$(echo ${{ github.event.release.name }} \ @@ -106,10 +54,7 @@ jobs: >> "${GITHUB_ENV}" fi - ######################### - # Update deployment API # - ######################### - - name: Start deployment + - name: Start deployment for the ${{ matrix.images.deployment-environment-identifier }} environment uses: bobheadxi/deployments@v1.3.0 id: deployment with: @@ -117,33 +62,19 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} env: ${{ matrix.images.deployment-environment-identifier }} - ###################################### - # Build the docker image and push it # - ###################################### - - name: Build Docker image - ${{ matrix.images.image-id }} - uses: docker/build-push-action@v3.1.1 + # We don't rebuild the image to avoid that the latest tag and the release tags don't point to what the release tag is pointing to. + # Instead, we pull the latest image and tag it. + - name: Add release tags and push image + uses: akhilerm/tag-push-action@v2.0.0 with: - context: . - file: ./Dockerfile - build-args: | - BUILD_DATE=${{ env.BUILD_DATE }} - BUILD_REVISION=${{ github.sha }} - BUILD_VERSION=${{ github.sha }} - load: false - push: true - tags: | - github/super-linter:${{ matrix.images.container-image-id-prefix }}latest + src: ghcr.io/github/super-linter:${{ matrix.images.container-image-id-prefix }}latest + dst: | github/super-linter:${{ matrix.images.container-image-id-prefix }}v4 github/super-linter:${{ matrix.images.container-image-id-prefix }}${{ env.RELEASE_VERSION }} - ghcr.io/github/super-linter:${{ matrix.images.container-image-id-prefix }}latest ghcr.io/github/super-linter:${{ matrix.images.container-image-id-prefix }}v4 ghcr.io/github/super-linter:${{ matrix.images.container-image-id-prefix }}${{ env.RELEASE_VERSION }} - target: "${{ matrix.images.container-build-target }}" - ######################### - # Update Deployment API # - ######################### - - name: Update deployment status + - name: Update the deployment status for the ${{ matrix.images.deployment-environment-identifier }} environment uses: bobheadxi/deployments@v1.3.0 if: always() with: @@ -154,10 +85,7 @@ jobs: env: ${{ steps.deployment.outputs.env }} env_url: https://github.com/github/super-linter/releases/tag/${{ env.RELEASE_VERSION }} - ####################################################### - # Create a GitHub Issue with the info from this build # - ####################################################### - - name: Create GitHub Issue for failure + - name: Create a GitHub issue on failure if: failure() uses: actions/github-script@v6 id: create-issue @@ -174,14 +102,10 @@ jobs: console.log('create', create) return create.data.number - ############################ - # Assign admins on failure # - ############################ - - name: Assign Admins on failure + - name: Assign admins to the issue created on failure uses: actions/github-script@v6 if: failure() with: - # https://octokit.github.io/rest.js/v18#issues-create github-token: ${{secrets.GITHUB_TOKEN}} script: | github.rest.issues.addAssignees({ diff --git a/.github/workflows/repo-visualization.yml b/.github/workflows/repo-visualization.yml deleted file mode 100644 index 3824e3ba..00000000 --- a/.github/workflows/repo-visualization.yml +++ /dev/null @@ -1,71 +0,0 @@ ---- -############################## -############################## -## Repository Visualization ## -############################## -############################## - -# -# Documentation: -# https://help.github.com/en/articles/workflow-syntax-for-github-actions -# - -name: Repository Visualization -########################################### -# Start the job on all push or PR to main # -########################################### -on: - schedule: - # Sunday at 5:00pm - - cron: "0 17 * * 0" - -############### -# Set the Job # -############### -permissions: - contents: read - -jobs: - build: - # Name the Job - name: Repository Visualization - # Set the agent to run on - runs-on: ubuntu-latest - timeout-minutes: 60 - - ############### - # Steps below # - ############### - steps: - ############################ - # Checkout the source code # - ############################ - - name: Checkout Code - uses: actions/checkout@v3 - - ############################## - # Create Visualization Files # - ############################## - - name: Create Visualization - uses: githubocto/repo-visualizer@0.8.2 - with: - output_file: "./diagram.svg" - should_push: false - - ########################### - # Configure the AWS creds # - ########################### - - name: Configure AWS Credentials - if: success() - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - ############################ - # Push the image to aws s3 # - ############################ - - name: Push diagram to s3 - if: success() - run: aws s3 cp diagram.svg s3://super-linter/diagram.svg diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 7fc95e75..2526dc00 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -24,13 +24,6 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - # ########################## - # # Build the docker image # - # ########################## - - name: Build an image from Dockerfile - run: | - docker build -t "docker.io/github/super-linter:${{ github.sha }}" . - ################################# # Run Trivy Scan of source code # ################################# @@ -48,6 +41,5 @@ jobs: ################################# - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@v2 - if: always() with: sarif_file: 'report.sarif' diff --git a/Dockerfile b/Dockerfile index 9b1c82b7..9800a9c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -207,6 +207,10 @@ RUN mkdir -p /home/r-library \ && Rscript -e "install.packages(c('lintr','purrr'), repos = 'https://cloud.r-project.org/')" \ && R -e "install.packages(list.dirs('/home/r-library',recursive = FALSE), repos = NULL, type = 'source')" +# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub +# Store the key here because the above host is sometimes down, and breaks our builds +COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub + ################### # Install Kubeval # ################### @@ -214,9 +218,9 @@ RUN wget --tries=5 -q -O kubeval-linux-amd64.tar.gz https://github.com/instrumen && tar xf kubeval-linux-amd64.tar.gz \ && mv kubeval /usr/local/bin \ && rm kubeval-linux-amd64.tar.gz \ -################## -# Install ktlint # -################## + ################## + # Install ktlint # + ################## && curl --retry 5 --retry-delay 5 -sSLO https://github.com/pinterest/ktlint/releases/latest/download/ktlint \ && chmod a+x ktlint \ && mv "ktlint" /usr/bin/ \ @@ -225,7 +229,6 @@ RUN wget --tries=5 -q -O kubeval-linux-amd64.tar.gz https://github.com/instrumen #################### # Install dart-sdk # #################### - && wget --tries=5 -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub \ && wget --tries=5 -q https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk \ && apk add --no-cache glibc-${GLIBC_VERSION}.apk \ && rm glibc-${GLIBC_VERSION}.apk \ @@ -329,11 +332,14 @@ ENV BUILD_REVISION=$BUILD_REVISION ENV BUILD_VERSION=$BUILD_VERSION ENV IMAGE="slim" +# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub +# Store the key here because the above host is sometimes down, and breaks our builds +COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub + ###################################### # Install Phive dependencies and git # ###################################### -RUN wget --tries=5 -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub \ - && wget --tries=5 -q https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk \ +RUN wget --tries=5 -q https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk \ && apk add --no-cache \ bash \ ca-certificates \ diff --git a/README.md b/README.md index 50cf0069..026609d0 100644 --- a/README.md +++ b/README.md @@ -48,10 +48,6 @@ The super-linter finds issues and reports them to the console output. Fixes are The design of the **Super-Linter** is currently to allow linting to occur in **GitHub Actions** as a part of continuous integration occurring on pull requests as the commits get pushed. It works best when commits are being pushed early and often to a branch with an open or draft pull request. There is some desire to move this closer to local development for faster feedback on linting errors but this is not yet supported. -### Repository Visualization - -![Visualization of the codebase](https://super-linter.s3.us-west-2.amazonaws.com/diagram.svg) - ## Supported Linters Developers on **GitHub** can call the **GitHub Action** to lint their codebase with the following list of linters: @@ -60,7 +56,7 @@ Developers on **GitHub** can call the **GitHub Action** to lint their codebase w | --------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | **Ansible** | [ansible-lint](https://github.com/ansible/ansible-lint) | | **AWS CloudFormation templates** | [cfn-lint](https://github.com/aws-cloudformation/cfn-python-lint/) | -| **Azure Resource Manager (ARM)** | [arm-ttk](https://github.com/azure/arm-ttk) | +| **Azure Resource Manager (ARM)** | [arm-ttk](https://github.com/azure/arm-ttk) | | **C++** | [cpp-lint](https://github.com/cpplint/cpplint) / [clang-format](https://clang.llvm.org/docs/ClangFormatStyleOptions.html) | | **C#** | [dotnet-format](https://github.com/dotnet/format) / [clang-format](https://clang.llvm.org/docs/ClangFormatStyleOptions.html) | | **CSS** | [stylelint](https://stylelint.io/) | @@ -141,11 +137,6 @@ This file should have the following code: ################################# name: Lint Code Base -# -# Documentation: -# https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions -# - ############################# # Start the job on all push # ############################# @@ -448,11 +439,6 @@ Example workflow: ################################# name: Lint Code Base -# -# Documentation: -# https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions -# - ############################# # Start the job on all push # ############################# diff --git a/dependencies/sgerrand.rsa.pub b/dependencies/sgerrand.rsa.pub new file mode 100644 index 00000000..3bf38b76 --- /dev/null +++ b/dependencies/sgerrand.rsa.pub @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApZ2u1KJKUu/fW4A25y9m +y70AGEa/J3Wi5ibNVGNn1gT1r0VfgeWd0pUybS4UmcHdiNzxJPgoWQhV2SSW1JYu +tOqKZF5QSN6X937PTUpNBjUvLtTQ1ve1fp39uf/lEXPpFpOPL88LKnDBgbh7wkCp +m2KzLVGChf83MS0ShL6G9EQIAUxLm99VpgRjwqTQ/KfzGtpke1wqws4au0Ab4qPY +KXvMLSPLUp7cfulWvhmZSegr5AdhNw5KNizPqCJT8ZrGvgHypXyiFvvAH5YRtSsc +Zvo9GI2e2MaZyo9/lvb+LbLEJZKEQckqRj4P26gmASrZEPStwc+yqy1ShHLA0j6m +1QIDAQAB +-----END PUBLIC KEY-----