Fix issues in the build workflow (#3357)

* Fix issues in the build workflow

* Don't build the image when running trivy

* Move the alpine glibc package key in the repo

* Move the alpine glibc package key in the repo

* Update labels

* Pull and tag
This commit is contained in:
Marco Ferrari 2022-09-27 16:30:54 +02:00 committed by GitHub
parent e4f768397e
commit 23e8ced872
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 62 additions and 306 deletions

View file

@ -1,31 +1,13 @@
---
#############################################
#############################################
## Deploy Docker Image test and Production ##
#############################################
#############################################
name: Build, test, and eventually deploy super-linter container images
#
# Documentation:
# https://help.github.com/en/articles/workflow-syntax-for-github-actions
#
name: Deploy Production
###########################################
# Start the job on all push or PR to main #
###########################################
on:
pull_request:
push:
###############
# Set the Job #
###############
jobs:
build:
# Name the Job
name: Deploy Docker Image - DEV
# Set the agent to run on
name: Build, test, and eventually push the container image
runs-on: ubuntu-latest
strategy:
fail-fast: false
@ -41,13 +23,7 @@ jobs:
image-id: standard
timeout-minutes: 60
###############
# Steps below #
###############
steps:
############################
# Checkout the source code #
############################
- name: Checkout Code
uses: actions/checkout@v3
with:
@ -55,23 +31,14 @@ jobs:
# of changed files within `super-linter`
fetch-depth: 0
###########################
# Set current date to ENV #
###########################
- name: Get current date
- name: Get current date and make it available as an environment variable
run: |
echo "Appending the build date contents to GITHUB_ENV..."
echo "BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> "${GITHUB_ENV}"
#######################
# Setup Docker BuildX #
#######################
- name: Setup BuildX
- name: Setup Docker BuildX
uses: docker/setup-buildx-action@v2.0.0
#######################################
# Build local docker images for tests #
#######################################
- name: Build Docker image - ${{ matrix.images.image-id }}
if: ${{ github.repository == 'github/super-linter' && github.ref != 'refs/heads/main' }}
uses: docker/build-push-action@v3.1.1
@ -89,25 +56,16 @@ jobs:
ghcr.io/github/super-linter:${{ matrix.images.container-image-id-prefix }}test
target: "${{ matrix.images.container-build-target }}"
################################
# Run local docker labels test #
################################
- name: Run Docker label test cases
- name: Run container image label test cases
if: ${{ github.repository == 'github/super-linter' && github.ref != 'refs/heads/main' }}
shell: bash
run: .automation/validate-docker-labels.sh "${{ matrix.images.image-id }}"
#######################################
# Edit the action.yml for local tests #
#######################################
- name: Edit an action.yml file for test local build
if: ${{ github.repository == 'github/super-linter' && github.ref != 'refs/heads/main' }}
run: |
sed -i "s/super-linter:.*/super-linter:${{ matrix.images.container-image-id-prefix }}${GITHUB_SHA}'/g" action.yml
######################
# Gather information #
######################
- name: Gather information about the runtime environment
if: ${{ github.repository == 'github/super-linter' && github.ref != 'refs/heads/main' }}
shell: bash
@ -129,29 +87,19 @@ jobs:
DEFAULT_BRANCH: main
LOCAL_UPDATES: true
###############################################################
# Fix file and dir ownership. #
# Workaround for https://github.com/actions/runner/issues/434 #
###############################################################
# Workaround for https://github.com/actions/runner/issues/434
- name: Fix file and directory ownership
if: ${{ github.repository == 'github/super-linter' && github.ref != 'refs/heads/main' }}
shell: bash
run: |
sudo chown -R "$(id -u)":"$(id -g)" "$(pwd)"
########################
# Run local make tests #
########################
- name: Run the test suite
if: ${{ github.repository == 'github/super-linter' && github.ref != 'refs/heads/main' }}
shell: bash
run: |
make IMAGE=${{ matrix.images.image-id }} test
##########################
# Codacy Coverage Report #
##########################
- name: Upload the code coverage report
if: ${{ github.repository == 'github/super-linter' && github.ref != 'refs/heads/main' }}
uses: codacy/codacy-coverage-reporter-action@v1.1
@ -161,10 +109,7 @@ jobs:
project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
coverage-reports: test/reports/cobertura/runTests.sh/cobertura.xml
#####################################
# Run Linter against Test code base #
#####################################
- name: Run Test Cases - ${{ matrix.images.image-id }}
- name: Run test cases - ${{ matrix.images.image-id }}
if: ${{ github.repository == 'github/super-linter' && github.ref != 'refs/heads/main' }}
shell: bash
run: |
@ -177,17 +122,11 @@ jobs:
-v "${GITHUB_WORKSPACE}:/tmp/lint" \
"ghcr.io/github/super-linter:${{ matrix.images.container-image-id-prefix }}${GITHUB_SHA}"
#########################################
# Clean code base to run against it all #
#########################################
- name: Clean Test code base for additional testing
- name: Clean the working directory for additional testing
if: ${{ github.repository == 'github/super-linter' && github.ref != 'refs/heads/main' }}
shell: bash
run: .automation/clean-code-base-for-tests.sh
############################################
# Run Linter against ALL cleaned code base #
############################################
- name: Run against all code base - ${{ matrix.images.image-id }}
if: ${{ github.repository == 'github/super-linter' && github.ref != 'refs/heads/main' }}
shell: bash
@ -200,26 +139,13 @@ jobs:
-v "${GITHUB_WORKSPACE}:/tmp/lint" \
"ghcr.io/github/super-linter:${{ matrix.images.container-image-id-prefix }}${GITHUB_SHA}"
#############################################################
#############################################################
## The following steps are only run if the PR is merges ##
## into the 'main' branch and push the image to registries ##
#############################################################
#############################################################
######################
# Login to DockerHub #
######################
- name: Login to DockerHub
- name: Login to Docker Hub
if: ${{ github.repository == 'github/super-linter' && github.ref == 'refs/heads/main' }}
uses: docker/login-action@v2.0.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
####################
# Login to GHCR.io #
####################
- name: Login to GitHub Container Registry
if: ${{ github.repository == 'github/super-linter' && github.ref == 'refs/heads/main' }}
uses: docker/login-action@v2.0.0
@ -228,10 +154,7 @@ jobs:
username: ${{ secrets.GCR_USERNAME }}
password: ${{ secrets.GCR_TOKEN }}
#########################
# Update deployment API #
#########################
- name: Start deployment
- name: Start the deployment for the ${{ matrix.images.deployment-environment-identifier }} environment
if: ${{ github.repository == 'github/super-linter' && github.ref == 'refs/heads/main' }}
uses: bobheadxi/deployments@v1.3.0
id: deployment
@ -240,10 +163,7 @@ jobs:
token: ${{ secrets.GITHUB_TOKEN }}
env: ${{ matrix.images.deployment-environment-identifier }}
######################################
# Build the docker image and push it #
######################################
- name: Build Docker image - ${{ matrix.images.image-id }}
- name: Build the container image and push it - ${{ matrix.images.image-id }}
if: ${{ github.repository == 'github/super-linter' && github.ref == 'refs/heads/main' }}
uses: docker/build-push-action@v3.1.1
with:
@ -260,10 +180,18 @@ jobs:
ghcr.io/github/super-linter:${{ matrix.images.container-image-id-prefix }}latest
target: "${{ matrix.images.container-build-target }}"
#######################################################
# Create a GitHub Issue with the info from this build #
#######################################################
- name: Create GitHub Issue for failure
- name: Update the deployment status for the ${{ matrix.images.deployment-environment-identifier }} environment
if: ${{ github.repository == 'github/super-linter' && github.ref == 'refs/heads/main' }}
uses: bobheadxi/deployments@v1.3.0
with:
step: finish
token: ${{ secrets.GITHUB_TOKEN }}
status: ${{ job.status }}
deployment_id: ${{ steps.deployment.outputs.deployment_id }}
env: ${{ steps.deployment.outputs.env }}
env_url: https://github.com/github/super-linter
- name: Create a GitHub issue on failure
if: ${{ github.repository == 'github/super-linter' && github.ref == 'refs/heads/main' && failure() }}
uses: actions/github-script@v6
id: create-issue
@ -280,14 +208,10 @@ jobs:
console.log('create', create)
return create.data.number
####################################
# Deploy was failure, alert admins #
####################################
- name: Assign Admins on failure
- name: Assign admins to the issue created on failure
uses: actions/github-script@v6
if: ${{ github.repository == 'github/super-linter' && github.ref == 'refs/heads/main' && failure() }}
with:
# https://octokit.github.io/rest.js/v18#issues-create
github-token: ${{secrets.GITHUB_TOKEN}}
script: |
github.rest.issues.addAssignees({
@ -304,17 +228,3 @@ jobs:
'ferrarimarco'
]
})
#########################
# Update Deployment API #
#########################
- name: Update deployment status
if: ${{ github.repository == 'github/super-linter' && github.ref == 'refs/heads/main' }}
uses: bobheadxi/deployments@v1.3.0
with:
step: finish
token: ${{ secrets.GITHUB_TOKEN }}
status: ${{ job.status }}
deployment_id: ${{ steps.deployment.outputs.deployment_id }}
env: ${{ steps.deployment.outputs.env }}
env_url: https://github.com/github/super-linter

View file

@ -1,89 +1,40 @@
---
#################################
#################################
## Deploy Docker Image Release ##
#################################
#################################
name: Deploy super-linter release container images
#
# Documentation:
# https://help.github.com/en/articles/workflow-syntax-for-github-actions
#
name: Deploy Release
###########################################
# Start the job upon publishing a release #
###########################################
on:
# Start when a release is published
release:
types: [published]
workflow_dispatch:
# Set the input variables you want to pull in
inputs:
release_version:
description: 'version to release. Ex: v4.3.2'
required: true
default: 'v'
###############
# Set the Job #
###############
jobs:
build:
# Name the Job
name: Deploy Docker Image - Release
# Set the agent to run on
name: Deploy container image - Release
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
images:
- container-build-target: final_slim
container-image-id-prefix: slim-
- container-image-id-prefix: slim-
deployment-environment-identifier: Release-SLIM
image-id: slim
- container-build-target: final_standard
container-image-id-prefix: ""
- container-image-id-prefix: ""
deployment-environment-identifier: Release
image-id: standard
timeout-minutes: 60
###############
# Steps below #
###############
steps:
############################
# Checkout the source code #
############################
- name: Checkout Code
uses: actions/checkout@v3
###########################
# Set current date to ENV #
###########################
- name: Get current date
run: |
echo "Appending the build date contents to GITHUB_ENV..."
echo "BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> "${GITHUB_ENV}"
#######################
# Setup Docker BuildX #
#######################
- name: Setup BuildX
- name: Setup Docker BuildX
uses: docker/setup-buildx-action@v2.0.0
######################
# Login to DockerHub #
######################
- name: Login to DockerHub
uses: docker/login-action@v2.0.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
####################
# Login to GHCR.io #
####################
- name: Login to GitHub Container Registry
uses: docker/login-action@v2.0.0
with:
@ -91,10 +42,7 @@ jobs:
username: ${{ secrets.GCR_USERNAME }}
password: ${{ secrets.GCR_TOKEN }}
###########################
# Get the current release #
###########################
- name: Get current Release number
- name: Get current release identifier
# shellcheck disable=SC2062
run: |
echo "RELEASE_VERSION=$(echo ${{ github.event.release.name }} \
@ -106,10 +54,7 @@ jobs:
>> "${GITHUB_ENV}"
fi
#########################
# Update deployment API #
#########################
- name: Start deployment
- name: Start deployment for the ${{ matrix.images.deployment-environment-identifier }} environment
uses: bobheadxi/deployments@v1.3.0
id: deployment
with:
@ -117,33 +62,19 @@ jobs:
token: ${{ secrets.GITHUB_TOKEN }}
env: ${{ matrix.images.deployment-environment-identifier }}
######################################
# Build the docker image and push it #
######################################
- name: Build Docker image - ${{ matrix.images.image-id }}
uses: docker/build-push-action@v3.1.1
# We don't rebuild the image to avoid that the latest tag and the release tags don't point to what the release tag is pointing to.
# Instead, we pull the latest image and tag it.
- name: Add release tags and push image
uses: akhilerm/tag-push-action@v2.0.0
with:
context: .
file: ./Dockerfile
build-args: |
BUILD_DATE=${{ env.BUILD_DATE }}
BUILD_REVISION=${{ github.sha }}
BUILD_VERSION=${{ github.sha }}
load: false
push: true
tags: |
github/super-linter:${{ matrix.images.container-image-id-prefix }}latest
src: ghcr.io/github/super-linter:${{ matrix.images.container-image-id-prefix }}latest
dst: |
github/super-linter:${{ matrix.images.container-image-id-prefix }}v4
github/super-linter:${{ matrix.images.container-image-id-prefix }}${{ env.RELEASE_VERSION }}
ghcr.io/github/super-linter:${{ matrix.images.container-image-id-prefix }}latest
ghcr.io/github/super-linter:${{ matrix.images.container-image-id-prefix }}v4
ghcr.io/github/super-linter:${{ matrix.images.container-image-id-prefix }}${{ env.RELEASE_VERSION }}
target: "${{ matrix.images.container-build-target }}"
#########################
# Update Deployment API #
#########################
- name: Update deployment status
- name: Update the deployment status for the ${{ matrix.images.deployment-environment-identifier }} environment
uses: bobheadxi/deployments@v1.3.0
if: always()
with:
@ -154,10 +85,7 @@ jobs:
env: ${{ steps.deployment.outputs.env }}
env_url: https://github.com/github/super-linter/releases/tag/${{ env.RELEASE_VERSION }}
#######################################################
# Create a GitHub Issue with the info from this build #
#######################################################
- name: Create GitHub Issue for failure
- name: Create a GitHub issue on failure
if: failure()
uses: actions/github-script@v6
id: create-issue
@ -174,14 +102,10 @@ jobs:
console.log('create', create)
return create.data.number
############################
# Assign admins on failure #
############################
- name: Assign Admins on failure
- name: Assign admins to the issue created on failure
uses: actions/github-script@v6
if: failure()
with:
# https://octokit.github.io/rest.js/v18#issues-create
github-token: ${{secrets.GITHUB_TOKEN}}
script: |
github.rest.issues.addAssignees({

View file

@ -1,71 +0,0 @@
---
##############################
##############################
## Repository Visualization ##
##############################
##############################
#
# Documentation:
# https://help.github.com/en/articles/workflow-syntax-for-github-actions
#
name: Repository Visualization
###########################################
# Start the job on all push or PR to main #
###########################################
on:
schedule:
# Sunday at 5:00pm
- cron: "0 17 * * 0"
###############
# Set the Job #
###############
permissions:
contents: read
jobs:
build:
# Name the Job
name: Repository Visualization
# Set the agent to run on
runs-on: ubuntu-latest
timeout-minutes: 60
###############
# Steps below #
###############
steps:
############################
# Checkout the source code #
############################
- name: Checkout Code
uses: actions/checkout@v3
##############################
# Create Visualization Files #
##############################
- name: Create Visualization
uses: githubocto/repo-visualizer@0.8.2
with:
output_file: "./diagram.svg"
should_push: false
###########################
# Configure the AWS creds #
###########################
- name: Configure AWS Credentials
if: success()
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }}
aws-region: ${{ secrets.AWS_REGION }}
############################
# Push the image to aws s3 #
############################
- name: Push diagram to s3
if: success()
run: aws s3 cp diagram.svg s3://super-linter/diagram.svg

View file

@ -24,13 +24,6 @@ jobs:
- name: Checkout code
uses: actions/checkout@v3
# ##########################
# # Build the docker image #
# ##########################
- name: Build an image from Dockerfile
run: |
docker build -t "docker.io/github/super-linter:${{ github.sha }}" .
#################################
# Run Trivy Scan of source code #
#################################
@ -48,6 +41,5 @@ jobs:
#################################
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
if: always()
with:
sarif_file: 'report.sarif'

View file

@ -207,6 +207,10 @@ RUN mkdir -p /home/r-library \
&& Rscript -e "install.packages(c('lintr','purrr'), repos = 'https://cloud.r-project.org/')" \
&& R -e "install.packages(list.dirs('/home/r-library',recursive = FALSE), repos = NULL, type = 'source')"
# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub
# Store the key here because the above host is sometimes down, and breaks our builds
COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub
###################
# Install Kubeval #
###################
@ -214,9 +218,9 @@ RUN wget --tries=5 -q -O kubeval-linux-amd64.tar.gz https://github.com/instrumen
&& tar xf kubeval-linux-amd64.tar.gz \
&& mv kubeval /usr/local/bin \
&& rm kubeval-linux-amd64.tar.gz \
##################
# Install ktlint #
##################
##################
# Install ktlint #
##################
&& curl --retry 5 --retry-delay 5 -sSLO https://github.com/pinterest/ktlint/releases/latest/download/ktlint \
&& chmod a+x ktlint \
&& mv "ktlint" /usr/bin/ \
@ -225,7 +229,6 @@ RUN wget --tries=5 -q -O kubeval-linux-amd64.tar.gz https://github.com/instrumen
####################
# Install dart-sdk #
####################
&& wget --tries=5 -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub \
&& wget --tries=5 -q https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk \
&& apk add --no-cache glibc-${GLIBC_VERSION}.apk \
&& rm glibc-${GLIBC_VERSION}.apk \
@ -329,11 +332,14 @@ ENV BUILD_REVISION=$BUILD_REVISION
ENV BUILD_VERSION=$BUILD_VERSION
ENV IMAGE="slim"
# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub
# Store the key here because the above host is sometimes down, and breaks our builds
COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub
######################################
# Install Phive dependencies and git #
######################################
RUN wget --tries=5 -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub \
&& wget --tries=5 -q https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk \
RUN wget --tries=5 -q https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk \
&& apk add --no-cache \
bash \
ca-certificates \

View file

@ -48,10 +48,6 @@ The super-linter finds issues and reports them to the console output. Fixes are
The design of the **Super-Linter** is currently to allow linting to occur in **GitHub Actions** as a part of continuous integration occurring on pull requests as the commits get pushed. It works best when commits are being pushed early and often to a branch with an open or draft pull request. There is some desire to move this closer to local development for faster feedback on linting errors but this is not yet supported.
### Repository Visualization
![Visualization of the codebase](https://super-linter.s3.us-west-2.amazonaws.com/diagram.svg)
## Supported Linters
Developers on **GitHub** can call the **GitHub Action** to lint their codebase with the following list of linters:
@ -60,7 +56,7 @@ Developers on **GitHub** can call the **GitHub Action** to lint their codebase w
| --------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| **Ansible** | [ansible-lint](https://github.com/ansible/ansible-lint) |
| **AWS CloudFormation templates** | [cfn-lint](https://github.com/aws-cloudformation/cfn-python-lint/) |
| **Azure Resource Manager (ARM)** | [arm-ttk](https://github.com/azure/arm-ttk) |
| **Azure Resource Manager (ARM)** | [arm-ttk](https://github.com/azure/arm-ttk) |
| **C++** | [cpp-lint](https://github.com/cpplint/cpplint) / [clang-format](https://clang.llvm.org/docs/ClangFormatStyleOptions.html) |
| **C#** | [dotnet-format](https://github.com/dotnet/format) / [clang-format](https://clang.llvm.org/docs/ClangFormatStyleOptions.html) |
| **CSS** | [stylelint](https://stylelint.io/) |
@ -141,11 +137,6 @@ This file should have the following code:
#################################
name: Lint Code Base
#
# Documentation:
# https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions
#
#############################
# Start the job on all push #
#############################
@ -448,11 +439,6 @@ Example workflow:
#################################
name: Lint Code Base
#
# Documentation:
# https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions
#
#############################
# Start the job on all push #
#############################

9
dependencies/sgerrand.rsa.pub vendored Normal file
View file

@ -0,0 +1,9 @@
-----BEGIN PUBLIC KEY-----
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApZ2u1KJKUu/fW4A25y9m
y70AGEa/J3Wi5ibNVGNn1gT1r0VfgeWd0pUybS4UmcHdiNzxJPgoWQhV2SSW1JYu
tOqKZF5QSN6X937PTUpNBjUvLtTQ1ve1fp39uf/lEXPpFpOPL88LKnDBgbh7wkCp
m2KzLVGChf83MS0ShL6G9EQIAUxLm99VpgRjwqTQ/KfzGtpke1wqws4au0Ab4qPY
KXvMLSPLUp7cfulWvhmZSegr5AdhNw5KNizPqCJT8ZrGvgHypXyiFvvAH5YRtSsc
Zvo9GI2e2MaZyo9/lvb+LbLEJZKEQckqRj4P26gmASrZEPStwc+yqy1ShHLA0j6m
1QIDAQAB
-----END PUBLIC KEY-----