From 83badbc49c1852364afbcfbbf65bbfc75a92b957 Mon Sep 17 00:00:00 2001 From: Lukas Gravley Date: Tue, 25 May 2021 10:14:43 -0500 Subject: [PATCH] Image 4.0 (#1565) * adding new one... * adding automation * linter * pull apart for faster builds * maybe * update automation * fix that * prime is smart * fixed it * quotes * fix build image * inpec * typo * forgot backslash * fixed hairbrain logic * this should help cleanup * i hate r * order * maybe fix r * maybe fix r * update readme * remove spaces * update --- .automation/validate-docker-labels.sh | 8 +- .github/CODEOWNERS | 2 +- .github/workflows/deploy-DEV-slim.yml | 171 +++++++++ ...deploy-DEV.yml => deploy-DEV-standard.yml} | 4 +- .github/workflows/deploy-PROD.yml | 19 +- .github/workflows/deploy-RELEASE.yml | 45 ++- Dockerfile | 17 +- Dockerfile-slim | 340 ++++++++++++++++++ Makefile | 16 +- README.md | 50 +++ lib/linter.sh | 41 +++ .../super-linter/controls/super_linter.rb | 138 ++++--- 12 files changed, 780 insertions(+), 71 deletions(-) create mode 100644 .github/workflows/deploy-DEV-slim.yml rename .github/workflows/{deploy-DEV.yml => deploy-DEV-standard.yml} (98%) create mode 100644 Dockerfile-slim diff --git a/.automation/validate-docker-labels.sh b/.automation/validate-docker-labels.sh index fb3bee5f..58bc77b8 100755 --- a/.automation/validate-docker-labels.sh +++ b/.automation/validate-docker-labels.sh @@ -10,6 +10,7 @@ # GITHUB_WORKSPACE="${GITHUB_WORKSPACE}" # GitHub Workspace # GITHUB_SHA="${GITHUB_SHA}" # Sha used to create this branch # BUILD_DATE="${BUILD_DATE}" # Date the container was built +IMAGE="${1}" # Image of the super-linter we build BUILD_REVISION="${GITHUB_SHA}" # GitHub Sha BUILD_VERSION="${GITHUB_SHA}" # Version of the container ORG_REPO="github/super-linter" # Org/repo @@ -46,7 +47,12 @@ ValidateLabel() { ######################## # Get the docker label # ######################## - LABEL=$(docker inspect --format "{{ index .Config.Labels \"${CONTAINER_KEY}\" }}" "${REGISTRY}/${ORG_REPO}:${GITHUB_SHA}") + LABEL='' + if [[ "${IMAGE}" == "slim" ]]; then + LABEL=$(docker inspect --format "{{ index .Config.Labels \"${CONTAINER_KEY}\" }}" "${REGISTRY}/${ORG_REPO}:slim-${GITHUB_SHA}") + else + LABEL=$(docker inspect --format "{{ index .Config.Labels \"${CONTAINER_KEY}\" }}" "${REGISTRY}/${ORG_REPO}:${GITHUB_SHA}") + fi ################### # Check the value # diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 13c68201..5bcd67b1 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,4 +1,4 @@ ###################################################################### # These owners will be the default owners for everything in the repo # ###################################################################### -* @admiralawkbar @jwiebalk @zkoppert @IAmHughes @nemchik @Hanse00 @github-actions @GaboFDC @ferrarimarco +* @admiralawkbar @jwiebalk @zkoppert @nemchik @Hanse00 @github-actions @GaboFDC @ferrarimarco diff --git a/.github/workflows/deploy-DEV-slim.yml b/.github/workflows/deploy-DEV-slim.yml new file mode 100644 index 00000000..5717b935 --- /dev/null +++ b/.github/workflows/deploy-DEV-slim.yml @@ -0,0 +1,171 @@ +--- +######################### +######################### +## Deploy Docker Image ## +######################### +######################### + +# +# Documentation: +# https://help.github.com/en/articles/workflow-syntax-for-github-actions +# + +####################################### +# Start the job on all push to master # +####################################### +############################# +# Start the job on all push # +############################# +on: + push: + branches-ignore: [master] + pull_request: + branches-ignore: [ ] + +############### +# Set the Job # +############### +jobs: + build: + # Name the Job + name: Deploy Docker Image - DEV - SLIM + # Set the agent to run on + runs-on: ubuntu-latest + # Prevent duplicate run from happening when a forked push is committed + if: github.event_name == 'push' || + github.event.pull_request.head.repo.full_name != github.repository + ################## + # Load all steps # + ################## + steps: + ########################## + # Checkout the code base # + ########################## + - name: Checkout Code + uses: actions/checkout@v2.3.4 + with: + # Full git history is needed to get a proper list + # of changed files within `super-linter` + fetch-depth: 0 + + ######################## + # Get the current date # + ######################## + - name: Get current date + run: echo "BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> ${GITHUB_ENV} + + ################################### + # Build image locally for testing # + ################################### + + ########################################## + # Build and Push containers to registries # + ########################################### + - name: Build Docker image - SLIM + run: | + docker build \ + --build-arg BUILD_DATE=${{ env.BUILD_DATE }} \ + --build-arg BUILD_REVISION=${{ github.sha }} \ + --build-arg BUILD_VERSION=${{ github.sha }} \ + -t ghcr.io/github/super-linter:slim-${{ github.sha }} \ + -t ghcr.io/github/super-linter:slim-test -f Dockerfile-slim . + + ######################################## + # Validates the metadata docker labels # + ######################################## + - name: Run Docker label test cases + shell: bash + run: .automation/validate-docker-labels.sh "slim" + + ######################################## + # Edit action.yml for test local build # + ######################################## + - name: Edit an action.yml file for test local build + run: | + sed -i "s/super-linter:.*/super-linter:slim-${GITHUB_SHA}'/g" action.yml + + ###################### + # Gather information # + ###################### + - name: Gather information about the runtime environment + shell: bash + run: | + make info + + ########################## + # Test in action context # + ########################## + # Test the built image in the actions context. + # Not the container directly, and not using RUN_LOCAL=true + - name: Test the local action + uses: ./ + env: + ACTIONS_RUNNER_DEBUG: true + ERROR_ON_MISSING_EXEC_BIT: true + VALIDATE_ALL_CODEBASE: false + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + ############################################################### + # Fix file and dir ownership. # + # Workaround for https://github.com/actions/runner/issues/434 # + ############################################################### + - name: Fix file and directory ownership + shell: bash + run: | + sudo chown -R "$(id -u)":"$(id -g)" "$(pwd)" + + ################## + # Run test cases # + ################## + - name: Run the test suite + shell: bash + run: | + make IMAGE=slim test + + ########################## + # Codacy Coverage Report # + ########################## + - name: Upload the code coverage report + uses: codacy/codacy-coverage-reporter-action@1.0.1 + # Dependabot does not have priv to see the secret, so will + # fail opn bump jobs... + continue-on-error: true + with: + project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} + coverage-reports: test/reports/cobertura/runTests.sh/cobertura.xml + + ##################################### + # Run Linter against Test code base # + ##################################### + - name: Run Test Cases - SLIM + shell: bash + run: | + docker run \ + -e RUN_LOCAL=true \ + -e TEST_CASE_RUN=true \ + -e ANSIBLE_DIRECTORY=.automation/test/ansible \ + -e ACTIONS_RUNNER_DEBUG=true \ + -e ERROR_ON_MISSING_EXEC_BIT=true \ + -v ${GITHUB_WORKSPACE}:/tmp/lint \ + ghcr.io/github/super-linter:slim-${GITHUB_SHA} + + ######################################### + # Clean code base to run against it all # + ######################################### + - name: Clean Test code base for additional testing + shell: bash + run: .automation/clean-code-base-for-tests.sh + + ############################################ + # Run Linter against ALL cleaned code base # + ############################################ + - name: Run against all code base - SLIM + shell: bash + run: | + docker run \ + -e RUN_LOCAL=true \ + -e OUTPUT_DETAILS=detailed \ + -e ACTIONS_RUNNER_DEBUG=true \ + -e ERROR_ON_MISSING_EXEC_BIT=true \ + -v ${GITHUB_WORKSPACE}:/tmp/lint \ + ghcr.io/github/super-linter:slim-${GITHUB_SHA} diff --git a/.github/workflows/deploy-DEV.yml b/.github/workflows/deploy-DEV-standard.yml similarity index 98% rename from .github/workflows/deploy-DEV.yml rename to .github/workflows/deploy-DEV-standard.yml index fb7bb111..b2524ab4 100644 --- a/.github/workflows/deploy-DEV.yml +++ b/.github/workflows/deploy-DEV-standard.yml @@ -60,7 +60,7 @@ jobs: ########################################### # Build and Push containers to registries # ########################################### - - name: Build Docker image + - name: Build Docker standard image run: | docker build \ --build-arg BUILD_DATE=${{ env.BUILD_DATE }} \ @@ -136,7 +136,7 @@ jobs: ##################################### # Run Linter against Test code base # ##################################### - - name: Run Test Cases + - name: Run Test Cases - standard shell: bash run: | docker run \ diff --git a/.github/workflows/deploy-PROD.yml b/.github/workflows/deploy-PROD.yml index fd9e24bf..2184e994 100644 --- a/.github/workflows/deploy-PROD.yml +++ b/.github/workflows/deploy-PROD.yml @@ -85,7 +85,7 @@ jobs: ########################################### # Build and Push containers to registries # ########################################### - - name: Build and push + - name: Build and push - Standard uses: docker/build-push-action@v2.4.0 with: context: . @@ -99,6 +99,23 @@ jobs: github/super-linter:latest ghcr.io/github/super-linter:latest + ########################################### + # Build and Push containers to registries # + ########################################### + - name: Build and push - SLIM + uses: docker/build-push-action@v2.4.0 + with: + context: . + file: ./Dockerfile-slim + build-args: | + BUILD_DATE=${{ env.BUILD_DATE }} + BUILD_REVISION=${{ github.sha }} + BUILD_VERSION=${{ github.sha }} + push: true + tags: | + github/super-linter:slim-latest + ghcr.io/github/super-linter:slim-latest + ####################################################### # Create a GitHub Issue with the info from this build # ####################################################### diff --git a/.github/workflows/deploy-RELEASE.yml b/.github/workflows/deploy-RELEASE.yml index 82cabbac..8f125e06 100644 --- a/.github/workflows/deploy-RELEASE.yml +++ b/.github/workflows/deploy-RELEASE.yml @@ -126,7 +126,7 @@ jobs: ########################################### # Build and Push containers to registries # ########################################### - - name: Build and push + - name: Build and push - Standard uses: docker/build-push-action@v2.4.0 with: context: . @@ -144,6 +144,35 @@ jobs: ghcr.io/github/super-linter:v3 ghcr.io/github/super-linter:${{ env.RELEASE_VERSION }} + ########################################### + # Build and Push containers to registries # + ########################################### + - name: Build and push - SLIM + uses: docker/build-push-action@v2.4.0 + with: + context: . + file: ./Dockerfile-slim + build-args: | + BUILD_DATE=${{ env.BUILD_DATE }} + BUILD_REVISION=${{ github.sha }} + BUILD_VERSION=${{ github.sha }} + push: true + tags: | + github/super-linter:slim-latest + github/super-linter:slim-v3 + github/super-linter:slim-${{ env.RELEASE_VERSION }} + ghcr.io/github/super-linter:slim-latest + ghcr.io/github/super-linter:slim-v3 + ghcr.io/github/super-linter:slim-${{ env.RELEASE_VERSION }} + + ######################## + # Get the current date # + ######################## + - name: Update Release Body String + run: | + echo "UPDATED_BODY_STRING=$(echo "${{ github.event.issue.body }}" | \ + sed -E ':a;N;$!ba;s/\r{0,1}\n/\\n/g')" >> ${GITHUB_ENV} + ############################# # Create the GitHub Release # ############################# @@ -151,16 +180,20 @@ jobs: if: success() id: create_release run: | - UPDATED_BODY_STRING=$(echo "${{ github.event.issue.body }}" | sed -E ':a;N;$!ba;s/\r{0,1}\n/\\n/g') + UPDATED_BODY_STRING=$(echo "${{ github.event.issue.body }}" | \ + sed -E ':a;N;$!ba;s/\r{0,1}\n/\\n/g') curl --fail -X POST \ --url https://api.github.com/repos/${{ github.repository }}/releases \ -H 'Accept: application/vnd.github.v3+json' \ -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ -H 'Content-Type: application/json' \ - --data "{ \"tag_name\": \"${{ env.RELEASE_VERSION }}\", \"target_commitish\": \"${{ env.PR_REF }}\", \ - \"name\": \"Release ${{ env.RELEASE_VERSION }}\", \"draft\": false, \"prerelease\": false, \ - \"body\": \"${UPDATED_BODY_STRING}\" \ - \"repository_action_release_attributes\": { \"published_on_marketplace\": true }}" + --data "{ \"tag_name\": \"${{ env.RELEASE_VERSION }}\", \ + \"target_commitish\": \"${{ env.PR_REF }}\", \ + \"name\": \"Release ${{ env.RELEASE_VERSION }}\", \ + \"draft\": false, \"prerelease\": false, \ + \"body\": \"${{ env.UPDATED_BODY_STRING }}\" \ + \"repository_action_release_attributes\": \ + { \"published_on_marketplace\": true }}" ##################################################### # Create the Required status check for Stack Linter # diff --git a/Dockerfile b/Dockerfile index 2b0de134..46f004c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,9 +18,8 @@ FROM alpine/terragrunt:0.15.3 as terragrunt FROM mvdan/shfmt:v3.2.4 as shfmt FROM accurics/terrascan:1.6.0 as terrascan FROM hadolint/hadolint:latest-alpine as dockerfile-lint -FROM ghcr.io/assignuser/chktex-alpine:0.1.1 as chktex +FROM assignuser/chktex-alpine:v0.1.1 as chktex FROM garethr/kubeval:0.15.0 as kubeval -FROM ghcr.io/assignuser/lintr-lib:0.2.0 as lintr-lib ################## # Get base image # @@ -226,11 +225,6 @@ COPY --from=kubeval /kubeval /usr/bin/ ################# COPY --from=shfmt /bin/shfmt /usr/bin/ -################# -# Install Litnr # -################# -COPY --from=lintr-lib /usr/lib/R/library/ /home/r-library - ################## # Install ktlint # ################## @@ -239,7 +233,6 @@ RUN curl --retry 5 --retry-delay 5 -sSLO https://github.com/pinterest/ktlint/rel && mv "ktlint" /usr/bin/ \ && terrascan init \ && cd ~ && touch .chktexrc \ - && R -e "install.packages(list.dirs('/home/r-library',recursive = FALSE), repos = NULL, type = 'source')" \ #################### # Install dart-sdk # #################### @@ -297,6 +290,7 @@ RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/community/" >> /etc/apk/repo ################################################################################ # Grab small clean image ####################################################### ################################################################################ +FROM ghcr.io/assignuser/lintr-lib:0.2.0 as lintr-lib FROM alpine:3.13.5 as final ############################ @@ -377,7 +371,12 @@ COPY --from=base_image /usr/include/ /usr/include/ COPY --from=base_image /lib/ /lib/ COPY --from=base_image /bin/ /bin/ COPY --from=base_image /node_modules/ /node_modules/ -COPY --from=base_image /home/r-library /home/r-library + +################# +# Install Litnr # +################# +COPY --from=lintr-lib /usr/lib/R/library/ /home/r-library +RUN R -e "install.packages(list.dirs('/home/r-library',recursive = FALSE), repos = NULL, type = 'source')" ######################################## # Add node packages to path and dotnet # diff --git a/Dockerfile-slim b/Dockerfile-slim new file mode 100644 index 00000000..0da0f21f --- /dev/null +++ b/Dockerfile-slim @@ -0,0 +1,340 @@ +########################################### +########################################### +## Dockerfile to run GitHub Super-Linter ## +## SLIM IMAGE ## +########################################### +########################################### + +# Removed Linters: +# - rust +# - dotenv +# - armttk +# - pwsh + +######################################### +# Get dependency images as build stages # +######################################### +FROM cljkondo/clj-kondo:2021.04.23-alpine as clj-kondo +FROM mstruebing/editorconfig-checker:2.3.5 as editorconfig-checker +FROM yoheimuta/protolint:v0.31.0 as protolint +FROM golangci/golangci-lint:v1.40.0 as golangci-lint +FROM koalaman/shellcheck:v0.7.2 as shellcheck +FROM wata727/tflint:0.28.1 as tflint +FROM alpine/terragrunt:0.15.3 as terragrunt +FROM mvdan/shfmt:v3.2.4 as shfmt +FROM accurics/terrascan:1.6.0 as terrascan +FROM hadolint/hadolint:latest-alpine as dockerfile-lint +FROM assignuser/chktex-alpine:v0.1.1 as chktex +FROM garethr/kubeval:0.15.0 as kubeval + +################## +# Get base image # +################## +FROM python:3.9-alpine as base_image + +################################ +# Set ARG values used in Build # +################################ +# Dart Linter +## stable dart sdk: https://dart.dev/get-dart#release-channels +ARG DART_VERSION='2.8.4' +## install alpine-pkg-glibc (glibc compatibility layer package for Alpine Linux) +ARG GLIBC_VERSION='2.31-r0' + +#################### +# Run APK installs # +#################### +RUN apk add --no-cache \ + bash \ + cargo \ + coreutils \ + curl \ + file \ + gcc \ + git git-lfs\ + go \ + gnupg \ + icu-libs \ + jq \ + krb5-libs \ + libc-dev libcurl libffi-dev libgcc \ + libintl libssl1.1 libstdc++ \ + libxml2-dev libxml2-utils \ + linux-headers \ + lttng-ust-dev \ + make \ + musl-dev \ + npm nodejs-current \ + openjdk8-jre \ + openssl-dev \ + perl perl-dev \ + py3-setuptools python3-dev \ + R R-dev R-doc \ + readline-dev \ + ruby ruby-dev ruby-bundler ruby-rdoc \ + zlib zlib-dev + +######################################## +# Copy dependencies files to container # +######################################## +COPY dependencies/* / + +################################ +# Installs python dependencies # +################################ +RUN pip3 install --no-cache-dir pipenv \ + # Bug in hadolint thinks pipenv is pip + # hadolint ignore=DL3042 + && pipenv install --clear --system \ +#################### +# Run NPM Installs # +#################### + && npm config set package-lock false \ + && npm config set loglevel error \ + && npm --no-cache install \ + && npm audit fix \ +############################## +# Installs ruby dependencies # +############################## + && bundle install + +############################## +# Installs Perl dependencies # +############################## +RUN curl --retry 5 --retry-delay 5 -sL https://cpanmin.us/ | perl - -nq --no-wget Perl::Critic + +###################### +# Install shellcheck # +###################### +COPY --from=shellcheck /bin/shellcheck /usr/bin/ + +##################### +# Install Go Linter # +##################### +COPY --from=golangci-lint /usr/bin/golangci-lint /usr/bin/ + +################## +# Install TFLint # +################## +COPY --from=tflint /usr/local/bin/tflint /usr/bin/ + +##################### +# Install Terrascan # +##################### +COPY --from=terrascan /go/bin/terrascan /usr/bin/ + +###################### +# Install Terragrunt # +###################### +COPY --from=terragrunt /usr/local/bin/terragrunt /usr/bin/ + +###################### +# Install protolint # +###################### +COPY --from=protolint /usr/local/bin/protolint /usr/bin/ + +##################### +# Install clj-kondo # +##################### +COPY --from=clj-kondo /bin/clj-kondo /usr/bin/ + +################################ +# Install editorconfig-checker # +################################ +COPY --from=editorconfig-checker /usr/bin/ec /usr/bin/editorconfig-checker + +############################### +# Install hadolint dockerfile # +############################### +COPY --from=dockerfile-lint /bin/hadolint /usr/bin/hadolint + +################## +# Install chktex # +################## +COPY --from=chktex /usr/bin/chktex /usr/bin/ + +################### +# Install kubeval # +################### +COPY --from=kubeval /kubeval /usr/bin/ + +################# +# Install shfmt # +################# +COPY --from=shfmt /bin/shfmt /usr/bin/ + +################## +# Install ktlint # +################## +RUN curl --retry 5 --retry-delay 5 -sSLO https://github.com/pinterest/ktlint/releases/latest/download/ktlint \ + && chmod a+x ktlint \ + && mv "ktlint" /usr/bin/ \ + && terrascan init \ + && cd ~ && touch .chktexrc \ +#################### +# Install dart-sdk # +#################### + && wget --tries=5 -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub \ + && wget --tries=5 -q https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk \ + && apk add --no-cache glibc-${GLIBC_VERSION}.apk \ + && rm glibc-${GLIBC_VERSION}.apk \ + && wget --tries=5 -q https://storage.googleapis.com/dart-archive/channels/stable/release/${DART_VERSION}/sdk/dartsdk-linux-x64-release.zip -O - -q | unzip -q - \ + && chmod +x dart-sdk/bin/dart* \ + && mv dart-sdk/bin/* /usr/bin/ && mv dart-sdk/lib/* /usr/lib/ && mv dart-sdk/include/* /usr/include/ \ + && rm -r dart-sdk/ \ +################################ +# Create and install Bash-Exec # +################################ + && printf '#!/bin/bash \n\nif [[ -x "$1" ]]; then exit 0; else echo "Error: File:[$1] is not executable"; exit 1; fi' > /usr/bin/bash-exec \ + && chmod +x /usr/bin/bash-exec + +################################################# +# Install Raku and additional Edge dependencies # +################################################# +# Basic setup, programs and init +RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/community/" >> /etc/apk/repositories \ + && apk add --no-cache rakudo zef \ +###################### +# Install CheckStyle # +###################### + && CHECKSTYLE_LATEST=$(curl -s https://api.github.com/repos/checkstyle/checkstyle/releases/latest \ + | grep browser_download_url \ + | grep ".jar" \ + | cut -d '"' -f 4) \ + && curl --retry 5 --retry-delay 5 -sSL "$CHECKSTYLE_LATEST" \ + --output /usr/bin/checkstyle \ +################################# +# Install luacheck and luarocks # +################################# + && wget --tries=5 -q https://www.lua.org/ftp/lua-5.3.5.tar.gz -O - -q | tar -xzf - \ + && cd lua-5.3.5 \ + && make linux \ + && make install \ + && cd .. && rm -r lua-5.3.5/ \ + && wget --tries=5 -q https://github.com/cvega/luarocks/archive/v3.3.1-super-linter.tar.gz -O - -q | tar -xzf - \ + && cd luarocks-3.3.1-super-linter \ + && ./configure --with-lua-include=/usr/local/include \ + && make \ + && make -b install \ + && cd .. \ + && rm -r luarocks-3.3.1-super-linter/ \ + && luarocks install luacheck \ + && mv /etc/R/* /usr/lib/R/etc/ \ + && find /node_modules/ -type f -name 'LICENSE' -exec rm {} + \ + && find /node_modules/ -type f -name '*.md' -exec rm {} + \ + && find /node_modules/ -type f -name '*.txt' -exec rm {} + \ + && find /usr/ -type f -name '*.md' -exec rm {} + + +################################################################################ +# Grab small clean image ####################################################### +################################################################################ +FROM ghcr.io/assignuser/lintr-lib:0.2.0 as lintr-lib +FROM alpine:3.13.5 as final + +############################ +# Get the build arguements # +############################ +ARG BUILD_DATE +ARG BUILD_REVISION +ARG BUILD_VERSION +## install alpine-pkg-glibc (glibc compatibility layer package for Alpine Linux) +ARG GLIBC_VERSION='2.31-r0' + +######################################### +# Label the instance and set maintainer # +######################################### +LABEL com.github.actions.name="GitHub Super-Linter" \ + com.github.actions.description="Lint your code base with GitHub Actions" \ + com.github.actions.icon="code" \ + com.github.actions.color="red" \ + maintainer="GitHub DevOps " \ + org.opencontainers.image.created=$BUILD_DATE \ + org.opencontainers.image.revision=$BUILD_REVISION \ + org.opencontainers.image.version=$BUILD_VERSION \ + org.opencontainers.image.authors="GitHub DevOps " \ + org.opencontainers.image.url="https://github.com/github/super-linter" \ + org.opencontainers.image.source="https://github.com/github/super-linter" \ + org.opencontainers.image.documentation="https://github.com/github/super-linter" \ + org.opencontainers.image.vendor="GitHub" \ + org.opencontainers.image.description="Lint your code base with GitHub Actions" + +################################################# +# Set ENV values used for debugging the version # +################################################# +ENV BUILD_DATE=$BUILD_DATE +ENV BUILD_REVISION=$BUILD_REVISION +ENV BUILD_VERSION=$BUILD_VERSION +ENV IMAGE="slim" + +###################################### +# Install Phive dependencies and git # +###################################### +RUN wget --tries=5 -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub \ + && wget --tries=5 -q https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk \ + && apk add --no-cache \ + bash \ + ca-certificates \ + git git-lfs \ + glibc-${GLIBC_VERSION}.apk \ + gnupg \ + php7 php7-phar php7-json php7-mbstring php-xmlwriter \ + php7-tokenizer php7-ctype php7-curl php7-dom php7-simplexml \ + && rm glibc-${GLIBC_VERSION}.apk \ + && wget -q --tries=5 -O phive.phar https://phar.io/releases/phive.phar \ + && wget -q --tries=5 -O phive.phar.asc https://phar.io/releases/phive.phar.asc \ + && PHAR_KEY_ID="0x9D8A98B29B2D5D79" \ + && ( gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$PHAR_KEY_ID" \ + || gpg --keyserver pgp.mit.edu --recv-keys "$PHAR_KEY_ID" \ + || gpg --keyserver keyserver.pgp.com --recv-keys "$PHAR_KEY_ID" ) \ + && gpg --verify phive.phar.asc phive.phar \ + && chmod +x phive.phar \ + && mv phive.phar /usr/local/bin/phive \ + && rm phive.phar.asc \ + && phive --no-progress install --trust-gpg-keys \ + 31C7E470E2138192,CF1A108D0E7AE720,8A03EA3B385DBAA1,12CE0F1D262429A5 \ + --target /usr/bin phpstan@^0.12.64 psalm@^3.18.2 phpcs@^3.5.8 + +################################# +# Copy the libraries into image # +################################# +COPY --from=base_image /usr/bin/ /usr/bin/ +COPY --from=base_image /usr/local/bin/ /usr/local/bin/ +COPY --from=base_image /usr/local/lib/ /usr/local/lib/ +COPY --from=base_image /usr/local/share/ /usr/local/share/ +COPY --from=base_image /usr/lib/ /usr/lib/ +COPY --from=base_image /usr/share/ /usr/share/ +COPY --from=base_image /usr/include/ /usr/include/ +COPY --from=base_image /lib/ /lib/ +COPY --from=base_image /bin/ /bin/ +COPY --from=base_image /node_modules/ /node_modules/ + +################# +# Install Litnr # +################# +COPY --from=lintr-lib /usr/lib/R/library/ /home/r-library +RUN R -e "install.packages(list.dirs('/home/r-library',recursive = FALSE), repos = NULL, type = 'source')" + +######################################## +# Add node packages to path and dotnet # +######################################## +ENV PATH="${PATH}:/node_modules/.bin" + +############################# +# Copy scripts to container # +############################# +COPY lib /action/lib + +################################## +# Copy linter rules to container # +################################## +COPY TEMPLATES /action/lib/.automation + +################################################ +# Run to build version file and validate image # +################################################ +RUN ACTIONS_RUNNER_DEBUG=true WRITE_LINTER_VERSIONS_FILE=true IMAGE=slim /action/lib/linter.sh + +###################### +# Set the entrypoint # +###################### +ENTRYPOINT ["/action/lib/linter.sh"] diff --git a/Makefile b/Makefile index c6a0f33e..cf93cf71 100644 --- a/Makefile +++ b/Makefile @@ -71,18 +71,30 @@ inspec-check: ## Validate inspec profiles test/inspec/super-linter SUPER_LINTER_TEST_CONTAINER_NAME := "super-linter-test" -SUPER_LINTER_TEST_CONTINER_URL := "ghcr.io/github/super-linter:test" +SUPER_LINTER_TEST_CONTINER_URL := '' +DOCKERFILE := '' +IMAGE := '' +ifeq ($(IMAGE),slim) + SUPER_LINTER_TEST_CONTINER_URL := "ghcr.io/github/super-linter:slim-test" + DOCKERFILE := "Dockerfile-slim" + IMAGE := "slim" +else + SUPER_LINTER_TEST_CONTINER_URL := "ghcr.io/github/super-linter:test" + DOCKERFILE := "Dockerfile" + IMAGE := "standard" +endif .PHONY: inspec inspec: inspec-check ## Run InSpec tests DOCKER_CONTAINER_STATE="$$(docker inspect --format "{{.State.Running}}" "$(SUPER_LINTER_TEST_CONTAINER_NAME)" 2>/dev/null || echo "")"; \ if [ "$$DOCKER_CONTAINER_STATE" = "true" ]; then docker kill "$(SUPER_LINTER_TEST_CONTAINER_NAME)"; fi && \ - docker build -t $(SUPER_LINTER_TEST_CONTAINER_NAME) -f Dockerfile . && \ + docker build -t $(SUPER_LINTER_TEST_CONTAINER_NAME) -f $(DOCKERFILE) . && \ SUPER_LINTER_TEST_CONTAINER_ID="$$(docker run -d --name "$(SUPER_LINTER_TEST_CONTAINER_NAME)" --rm -it --entrypoint /bin/ash "$(SUPER_LINTER_TEST_CONTAINER_NAME)" -c "while true; do sleep 1; done")" \ && docker run $(DOCKER_FLAGS) \ --rm \ -v "$(CURDIR)":/workspace \ -v /var/run/docker.sock:/var/run/docker.sock \ + -e IMAGE=$(IMAGE) \ -w="/workspace" \ chef/inspec exec test/inspec/super-linter\ --chef-license=accept \ diff --git a/README.md b/README.md index df12aaf7..285468df 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,9 @@ It is a simple combination of various linters, written in `bash`, to help valida - [How to use](#how-to-use) - [Example connecting GitHub Action Workflow](#example-connecting-github-action-workflow) - [Add Super-Linter badge in your repository README](#add-super-linter-badge-in-your-repository-readme) + - [Images](#images) + - [Standard Image](#standard-image) + - [Slim Image](#slim-image) - [Environment variables](#environment-variables) - [Template rules files](#template-rules-files) - [Using your own rules files](#using-your-own-rules-files) @@ -196,6 +199,53 @@ Example: _Note:_ IF you did not use `Lint Code Base` as GitHub Action name, please read [GitHub Actions Badges documentation](https://docs.github.com/en/actions/configuring-and-managing-workflows/configuring-a-workflow#adding-a-workflow-status-badge-to-your-repository) +### Images + +The **GitHub Super-Linter** now builds and supports `multiple` images. We have found as we added more linters, the image size expanded drastically. +After further investigation, we were able to see that a few linters were very disk heavy. We removed those linters and created the `slim` image. +This allows users to choose which **Super-Linter** they want to run and potentially speed up their build time. +The available images: +- `github/super-linter:v4` +- `github/super-linter:slim-v4` + +#### Standard Image + +The standard `github/super-linter:v4` comes with all supported linters. +Example usage: +```yml +################################ +# Run Linter against code base # +################################ +- name: Lint Code Base + uses: github/super-linter@v4 + env: + VALIDATE_ALL_CODEBASE: false + DEFAULT_BRANCH: master + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +``` + +#### Slim Image + +The slim `github/super-linter:slim-v4` comes with all supported linters but removes the following: +- `rust` linters +- `dotenv` linters +- `armttk` linters +- `pwsh` linters +By removing these linters, we were able to bring the image size down by `2gb` and drastically speed up the build and download time. +The behavior will be the same for non-supported languages, and will skip languages at run time. +Example usage: +```yml +################################ +# Run Linter against code base # +################################ +- name: Lint Code Base + uses: ghcr.io://github/super-linter@slim-v4 + env: + VALIDATE_ALL_CODEBASE: false + DEFAULT_BRANCH: master + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +``` + ## Environment variables The super-linter allows you to pass the following `ENV` variables to be able to trigger different functionality. diff --git a/lib/linter.sh b/lib/linter.sh index 6a0b0f14..02386def 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -12,6 +12,7 @@ ################################################################## # RUN_LOCAL="${RUN_LOCAL}" # Boolean to see if we are running locally ACTIONS_RUNNER_DEBUG="${ACTIONS_RUNNER_DEBUG:-false}" # Boolean to see even more info (debug) +IMAGE="${IMAGE:-standard}" # Version of the Super-linter (standard,slim,etc) ################################################################## # Log Vars # @@ -685,6 +686,41 @@ Footer() { exit 0 } ################################################################################ +#### Function UpdateLoopsForImage ############################################## +UpdateLoopsForImage() { + ###################################################################### + # Need to clean the array lists of the linters removed for the image # + ###################################################################### + if [[ "${IMAGE}" == "slim" ]]; then + ############################################# + # Need to remove linters for the slim image # + ############################################# + REMOVE_ARRAY=("ARM" "CSHARP" "ENV" "POWERSHELL" "RUST_2015" "RUST_2018" "RUST_CLIPPY") + + # Remove from LANGUAGE_ARRAY + echo "Removing Languages from LANGUAGE_ARRAY for slim image..." + for REMOVE_LANGUAGE in "${REMOVE_ARRAY[@]}"; do + for INDEX in "${!LANGUAGE_ARRAY[@]}"; do + if [[ ${LANGUAGE_ARRAY[INDEX]} = "${REMOVE_LANGUAGE}" ]]; then + echo "found item:[${REMOVE_LANGUAGE}], removing Language..." + unset 'LANGUAGE_ARRAY[INDEX]' + fi + done + done + + # Remove from LINTER_NAMES_ARRAY + echo "Removing Linters from LINTER_NAMES_ARRAY for slim image..." + for REMOVE_LINTER in "${REMOVE_ARRAY[@]}"; do + for INDEX in "${!LINTER_NAMES_ARRAY[@]}"; do + if [[ ${INDEX} = "${REMOVE_LINTER}" ]]; then + echo "found item:[${REMOVE_LINTER}], removing linter..." + unset 'LINTER_NAMES_ARRAY[$INDEX]' + fi + done + done + fi +} +################################################################################ #### Function Cleanup ########################################################## cleanup() { local -ri EXIT_CODE=$? @@ -704,6 +740,11 @@ trap 'cleanup' 0 1 2 3 6 14 15 ########## Header +################################################ +# Need to update the loops for the image style # +################################################ +UpdateLoopsForImage + ################################## # Get and print all version info # ################################## diff --git a/test/inspec/super-linter/controls/super_linter.rb b/test/inspec/super-linter/controls/super_linter.rb index dbd6dfab..5bc3811d 100644 --- a/test/inspec/super-linter/controls/super_linter.rb +++ b/test/inspec/super-linter/controls/super_linter.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# PUll in env vars passed +image = ENV["IMAGE"] + ################################################## # Check to see all system packages are installed # ################################################## @@ -57,12 +60,20 @@ control "super-linter-installed-packages" do "zlib" ] + # Removed linters from slim image + SLIM_IMAGE_REMOVED_PACKAGES=%w( + rustup + ) + packages.each do |item| - describe package(item) do - it { should be_installed } + if(image == "slim" && SLIM_IMAGE_REMOVED_PACKAGES.include?(item)) + next + else + describe package(item) do + it { should be_installed } + end end end - end ########################################### @@ -140,53 +151,69 @@ control "super-linter-installed-commands" do { linter_name: "yamllint"}, ] + # Removed linters from slim image + SLIM_IMAGE_REMOVED_LINTERS=%w( + arm-ttk + clippy + dotnet-format + dotenv-linter + pwsh + rustfmt + ) + linters.each do |linter| # If we didn't specify a linter command, use the linter name as a linter # command because the vast majority of linters have name == command - if(linter.key?(:linter_command)) - linter_command = linter[:linter_command] + linter_command = "" + + if(image == "slim" && SLIM_IMAGE_REMOVED_LINTERS.include?(linter[:linter_name])) + next else - linter_command = linter[:linter_name] - end - - describe command("command -v #{linter_command}") do - its("exit_status") { should eq 0 } - end - - # A few linters have a command that it's different than linter_command - if(linter.key?(:version_command)) - version_command = linter[:version_command] - else - # Check if the linter needs an option that is different from the one that - # the vast majority of linters use to get the version - if(linter.key?(:version_option)) - version_option = linter[:version_option] + if(linter.key?(:linter_command)) + linter_command = linter[:linter_command] else - version_option = default_version_option + linter_command = linter[:linter_name] end - version_command = "#{linter_command} #{version_option}" - - if(linter.key?(:expected_exit_status)) - expected_exit_status = linter[:expected_exit_status] - else - expected_exit_status = default_version_expected_exit_status + describe command("command -v #{linter_command}") do + its("exit_status") { should eq 0 } end - if(linter.key?(:expected_stdout_regex)) - expected_stdout_regex = linter[:expected_stdout_regex] + # A few linters have a command that it's different than linter_command + if(linter.key?(:version_command)) + version_command = linter[:version_command] else - expected_stdout_regex = default_expected_stdout_regex - end + # Check if the linter needs an option that is different from the one that + # the vast majority of linters use to get the version + if(linter.key?(:version_option)) + version_option = linter[:version_option] + else + version_option = default_version_option + end - ########################################################## - # Being able to run the command `linter --version` helps # - # achieve that the linter is installed, ini PATH, and # - # has the libraries needed to be able to basically run # - ########################################################## - describe command(version_command) do - its("exit_status") { should eq expected_exit_status } - its("stdout") { should match (expected_stdout_regex) } + version_command = "#{linter_command} #{version_option}" + + if(linter.key?(:expected_exit_status)) + expected_exit_status = linter[:expected_exit_status] + else + expected_exit_status = default_version_expected_exit_status + end + + if(linter.key?(:expected_stdout_regex)) + expected_stdout_regex = linter[:expected_stdout_regex] + else + expected_stdout_regex = default_expected_stdout_regex + end + + ########################################################## + # Being able to run the command `linter --version` helps # + # achieve that the linter is installed, ini PATH, and # + # has the libraries needed to be able to basically run # + ########################################################## + describe command(version_command) do + its("exit_status") { should eq expected_exit_status } + its("stdout") { should match (expected_stdout_regex) } + end end end end @@ -328,10 +355,19 @@ control "super-linter-validate-directories" do "/usr/local/share/" ] + # Removed linters from slim image + SLIM_IMAGE_REMOVED_DIRS=%w( + /home/r-library + ) + dirs.each do |item| - describe directory(item) do - it { should exist } - it { should be_directory } + if(image == "slim" && SLIM_IMAGE_REMOVED_DIRS.include?(item)) + next + else + describe directory(item) do + it { should exist } + it { should be_directory } + end end end end @@ -412,13 +448,17 @@ control "super-linter-validate-powershell-modules" do title "Super-Linter validate Powershell Modules" desc "Check that Powershell modules that Super-Linter needs are installed." - describe command("pwsh -c \"(Get-Module -Name PSScriptAnalyzer -ListAvailable | Select-Object -First 1).Name\" 2>&1") do - its("exit_status") { should eq 0 } - its("stdout") { should eq "PSScriptAnalyzer\n" } - end + if(image == "slim") + next + else + describe command("pwsh -c \"(Get-Module -Name PSScriptAnalyzer -ListAvailable | Select-Object -First 1).Name\" 2>&1") do + its("exit_status") { should eq 0 } + its("stdout") { should eq "PSScriptAnalyzer\n" } + end - describe command("pwsh -c \"(Get-Command Invoke-ScriptAnalyzer | Select-Object -First 1).Name\" 2>&1") do - its("exit_status") { should eq 0 } - its("stdout") { should eq "Invoke-ScriptAnalyzer\n" } + describe command("pwsh -c \"(Get-Command Invoke-ScriptAnalyzer | Select-Object -First 1).Name\" 2>&1") do + its("exit_status") { should eq 0 } + its("stdout") { should eq "Invoke-ScriptAnalyzer\n" } + end end end