From 22b8624f612cfd4752501a3944b35e7dd96ea7c8 Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Fri, 15 Dec 2023 11:59:36 +0100 Subject: [PATCH] Simplify container image build (#4962) --- .github/CODEOWNERS | 5 +- .github/workflows/cd.yml | 2 +- .github/workflows/release.yml | 3 +- Dockerfile | 334 ++++++++---------- Makefile | 2 +- dependencies/phive.xml | 7 +- scripts/install-arm-ttk.sh | 11 +- scripts/install-checkstyle.sh | 18 +- scripts/install-dart-sdk.sh | 19 +- scripts/install-glibc.sh | 37 ++ scripts/install-google-java-format.sh | 18 +- scripts/install-ktlint.sh | 15 +- scripts/install-lua.sh | 12 +- scripts/install-phive.sh | 50 +-- scripts/install-pwsh.sh | 15 +- .../super-linter/controls/super_linter.rb | 118 ++++++- 16 files changed, 354 insertions(+), 312 deletions(-) create mode 100755 scripts/install-glibc.sh diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 837e680e..6559d709 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,4 +1 @@ -###################################################################### -# These owners will be the default owners for everything in the repo # -###################################################################### -* @admiralawkbar @lindluni @zkoppert @nemchik @Hanse00 @GaboFDC @ferrarimarco +* @zkoppert @Hanse00 @ferrarimarco diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 78f24569..0f1c0133 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -149,6 +149,6 @@ jobs: title: "Failed to deploy to production", body: "Automation has failed us!\nMore information can be found at:\n - ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}", assignees: [ - "zkoppert", "Hanse00" + "zkoppert", "Hanse00", "ferrarimarco" ] }) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2fbafba4..0087fa75 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -132,6 +132,7 @@ jobs: body: "Automation has failed us! Failed to push release ${{ env.RELEASE_VERSION }}\nMore information can be found at:\n - ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}", assignees: [ 'zkoppert', - 'Hanse00' + 'Hanse00', + 'ferrarimarco' ] }) diff --git a/Dockerfile b/Dockerfile index 692fe3a1..8eb4f619 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,8 @@ ########################################### ########################################### +ARG GLIBC_VERSION='2.34-r0' + ######################################### # Get dependency images as build stages # ######################################### @@ -16,7 +18,7 @@ FROM ghcr.io/terraform-linters/tflint:v0.48.0 as tflint FROM ghcr.io/yannh/kubeconform:v0.6.4 as kubeconfrm FROM golang:1.21.5-alpine as golang FROM golangci/golangci-lint:v1.55.2 as golangci-lint -FROM hadolint/hadolint:latest-alpine as dockerfile-lint +FROM hadolint/hadolint:v2.12.0-alpine as dockerfile-lint FROM hashicorp/terraform:1.6.5 as terraform FROM koalaman/shellcheck:v0.9.0 as shellcheck FROM mstruebing/editorconfig-checker:2.7.2 as editorconfig-checker @@ -26,72 +28,84 @@ FROM scalameta/scalafmt:v3.7.17 as scalafmt FROM zricethezav/gitleaks:v8.18.1 as gitleaks FROM yoheimuta/protolint:0.46.3 as protolint -################## -# Get base image # -################## -FROM python:3.11.5-alpine3.17 as base_image +FROM python:3.12.1-alpine3.19 as slim + +LABEL com.github.actions.name="Super-Linter" \ + com.github.actions.description="A collection of code linters and analyzers." \ + com.github.actions.icon="code" \ + com.github.actions.color="red" \ + maintainer="@Hanse00, @ferrarimarco, @zkoppert" \ + org.opencontainers.image.authors="Super Linter Contributors: https://github.com/super-linter/super-linter/graphs/contributors" \ + org.opencontainers.image.url="https://github.com/super-linter/super-linter" \ + org.opencontainers.image.source="https://github.com/super-linter/super-linter" \ + org.opencontainers.image.documentation="https://github.com/super-linter/super-linter" \ + org.opencontainers.image.vendor="GitHub" \ + org.opencontainers.image.description="A collection of code linters and analyzers." -################################ -# Set ARG values used in Build # -################################ -ARG CLJ_KONDO_VERSION='2023.05.18' -# Dart Linter -## stable dart sdk: https://dart.dev/get-dart#release-channels -ARG DART_VERSION='2.8.4' -## install alpine-pkg-glibc (glibc compatibility layer package for Alpine Linux) -ARG GLIBC_VERSION='2.34-r0' -ARG KTLINT_VERSION='0.47.1' -# PowerShell & PSScriptAnalyzer linter -ARG PSSA_VERSION='1.21.0' -ARG PWSH_DIRECTORY='/usr/lib/microsoft/powershell' -ARG PWSH_VERSION='v7.3.1' # https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope ARG TARGETARCH -#################### -# Run APK installs # -#################### +# Install bash first so we can use it +RUN apk add --no-cache \ + bash + +SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-c"] + RUN apk add --no-cache \ - bash \ ca-certificates \ cargo \ cmake \ coreutils \ curl \ file \ - gcc \ g++ \ - git git-lfs \ + gcc \ + git \ + git-lfs \ gnupg \ icu-libs \ jpeg-dev \ jq \ krb5-libs \ - libc-dev libcurl libffi-dev libgcc \ - libintl libssl1.1 libstdc++ \ - libxml2-dev libxml2-utils \ + libc-dev \ + libcurl \ + libffi-dev \ + libgcc \ + libintl \ + libssl3 \ + libstdc++ \ + libxml2-dev \ + libxml2-utils \ linux-headers \ lttng-ust-dev \ make \ musl-dev \ net-snmp-dev \ - npm nodejs-current \ + nodejs-current \ + npm \ openjdk17-jre \ openssh-client \ openssl-dev \ parallel \ - perl perl-dev \ - py3-setuptools python3-dev \ + perl \ + perl-dev \ py3-pyflakes \ - R R-dev R-doc \ + py3-setuptools \ + python3-dev \ + R \ + R-dev \ + R-doc \ readline-dev \ - ruby ruby-dev ruby-bundler ruby-rdoc \ + ruby \ + ruby-bundler \ + ruby-dev \ + ruby-rdoc \ rustup \ - zlib zlib-dev + tar \ + zlib \ + zlib-dev \ + zstd -######################################## -# Copy dependencies files to container # -######################################## COPY dependencies/ / ################################################################### @@ -191,44 +205,70 @@ COPY --from=actionlint /usr/local/bin/actionlint /usr/bin/ ###################### COPY --from=kubeconfrm /kubeconform /usr/bin/ +# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub +# Store the key here because the above host is sometimes down, and breaks our builds +COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub + +################# +# Install glibc # +################# +ARG GLIBC_VERSION +COPY scripts/install-glibc.sh / +RUN --mount=type=secret,id=GITHUB_TOKEN /install-glibc.sh && rm -rf /install-glibc.sh + ################# # Install Lintr # ################# COPY scripts/install-lintr.sh / RUN /install-lintr.sh && rm -rf /install-lintr.sh +################################# +# Install luacheck and luarocks # +################################# +COPY scripts/install-lua.sh / +RUN --mount=type=secret,id=GITHUB_TOKEN /install-lua.sh && rm -rf /install-lua.sh + +##################################### +# Build python virtual environments # +##################################### +COPY dependencies/python/ /stage +WORKDIR /stage +RUN ./build-venvs.sh +# Set work directory back to root because some scripts depend on it +WORKDIR / + +############################## +# Install Phive dependencies # +############################## +COPY scripts/install-phive.sh / +RUN /install-phive.sh && rm -rf /install-phive.sh + ##################### # Install clj-kondo # ##################### +ARG CLJ_KONDO_VERSION='2023.05.18' COPY scripts/install-clj-kondo.sh / RUN /install-clj-kondo.sh && rm -rf /install-clj-kondo.sh -# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub -# Store the key here because the above host is sometimes down, and breaks our builds -COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub - ################## # Install ktlint # ################## +ARG KTLINT_VERSION='0.47.1' COPY scripts/install-ktlint.sh / RUN --mount=type=secret,id=GITHUB_TOKEN /install-ktlint.sh && rm -rf /install-ktlint.sh -#################### -# Install dart-sdk # -#################### -COPY scripts/install-dart-sdk.sh / -RUN --mount=type=secret,id=GITHUB_TOKEN /install-dart-sdk.sh && rm -rf /install-dart-sdk.sh - -################################ -# Install Bash-Exec # -################################ -COPY --chmod=555 scripts/bash-exec.sh /usr/bin/bash-exec - ################################################# # Install Raku and additional Edge dependencies # ################################################# RUN apk add --no-cache rakudo zef +#################### +# Install dart-sdk # +#################### +ARG DART_VERSION='2.8.4' +COPY scripts/install-dart-sdk.sh / +RUN /install-dart-sdk.sh && rm -rf /install-dart-sdk.sh + ###################### # Install CheckStyle # ###################### @@ -241,118 +281,32 @@ RUN --mount=type=secret,id=GITHUB_TOKEN /install-checkstyle.sh && rm -rf /instal COPY scripts/install-google-java-format.sh / RUN --mount=type=secret,id=GITHUB_TOKEN /install-google-java-format.sh && rm -rf /install-google-java-format.sh -################################# -# Install luacheck and luarocks # -################################# -COPY scripts/install-lua.sh / -RUN --mount=type=secret,id=GITHUB_TOKEN /install-lua.sh && rm -rf /install-lua.sh - ######################### # Clean to shrink image # ######################### RUN find /usr/ -type f -name '*.md' -exec rm {} + -################################################################################ -# Grab small clean image to build python packages ############################## -################################################################################ -FROM python:3.11.5-alpine3.17 as python_builder -RUN apk add --no-cache bash g++ git libffi-dev -COPY dependencies/python/ /stage -WORKDIR /stage -RUN ./build-venvs.sh +##################### +# Install Bash-Exec # +##################### +COPY --chmod=555 scripts/bash-exec.sh /usr/bin/bash-exec -################################################################################ -# Grab small clean image to build slim ################################### -################################################################################ -FROM alpine:3.19.0 as slim +################################# +# Copy super-linter executables # +################################# +COPY lib /action/lib -############################ -# Get the build arguements # -############################ -ARG BUILD_DATE -ARG BUILD_REVISION -ARG BUILD_VERSION -## install alpine-pkg-glibc (glibc compatibility layer package for Alpine Linux) -ARG GLIBC_VERSION='2.34-r0' -# https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope -ARG TARGETARCH +################################### +# Copy linter configuration files # +################################### +COPY TEMPLATES /action/lib/.automation -######################################### -# Label the instance and set maintainer # -######################################### -LABEL com.github.actions.name="GitHub Super-Linter" \ - com.github.actions.description="Lint your code base with GitHub Actions" \ - com.github.actions.icon="code" \ - com.github.actions.color="red" \ - maintainer="@Hanse00, @ferrarimarco, @zkoppert" \ - org.opencontainers.image.created=$BUILD_DATE \ - org.opencontainers.image.revision=$BUILD_REVISION \ - org.opencontainers.image.version=$BUILD_VERSION \ - org.opencontainers.image.authors="Super Linter Contributors: https://github.com/super-linter/super-linter/graphs/contributors" \ - org.opencontainers.image.url="https://github.com/super-linter/super-linter" \ - org.opencontainers.image.source="https://github.com/super-linter/super-linter" \ - org.opencontainers.image.documentation="https://github.com/super-linter/super-linter" \ - org.opencontainers.image.vendor="GitHub" \ - org.opencontainers.image.description="Lint your code base with GitHub Actions" - -################################################# -# Set ENV values used for debugging the version # -################################################# -ENV BUILD_DATE=$BUILD_DATE -ENV BUILD_REVISION=$BUILD_REVISION -ENV BUILD_VERSION=$BUILD_VERSION +######################### +# Configure Environment # +######################### +# Set image variant ENV IMAGE="slim" -# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub -# Store the key here because the above host is sometimes down, and breaks our builds -COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub - -############### -# Install Git # -############### -RUN apk add --no-cache bash git git-lfs - -############################## -# Install Phive dependencies # -############################## -COPY scripts/install-phive.sh / -RUN --mount=type=secret,id=GITHUB_TOKEN /install-phive.sh && rm -rf /install-phive.sh - -#################################################### -# Install Composer after all Libs have been copied # -#################################################### -RUN sh -c 'curl --retry 5 --retry-delay 5 --show-error -sS https://getcomposer.org/installer | php -- --install-dir=/usr/bin --filename=composer' - -################################# -# Copy the libraries into image # -################################# -COPY --from=base_image /usr/bin/ /usr/bin/ -COPY --from=base_image /usr/local/bin/ /usr/local/bin/ -COPY --from=base_image /usr/local/lib/ /usr/local/lib/ -COPY --from=base_image /usr/local/share/ /usr/local/share/ -COPY --from=base_image /usr/local/include/ /usr/local/include/ -COPY --from=base_image /usr/lib/ /usr/lib/ -COPY --from=base_image /usr/share/ /usr/share/ -COPY --from=base_image /usr/include/ /usr/include/ -COPY --from=base_image /lib/ /lib/ -COPY --from=base_image /bin/ /bin/ -COPY --from=base_image /node_modules/ /node_modules/ -COPY --from=base_image /home/r-library /home/r-library -COPY --from=python_builder /venvs/ /venvs/ - -################################## -# Configure TFLint plugin folder # -################################## -ENV TFLINT_PLUGIN_DIR="/root/.tflint.d/plugins" - -######################################## -# Add node packages to path and dotnet # -######################################## -ENV PATH="${PATH}:/node_modules/.bin" - -############################### -# Add python packages to path # -############################### ENV PATH="${PATH}:/venvs/ansible-lint/bin" ENV PATH="${PATH}:/venvs/black/bin" ENV PATH="${PATH}:/venvs/cfn-lint/bin" @@ -366,55 +320,49 @@ ENV PATH="${PATH}:/venvs/snakemake/bin" ENV PATH="${PATH}:/venvs/sqlfluff/bin" ENV PATH="${PATH}:/venvs/yamllint/bin" ENV PATH="${PATH}:/venvs/yq/bin" - -################## -# Add go to path # -################## +ENV PATH="${PATH}:/node_modules/.bin" ENV PATH="${PATH}:/usr/lib/go/bin" -############################# -# Copy scripts to container # -############################# -COPY lib /action/lib +# Configure TFLint plugin folder +ENV TFLINT_PLUGIN_DIR="/root/.tflint.d/plugins" -################################## -# Copy linter rules to container # -################################## -COPY TEMPLATES /action/lib/.automation +# Initialize TFLint plugins so we get plugin versions listed when we ask for TFLint version +# Run to build version file and validate image +RUN tflint --init -c /action/lib/.automation/.tflint.hcl \ + && ACTIONS_RUNNER_DEBUG=true WRITE_LINTER_VERSIONS_FILE=true IMAGE="${IMAGE}" /action/lib/linter.sh -################ -# Pull in libs # -################ -COPY --from=base_image /usr/libexec/ /usr/libexec/ - -################################################ -# Run to build version file and validate image # -################################################ -RUN ACTIONS_RUNNER_DEBUG=true WRITE_LINTER_VERSIONS_FILE=true IMAGE="${IMAGE}" /action/lib/linter.sh - -###################### -# Set the entrypoint # -###################### ENTRYPOINT ["/action/lib/linter.sh"] -################################################################################ -# Grab small clean image to build standard ############################### -################################################################################ +# Initialize Terrascan +# Initialize ChkTeX config file +RUN terrascan init \ + && touch ~/.chktexrc + +# Set build metadata here so we don't invalidate the container image cache if we +# change the values of these arguments +ARG BUILD_DATE +ARG BUILD_REVISION +ARG BUILD_VERSION + +LABEL org.opencontainers.image.created=$BUILD_DATE \ + org.opencontainers.image.revision=$BUILD_REVISION \ + org.opencontainers.image.version=$BUILD_VERSION + +ENV BUILD_DATE=$BUILD_DATE +ENV BUILD_REVISION=$BUILD_REVISION +ENV BUILD_VERSION=$BUILD_VERSION + +############################## +# Build the standard variant # +############################## FROM slim as standard -############### -# Set up args # -############### -ARG GITHUB_TOKEN +# https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope +ARG TARGETARCH ARG PWSH_VERSION='latest' ARG PWSH_DIRECTORY='/usr/lib/microsoft/powershell' ARG PSSA_VERSION='1.21.0' -# https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope -ARG TARGETARCH -################ -# Set ENV vars # -################ ENV ARM_TTK_PSD1="/usr/lib/microsoft/arm-ttk/arm-ttk.psd1" ENV IMAGE="standard" ENV PATH="${PATH}:/var/cache/dotnet/tools:/usr/share/dotnet" @@ -449,7 +397,5 @@ RUN --mount=type=secret,id=GITHUB_TOKEN /install-pwsh.sh && rm -rf /install-pwsh COPY scripts/install-arm-ttk.sh / RUN --mount=type=secret,id=GITHUB_TOKEN /install-arm-ttk.sh && rm -rf /install-arm-ttk.sh -######################################################################################## -# Run to build version file and validate image again because we installed more linters # -######################################################################################## +# Run to build version file and validate image again because we installed more linters RUN ACTIONS_RUNNER_DEBUG=true WRITE_LINTER_VERSIONS_FILE=true IMAGE="${IMAGE}" /action/lib/linter.sh diff --git a/Makefile b/Makefile index 6ce9ab5c..7558505f 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,7 @@ inspec: inspec-check ## Run InSpec tests -v /var/run/docker.sock:/var/run/docker.sock \ -e IMAGE=$(IMAGE) \ -w="/workspace" \ - chef/inspec exec test/inspec/super-linter\ + chef/inspec exec test/inspec/super-linter \ --chef-license=accept \ --diagnose \ --log-level=debug \ diff --git a/dependencies/phive.xml b/dependencies/phive.xml index c9998882..e14a013d 100644 --- a/dependencies/phive.xml +++ b/dependencies/phive.xml @@ -1,7 +1,8 @@ - - - + + + + diff --git a/scripts/install-arm-ttk.sh b/scripts/install-arm-ttk.sh index 09d104fa..338bc998 100755 --- a/scripts/install-arm-ttk.sh +++ b/scripts/install-arm-ttk.sh @@ -6,10 +6,13 @@ set -euo pipefail # Reference https://github.com/Azure/arm-ttk # Reference https://docs.microsoft.com/en-us/azure/azure-resource-manager/templates/test-toolkit -url=$(curl -s \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ - https://api.github.com/repos/Azure/arm-ttk/releases/latest | jq -r '.tarball_url') +url=$( + set -euo pipefail + curl -s \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ + https://api.github.com/repos/Azure/arm-ttk/releases/latest | jq -r '.tarball_url' +) mkdir -p /usr/lib/microsoft curl --retry 5 --retry-delay 5 -sL \ -H "Accept: application/vnd.github+json" \ diff --git a/scripts/install-checkstyle.sh b/scripts/install-checkstyle.sh index ebeb272a..66a38bae 100755 --- a/scripts/install-checkstyle.sh +++ b/scripts/install-checkstyle.sh @@ -2,14 +2,20 @@ set -euo pipefail -CHECKSTYLE_VERSION="$(grep <"checkstyle/build.gradle" "checkstyle" | awk -F ':' '{print $3}' | tr -d "'")" +CHECKSTYLE_VERSION="$( + set -euo pipefail + grep <"checkstyle/build.gradle" "checkstyle" | awk -F ':' '{print $3}' | tr -d "'" +)" echo "Installing Checkstyle: ${CHECKSTYLE_VERSION}" -url=$(curl -s \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ - "https://api.github.com/repos/checkstyle/checkstyle/releases/tags/checkstyle-${CHECKSTYLE_VERSION}" | - jq --arg name "checkstyle-${CHECKSTYLE_VERSION}-all.jar" -r '.assets | .[] | select(.name==$name) | .url') +url=$( + set -euo pipefail + curl -s \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ + "https://api.github.com/repos/checkstyle/checkstyle/releases/tags/checkstyle-${CHECKSTYLE_VERSION}" | + jq --arg name "checkstyle-${CHECKSTYLE_VERSION}-all.jar" -r '.assets | .[] | select(.name==$name) | .url' +) curl --retry 5 --retry-delay 5 -sL -o /usr/bin/checkstyle \ -H "Accept: application/octet-stream" \ -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ diff --git a/scripts/install-dart-sdk.sh b/scripts/install-dart-sdk.sh index 837d2310..9fe8fe98 100755 --- a/scripts/install-dart-sdk.sh +++ b/scripts/install-dart-sdk.sh @@ -15,19 +15,10 @@ arm64) ;; esac -url=$(curl -s \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ - "https://api.github.com/repos/sgerrand/alpine-pkg-glibc/releases/tags/${GLIBC_VERSION}" | - jq --arg name "glibc-${GLIBC_VERSION}.apk" -r '.assets | .[] | select(.name | contains($name)) | .url') -curl --retry 5 --retry-delay 5 -sL -o "glibc-${GLIBC_VERSION}.apk" \ - -H "Accept: application/octet-stream" \ - -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ - "${url}" -apk add --no-cache --force-overwrite "glibc-${GLIBC_VERSION}.apk" -rm "glibc-${GLIBC_VERSION}.apk" - curl --retry 5 --retry-delay 5 -sO "https://storage.googleapis.com/dart-archive/channels/stable/release/${DART_VERSION}/sdk/dartsdk-linux-${target}-release.zip" unzip -q dartsdk-linux-${target}-release.zip -chmod +x dart-sdk/bin/dart* && mv dart-sdk/bin/* /usr/bin/ && mv dart-sdk/lib/* /usr/lib/ && mv dart-sdk/include/* /usr/include/ -rm -r dart-sdk/ dartsdk-linux-${target}-release.zip +chmod +x dart-sdk/bin/dart* +mv -v dart-sdk/bin/* /usr/bin/ +mv -v dart-sdk/lib/* /usr/lib/ +mv -v dart-sdk/include/* /usr/include/ +rm -rv dart-sdk/ dartsdk-linux-${target}-release.zip diff --git a/scripts/install-glibc.sh b/scripts/install-glibc.sh new file mode 100755 index 00000000..c061b6a8 --- /dev/null +++ b/scripts/install-glibc.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -euo pipefail + +case $TARGETARCH in +amd64) + target=x86_64 + ;; +arm64) + target=arm64 + ;; +*) + echo "$TARGETARCH is not supported" + exit 1 + ;; +esac + +url=$( + set -euo pipefail + curl -s \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ + "https://api.github.com/repos/sgerrand/alpine-pkg-glibc/releases/tags/${GLIBC_VERSION}" | + jq --arg name "glibc-${GLIBC_VERSION}.apk" -r '.assets | .[] | select(.name | contains($name)) | .url' +) +curl --retry 5 --retry-delay 5 -sL -o "glibc-${GLIBC_VERSION}.apk" \ + -H "Accept: application/octet-stream" \ + -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ + "${url}" +apk add --no-cache --force-overwrite "glibc-${GLIBC_VERSION}.apk" +rm "glibc-${GLIBC_VERSION}.apk" + +# Install zlib +mkdir /tmp/libz +curl --retry 5 --retry-delay 5 -sL https://www.archlinux.org/packages/core/${target}/zlib/download | tar -x --zstd -C /tmp/libz +mv /tmp/libz/usr/lib/libz.so* /usr/glibc-compat/lib +rm -rf /tmp/libz diff --git a/scripts/install-google-java-format.sh b/scripts/install-google-java-format.sh index 58413698..32532dbc 100755 --- a/scripts/install-google-java-format.sh +++ b/scripts/install-google-java-format.sh @@ -2,14 +2,20 @@ set -euo pipefail -GOOGLE_JAVA_FORMAT_VERSION="$(grep <"google-java-format/build.gradle" "google-java-format" | awk -F ':' '{print $3}' | tr -d "'")" +GOOGLE_JAVA_FORMAT_VERSION="$( + set -euo pipefail + grep <"google-java-format/build.gradle" "google-java-format" | awk -F ':' '{print $3}' | tr -d "'" +)" echo "Installing Google Java Format: ${GOOGLE_JAVA_FORMAT_VERSION}" -url=$(curl -s \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ - "https://api.github.com/repos/google/google-java-format/releases/tags/v${GOOGLE_JAVA_FORMAT_VERSION}" | - jq --arg name "google-java-format-${GOOGLE_JAVA_FORMAT_VERSION}-all-deps.jar" -r '.assets | .[] | select(.name==$name) | .url') +url=$( + set -euo pipefail + curl -s \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ + "https://api.github.com/repos/google/google-java-format/releases/tags/v${GOOGLE_JAVA_FORMAT_VERSION}" | + jq --arg name "google-java-format-${GOOGLE_JAVA_FORMAT_VERSION}-all-deps.jar" -r '.assets | .[] | select(.name==$name) | .url' +) curl --retry 5 --retry-delay 5 -sL -o /usr/bin/google-java-format \ -H "Accept: application/octet-stream" \ -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ diff --git a/scripts/install-ktlint.sh b/scripts/install-ktlint.sh index a2a93109..a69d573b 100755 --- a/scripts/install-ktlint.sh +++ b/scripts/install-ktlint.sh @@ -2,15 +2,16 @@ set -euo pipefail -url=$(curl -s \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ - "https://api.github.com/repos/pinterest/ktlint/releases/tags/${KTLINT_VERSION}" | - jq -r '.assets | .[] | select(.name=="ktlint") | .url') +url=$( + set -euo pipefail + curl -s \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ + "https://api.github.com/repos/pinterest/ktlint/releases/tags/${KTLINT_VERSION}" | + jq -r '.assets | .[] | select(.name=="ktlint") | .url' +) curl --retry 5 --retry-delay 5 -sL -o "/usr/bin/ktlint" \ -H "Accept: application/octet-stream" \ -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ "${url}" chmod a+x /usr/bin/ktlint -terrascan init -cd ~ && touch .chktexrc diff --git a/scripts/install-lua.sh b/scripts/install-lua.sh index e0457b18..cbed2df4 100755 --- a/scripts/install-lua.sh +++ b/scripts/install-lua.sh @@ -8,10 +8,14 @@ make linux make install cd .. && rm -r lua-5.3.5/ -url=$(curl -s \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ - https://api.github.com/repos/cvega/luarocks/releases/latest | jq -r '.tarball_url') +url=$( + set -euo pipefail + curl -s \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ + https://api.github.com/repos/cvega/luarocks/releases/latest | + jq -r '.tarball_url' +) curl --retry 5 --retry-delay 5 -sL \ -H "Accept: application/vnd.github+json" \ -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ diff --git a/scripts/install-phive.sh b/scripts/install-phive.sh index a55cd947..7d8e81c7 100755 --- a/scripts/install-phive.sh +++ b/scripts/install-phive.sh @@ -1,44 +1,13 @@ #!/usr/bin/env bash set -euo pipefail -set -x -case $TARGETARCH in -amd64) - target=x86_64 - ;; -arm64) - target=aarch64 - ;; -*) - echo "$TARGETARCH is not supported" - exit 1 - ;; -esac - -apk add curl jq -url=$(curl -s \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ - "https://api.github.com/repos/sgerrand/alpine-pkg-glibc/releases/tags/${GLIBC_VERSION}" | - jq --arg name "glibc-${GLIBC_VERSION}.apk" -r '.assets | .[] | select(.name | contains($name)) | .url') -curl --retry 5 --retry-delay 5 -sL -o "glibc-${GLIBC_VERSION}.apk" \ - -H "Accept: application/octet-stream" \ - -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ - "${url}" -apk add --no-cache --force-overwrite \ - bash \ - ca-certificates \ - "glibc-${GLIBC_VERSION}.apk" \ - gnupg \ +# Install PHP +apk add --no-cache \ php82 php82-curl php82-ctype php82-dom php82-iconv php82-mbstring \ - php82-openssl php82-phar php82-simplexml php82-tokenizer php82-xmlwriter \ - tar zstd -rm "glibc-${GLIBC_VERSION}.apk" -mkdir /tmp/libz -curl --retry 5 --retry-delay 5 -sL https://www.archlinux.org/packages/core/${target}/zlib/download | tar -x --zstd -C /tmp/libz -mv /tmp/libz/usr/lib/libz.so* /usr/glibc-compat/lib -rm -rf /tmp/libz + php82-openssl php82-phar php82-simplexml php82-tokenizer php82-xmlwriter + +# Install phive curl --retry 5 --retry-delay 5 -sLO https://phar.io/releases/phive.phar curl --retry 5 --retry-delay 5 -sLO https://phar.io/releases/phive.phar.asc gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "0x9D8A98B29B2D5D79" @@ -47,10 +16,7 @@ chmod +x phive.phar mv phive.phar /usr/local/bin/phive rm phive.phar.asc -# Install PHARs +# Install the PHARs listed in phive.xml phive --no-progress install \ - --trust-gpg-keys 31C7E470E2138192,CF1A108D0E7AE720,8A03EA3B385DBAA1,12CE0F1D262429A5,5E6DDE998AB73B8E,51C67305FFC2E5C0 \ - --target /usr/bin \ - phpstan@^1.10.48 \ - psalm@^5.17.0 \ - phpcs@^3.8.0 + --trust-gpg-keys 31C7E470E2138192,CF1A108D0E7AE720,8A03EA3B385DBAA1,12CE0F1D262429A5,5E6DDE998AB73B8E,51C67305FFC2E5C0,CBB3D576F2A0946F \ + --target /usr/bin diff --git a/scripts/install-pwsh.sh b/scripts/install-pwsh.sh index a1b3a7e2..29627b6f 100755 --- a/scripts/install-pwsh.sh +++ b/scripts/install-pwsh.sh @@ -10,8 +10,6 @@ case $TARGETARCH in amd64) target=x64 ;; -# arm64) -# only on ubuntu *) echo "$TARGETARCH is not supported" exit 1 @@ -19,11 +17,14 @@ amd64) esac mkdir -p "${PWSH_DIRECTORY}" -url=$(curl -s \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ - "https://api.github.com/repos/powershell/powershell/releases/${PWSH_VERSION}" | - jq --arg target "${target}" -r '.assets | .[] | select(.name | contains("linux-musl-" + $target)) | .url') +url=$( + set -euo pipefail + curl -s \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ + "https://api.github.com/repos/powershell/powershell/releases/${PWSH_VERSION}" | + jq --arg target "${target}" -r '.assets | .[] | select(.name | contains("linux-musl-" + $target)) | .url' +) curl --retry 5 --retry-delay 5 -sL \ -H "Accept: application/octet-stream" \ -H "Authorization: Bearer $(cat /run/secrets/GITHUB_TOKEN)" \ diff --git a/test/inspec/super-linter/controls/super_linter.rb b/test/inspec/super-linter/controls/super_linter.rb index b592f4cf..eda1cf94 100644 --- a/test/inspec/super-linter/controls/super_linter.rb +++ b/test/inspec/super-linter/controls/super_linter.rb @@ -14,8 +14,12 @@ control "super-linter-installed-packages" do packages = [ "bash", "ca-certificates", + "cargo", + "cmake", "coreutils", "curl", + "file", + "g++", "gcc", "git-lfs", "git", @@ -23,6 +27,7 @@ control "super-linter-installed-packages" do "gnupg", "go", "icu-libs", + "jpeg-dev", "jq", "krb5-libs", "libc-dev", @@ -30,7 +35,7 @@ control "super-linter-installed-packages" do "libffi-dev", "libgcc", "libintl", - "libssl1.1", + "libssl3", "libstdc++", "libxml2-dev", "libxml2-utils", @@ -38,28 +43,44 @@ control "super-linter-installed-packages" do "lttng-ust-dev", "make", "musl-dev", - "npm", "nodejs-current", + "npm", + "net-snmp-dev", "openjdk17-jre", + "openssh-client", "openssl-dev", + "parallel", "perl-dev", "perl", + "php82", + "php82-ctype", + "php82-curl", + "php82-dom", + "php82-iconv", + "php82-mbstring", + "php82-openssl", + "php82-phar", + "php82-simplexml", + "php82-tokenizer", + "php82-xmlwriter", "py3-pyflakes", "py3-setuptools", "python3-dev", - "rakudo", "R-dev", "R-doc", "R", + "rakudo", "readline-dev", "ruby-bundler", "ruby-dev", "ruby-rdoc", "ruby", "rustup", + "tar", "zef", "zlib-dev", - "zlib" + "zlib", + "zstd" ] # Removed linters from slim image @@ -94,26 +115,28 @@ control "super-linter-installed-commands" do { linter_name: "actionlint"}, { linter_name: "ansible-lint", expected_stdout_regex: /(.*)/}, { linter_name: "arm-ttk", version_command: "grep -iE 'version' '/usr/bin/arm-ttk' | xargs"}, - { linter_name: "asl-validator", expected_exit_status: 0}, + { linter_name: "asl-validator"}, { linter_name: "bash-exec", expected_exit_status: 1}, # expect a return code = 1 because this linter doesn't support a "get linter version" command { linter_name: "black"}, - { linter_name: "clang-format"}, { linter_name: "cfn-lint"}, { linter_name: "checkstyle", version_command: "java -jar /usr/bin/checkstyle --version"}, - { linter_name: "google-java-format", version_command: "java -jar /usr/bin/google-java-format --version"}, { linter_name: "chktex"}, + { linter_name: "clang-format"}, { linter_name: "clippy", linter_command: "clippy", version_command: "cargo-clippy --version"}, { linter_name: "clj-kondo"}, { linter_name: "coffeelint"}, + { linter_name: "composer"}, { linter_name: "cpplint"}, { linter_name: "dart"}, - { linter_name: "dotnet"}, { linter_name: "dotenv-linter"}, + { linter_name: "dotnet"}, { linter_name: "editorconfig-checker", version_option: "-version"}, { linter_name: "eslint"}, { linter_name: "flake8"}, { linter_name: "gherkin-lint", expected_exit_status: 1}, # expect a return code = 1 because this linter doesn't support a "get linter version" command + { linter_name: "gitleaks", version_option: "version"}, { linter_name: "golangci-lint"}, + { linter_name: "google-java-format", version_command: "java -jar /usr/bin/google-java-format --version"}, { linter_name: "hadolint"}, { linter_name: "htmlhint"}, { linter_name: "isort"}, @@ -128,6 +151,7 @@ control "super-linter-installed-commands" do { linter_name: "php"}, { linter_name: "phpcs"}, { linter_name: "phpstan"}, + { linter_name: "prettier"}, { linter_name: "protolint", version_option: "version"}, { linter_name: "psalm"}, { linter_name: "pwsh"}, @@ -136,6 +160,7 @@ control "super-linter-installed-commands" do lintr_ver <- packageVersion('lintr'); \ glue::glue('lintr { lintr_ver } on { r_ver }')\""}, { linter_name: "raku", version_command: "raku --version | strings -n 8"}, + { linter_name: "renovate-config-validator", version_command: "renovate --version"}, { linter_name: "rubocop"}, { linter_name: "rustfmt"}, { linter_name: "scalafmt"}, @@ -152,7 +177,9 @@ control "super-linter-installed-commands" do { linter_name: "terraform"}, { linter_name: "terragrunt"}, { linter_name: "terrascan", version_option: "version"}, + { linter_name: "textlint"}, { linter_name: "tflint"}, + { linter_name: "ts-standard"}, { linter_name: "xmllint"}, { linter_name: "yamllint"}, ] @@ -242,9 +269,12 @@ control "super-linter-installed-ruby-gems" do gems = [ "rubocop", "rubocop-github", + "rubocop-minitest", "rubocop-performance", "rubocop-rails", - "rubocop-rspec" + "rubocop-rake", + "rubocop-rspec", + "standard" ] gems.each do |item| @@ -252,7 +282,6 @@ control "super-linter-installed-ruby-gems" do it { should be_installed } end end - end ############################################### @@ -268,38 +297,61 @@ control "super-linter-installed-npm-packages" do "@babel/preset-react", "@babel/preset-typescript", "@coffeelint/cli", + "@react-native-community/eslint-config", + "@react-native-community/eslint-plugin", "@stoplight/spectral-cli", "@typescript-eslint/eslint-plugin", "@typescript-eslint/parser", "asl-validator", - #"axios", - #"eslint", + "axios", + "eslint", "eslint-config-airbnb", + "eslint-config-airbnb-typescript", "eslint-config-prettier", "eslint-plugin-jest", "eslint-plugin-json", "eslint-plugin-jsonc", "eslint-plugin-jsx-a11y", "eslint-plugin-prettier", + "eslint-plugin-react", + "eslint-plugin-react-hooks", + "eslint-plugin-vue", "gherkin-lint", "htmlhint", - #"immer", - #"ini", + "immer", + "ini", "jscpd", - #"lodash", + "lodash", "markdownlint-cli", - #"node-fetch", + "next", + "next-pwa", + "node-fetch", "npm-groovy-lint", + "postcss-less", "prettier", "prettyjson", - #"pug", + "pug", + "react", + "react-dom", + "react-intl", + "react-redux", + "react-router-dom", + "renovate", "sql-lint", "standard", "stylelint", + "stylelint-config-recommended-scss", "stylelint-config-sass-guidelines", "stylelint-config-standard", - #"stylelint-scss", + "stylelint-config-standard-scss", + "stylelint-prettier", + "stylelint-scss", "tekton-lint", + "textlint", + "textlint-filter-rule-allowlist", + "textlint-filter-rule-comments", + "textlint-rule-terminology", + "ts-standard", "typescript" ] @@ -308,7 +360,37 @@ control "super-linter-installed-npm-packages" do it { should be_installed } end end +end +############################################### +# Check to see if PyPi packages are installed # +############################################### +control "super-linter-installed-pypi-packages" do + impact 1 + title "Super-Linter installed PyPi packages check" + desc "Check that PyPi packages that Super-Linter needs are installed." + + pypi_packages = [ + "ansible-lint", + "black", + "cfn-lint", + "cpplint", + "flake8", + "isort", + "mypy", + "pylint", + "snakefmt", + "snakemake", + "sqlfluff", + "yamllint", + "yq" + ] + + pypi_packages.each do |item| + describe pip(item, "/venvs/#{item}/bin/pip") do + it { should be_installed } + end + end end #####################################