From 1ca3ebccd6b63a0d880c6b7603ff2dddb6f37ac4 Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Fri, 29 Dec 2023 19:30:58 +0100 Subject: [PATCH] build: reduce container image size (#5072) - Remove build-time dependencies - Remove cached NPM packages - Remove cached PyPi packages - Remove dependency descriptors. These still count against the total space, although it's a few KBs - Install rust-clippy and rust-fmt using the OS package manager instead of maintaining our own installation script - Add tests for build time dependencies that are not supposed to be installed --- Dockerfile | 183 +++++++++--------- dependencies/python/build-venvs.sh | 12 +- scripts/clippy.sh | 11 ++ scripts/install-dotnet.sh | 1 + scripts/install-glibc.sh | 7 + scripts/install-lintr.sh | 11 ++ scripts/install-lua.sh | 8 + scripts/install-phive.sh | 8 +- scripts/install-rustfmt.sh | 38 ---- .../super-linter/controls/super_linter.rb | 73 ++++--- 10 files changed, 183 insertions(+), 169 deletions(-) create mode 100755 scripts/clippy.sh delete mode 100755 scripts/install-rustfmt.sh diff --git a/Dockerfile b/Dockerfile index ebd219b0..f363904e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,8 @@ -########################################### -########################################### -## Dockerfile to run GitHub Super-Linter ## -########################################### -########################################### - -ARG GLIBC_VERSION='2.34-r0' +#################################### +#################################### +## Dockerfile to run Super-Linter ## +#################################### +#################################### ######################################### # Get dependency images as build stages # @@ -47,79 +45,89 @@ LABEL com.github.actions.name="Super-Linter" \ ARG TARGETARCH # Install bash first so we can use it +# This is also a super-linter runtime dependency RUN apk add --no-cache \ bash SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-c"] +# Install super-linter runtime dependencies RUN apk add --no-cache \ ca-certificates \ - cargo \ - cmake \ coreutils \ curl \ file \ - g++ \ - gcc \ git \ git-lfs \ - gnupg \ - icu-libs \ - jpeg-dev \ jq \ - krb5-libs \ - libc-dev \ - libcurl \ - libffi-dev \ - libgcc \ - libintl \ - libssl3 \ - libstdc++ \ - libxml2-dev \ libxml2-utils \ - linux-headers \ - lttng-ust-dev \ - make \ - musl-dev \ - net-snmp-dev \ nodejs-current \ - npm \ openjdk17-jre \ openssh-client \ - openssl-dev \ - parallel \ perl \ - perl-dev \ - py3-pyflakes \ - py3-setuptools \ - python3-dev \ + php82 \ + php82-ctype \ + php82-curl \ + php82-dom \ + php82-iconv \ + php82-mbstring \ + php82-openssl \ + php82-phar \ + php82-simplexml \ + php82-tokenizer \ + php82-xmlwriter \ R \ - R-dev \ - R-doc \ - readline-dev \ + rakudo \ ruby \ + zef + +# Install Node tools +# The chown fixes broken uid/gid in ast-types-flow dependency +# (see https://github.com/super-linter/super-linter/issues/3901) +# Npm is not a runtime dependency but we need it to ensure that npm packages +# are installed when we run the test suite. If we decide to remove it, add +# the following command to the RUN instruction below: +# apk del --no-network --purge .node-build-deps +COPY dependencies/package.json dependencies/package-lock.json / +RUN apk add --no-cache --virtual .node-build-deps \ + npm \ + && npm install \ + && npm cache clean --force \ + && chown -R "$(id -u)":"$(id -g)" node_modules \ + && rm -rfv package.json package-lock.json + +# Install Ruby tools +COPY dependencies/Gemfile dependencies/Gemfile.lock / +RUN apk add --no-cache --virtual .ruby-build-deps \ + gcc \ + make \ + musl-dev \ ruby-bundler \ ruby-dev \ ruby-rdoc \ - rustup \ - tar \ - zlib \ - zlib-dev \ - zstd - -COPY dependencies/ / - -################################################################### -# Install Dependencies # -# The chown fixes broken uid/gid in ast-types-flow dependency # -# (see https://github.com/super-linter/super-linter/issues/3901) # -################################################################### -RUN npm install && chown -R "$(id -u)":"$(id -g)" node_modules && bundle install + && bundle install \ + && apk del --no-network --purge .ruby-build-deps \ + && rm -rf Gemfile Gemfile.lock ############################## # Installs Perl dependencies # ############################## -RUN curl --retry 5 --retry-delay 5 -sL https://cpanmin.us/ | perl - -nq --no-wget Perl::Critic Perl::Critic::Community Perl::Critic::More Perl::Critic::Bangs Perl::Critic::Lax Perl::Critic::StricterSubs Perl::Critic::Swift Perl::Critic::Tics +RUN apk add --no-cache --virtual .perl-build-deps \ + gcc \ + make \ + musl-dev \ + perl-dev \ + && curl --retry 5 --retry-delay 5 -sL https://cpanmin.us/ \ + | perl - -nq --no-wget \ + Perl::Critic \ + Perl::Critic::Bangs \ + Perl::Critic::Community \ + Perl::Critic::Lax \ + Perl::Critic::More \ + Perl::Critic::StricterSubs \ + Perl::Critic::Swift \ + Perl::Critic::Tics \ + && apk del --no-network --purge .perl-build-deps ###################### # Install shellcheck # @@ -206,16 +214,16 @@ COPY --from=actionlint /usr/local/bin/actionlint /usr/bin/ ###################### COPY --from=kubeconfrm /kubeconform /usr/bin/ -# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub -# Store the key here because the above host is sometimes down, and breaks our builds -COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub - ################# # Install glibc # ################# -ARG GLIBC_VERSION +# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub +# Store the key here because the above host is sometimes down, and breaks our builds +COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub +ARG GLIBC_VERSION='2.34-r0' COPY scripts/install-glibc.sh / -RUN --mount=type=secret,id=GITHUB_TOKEN /install-glibc.sh && rm -rf /install-glibc.sh +RUN --mount=type=secret,id=GITHUB_TOKEN /install-glibc.sh \ + && rm -rf /install-glibc.sh /sgerrand.rsa.pub ##################### # Install clj-kondo # @@ -246,43 +254,41 @@ RUN --mount=type=secret,id=GITHUB_TOKEN /install-lua.sh && rm -rf /install-lua.s ##################################### COPY dependencies/python/ /stage WORKDIR /stage -RUN ./build-venvs.sh +RUN ./build-venvs.sh && rm -rfv /stage # Set work directory back to root because some scripts depend on it WORKDIR / ############################## # Install Phive dependencies # ############################## +COPY dependencies/phive.xml /phive.xml COPY scripts/install-phive.sh / -RUN /install-phive.sh && rm -rf /install-phive.sh +RUN /install-phive.sh \ + && rm -rfv /install-phive.sh /phive.xml ################## # Install ktlint # ################## COPY scripts/install-ktlint.sh / -RUN --mount=type=secret,id=GITHUB_TOKEN /install-ktlint.sh && rm -rf /install-ktlint.sh - -################################################# -# Install Raku and additional Edge dependencies # -################################################# -RUN apk add --no-cache rakudo zef +COPY dependencies/ktlint /ktlint +RUN --mount=type=secret,id=GITHUB_TOKEN /install-ktlint.sh \ + && rm -rfv /install-ktlint.sh /ktlint ###################### # Install CheckStyle # ###################### COPY scripts/install-checkstyle.sh / -RUN --mount=type=secret,id=GITHUB_TOKEN /install-checkstyle.sh && rm -rf /install-checkstyle.sh +COPY dependencies/checkstyle /checkstyle +RUN --mount=type=secret,id=GITHUB_TOKEN /install-checkstyle.sh \ + && rm -rfv /install-checkstyle.sh /checkstyle ############################## # Install google-java-format # ############################## COPY scripts/install-google-java-format.sh / -RUN --mount=type=secret,id=GITHUB_TOKEN /install-google-java-format.sh && rm -rf /install-google-java-format.sh - -######################### -# Clean to shrink image # -######################### -RUN find /usr/ -type f -name '*.md' -exec rm {} + +COPY dependencies/google-java-format /google-java-format +RUN --mount=type=secret,id=GITHUB_TOKEN /install-google-java-format.sh \ + && rm -rfv /install-google-java-format.sh /google-java-format ##################### # Install Bash-Exec # @@ -327,17 +333,17 @@ ENV PATH="${PATH}:${DART_SDK}/bin:/root/.pub-cache/bin" ENV TFLINT_PLUGIN_DIR="/root/.tflint.d/plugins" # Initialize TFLint plugins so we get plugin versions listed when we ask for TFLint version -# Run to build version file and validate image -RUN tflint --init -c /action/lib/.automation/.tflint.hcl \ - && ACTIONS_RUNNER_DEBUG=true WRITE_LINTER_VERSIONS_FILE=true IMAGE="${IMAGE}" /action/lib/linter.sh - -ENTRYPOINT ["/action/lib/linter.sh"] - # Initialize Terrascan # Initialize ChkTeX config file -RUN terrascan init \ +RUN tflint --init -c /action/lib/.automation/.tflint.hcl \ + && terrascan init \ && touch ~/.chktexrc +# Run to build version file and validate image +RUN ACTIONS_RUNNER_DEBUG=true WRITE_LINTER_VERSIONS_FILE=true IMAGE="${IMAGE}" /action/lib/linter.sh + +ENTRYPOINT ["/action/lib/linter.sh"] + FROM base_image as slim # Set build metadata here so we don't invalidate the container image cache if we @@ -369,6 +375,14 @@ ENV ARM_TTK_PSD1="/usr/lib/microsoft/arm-ttk/arm-ttk.psd1" ENV IMAGE="standard" ENV PATH="${PATH}:/var/cache/dotnet/tools:/usr/share/dotnet" +# Install super-linter runtime dependencies +RUN apk add --no-cache \ + rust-clippy \ + rustfmt + +COPY scripts/clippy.sh /usr/bin/clippy +RUN chmod +x /usr/bin/clippy + ######################### # Install dotenv-linter # ######################### @@ -380,13 +394,6 @@ COPY --from=dotenv-linter /dotenv-linter /usr/bin/ COPY scripts/install-dotnet.sh / RUN /install-dotnet.sh && rm -rf /install-dotnet.sh -############################## -# Install rustfmt & clippy # -############################## -ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 -COPY scripts/install-rustfmt.sh / -RUN /install-rustfmt.sh && rm -rf /install-rustfmt.sh - ######################################### # Install Powershell + PSScriptAnalyzer # ######################################### diff --git a/dependencies/python/build-venvs.sh b/dependencies/python/build-venvs.sh index b2f34409..4b1fcfa7 100755 --- a/dependencies/python/build-venvs.sh +++ b/dependencies/python/build-venvs.sh @@ -8,6 +8,12 @@ ##################### set -euo pipefail +apk add --no-cache --virtual .python-build-deps \ + gcc \ + linux-headers \ + musl-dev \ + python3-dev + ############################ # Create staging directory # ############################ @@ -32,9 +38,13 @@ for DEP_FILE in *.txt; do virtualenv . # shellcheck disable=SC1091 source bin/activate - pip install -r requirements.txt + pip install \ + --no-cache-dir \ + --requirement requirements.txt # deactivate the python virtualenv deactivate # pop the stack popd done + +apk del --no-network --purge .python-build-deps diff --git a/scripts/clippy.sh b/scripts/clippy.sh new file mode 100755 index 00000000..0665cc14 --- /dev/null +++ b/scripts/clippy.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +pushd "$(dirname "$1")" || exit 1 + +cargo-clippy + +rc=$? + +popd || exit 1 + +exit $rc diff --git a/scripts/install-dotnet.sh b/scripts/install-dotnet.sh index 414688f0..8fbe6a75 100755 --- a/scripts/install-dotnet.sh +++ b/scripts/install-dotnet.sh @@ -5,3 +5,4 @@ set -euo pipefail curl --retry 5 --retry-delay 5 -sLO https://dot.net/v1/dotnet-install.sh chmod +x dotnet-install.sh ./dotnet-install.sh --install-dir /usr/share/dotnet -channel LTS -version latest +rm -rfv ./dotnet-install.sh diff --git a/scripts/install-glibc.sh b/scripts/install-glibc.sh index c061b6a8..3abfd53c 100755 --- a/scripts/install-glibc.sh +++ b/scripts/install-glibc.sh @@ -32,6 +32,13 @@ rm "glibc-${GLIBC_VERSION}.apk" # Install zlib mkdir /tmp/libz + +apk add --no-cache --virtual .glibc-build-deps \ + tar \ + zstd + curl --retry 5 --retry-delay 5 -sL https://www.archlinux.org/packages/core/${target}/zlib/download | tar -x --zstd -C /tmp/libz mv /tmp/libz/usr/lib/libz.so* /usr/glibc-compat/lib rm -rf /tmp/libz + +apk del --no-network --purge .glibc-build-deps diff --git a/scripts/install-lintr.sh b/scripts/install-lintr.sh index 7a454ef5..9d189d8c 100755 --- a/scripts/install-lintr.sh +++ b/scripts/install-lintr.sh @@ -2,4 +2,15 @@ set -euo pipefail +apk add --no-cache --virtual .r-build-deps \ + g++ \ + gcc \ + libxml2-dev \ + linux-headers \ + make \ + R-dev \ + R-doc + Rscript --no-save /install-r-package-or-fail.R lintr purrr remotes + +apk del --no-network --purge .r-build-deps diff --git a/scripts/install-lua.sh b/scripts/install-lua.sh index cbed2df4..cdc62622 100755 --- a/scripts/install-lua.sh +++ b/scripts/install-lua.sh @@ -2,6 +2,12 @@ set -euo pipefail +apk add --no-cache --virtual .lua-build-deps \ + gcc \ + make \ + musl-dev \ + readline-dev + curl --retry 5 --retry-delay 5 -s https://www.lua.org/ftp/lua-5.3.5.tar.gz | tar -xz cd lua-5.3.5 make linux @@ -30,3 +36,5 @@ rm -r cvega-luarocks-6b1aee6 luarocks install luacheck luarocks install argparse luarocks install luafilesystem + +apk del --no-network --purge .lua-build-deps diff --git a/scripts/install-phive.sh b/scripts/install-phive.sh index 7d8e81c7..d7db77ed 100755 --- a/scripts/install-phive.sh +++ b/scripts/install-phive.sh @@ -2,10 +2,8 @@ set -euo pipefail -# Install PHP -apk add --no-cache \ - php82 php82-curl php82-ctype php82-dom php82-iconv php82-mbstring \ - php82-openssl php82-phar php82-simplexml php82-tokenizer php82-xmlwriter +apk add --no-cache --virtual .php-build-deps \ + gnupg # Install phive curl --retry 5 --retry-delay 5 -sLO https://phar.io/releases/phive.phar @@ -20,3 +18,5 @@ rm phive.phar.asc phive --no-progress install \ --trust-gpg-keys 31C7E470E2138192,CF1A108D0E7AE720,8A03EA3B385DBAA1,12CE0F1D262429A5,5E6DDE998AB73B8E,51C67305FFC2E5C0,CBB3D576F2A0946F \ --target /usr/bin + +apk del --no-network --purge .php-build-deps diff --git a/scripts/install-rustfmt.sh b/scripts/install-rustfmt.sh deleted file mode 100755 index c47a714a..00000000 --- a/scripts/install-rustfmt.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash - -# https://doc.rust-lang.org/rustc/platform-support.html - -set -euo pipefail - -case $TARGETARCH in -amd64) - target=x86_64 - ;; -arm64) - target=aarch64 - ;; -*) - echo "$TARGETARCH is not supported" - exit 1 - ;; -esac - -ln -s /usr/bin/rustup-init /usr/bin/rustup -rustup toolchain install stable-${target}-unknown-linux-musl -rustup component add rustfmt --toolchain=stable-${target}-unknown-linux-musl -rustup component add clippy --toolchain=stable-${target}-unknown-linux-musl -mv /root/.rustup /usr/lib/.rustup -ln -fsv /usr/lib/.rustup/toolchains/stable-${target}-unknown-linux-musl/bin/rustfmt /usr/bin/rustfmt -ln -fsv /usr/lib/.rustup/toolchains/stable-${target}-unknown-linux-musl/bin/rustc /usr/bin/rustc -ln -fsv /usr/lib/.rustup/toolchains/stable-${target}-unknown-linux-musl/bin/cargo /usr/bin/cargo -ln -fsv /usr/lib/.rustup/toolchains/stable-${target}-unknown-linux-musl/bin/cargo-clippy /usr/bin/cargo-clippy - -cat <<'EOF' >/usr/bin/clippy -#!/usr/bin/env bash -pushd $(dirname $1) -cargo-clippy -rc=$? -popd -exit $rc -EOF -chmod +x /usr/bin/clippy diff --git a/test/inspec/super-linter/controls/super_linter.rb b/test/inspec/super-linter/controls/super_linter.rb index 02d9ed01..9f6af1f8 100644 --- a/test/inspec/super-linter/controls/super_linter.rb +++ b/test/inspec/super-linter/controls/super_linter.rb @@ -14,43 +14,20 @@ control "super-linter-installed-packages" do packages = [ "bash", "ca-certificates", - "cargo", - "cmake", "coreutils", "curl", "file", - "g++", - "gcc", "git-lfs", "git", "glibc", - "gnupg", "go", - "icu-libs", - "jpeg-dev", "jq", - "krb5-libs", - "libc-dev", - "libcurl", - "libffi-dev", - "libgcc", - "libintl", - "libssl3", - "libstdc++", - "libxml2-dev", "libxml2-utils", - "linux-headers", - "lttng-ust-dev", - "make", - "musl-dev", "nodejs-current", "npm", - "net-snmp-dev", "openjdk17-jre", "openssh-client", - "openssl-dev", "parallel", - "perl-dev", "perl", "php82", "php82-ctype", @@ -63,29 +40,19 @@ control "super-linter-installed-packages" do "php82-simplexml", "php82-tokenizer", "php82-xmlwriter", - "py3-pyflakes", - "py3-setuptools", - "python3-dev", - "R-dev", - "R-doc", "R", "rakudo", - "readline-dev", - "ruby-bundler", - "ruby-dev", - "ruby-rdoc", "ruby", - "rustup", + "rust-clippy", + "rustfmt", "tar", - "zef", - "zlib-dev", - "zlib", - "zstd" + "zef" ] # Removed linters from slim image SLIM_IMAGE_REMOVED_PACKAGES=%w( - rustup + rust-clippy + rustfmt ) packages.each do |item| @@ -99,6 +66,36 @@ control "super-linter-installed-packages" do end end +control "super-linter-uninstalled-packages" do + impact 1 + title "Super-Linter uninstalled packages check" + desc "Check that packages that Super-Linter doesn't need are not installed." + + packages = [ + "cmake", + "g++", + "gnupg", + "libc-dev", + "libxml2-dev", + "linux-headers", + "make", + "perl-dev", + "python3-dev", + "R-dev", + "R-doc", + "readline-dev", + "ruby-bundler", + "ruby-dev", + "ruby-rdoc" + ] + + packages.each do |item| + describe package(item) do + it { should_not be_installed } + end + end +end + ########################################### # Check to see all binaries are installed # ###########################################