build: reduce container image size (#5072)

- Remove build-time dependencies - Remove cached NPM packages - Remove cached PyPi packages - Remove dependency descriptors. These still count against the total space, although it's a few KBs - Install rust-clippy and rust-fmt using the OS package manager instead of maintaining our own installation script - Add tests for build time dependencies that are not supposed to be installed
2024-11-23 17:21:04 -05:00 · 2023-12-29 19:30:58 +01:00 · 2023-12-29 19:30:58 +01:00 · 1ca3ebccd6
commit 1ca3ebccd6
parent 0355c996e0
10 changed files with 183 additions and 169 deletions
--- a/183
+++ b/183
@ -1,10 +1,8 @@
-###########################################
-###########################################
-## Dockerfile to run GitHub Super-Linter ##
-###########################################
-###########################################
-
-ARG GLIBC_VERSION='2.34-r0'
+####################################
+####################################
+## Dockerfile to run Super-Linter ##
+####################################
+####################################

 #########################################
 # Get dependency images as build stages #
@ -47,79 +45,89 @@ LABEL com.github.actions.name="Super-Linter" \
 ARG TARGETARCH

 # Install bash first so we can use it
+# This is also a super-linter runtime dependency
 RUN apk add --no-cache \
    bash

 SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-c"]

+# Install super-linter runtime dependencies
 RUN apk add --no-cache \
    ca-certificates \
-    cargo \
-    cmake \
    coreutils \
    curl \
    file \
-    g++ \
-    gcc \
    git \
    git-lfs \
-    gnupg \
-    icu-libs \
-    jpeg-dev \
    jq \
-    krb5-libs \
-    libc-dev \
-    libcurl \
-    libffi-dev \
-    libgcc \
-    libintl \
-    libssl3 \
-    libstdc++ \
-    libxml2-dev \
    libxml2-utils \
-    linux-headers \
-    lttng-ust-dev \
-    make \
-    musl-dev \
-    net-snmp-dev \
    nodejs-current \
-    npm \
    openjdk17-jre \
    openssh-client \
-    openssl-dev \
-    parallel \
    perl \
-    perl-dev \
-    py3-pyflakes \
-    py3-setuptools \
-    python3-dev  \
+    php82 \
+    php82-ctype \
+    php82-curl \
+    php82-dom \
+    php82-iconv \
+    php82-mbstring \
+    php82-openssl \
+    php82-phar \
+    php82-simplexml \
+    php82-tokenizer \
+    php82-xmlwriter \
    R \
-    R-dev \
-    R-doc \
-    readline-dev \
+    rakudo \
    ruby \
+    zef
+
+# Install Node tools
+# The chown fixes broken uid/gid in ast-types-flow dependency
+# (see https://github.com/super-linter/super-linter/issues/3901)
+# Npm is not a runtime dependency but we need it to ensure that npm packages
+# are installed when we run the test suite. If we decide to remove it, add
+# the following command to the RUN instruction below:
+# apk del --no-network --purge .node-build-deps
+COPY dependencies/package.json dependencies/package-lock.json /
+RUN apk add --no-cache --virtual .node-build-deps \
+    npm \
+    && npm install \
+    && npm cache clean --force \
+    && chown -R "$(id -u)":"$(id -g)" node_modules \
+    && rm -rfv package.json package-lock.json
+
+# Install Ruby tools
+COPY dependencies/Gemfile dependencies/Gemfile.lock /
+RUN apk add --no-cache --virtual .ruby-build-deps \
+    gcc \
+    make \
+    musl-dev \
    ruby-bundler \
    ruby-dev \
    ruby-rdoc \
-    rustup \
-    tar \
-    zlib \
-    zlib-dev \
-    zstd
-
-COPY dependencies/ /
-
-###################################################################
-# Install Dependencies                                            #
-# The chown fixes broken uid/gid in ast-types-flow dependency     #
-# (see https://github.com/super-linter/super-linter/issues/3901)  #
-###################################################################
-RUN npm install && chown -R "$(id -u)":"$(id -g)" node_modules && bundle install
+    && bundle install \
+    && apk del --no-network --purge .ruby-build-deps \
+    && rm -rf Gemfile Gemfile.lock

 ##############################
 # Installs Perl dependencies #
 ##############################
-RUN curl --retry 5 --retry-delay 5 -sL https://cpanmin.us/ | perl - -nq --no-wget Perl::Critic Perl::Critic::Community Perl::Critic::More Perl::Critic::Bangs Perl::Critic::Lax Perl::Critic::StricterSubs Perl::Critic::Swift Perl::Critic::Tics
+RUN apk add --no-cache --virtual .perl-build-deps \
+    gcc \
+    make \
+    musl-dev \
+    perl-dev \
+    && curl --retry 5 --retry-delay 5 -sL https://cpanmin.us/ \
+    | perl - -nq --no-wget \
+    Perl::Critic \
+    Perl::Critic::Bangs \
+    Perl::Critic::Community \
+    Perl::Critic::Lax \
+    Perl::Critic::More \
+    Perl::Critic::StricterSubs \
+    Perl::Critic::Swift \
+    Perl::Critic::Tics \
+    && apk del --no-network --purge .perl-build-deps

 ######################
 # Install shellcheck #
@ -206,16 +214,16 @@ COPY --from=actionlint /usr/local/bin/actionlint /usr/bin/
 ######################
 COPY --from=kubeconfrm /kubeconform /usr/bin/

-# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub
-# Store the key here because the above host is sometimes down, and breaks our builds
-COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub
-
 #################
 # Install glibc #
 #################
-ARG GLIBC_VERSION
+# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub
+# Store the key here because the above host is sometimes down, and breaks our builds
+COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub
+ARG GLIBC_VERSION='2.34-r0'
 COPY scripts/install-glibc.sh /
-RUN --mount=type=secret,id=GITHUB_TOKEN /install-glibc.sh && rm -rf /install-glibc.sh
+RUN --mount=type=secret,id=GITHUB_TOKEN /install-glibc.sh \
+    && rm -rf /install-glibc.sh /sgerrand.rsa.pub

 #####################
 # Install clj-kondo #
@ -246,43 +254,41 @@ RUN --mount=type=secret,id=GITHUB_TOKEN /install-lua.sh && rm -rf /install-lua.s
 #####################################
 COPY dependencies/python/ /stage
 WORKDIR /stage
-RUN ./build-venvs.sh
+RUN ./build-venvs.sh && rm -rfv /stage
 # Set work directory back to root because some scripts depend on it
 WORKDIR /

 ##############################
 # Install Phive dependencies #
 ##############################
+COPY dependencies/phive.xml /phive.xml
 COPY scripts/install-phive.sh /
-RUN /install-phive.sh && rm -rf /install-phive.sh
+RUN /install-phive.sh \
+    && rm -rfv /install-phive.sh /phive.xml

 ##################
 # Install ktlint #
 ##################
 COPY scripts/install-ktlint.sh /
-RUN --mount=type=secret,id=GITHUB_TOKEN /install-ktlint.sh && rm -rf /install-ktlint.sh
-
-#################################################
-# Install Raku and additional Edge dependencies #
-#################################################
-RUN apk add --no-cache rakudo zef
+COPY dependencies/ktlint /ktlint
+RUN --mount=type=secret,id=GITHUB_TOKEN /install-ktlint.sh \
+    && rm -rfv /install-ktlint.sh /ktlint

 ######################
 # Install CheckStyle #
 ######################
 COPY scripts/install-checkstyle.sh /
-RUN --mount=type=secret,id=GITHUB_TOKEN /install-checkstyle.sh && rm -rf /install-checkstyle.sh
+COPY dependencies/checkstyle /checkstyle
+RUN --mount=type=secret,id=GITHUB_TOKEN /install-checkstyle.sh \
+    && rm -rfv /install-checkstyle.sh /checkstyle

 ##############################
 # Install google-java-format #
 ##############################
 COPY scripts/install-google-java-format.sh /
-RUN --mount=type=secret,id=GITHUB_TOKEN /install-google-java-format.sh && rm -rf /install-google-java-format.sh
-
-#########################
-# Clean to shrink image #
-#########################
-RUN find /usr/ -type f -name '*.md' -exec rm {} +
+COPY dependencies/google-java-format /google-java-format
+RUN --mount=type=secret,id=GITHUB_TOKEN /install-google-java-format.sh \
+    && rm -rfv /install-google-java-format.sh /google-java-format

 #####################
 # Install Bash-Exec #
@ -327,17 +333,17 @@ ENV PATH="${PATH}:${DART_SDK}/bin:/root/.pub-cache/bin"
 ENV TFLINT_PLUGIN_DIR="/root/.tflint.d/plugins"

 # Initialize TFLint plugins so we get plugin versions listed when we ask for TFLint version
-# Run to build version file and validate image
-RUN tflint --init -c /action/lib/.automation/.tflint.hcl \
-    && ACTIONS_RUNNER_DEBUG=true WRITE_LINTER_VERSIONS_FILE=true IMAGE="${IMAGE}" /action/lib/linter.sh
-
-ENTRYPOINT ["/action/lib/linter.sh"]
-
 # Initialize Terrascan
 # Initialize ChkTeX config file
-RUN terrascan init \
+RUN tflint --init -c /action/lib/.automation/.tflint.hcl \
+    && terrascan init \
    && touch ~/.chktexrc

+# Run to build version file and validate image
+RUN ACTIONS_RUNNER_DEBUG=true WRITE_LINTER_VERSIONS_FILE=true IMAGE="${IMAGE}" /action/lib/linter.sh
+
+ENTRYPOINT ["/action/lib/linter.sh"]
+
 FROM base_image as slim

 # Set build metadata here so we don't invalidate the container image cache if we
@ -369,6 +375,14 @@ ENV ARM_TTK_PSD1="/usr/lib/microsoft/arm-ttk/arm-ttk.psd1"
 ENV IMAGE="standard"
 ENV PATH="${PATH}:/var/cache/dotnet/tools:/usr/share/dotnet"

+# Install super-linter runtime dependencies
+RUN apk add --no-cache \
+    rust-clippy \
+    rustfmt
+
+COPY scripts/clippy.sh /usr/bin/clippy
+RUN chmod +x /usr/bin/clippy
+
 #########################
 # Install dotenv-linter #
 #########################
@ -380,13 +394,6 @@ COPY --from=dotenv-linter /dotenv-linter /usr/bin/
 COPY scripts/install-dotnet.sh /
 RUN /install-dotnet.sh && rm -rf /install-dotnet.sh

-##############################
-# Install rustfmt & clippy   #
-##############################
-ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
-COPY scripts/install-rustfmt.sh /
-RUN /install-rustfmt.sh && rm -rf /install-rustfmt.sh
-
 #########################################
 # Install Powershell + PSScriptAnalyzer #
 #########################################
--- a/dependencies/python/build-venvs.sh
+++ b/dependencies/python/build-venvs.sh
@ -8,6 +8,12 @@
 #####################
 set -euo pipefail

+apk add --no-cache --virtual .python-build-deps \
+  gcc \
+  linux-headers \
+  musl-dev \
+  python3-dev
+
 ############################
 # Create staging directory #
 ############################
@ -32,9 +38,13 @@ for DEP_FILE in *.txt; do
  virtualenv .
  # shellcheck disable=SC1091
  source bin/activate
-  pip install -r requirements.txt
+  pip install \
+    --no-cache-dir \
+    --requirement requirements.txt
  # deactivate the python virtualenv
  deactivate
  # pop the stack
  popd
 done
+
+apk del --no-network --purge .python-build-deps
--- a/scripts/clippy.sh
+++ b/scripts/clippy.sh
@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+pushd "$(dirname "$1")" || exit 1
+
+cargo-clippy
+
+rc=$?
+
+popd || exit 1
+
+exit $rc
--- a/scripts/install-dotnet.sh
+++ b/scripts/install-dotnet.sh
@ -5,3 +5,4 @@ set -euo pipefail
 curl --retry 5 --retry-delay 5 -sLO https://dot.net/v1/dotnet-install.sh
 chmod +x dotnet-install.sh
 ./dotnet-install.sh --install-dir /usr/share/dotnet -channel LTS -version latest
+rm -rfv ./dotnet-install.sh
--- a/scripts/install-glibc.sh
+++ b/scripts/install-glibc.sh
@ -32,6 +32,13 @@ rm "glibc-${GLIBC_VERSION}.apk"

 # Install zlib
 mkdir /tmp/libz
+
+apk add --no-cache --virtual .glibc-build-deps \
+  tar \
+  zstd
+
 curl --retry 5 --retry-delay 5 -sL https://www.archlinux.org/packages/core/${target}/zlib/download | tar -x --zstd -C /tmp/libz
 mv /tmp/libz/usr/lib/libz.so* /usr/glibc-compat/lib
 rm -rf /tmp/libz
+
+apk del --no-network --purge .glibc-build-deps
--- a/scripts/install-lintr.sh
+++ b/scripts/install-lintr.sh
@ -2,4 +2,15 @@

 set -euo pipefail

+apk add --no-cache --virtual .r-build-deps \
+  g++ \
+  gcc \
+  libxml2-dev \
+  linux-headers \
+  make \
+  R-dev \
+  R-doc
+
 Rscript --no-save /install-r-package-or-fail.R lintr purrr remotes
+
+apk del --no-network --purge .r-build-deps
--- a/scripts/install-lua.sh
+++ b/scripts/install-lua.sh
@ -2,6 +2,12 @@

 set -euo pipefail

+apk add --no-cache --virtual .lua-build-deps \
+  gcc \
+  make \
+  musl-dev \
+  readline-dev
+
 curl --retry 5 --retry-delay 5 -s https://www.lua.org/ftp/lua-5.3.5.tar.gz | tar -xz
 cd lua-5.3.5
 make linux
@ -30,3 +36,5 @@ rm -r cvega-luarocks-6b1aee6
 luarocks install luacheck
 luarocks install argparse
 luarocks install luafilesystem
+
+apk del --no-network --purge .lua-build-deps
--- a/scripts/install-phive.sh
+++ b/scripts/install-phive.sh
@ -2,10 +2,8 @@

 set -euo pipefail

-# Install PHP
-apk add --no-cache \
-  php82 php82-curl php82-ctype php82-dom php82-iconv php82-mbstring \
-  php82-openssl php82-phar php82-simplexml php82-tokenizer php82-xmlwriter
+apk add --no-cache --virtual .php-build-deps \
+  gnupg

 # Install phive
 curl --retry 5 --retry-delay 5 -sLO https://phar.io/releases/phive.phar
@ -20,3 +18,5 @@ rm phive.phar.asc
 phive --no-progress install \
  --trust-gpg-keys 31C7E470E2138192,CF1A108D0E7AE720,8A03EA3B385DBAA1,12CE0F1D262429A5,5E6DDE998AB73B8E,51C67305FFC2E5C0,CBB3D576F2A0946F \
  --target /usr/bin
+
+apk del --no-network --purge .php-build-deps
--- a/scripts/install-rustfmt.sh
+++ b/scripts/install-rustfmt.sh
@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-
-# https://doc.rust-lang.org/rustc/platform-support.html
-
-set -euo pipefail
-
-case $TARGETARCH in
-amd64)
-  target=x86_64
-  ;;
-arm64)
-  target=aarch64
-  ;;
-*)
-  echo "$TARGETARCH is not supported"
-  exit 1
-  ;;
-esac
-
-ln -s /usr/bin/rustup-init /usr/bin/rustup
-rustup toolchain install stable-${target}-unknown-linux-musl
-rustup component add rustfmt --toolchain=stable-${target}-unknown-linux-musl
-rustup component add clippy --toolchain=stable-${target}-unknown-linux-musl
-mv /root/.rustup /usr/lib/.rustup
-ln -fsv /usr/lib/.rustup/toolchains/stable-${target}-unknown-linux-musl/bin/rustfmt /usr/bin/rustfmt
-ln -fsv /usr/lib/.rustup/toolchains/stable-${target}-unknown-linux-musl/bin/rustc /usr/bin/rustc
-ln -fsv /usr/lib/.rustup/toolchains/stable-${target}-unknown-linux-musl/bin/cargo /usr/bin/cargo
-ln -fsv /usr/lib/.rustup/toolchains/stable-${target}-unknown-linux-musl/bin/cargo-clippy /usr/bin/cargo-clippy
-
-cat <<'EOF' >/usr/bin/clippy
-#!/usr/bin/env bash
-pushd $(dirname $1)
-cargo-clippy
-rc=$?
-popd
-exit $rc
-EOF
-chmod +x /usr/bin/clippy
--- a/test/inspec/super-linter/controls/super_linter.rb
+++ b/test/inspec/super-linter/controls/super_linter.rb
@ -14,43 +14,20 @@ control "super-linter-installed-packages" do
  packages = [
    "bash",
    "ca-certificates",
-    "cargo",
-    "cmake",
    "coreutils",
    "curl",
    "file",
-    "g++",
-    "gcc",
    "git-lfs",
    "git",
    "glibc",
-    "gnupg",
    "go",
-    "icu-libs",
-    "jpeg-dev",
    "jq",
-    "krb5-libs",
-    "libc-dev",
-    "libcurl",
-    "libffi-dev",
-    "libgcc",
-    "libintl",
-    "libssl3",
-    "libstdc++",
-    "libxml2-dev",
    "libxml2-utils",
-    "linux-headers",
-    "lttng-ust-dev",
-    "make",
-    "musl-dev",
    "nodejs-current",
    "npm",
-    "net-snmp-dev",
    "openjdk17-jre",
    "openssh-client",
-    "openssl-dev",
    "parallel",
-    "perl-dev",
    "perl",
    "php82",
    "php82-ctype",
@ -63,29 +40,19 @@ control "super-linter-installed-packages" do
    "php82-simplexml",
    "php82-tokenizer",
    "php82-xmlwriter",
-    "py3-pyflakes",
-    "py3-setuptools",
-    "python3-dev",
-    "R-dev",
-    "R-doc",
    "R",
    "rakudo",
-    "readline-dev",
-    "ruby-bundler",
-    "ruby-dev",
-    "ruby-rdoc",
    "ruby",
-    "rustup",
+    "rust-clippy",
+    "rustfmt",
    "tar",
-    "zef",
-    "zlib-dev",
-    "zlib",
-    "zstd"
+    "zef"
  ]

  # Removed linters from slim image
  SLIM_IMAGE_REMOVED_PACKAGES=%w(
-    rustup
+    rust-clippy
+    rustfmt
  )

  packages.each do |item|
@ -99,6 +66,36 @@ control "super-linter-installed-packages" do
  end
 end

+control "super-linter-uninstalled-packages" do
+  impact 1
+  title "Super-Linter uninstalled packages check"
+  desc "Check that packages that Super-Linter doesn't need are not installed."
+
+  packages = [
+    "cmake",
+    "g++",
+    "gnupg",
+    "libc-dev",
+    "libxml2-dev",
+    "linux-headers",
+    "make",
+    "perl-dev",
+    "python3-dev",
+    "R-dev",
+    "R-doc",
+    "readline-dev",
+    "ruby-bundler",
+    "ruby-dev",
+    "ruby-rdoc"
+  ]
+
+  packages.each do |item|
+    describe package(item) do
+      it { should_not be_installed }
+    end
+  end
+end
+
 ###########################################
 # Check to see all binaries are installed #
 ###########################################