build: reduce container image size (#5072)

- Remove build-time dependencies
- Remove cached NPM packages
- Remove cached PyPi packages
- Remove dependency descriptors. These still count against the total space,
  although it's a few KBs
- Install rust-clippy and rust-fmt using the OS package manager instead of
  maintaining our own installation script
- Add tests for build time dependencies that are not supposed to be installed

Dockerfile

@@ -1,10 +1,8 @@
-###########################################
+####################################
-###########################################
+####################################
-## Dockerfile to run GitHub Super-Linter ##
+## Dockerfile to run Super-Linter ##
-###########################################
+####################################
-###########################################
+####################################
-
-ARG GLIBC_VERSION='2.34-r0'
 
 #########################################
 # Get dependency images as build stages #
@@ -47,79 +45,89 @@ LABEL com.github.actions.name="Super-Linter" \
 ARG TARGETARCH
 
 # Install bash first so we can use it
+# This is also a super-linter runtime dependency
 RUN apk add --no-cache \
     bash
 
 SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-c"]
 
+# Install super-linter runtime dependencies
 RUN apk add --no-cache \
     ca-certificates \
-    cargo \
-    cmake \
     coreutils \
     curl \
     file \
-    g++ \
-    gcc \
     git \
     git-lfs \
-    gnupg \
-    icu-libs \
-    jpeg-dev \
     jq \
-    krb5-libs \
-    libc-dev \
-    libcurl \
-    libffi-dev \
-    libgcc \
-    libintl \
-    libssl3 \
-    libstdc++ \
-    libxml2-dev \
     libxml2-utils \
-    linux-headers \
-    lttng-ust-dev \
-    make \
-    musl-dev \
-    net-snmp-dev \
     nodejs-current \
-    npm \
     openjdk17-jre \
     openssh-client \
-    openssl-dev \
-    parallel \
     perl \
-    perl-dev \
+    php82 \
-    py3-pyflakes \
+    php82-ctype \
-    py3-setuptools \
+    php82-curl \
-    python3-dev  \
+    php82-dom \
+    php82-iconv \
+    php82-mbstring \
+    php82-openssl \
+    php82-phar \
+    php82-simplexml \
+    php82-tokenizer \
+    php82-xmlwriter \
     R \
-    R-dev \
+    rakudo \
-    R-doc \
-    readline-dev \
     ruby \
+    zef
+
+# Install Node tools
+# The chown fixes broken uid/gid in ast-types-flow dependency
+# (see https://github.com/super-linter/super-linter/issues/3901)
+# Npm is not a runtime dependency but we need it to ensure that npm packages
+# are installed when we run the test suite. If we decide to remove it, add
+# the following command to the RUN instruction below:
+# apk del --no-network --purge .node-build-deps
+COPY dependencies/package.json dependencies/package-lock.json /
+RUN apk add --no-cache --virtual .node-build-deps \
+    npm \
+    && npm install \
+    && npm cache clean --force \
+    && chown -R "$(id -u)":"$(id -g)" node_modules \
+    && rm -rfv package.json package-lock.json
+
+# Install Ruby tools
+COPY dependencies/Gemfile dependencies/Gemfile.lock /
+RUN apk add --no-cache --virtual .ruby-build-deps \
+    gcc \
+    make \
+    musl-dev \
     ruby-bundler \
     ruby-dev \
     ruby-rdoc \
-    rustup \
+    && bundle install \
-    tar \
+    && apk del --no-network --purge .ruby-build-deps \
-    zlib \
+    && rm -rf Gemfile Gemfile.lock
-    zlib-dev \
-    zstd
-
-COPY dependencies/ /
-
-###################################################################
-# Install Dependencies                                            #
-# The chown fixes broken uid/gid in ast-types-flow dependency     #
-# (see https://github.com/super-linter/super-linter/issues/3901)  #
-###################################################################
-RUN npm install && chown -R "$(id -u)":"$(id -g)" node_modules && bundle install
 
 ##############################
 # Installs Perl dependencies #
 ##############################
-RUN curl --retry 5 --retry-delay 5 -sL https://cpanmin.us/ | perl - -nq --no-wget Perl::Critic Perl::Critic::Community Perl::Critic::More Perl::Critic::Bangs Perl::Critic::Lax Perl::Critic::StricterSubs Perl::Critic::Swift Perl::Critic::Tics
+RUN apk add --no-cache --virtual .perl-build-deps \
+    gcc \
+    make \
+    musl-dev \
+    perl-dev \
+    && curl --retry 5 --retry-delay 5 -sL https://cpanmin.us/ \
+    | perl - -nq --no-wget \
+    Perl::Critic \
+    Perl::Critic::Bangs \
+    Perl::Critic::Community \
+    Perl::Critic::Lax \
+    Perl::Critic::More \
+    Perl::Critic::StricterSubs \
+    Perl::Critic::Swift \
+    Perl::Critic::Tics \
+    && apk del --no-network --purge .perl-build-deps
 
 ######################
 # Install shellcheck #
@@ -206,16 +214,16 @@ COPY --from=actionlint /usr/local/bin/actionlint /usr/bin/
 ######################
 COPY --from=kubeconfrm /kubeconform /usr/bin/
 
-# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub
-# Store the key here because the above host is sometimes down, and breaks our builds
-COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub
-
 #################
 # Install glibc #
 #################
-ARG GLIBC_VERSION
+# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub
+# Store the key here because the above host is sometimes down, and breaks our builds
+COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub
+ARG GLIBC_VERSION='2.34-r0'
 COPY scripts/install-glibc.sh /
-RUN --mount=type=secret,id=GITHUB_TOKEN /install-glibc.sh && rm -rf /install-glibc.sh
+RUN --mount=type=secret,id=GITHUB_TOKEN /install-glibc.sh \
+    && rm -rf /install-glibc.sh /sgerrand.rsa.pub
 
 #####################
 # Install clj-kondo #
@@ -246,43 +254,41 @@ RUN --mount=type=secret,id=GITHUB_TOKEN /install-lua.sh && rm -rf /install-lua.s
 #####################################
 COPY dependencies/python/ /stage
 WORKDIR /stage
-RUN ./build-venvs.sh
+RUN ./build-venvs.sh && rm -rfv /stage
 # Set work directory back to root because some scripts depend on it
 WORKDIR /
 
 ##############################
 # Install Phive dependencies #
 ##############################
+COPY dependencies/phive.xml /phive.xml
 COPY scripts/install-phive.sh /
-RUN /install-phive.sh && rm -rf /install-phive.sh
+RUN /install-phive.sh \
+    && rm -rfv /install-phive.sh /phive.xml
 
 ##################
 # Install ktlint #
 ##################
 COPY scripts/install-ktlint.sh /
-RUN --mount=type=secret,id=GITHUB_TOKEN /install-ktlint.sh && rm -rf /install-ktlint.sh
+COPY dependencies/ktlint /ktlint
-
+RUN --mount=type=secret,id=GITHUB_TOKEN /install-ktlint.sh \
-#################################################
+    && rm -rfv /install-ktlint.sh /ktlint
-# Install Raku and additional Edge dependencies #
-#################################################
-RUN apk add --no-cache rakudo zef
 
 ######################
 # Install CheckStyle #
 ######################
 COPY scripts/install-checkstyle.sh /
-RUN --mount=type=secret,id=GITHUB_TOKEN /install-checkstyle.sh && rm -rf /install-checkstyle.sh
+COPY dependencies/checkstyle /checkstyle
+RUN --mount=type=secret,id=GITHUB_TOKEN /install-checkstyle.sh \
+    && rm -rfv /install-checkstyle.sh /checkstyle
 
 ##############################
 # Install google-java-format #
 ##############################
 COPY scripts/install-google-java-format.sh /
-RUN --mount=type=secret,id=GITHUB_TOKEN /install-google-java-format.sh && rm -rf /install-google-java-format.sh
+COPY dependencies/google-java-format /google-java-format
-
+RUN --mount=type=secret,id=GITHUB_TOKEN /install-google-java-format.sh \
-#########################
+    && rm -rfv /install-google-java-format.sh /google-java-format
-# Clean to shrink image #
-#########################
-RUN find /usr/ -type f -name '*.md' -exec rm {} +
 
 #####################
 # Install Bash-Exec #
@@ -327,17 +333,17 @@ ENV PATH="${PATH}:${DART_SDK}/bin:/root/.pub-cache/bin"
 ENV TFLINT_PLUGIN_DIR="/root/.tflint.d/plugins"
 
 # Initialize TFLint plugins so we get plugin versions listed when we ask for TFLint version
-# Run to build version file and validate image
-RUN tflint --init -c /action/lib/.automation/.tflint.hcl \
-    && ACTIONS_RUNNER_DEBUG=true WRITE_LINTER_VERSIONS_FILE=true IMAGE="${IMAGE}" /action/lib/linter.sh
-
-ENTRYPOINT ["/action/lib/linter.sh"]
-
 # Initialize Terrascan
 # Initialize ChkTeX config file
-RUN terrascan init \
+RUN tflint --init -c /action/lib/.automation/.tflint.hcl \
+    && terrascan init \
     && touch ~/.chktexrc
 
+# Run to build version file and validate image
+RUN ACTIONS_RUNNER_DEBUG=true WRITE_LINTER_VERSIONS_FILE=true IMAGE="${IMAGE}" /action/lib/linter.sh
+
+ENTRYPOINT ["/action/lib/linter.sh"]
+
 FROM base_image as slim
 
 # Set build metadata here so we don't invalidate the container image cache if we
@@ -369,6 +375,14 @@ ENV ARM_TTK_PSD1="/usr/lib/microsoft/arm-ttk/arm-ttk.psd1"
 ENV IMAGE="standard"
 ENV PATH="${PATH}:/var/cache/dotnet/tools:/usr/share/dotnet"
 
+# Install super-linter runtime dependencies
+RUN apk add --no-cache \
+    rust-clippy \
+    rustfmt
+
+COPY scripts/clippy.sh /usr/bin/clippy
+RUN chmod +x /usr/bin/clippy
+
 #########################
 # Install dotenv-linter #
 #########################
@@ -380,13 +394,6 @@ COPY --from=dotenv-linter /dotenv-linter /usr/bin/
 COPY scripts/install-dotnet.sh /
 RUN /install-dotnet.sh && rm -rf /install-dotnet.sh
 
-##############################
-# Install rustfmt & clippy   #
-##############################
-ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
-COPY scripts/install-rustfmt.sh /
-RUN /install-rustfmt.sh && rm -rf /install-rustfmt.sh
-
 #########################################
 # Install Powershell + PSScriptAnalyzer #
 #########################################

dependencies/python/build-venvs.sh

@@ -8,6 +8,12 @@
 #####################
 set -euo pipefail
 
+apk add --no-cache --virtual .python-build-deps \
+  gcc \
+  linux-headers \
+  musl-dev \
+  python3-dev
+
 ############################
 # Create staging directory #
 ############################
@@ -32,9 +38,13 @@ for DEP_FILE in *.txt; do
   virtualenv .
   # shellcheck disable=SC1091
   source bin/activate
-  pip install -r requirements.txt
+  pip install \
+    --no-cache-dir \
+    --requirement requirements.txt
   # deactivate the python virtualenv
   deactivate
   # pop the stack
   popd
 done
+
+apk del --no-network --purge .python-build-deps

scripts/clippy.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+pushd "$(dirname "$1")" || exit 1
+
+cargo-clippy
+
+rc=$?
+
+popd || exit 1
+
+exit $rc

scripts/install-dotnet.sh

@@ -5,3 +5,4 @@ set -euo pipefail
 curl --retry 5 --retry-delay 5 -sLO https://dot.net/v1/dotnet-install.sh
 chmod +x dotnet-install.sh
 ./dotnet-install.sh --install-dir /usr/share/dotnet -channel LTS -version latest
+rm -rfv ./dotnet-install.sh

scripts/install-glibc.sh

@@ -32,6 +32,13 @@ rm "glibc-${GLIBC_VERSION}.apk"
 
 # Install zlib
 mkdir /tmp/libz
+
+apk add --no-cache --virtual .glibc-build-deps \
+  tar \
+  zstd
+
 curl --retry 5 --retry-delay 5 -sL https://www.archlinux.org/packages/core/${target}/zlib/download | tar -x --zstd -C /tmp/libz
 mv /tmp/libz/usr/lib/libz.so* /usr/glibc-compat/lib
 rm -rf /tmp/libz
+
+apk del --no-network --purge .glibc-build-deps

scripts/install-lintr.sh

@@ -2,4 +2,15 @@
 
 set -euo pipefail
 
+apk add --no-cache --virtual .r-build-deps \
+  g++ \
+  gcc \
+  libxml2-dev \
+  linux-headers \
+  make \
+  R-dev \
+  R-doc
+
 Rscript --no-save /install-r-package-or-fail.R lintr purrr remotes
+
+apk del --no-network --purge .r-build-deps

scripts/install-lua.sh

@@ -2,6 +2,12 @@
 
 set -euo pipefail
 
+apk add --no-cache --virtual .lua-build-deps \
+  gcc \
+  make \
+  musl-dev \
+  readline-dev
+
 curl --retry 5 --retry-delay 5 -s https://www.lua.org/ftp/lua-5.3.5.tar.gz | tar -xz
 cd lua-5.3.5
 make linux
@@ -30,3 +36,5 @@ rm -r cvega-luarocks-6b1aee6
 luarocks install luacheck
 luarocks install argparse
 luarocks install luafilesystem
+
+apk del --no-network --purge .lua-build-deps

scripts/install-phive.sh

@@ -2,10 +2,8 @@
 
 set -euo pipefail
 
-# Install PHP
+apk add --no-cache --virtual .php-build-deps \
-apk add --no-cache \
+  gnupg
-  php82 php82-curl php82-ctype php82-dom php82-iconv php82-mbstring \
-  php82-openssl php82-phar php82-simplexml php82-tokenizer php82-xmlwriter
 
 # Install phive
 curl --retry 5 --retry-delay 5 -sLO https://phar.io/releases/phive.phar
@@ -20,3 +18,5 @@ rm phive.phar.asc
 phive --no-progress install \
   --trust-gpg-keys 31C7E470E2138192,CF1A108D0E7AE720,8A03EA3B385DBAA1,12CE0F1D262429A5,5E6DDE998AB73B8E,51C67305FFC2E5C0,CBB3D576F2A0946F \
   --target /usr/bin
+
+apk del --no-network --purge .php-build-deps

scripts/install-rustfmt.sh

@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-
-# https://doc.rust-lang.org/rustc/platform-support.html
-
-set -euo pipefail
-
-case $TARGETARCH in
-amd64)
-  target=x86_64
-  ;;
-arm64)
-  target=aarch64
-  ;;
-*)
-  echo "$TARGETARCH is not supported"
-  exit 1
-  ;;
-esac
-
-ln -s /usr/bin/rustup-init /usr/bin/rustup
-rustup toolchain install stable-${target}-unknown-linux-musl
-rustup component add rustfmt --toolchain=stable-${target}-unknown-linux-musl
-rustup component add clippy --toolchain=stable-${target}-unknown-linux-musl
-mv /root/.rustup /usr/lib/.rustup
-ln -fsv /usr/lib/.rustup/toolchains/stable-${target}-unknown-linux-musl/bin/rustfmt /usr/bin/rustfmt
-ln -fsv /usr/lib/.rustup/toolchains/stable-${target}-unknown-linux-musl/bin/rustc /usr/bin/rustc
-ln -fsv /usr/lib/.rustup/toolchains/stable-${target}-unknown-linux-musl/bin/cargo /usr/bin/cargo
-ln -fsv /usr/lib/.rustup/toolchains/stable-${target}-unknown-linux-musl/bin/cargo-clippy /usr/bin/cargo-clippy
-
-cat <<'EOF' >/usr/bin/clippy
-#!/usr/bin/env bash
-pushd $(dirname $1)
-cargo-clippy
-rc=$?
-popd
-exit $rc
-EOF
-chmod +x /usr/bin/clippy

test/inspec/super-linter/controls/super_linter.rb

@@ -14,43 +14,20 @@ control "super-linter-installed-packages" do
   packages = [
     "bash",
     "ca-certificates",
-    "cargo",
-    "cmake",
     "coreutils",
     "curl",
     "file",
-    "g++",
-    "gcc",
     "git-lfs",
     "git",
     "glibc",
-    "gnupg",
     "go",
-    "icu-libs",
-    "jpeg-dev",
     "jq",
-    "krb5-libs",
-    "libc-dev",
-    "libcurl",
-    "libffi-dev",
-    "libgcc",
-    "libintl",
-    "libssl3",
-    "libstdc++",
-    "libxml2-dev",
     "libxml2-utils",
-    "linux-headers",
-    "lttng-ust-dev",
-    "make",
-    "musl-dev",
     "nodejs-current",
     "npm",
-    "net-snmp-dev",
     "openjdk17-jre",
     "openssh-client",
-    "openssl-dev",
     "parallel",
-    "perl-dev",
     "perl",
     "php82",
     "php82-ctype",
@@ -63,29 +40,19 @@ control "super-linter-installed-packages" do
     "php82-simplexml",
     "php82-tokenizer",
     "php82-xmlwriter",
-    "py3-pyflakes",
-    "py3-setuptools",
-    "python3-dev",
-    "R-dev",
-    "R-doc",
     "R",
     "rakudo",
-    "readline-dev",
-    "ruby-bundler",
-    "ruby-dev",
-    "ruby-rdoc",
     "ruby",
-    "rustup",
+    "rust-clippy",
+    "rustfmt",
     "tar",
-    "zef",
+    "zef"
-    "zlib-dev",
-    "zlib",
-    "zstd"
   ]
 
   # Removed linters from slim image
   SLIM_IMAGE_REMOVED_PACKAGES=%w(
-    rustup
+    rust-clippy
+    rustfmt
   )
 
   packages.each do |item|
@@ -99,6 +66,36 @@ control "super-linter-installed-packages" do
   end
 end
 
+control "super-linter-uninstalled-packages" do
+  impact 1
+  title "Super-Linter uninstalled packages check"
+  desc "Check that packages that Super-Linter doesn't need are not installed."
+
+  packages = [
+    "cmake",
+    "g++",
+    "gnupg",
+    "libc-dev",
+    "libxml2-dev",
+    "linux-headers",
+    "make",
+    "perl-dev",
+    "python3-dev",
+    "R-dev",
+    "R-doc",
+    "readline-dev",
+    "ruby-bundler",
+    "ruby-dev",
+    "ruby-rdoc"
+  ]
+
+  packages.each do |item|
+    describe package(item) do
+      it { should_not be_installed }
+    end
+  end
+end
+
 ###########################################
 # Check to see all binaries are installed #
 ###########################################