From df911171c4edf0a1937552e3b3b10858e11f2121 Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Wed, 10 Jan 2024 11:09:53 +0100 Subject: [PATCH] build: python venvs and npm in dedicated stages (#5078) Move the following tasks to dedicated stages so we can run these steps in parallel with other stages: - Build Python virtual environments - Install npm packages - Build clang-format - Download and install TFlint plugins Add missing target stage when building the container image --- Dockerfile | 215 ++++++++++-------- Makefile | 1 + .../super-linter/controls/super_linter.rb | 3 + 3 files changed, 125 insertions(+), 94 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1242894b..a36da217 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,6 +53,48 @@ RUN cmake \ && ninja clang-format \ && mv /tmp/llvm-project/llvm/build/bin/clang-format /usr/bin +FROM python:3.12.1-alpine3.19 as python-builder + +RUN apk add --no-cache \ + bash + +SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-c"] + +COPY dependencies/python/ /stage +WORKDIR /stage +RUN ./build-venvs.sh && rm -rfv /stage + +FROM python:3.12.1-alpine3.19 as npm-builder + +RUN apk add --no-cache \ + bash \ + nodejs-current + +# The chown fixes broken uid/gid in ast-types-flow dependency +# (see https://github.com/super-linter/super-linter/issues/3901) +# Npm is not a runtime dependency but we need it to ensure that npm packages +# are installed when we run the test suite. If we decide to remove it, add +# the following command to the RUN instruction below: +# apk del --no-network --purge .node-build-deps +COPY dependencies/package.json dependencies/package-lock.json / +RUN apk add --no-cache --virtual .node-build-deps \ + npm \ + && npm install \ + && npm cache clean --force \ + && chown -R "$(id -u)":"$(id -g)" node_modules \ + && rm -rfv package.json package-lock.json + +FROM tflint as tflint-plugins + +# Configure TFLint plugin folder +ENV TFLINT_PLUGIN_DIR="/root/.tflint.d/plugins" + +# Copy TFlint configuration file because it contains plugin definitions +COPY TEMPLATES/.tflint.hcl /action/lib/.automation/ + +# Initialize TFLint plugins so we get plugin versions listed when we ask for TFLint version +RUN tflint --init -c /action/lib/.automation/.tflint.hcl + FROM python:3.12.1-alpine3.19 as base_image LABEL com.github.actions.name="Super-Linter" \ @@ -77,6 +119,8 @@ RUN apk add --no-cache \ SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-c"] # Install super-linter runtime dependencies +# Npm is not a runtime dependency but we need it to ensure that npm packages +# are installed when we run the test suite. RUN apk add --no-cache \ ca-certificates \ coreutils \ @@ -86,6 +130,7 @@ RUN apk add --no-cache \ git-lfs \ jq \ libxml2-utils \ + npm \ nodejs-current \ openjdk17-jre \ openssh-client \ @@ -106,21 +151,6 @@ RUN apk add --no-cache \ ruby \ zef -# Install Node tools -# The chown fixes broken uid/gid in ast-types-flow dependency -# (see https://github.com/super-linter/super-linter/issues/3901) -# Npm is not a runtime dependency but we need it to ensure that npm packages -# are installed when we run the test suite. If we decide to remove it, add -# the following command to the RUN instruction below: -# apk del --no-network --purge .node-build-deps -COPY dependencies/package.json dependencies/package-lock.json / -RUN apk add --no-cache --virtual .node-build-deps \ - npm \ - && npm install \ - && npm cache clean --force \ - && chown -R "$(id -u)":"$(id -g)" node_modules \ - && rm -rfv package.json package-lock.json - # Install Ruby tools COPY dependencies/Gemfile dependencies/Gemfile.lock / RUN apk add --no-cache --virtual .ruby-build-deps \ @@ -154,6 +184,17 @@ RUN apk add --no-cache --virtual .perl-build-deps \ Perl::Critic::Tics \ && apk del --no-network --purge .perl-build-deps +################# +# Install glibc # +################# +# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub +# Store the key here because the above host is sometimes down, and breaks our builds +COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub +ARG GLIBC_VERSION='2.34-r0' +COPY scripts/install-glibc.sh / +RUN --mount=type=secret,id=GITHUB_TOKEN /install-glibc.sh \ + && rm -rf /install-glibc.sh /sgerrand.rsa.pub + ################## # Install chktex # ################## @@ -162,6 +203,53 @@ RUN --mount=type=secret,id=GITHUB_TOKEN /install-chktex.sh && rm -rf /install-ch # Set work directory back to root because some scripts depend on it WORKDIR / +################# +# Install Lintr # +################# +COPY scripts/install-lintr.sh scripts/install-r-package-or-fail.R / +RUN /install-lintr.sh && rm -rf /install-lintr.sh /install-r-package-or-fail.R + +################################# +# Install luacheck and luarocks # +################################# +COPY scripts/install-lua.sh / +RUN --mount=type=secret,id=GITHUB_TOKEN /install-lua.sh && rm -rf /install-lua.sh + +############################## +# Install Phive dependencies # +############################## +COPY dependencies/phive.xml /phive.xml +COPY scripts/install-phive.sh / +RUN /install-phive.sh \ + && rm -rfv /install-phive.sh /phive.xml + +################## +# Install ktlint # +################## +COPY scripts/install-ktlint.sh / +COPY dependencies/ktlint /ktlint +RUN --mount=type=secret,id=GITHUB_TOKEN /install-ktlint.sh \ + && rm -rfv /install-ktlint.sh /ktlint + +###################### +# Install CheckStyle # +###################### +COPY scripts/install-checkstyle.sh / +COPY dependencies/checkstyle /checkstyle +RUN --mount=type=secret,id=GITHUB_TOKEN /install-checkstyle.sh \ + && rm -rfv /install-checkstyle.sh /checkstyle + +############################## +# Install google-java-format # +############################## +COPY scripts/install-google-java-format.sh / +COPY dependencies/google-java-format /google-java-format +RUN --mount=type=secret,id=GITHUB_TOKEN /install-google-java-format.sh \ + && rm -rfv /install-google-java-format.sh /google-java-format + +# Copy Node tools +COPY --from=npm-builder /node_modules /node_modules + ###################### # Install shellcheck # ###################### @@ -185,7 +273,10 @@ COPY --from=terraform /bin/terraform /usr/bin/ ################## # Install TFLint # ################## +# Configure TFLint plugin folder +ENV TFLINT_PLUGIN_DIR="/root/.tflint.d/plugins" COPY --from=tflint /usr/local/bin/tflint /usr/bin/ +COPY --from=tflint-plugins "${TFLINT_PLUGIN_DIR}" "${TFLINT_PLUGIN_DIR}" ##################### # Install Terrascan # @@ -237,17 +328,6 @@ COPY --from=actionlint /usr/local/bin/actionlint /usr/bin/ ###################### COPY --from=kubeconfrm /kubeconform /usr/bin/ -################# -# Install glibc # -################# -# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub -# Store the key here because the above host is sometimes down, and breaks our builds -COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub -ARG GLIBC_VERSION='2.34-r0' -COPY scripts/install-glibc.sh / -RUN --mount=type=secret,id=GITHUB_TOKEN /install-glibc.sh \ - && rm -rf /install-glibc.sh /sgerrand.rsa.pub - ##################### # Install clj-kondo # ##################### @@ -265,74 +345,16 @@ RUN chmod 755 "${DART_SDK}" && chmod 755 "${DART_SDK}/bin" ######################## COPY --from=clang-format /usr/bin/clang-format /usr/bin/ -################# -# Install Lintr # -################# -COPY scripts/install-lintr.sh scripts/install-r-package-or-fail.R / -RUN /install-lintr.sh && rm -rf /install-lintr.sh /install-r-package-or-fail.R - -################################# -# Install luacheck and luarocks # -################################# -COPY scripts/install-lua.sh / -RUN --mount=type=secret,id=GITHUB_TOKEN /install-lua.sh && rm -rf /install-lua.sh - -##################################### -# Build python virtual environments # -##################################### -COPY dependencies/python/ /stage -WORKDIR /stage -RUN ./build-venvs.sh && rm -rfv /stage -# Set work directory back to root because some scripts depend on it -WORKDIR / - -############################## -# Install Phive dependencies # -############################## -COPY dependencies/phive.xml /phive.xml -COPY scripts/install-phive.sh / -RUN /install-phive.sh \ - && rm -rfv /install-phive.sh /phive.xml - -################## -# Install ktlint # -################## -COPY scripts/install-ktlint.sh / -COPY dependencies/ktlint /ktlint -RUN --mount=type=secret,id=GITHUB_TOKEN /install-ktlint.sh \ - && rm -rfv /install-ktlint.sh /ktlint - -###################### -# Install CheckStyle # -###################### -COPY scripts/install-checkstyle.sh / -COPY dependencies/checkstyle /checkstyle -RUN --mount=type=secret,id=GITHUB_TOKEN /install-checkstyle.sh \ - && rm -rfv /install-checkstyle.sh /checkstyle - -############################## -# Install google-java-format # -############################## -COPY scripts/install-google-java-format.sh / -COPY dependencies/google-java-format /google-java-format -RUN --mount=type=secret,id=GITHUB_TOKEN /install-google-java-format.sh \ - && rm -rfv /install-google-java-format.sh /google-java-format +######################## +# Install python tools # +######################## +COPY --from=python-builder /venvs /venvs ##################### # Install Bash-Exec # ##################### COPY --chmod=555 scripts/bash-exec.sh /usr/bin/bash-exec -################################# -# Copy super-linter executables # -################################# -COPY lib /action/lib - -################################### -# Copy linter configuration files # -################################### -COPY TEMPLATES /action/lib/.automation - ######################### # Configure Environment # ######################### @@ -357,16 +379,21 @@ ENV PATH="${PATH}:/node_modules/.bin" ENV PATH="${PATH}:/usr/lib/go/bin" ENV PATH="${PATH}:${DART_SDK}/bin:/root/.pub-cache/bin" -# Configure TFLint plugin folder -ENV TFLINT_PLUGIN_DIR="/root/.tflint.d/plugins" - -# Initialize TFLint plugins so we get plugin versions listed when we ask for TFLint version # Initialize Terrascan # Initialize ChkTeX config file -RUN tflint --init -c /action/lib/.automation/.tflint.hcl \ - && terrascan init \ +RUN terrascan init \ && touch ~/.chktexrc +################################### +# Copy linter configuration files # +################################### +COPY TEMPLATES /action/lib/.automation + +################################# +# Copy super-linter executables # +################################# +COPY lib /action/lib + # Run to build version file and validate image RUN ACTIONS_RUNNER_DEBUG=true WRITE_LINTER_VERSIONS_FILE=true IMAGE="${IMAGE}" /action/lib/linter.sh diff --git a/Makefile b/Makefile index 9fd1664f..55d00186 100644 --- a/Makefile +++ b/Makefile @@ -113,6 +113,7 @@ docker: check-github-token ## Build the container image --build-arg BUILD_REVISION=$(BUILD_REVISION) \ --build-arg BUILD_VERSION=$(BUILD_VERSION) \ --secret id=GITHUB_TOKEN,src=$(GITHUB_TOKEN_PATH) \ + --target $(IMAGE) \ -t $(SUPER_LINTER_TEST_CONTAINER_URL) . .phony: docker-pull diff --git a/test/inspec/super-linter/controls/super_linter.rb b/test/inspec/super-linter/controls/super_linter.rb index 25b0a09c..b06b4516 100644 --- a/test/inspec/super-linter/controls/super_linter.rb +++ b/test/inspec/super-linter/controls/super_linter.rb @@ -400,6 +400,9 @@ control "super-linter-validate-directories" do "/action/lib", "/action/lib/functions", "/action/lib/.automation", + "/root/.tflint.d/plugins/github.com/terraform-linters/tflint-ruleset-aws", + "/root/.tflint.d/plugins/github.com/terraform-linters/tflint-ruleset-azurerm", + "/root/.tflint.d/plugins/github.com/terraform-linters/tflint-ruleset-google", "/usr/local/lib/", "/usr/local/share/" ]