build: python venvs and npm in dedicated stages (#5078)

Move the following tasks to dedicated stages so we can run these steps
in parallel with other stages:

- Build Python virtual environments
- Install npm packages
- Build clang-format
- Download and install TFlint plugins

Add missing target stage when building the container image
This commit is contained in:
Marco Ferrari 2024-01-10 11:09:53 +01:00 committed by GitHub
parent 3fb1c3467c
commit df911171c4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 125 additions and 94 deletions

View file

@ -53,6 +53,48 @@ RUN cmake \
&& ninja clang-format \ && ninja clang-format \
&& mv /tmp/llvm-project/llvm/build/bin/clang-format /usr/bin && mv /tmp/llvm-project/llvm/build/bin/clang-format /usr/bin
FROM python:3.12.1-alpine3.19 as python-builder
RUN apk add --no-cache \
bash
SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-c"]
COPY dependencies/python/ /stage
WORKDIR /stage
RUN ./build-venvs.sh && rm -rfv /stage
FROM python:3.12.1-alpine3.19 as npm-builder
RUN apk add --no-cache \
bash \
nodejs-current
# The chown fixes broken uid/gid in ast-types-flow dependency
# (see https://github.com/super-linter/super-linter/issues/3901)
# Npm is not a runtime dependency but we need it to ensure that npm packages
# are installed when we run the test suite. If we decide to remove it, add
# the following command to the RUN instruction below:
# apk del --no-network --purge .node-build-deps
COPY dependencies/package.json dependencies/package-lock.json /
RUN apk add --no-cache --virtual .node-build-deps \
npm \
&& npm install \
&& npm cache clean --force \
&& chown -R "$(id -u)":"$(id -g)" node_modules \
&& rm -rfv package.json package-lock.json
FROM tflint as tflint-plugins
# Configure TFLint plugin folder
ENV TFLINT_PLUGIN_DIR="/root/.tflint.d/plugins"
# Copy TFlint configuration file because it contains plugin definitions
COPY TEMPLATES/.tflint.hcl /action/lib/.automation/
# Initialize TFLint plugins so we get plugin versions listed when we ask for TFLint version
RUN tflint --init -c /action/lib/.automation/.tflint.hcl
FROM python:3.12.1-alpine3.19 as base_image FROM python:3.12.1-alpine3.19 as base_image
LABEL com.github.actions.name="Super-Linter" \ LABEL com.github.actions.name="Super-Linter" \
@ -77,6 +119,8 @@ RUN apk add --no-cache \
SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-c"] SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-c"]
# Install super-linter runtime dependencies # Install super-linter runtime dependencies
# Npm is not a runtime dependency but we need it to ensure that npm packages
# are installed when we run the test suite.
RUN apk add --no-cache \ RUN apk add --no-cache \
ca-certificates \ ca-certificates \
coreutils \ coreutils \
@ -86,6 +130,7 @@ RUN apk add --no-cache \
git-lfs \ git-lfs \
jq \ jq \
libxml2-utils \ libxml2-utils \
npm \
nodejs-current \ nodejs-current \
openjdk17-jre \ openjdk17-jre \
openssh-client \ openssh-client \
@ -106,21 +151,6 @@ RUN apk add --no-cache \
ruby \ ruby \
zef zef
# Install Node tools
# The chown fixes broken uid/gid in ast-types-flow dependency
# (see https://github.com/super-linter/super-linter/issues/3901)
# Npm is not a runtime dependency but we need it to ensure that npm packages
# are installed when we run the test suite. If we decide to remove it, add
# the following command to the RUN instruction below:
# apk del --no-network --purge .node-build-deps
COPY dependencies/package.json dependencies/package-lock.json /
RUN apk add --no-cache --virtual .node-build-deps \
npm \
&& npm install \
&& npm cache clean --force \
&& chown -R "$(id -u)":"$(id -g)" node_modules \
&& rm -rfv package.json package-lock.json
# Install Ruby tools # Install Ruby tools
COPY dependencies/Gemfile dependencies/Gemfile.lock / COPY dependencies/Gemfile dependencies/Gemfile.lock /
RUN apk add --no-cache --virtual .ruby-build-deps \ RUN apk add --no-cache --virtual .ruby-build-deps \
@ -154,6 +184,17 @@ RUN apk add --no-cache --virtual .perl-build-deps \
Perl::Critic::Tics \ Perl::Critic::Tics \
&& apk del --no-network --purge .perl-build-deps && apk del --no-network --purge .perl-build-deps
#################
# Install glibc #
#################
# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub
# Store the key here because the above host is sometimes down, and breaks our builds
COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub
ARG GLIBC_VERSION='2.34-r0'
COPY scripts/install-glibc.sh /
RUN --mount=type=secret,id=GITHUB_TOKEN /install-glibc.sh \
&& rm -rf /install-glibc.sh /sgerrand.rsa.pub
################## ##################
# Install chktex # # Install chktex #
################## ##################
@ -162,6 +203,53 @@ RUN --mount=type=secret,id=GITHUB_TOKEN /install-chktex.sh && rm -rf /install-ch
# Set work directory back to root because some scripts depend on it # Set work directory back to root because some scripts depend on it
WORKDIR / WORKDIR /
#################
# Install Lintr #
#################
COPY scripts/install-lintr.sh scripts/install-r-package-or-fail.R /
RUN /install-lintr.sh && rm -rf /install-lintr.sh /install-r-package-or-fail.R
#################################
# Install luacheck and luarocks #
#################################
COPY scripts/install-lua.sh /
RUN --mount=type=secret,id=GITHUB_TOKEN /install-lua.sh && rm -rf /install-lua.sh
##############################
# Install Phive dependencies #
##############################
COPY dependencies/phive.xml /phive.xml
COPY scripts/install-phive.sh /
RUN /install-phive.sh \
&& rm -rfv /install-phive.sh /phive.xml
##################
# Install ktlint #
##################
COPY scripts/install-ktlint.sh /
COPY dependencies/ktlint /ktlint
RUN --mount=type=secret,id=GITHUB_TOKEN /install-ktlint.sh \
&& rm -rfv /install-ktlint.sh /ktlint
######################
# Install CheckStyle #
######################
COPY scripts/install-checkstyle.sh /
COPY dependencies/checkstyle /checkstyle
RUN --mount=type=secret,id=GITHUB_TOKEN /install-checkstyle.sh \
&& rm -rfv /install-checkstyle.sh /checkstyle
##############################
# Install google-java-format #
##############################
COPY scripts/install-google-java-format.sh /
COPY dependencies/google-java-format /google-java-format
RUN --mount=type=secret,id=GITHUB_TOKEN /install-google-java-format.sh \
&& rm -rfv /install-google-java-format.sh /google-java-format
# Copy Node tools
COPY --from=npm-builder /node_modules /node_modules
###################### ######################
# Install shellcheck # # Install shellcheck #
###################### ######################
@ -185,7 +273,10 @@ COPY --from=terraform /bin/terraform /usr/bin/
################## ##################
# Install TFLint # # Install TFLint #
################## ##################
# Configure TFLint plugin folder
ENV TFLINT_PLUGIN_DIR="/root/.tflint.d/plugins"
COPY --from=tflint /usr/local/bin/tflint /usr/bin/ COPY --from=tflint /usr/local/bin/tflint /usr/bin/
COPY --from=tflint-plugins "${TFLINT_PLUGIN_DIR}" "${TFLINT_PLUGIN_DIR}"
##################### #####################
# Install Terrascan # # Install Terrascan #
@ -237,17 +328,6 @@ COPY --from=actionlint /usr/local/bin/actionlint /usr/bin/
###################### ######################
COPY --from=kubeconfrm /kubeconform /usr/bin/ COPY --from=kubeconfrm /kubeconform /usr/bin/
#################
# Install glibc #
#################
# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub
# Store the key here because the above host is sometimes down, and breaks our builds
COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub
ARG GLIBC_VERSION='2.34-r0'
COPY scripts/install-glibc.sh /
RUN --mount=type=secret,id=GITHUB_TOKEN /install-glibc.sh \
&& rm -rf /install-glibc.sh /sgerrand.rsa.pub
##################### #####################
# Install clj-kondo # # Install clj-kondo #
##################### #####################
@ -265,74 +345,16 @@ RUN chmod 755 "${DART_SDK}" && chmod 755 "${DART_SDK}/bin"
######################## ########################
COPY --from=clang-format /usr/bin/clang-format /usr/bin/ COPY --from=clang-format /usr/bin/clang-format /usr/bin/
################# ########################
# Install Lintr # # Install python tools #
################# ########################
COPY scripts/install-lintr.sh scripts/install-r-package-or-fail.R / COPY --from=python-builder /venvs /venvs
RUN /install-lintr.sh && rm -rf /install-lintr.sh /install-r-package-or-fail.R
#################################
# Install luacheck and luarocks #
#################################
COPY scripts/install-lua.sh /
RUN --mount=type=secret,id=GITHUB_TOKEN /install-lua.sh && rm -rf /install-lua.sh
#####################################
# Build python virtual environments #
#####################################
COPY dependencies/python/ /stage
WORKDIR /stage
RUN ./build-venvs.sh && rm -rfv /stage
# Set work directory back to root because some scripts depend on it
WORKDIR /
##############################
# Install Phive dependencies #
##############################
COPY dependencies/phive.xml /phive.xml
COPY scripts/install-phive.sh /
RUN /install-phive.sh \
&& rm -rfv /install-phive.sh /phive.xml
##################
# Install ktlint #
##################
COPY scripts/install-ktlint.sh /
COPY dependencies/ktlint /ktlint
RUN --mount=type=secret,id=GITHUB_TOKEN /install-ktlint.sh \
&& rm -rfv /install-ktlint.sh /ktlint
######################
# Install CheckStyle #
######################
COPY scripts/install-checkstyle.sh /
COPY dependencies/checkstyle /checkstyle
RUN --mount=type=secret,id=GITHUB_TOKEN /install-checkstyle.sh \
&& rm -rfv /install-checkstyle.sh /checkstyle
##############################
# Install google-java-format #
##############################
COPY scripts/install-google-java-format.sh /
COPY dependencies/google-java-format /google-java-format
RUN --mount=type=secret,id=GITHUB_TOKEN /install-google-java-format.sh \
&& rm -rfv /install-google-java-format.sh /google-java-format
##################### #####################
# Install Bash-Exec # # Install Bash-Exec #
##################### #####################
COPY --chmod=555 scripts/bash-exec.sh /usr/bin/bash-exec COPY --chmod=555 scripts/bash-exec.sh /usr/bin/bash-exec
#################################
# Copy super-linter executables #
#################################
COPY lib /action/lib
###################################
# Copy linter configuration files #
###################################
COPY TEMPLATES /action/lib/.automation
######################### #########################
# Configure Environment # # Configure Environment #
######################### #########################
@ -357,16 +379,21 @@ ENV PATH="${PATH}:/node_modules/.bin"
ENV PATH="${PATH}:/usr/lib/go/bin" ENV PATH="${PATH}:/usr/lib/go/bin"
ENV PATH="${PATH}:${DART_SDK}/bin:/root/.pub-cache/bin" ENV PATH="${PATH}:${DART_SDK}/bin:/root/.pub-cache/bin"
# Configure TFLint plugin folder
ENV TFLINT_PLUGIN_DIR="/root/.tflint.d/plugins"
# Initialize TFLint plugins so we get plugin versions listed when we ask for TFLint version
# Initialize Terrascan # Initialize Terrascan
# Initialize ChkTeX config file # Initialize ChkTeX config file
RUN tflint --init -c /action/lib/.automation/.tflint.hcl \ RUN terrascan init \
&& terrascan init \
&& touch ~/.chktexrc && touch ~/.chktexrc
###################################
# Copy linter configuration files #
###################################
COPY TEMPLATES /action/lib/.automation
#################################
# Copy super-linter executables #
#################################
COPY lib /action/lib
# Run to build version file and validate image # Run to build version file and validate image
RUN ACTIONS_RUNNER_DEBUG=true WRITE_LINTER_VERSIONS_FILE=true IMAGE="${IMAGE}" /action/lib/linter.sh RUN ACTIONS_RUNNER_DEBUG=true WRITE_LINTER_VERSIONS_FILE=true IMAGE="${IMAGE}" /action/lib/linter.sh

View file

@ -113,6 +113,7 @@ docker: check-github-token ## Build the container image
--build-arg BUILD_REVISION=$(BUILD_REVISION) \ --build-arg BUILD_REVISION=$(BUILD_REVISION) \
--build-arg BUILD_VERSION=$(BUILD_VERSION) \ --build-arg BUILD_VERSION=$(BUILD_VERSION) \
--secret id=GITHUB_TOKEN,src=$(GITHUB_TOKEN_PATH) \ --secret id=GITHUB_TOKEN,src=$(GITHUB_TOKEN_PATH) \
--target $(IMAGE) \
-t $(SUPER_LINTER_TEST_CONTAINER_URL) . -t $(SUPER_LINTER_TEST_CONTAINER_URL) .
.phony: docker-pull .phony: docker-pull

View file

@ -400,6 +400,9 @@ control "super-linter-validate-directories" do
"/action/lib", "/action/lib",
"/action/lib/functions", "/action/lib/functions",
"/action/lib/.automation", "/action/lib/.automation",
"/root/.tflint.d/plugins/github.com/terraform-linters/tflint-ruleset-aws",
"/root/.tflint.d/plugins/github.com/terraform-linters/tflint-ruleset-azurerm",
"/root/.tflint.d/plugins/github.com/terraform-linters/tflint-ruleset-google",
"/usr/local/lib/", "/usr/local/lib/",
"/usr/local/share/" "/usr/local/share/"
] ]