mirror of
https://github.com/super-linter/super-linter.git
synced 2024-09-13 07:57:10 -04:00
build: python venvs and npm in dedicated stages (#5078)
Move the following tasks to dedicated stages so we can run these steps in parallel with other stages: - Build Python virtual environments - Install npm packages - Build clang-format - Download and install TFlint plugins Add missing target stage when building the container image
This commit is contained in:
Marco Ferrari
GitHub
parent
3fb1c3467c
commit
df911171c4
3 changed files with 125 additions and 94 deletions
215
Dockerfile
215
Dockerfile
|
|
@ -53,6 +53,48 @@ RUN cmake \
|
|||
&& ninja clang-format \
|
||||
&& mv /tmp/llvm-project/llvm/build/bin/clang-format /usr/bin
|
||||
|
||||
FROM python:3.12.1-alpine3.19 as python-builder
|
||||
|
||||
RUN apk add --no-cache \
|
||||
bash
|
||||
|
||||
SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-c"]
|
||||
|
||||
COPY dependencies/python/ /stage
|
||||
WORKDIR /stage
|
||||
RUN ./build-venvs.sh && rm -rfv /stage
|
||||
|
||||
FROM python:3.12.1-alpine3.19 as npm-builder
|
||||
|
||||
RUN apk add --no-cache \
|
||||
bash \
|
||||
nodejs-current
|
||||
|
||||
# The chown fixes broken uid/gid in ast-types-flow dependency
|
||||
# (see https://github.com/super-linter/super-linter/issues/3901)
|
||||
# Npm is not a runtime dependency but we need it to ensure that npm packages
|
||||
# are installed when we run the test suite. If we decide to remove it, add
|
||||
# the following command to the RUN instruction below:
|
||||
# apk del --no-network --purge .node-build-deps
|
||||
COPY dependencies/package.json dependencies/package-lock.json /
|
||||
RUN apk add --no-cache --virtual .node-build-deps \
|
||||
npm \
|
||||
&& npm install \
|
||||
&& npm cache clean --force \
|
||||
&& chown -R "$(id -u)":"$(id -g)" node_modules \
|
||||
&& rm -rfv package.json package-lock.json
|
||||
|
||||
FROM tflint as tflint-plugins
|
||||
|
||||
# Configure TFLint plugin folder
|
||||
ENV TFLINT_PLUGIN_DIR="/root/.tflint.d/plugins"
|
||||
|
||||
# Copy TFlint configuration file because it contains plugin definitions
|
||||
COPY TEMPLATES/.tflint.hcl /action/lib/.automation/
|
||||
|
||||
# Initialize TFLint plugins so we get plugin versions listed when we ask for TFLint version
|
||||
RUN tflint --init -c /action/lib/.automation/.tflint.hcl
|
||||
|
||||
FROM python:3.12.1-alpine3.19 as base_image
|
||||
|
||||
LABEL com.github.actions.name="Super-Linter" \
|
||||
|
|
@ -77,6 +119,8 @@ RUN apk add --no-cache \
|
|||
SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-c"]
|
||||
|
||||
# Install super-linter runtime dependencies
|
||||
# Npm is not a runtime dependency but we need it to ensure that npm packages
|
||||
# are installed when we run the test suite.
|
||||
RUN apk add --no-cache \
|
||||
ca-certificates \
|
||||
coreutils \
|
||||
|
|
@ -86,6 +130,7 @@ RUN apk add --no-cache \
|
|||
git-lfs \
|
||||
jq \
|
||||
libxml2-utils \
|
||||
npm \
|
||||
nodejs-current \
|
||||
openjdk17-jre \
|
||||
openssh-client \
|
||||
|
|
@ -106,21 +151,6 @@ RUN apk add --no-cache \
|
|||
ruby \
|
||||
zef
|
||||
|
||||
# Install Node tools
|
||||
# The chown fixes broken uid/gid in ast-types-flow dependency
|
||||
# (see https://github.com/super-linter/super-linter/issues/3901)
|
||||
# Npm is not a runtime dependency but we need it to ensure that npm packages
|
||||
# are installed when we run the test suite. If we decide to remove it, add
|
||||
# the following command to the RUN instruction below:
|
||||
# apk del --no-network --purge .node-build-deps
|
||||
COPY dependencies/package.json dependencies/package-lock.json /
|
||||
RUN apk add --no-cache --virtual .node-build-deps \
|
||||
npm \
|
||||
&& npm install \
|
||||
&& npm cache clean --force \
|
||||
&& chown -R "$(id -u)":"$(id -g)" node_modules \
|
||||
&& rm -rfv package.json package-lock.json
|
||||
|
||||
# Install Ruby tools
|
||||
COPY dependencies/Gemfile dependencies/Gemfile.lock /
|
||||
RUN apk add --no-cache --virtual .ruby-build-deps \
|
||||
|
|
@ -154,6 +184,17 @@ RUN apk add --no-cache --virtual .perl-build-deps \
|
|||
Perl::Critic::Tics \
|
||||
&& apk del --no-network --purge .perl-build-deps
|
||||
|
||||
#################
|
||||
# Install glibc #
|
||||
#################
|
||||
# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub
|
||||
# Store the key here because the above host is sometimes down, and breaks our builds
|
||||
COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub
|
||||
ARG GLIBC_VERSION='2.34-r0'
|
||||
COPY scripts/install-glibc.sh /
|
||||
RUN --mount=type=secret,id=GITHUB_TOKEN /install-glibc.sh \
|
||||
&& rm -rf /install-glibc.sh /sgerrand.rsa.pub
|
||||
|
||||
##################
|
||||
# Install chktex #
|
||||
##################
|
||||
|
|
@ -162,6 +203,53 @@ RUN --mount=type=secret,id=GITHUB_TOKEN /install-chktex.sh && rm -rf /install-ch
|
|||
# Set work directory back to root because some scripts depend on it
|
||||
WORKDIR /
|
||||
|
||||
#################
|
||||
# Install Lintr #
|
||||
#################
|
||||
COPY scripts/install-lintr.sh scripts/install-r-package-or-fail.R /
|
||||
RUN /install-lintr.sh && rm -rf /install-lintr.sh /install-r-package-or-fail.R
|
||||
|
||||
#################################
|
||||
# Install luacheck and luarocks #
|
||||
#################################
|
||||
COPY scripts/install-lua.sh /
|
||||
RUN --mount=type=secret,id=GITHUB_TOKEN /install-lua.sh && rm -rf /install-lua.sh
|
||||
|
||||
##############################
|
||||
# Install Phive dependencies #
|
||||
##############################
|
||||
COPY dependencies/phive.xml /phive.xml
|
||||
COPY scripts/install-phive.sh /
|
||||
RUN /install-phive.sh \
|
||||
&& rm -rfv /install-phive.sh /phive.xml
|
||||
|
||||
##################
|
||||
# Install ktlint #
|
||||
##################
|
||||
COPY scripts/install-ktlint.sh /
|
||||
COPY dependencies/ktlint /ktlint
|
||||
RUN --mount=type=secret,id=GITHUB_TOKEN /install-ktlint.sh \
|
||||
&& rm -rfv /install-ktlint.sh /ktlint
|
||||
|
||||
######################
|
||||
# Install CheckStyle #
|
||||
######################
|
||||
COPY scripts/install-checkstyle.sh /
|
||||
COPY dependencies/checkstyle /checkstyle
|
||||
RUN --mount=type=secret,id=GITHUB_TOKEN /install-checkstyle.sh \
|
||||
&& rm -rfv /install-checkstyle.sh /checkstyle
|
||||
|
||||
##############################
|
||||
# Install google-java-format #
|
||||
##############################
|
||||
COPY scripts/install-google-java-format.sh /
|
||||
COPY dependencies/google-java-format /google-java-format
|
||||
RUN --mount=type=secret,id=GITHUB_TOKEN /install-google-java-format.sh \
|
||||
&& rm -rfv /install-google-java-format.sh /google-java-format
|
||||
|
||||
# Copy Node tools
|
||||
COPY --from=npm-builder /node_modules /node_modules
|
||||
|
||||
######################
|
||||
# Install shellcheck #
|
||||
######################
|
||||
|
|
@ -185,7 +273,10 @@ COPY --from=terraform /bin/terraform /usr/bin/
|
|||
##################
|
||||
# Install TFLint #
|
||||
##################
|
||||
# Configure TFLint plugin folder
|
||||
ENV TFLINT_PLUGIN_DIR="/root/.tflint.d/plugins"
|
||||
COPY --from=tflint /usr/local/bin/tflint /usr/bin/
|
||||
COPY --from=tflint-plugins "${TFLINT_PLUGIN_DIR}" "${TFLINT_PLUGIN_DIR}"
|
||||
|
||||
#####################
|
||||
# Install Terrascan #
|
||||
|
|
@ -237,17 +328,6 @@ COPY --from=actionlint /usr/local/bin/actionlint /usr/bin/
|
|||
######################
|
||||
COPY --from=kubeconfrm /kubeconform /usr/bin/
|
||||
|
||||
#################
|
||||
# Install glibc #
|
||||
#################
|
||||
# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub
|
||||
# Store the key here because the above host is sometimes down, and breaks our builds
|
||||
COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub
|
||||
ARG GLIBC_VERSION='2.34-r0'
|
||||
COPY scripts/install-glibc.sh /
|
||||
RUN --mount=type=secret,id=GITHUB_TOKEN /install-glibc.sh \
|
||||
&& rm -rf /install-glibc.sh /sgerrand.rsa.pub
|
||||
|
||||
#####################
|
||||
# Install clj-kondo #
|
||||
#####################
|
||||
|
|
@ -265,74 +345,16 @@ RUN chmod 755 "${DART_SDK}" && chmod 755 "${DART_SDK}/bin"
|
|||
########################
|
||||
COPY --from=clang-format /usr/bin/clang-format /usr/bin/
|
||||
|
||||
#################
|
||||
# Install Lintr #
|
||||
#################
|
||||
COPY scripts/install-lintr.sh scripts/install-r-package-or-fail.R /
|
||||
RUN /install-lintr.sh && rm -rf /install-lintr.sh /install-r-package-or-fail.R
|
||||
|
||||
#################################
|
||||
# Install luacheck and luarocks #
|
||||
#################################
|
||||
COPY scripts/install-lua.sh /
|
||||
RUN --mount=type=secret,id=GITHUB_TOKEN /install-lua.sh && rm -rf /install-lua.sh
|
||||
|
||||
#####################################
|
||||
# Build python virtual environments #
|
||||
#####################################
|
||||
COPY dependencies/python/ /stage
|
||||
WORKDIR /stage
|
||||
RUN ./build-venvs.sh && rm -rfv /stage
|
||||
# Set work directory back to root because some scripts depend on it
|
||||
WORKDIR /
|
||||
|
||||
##############################
|
||||
# Install Phive dependencies #
|
||||
##############################
|
||||
COPY dependencies/phive.xml /phive.xml
|
||||
COPY scripts/install-phive.sh /
|
||||
RUN /install-phive.sh \
|
||||
&& rm -rfv /install-phive.sh /phive.xml
|
||||
|
||||
##################
|
||||
# Install ktlint #
|
||||
##################
|
||||
COPY scripts/install-ktlint.sh /
|
||||
COPY dependencies/ktlint /ktlint
|
||||
RUN --mount=type=secret,id=GITHUB_TOKEN /install-ktlint.sh \
|
||||
&& rm -rfv /install-ktlint.sh /ktlint
|
||||
|
||||
######################
|
||||
# Install CheckStyle #
|
||||
######################
|
||||
COPY scripts/install-checkstyle.sh /
|
||||
COPY dependencies/checkstyle /checkstyle
|
||||
RUN --mount=type=secret,id=GITHUB_TOKEN /install-checkstyle.sh \
|
||||
&& rm -rfv /install-checkstyle.sh /checkstyle
|
||||
|
||||
##############################
|
||||
# Install google-java-format #
|
||||
##############################
|
||||
COPY scripts/install-google-java-format.sh /
|
||||
COPY dependencies/google-java-format /google-java-format
|
||||
RUN --mount=type=secret,id=GITHUB_TOKEN /install-google-java-format.sh \
|
||||
&& rm -rfv /install-google-java-format.sh /google-java-format
|
||||
########################
|
||||
# Install python tools #
|
||||
########################
|
||||
COPY --from=python-builder /venvs /venvs
|
||||
|
||||
#####################
|
||||
# Install Bash-Exec #
|
||||
#####################
|
||||
COPY --chmod=555 scripts/bash-exec.sh /usr/bin/bash-exec
|
||||
|
||||
#################################
|
||||
# Copy super-linter executables #
|
||||
#################################
|
||||
COPY lib /action/lib
|
||||
|
||||
###################################
|
||||
# Copy linter configuration files #
|
||||
###################################
|
||||
COPY TEMPLATES /action/lib/.automation
|
||||
|
||||
#########################
|
||||
# Configure Environment #
|
||||
#########################
|
||||
|
|
@ -357,16 +379,21 @@ ENV PATH="${PATH}:/node_modules/.bin"
|
|||
ENV PATH="${PATH}:/usr/lib/go/bin"
|
||||
ENV PATH="${PATH}:${DART_SDK}/bin:/root/.pub-cache/bin"
|
||||
|
||||
# Configure TFLint plugin folder
|
||||
ENV TFLINT_PLUGIN_DIR="/root/.tflint.d/plugins"
|
||||
|
||||
# Initialize TFLint plugins so we get plugin versions listed when we ask for TFLint version
|
||||
# Initialize Terrascan
|
||||
# Initialize ChkTeX config file
|
||||
RUN tflint --init -c /action/lib/.automation/.tflint.hcl \
|
||||
&& terrascan init \
|
||||
RUN terrascan init \
|
||||
&& touch ~/.chktexrc
|
||||
|
||||
###################################
|
||||
# Copy linter configuration files #
|
||||
###################################
|
||||
COPY TEMPLATES /action/lib/.automation
|
||||
|
||||
#################################
|
||||
# Copy super-linter executables #
|
||||
#################################
|
||||
COPY lib /action/lib
|
||||
|
||||
# Run to build version file and validate image
|
||||
RUN ACTIONS_RUNNER_DEBUG=true WRITE_LINTER_VERSIONS_FILE=true IMAGE="${IMAGE}" /action/lib/linter.sh
|
||||
|
||||
|
|
|
|||
1
Makefile
1
Makefile
|
|
@ -113,6 +113,7 @@ docker: check-github-token ## Build the container image
|
|||
--build-arg BUILD_REVISION=$(BUILD_REVISION) \
|
||||
--build-arg BUILD_VERSION=$(BUILD_VERSION) \
|
||||
--secret id=GITHUB_TOKEN,src=$(GITHUB_TOKEN_PATH) \
|
||||
--target $(IMAGE) \
|
||||
-t $(SUPER_LINTER_TEST_CONTAINER_URL) .
|
||||
|
||||
.phony: docker-pull
|
||||
|
|
|
|||
|
|
@ -400,6 +400,9 @@ control "super-linter-validate-directories" do
|
|||
"/action/lib",
|
||||
"/action/lib/functions",
|
||||
"/action/lib/.automation",
|
||||
"/root/.tflint.d/plugins/github.com/terraform-linters/tflint-ruleset-aws",
|
||||
"/root/.tflint.d/plugins/github.com/terraform-linters/tflint-ruleset-azurerm",
|
||||
"/root/.tflint.d/plugins/github.com/terraform-linters/tflint-ruleset-google",
|
||||
"/usr/local/lib/",
|
||||
"/usr/local/share/"
|
||||
]
|
||||
|
|
|
|||
Loading…
Reference in a new issue