mirror of
https://github.com/super-linter/super-linter.git
synced 2024-11-25 02:01:02 -05:00
build: python venvs and npm in dedicated stages (#5078)
Move the following tasks to dedicated stages so we can run these steps in parallel with other stages: - Build Python virtual environments - Install npm packages - Build clang-format - Download and install TFlint plugins Add missing target stage when building the container image
This commit is contained in:
parent
3fb1c3467c
commit
df911171c4
3 changed files with 125 additions and 94 deletions
215
Dockerfile
215
Dockerfile
|
@ -53,6 +53,48 @@ RUN cmake \
|
||||||
&& ninja clang-format \
|
&& ninja clang-format \
|
||||||
&& mv /tmp/llvm-project/llvm/build/bin/clang-format /usr/bin
|
&& mv /tmp/llvm-project/llvm/build/bin/clang-format /usr/bin
|
||||||
|
|
||||||
|
FROM python:3.12.1-alpine3.19 as python-builder
|
||||||
|
|
||||||
|
RUN apk add --no-cache \
|
||||||
|
bash
|
||||||
|
|
||||||
|
SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-c"]
|
||||||
|
|
||||||
|
COPY dependencies/python/ /stage
|
||||||
|
WORKDIR /stage
|
||||||
|
RUN ./build-venvs.sh && rm -rfv /stage
|
||||||
|
|
||||||
|
FROM python:3.12.1-alpine3.19 as npm-builder
|
||||||
|
|
||||||
|
RUN apk add --no-cache \
|
||||||
|
bash \
|
||||||
|
nodejs-current
|
||||||
|
|
||||||
|
# The chown fixes broken uid/gid in ast-types-flow dependency
|
||||||
|
# (see https://github.com/super-linter/super-linter/issues/3901)
|
||||||
|
# Npm is not a runtime dependency but we need it to ensure that npm packages
|
||||||
|
# are installed when we run the test suite. If we decide to remove it, add
|
||||||
|
# the following command to the RUN instruction below:
|
||||||
|
# apk del --no-network --purge .node-build-deps
|
||||||
|
COPY dependencies/package.json dependencies/package-lock.json /
|
||||||
|
RUN apk add --no-cache --virtual .node-build-deps \
|
||||||
|
npm \
|
||||||
|
&& npm install \
|
||||||
|
&& npm cache clean --force \
|
||||||
|
&& chown -R "$(id -u)":"$(id -g)" node_modules \
|
||||||
|
&& rm -rfv package.json package-lock.json
|
||||||
|
|
||||||
|
FROM tflint as tflint-plugins
|
||||||
|
|
||||||
|
# Configure TFLint plugin folder
|
||||||
|
ENV TFLINT_PLUGIN_DIR="/root/.tflint.d/plugins"
|
||||||
|
|
||||||
|
# Copy TFlint configuration file because it contains plugin definitions
|
||||||
|
COPY TEMPLATES/.tflint.hcl /action/lib/.automation/
|
||||||
|
|
||||||
|
# Initialize TFLint plugins so we get plugin versions listed when we ask for TFLint version
|
||||||
|
RUN tflint --init -c /action/lib/.automation/.tflint.hcl
|
||||||
|
|
||||||
FROM python:3.12.1-alpine3.19 as base_image
|
FROM python:3.12.1-alpine3.19 as base_image
|
||||||
|
|
||||||
LABEL com.github.actions.name="Super-Linter" \
|
LABEL com.github.actions.name="Super-Linter" \
|
||||||
|
@ -77,6 +119,8 @@ RUN apk add --no-cache \
|
||||||
SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-c"]
|
SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-c"]
|
||||||
|
|
||||||
# Install super-linter runtime dependencies
|
# Install super-linter runtime dependencies
|
||||||
|
# Npm is not a runtime dependency but we need it to ensure that npm packages
|
||||||
|
# are installed when we run the test suite.
|
||||||
RUN apk add --no-cache \
|
RUN apk add --no-cache \
|
||||||
ca-certificates \
|
ca-certificates \
|
||||||
coreutils \
|
coreutils \
|
||||||
|
@ -86,6 +130,7 @@ RUN apk add --no-cache \
|
||||||
git-lfs \
|
git-lfs \
|
||||||
jq \
|
jq \
|
||||||
libxml2-utils \
|
libxml2-utils \
|
||||||
|
npm \
|
||||||
nodejs-current \
|
nodejs-current \
|
||||||
openjdk17-jre \
|
openjdk17-jre \
|
||||||
openssh-client \
|
openssh-client \
|
||||||
|
@ -106,21 +151,6 @@ RUN apk add --no-cache \
|
||||||
ruby \
|
ruby \
|
||||||
zef
|
zef
|
||||||
|
|
||||||
# Install Node tools
|
|
||||||
# The chown fixes broken uid/gid in ast-types-flow dependency
|
|
||||||
# (see https://github.com/super-linter/super-linter/issues/3901)
|
|
||||||
# Npm is not a runtime dependency but we need it to ensure that npm packages
|
|
||||||
# are installed when we run the test suite. If we decide to remove it, add
|
|
||||||
# the following command to the RUN instruction below:
|
|
||||||
# apk del --no-network --purge .node-build-deps
|
|
||||||
COPY dependencies/package.json dependencies/package-lock.json /
|
|
||||||
RUN apk add --no-cache --virtual .node-build-deps \
|
|
||||||
npm \
|
|
||||||
&& npm install \
|
|
||||||
&& npm cache clean --force \
|
|
||||||
&& chown -R "$(id -u)":"$(id -g)" node_modules \
|
|
||||||
&& rm -rfv package.json package-lock.json
|
|
||||||
|
|
||||||
# Install Ruby tools
|
# Install Ruby tools
|
||||||
COPY dependencies/Gemfile dependencies/Gemfile.lock /
|
COPY dependencies/Gemfile dependencies/Gemfile.lock /
|
||||||
RUN apk add --no-cache --virtual .ruby-build-deps \
|
RUN apk add --no-cache --virtual .ruby-build-deps \
|
||||||
|
@ -154,6 +184,17 @@ RUN apk add --no-cache --virtual .perl-build-deps \
|
||||||
Perl::Critic::Tics \
|
Perl::Critic::Tics \
|
||||||
&& apk del --no-network --purge .perl-build-deps
|
&& apk del --no-network --purge .perl-build-deps
|
||||||
|
|
||||||
|
#################
|
||||||
|
# Install glibc #
|
||||||
|
#################
|
||||||
|
# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub
|
||||||
|
# Store the key here because the above host is sometimes down, and breaks our builds
|
||||||
|
COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub
|
||||||
|
ARG GLIBC_VERSION='2.34-r0'
|
||||||
|
COPY scripts/install-glibc.sh /
|
||||||
|
RUN --mount=type=secret,id=GITHUB_TOKEN /install-glibc.sh \
|
||||||
|
&& rm -rf /install-glibc.sh /sgerrand.rsa.pub
|
||||||
|
|
||||||
##################
|
##################
|
||||||
# Install chktex #
|
# Install chktex #
|
||||||
##################
|
##################
|
||||||
|
@ -162,6 +203,53 @@ RUN --mount=type=secret,id=GITHUB_TOKEN /install-chktex.sh && rm -rf /install-ch
|
||||||
# Set work directory back to root because some scripts depend on it
|
# Set work directory back to root because some scripts depend on it
|
||||||
WORKDIR /
|
WORKDIR /
|
||||||
|
|
||||||
|
#################
|
||||||
|
# Install Lintr #
|
||||||
|
#################
|
||||||
|
COPY scripts/install-lintr.sh scripts/install-r-package-or-fail.R /
|
||||||
|
RUN /install-lintr.sh && rm -rf /install-lintr.sh /install-r-package-or-fail.R
|
||||||
|
|
||||||
|
#################################
|
||||||
|
# Install luacheck and luarocks #
|
||||||
|
#################################
|
||||||
|
COPY scripts/install-lua.sh /
|
||||||
|
RUN --mount=type=secret,id=GITHUB_TOKEN /install-lua.sh && rm -rf /install-lua.sh
|
||||||
|
|
||||||
|
##############################
|
||||||
|
# Install Phive dependencies #
|
||||||
|
##############################
|
||||||
|
COPY dependencies/phive.xml /phive.xml
|
||||||
|
COPY scripts/install-phive.sh /
|
||||||
|
RUN /install-phive.sh \
|
||||||
|
&& rm -rfv /install-phive.sh /phive.xml
|
||||||
|
|
||||||
|
##################
|
||||||
|
# Install ktlint #
|
||||||
|
##################
|
||||||
|
COPY scripts/install-ktlint.sh /
|
||||||
|
COPY dependencies/ktlint /ktlint
|
||||||
|
RUN --mount=type=secret,id=GITHUB_TOKEN /install-ktlint.sh \
|
||||||
|
&& rm -rfv /install-ktlint.sh /ktlint
|
||||||
|
|
||||||
|
######################
|
||||||
|
# Install CheckStyle #
|
||||||
|
######################
|
||||||
|
COPY scripts/install-checkstyle.sh /
|
||||||
|
COPY dependencies/checkstyle /checkstyle
|
||||||
|
RUN --mount=type=secret,id=GITHUB_TOKEN /install-checkstyle.sh \
|
||||||
|
&& rm -rfv /install-checkstyle.sh /checkstyle
|
||||||
|
|
||||||
|
##############################
|
||||||
|
# Install google-java-format #
|
||||||
|
##############################
|
||||||
|
COPY scripts/install-google-java-format.sh /
|
||||||
|
COPY dependencies/google-java-format /google-java-format
|
||||||
|
RUN --mount=type=secret,id=GITHUB_TOKEN /install-google-java-format.sh \
|
||||||
|
&& rm -rfv /install-google-java-format.sh /google-java-format
|
||||||
|
|
||||||
|
# Copy Node tools
|
||||||
|
COPY --from=npm-builder /node_modules /node_modules
|
||||||
|
|
||||||
######################
|
######################
|
||||||
# Install shellcheck #
|
# Install shellcheck #
|
||||||
######################
|
######################
|
||||||
|
@ -185,7 +273,10 @@ COPY --from=terraform /bin/terraform /usr/bin/
|
||||||
##################
|
##################
|
||||||
# Install TFLint #
|
# Install TFLint #
|
||||||
##################
|
##################
|
||||||
|
# Configure TFLint plugin folder
|
||||||
|
ENV TFLINT_PLUGIN_DIR="/root/.tflint.d/plugins"
|
||||||
COPY --from=tflint /usr/local/bin/tflint /usr/bin/
|
COPY --from=tflint /usr/local/bin/tflint /usr/bin/
|
||||||
|
COPY --from=tflint-plugins "${TFLINT_PLUGIN_DIR}" "${TFLINT_PLUGIN_DIR}"
|
||||||
|
|
||||||
#####################
|
#####################
|
||||||
# Install Terrascan #
|
# Install Terrascan #
|
||||||
|
@ -237,17 +328,6 @@ COPY --from=actionlint /usr/local/bin/actionlint /usr/bin/
|
||||||
######################
|
######################
|
||||||
COPY --from=kubeconfrm /kubeconform /usr/bin/
|
COPY --from=kubeconfrm /kubeconform /usr/bin/
|
||||||
|
|
||||||
#################
|
|
||||||
# Install glibc #
|
|
||||||
#################
|
|
||||||
# Source: https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub
|
|
||||||
# Store the key here because the above host is sometimes down, and breaks our builds
|
|
||||||
COPY dependencies/sgerrand.rsa.pub /etc/apk/keys/sgerrand.rsa.pub
|
|
||||||
ARG GLIBC_VERSION='2.34-r0'
|
|
||||||
COPY scripts/install-glibc.sh /
|
|
||||||
RUN --mount=type=secret,id=GITHUB_TOKEN /install-glibc.sh \
|
|
||||||
&& rm -rf /install-glibc.sh /sgerrand.rsa.pub
|
|
||||||
|
|
||||||
#####################
|
#####################
|
||||||
# Install clj-kondo #
|
# Install clj-kondo #
|
||||||
#####################
|
#####################
|
||||||
|
@ -265,74 +345,16 @@ RUN chmod 755 "${DART_SDK}" && chmod 755 "${DART_SDK}/bin"
|
||||||
########################
|
########################
|
||||||
COPY --from=clang-format /usr/bin/clang-format /usr/bin/
|
COPY --from=clang-format /usr/bin/clang-format /usr/bin/
|
||||||
|
|
||||||
#################
|
########################
|
||||||
# Install Lintr #
|
# Install python tools #
|
||||||
#################
|
########################
|
||||||
COPY scripts/install-lintr.sh scripts/install-r-package-or-fail.R /
|
COPY --from=python-builder /venvs /venvs
|
||||||
RUN /install-lintr.sh && rm -rf /install-lintr.sh /install-r-package-or-fail.R
|
|
||||||
|
|
||||||
#################################
|
|
||||||
# Install luacheck and luarocks #
|
|
||||||
#################################
|
|
||||||
COPY scripts/install-lua.sh /
|
|
||||||
RUN --mount=type=secret,id=GITHUB_TOKEN /install-lua.sh && rm -rf /install-lua.sh
|
|
||||||
|
|
||||||
#####################################
|
|
||||||
# Build python virtual environments #
|
|
||||||
#####################################
|
|
||||||
COPY dependencies/python/ /stage
|
|
||||||
WORKDIR /stage
|
|
||||||
RUN ./build-venvs.sh && rm -rfv /stage
|
|
||||||
# Set work directory back to root because some scripts depend on it
|
|
||||||
WORKDIR /
|
|
||||||
|
|
||||||
##############################
|
|
||||||
# Install Phive dependencies #
|
|
||||||
##############################
|
|
||||||
COPY dependencies/phive.xml /phive.xml
|
|
||||||
COPY scripts/install-phive.sh /
|
|
||||||
RUN /install-phive.sh \
|
|
||||||
&& rm -rfv /install-phive.sh /phive.xml
|
|
||||||
|
|
||||||
##################
|
|
||||||
# Install ktlint #
|
|
||||||
##################
|
|
||||||
COPY scripts/install-ktlint.sh /
|
|
||||||
COPY dependencies/ktlint /ktlint
|
|
||||||
RUN --mount=type=secret,id=GITHUB_TOKEN /install-ktlint.sh \
|
|
||||||
&& rm -rfv /install-ktlint.sh /ktlint
|
|
||||||
|
|
||||||
######################
|
|
||||||
# Install CheckStyle #
|
|
||||||
######################
|
|
||||||
COPY scripts/install-checkstyle.sh /
|
|
||||||
COPY dependencies/checkstyle /checkstyle
|
|
||||||
RUN --mount=type=secret,id=GITHUB_TOKEN /install-checkstyle.sh \
|
|
||||||
&& rm -rfv /install-checkstyle.sh /checkstyle
|
|
||||||
|
|
||||||
##############################
|
|
||||||
# Install google-java-format #
|
|
||||||
##############################
|
|
||||||
COPY scripts/install-google-java-format.sh /
|
|
||||||
COPY dependencies/google-java-format /google-java-format
|
|
||||||
RUN --mount=type=secret,id=GITHUB_TOKEN /install-google-java-format.sh \
|
|
||||||
&& rm -rfv /install-google-java-format.sh /google-java-format
|
|
||||||
|
|
||||||
#####################
|
#####################
|
||||||
# Install Bash-Exec #
|
# Install Bash-Exec #
|
||||||
#####################
|
#####################
|
||||||
COPY --chmod=555 scripts/bash-exec.sh /usr/bin/bash-exec
|
COPY --chmod=555 scripts/bash-exec.sh /usr/bin/bash-exec
|
||||||
|
|
||||||
#################################
|
|
||||||
# Copy super-linter executables #
|
|
||||||
#################################
|
|
||||||
COPY lib /action/lib
|
|
||||||
|
|
||||||
###################################
|
|
||||||
# Copy linter configuration files #
|
|
||||||
###################################
|
|
||||||
COPY TEMPLATES /action/lib/.automation
|
|
||||||
|
|
||||||
#########################
|
#########################
|
||||||
# Configure Environment #
|
# Configure Environment #
|
||||||
#########################
|
#########################
|
||||||
|
@ -357,16 +379,21 @@ ENV PATH="${PATH}:/node_modules/.bin"
|
||||||
ENV PATH="${PATH}:/usr/lib/go/bin"
|
ENV PATH="${PATH}:/usr/lib/go/bin"
|
||||||
ENV PATH="${PATH}:${DART_SDK}/bin:/root/.pub-cache/bin"
|
ENV PATH="${PATH}:${DART_SDK}/bin:/root/.pub-cache/bin"
|
||||||
|
|
||||||
# Configure TFLint plugin folder
|
|
||||||
ENV TFLINT_PLUGIN_DIR="/root/.tflint.d/plugins"
|
|
||||||
|
|
||||||
# Initialize TFLint plugins so we get plugin versions listed when we ask for TFLint version
|
|
||||||
# Initialize Terrascan
|
# Initialize Terrascan
|
||||||
# Initialize ChkTeX config file
|
# Initialize ChkTeX config file
|
||||||
RUN tflint --init -c /action/lib/.automation/.tflint.hcl \
|
RUN terrascan init \
|
||||||
&& terrascan init \
|
|
||||||
&& touch ~/.chktexrc
|
&& touch ~/.chktexrc
|
||||||
|
|
||||||
|
###################################
|
||||||
|
# Copy linter configuration files #
|
||||||
|
###################################
|
||||||
|
COPY TEMPLATES /action/lib/.automation
|
||||||
|
|
||||||
|
#################################
|
||||||
|
# Copy super-linter executables #
|
||||||
|
#################################
|
||||||
|
COPY lib /action/lib
|
||||||
|
|
||||||
# Run to build version file and validate image
|
# Run to build version file and validate image
|
||||||
RUN ACTIONS_RUNNER_DEBUG=true WRITE_LINTER_VERSIONS_FILE=true IMAGE="${IMAGE}" /action/lib/linter.sh
|
RUN ACTIONS_RUNNER_DEBUG=true WRITE_LINTER_VERSIONS_FILE=true IMAGE="${IMAGE}" /action/lib/linter.sh
|
||||||
|
|
||||||
|
|
1
Makefile
1
Makefile
|
@ -113,6 +113,7 @@ docker: check-github-token ## Build the container image
|
||||||
--build-arg BUILD_REVISION=$(BUILD_REVISION) \
|
--build-arg BUILD_REVISION=$(BUILD_REVISION) \
|
||||||
--build-arg BUILD_VERSION=$(BUILD_VERSION) \
|
--build-arg BUILD_VERSION=$(BUILD_VERSION) \
|
||||||
--secret id=GITHUB_TOKEN,src=$(GITHUB_TOKEN_PATH) \
|
--secret id=GITHUB_TOKEN,src=$(GITHUB_TOKEN_PATH) \
|
||||||
|
--target $(IMAGE) \
|
||||||
-t $(SUPER_LINTER_TEST_CONTAINER_URL) .
|
-t $(SUPER_LINTER_TEST_CONTAINER_URL) .
|
||||||
|
|
||||||
.phony: docker-pull
|
.phony: docker-pull
|
||||||
|
|
|
@ -400,6 +400,9 @@ control "super-linter-validate-directories" do
|
||||||
"/action/lib",
|
"/action/lib",
|
||||||
"/action/lib/functions",
|
"/action/lib/functions",
|
||||||
"/action/lib/.automation",
|
"/action/lib/.automation",
|
||||||
|
"/root/.tflint.d/plugins/github.com/terraform-linters/tflint-ruleset-aws",
|
||||||
|
"/root/.tflint.d/plugins/github.com/terraform-linters/tflint-ruleset-azurerm",
|
||||||
|
"/root/.tflint.d/plugins/github.com/terraform-linters/tflint-ruleset-google",
|
||||||
"/usr/local/lib/",
|
"/usr/local/lib/",
|
||||||
"/usr/local/share/"
|
"/usr/local/share/"
|
||||||
]
|
]
|
||||||
|
|
Loading…
Reference in a new issue