From b94bec19c922fd19386dd65650b7dd88e70f7fc1 Mon Sep 17 00:00:00 2001 From: Zack Koppert Date: Thu, 22 Apr 2021 12:38:05 -0700 Subject: [PATCH] multistage build changes (#1461) * format lines * minimize layers by using `&& \` * stages * version * Fix multiple consecutive `RUN` instructions * adding them lins * make linter happy * slowly but shurly... * adding more * its alive now * cleaner * bad commit * cleaner * fix vars * fix env * cleaner Co-authored-by: Admiral Awkbar --- .../test/arm/reports/expected-ARM-2.tap | 7 + .gitignore | 1 + Dockerfile | 312 ++++++++++-------- 3 files changed, 175 insertions(+), 145 deletions(-) create mode 100644 .automation/test/arm/reports/expected-ARM-2.tap diff --git a/.automation/test/arm/reports/expected-ARM-2.tap b/.automation/test/arm/reports/expected-ARM-2.tap new file mode 100644 index 00000000..e83754aa --- /dev/null +++ b/.automation/test/arm/reports/expected-ARM-2.tap @@ -0,0 +1,7 @@ +TAP version 13 +1..2 +not ok 1 - arm_bad_1.json + --- + message: \nValidating arm\arm_bad_1.json\n deploymentTemplate\n [+] adminUsername Should Not Be A Literal\n [+] apiVersions Should Be Recent\n [+] artifacts parameter\n [+] CommandToExecute Must Use ProtectedSettings For Secrets\n [+] DependsOn Best Practices\n [+] Deployment Resources Must Not Be Debug\n [+] DeploymentTemplate Must Not Contain Hardcoded Uri\n [+] DeploymentTemplate Schema Is Correct\n [+] Dynamic Variable References Should Not Use Concat\n [+] IDs Should Be Derived From ResourceIDs\n [+] Location Should Not Be Hardcoded\n [+] ManagedIdentityExtension must not be used\n [+] Min And Max Value Are Numbers\n [+] Outputs Must Not Contain Secrets\n [+] Parameters Must Be Referenced\n [+] providers apiVersions Is Not Permitted\n [+] ResourceIds should not contain\n [+] Resources Should Have Location\n [+] Resources Should Not Be Ambiguous\n [+] Secure String Parameters Cannot Have Default\n [-] Template Should Not Contain Blanks\n Empty property [] found on line 28 Index 1331\n Empty property "" found on line 3 Index 121\n\n [-] Variables Must Be Referenced\n Unreferenced variable Network.Location\n Unreferenced variable Test.Blank.Variable\n\n [+] Virtual Machines Should Not Be Preview\n [+] VM Images Should Use Latest Version\n [+] VM Size Should Be A Parameter\n \n + ... +ok 2 - arm_good_1.json diff --git a/.gitignore b/.gitignore index 06b66476..546c1d83 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ lib-cov # Coverage directory used by tools like istanbul coverage +test/.coverage/ # nyc test coverage .nyc_output diff --git a/Dockerfile b/Dockerfile index ab411b2b..0d156502 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,39 +25,7 @@ FROM garethr/kubeval:0.15.0 as kubeval ################## # Get base image # ################## -FROM python:3.9-alpine - -############################ -# Get the build arguements # -############################ -ARG BUILD_DATE -ARG BUILD_REVISION -ARG BUILD_VERSION - -######################################### -# Label the instance and set maintainer # -######################################### -LABEL com.github.actions.name="GitHub Super-Linter" \ - com.github.actions.description="Lint your code base with GitHub Actions" \ - com.github.actions.icon="code" \ - com.github.actions.color="red" \ - maintainer="GitHub DevOps " \ - org.opencontainers.image.created=$BUILD_DATE \ - org.opencontainers.image.revision=$BUILD_REVISION \ - org.opencontainers.image.version=$BUILD_VERSION \ - org.opencontainers.image.authors="GitHub DevOps " \ - org.opencontainers.image.url="https://github.com/github/super-linter" \ - org.opencontainers.image.source="https://github.com/github/super-linter" \ - org.opencontainers.image.documentation="https://github.com/github/super-linter" \ - org.opencontainers.image.vendor="GitHub" \ - org.opencontainers.image.description="Lint your code base with GitHub Actions" - -################################################# -# Set ENV values used for debugging the version # -################################################# -ENV BUILD_DATE=$BUILD_DATE -ENV BUILD_REVISION=$BUILD_REVISION -ENV BUILD_VERSION=$BUILD_VERSION +FROM python:3.9-alpine as base_image ################################ # Set ARG values used in Build # @@ -102,8 +70,6 @@ RUN apk add --no-cache \ openjdk8-jre \ openssl-dev \ perl perl-dev \ - php7 php7-phar php7-json php7-mbstring php-xmlwriter \ - php7-tokenizer php7-ctype php7-curl php7-dom php7-simplexml \ py3-setuptools python3-dev\ R R-dev R-doc \ readline-dev \ @@ -140,67 +106,40 @@ COPY dependencies/* / ################################ # Installs python dependencies # ################################ -RUN pip3 install --no-cache-dir pipenv -# Bug in hadolint thinks pipenv is pip -# hadolint ignore=DL3042 -RUN pipenv install --clear --system - +RUN pip3 install --no-cache-dir pipenv \ + # Bug in hadolint thinks pipenv is pip + # hadolint ignore=DL3042 + && pipenv install --clear --system \ #################### # Run NPM Installs # #################### -RUN npm config set package-lock false \ + && npm config set package-lock false \ && npm config set loglevel error \ && npm --no-cache install \ - && npm audit fix - -############################# -# Add node packages to path # -############################# -ENV PATH="/node_modules/.bin:${PATH}" - + && npm audit fix \ ############################## # Installs ruby dependencies # ############################## -RUN bundle install - + && bundle install \ ################################### # Install DotNet and Dependencies # ################################### -RUN wget --tries=5 -q -O dotnet-install.sh https://dot.net/v1/dotnet-install.sh \ + && wget --tries=5 -q -O dotnet-install.sh https://dot.net/v1/dotnet-install.sh \ && chmod +x dotnet-install.sh \ && ./dotnet-install.sh --install-dir /usr/share/dotnet -channel Current -version latest \ - && /usr/share/dotnet/dotnet tool install --tool-path /var/cache/dotnet/tools dotnet-format - -ENV PATH="${PATH}:/var/cache/dotnet/tools:/usr/share/dotnet" + && /usr/share/dotnet/dotnet tool install --tool-path /usr/bin dotnet-format ############################## # Installs Perl dependencies # ############################## -RUN curl --retry 5 --retry-delay 5 -sL https://cpanmin.us/ | perl - -nq --no-wget Perl::Critic - -############################## -# Install Phive dependencies # -############################## -RUN wget -q --tries=5 -O phive.phar https://phar.io/releases/phive.phar \ - && wget -q --tries=5 -O phive.phar.asc https://phar.io/releases/phive.phar.asc \ - && PHAR_KEY_ID="0x9D8A98B29B2D5D79" \ - && ( gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$PHAR_KEY_ID" \ - || gpg --keyserver pgp.mit.edu --recv-keys "$PHAR_KEY_ID" \ - || gpg --keyserver keyserver.pgp.com --recv-keys "$PHAR_KEY_ID" ) \ - && gpg --verify phive.phar.asc phive.phar \ - && chmod +x phive.phar \ - && mv phive.phar /usr/local/bin/phive \ - && rm phive.phar.asc \ - && phive install --trust-gpg-keys 31C7E470E2138192,CF1A108D0E7AE720,8A03EA3B385DBAA1 -# Trusted GPG keys for PHP linters: phpcs, phpstan, psalm - +RUN curl --retry 5 --retry-delay 5 -sL https://cpanmin.us/ | perl - -nq --no-wget Perl::Critic \ ######################################### # Install Powershell + PSScriptAnalyzer # ######################################### # Reference: https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-linux?view=powershell-7 # Slightly modified to always retrieve latest stable Powershell version # If changing PWSH_VERSION='latest' to a specific version, use format PWSH_VERSION='tags/v7.0.2' -RUN mkdir -p ${PWSH_DIRECTORY} \ + && mkdir -p ${PWSH_DIRECTORY} \ && curl --retry 5 --retry-delay 5 -s https://api.github.com/repos/powershell/powershell/releases/${PWSH_VERSION} \ | grep browser_download_url \ | grep linux-alpine-x64 \ @@ -237,11 +176,10 @@ COPY --from=golangci-lint /usr/bin/golangci-lint /usr/bin/ ################## COPY --from=tflint /usr/local/bin/tflint /usr/bin/ -################## +##################### # Install Terrascan # -################## +##################### COPY --from=terrascan /go/bin/terrascan /usr/bin/ -RUN terrascan init ###################### # Install Terragrunt # @@ -273,75 +211,15 @@ COPY --from=editorconfig-checker /usr/bin/ec /usr/bin/editorconfig-checker ############################### COPY --from=dockerfile-lint /bin/hadolint /usr/bin/hadolint -################## -# Install ktlint # -################## -RUN curl --retry 5 --retry-delay 5 -sSLO https://github.com/pinterest/ktlint/releases/latest/download/ktlint && chmod a+x ktlint \ - && mv "ktlint" /usr/bin/ - -#################### -# Install dart-sdk # -#################### -RUN wget --tries=5 -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub -RUN wget --tries=5 -q https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk -RUN apk add --no-cache glibc-${GLIBC_VERSION}.apk && rm glibc-${GLIBC_VERSION}.apk -RUN wget --tries=5 -q https://storage.googleapis.com/dart-archive/channels/stable/release/${DART_VERSION}/sdk/dartsdk-linux-x64-release.zip -O - -q | unzip -q - \ - && chmod +x dart-sdk/bin/dart* \ - && mv dart-sdk/bin/* /usr/bin/ && mv dart-sdk/lib/* /usr/lib/ && mv dart-sdk/include/* /usr/include/ \ - && rm -r dart-sdk/ - -################################ -# Create and install Bash-Exec # -################################ -RUN printf '#!/bin/bash \n\nif [[ -x "$1" ]]; then exit 0; else echo "Error: File:[$1] is not executable"; exit 1; fi' > /usr/bin/bash-exec \ - && chmod +x /usr/bin/bash-exec - -################################################# -# Install Raku and additional Edge dependencies # -################################################# -# Basic setup, programs and init -RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/community/" >> /etc/apk/repositories \ - && apk add --no-cache rakudo zef - -###################### -# Install CheckStyle # -###################### -RUN CHECKSTYLE_LATEST=$(curl -s https://api.github.com/repos/checkstyle/checkstyle/releases/latest \ - | grep browser_download_url \ - | grep ".jar" \ - | cut -d '"' -f 4) \ - && curl --retry 5 --retry-delay 5 -sSL "$CHECKSTYLE_LATEST" \ - --output /usr/bin/checkstyle - -#################### -# Install luacheck # -#################### -RUN wget --tries=5 -q https://www.lua.org/ftp/lua-5.3.5.tar.gz -O - -q | tar -xzf - \ - && cd lua-5.3.5 \ - && make linux \ - && make install \ - && cd .. && rm -r lua-5.3.5/ - -RUN wget --tries=5 -q https://github.com/cvega/luarocks/archive/v3.3.1-super-linter.tar.gz -O - -q | tar -xzf - \ - && cd luarocks-3.3.1-super-linter \ - && ./configure --with-lua-include=/usr/local/include \ - && make \ - && make -b install \ - && cd .. && rm -r luarocks-3.3.1-super-linter/ - -RUN luarocks install luacheck - ################# # Install lintr # ################# COPY --from=lintr-lib /usr/lib/R/library/ /home/r-library -RUN R -e "install.packages(list.dirs('/home/r-library',recursive = FALSE), repos = NULL, type = 'source')" ################## # Install chktex # ################## COPY --from=chktex /usr/bin/chktex /usr/bin/ -RUN cd ~ && touch .chktexrc ################### # Install kubeval # @@ -353,6 +231,154 @@ COPY --from=kubeval /kubeval /usr/bin/ ################# COPY --from=shfmt /bin/shfmt /usr/bin/ +################## +# Install ktlint # +################## +RUN curl --retry 5 --retry-delay 5 -sSLO https://github.com/pinterest/ktlint/releases/latest/download/ktlint \ + && chmod a+x ktlint \ + && mv "ktlint" /usr/bin/ \ + && terrascan init \ + && cd ~ && touch .chktexrc \ + && R -e "install.packages(list.dirs('/home/r-library',recursive = FALSE), repos = NULL, type = 'source')" \ +#################### +# Install dart-sdk # +#################### + && wget --tries=5 -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub \ + && wget --tries=5 -q https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk \ + && apk add --no-cache glibc-${GLIBC_VERSION}.apk \ + && rm glibc-${GLIBC_VERSION}.apk \ + && wget --tries=5 -q https://storage.googleapis.com/dart-archive/channels/stable/release/${DART_VERSION}/sdk/dartsdk-linux-x64-release.zip -O - -q | unzip -q - \ + && chmod +x dart-sdk/bin/dart* \ + && mv dart-sdk/bin/* /usr/bin/ && mv dart-sdk/lib/* /usr/lib/ && mv dart-sdk/include/* /usr/include/ \ + && rm -r dart-sdk/ \ +################################ +# Create and install Bash-Exec # +################################ + && printf '#!/bin/bash \n\nif [[ -x "$1" ]]; then exit 0; else echo "Error: File:[$1] is not executable"; exit 1; fi' > /usr/bin/bash-exec \ + && chmod +x /usr/bin/bash-exec + +################################################# +# Install Raku and additional Edge dependencies # +################################################# +# Basic setup, programs and init +RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/community/" >> /etc/apk/repositories \ + && apk add --no-cache rakudo zef \ +###################### +# Install CheckStyle # +###################### + && CHECKSTYLE_LATEST=$(curl -s https://api.github.com/repos/checkstyle/checkstyle/releases/latest \ + | grep browser_download_url \ + | grep ".jar" \ + | cut -d '"' -f 4) \ + && curl --retry 5 --retry-delay 5 -sSL "$CHECKSTYLE_LATEST" \ + --output /usr/bin/checkstyle \ +################################# +# Install luacheck and luarocks # +################################# + && wget --tries=5 -q https://www.lua.org/ftp/lua-5.3.5.tar.gz -O - -q | tar -xzf - \ + && cd lua-5.3.5 \ + && make linux \ + && make install \ + && cd .. && rm -r lua-5.3.5/ \ + && wget --tries=5 -q https://github.com/cvega/luarocks/archive/v3.3.1-super-linter.tar.gz -O - -q | tar -xzf - \ + && cd luarocks-3.3.1-super-linter \ + && ./configure --with-lua-include=/usr/local/include \ + && make \ + && make -b install \ + && cd .. \ + && rm -r luarocks-3.3.1-super-linter/ \ + && luarocks install luacheck + +################################################################################ +# Grab small clean image ####################################################### +################################################################################ +FROM alpine:3.13.5 as final + +############################ +# Get the build arguements # +############################ +ARG BUILD_DATE +ARG BUILD_REVISION +ARG BUILD_VERSION +## install alpine-pkg-glibc (glibc compatibility layer package for Alpine Linux) +ARG GLIBC_VERSION='2.31-r0' +ARG ARM_TTK_DIRECTORY='/opt/microsoft' + +######################################### +# Label the instance and set maintainer # +######################################### +LABEL com.github.actions.name="GitHub Super-Linter" \ + com.github.actions.description="Lint your code base with GitHub Actions" \ + com.github.actions.icon="code" \ + com.github.actions.color="red" \ + maintainer="GitHub DevOps " \ + org.opencontainers.image.created=$BUILD_DATE \ + org.opencontainers.image.revision=$BUILD_REVISION \ + org.opencontainers.image.version=$BUILD_VERSION \ + org.opencontainers.image.authors="GitHub DevOps " \ + org.opencontainers.image.url="https://github.com/github/super-linter" \ + org.opencontainers.image.source="https://github.com/github/super-linter" \ + org.opencontainers.image.documentation="https://github.com/github/super-linter" \ + org.opencontainers.image.vendor="GitHub" \ + org.opencontainers.image.description="Lint your code base with GitHub Actions" + +################################################# +# Set ENV values used for debugging the version # +################################################# +ENV BUILD_DATE=$BUILD_DATE +ENV BUILD_REVISION=$BUILD_REVISION +ENV BUILD_VERSION=$BUILD_VERSION +ENV ARM_TTK_PSD1="${ARM_TTK_DIRECTORY}/arm-ttk-master/arm-ttk/arm-ttk.psd1" + +############################## +# Install Phive dependencies # +############################## +RUN wget --tries=5 -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub \ + && wget --tries=5 -q https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk \ + && apk add --no-cache \ + bash \ + glibc-${GLIBC_VERSION}.apk \ + gnupg \ + php7 php7-phar php7-json php7-mbstring php-xmlwriter \ + php7-tokenizer php7-ctype php7-curl php7-dom php7-simplexml \ + && rm glibc-${GLIBC_VERSION}.apk \ + && wget -q --tries=5 -O phive.phar https://phar.io/releases/phive.phar \ + && wget -q --tries=5 -O phive.phar.asc https://phar.io/releases/phive.phar.asc \ + && PHAR_KEY_ID="0x9D8A98B29B2D5D79" \ + && ( gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$PHAR_KEY_ID" \ + || gpg --keyserver pgp.mit.edu --recv-keys "$PHAR_KEY_ID" \ + || gpg --keyserver keyserver.pgp.com --recv-keys "$PHAR_KEY_ID" ) \ + && gpg --verify phive.phar.asc phive.phar \ + && chmod +x phive.phar \ + && mv phive.phar /usr/local/bin/phive \ + && rm phive.phar.asc \ + && phive --no-progress install --trust-gpg-keys \ + 31C7E470E2138192,CF1A108D0E7AE720,8A03EA3B385DBAA1,12CE0F1D262429A5 \ + --target /usr/bin phpstan@^0.12.64 psalm@^3.18.2 phpcs@^3.5.8 + +################################# +# Copy the libraries into image # +################################# +COPY --from=base_image /usr/bin/ /usr/bin/ +COPY --from=base_image /usr/local/bin/ /usr/local/bin/ +COPY --from=base_image /usr/local/lib/ /usr/local/lib/ +COPY --from=base_image /usr/local/share/ /usr/local/share/ +COPY --from=base_image /usr/lib /usr/lib/ +COPY --from=base_image /usr/share/ /usr/share/ +COPY --from=base_image /usr/include/ /usr/include/ +COPY --from=base_image /lib/ /lib/ +COPY --from=base_image /bin/ /bin/ +COPY --from=base_image /opt/microsoft/ /opt/microsoft/ +COPY --from=base_image /node_modules/ /node_modules/ +COPY --from=base_image /tmp/.rustup/ /tmp/.rustup/ +COPY --from=base_image /etc/R/ /etc/R/ + +######################################## +# Add node packages to path and dotnet # +######################################## +ENV PATH="${PATH}:/var/cache/dotnet/tools:/usr/share/dotnet:/node_modules/.bin" +ENV ARM_TTK_PSD1="${ARM_TTK_DIRECTORY}/arm-ttk-master/arm-ttk/arm-ttk.psd1" + ############################# # Copy scripts to container # ############################# @@ -363,15 +389,11 @@ COPY lib /action/lib ################################## COPY TEMPLATES /action/lib/.automation -################################### -# Run to build file with versions # -################################### -RUN ACTIONS_RUNNER_DEBUG=true WRITE_LINTER_VERSIONS_FILE=true /action/lib/linter.sh - -##################################4 -# Run validations of built image # -################################## -RUN /action/lib/functions/validateDocker.sh +################################################ +# Run to build version file and validate image # +################################################ +RUN ACTIONS_RUNNER_DEBUG=true WRITE_LINTER_VERSIONS_FILE=true /action/lib/linter.sh \ + && /action/lib/functions/validateDocker.sh ###################### # Set the entrypoint #