Use Virtual Environment for Python Deps (#2294)

* Build static python depenencies

* Address linting

* Fix copy path

* cleaner

* Stage virtual environments

* Update Dockerfile to support virtual environments

* Remove old python builds

* Remove unnecessary RUN step

* Fix merge conflicts

* Remove test checking for PIP packages

We use virtual environments and no longer install the packages
via pip directly in the image. It should be enough that the version
tests check for the existence already and that the version
comes back correctly.

* Remove binary installation of black

* cleaner

* Remove pip

* pretty

Co-authored-by: Admiral Awkbar <admiralawkbar@github.com>
This commit is contained in:
Brett Logan 2022-01-10 16:23:22 -05:00 committed by GitHub
parent 57e304bdb4
commit fc6c5b34d9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 327 additions and 1467 deletions

View file

@ -7,24 +7,24 @@
#########################################
# Get dependency images as build stages #
#########################################
FROM accurics/terrascan:1.13.0 as terrascan
FROM alpine/terragrunt:1.1.3 as terragrunt
FROM assignuser/chktex-alpine:v0.1.1 as chktex
FROM cljkondo/clj-kondo:2021.12.19-alpine as clj-kondo
FROM dotenvlinter/dotenv-linter:3.1.1 as dotenv-linter
FROM mstruebing/editorconfig-checker:2.4.0 as editorconfig-checker
FROM yoheimuta/protolint:v0.35.2 as protolint
FROM golangci/golangci-lint:v1.43.0 as golangci-lint
FROM koalaman/shellcheck:v0.8.0 as shellcheck
FROM ghcr.io/terraform-linters/tflint-bundle:v0.34.1.1 as tflint
FROM hashicorp/terraform:1.1.3 as terraform
FROM alpine/terragrunt:1.1.3 as terragrunt
FROM mvdan/shfmt:v3.4.2 as shfmt
FROM accurics/terrascan:1.13.0 as terrascan
FROM hadolint/hadolint:latest-alpine as dockerfile-lint
FROM assignuser/chktex-alpine:v0.1.1 as chktex
FROM zricethezav/gitleaks:v8.2.7 as gitleaks
FROM garethr/kubeval:0.15.0 as kubeval
FROM ghcr.io/awkbar-devops/clang-format:v1.0.2 as clang-format
FROM scalameta/scalafmt:v3.3.1 as scalafmt
FROM ghcr.io/terraform-linters/tflint-bundle:v0.34.1.1 as tflint
FROM golangci/golangci-lint:v1.43.0 as golangci-lint
FROM hadolint/hadolint:latest-alpine as dockerfile-lint
FROM hashicorp/terraform:1.1.3 as terraform
FROM koalaman/shellcheck:v0.8.0 as shellcheck
FROM mstruebing/editorconfig-checker:2.4.0 as editorconfig-checker
FROM mvdan/shfmt:v3.4.2 as shfmt
FROM rhysd/actionlint:1.6.8 as actionlint
FROM scalameta/scalafmt:v3.3.1 as scalafmt
FROM yoheimuta/protolint:v0.35.2 as protolint
FROM zricethezav/gitleaks:v8.2.7 as gitleaks
##################
# Get base image #
@ -34,11 +34,19 @@ FROM python:3.10.1-alpine as base_image
################################
# Set ARG values used in Build #
################################
# arm-ttk Linter
ARG ARM_TTK_NAME='master.zip'
ARG ARM_TTK_URI='https://github.com/Azure/arm-ttk/archive/master.zip'
ARG ARM_TTK_DIRECTORY='/usr/lib/microsoft'
# Dart Linter
## stable dart sdk: https://dart.dev/get-dart#release-channels
ARG DART_VERSION='2.8.4'
## install alpine-pkg-glibc (glibc compatibility layer package for Alpine Linux)
ARG GLIBC_VERSION='2.31-r0'
# PowerShell & PSScriptAnalyzer linter
ARG PSSA_VERSION='latest'
ARG PWSH_DIRECTORY='/usr/lib/microsoft/powershell'
ARG PWSH_VERSION='latest'
####################
# Run APK installs #
@ -53,6 +61,7 @@ RUN apk add --no-cache \
gcc \
g++ \
git git-lfs \
go \
gnupg \
go \
icu-libs \
@ -86,14 +95,7 @@ COPY dependencies/* /
################################
# Installs python dependencies #
################################
RUN pip3 install --no-cache-dir pipenv \
# Bug in hadolint thinks pipenv is pip
# hadolint ignore=DL3042
&& pipenv install --clear --system \
####################
# Run NPM Installs #
####################
&& npm config set package-lock true \
RUN npm config set package-lock true \
&& npm config set loglevel error \
&& npm --no-cache install \
&& npm audit fix --audit-level=critical \
@ -106,11 +108,6 @@ RUN pip3 install --no-cache-dir pipenv \
# Installs Perl dependencies #
##############################
RUN curl --retry 5 --retry-delay 5 -sL https://cpanmin.us/ | perl - -nq --no-wget Perl::Critic \
########################
# Install Python Black #
########################
&& wget --tries=5 -q -O /usr/local/bin/black https://github.com/psf/black/releases/download/21.11b1/black_linux \
&& chmod +x /usr/local/bin/black \
#######################
# Installs ActionLint #
#######################
@ -286,7 +283,16 @@ RUN apk add --no-cache rakudo zef \
&& find /usr/ -type f -name '*.md' -exec rm {} +
################################################################################
# Grab small clean image #######################################################
# Grab small clean image to build python packages ##############################
################################################################################
FROM python:3.10.1-alpine as python_builder
RUN apk add --no-cache bash g++ git libffi-dev
COPY dependencies/python/ /stage
WORKDIR /stage
RUN ./build-venvs.sh
################################################################################
# Grab small clean image to build final_slim ###################################
################################################################################
FROM alpine:3.15.0 as final_slim
@ -372,6 +378,7 @@ COPY --from=base_image /bin/ /bin/
COPY --from=base_image /node_modules/ /node_modules/
COPY --from=base_image /home/r-library /home/r-library
COPY --from=base_image /root/.tflint.d/ /root/.tflint.d/
COPY --from=python_builder /venvs/ /venvs/
####################################################
# Install Composer after all Libs have been copied #
@ -383,6 +390,23 @@ RUN sh -c 'curl -sS https://getcomposer.org/installer | php -- --install-dir=/us
########################################
ENV PATH="${PATH}:/node_modules/.bin"
###############################
# Add python packages to path #
###############################
ENV PATH="${PATH}:/venvs/ansible-lint/bin"
ENV PATH="${PATH}:/venvs/black/bin"
ENV PATH="${PATH}:/venvs/cfn-lint/bin"
ENV PATH="${PATH}:/venvs/cpplint/bin"
ENV PATH="${PATH}:/venvs/flake8/bin"
ENV PATH="${PATH}:/venvs/isort/bin"
ENV PATH="${PATH}:/venvs/mypy/bin"
ENV PATH="${PATH}:/venvs/pylint/bin"
ENV PATH="${PATH}:/venvs/snakefmt/bin"
ENV PATH="${PATH}:/venvs/snakemake/bin"
ENV PATH="${PATH}:/venvs/sqlfluff/bin"
ENV PATH="${PATH}:/venvs/yamllint/bin"
ENV PATH="${PATH}:/venvs/yq/bin"
#############################
# Copy scripts to container #
#############################
@ -403,25 +427,33 @@ RUN ACTIONS_RUNNER_DEBUG=true WRITE_LINTER_VERSIONS_FILE=true IMAGE="${IMAGE}" /
######################
ENTRYPOINT ["/action/lib/linter.sh"]
################################################################################
# Grab small clean image to build final_standard ###############################
################################################################################
FROM final_slim as final_standard
ARG ARM_TTK_DIRECTORY='/usr/lib/microsoft'
# PowerShell & PSScriptAnalyzer
ARG PWSH_VERSION='latest'
ARG PWSH_DIRECTORY='/usr/lib/microsoft/powershell'
ARG PSSA_VERSION='latest'
###############
# Set up args #
###############
# arm-ttk
ARG ARM_TTK_NAME='master.zip'
ARG ARM_TTK_URI='https://github.com/Azure/arm-ttk/archive/master.zip'
ARG ARM_TTK_DIRECTORY='/usr/lib/microsoft'
# PowerShell & PSScriptAnalyzer
ARG PWSH_VERSION='latest'
ARG PWSH_DIRECTORY='/usr/lib/microsoft/powershell'
ARG PSSA_VERSION='latest'
ENV IMAGE="standard"
################
# Set ENV vars #
################
ENV ARM_TTK_PSD1="${ARM_TTK_DIRECTORY}/arm-ttk-master/arm-ttk/arm-ttk.psd1"
ENV IMAGE="standard"
ENV PATH="${PATH}:/var/cache/dotnet/tools:/usr/share/dotnet"
################
# Pull in libs #
################
COPY --from=base_image /usr/libexec/ /usr/libexec/
#########################

32
dependencies/Pipfile vendored
View file

@ -1,32 +0,0 @@
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
[packages]
ansible-lint = {extras = ["core"], version = "*"}
black = "*"
cfn-lint = "*"
cpplint = "*"
cython = "*"
flake8 = "*"
isort = "*"
kiwisolver = "*"
mypy = "*"
pylint = "*"
pybind11 = "*"
snakefmt = "*"
snakemake = "*"
sqlfluff = "*"
typing_extensions = "*"
types-requests = "*"
yamllint = "*"
yq = "*"
[requires]
python_version = "3.9"
[pipenv]
allow_prereleases = true

1365
dependencies/Pipfile.lock generated vendored

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,26 @@
ansible-core==2.12.1
ansible-lint==5.3.1
bracex==2.2.1
cffi==1.15.0
colorama==0.4.4
commonmark==0.9.1
cryptography==36.0.1
distlib==0.3.4
enrich==1.2.6
filelock==3.4.2
Jinja2==3.0.3
MarkupSafe==2.0.1
packaging==21.3
platformdirs==2.4.1
pycparser==2.21
Pygments==2.11.1
pyparsing==3.0.6
PyYAML==6.0
resolvelib==0.5.4
rich==10.16.2
ruamel.yaml==0.17.20
ruamel.yaml.clib==0.2.6
six==1.16.0
tenacity==8.0.1
virtualenv==20.13.0
wcmatch==8.3

View file

@ -0,0 +1,11 @@
black==21.12b0
click==8.0.3
distlib==0.3.4
filelock==3.4.2
mypy-extensions==0.4.3
pathspec==0.9.0
platformdirs==2.4.1
six==1.16.0
tomli==1.2.3
typing_extensions==4.0.1
virtualenv==20.13.0

43
dependencies/python/build-venvs.sh vendored Executable file
View file

@ -0,0 +1,43 @@
#!/usr/bin/env bash
################################################################################
########################### Install Python Dependancies ########################
################################################################################
#####################
# Set fail on error #
#####################
set -euo pipefail
############################
# Create staging directory #
############################
mkdir -p /venvs
########################################
# Install basic libs to run installers #
########################################
pip install virtualenv
#########################################################
# Itterate through requirments.txt to install bainaries #
#########################################################
while read -r LINE; do
# split the package name from its version
PACKAGE_NAME=$(cut -d'=' -f1 <<<"${LINE}")
if [[ "${PACKAGE_NAME}" == *"["* ]]; then
PACKAGE_NAME=$(cut -d'[' -f1 <<<"${PACKAGE_NAME}")
fi
echo "-------------------------------------------"
mkdir -p "/venvs/${PACKAGE_NAME}"
cp "${PACKAGE_NAME}/requirements.txt" "/venvs/${PACKAGE_NAME}/requirements.txt"
echo "Generating virtualenv for: [${PACKAGE_NAME}]"
pushd "/venvs/${PACKAGE_NAME}"
virtualenv .
# shellcheck disable=SC1091
source bin/activate
pip install -r requirements.txt
# deactivate the python virtualenv
deactivate
# pop the stack
popd
done <packages.txt

View file

@ -0,0 +1,25 @@
attrs==21.4.0
aws-sam-translator==1.42.0
boto3==1.20.29
botocore==1.23.29
cfn-lint==0.56.3
distlib==0.3.4
filelock==3.4.2
jmespath==0.10.0
jschema-to-python==1.2.3
jsonpatch==1.32
jsonpickle==2.0.0
jsonpointer==2.2
jsonschema==3.2.0
junit-xml==1.9
networkx==2.6.3
pbr==5.8.0
platformdirs==2.4.1
pyrsistent==0.18.0
python-dateutil==2.8.2
PyYAML==6.0
s3transfer==0.5.0
sarif-om==1.0.4
six==1.16.0
urllib3==1.26.7
virtualenv==20.13.0

View file

@ -0,0 +1,6 @@
cpplint==1.5.5
distlib==0.3.4
filelock==3.4.2
platformdirs==2.4.1
six==1.16.0
virtualenv==20.13.0

View file

@ -0,0 +1,9 @@
distlib==0.3.4
filelock==3.4.2
flake8==4.0.1
mccabe==0.6.1
platformdirs==2.4.1
pycodestyle==2.8.0
pyflakes==2.4.0
six==1.16.0
virtualenv==20.13.0

View file

@ -0,0 +1,6 @@
distlib==0.3.4
filelock==3.4.2
isort==5.10.1
platformdirs==2.4.1
six==1.16.0
virtualenv==20.13.0

View file

@ -0,0 +1,9 @@
distlib==0.3.4
filelock==3.4.2
mypy==0.930
mypy-extensions==0.4.3
platformdirs==2.4.1
six==1.16.0
tomli==2.0.0
typing_extensions==4.0.1
virtualenv==20.13.0

13
dependencies/python/packages.txt vendored Normal file
View file

@ -0,0 +1,13 @@
ansible-lint[core]
black
cfn-lint
cpplint
flake8
isort
mypy
pylint
snakefmt
snakemake
sqlfluff
yamllint
yq

View file

@ -0,0 +1,12 @@
astroid==2.9.2
distlib==0.3.4
filelock==3.4.2
isort==5.10.1
lazy-object-proxy==1.7.1
mccabe==0.6.1
platformdirs==2.4.1
pylint==2.12.2
six==1.16.0
toml==0.10.2
virtualenv==20.13.0
wrapt==1.13.3

View file

@ -0,0 +1,15 @@
black==21.12b0
click==7.1.2
distlib==0.3.4
filelock==3.4.2
importlib-metadata==1.7.0
mypy-extensions==0.4.3
pathspec==0.9.0
platformdirs==2.4.1
six==1.16.0
snakefmt==0.4.4
toml==0.10.2
tomli==1.2.3
typing_extensions==4.0.1
virtualenv==20.13.0
zipp==3.7.0

View file

@ -0,0 +1,35 @@
appdirs==1.4.4
attrs==21.4.0
certifi==2021.10.8
charset-normalizer==2.0.10
ConfigArgParse==1.5.3
connection-pool==0.0.3
datrie==0.8.2
distlib==0.3.4
docutils==0.18.1
filelock==3.4.2
gitdb==4.0.9
GitPython==3.1.24
idna==3.3
ipython-genutils==0.2.0
jsonschema==4.3.3
jupyter-core==4.9.1
nbformat==5.1.3
platformdirs==2.4.1
psutil==5.9.0
PuLP==2.6.0
pyrsistent==0.18.0
PyYAML==6.0
ratelimiter==1.2.0.post0
requests==2.27.1
six==1.16.0
smart-open==5.2.1
smmap==5.0.0
snakemake==6.12.3
stopit==1.1.2
tabulate==0.8.9
toposort==1.7
traitlets==5.1.1
urllib3==1.26.7
virtualenv==20.13.0
wrapt==1.13.3

View file

@ -0,0 +1,30 @@
appdirs==1.4.4
attrs==21.4.0
cached-property==1.5.2
chardet==4.0.0
click==8.0.3
colorama==0.4.4
diff-cover==6.4.4
distlib==0.3.4
filelock==3.4.2
iniconfig==1.1.1
Jinja2==3.0.3
MarkupSafe==2.0.1
oyaml==1.0
packaging==21.3
pathspec==0.9.0
platformdirs==2.4.1
pluggy==1.0.0
py==1.11.0
Pygments==2.11.1
pyparsing==3.0.6
pytest==6.2.5
PyYAML==6.0
regex==2021.11.10
six==1.16.0
sqlfluff==0.9.0
tblib==1.7.0
toml==0.10.2
tqdm==4.62.3
typing_extensions==4.0.1
virtualenv==20.13.0

View file

@ -0,0 +1,8 @@
distlib==0.3.4
filelock==3.4.2
pathspec==0.9.0
platformdirs==2.4.1
PyYAML==6.0
six==1.16.0
virtualenv==20.13.0
yamllint==1.26.3

10
dependencies/python/yq/requirements.txt vendored Normal file
View file

@ -0,0 +1,10 @@
argcomplete==2.0.0
distlib==0.3.4
filelock==3.4.2
platformdirs==2.4.1
PyYAML==6.0
six==1.16.0
toml==0.10.2
virtualenv==20.13.0
xmltodict==0.12.0
yq==2.13.0

View file

@ -252,39 +252,6 @@ control "super-linter-installed-ruby-gems" do
end
###############################################
# Check to see all PIP packages are installed #
###############################################
control "super-linter-installed-pip-packages" do
impact 1
title "Super-Linter installed PIP packages check"
desc "Check that PIP packages that Super-Linter needs are installed."
packages = [
"ansible-lint",
"cfn-lint",
"cpplint",
"cython",
"flake8",
"isort",
"mypy",
"pylint",
"snakefmt",
"snakemake",
"sqlfluff",
"typing_extensions",
"yamllint",
"yq"
]
packages.each do |item|
describe pip(item) do
it { should be_installed }
end
end
end
###############################################
# Check to see all NPM packages are installed #
###############################################