Use Virtual Environment for Python Deps (#2294)

* Build static python depenencies

* Address linting

* Fix copy path

* cleaner

* Stage virtual environments

* Update Dockerfile to support virtual environments

* Remove old python builds

* Remove unnecessary RUN step

* Fix merge conflicts

* Remove test checking for PIP packages

We use virtual environments and no longer install the packages
via pip directly in the image. It should be enough that the version
tests check for the existence already and that the version
comes back correctly.

* Remove binary installation of black

* cleaner

* Remove pip

* pretty

Co-authored-by: Admiral Awkbar <admiralawkbar@github.com>

Dockerfile

@@ -7,24 +7,24 @@
 #########################################
 # Get dependency images as build stages #
 #########################################
+FROM accurics/terrascan:1.13.0 as terrascan
+FROM alpine/terragrunt:1.1.3 as terragrunt
+FROM assignuser/chktex-alpine:v0.1.1 as chktex
 FROM cljkondo/clj-kondo:2021.12.19-alpine as clj-kondo
 FROM dotenvlinter/dotenv-linter:3.1.1 as dotenv-linter
-FROM mstruebing/editorconfig-checker:2.4.0 as editorconfig-checker
-FROM yoheimuta/protolint:v0.35.2 as protolint
-FROM golangci/golangci-lint:v1.43.0 as golangci-lint
-FROM koalaman/shellcheck:v0.8.0 as shellcheck
-FROM ghcr.io/terraform-linters/tflint-bundle:v0.34.1.1 as tflint
-FROM hashicorp/terraform:1.1.3 as terraform
-FROM alpine/terragrunt:1.1.3 as terragrunt
-FROM mvdan/shfmt:v3.4.2 as shfmt
-FROM accurics/terrascan:1.13.0 as terrascan
-FROM hadolint/hadolint:latest-alpine as dockerfile-lint
-FROM assignuser/chktex-alpine:v0.1.1 as chktex
-FROM zricethezav/gitleaks:v8.2.7 as gitleaks
 FROM garethr/kubeval:0.15.0 as kubeval
 FROM ghcr.io/awkbar-devops/clang-format:v1.0.2 as clang-format
-FROM scalameta/scalafmt:v3.3.1 as scalafmt
+FROM ghcr.io/terraform-linters/tflint-bundle:v0.34.1.1 as tflint
+FROM golangci/golangci-lint:v1.43.0 as golangci-lint
+FROM hadolint/hadolint:latest-alpine as dockerfile-lint
+FROM hashicorp/terraform:1.1.3 as terraform
+FROM koalaman/shellcheck:v0.8.0 as shellcheck
+FROM mstruebing/editorconfig-checker:2.4.0 as editorconfig-checker
+FROM mvdan/shfmt:v3.4.2 as shfmt
 FROM rhysd/actionlint:1.6.8 as actionlint
+FROM scalameta/scalafmt:v3.3.1 as scalafmt
+FROM yoheimuta/protolint:v0.35.2 as protolint
+FROM zricethezav/gitleaks:v8.2.7 as gitleaks
 
 ##################
 # Get base image #
@@ -34,11 +34,19 @@ FROM python:3.10.1-alpine as base_image
 ################################
 # Set ARG values used in Build #
 ################################
+# arm-ttk Linter
+ARG ARM_TTK_NAME='master.zip'
+ARG ARM_TTK_URI='https://github.com/Azure/arm-ttk/archive/master.zip'
+ARG ARM_TTK_DIRECTORY='/usr/lib/microsoft'
 # Dart Linter
 ## stable dart sdk: https://dart.dev/get-dart#release-channels
 ARG DART_VERSION='2.8.4'
 ## install alpine-pkg-glibc (glibc compatibility layer package for Alpine Linux)
 ARG GLIBC_VERSION='2.31-r0'
+# PowerShell & PSScriptAnalyzer linter
+ARG PSSA_VERSION='latest'
+ARG PWSH_DIRECTORY='/usr/lib/microsoft/powershell'
+ARG PWSH_VERSION='latest'
 
 ####################
 # Run APK installs #
@@ -52,7 +60,8 @@ RUN apk add --no-cache \
     file \
     gcc \
     g++ \
-    git git-lfs\
+    git git-lfs \
+    go \
     gnupg \
     go \
     icu-libs \
@@ -86,14 +95,7 @@ COPY dependencies/* /
 ################################
 # Installs python dependencies #
 ################################
-RUN pip3 install --no-cache-dir pipenv \
-    # Bug in hadolint thinks pipenv is pip
-    # hadolint ignore=DL3042
-    && pipenv install --clear --system \
-    ####################
-    # Run NPM Installs #
-    ####################
-    && npm config set package-lock true \
+RUN npm config set package-lock true  \
     && npm config set loglevel error \
     && npm --no-cache install \
     && npm audit fix --audit-level=critical \
@@ -106,11 +108,6 @@ RUN pip3 install --no-cache-dir pipenv \
 # Installs Perl dependencies #
 ##############################
 RUN curl --retry 5 --retry-delay 5 -sL https://cpanmin.us/ | perl - -nq --no-wget Perl::Critic \
-    ########################
-    # Install Python Black #
-    ########################
-    && wget --tries=5 -q -O /usr/local/bin/black https://github.com/psf/black/releases/download/21.11b1/black_linux \
-    && chmod +x /usr/local/bin/black \
     #######################
     # Installs ActionLint #
     #######################
@@ -286,7 +283,16 @@ RUN apk add --no-cache rakudo zef \
     && find /usr/ -type f -name '*.md' -exec rm {} +
 
 ################################################################################
-# Grab small clean image #######################################################
+# Grab small clean image to build python packages ##############################
+################################################################################
+FROM python:3.10.1-alpine as python_builder
+RUN apk add --no-cache bash g++ git libffi-dev
+COPY dependencies/python/ /stage
+WORKDIR /stage
+RUN ./build-venvs.sh
+
+################################################################################
+# Grab small clean image to build final_slim ###################################
 ################################################################################
 FROM alpine:3.15.0 as final_slim
 
@@ -372,6 +378,7 @@ COPY --from=base_image /bin/ /bin/
 COPY --from=base_image /node_modules/ /node_modules/
 COPY --from=base_image /home/r-library /home/r-library
 COPY --from=base_image /root/.tflint.d/ /root/.tflint.d/
+COPY --from=python_builder /venvs/ /venvs/
 
 ####################################################
 # Install Composer after all Libs have been copied #
@@ -383,6 +390,23 @@ RUN sh -c 'curl -sS https://getcomposer.org/installer | php -- --install-dir=/us
 ########################################
 ENV PATH="${PATH}:/node_modules/.bin"
 
+###############################
+# Add python packages to path #
+###############################
+ENV PATH="${PATH}:/venvs/ansible-lint/bin"
+ENV PATH="${PATH}:/venvs/black/bin"
+ENV PATH="${PATH}:/venvs/cfn-lint/bin"
+ENV PATH="${PATH}:/venvs/cpplint/bin"
+ENV PATH="${PATH}:/venvs/flake8/bin"
+ENV PATH="${PATH}:/venvs/isort/bin"
+ENV PATH="${PATH}:/venvs/mypy/bin"
+ENV PATH="${PATH}:/venvs/pylint/bin"
+ENV PATH="${PATH}:/venvs/snakefmt/bin"
+ENV PATH="${PATH}:/venvs/snakemake/bin"
+ENV PATH="${PATH}:/venvs/sqlfluff/bin"
+ENV PATH="${PATH}:/venvs/yamllint/bin"
+ENV PATH="${PATH}:/venvs/yq/bin"
+
 #############################
 # Copy scripts to container #
 #############################
@@ -403,25 +427,33 @@ RUN ACTIONS_RUNNER_DEBUG=true WRITE_LINTER_VERSIONS_FILE=true IMAGE="${IMAGE}" /
 ######################
 ENTRYPOINT ["/action/lib/linter.sh"]
 
+################################################################################
+# Grab small clean image to build final_standard ###############################
+################################################################################
 FROM final_slim as final_standard
 
-ARG ARM_TTK_DIRECTORY='/usr/lib/microsoft'
-
-# PowerShell & PSScriptAnalyzer
-ARG PWSH_VERSION='latest'
-ARG PWSH_DIRECTORY='/usr/lib/microsoft/powershell'
-ARG PSSA_VERSION='latest'
+###############
+# Set up args #
+###############
 # arm-ttk
 ARG ARM_TTK_NAME='master.zip'
 ARG ARM_TTK_URI='https://github.com/Azure/arm-ttk/archive/master.zip'
 ARG ARM_TTK_DIRECTORY='/usr/lib/microsoft'
+# PowerShell & PSScriptAnalyzer
+ARG PWSH_VERSION='latest'
+ARG PWSH_DIRECTORY='/usr/lib/microsoft/powershell'
+ARG PSSA_VERSION='latest'
 
-ENV IMAGE="standard"
-
+################
+# Set ENV vars #
+################
 ENV ARM_TTK_PSD1="${ARM_TTK_DIRECTORY}/arm-ttk-master/arm-ttk/arm-ttk.psd1"
-
+ENV IMAGE="standard"
 ENV PATH="${PATH}:/var/cache/dotnet/tools:/usr/share/dotnet"
 
+################
+# Pull in libs #
+################
 COPY --from=base_image /usr/libexec/ /usr/libexec/
 
 #########################

dependencies/Pipfile

@@ -1,32 +0,0 @@
-[[source]]
-name = "pypi"
-url = "https://pypi.org/simple"
-verify_ssl = true
-
-[dev-packages]
-
-[packages]
-ansible-lint = {extras = ["core"], version = "*"}
-black = "*"
-cfn-lint = "*"
-cpplint = "*"
-cython = "*"
-flake8 = "*"
-isort = "*"
-kiwisolver = "*"
-mypy = "*"
-pylint = "*"
-pybind11 = "*"
-snakefmt = "*"
-snakemake = "*"
-sqlfluff = "*"
-typing_extensions = "*"
-types-requests = "*"
-yamllint = "*"
-yq = "*"
-
-[requires]
-python_version = "3.9"
-
-[pipenv]
-allow_prereleases = true

dependencies/python/ansible-lint/requirements.txt

@@ -0,0 +1,26 @@
+ansible-core==2.12.1
+ansible-lint==5.3.1
+bracex==2.2.1
+cffi==1.15.0
+colorama==0.4.4
+commonmark==0.9.1
+cryptography==36.0.1
+distlib==0.3.4
+enrich==1.2.6
+filelock==3.4.2
+Jinja2==3.0.3
+MarkupSafe==2.0.1
+packaging==21.3
+platformdirs==2.4.1
+pycparser==2.21
+Pygments==2.11.1
+pyparsing==3.0.6
+PyYAML==6.0
+resolvelib==0.5.4
+rich==10.16.2
+ruamel.yaml==0.17.20
+ruamel.yaml.clib==0.2.6
+six==1.16.0
+tenacity==8.0.1
+virtualenv==20.13.0
+wcmatch==8.3

dependencies/python/black/requirements.txt

@@ -0,0 +1,11 @@
+black==21.12b0
+click==8.0.3
+distlib==0.3.4
+filelock==3.4.2
+mypy-extensions==0.4.3
+pathspec==0.9.0
+platformdirs==2.4.1
+six==1.16.0
+tomli==1.2.3
+typing_extensions==4.0.1
+virtualenv==20.13.0

dependencies/python/build-venvs.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+################################################################################
+########################### Install Python Dependancies ########################
+################################################################################
+
+#####################
+# Set fail on error #
+#####################
+set -euo pipefail
+
+############################
+# Create staging directory #
+############################
+mkdir -p /venvs
+
+########################################
+# Install basic libs to run installers #
+########################################
+pip install virtualenv
+
+#########################################################
+# Itterate through requirments.txt to install bainaries #
+#########################################################
+while read -r LINE; do
+  # split the package name from its version
+  PACKAGE_NAME=$(cut -d'=' -f1 <<<"${LINE}")
+  if [[ "${PACKAGE_NAME}" == *"["* ]]; then
+    PACKAGE_NAME=$(cut -d'[' -f1 <<<"${PACKAGE_NAME}")
+  fi
+  echo "-------------------------------------------"
+  mkdir -p "/venvs/${PACKAGE_NAME}"
+  cp "${PACKAGE_NAME}/requirements.txt" "/venvs/${PACKAGE_NAME}/requirements.txt"
+  echo "Generating virtualenv for: [${PACKAGE_NAME}]"
+  pushd "/venvs/${PACKAGE_NAME}"
+  virtualenv .
+  # shellcheck disable=SC1091
+  source bin/activate
+  pip install -r requirements.txt
+  # deactivate the python virtualenv
+  deactivate
+  # pop the stack
+  popd
+done <packages.txt

dependencies/python/cfn-lint/requirements.txt

@@ -0,0 +1,25 @@
+attrs==21.4.0
+aws-sam-translator==1.42.0
+boto3==1.20.29
+botocore==1.23.29
+cfn-lint==0.56.3
+distlib==0.3.4
+filelock==3.4.2
+jmespath==0.10.0
+jschema-to-python==1.2.3
+jsonpatch==1.32
+jsonpickle==2.0.0
+jsonpointer==2.2
+jsonschema==3.2.0
+junit-xml==1.9
+networkx==2.6.3
+pbr==5.8.0
+platformdirs==2.4.1
+pyrsistent==0.18.0
+python-dateutil==2.8.2
+PyYAML==6.0
+s3transfer==0.5.0
+sarif-om==1.0.4
+six==1.16.0
+urllib3==1.26.7
+virtualenv==20.13.0

dependencies/python/cpplint/requirements.txt

@@ -0,0 +1,6 @@
+cpplint==1.5.5
+distlib==0.3.4
+filelock==3.4.2
+platformdirs==2.4.1
+six==1.16.0
+virtualenv==20.13.0

dependencies/python/flake8/requirements.txt

@@ -0,0 +1,9 @@
+distlib==0.3.4
+filelock==3.4.2
+flake8==4.0.1
+mccabe==0.6.1
+platformdirs==2.4.1
+pycodestyle==2.8.0
+pyflakes==2.4.0
+six==1.16.0
+virtualenv==20.13.0

dependencies/python/isort/requirements.txt

@@ -0,0 +1,6 @@
+distlib==0.3.4
+filelock==3.4.2
+isort==5.10.1
+platformdirs==2.4.1
+six==1.16.0
+virtualenv==20.13.0

dependencies/python/mypy/requirements.txt

@@ -0,0 +1,9 @@
+distlib==0.3.4
+filelock==3.4.2
+mypy==0.930
+mypy-extensions==0.4.3
+platformdirs==2.4.1
+six==1.16.0
+tomli==2.0.0
+typing_extensions==4.0.1
+virtualenv==20.13.0

dependencies/python/packages.txt

@@ -0,0 +1,13 @@
+ansible-lint[core]
+black
+cfn-lint
+cpplint
+flake8
+isort
+mypy
+pylint
+snakefmt
+snakemake
+sqlfluff
+yamllint
+yq

dependencies/python/pylint/requirements.txt

@@ -0,0 +1,12 @@
+astroid==2.9.2
+distlib==0.3.4
+filelock==3.4.2
+isort==5.10.1
+lazy-object-proxy==1.7.1
+mccabe==0.6.1
+platformdirs==2.4.1
+pylint==2.12.2
+six==1.16.0
+toml==0.10.2
+virtualenv==20.13.0
+wrapt==1.13.3

dependencies/python/snakefmt/requirements.txt

@@ -0,0 +1,15 @@
+black==21.12b0
+click==7.1.2
+distlib==0.3.4
+filelock==3.4.2
+importlib-metadata==1.7.0
+mypy-extensions==0.4.3
+pathspec==0.9.0
+platformdirs==2.4.1
+six==1.16.0
+snakefmt==0.4.4
+toml==0.10.2
+tomli==1.2.3
+typing_extensions==4.0.1
+virtualenv==20.13.0
+zipp==3.7.0

dependencies/python/snakemake/requirements.txt

@@ -0,0 +1,35 @@
+appdirs==1.4.4
+attrs==21.4.0
+certifi==2021.10.8
+charset-normalizer==2.0.10
+ConfigArgParse==1.5.3
+connection-pool==0.0.3
+datrie==0.8.2
+distlib==0.3.4
+docutils==0.18.1
+filelock==3.4.2
+gitdb==4.0.9
+GitPython==3.1.24
+idna==3.3
+ipython-genutils==0.2.0
+jsonschema==4.3.3
+jupyter-core==4.9.1
+nbformat==5.1.3
+platformdirs==2.4.1
+psutil==5.9.0
+PuLP==2.6.0
+pyrsistent==0.18.0
+PyYAML==6.0
+ratelimiter==1.2.0.post0
+requests==2.27.1
+six==1.16.0
+smart-open==5.2.1
+smmap==5.0.0
+snakemake==6.12.3
+stopit==1.1.2
+tabulate==0.8.9
+toposort==1.7
+traitlets==5.1.1
+urllib3==1.26.7
+virtualenv==20.13.0
+wrapt==1.13.3

dependencies/python/sqlfluff/requirements.txt

@@ -0,0 +1,30 @@
+appdirs==1.4.4
+attrs==21.4.0
+cached-property==1.5.2
+chardet==4.0.0
+click==8.0.3
+colorama==0.4.4
+diff-cover==6.4.4
+distlib==0.3.4
+filelock==3.4.2
+iniconfig==1.1.1
+Jinja2==3.0.3
+MarkupSafe==2.0.1
+oyaml==1.0
+packaging==21.3
+pathspec==0.9.0
+platformdirs==2.4.1
+pluggy==1.0.0
+py==1.11.0
+Pygments==2.11.1
+pyparsing==3.0.6
+pytest==6.2.5
+PyYAML==6.0
+regex==2021.11.10
+six==1.16.0
+sqlfluff==0.9.0
+tblib==1.7.0
+toml==0.10.2
+tqdm==4.62.3
+typing_extensions==4.0.1
+virtualenv==20.13.0

dependencies/python/yamllint/requirements.txt

@@ -0,0 +1,8 @@
+distlib==0.3.4
+filelock==3.4.2
+pathspec==0.9.0
+platformdirs==2.4.1
+PyYAML==6.0
+six==1.16.0
+virtualenv==20.13.0
+yamllint==1.26.3

dependencies/python/yq/requirements.txt

@@ -0,0 +1,10 @@
+argcomplete==2.0.0
+distlib==0.3.4
+filelock==3.4.2
+platformdirs==2.4.1
+PyYAML==6.0
+six==1.16.0
+toml==0.10.2
+virtualenv==20.13.0
+xmltodict==0.12.0
+yq==2.13.0

test/inspec/super-linter/controls/super_linter.rb

@@ -252,39 +252,6 @@ control "super-linter-installed-ruby-gems" do
 
 end
 
-###############################################
-# Check to see all PIP packages are installed #
-###############################################
-control "super-linter-installed-pip-packages" do
-  impact 1
-  title "Super-Linter installed PIP packages check"
-  desc "Check that PIP packages that Super-Linter needs are installed."
-
-  packages = [
-    "ansible-lint",
-    "cfn-lint",
-    "cpplint",
-    "cython",
-    "flake8",
-    "isort",
-    "mypy",
-    "pylint",
-    "snakefmt",
-    "snakemake",
-    "sqlfluff",
-    "typing_extensions",
-    "yamllint",
-    "yq"
-  ]
-
-  packages.each do |item|
-    describe pip(item) do
-      it { should be_installed }
-    end
-  end
-
-end
-
 ###############################################
 # Check to see all NPM packages are installed #
 ###############################################