From 0c8db849aa77c440e06aa78c0c0615a3464148a1 Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Fri, 30 Apr 2021 18:29:15 +0200 Subject: [PATCH] Implement a test suite with InSpec (#1466) * Implement a test suite with InSpec * make ruby happy * adding file * Fix inspec and add make target * Run inspec * adding binaries * make ruby happy * fix linter order * cleanup * adding version check * adding notes * cleanup * cleanup * fixed r test * fixed copy paste * dynamic tests * fix hash * fix notation * docker ps * Fix makefile * Fix makefile Co-authored-by: Admiral Awkbar --- Makefile | 35 +++- lib/functions/linterVersions.sh | 6 +- lib/linter.sh | 16 -- test/inspec/inspec.lock | 3 + .../super-linter/controls/super_linter.rb | 198 ++++++++++++++++++ test/inspec/super-linter/inspec.lock | 3 + test/inspec/super-linter/inspec.yml | 11 + 7 files changed, 253 insertions(+), 19 deletions(-) create mode 100644 test/inspec/inspec.lock create mode 100644 test/inspec/super-linter/controls/super_linter.rb create mode 100644 test/inspec/super-linter/inspec.lock create mode 100644 test/inspec/super-linter/inspec.yml diff --git a/Makefile b/Makefile index 809dcc18..6857c3d7 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ all: info test ## Run all targets. .PHONY: test -test: info clean kcov prepare-test-reports ## Run tests +test: info clean inspec kcov prepare-test-reports ## Run tests # if this session isn't interactive, then we don't want to allocate a # TTY, which would fail, but if it is interactive, we do want to attach @@ -19,7 +19,8 @@ info: ## Gather information about the runtime environment echo "whoami: $$(whoami)"; \ echo "pwd: $$(pwd)"; \ echo "ls -ahl: $$(ls -ahl)"; \ - docker images + docker images; \ + docker ps .PHONY: kcov kcov: ## Run kcov @@ -58,3 +59,33 @@ clean: ## Clean the workspace .PHONY: help help: ## Show help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +.PHONY: inspec-check +inspec-check: ## Validate inspec profiles + docker run $(DOCKER_FLAGS) \ + --rm \ + -v "$(CURDIR)":/workspace \ + -w="/workspace" \ + chef/inspec check \ + --chef-license=accept \ + test/inspec/super-linter + +SUPER_LINTER_TEST_CONTAINER_NAME := "super-linter-test" + +.PHONY: inspec +inspec: inspec-check ## Run InSpec tests + DOCKER_CONTAINER_STATE="$$(docker inspect --format "{{.State.Running}}" "$(SUPER_LINTER_TEST_CONTAINER_NAME)" 2>/dev/null || echo "")"; \ + if [ "$$DOCKER_CONTAINER_STATE" = "true" ]; then docker kill "$(SUPER_LINTER_TEST_CONTAINER_NAME)"; fi && \ + SUPER_LINTER_TEST_CONTAINER_ID="$$(docker run -d --name "$(SUPER_LINTER_TEST_CONTAINER_NAME)" --rm -it --entrypoint /bin/ash ghcr.io/github/super-linter -c "while true; do sleep 1; done")" \ + && docker run $(DOCKER_FLAGS) \ + --rm \ + -v "$(CURDIR)":/workspace \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -w="/workspace" \ + chef/inspec exec test/inspec/super-linter\ + --chef-license=accept \ + --diagnose \ + --log-level=debug \ + -t "docker://$${SUPER_LINTER_TEST_CONTAINER_ID}" \ + && docker ps \ + && docker kill "$(SUPER_LINTER_TEST_CONTAINER_NAME)" diff --git a/lib/functions/linterVersions.sh b/lib/functions/linterVersions.sh index 5d51a8f1..227eba35 100755 --- a/lib/functions/linterVersions.sh +++ b/lib/functions/linterVersions.sh @@ -68,7 +68,7 @@ BuildLinterVersions() { if [[ ${LINTER} == "arm-ttk" ]]; then # Need specific command for ARM GET_VERSION_CMD="$(grep -iE 'version' "/usr/bin/arm-ttk" | xargs 2>&1)" - elif [[ ${LINTER} == "protolint" ]] || [[ ${LINTER} == "editorconfig-checker" ]] || [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]]; then + elif [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]]; then # Need specific command for Protolint and editorconfig-checker GET_VERSION_CMD="$(echo "--version not supported")" elif [[ ${LINTER} == "jsonlint" ]]; then @@ -89,6 +89,10 @@ BuildLinterVersions() { GET_VERSION_CMD="$(java -jar "/usr/bin/${LINTER}" --version 2>&1)" elif [[ ${LINTER} == "clippy" ]]; then GET_VERSION_CMD="$(cargo-clippy --version 2>&1)" + elif [[ ${LINTER} == "protolint" ]]; then + GET_VERSION_CMD="$(${LINTER} version)" + elif [[ ${LINTER} == "editorconfig-checker" ]]; then + GET_VERSION_CMD="$(${LINTER} -version)" else # Standard version command GET_VERSION_CMD="$("${LINTER}" --version 2>&1)" diff --git a/lib/linter.sh b/lib/linter.sh index 00c31d72..4eb0f040 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -323,22 +323,6 @@ for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do eval "${FILE_ARRAY_VARIABLE_NAME}=()" done -##################################### -# Validate we have linter installed # -##################################### -for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do - LINTER_NAME="${LINTER_NAMES_ARRAY["${LANGUAGE}"]}" - debug "Checking if linter with name ${LINTER_NAME} for the ${LANGUAGE} language is available..." - - if ! command -v "${LINTER_NAME}" 1 &>/dev/null 2>&1; then - # Failed - fatal "Failed to find [${LINTER_NAME}] in system!" - else - # Success - debug "Successfully found binary for ${F[W]}[${LINTER_NAME}]${F[B]}." - fi -done - ################################################################################ ########################## FUNCTIONS BELOW ##################################### ################################################################################ diff --git a/test/inspec/inspec.lock b/test/inspec/inspec.lock new file mode 100644 index 00000000..e687b9b4 --- /dev/null +++ b/test/inspec/inspec.lock @@ -0,0 +1,3 @@ +--- +lockfile_version: 1 +depends: [] diff --git a/test/inspec/super-linter/controls/super_linter.rb b/test/inspec/super-linter/controls/super_linter.rb new file mode 100644 index 00000000..0b8527cf --- /dev/null +++ b/test/inspec/super-linter/controls/super_linter.rb @@ -0,0 +1,198 @@ +# frozen_string_literal: true + +################################################## +# Check to see all system packages are installed # +################################################## +control "super-linter-installed-packages" do + impact 1 + title "Super-Linter installed packages check" + desc "Check that packages that Super-Linter needs are installed." + + packages = [ + "bash", + "coreutils", + "curl", + "gcc", + "git-lfs", + "git", + "glibc", + "gnupg", + "go", + "icu-libs", + "jq", + "krb5-libs", + "libc-dev", + "libcurl", + "libffi-dev", + "libgcc", + "libintl", + "libssl1.1", + "libstdc++", + "libxml2-dev", + "libxml2-utils", + "linux-headers", + "lttng-ust-dev", + "make", + "musl-dev", + "npm", + "nodejs-current", + "openjdk8-jre", + "openssl-dev", + "perl-dev", + "perl", + "py3-setuptools", + "python3-dev", + "rakudo", + "R-dev", + "R-doc", + "R", + "readline-dev", + "ruby-bundler", + "ruby-dev", + "ruby-rdoc", + "ruby", + "rustup", + "zef", + "zlib-dev", + "zlib" + ] + + packages.each do |item| + describe package(item) do + it { should be_installed } + end + end + +end + +########################################### +# Check to see all binaries are installed # +########################################### +control "super-linter-installed-commands" do + impact 1 + title "Super-Linter installed commands check" + desc "Check that commands that Super-Linter needs are installed." + + default_version_option = "--version" + default_version_expected_exit_status = 0 + default_expected_stdout_regex = /(.*?)/s + + linters = [ + { linter_name: "ansible-lint"}, + { linter_name: "arm-ttk", version_command: "grep -iE 'version' '/usr/bin/arm-ttk' | xargs"}, + { linter_name: "asl-validator"}, + { linter_name: "bash-exec", expected_exit_status: 1}, # expect a return code = 1 because this linter doesn't support a "get linter version" command + { linter_name: "black"}, + { linter_name: "cfn-lint"}, + { linter_name: "checkstyle", version_command: "java -jar /usr/bin/checkstyle --version"}, + { linter_name: "chktex"}, + { linter_name: "clippy", linter_command: "clippy", version_command: "cargo-clippy --version"}, + { linter_name: "clj-kondo"}, + { linter_name: "coffeelint"}, + { linter_name: "dart"}, + { linter_name: "dockerfilelint"}, + { linter_name: "dotnet-format"}, + { linter_name: "dotenv-linter"}, + { linter_name: "editorconfig-checker", version_option: "-version"}, + { linter_name: "eslint"}, + { linter_name: "flake8"}, + { linter_name: "gherkin-lint", expected_exit_status: 1}, # expect a return code = 1 because this linter doesn't support a "get linter version" command + { linter_name: "golangci-lint"}, + { linter_name: "hadolint"}, + { linter_name: "htmlhint"}, + { linter_name: "isort"}, + { linter_name: "jscpd"}, + { linter_name: "jsonlint", expected_exit_status: 1, expected_stdout_regex: /\d+\.\d+\.\d+/}, + { linter_name: "ktlint"}, + { linter_name: "kubeval"}, + { linter_name: "lua", version_option: "-v"}, + { linter_name: "markdownlint"}, + { linter_name: "mypy"}, + { linter_name: "npm-groovy-lint"}, + { linter_name: "perl"}, + { linter_name: "php"}, + { linter_name: "phpcs"}, + { linter_name: "phpstan"}, + { linter_name: "protolint", version_option: "version"}, + { linter_name: "psalm"}, + { linter_name: "pwsh"}, + { linter_name: "pylint"}, + { linter_name: "R", version_command: "R --slave -e \"r_ver <- R.Version()\\$version.string; \ + lintr_ver <- packageVersion('lintr'); \ + glue::glue('lintr { lintr_ver } on { r_ver }')\""}, + { linter_name: "raku"}, + { linter_name: "rubocop"}, + { linter_name: "rustfmt"}, + { linter_name: "shellcheck"}, + { linter_name: "shfmt"}, + { linter_name: "snakefmt"}, + { linter_name: "snakemake"}, + { linter_name: "spectral"}, + { linter_name: "sql-lint"}, + { linter_name: "standard"}, + { linter_name: "stylelint"}, + { linter_name: "tekton-lint"}, + { linter_name: "terragrunt"}, + { linter_name: "terrascan", version_option: "version"}, + { linter_name: "tflint"}, + { linter_name: "xmllint"}, + { linter_name: "yamllint"}, + ] + + linters.each do |linter| + # If we didn't specify a linter command, use the linter name as a linter + # command because the vast majority of linters have name == command + if(linter.key?(:linter_command)) + linter_command = linter[:linter_command] + else + linter_command = linter[:linter_name] + end + + describe command("command -v #{linter_command}") do + its("exit_status") { should eq 0 } + end + + # A few linters have a command that it's different than linter_command + if(linter.key?(:version_command)) + version_command = linter[:version_command] + else + # Check if the linter needs an option that is different from the one that + # the vast majority of linters use to get the version + if(linter.key?(:version_option)) + version_option = linter[:version_option] + else + version_option = default_version_option + end + + version_command = "#{linter_command} #{version_option}" + + if(linter.key?(:expected_exit_status)) + expected_exit_status = linter[:expected_exit_status] + else + expected_exit_status = default_version_expected_exit_status + end + + if(linter.key?(:expected_stdout_regex)) + expected_stdout_regex = linter[:expected_stdout_regex] + else + expected_stdout_regex = default_expected_stdout_regex + end + + ########################################################## + # Being able to run the command `linter --version` helps # + # achieve that the linter is installed, ini PATH, and # + # has the libraries needed to be able to basically run # + ########################################################## + describe command(version_command) do + its("exit_status") { should eq expected_exit_status } + its("stdout") { should match (expected_stdout_regex) } + end + end + end +end + +################################### +# Linters with no version command # +# protolint editorconfig-checker # +# bash-exec gherkin-lint # +################################### diff --git a/test/inspec/super-linter/inspec.lock b/test/inspec/super-linter/inspec.lock new file mode 100644 index 00000000..e687b9b4 --- /dev/null +++ b/test/inspec/super-linter/inspec.lock @@ -0,0 +1,3 @@ +--- +lockfile_version: 1 +depends: [] diff --git a/test/inspec/super-linter/inspec.yml b/test/inspec/super-linter/inspec.yml new file mode 100644 index 00000000..a5ecc16e --- /dev/null +++ b/test/inspec/super-linter/inspec.yml @@ -0,0 +1,11 @@ +--- +name: super-linter +title: super-linter InSpec Profile +maintainer: GitHub DevOps +copyright: GitHub +copyright_email: github_devops@github.com +license: Apache-2.0 +summary: InSpec profile to validate super-linter +version: 1.0.0 +supports: + platform: linux