Implement a test suite with InSpec (#1466)

* Implement a test suite with InSpec

* make ruby happy

* adding file

* Fix inspec and add make target

* Run inspec

* adding binaries

* make ruby happy

* fix linter order

* cleanup

* adding version check

* adding notes

* cleanup

* cleanup

* fixed r test

* fixed copy paste

* dynamic tests

* fix hash

* fix notation

* docker ps

* Fix makefile

* Fix makefile

Co-authored-by: Admiral Awkbar <admiralawkbar@github.com>
This commit is contained in:
Marco Ferrari 2021-04-30 18:29:15 +02:00 committed by GitHub
parent 80b7d1377a
commit 0c8db849aa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 253 additions and 19 deletions

View file

@ -4,7 +4,7 @@
all: info test ## Run all targets. all: info test ## Run all targets.
.PHONY: test .PHONY: test
test: info clean kcov prepare-test-reports ## Run tests test: info clean inspec kcov prepare-test-reports ## Run tests
# if this session isn't interactive, then we don't want to allocate a # if this session isn't interactive, then we don't want to allocate a
# TTY, which would fail, but if it is interactive, we do want to attach # TTY, which would fail, but if it is interactive, we do want to attach
@ -19,7 +19,8 @@ info: ## Gather information about the runtime environment
echo "whoami: $$(whoami)"; \ echo "whoami: $$(whoami)"; \
echo "pwd: $$(pwd)"; \ echo "pwd: $$(pwd)"; \
echo "ls -ahl: $$(ls -ahl)"; \ echo "ls -ahl: $$(ls -ahl)"; \
docker images docker images; \
docker ps
.PHONY: kcov .PHONY: kcov
kcov: ## Run kcov kcov: ## Run kcov
@ -58,3 +59,33 @@ clean: ## Clean the workspace
.PHONY: help .PHONY: help
help: ## Show help help: ## Show help
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
.PHONY: inspec-check
inspec-check: ## Validate inspec profiles
docker run $(DOCKER_FLAGS) \
--rm \
-v "$(CURDIR)":/workspace \
-w="/workspace" \
chef/inspec check \
--chef-license=accept \
test/inspec/super-linter
SUPER_LINTER_TEST_CONTAINER_NAME := "super-linter-test"
.PHONY: inspec
inspec: inspec-check ## Run InSpec tests
DOCKER_CONTAINER_STATE="$$(docker inspect --format "{{.State.Running}}" "$(SUPER_LINTER_TEST_CONTAINER_NAME)" 2>/dev/null || echo "")"; \
if [ "$$DOCKER_CONTAINER_STATE" = "true" ]; then docker kill "$(SUPER_LINTER_TEST_CONTAINER_NAME)"; fi && \
SUPER_LINTER_TEST_CONTAINER_ID="$$(docker run -d --name "$(SUPER_LINTER_TEST_CONTAINER_NAME)" --rm -it --entrypoint /bin/ash ghcr.io/github/super-linter -c "while true; do sleep 1; done")" \
&& docker run $(DOCKER_FLAGS) \
--rm \
-v "$(CURDIR)":/workspace \
-v /var/run/docker.sock:/var/run/docker.sock \
-w="/workspace" \
chef/inspec exec test/inspec/super-linter\
--chef-license=accept \
--diagnose \
--log-level=debug \
-t "docker://$${SUPER_LINTER_TEST_CONTAINER_ID}" \
&& docker ps \
&& docker kill "$(SUPER_LINTER_TEST_CONTAINER_NAME)"

View file

@ -68,7 +68,7 @@ BuildLinterVersions() {
if [[ ${LINTER} == "arm-ttk" ]]; then if [[ ${LINTER} == "arm-ttk" ]]; then
# Need specific command for ARM # Need specific command for ARM
GET_VERSION_CMD="$(grep -iE 'version' "/usr/bin/arm-ttk" | xargs 2>&1)" GET_VERSION_CMD="$(grep -iE 'version' "/usr/bin/arm-ttk" | xargs 2>&1)"
elif [[ ${LINTER} == "protolint" ]] || [[ ${LINTER} == "editorconfig-checker" ]] || [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]]; then elif [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]]; then
# Need specific command for Protolint and editorconfig-checker # Need specific command for Protolint and editorconfig-checker
GET_VERSION_CMD="$(echo "--version not supported")" GET_VERSION_CMD="$(echo "--version not supported")"
elif [[ ${LINTER} == "jsonlint" ]]; then elif [[ ${LINTER} == "jsonlint" ]]; then
@ -89,6 +89,10 @@ BuildLinterVersions() {
GET_VERSION_CMD="$(java -jar "/usr/bin/${LINTER}" --version 2>&1)" GET_VERSION_CMD="$(java -jar "/usr/bin/${LINTER}" --version 2>&1)"
elif [[ ${LINTER} == "clippy" ]]; then elif [[ ${LINTER} == "clippy" ]]; then
GET_VERSION_CMD="$(cargo-clippy --version 2>&1)" GET_VERSION_CMD="$(cargo-clippy --version 2>&1)"
elif [[ ${LINTER} == "protolint" ]]; then
GET_VERSION_CMD="$(${LINTER} version)"
elif [[ ${LINTER} == "editorconfig-checker" ]]; then
GET_VERSION_CMD="$(${LINTER} -version)"
else else
# Standard version command # Standard version command
GET_VERSION_CMD="$("${LINTER}" --version 2>&1)" GET_VERSION_CMD="$("${LINTER}" --version 2>&1)"

View file

@ -323,22 +323,6 @@ for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do
eval "${FILE_ARRAY_VARIABLE_NAME}=()" eval "${FILE_ARRAY_VARIABLE_NAME}=()"
done done
#####################################
# Validate we have linter installed #
#####################################
for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do
LINTER_NAME="${LINTER_NAMES_ARRAY["${LANGUAGE}"]}"
debug "Checking if linter with name ${LINTER_NAME} for the ${LANGUAGE} language is available..."
if ! command -v "${LINTER_NAME}" 1 &>/dev/null 2>&1; then
# Failed
fatal "Failed to find [${LINTER_NAME}] in system!"
else
# Success
debug "Successfully found binary for ${F[W]}[${LINTER_NAME}]${F[B]}."
fi
done
################################################################################ ################################################################################
########################## FUNCTIONS BELOW ##################################### ########################## FUNCTIONS BELOW #####################################
################################################################################ ################################################################################

3
test/inspec/inspec.lock Normal file
View file

@ -0,0 +1,3 @@
---
lockfile_version: 1
depends: []

View file

@ -0,0 +1,198 @@
# frozen_string_literal: true
##################################################
# Check to see all system packages are installed #
##################################################
control "super-linter-installed-packages" do
impact 1
title "Super-Linter installed packages check"
desc "Check that packages that Super-Linter needs are installed."
packages = [
"bash",
"coreutils",
"curl",
"gcc",
"git-lfs",
"git",
"glibc",
"gnupg",
"go",
"icu-libs",
"jq",
"krb5-libs",
"libc-dev",
"libcurl",
"libffi-dev",
"libgcc",
"libintl",
"libssl1.1",
"libstdc++",
"libxml2-dev",
"libxml2-utils",
"linux-headers",
"lttng-ust-dev",
"make",
"musl-dev",
"npm",
"nodejs-current",
"openjdk8-jre",
"openssl-dev",
"perl-dev",
"perl",
"py3-setuptools",
"python3-dev",
"rakudo",
"R-dev",
"R-doc",
"R",
"readline-dev",
"ruby-bundler",
"ruby-dev",
"ruby-rdoc",
"ruby",
"rustup",
"zef",
"zlib-dev",
"zlib"
]
packages.each do |item|
describe package(item) do
it { should be_installed }
end
end
end
###########################################
# Check to see all binaries are installed #
###########################################
control "super-linter-installed-commands" do
impact 1
title "Super-Linter installed commands check"
desc "Check that commands that Super-Linter needs are installed."
default_version_option = "--version"
default_version_expected_exit_status = 0
default_expected_stdout_regex = /(.*?)/s
linters = [
{ linter_name: "ansible-lint"},
{ linter_name: "arm-ttk", version_command: "grep -iE 'version' '/usr/bin/arm-ttk' | xargs"},
{ linter_name: "asl-validator"},
{ linter_name: "bash-exec", expected_exit_status: 1}, # expect a return code = 1 because this linter doesn't support a "get linter version" command
{ linter_name: "black"},
{ linter_name: "cfn-lint"},
{ linter_name: "checkstyle", version_command: "java -jar /usr/bin/checkstyle --version"},
{ linter_name: "chktex"},
{ linter_name: "clippy", linter_command: "clippy", version_command: "cargo-clippy --version"},
{ linter_name: "clj-kondo"},
{ linter_name: "coffeelint"},
{ linter_name: "dart"},
{ linter_name: "dockerfilelint"},
{ linter_name: "dotnet-format"},
{ linter_name: "dotenv-linter"},
{ linter_name: "editorconfig-checker", version_option: "-version"},
{ linter_name: "eslint"},
{ linter_name: "flake8"},
{ linter_name: "gherkin-lint", expected_exit_status: 1}, # expect a return code = 1 because this linter doesn't support a "get linter version" command
{ linter_name: "golangci-lint"},
{ linter_name: "hadolint"},
{ linter_name: "htmlhint"},
{ linter_name: "isort"},
{ linter_name: "jscpd"},
{ linter_name: "jsonlint", expected_exit_status: 1, expected_stdout_regex: /\d+\.\d+\.\d+/},
{ linter_name: "ktlint"},
{ linter_name: "kubeval"},
{ linter_name: "lua", version_option: "-v"},
{ linter_name: "markdownlint"},
{ linter_name: "mypy"},
{ linter_name: "npm-groovy-lint"},
{ linter_name: "perl"},
{ linter_name: "php"},
{ linter_name: "phpcs"},
{ linter_name: "phpstan"},
{ linter_name: "protolint", version_option: "version"},
{ linter_name: "psalm"},
{ linter_name: "pwsh"},
{ linter_name: "pylint"},
{ linter_name: "R", version_command: "R --slave -e \"r_ver <- R.Version()\\$version.string; \
lintr_ver <- packageVersion('lintr'); \
glue::glue('lintr { lintr_ver } on { r_ver }')\""},
{ linter_name: "raku"},
{ linter_name: "rubocop"},
{ linter_name: "rustfmt"},
{ linter_name: "shellcheck"},
{ linter_name: "shfmt"},
{ linter_name: "snakefmt"},
{ linter_name: "snakemake"},
{ linter_name: "spectral"},
{ linter_name: "sql-lint"},
{ linter_name: "standard"},
{ linter_name: "stylelint"},
{ linter_name: "tekton-lint"},
{ linter_name: "terragrunt"},
{ linter_name: "terrascan", version_option: "version"},
{ linter_name: "tflint"},
{ linter_name: "xmllint"},
{ linter_name: "yamllint"},
]
linters.each do |linter|
# If we didn't specify a linter command, use the linter name as a linter
# command because the vast majority of linters have name == command
if(linter.key?(:linter_command))
linter_command = linter[:linter_command]
else
linter_command = linter[:linter_name]
end
describe command("command -v #{linter_command}") do
its("exit_status") { should eq 0 }
end
# A few linters have a command that it's different than linter_command
if(linter.key?(:version_command))
version_command = linter[:version_command]
else
# Check if the linter needs an option that is different from the one that
# the vast majority of linters use to get the version
if(linter.key?(:version_option))
version_option = linter[:version_option]
else
version_option = default_version_option
end
version_command = "#{linter_command} #{version_option}"
if(linter.key?(:expected_exit_status))
expected_exit_status = linter[:expected_exit_status]
else
expected_exit_status = default_version_expected_exit_status
end
if(linter.key?(:expected_stdout_regex))
expected_stdout_regex = linter[:expected_stdout_regex]
else
expected_stdout_regex = default_expected_stdout_regex
end
##########################################################
# Being able to run the command `linter --version` helps #
# achieve that the linter is installed, ini PATH, and #
# has the libraries needed to be able to basically run #
##########################################################
describe command(version_command) do
its("exit_status") { should eq expected_exit_status }
its("stdout") { should match (expected_stdout_regex) }
end
end
end
end
###################################
# Linters with no version command #
# protolint editorconfig-checker #
# bash-exec gherkin-lint #
###################################

View file

@ -0,0 +1,3 @@
---
lockfile_version: 1
depends: []

View file

@ -0,0 +1,11 @@
---
name: super-linter
title: super-linter InSpec Profile
maintainer: GitHub DevOps
copyright: GitHub
copyright_email: github_devops@github.com
license: Apache-2.0
summary: InSpec profile to validate super-linter
version: 1.0.0
supports:
platform: linux