Implement a test suite with InSpec (#1466)

* Implement a test suite with InSpec

* make ruby happy

* adding file

* Fix inspec and add make target

* Run inspec

* adding binaries

* make ruby happy

* fix linter order

* cleanup

* adding version check

* adding notes

* cleanup

* cleanup

* fixed r test

* fixed copy paste

* dynamic tests

* fix hash

* fix notation

* docker ps

* Fix makefile

* Fix makefile

Co-authored-by: Admiral Awkbar <admiralawkbar@github.com>

Makefile

@@ -4,7 +4,7 @@
 all: info test ## Run all targets.
 
 .PHONY: test
-test: info clean kcov prepare-test-reports ## Run tests
+test: info clean inspec kcov prepare-test-reports ## Run tests
 
 # if this session isn't interactive, then we don't want to allocate a
 # TTY, which would fail, but if it is interactive, we do want to attach
@@ -19,7 +19,8 @@ info: ## Gather information about the runtime environment
 	echo "whoami: $$(whoami)"; \
 	echo "pwd: $$(pwd)"; \
 	echo "ls -ahl: $$(ls -ahl)"; \
-	docker images
+	docker images; \
+	docker ps
 
 .PHONY: kcov
 kcov: ## Run kcov
@@ -58,3 +59,33 @@ clean: ## Clean the workspace
 .PHONY: help
 help: ## Show help
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+.PHONY: inspec-check
+inspec-check: ## Validate inspec profiles
+	docker run $(DOCKER_FLAGS) \
+		--rm \
+		-v "$(CURDIR)":/workspace \
+		-w="/workspace" \
+		chef/inspec check \
+		--chef-license=accept \
+		test/inspec/super-linter
+
+SUPER_LINTER_TEST_CONTAINER_NAME := "super-linter-test"
+
+.PHONY: inspec
+inspec: inspec-check ## Run InSpec tests
+	DOCKER_CONTAINER_STATE="$$(docker inspect --format "{{.State.Running}}" "$(SUPER_LINTER_TEST_CONTAINER_NAME)" 2>/dev/null || echo "")"; \
+	if [ "$$DOCKER_CONTAINER_STATE" = "true" ]; then docker kill "$(SUPER_LINTER_TEST_CONTAINER_NAME)"; fi && \
+	SUPER_LINTER_TEST_CONTAINER_ID="$$(docker run -d --name "$(SUPER_LINTER_TEST_CONTAINER_NAME)" --rm -it --entrypoint /bin/ash ghcr.io/github/super-linter -c "while true; do sleep 1; done")" \
+	&& docker run $(DOCKER_FLAGS) \
+		--rm \
+		-v "$(CURDIR)":/workspace \
+		-v /var/run/docker.sock:/var/run/docker.sock \
+		-w="/workspace" \
+		chef/inspec exec test/inspec/super-linter\
+		--chef-license=accept \
+		--diagnose \
+		--log-level=debug \
+		-t "docker://$${SUPER_LINTER_TEST_CONTAINER_ID}" \
+	&& docker ps \
+	&& docker kill "$(SUPER_LINTER_TEST_CONTAINER_NAME)"

lib/functions/linterVersions.sh

@@ -68,7 +68,7 @@ BuildLinterVersions() {
       if [[ ${LINTER} == "arm-ttk" ]]; then
         # Need specific command for ARM
         GET_VERSION_CMD="$(grep -iE 'version' "/usr/bin/arm-ttk" | xargs 2>&1)"
-      elif [[ ${LINTER} == "protolint" ]] || [[ ${LINTER} == "editorconfig-checker" ]] || [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]]; then
+      elif [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]]; then
         # Need specific command for Protolint and editorconfig-checker
         GET_VERSION_CMD="$(echo "--version not supported")"
       elif [[ ${LINTER} == "jsonlint" ]]; then
@@ -89,6 +89,10 @@ BuildLinterVersions() {
         GET_VERSION_CMD="$(java -jar "/usr/bin/${LINTER}" --version 2>&1)"
       elif [[ ${LINTER} == "clippy" ]]; then
         GET_VERSION_CMD="$(cargo-clippy --version 2>&1)"
+      elif [[ ${LINTER} == "protolint" ]]; then
+        GET_VERSION_CMD="$(${LINTER} version)"
+      elif [[ ${LINTER} == "editorconfig-checker" ]]; then
+        GET_VERSION_CMD="$(${LINTER} -version)"
       else
         # Standard version command
         GET_VERSION_CMD="$("${LINTER}" --version 2>&1)"

lib/linter.sh

@@ -323,22 +323,6 @@ for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do
   eval "${FILE_ARRAY_VARIABLE_NAME}=()"
 done
 
-#####################################
-# Validate we have linter installed #
-#####################################
-for LANGUAGE in "${LANGUAGE_ARRAY[@]}"; do
-  LINTER_NAME="${LINTER_NAMES_ARRAY["${LANGUAGE}"]}"
-  debug "Checking if linter with name ${LINTER_NAME} for the ${LANGUAGE} language is available..."
-
-  if ! command -v "${LINTER_NAME}" 1 &>/dev/null 2>&1; then
-    # Failed
-    fatal "Failed to find [${LINTER_NAME}] in system!"
-  else
-    # Success
-    debug "Successfully found binary for ${F[W]}[${LINTER_NAME}]${F[B]}."
-  fi
-done
-
 ################################################################################
 ########################## FUNCTIONS BELOW #####################################
 ################################################################################

test/inspec/inspec.lock

@@ -0,0 +1,3 @@
+---
+lockfile_version: 1
+depends: []

test/inspec/super-linter/controls/super_linter.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+##################################################
+# Check to see all system packages are installed #
+##################################################
+control "super-linter-installed-packages" do
+  impact 1
+  title "Super-Linter installed packages check"
+  desc "Check that packages that Super-Linter needs are installed."
+
+  packages = [
+    "bash",
+    "coreutils",
+    "curl",
+    "gcc",
+    "git-lfs",
+    "git",
+    "glibc",
+    "gnupg",
+    "go",
+    "icu-libs",
+    "jq",
+    "krb5-libs",
+    "libc-dev",
+    "libcurl",
+    "libffi-dev",
+    "libgcc",
+    "libintl",
+    "libssl1.1",
+    "libstdc++",
+    "libxml2-dev",
+    "libxml2-utils",
+    "linux-headers",
+    "lttng-ust-dev",
+    "make",
+    "musl-dev",
+    "npm",
+    "nodejs-current",
+    "openjdk8-jre",
+    "openssl-dev",
+    "perl-dev",
+    "perl",
+    "py3-setuptools",
+    "python3-dev",
+    "rakudo",
+    "R-dev",
+    "R-doc",
+    "R",
+    "readline-dev",
+    "ruby-bundler",
+    "ruby-dev",
+    "ruby-rdoc",
+    "ruby",
+    "rustup",
+    "zef",
+    "zlib-dev",
+    "zlib"
+  ]
+
+  packages.each do |item|
+    describe package(item) do
+      it { should be_installed }
+    end
+  end
+
+end
+
+###########################################
+# Check to see all binaries are installed #
+###########################################
+control "super-linter-installed-commands" do
+  impact 1
+  title "Super-Linter installed commands check"
+  desc "Check that commands that Super-Linter needs are installed."
+
+  default_version_option = "--version"
+  default_version_expected_exit_status = 0
+  default_expected_stdout_regex = /(.*?)/s
+
+  linters = [
+    { linter_name: "ansible-lint"},
+    { linter_name: "arm-ttk", version_command: "grep -iE 'version' '/usr/bin/arm-ttk' | xargs"},
+    { linter_name: "asl-validator"},
+    { linter_name: "bash-exec", expected_exit_status: 1}, # expect a return code = 1 because this linter doesn't support a "get linter version" command
+    { linter_name: "black"},
+    { linter_name: "cfn-lint"},
+    { linter_name: "checkstyle", version_command: "java -jar /usr/bin/checkstyle --version"},
+    { linter_name: "chktex"},
+    { linter_name: "clippy", linter_command: "clippy", version_command: "cargo-clippy --version"},
+    { linter_name: "clj-kondo"},
+    { linter_name: "coffeelint"},
+    { linter_name: "dart"},
+    { linter_name: "dockerfilelint"},
+    { linter_name: "dotnet-format"},
+    { linter_name: "dotenv-linter"},
+    { linter_name: "editorconfig-checker", version_option: "-version"},
+    { linter_name: "eslint"},
+    { linter_name: "flake8"},
+    { linter_name: "gherkin-lint", expected_exit_status: 1}, # expect a return code = 1 because this linter doesn't support a "get linter version" command
+    { linter_name: "golangci-lint"},
+    { linter_name: "hadolint"},
+    { linter_name: "htmlhint"},
+    { linter_name: "isort"},
+    { linter_name: "jscpd"},
+    { linter_name: "jsonlint", expected_exit_status: 1, expected_stdout_regex: /\d+\.\d+\.\d+/},
+    { linter_name: "ktlint"},
+    { linter_name: "kubeval"},
+    { linter_name: "lua", version_option: "-v"},
+    { linter_name: "markdownlint"},
+    { linter_name: "mypy"},
+    { linter_name: "npm-groovy-lint"},
+    { linter_name: "perl"},
+    { linter_name: "php"},
+    { linter_name: "phpcs"},
+    { linter_name: "phpstan"},
+    { linter_name: "protolint", version_option: "version"},
+    { linter_name: "psalm"},
+    { linter_name: "pwsh"},
+    { linter_name: "pylint"},
+    { linter_name: "R", version_command: "R --slave -e \"r_ver <- R.Version()\\$version.string; \
+            lintr_ver <- packageVersion('lintr'); \
+            glue::glue('lintr { lintr_ver } on { r_ver }')\""},
+    { linter_name: "raku"},
+    { linter_name: "rubocop"},
+    { linter_name: "rustfmt"},
+    { linter_name: "shellcheck"},
+    { linter_name: "shfmt"},
+    { linter_name: "snakefmt"},
+    { linter_name: "snakemake"},
+    { linter_name: "spectral"},
+    { linter_name: "sql-lint"},
+    { linter_name: "standard"},
+    { linter_name: "stylelint"},
+    { linter_name: "tekton-lint"},
+    { linter_name: "terragrunt"},
+    { linter_name: "terrascan", version_option: "version"},
+    { linter_name: "tflint"},
+    { linter_name: "xmllint"},
+    { linter_name: "yamllint"},
+  ]
+
+  linters.each do |linter|
+    # If we didn't specify a linter command, use the linter name as a linter
+    # command because the vast majority of linters have name == command
+    if(linter.key?(:linter_command))
+      linter_command = linter[:linter_command]
+    else
+      linter_command = linter[:linter_name]
+    end
+
+    describe command("command -v #{linter_command}") do
+      its("exit_status") { should eq 0 }
+    end
+
+    # A few linters have a command that it's different than linter_command
+    if(linter.key?(:version_command))
+      version_command = linter[:version_command]
+    else
+      # Check if the linter needs an option that is different from the one that
+      # the vast majority of linters use to get the version
+      if(linter.key?(:version_option))
+        version_option = linter[:version_option]
+      else
+        version_option = default_version_option
+      end
+
+      version_command = "#{linter_command} #{version_option}"
+
+      if(linter.key?(:expected_exit_status))
+        expected_exit_status = linter[:expected_exit_status]
+      else
+        expected_exit_status = default_version_expected_exit_status
+      end
+
+      if(linter.key?(:expected_stdout_regex))
+        expected_stdout_regex = linter[:expected_stdout_regex]
+      else
+        expected_stdout_regex = default_expected_stdout_regex
+      end
+
+      ##########################################################
+      # Being able to run the command `linter --version` helps #
+      # achieve that the linter is installed, ini PATH, and    #
+      # has the libraries needed to be able to basically run   #
+      ##########################################################
+      describe command(version_command) do
+        its("exit_status") { should eq expected_exit_status }
+        its("stdout") { should match (expected_stdout_regex) }
+      end
+    end
+  end
+end
+
+###################################
+# Linters with no version command #
+# protolint editorconfig-checker  #
+# bash-exec gherkin-lint          #
+###################################

test/inspec/super-linter/inspec.lock

@@ -0,0 +1,3 @@
+---
+lockfile_version: 1
+depends: []

test/inspec/super-linter/inspec.yml

@@ -0,0 +1,11 @@
+---
+name: super-linter
+title: super-linter InSpec Profile
+maintainer: GitHub DevOps
+copyright: GitHub
+copyright_email: github_devops@github.com
+license: Apache-2.0
+summary: InSpec profile to validate super-linter
+version: 1.0.0
+supports:
+  platform: linux