[tests] validate TAP format

.automation/test/ansible/reports/expected-ANSIBLE.tap

@@ -0,0 +1,7 @@
+TAP version 13
+1..2
+not ok 1 - ansible_bad_1.yml
+  ---
+  message: Traceback (most recent call last) \n  File "/usr/bin/ansible-lint", line 11, in <module>\n    load_entry_point('ansible-lint==4.2.0', 'console_scripts', 'ansible-lint')()\n  File "/usr/lib/python3.8/site-packages/ansiblelint/__main__.py", line 187, in main\n    matches.extend(runner.run())\n  File "/usr/lib/python3.8/site-packages/ansiblelint/__init__.py", line 267, in run\n    for child in ansiblelint.utils.find_children(arg, self.playbook_dir) \n  File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 163, in find_children\n    for child in play_children(basedir, item, playbook[1], playbook_dir) \n  File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 215, in play_children\n    return delegate_map[k](basedir, k, v, parent_type)\n  File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 246, in _taskshandlers_children\n    results.extend(_roles_children(basedir, k, [th['action'].get('name')],\n  File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 285, in _roles_children\n    results.extend(_look_for_role_files(basedir, role, main=main))\n  File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 330, in _look_for_role_files\n    role_path = _rolepath(basedir, role)\n  File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 299, in _rolepath\n    path_dwim(basedir, os.path.join('roles', role)),\n  File "/usr/lib/python3.8/posixpath.py", line 90, in join\n    genericpath._check_arg_types('join', a, *p)\n  File "/usr/lib/python3.8/genericpath.py", line 152, in _check_arg_types\n    raise TypeError(f'{funcname}() argument must be str, bytes, or '\nTypeError  join() argument must be str, bytes, or os.PathLike object, not 'NoneType'\n
+  ...
+ok 2 - ansible_good_1.yml

.automation/test/coffeescript/reports/expected-COFFEESCRIPT.tap

@@ -0,0 +1,7 @@
+TAP version 13
+1..2
+not ok 1 - coffeescript_bad_1.coffee
+  ---
+  message:   ✗ /tmp/lint/.automation/test/coffeescript/coffeescript_bad_1.coffee\n     ✗ #39  [stdin] 39 29  error  unmatched )\nmodule.exports = (robot) -> )\n                            ^. (coffeescript_error)\n\n✗ Lint! » 1 error and 0 warnings in 1 file\n
+  ...
+ok 2 - coffeescript_good_1.coffee

.automation/test/docker/reports/expected-DOCKER.tap

@@ -0,0 +1,7 @@
+TAP version 13
+1..2
+not ok 1 - Dockerfile
+  ---
+  message: \nFile    /tmp/lint/.automation/test/docker/bad/Dockerfile\nIssues  6\n\nLine 1  from node latest\nIssue  Category      Title                 Description\n    1  Clarity       Capitalize            For clarity and readability, all instructions in a Dockerfile\n                     Dockerfile            should be uppercase.\n                     Instructions          This is a convention adopted by most of the official images and\n                                           greatly improves readability in long Dockerfiles.  For an example\n                                           of\n                                           why this makes a difference, check out the current [redis\n                                           Dockerfile](https //github.com/docker-library/redis/blob/b375650fb6\n                                           9b7db819e90c0033433c705b28656e/3.0/Dockerfile)\n                                           and you should be able to easily see the instructions used.\n    2  Clarity       Base Image Latest     Base images should not use the latest tag.\n                     Tag\n\nLine 4  run mkdir -p /usr/src/app\nIssue  Category      Title                 Description\n    3  Clarity       Capitalize            For clarity and readability, all instructions in a Dockerfile\n                     Dockerfile            should be uppercase.\n                     Instructions          This is a convention adopted by most of the official images and\n                                           greatly improves readability in long Dockerfiles.  For an example\n                                           of\n                                           why this makes a difference, check out the current [redis\n                                           Dockerfile](https //github.com/docker-library/redis/blob/b375650fb6\n                                           9b7db819e90c0033433c705b28656e/3.0/Dockerfile)\n                                           and you should be able to easily see the instructions used.\n\nLine 8  copy package.json /usr/src/app/ /here/there\nIssue  Category      Title                 Description\n    4  Clarity       Capitalize            For clarity and readability, all instructions in a Dockerfile\n                     Dockerfile            should be uppercase.\n                     Instructions          This is a convention adopted by most of the official images and\n                                           greatly improves readability in long Dockerfiles.  For an example\n                                           of\n                                           why this makes a difference, check out the current [redis\n                                           Dockerfile](https //github.com/docker-library/redis/blob/b375650fb6\n                                           9b7db819e90c0033433c705b28656e/3.0/Dockerfile)\n                                           and you should be able to easily see the instructions used.\n\nLine 9  RUN sudo npm install\nIssue  Category      Title                 Description\n    5  Possible Bug  Use Of sudo Is Not    Use of `sudo` is not allowed in a Dockerfile.  From the official\n                     Allowed               document [Best practices for writing\n                                           Dockerfiles](https //docs.docker.com/engine/userguide/eng-image/doc\n                                           kerfile_best-practices/) \n                                           > You should avoid installing or using `sudo` since it has\n                                           unpredictable TTY and signal-forwarding behavior that can cause\n                                           more problems than it solves.\n                                           > If you absolutely need functionality similar to `sudo` (e.g.,\n                                           initializing the daemon as root but running it as non-root), you\n                                           may be able to use `gosu`.\n\nLine 14  ENtrypoint /tmp/here.sh\nIssue  Category      Title                 Description\n    6  Clarity       Capitalize            For clarity and readability, all instructions in a Dockerfile\n                     Dockerfile            should be uppercase.\n                     Instructions          This is a convention adopted by most of the official images and\n                                           greatly improves readability in long Dockerfiles.  For an example\n                                           of\n                                           why this makes a difference, check out the current [redis\n                                           Dockerfile](https //github.com/docker-library/redis/blob/b375650fb6\n                                           9b7db819e90c0033433c705b28656e/3.0/Dockerfile)\n                                           and you should be able to easily see the instructions used.\n
+  ...
+ok 2 - Dockerfile

.automation/test/javascript/reports/expected-JAVASCRIPT_ES.tap

@@ -0,0 +1,7 @@
+TAP version 13
+1..2
+not ok 1 - javascript_bad_1.js
+  ---
+  message: \n/tmp/lint/.automation/test/javascript/javascript_bad_1.js\n  4 39  error  Parsing error  Unterminated regular expression literal\n\n✖ 1 problem (1 error, 0 warnings)\n
+  ...
+ok 2 - javascript_good_1.js

.automation/test/javascript/reports/expected-JAVASCRIPT_STANDARD.tap

@@ -0,0 +1,7 @@
+TAP version 13
+1..2
+not ok 1 - javascript_bad_1.js
+  ---
+  message: standard  Use JavaScript Standard Style (https //standardjs.com)\n  /tmp/lint/.automation/test/javascript/javascript_bad_1.js 4 40  Parsing error  Unterminated regular expression\n
+  ...
+ok 2 - javascript_good_1.js

.automation/test/json/reports/expected-JSON.tap

@@ -0,0 +1,7 @@
+TAP version 13
+1..2
+not ok 1 - json_bad_1.json
+  ---
+  message: Error  Parse error on line 6 \n...ng"  {    "level"  'ignore',    "space\n----------------------^\nExpecting 'STRING', 'NUMBER', 'NULL', 'TRUE', 'FALSE', '{', '[', got 'undefined'\n    at Object.parseError (/usr/lib/node_modules/jsonlint/lib/jsonlint.js 55 11)\n    at Object.parse (/usr/lib/node_modules/jsonlint/lib/jsonlint.js 132 22)\n    at parse (/usr/lib/node_modules/jsonlint/lib/cli.js 82 14)\n    at main (/usr/lib/node_modules/jsonlint/lib/cli.js 135 14)\n    at Object.<anonymous> (/usr/lib/node_modules/jsonlint/lib/cli.js 179 1)\n    at Module._compile (internal/modules/cjs/loader.js 1138 30)\n    at Object.Module._extensions..js (internal/modules/cjs/loader.js 1158 10)\n    at Module.load (internal/modules/cjs/loader.js 986 32)\n    at Function.Module._load (internal/modules/cjs/loader.js 879 14)\n    at Function.executeUserEntryPoint [as runMain] (internal/modules/run_main.js 71 12)\n
+  ...
+ok 2 - json_good_1.json

.automation/test/kotlin/reports/expected-KOTLIN.tap

@@ -0,0 +1,7 @@
+TAP version 13
+1..2
+not ok 1 - kotlin_bad_1.kt
+  ---
+  message: /tmp/lint/.automation/test/kotlin/kotlin_bad_1.kt 1 1  File must end with a newline (\n)\n/tmp/lint/.automation/test/kotlin/kotlin_bad_1.kt 2 20  Unnecessary semicolon\n/tmp/lint/.automation/test/kotlin/kotlin_bad_1.kt 3 16  Redundant curly braces\n/tmp/lint/.automation/test/kotlin/kotlin_bad_1.kt 3 27  Unnecessary semicolon\n/tmp/lint/.automation/test/kotlin/kotlin_bad_1.kt 5 15  Unnecessary semicolon\n
+  ...
+ok 2 - kotlint_good_1.kt

.automation/test/markdown/reports/expected-MARKDOWN.tap

@@ -0,0 +1,8 @@
+TAP version 13
+1..3
+ok 1 - README.md
+not ok 2 - markdown_bad_1.md
+  ---
+  message: /tmp/lint/.automation/test/markdown/markdown_bad_1.md 1 MD041/first-line-heading/first-line-h1 First line in file should be a top level heading [Context  "## Bad Markdown"]\n/tmp/lint/.automation/test/markdown/markdown_bad_1.md 5 MD001/heading-increment/header-increment Heading levels should only increment by one level at a time [Expected  h3; Actual  h6]\n/tmp/lint/.automation/test/markdown/markdown_bad_1.md 11 1 MD007/ul-indent Unordered list indentation [Expected  4; Actual  6]\n/tmp/lint/.automation/test/markdown/markdown_bad_1.md 13 MD040/fenced-code-language Fenced code blocks should have a language specified [Context  "```"]\n/tmp/lint/.automation/test/markdown/markdown_bad_1.md 20 16 MD034/no-bare-urls Bare URL used [Context  "https //github.com"]\n
+  ...
+ok 3 - markdown_good_1.md

.automation/test/perl/reports/expected-PERL.tap

@@ -0,0 +1,7 @@
+TAP version 13
+1..2
+not ok 1 - perl_bad_1.pl
+  ---
+  message: Global symbol "$here" requires explicit package name (did you forget to declare "my $here"?) at /tmp/lint/.automation/test/perl/perl_bad_1.pl line 37.\n/tmp/lint/.automation/test/perl/perl_bad_1.pl had compilation errors.\n
+  ...
+ok 2 - perl_good_1.pl

.automation/test/php/reports/expected-PHP.tap

@@ -0,0 +1,7 @@
+TAP version 13
+1..2
+not ok 1 - php_bad_1.php
+  ---
+  message: PHP Parse error   syntax error, unexpected 'pe98y' (T_STRING) in /tmp/lint/.automation/test/php/php_bad_1.php on line 3\nErrors parsing /tmp/lint/.automation/test/php/php_bad_1.php\n
+  ...
+ok 2 - php_good_1.php

.automation/test/python/reports/expected-PYTHON.tap

@@ -0,0 +1,7 @@
+TAP version 13
+1..2
+not ok 1 - python_bad_1.py
+  ---
+  message: ************* Module python_bad_1\npython/python_bad_1.py 15 24  E0001  invalid syntax (<unknown>, line 15) (syntax-error)\n
+  ...
+ok 2 - python_good_1.py

.automation/test/ruby/reports/expected-RUBY.tap

@@ -0,0 +1,7 @@
+TAP version 13
+1..2
+not ok 1 - ruby_bad_1.rb
+  ---
+  message: /usr/lib/ruby/gems/2.7.0/gems/rubocop-0.74.0/lib/rubocop/config_loader_resolver.rb 76  warning  Using the last argument as keyword parameters is deprecated; maybe ** should be added to the call\n/usr/lib/ruby/gems/2.7.0/gems/rubocop-0.74.0/lib/rubocop/config_loader_resolver.rb 84  warning  The called method `merge' is defined here\nInspecting 1 file\nC\n\nOffenses \n\nruby/ruby_bad_1.rb 1 1  C  Style/FrozenStringLiteralComment  Missing magic comment # frozen_string_literal  true.\nruby/ruby_bad_1.rb 11 11  C  Style/StringLiterals  Prefer double-quoted strings unless you need single quotes to avoid extra backslashes for escaping.\nFile.open('/tmp/urls.txt', " w" ) do | file|\n          ^^^^^^^^^^^^^^^\nruby/ruby_bad_1.rb 11 32  C  Layout/SpaceInsideParens  Space inside parentheses detected.\nFile.open('/tmp/urls.txt', " w" ) do | file|\n                               ^\nruby/ruby_bad_1.rb 11 39  C  Layout/SpaceAroundBlockParameters  Space before first block parameter detected.\nFile.open('/tmp/urls.txt', " w" ) do | file|\n                                      ^\nruby/ruby_bad_1.rb 12 24  C  Layout/SpaceAroundBlockParameters  Space after last block parameter detected.\n  Hook.active.map do |h |\n                       ^\nruby/ruby_bad_1.rb 13 13  C  Layout/SpaceInsideArrayLiteralBrackets  Do not use space inside array brackets.\n    urls = [ ARRAY_OF_URLS_CALLING_INSTANCE]\n            ^\n\n1 file inspected, 6 offenses detected\n
+  ...
+ok 2 - ruby_good_1.rb

.automation/test/shell/reports/expected-BASH.tap

@@ -0,0 +1,7 @@
+TAP version 13
+1..2
+not ok 1 - shell_bad_1.sh
+  ---
+  message: \nIn /tmp/lint/.automation/test/shell/shell_bad_1.sh line 10 \nif [ $ERROR_CODE -ne 0]; then\n^-- SC1009  The mentioned syntax error was in this if expression.\n   ^-- SC1073  Couldn't parse this test expression. Fix to allow more checks.\n                       ^-- SC1020  You need a space before the ].\n                       ^-- SC1072  Missing space before ]. Fix any mentioned problems and try again.\n\nFor more information \n  https //www.shellcheck.net/wiki/SC1020 -- You need a space before the ].\n  https //www.shellcheck.net/wiki/SC1072 -- Missing space before ]. Fix any m...\n  https //www.shellcheck.net/wiki/SC1073 -- Couldn't parse this test expressi...\n
+  ...
+ok 2 - shell_good_1.sh

.automation/test/typescript/reports/expected-TYPESCRIPT_ES.tap

@@ -0,0 +1,7 @@
+TAP version 13
+1..2
+not ok 1 - typescript_bad_1.ts
+  ---
+  message: \n/tmp/lint/.automation/test/typescript/typescript_bad_1.ts\n  5 39  error  Parsing error  Unterminated regular expression literal\n\n✖ 1 problem (1 error, 0 warnings)\n
+  ...
+ok 2 - typescript_good_1.ts

.automation/test/typescript/reports/expected-TYPESCRIPT_STANDARD.tap

@@ -0,0 +1,7 @@
+TAP version 13
+1..2
+not ok 1 - typescript_bad_1.ts
+  ---
+  message: standard  Use JavaScript Standard Style (https //standardjs.com)\n  /tmp/lint/.automation/test/typescript/typescript_bad_1.ts 5 39  Parsing error  Unterminated regular expression literal.\n
+  ...
+ok 2 - typescript_good_1.ts

.automation/test/xml/reports/expected-XML.tap

@@ -0,0 +1,7 @@
+TAP version 13
+1..2
+not ok 1 - xml_bad_1.xml
+  ---
+  message: /tmp/lint/.automation/test/xml/xml_bad_1.xml 7  parser error   EndTag  '</' not found\n\n^\n
+  ...
+ok 2 - xml_good_1.xml

.automation/test/yml/reports/expected-YML.tap

@@ -0,0 +1,7 @@
+TAP version 13
+1..2
+not ok 1 - yml_bad_1.yml
+  ---
+  message: /tmp/lint/.automation/test/yml/yml_bad_1.yml\n  10 1      warning  missing document start "---"  (document-start)\n  11 16     error    syntax error  mapping values are not allowed here (syntax)\n
+  ...
+ok 2 - yml_good_1.yml

.github/workflows/deploy-DEV.yml

@@ -55,4 +55,4 @@ jobs:
       ################################
       - name: Run Test Cases
         shell: bash
-        run: docker run -e RUN_LOCAL=true -e TEST_CASE_RUN=true -v ${GITHUB_WORKSPACE}:/tmp/lint github/super-linter:${GITHUB_SHA}
+        run: docker run -e RUN_LOCAL=true -e TEST_CASE_RUN=true -e OUTPUT_FORMAT=tap -e OUTPUT_FOLDER=${GITHUB_SHA} -e OUTPUT_DETAILS=detailed -v ${GITHUB_WORKSPACE}:/tmp/lint github/super-linter:${GITHUB_SHA}

lib/worker.sh

@@ -307,7 +307,16 @@ function TestCodebase() {
   #################################
   # Get list of all files to lint #
   #################################
-  mapfile -t LIST_FILES < <(find "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER/$INDVIDUAL_TEST_FOLDER" -type f -regex "$FILE_EXTENSIONS" ! -path "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER/ansible/ghe-initialize/*" 2>&1)
+  mapfile -t LIST_FILES < <(find "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER/$INDVIDUAL_TEST_FOLDER" -type f -regex "$FILE_EXTENSIONS" ! -path "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER/ansible/ghe-initialize/*" | sort 2>&1)
+
+  ########################################
+  # Prepare context if TAP output format #
+  ########################################
+  if IsTAP ; then
+    TMPFILE=$(mktemp -q "/tmp/super-linter-${FILE_TYPE}.XXXXXX")
+    mkdir -p "${REPORT_OUTPUT_FOLDER}"
+    REPORT_OUTPUT_FILE="${REPORT_OUTPUT_FOLDER}/super-linter-${FILE_TYPE}.${OUTPUT_FORMAT}"
+  fi
 
   ##################
   # Lint the files #
@@ -406,6 +415,11 @@ function TestCodebase() {
     #######################
     ERROR_CODE=$?
 
+    ########################################
+    # Increment counter that check was ran #
+    ########################################
+    (("TESTS_RAN++"))
+
     ########################################
     # Check for if it was supposed to pass #
     ########################################
@@ -422,15 +436,17 @@ function TestCodebase() {
         echo -e "${NC}${B[R]}${F[W]}ERROR:${NC} Linter CMD:[$LINTER_COMMAND $FILE]${NC}"
         # Increment the error count
         (("ERRORS_FOUND_$FILE_TYPE++"))
-        # Increment counter that check was ran
-        ((TESTS_RAN++))
       else
         ###########
         # Success #
         ###########
         echo -e "${NC}${F[B]} - File:${F[W]}[$FILE_NAME]${F[B]} was linted with ${F[W]}[$LINTER_NAME]${F[B]} successfully${NC}"
-        # Increment counter that check was ran
+      fi
-        ((TESTS_RAN++))
+      #######################################################
+      # Store the linting as a temporary file in TAP format #
+      #######################################################
+      if IsTAP ; then
+        echo "ok ${TESTS_RAN} - ${FILE_NAME}" >> "${TMPFILE}"
       fi
     else
       #######################################
@@ -449,19 +465,60 @@ function TestCodebase() {
         echo -e "${NC}${B[R]}${F[W]}ERROR:${NC} Linter CMD:[$LINTER_COMMAND $FILE]${NC}"
         # Increment the error count
         (("ERRORS_FOUND_$FILE_TYPE++"))
-        # Increment counter that check was ran
-        ((TESTS_RAN++))
       else
         ###########
         # Success #
         ###########
         echo -e "${NC}${F[B]} - File:${F[W]}[$FILE_NAME]${F[B]} failed test case with ${F[W]}[$LINTER_NAME]${F[B]} successfully${NC}"
-        # Increment counter that check was ran
+      fi
-        ((TESTS_RAN++))
+      #######################################################
+      # Store the linting as a temporary file in TAP format #
+      #######################################################
+      if IsTAP ; then
+        echo "not ok ${TESTS_RAN} - ${FILE_NAME}" >> "${TMPFILE}"
+        ##########################################
+        # Report the detailed message if enabled #
+        ##########################################
+        DETAILED_MSG=$(TransformTAPDetails "$LINT_CMD")
+        if [ -n "${DETAILED_MSG}" ] ; then
+          printf "  ---\n  message: %s\n  ...\n" "${DETAILED_MSG}" >> "${TMPFILE}"
+        fi
       fi
     fi
   done
 
+  ###########################################################################
+  # Generate report in TAP format and validate with the expected TAP output #
+  ###########################################################################
+  if IsTAP && [ ${TESTS_RAN} -gt 0 ] ; then
+    printf "TAP version 13\n1..%s\n" "${TESTS_RAN}" > "${REPORT_OUTPUT_FILE}"
+    cat "${TMPFILE}" >> "${REPORT_OUTPUT_FILE}"
+
+    ########################################################################
+    # If expected TAP report exists then compare with the generated report #
+    ########################################################################
+    EXPECTED_FILE="$GITHUB_WORKSPACE/$TEST_CASE_FOLDER/$INDVIDUAL_TEST_FOLDER/reports/expected-${FILE_TYPE}.tap"
+    if [ -e "$EXPECTED_FILE" ] ; then
+      TMPFILE=$(mktemp -q "/tmp/diff-${FILE_TYPE}.XXXXXX")
+      ## Ignore white spaces, case sensitive
+      if ! diff -a -w -i "${EXPECTED_FILE}" "${REPORT_OUTPUT_FILE}" > "${TMPFILE}" 2>&1; then
+        #############################################
+        # We failed to compare the reporting output #
+        #############################################
+        echo -e "${NC}${B[R]}${F[W]}ERROR!${NC} Failed to assert TAP output:[$LINTER_NAME]${NC}"!
+        echo "Please validate the asserts!"
+        cat "${TMPFILE}"
+        exit 1
+      else
+        # Success
+        echo -e "${NC}${F[B]}Successfully validation in the expected TAP format for ${F[W]}[$LINTER_NAME]${NC}"
+      fi
+    else
+      echo -e "${NC}${F[Y]}WARN!${NC} No TAP expected file found at:[$EXPECTED_FILE]${NC}"
+      echo "skipping report assertions"
+    fi
+  fi
+
   ##############################
   # Validate we ran some tests #
   ##############################