diff --git a/.automation/test/ansible/reports/expected-ANSIBLE.tap b/.automation/test/ansible/reports/expected-ANSIBLE.tap new file mode 100644 index 00000000..5f81e1ed --- /dev/null +++ b/.automation/test/ansible/reports/expected-ANSIBLE.tap @@ -0,0 +1,7 @@ +TAP version 13 +1..2 +not ok 1 - ansible_bad_1.yml + --- + message: Traceback (most recent call last) \n File "/usr/bin/ansible-lint", line 11, in \n load_entry_point('ansible-lint==4.2.0', 'console_scripts', 'ansible-lint')()\n File "/usr/lib/python3.8/site-packages/ansiblelint/__main__.py", line 187, in main\n matches.extend(runner.run())\n File "/usr/lib/python3.8/site-packages/ansiblelint/__init__.py", line 267, in run\n for child in ansiblelint.utils.find_children(arg, self.playbook_dir) \n File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 163, in find_children\n for child in play_children(basedir, item, playbook[1], playbook_dir) \n File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 215, in play_children\n return delegate_map[k](basedir, k, v, parent_type)\n File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 246, in _taskshandlers_children\n results.extend(_roles_children(basedir, k, [th['action'].get('name')],\n File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 285, in _roles_children\n results.extend(_look_for_role_files(basedir, role, main=main))\n File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 330, in _look_for_role_files\n role_path = _rolepath(basedir, role)\n File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 299, in _rolepath\n path_dwim(basedir, os.path.join('roles', role)),\n File "/usr/lib/python3.8/posixpath.py", line 90, in join\n genericpath._check_arg_types('join', a, *p)\n File "/usr/lib/python3.8/genericpath.py", line 152, in _check_arg_types\n raise TypeError(f'{funcname}() argument must be str, bytes, or '\nTypeError join() argument must be str, bytes, or os.PathLike object, not 'NoneType'\n + ... +ok 2 - ansible_good_1.yml diff --git a/.automation/test/coffeescript/reports/expected-COFFEESCRIPT.tap b/.automation/test/coffeescript/reports/expected-COFFEESCRIPT.tap new file mode 100644 index 00000000..4ce33752 --- /dev/null +++ b/.automation/test/coffeescript/reports/expected-COFFEESCRIPT.tap @@ -0,0 +1,7 @@ +TAP version 13 +1..2 +not ok 1 - coffeescript_bad_1.coffee + --- + message: ✗ /tmp/lint/.automation/test/coffeescript/coffeescript_bad_1.coffee\n ✗ #39 [stdin] 39 29 error unmatched )\nmodule.exports = (robot) -> )\n ^. (coffeescript_error)\n\n✗ Lint! » 1 error and 0 warnings in 1 file\n + ... +ok 2 - coffeescript_good_1.coffee diff --git a/.automation/test/docker/reports/expected-DOCKER.tap b/.automation/test/docker/reports/expected-DOCKER.tap new file mode 100644 index 00000000..6aafb4c6 --- /dev/null +++ b/.automation/test/docker/reports/expected-DOCKER.tap @@ -0,0 +1,7 @@ +TAP version 13 +1..2 +not ok 1 - Dockerfile + --- + message: \nFile /tmp/lint/.automation/test/docker/bad/Dockerfile\nIssues 6\n\nLine 1 from node latest\nIssue Category Title Description\n 1 Clarity Capitalize For clarity and readability, all instructions in a Dockerfile\n Dockerfile should be uppercase.\n Instructions This is a convention adopted by most of the official images and\n greatly improves readability in long Dockerfiles. For an example\n of\n why this makes a difference, check out the current [redis\n Dockerfile](https //github.com/docker-library/redis/blob/b375650fb6\n 9b7db819e90c0033433c705b28656e/3.0/Dockerfile)\n and you should be able to easily see the instructions used.\n 2 Clarity Base Image Latest Base images should not use the latest tag.\n Tag\n\nLine 4 run mkdir -p /usr/src/app\nIssue Category Title Description\n 3 Clarity Capitalize For clarity and readability, all instructions in a Dockerfile\n Dockerfile should be uppercase.\n Instructions This is a convention adopted by most of the official images and\n greatly improves readability in long Dockerfiles. For an example\n of\n why this makes a difference, check out the current [redis\n Dockerfile](https //github.com/docker-library/redis/blob/b375650fb6\n 9b7db819e90c0033433c705b28656e/3.0/Dockerfile)\n and you should be able to easily see the instructions used.\n\nLine 8 copy package.json /usr/src/app/ /here/there\nIssue Category Title Description\n 4 Clarity Capitalize For clarity and readability, all instructions in a Dockerfile\n Dockerfile should be uppercase.\n Instructions This is a convention adopted by most of the official images and\n greatly improves readability in long Dockerfiles. For an example\n of\n why this makes a difference, check out the current [redis\n Dockerfile](https //github.com/docker-library/redis/blob/b375650fb6\n 9b7db819e90c0033433c705b28656e/3.0/Dockerfile)\n and you should be able to easily see the instructions used.\n\nLine 9 RUN sudo npm install\nIssue Category Title Description\n 5 Possible Bug Use Of sudo Is Not Use of `sudo` is not allowed in a Dockerfile. From the official\n Allowed document [Best practices for writing\n Dockerfiles](https //docs.docker.com/engine/userguide/eng-image/doc\n kerfile_best-practices/) \n > You should avoid installing or using `sudo` since it has\n unpredictable TTY and signal-forwarding behavior that can cause\n more problems than it solves.\n > If you absolutely need functionality similar to `sudo` (e.g.,\n initializing the daemon as root but running it as non-root), you\n may be able to use `gosu`.\n\nLine 14 ENtrypoint /tmp/here.sh\nIssue Category Title Description\n 6 Clarity Capitalize For clarity and readability, all instructions in a Dockerfile\n Dockerfile should be uppercase.\n Instructions This is a convention adopted by most of the official images and\n greatly improves readability in long Dockerfiles. For an example\n of\n why this makes a difference, check out the current [redis\n Dockerfile](https //github.com/docker-library/redis/blob/b375650fb6\n 9b7db819e90c0033433c705b28656e/3.0/Dockerfile)\n and you should be able to easily see the instructions used.\n + ... +ok 2 - Dockerfile diff --git a/.automation/test/javascript/reports/expected-JAVASCRIPT_ES.tap b/.automation/test/javascript/reports/expected-JAVASCRIPT_ES.tap new file mode 100644 index 00000000..9e605d09 --- /dev/null +++ b/.automation/test/javascript/reports/expected-JAVASCRIPT_ES.tap @@ -0,0 +1,7 @@ +TAP version 13 +1..2 +not ok 1 - javascript_bad_1.js + --- + message: \n/tmp/lint/.automation/test/javascript/javascript_bad_1.js\n 4 39 error Parsing error Unterminated regular expression literal\n\n✖ 1 problem (1 error, 0 warnings)\n + ... +ok 2 - javascript_good_1.js diff --git a/.automation/test/javascript/reports/expected-JAVASCRIPT_STANDARD.tap b/.automation/test/javascript/reports/expected-JAVASCRIPT_STANDARD.tap new file mode 100644 index 00000000..42d92b34 --- /dev/null +++ b/.automation/test/javascript/reports/expected-JAVASCRIPT_STANDARD.tap @@ -0,0 +1,7 @@ +TAP version 13 +1..2 +not ok 1 - javascript_bad_1.js + --- + message: standard Use JavaScript Standard Style (https //standardjs.com)\n /tmp/lint/.automation/test/javascript/javascript_bad_1.js 4 40 Parsing error Unterminated regular expression\n + ... +ok 2 - javascript_good_1.js diff --git a/.automation/test/json/reports/expected-JSON.tap b/.automation/test/json/reports/expected-JSON.tap new file mode 100644 index 00000000..2b43a1ec --- /dev/null +++ b/.automation/test/json/reports/expected-JSON.tap @@ -0,0 +1,7 @@ +TAP version 13 +1..2 +not ok 1 - json_bad_1.json + --- + message: Error Parse error on line 6 \n...ng" { "level" 'ignore', "space\n----------------------^\nExpecting 'STRING', 'NUMBER', 'NULL', 'TRUE', 'FALSE', '{', '[', got 'undefined'\n at Object.parseError (/usr/lib/node_modules/jsonlint/lib/jsonlint.js 55 11)\n at Object.parse (/usr/lib/node_modules/jsonlint/lib/jsonlint.js 132 22)\n at parse (/usr/lib/node_modules/jsonlint/lib/cli.js 82 14)\n at main (/usr/lib/node_modules/jsonlint/lib/cli.js 135 14)\n at Object. (/usr/lib/node_modules/jsonlint/lib/cli.js 179 1)\n at Module._compile (internal/modules/cjs/loader.js 1138 30)\n at Object.Module._extensions..js (internal/modules/cjs/loader.js 1158 10)\n at Module.load (internal/modules/cjs/loader.js 986 32)\n at Function.Module._load (internal/modules/cjs/loader.js 879 14)\n at Function.executeUserEntryPoint [as runMain] (internal/modules/run_main.js 71 12)\n + ... +ok 2 - json_good_1.json diff --git a/.automation/test/kotlin/reports/expected-KOTLIN.tap b/.automation/test/kotlin/reports/expected-KOTLIN.tap new file mode 100644 index 00000000..89c7fbc8 --- /dev/null +++ b/.automation/test/kotlin/reports/expected-KOTLIN.tap @@ -0,0 +1,7 @@ +TAP version 13 +1..2 +not ok 1 - kotlin_bad_1.kt + --- + message: /tmp/lint/.automation/test/kotlin/kotlin_bad_1.kt 1 1 File must end with a newline (\n)\n/tmp/lint/.automation/test/kotlin/kotlin_bad_1.kt 2 20 Unnecessary semicolon\n/tmp/lint/.automation/test/kotlin/kotlin_bad_1.kt 3 16 Redundant curly braces\n/tmp/lint/.automation/test/kotlin/kotlin_bad_1.kt 3 27 Unnecessary semicolon\n/tmp/lint/.automation/test/kotlin/kotlin_bad_1.kt 5 15 Unnecessary semicolon\n + ... +ok 2 - kotlint_good_1.kt diff --git a/.automation/test/markdown/reports/expected-MARKDOWN.tap b/.automation/test/markdown/reports/expected-MARKDOWN.tap new file mode 100644 index 00000000..8960605d --- /dev/null +++ b/.automation/test/markdown/reports/expected-MARKDOWN.tap @@ -0,0 +1,8 @@ +TAP version 13 +1..3 +ok 1 - README.md +not ok 2 - markdown_bad_1.md + --- + message: /tmp/lint/.automation/test/markdown/markdown_bad_1.md 1 MD041/first-line-heading/first-line-h1 First line in file should be a top level heading [Context "## Bad Markdown"]\n/tmp/lint/.automation/test/markdown/markdown_bad_1.md 5 MD001/heading-increment/header-increment Heading levels should only increment by one level at a time [Expected h3; Actual h6]\n/tmp/lint/.automation/test/markdown/markdown_bad_1.md 11 1 MD007/ul-indent Unordered list indentation [Expected 4; Actual 6]\n/tmp/lint/.automation/test/markdown/markdown_bad_1.md 13 MD040/fenced-code-language Fenced code blocks should have a language specified [Context "```"]\n/tmp/lint/.automation/test/markdown/markdown_bad_1.md 20 16 MD034/no-bare-urls Bare URL used [Context "https //github.com"]\n + ... +ok 3 - markdown_good_1.md diff --git a/.automation/test/perl/reports/expected-PERL.tap b/.automation/test/perl/reports/expected-PERL.tap new file mode 100644 index 00000000..932c3621 --- /dev/null +++ b/.automation/test/perl/reports/expected-PERL.tap @@ -0,0 +1,7 @@ +TAP version 13 +1..2 +not ok 1 - perl_bad_1.pl + --- + message: Global symbol "$here" requires explicit package name (did you forget to declare "my $here"?) at /tmp/lint/.automation/test/perl/perl_bad_1.pl line 37.\n/tmp/lint/.automation/test/perl/perl_bad_1.pl had compilation errors.\n + ... +ok 2 - perl_good_1.pl diff --git a/.automation/test/php/reports/expected-PHP.tap b/.automation/test/php/reports/expected-PHP.tap new file mode 100644 index 00000000..48408065 --- /dev/null +++ b/.automation/test/php/reports/expected-PHP.tap @@ -0,0 +1,7 @@ +TAP version 13 +1..2 +not ok 1 - php_bad_1.php + --- + message: PHP Parse error syntax error, unexpected 'pe98y' (T_STRING) in /tmp/lint/.automation/test/php/php_bad_1.php on line 3\nErrors parsing /tmp/lint/.automation/test/php/php_bad_1.php\n + ... +ok 2 - php_good_1.php diff --git a/.automation/test/python/reports/expected-PYTHON.tap b/.automation/test/python/reports/expected-PYTHON.tap new file mode 100644 index 00000000..1751d70f --- /dev/null +++ b/.automation/test/python/reports/expected-PYTHON.tap @@ -0,0 +1,7 @@ +TAP version 13 +1..2 +not ok 1 - python_bad_1.py + --- + message: ************* Module python_bad_1\npython/python_bad_1.py 15 24 E0001 invalid syntax (, line 15) (syntax-error)\n + ... +ok 2 - python_good_1.py diff --git a/.automation/test/ruby/reports/expected-RUBY.tap b/.automation/test/ruby/reports/expected-RUBY.tap new file mode 100644 index 00000000..3f0a1372 --- /dev/null +++ b/.automation/test/ruby/reports/expected-RUBY.tap @@ -0,0 +1,7 @@ +TAP version 13 +1..2 +not ok 1 - ruby_bad_1.rb + --- + message: /usr/lib/ruby/gems/2.7.0/gems/rubocop-0.74.0/lib/rubocop/config_loader_resolver.rb 76 warning Using the last argument as keyword parameters is deprecated; maybe ** should be added to the call\n/usr/lib/ruby/gems/2.7.0/gems/rubocop-0.74.0/lib/rubocop/config_loader_resolver.rb 84 warning The called method `merge' is defined here\nInspecting 1 file\nC\n\nOffenses \n\nruby/ruby_bad_1.rb 1 1 C Style/FrozenStringLiteralComment Missing magic comment # frozen_string_literal true.\nruby/ruby_bad_1.rb 11 11 C Style/StringLiterals Prefer double-quoted strings unless you need single quotes to avoid extra backslashes for escaping.\nFile.open('/tmp/urls.txt', " w" ) do | file|\n ^^^^^^^^^^^^^^^\nruby/ruby_bad_1.rb 11 32 C Layout/SpaceInsideParens Space inside parentheses detected.\nFile.open('/tmp/urls.txt', " w" ) do | file|\n ^\nruby/ruby_bad_1.rb 11 39 C Layout/SpaceAroundBlockParameters Space before first block parameter detected.\nFile.open('/tmp/urls.txt', " w" ) do | file|\n ^\nruby/ruby_bad_1.rb 12 24 C Layout/SpaceAroundBlockParameters Space after last block parameter detected.\n Hook.active.map do |h |\n ^\nruby/ruby_bad_1.rb 13 13 C Layout/SpaceInsideArrayLiteralBrackets Do not use space inside array brackets.\n urls = [ ARRAY_OF_URLS_CALLING_INSTANCE]\n ^\n\n1 file inspected, 6 offenses detected\n + ... +ok 2 - ruby_good_1.rb diff --git a/.automation/test/shell/reports/expected-BASH.tap b/.automation/test/shell/reports/expected-BASH.tap new file mode 100644 index 00000000..f3bedca8 --- /dev/null +++ b/.automation/test/shell/reports/expected-BASH.tap @@ -0,0 +1,7 @@ +TAP version 13 +1..2 +not ok 1 - shell_bad_1.sh + --- + message: \nIn /tmp/lint/.automation/test/shell/shell_bad_1.sh line 10 \nif [ $ERROR_CODE -ne 0]; then\n^-- SC1009 The mentioned syntax error was in this if expression.\n ^-- SC1073 Couldn't parse this test expression. Fix to allow more checks.\n ^-- SC1020 You need a space before the ].\n ^-- SC1072 Missing space before ]. Fix any mentioned problems and try again.\n\nFor more information \n https //www.shellcheck.net/wiki/SC1020 -- You need a space before the ].\n https //www.shellcheck.net/wiki/SC1072 -- Missing space before ]. Fix any m...\n https //www.shellcheck.net/wiki/SC1073 -- Couldn't parse this test expressi...\n + ... +ok 2 - shell_good_1.sh diff --git a/.automation/test/typescript/reports/expected-TYPESCRIPT_ES.tap b/.automation/test/typescript/reports/expected-TYPESCRIPT_ES.tap new file mode 100644 index 00000000..c08faa21 --- /dev/null +++ b/.automation/test/typescript/reports/expected-TYPESCRIPT_ES.tap @@ -0,0 +1,7 @@ +TAP version 13 +1..2 +not ok 1 - typescript_bad_1.ts + --- + message: \n/tmp/lint/.automation/test/typescript/typescript_bad_1.ts\n 5 39 error Parsing error Unterminated regular expression literal\n\n✖ 1 problem (1 error, 0 warnings)\n + ... +ok 2 - typescript_good_1.ts diff --git a/.automation/test/typescript/reports/expected-TYPESCRIPT_STANDARD.tap b/.automation/test/typescript/reports/expected-TYPESCRIPT_STANDARD.tap new file mode 100644 index 00000000..1271fc8f --- /dev/null +++ b/.automation/test/typescript/reports/expected-TYPESCRIPT_STANDARD.tap @@ -0,0 +1,7 @@ +TAP version 13 +1..2 +not ok 1 - typescript_bad_1.ts + --- + message: standard Use JavaScript Standard Style (https //standardjs.com)\n /tmp/lint/.automation/test/typescript/typescript_bad_1.ts 5 39 Parsing error Unterminated regular expression literal.\n + ... +ok 2 - typescript_good_1.ts diff --git a/.automation/test/xml/reports/expected-XML.tap b/.automation/test/xml/reports/expected-XML.tap new file mode 100644 index 00000000..4a3081b9 --- /dev/null +++ b/.automation/test/xml/reports/expected-XML.tap @@ -0,0 +1,7 @@ +TAP version 13 +1..2 +not ok 1 - xml_bad_1.xml + --- + message: /tmp/lint/.automation/test/xml/xml_bad_1.xml 7 parser error EndTag '&1) + mapfile -t LIST_FILES < <(find "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER/$INDVIDUAL_TEST_FOLDER" -type f -regex "$FILE_EXTENSIONS" ! -path "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER/ansible/ghe-initialize/*" | sort 2>&1) + + ######################################## + # Prepare context if TAP output format # + ######################################## + if IsTAP ; then + TMPFILE=$(mktemp -q "/tmp/super-linter-${FILE_TYPE}.XXXXXX") + mkdir -p "${REPORT_OUTPUT_FOLDER}" + REPORT_OUTPUT_FILE="${REPORT_OUTPUT_FOLDER}/super-linter-${FILE_TYPE}.${OUTPUT_FORMAT}" + fi ################## # Lint the files # @@ -406,6 +415,11 @@ function TestCodebase() { ####################### ERROR_CODE=$? + ######################################## + # Increment counter that check was ran # + ######################################## + (("TESTS_RAN++")) + ######################################## # Check for if it was supposed to pass # ######################################## @@ -422,15 +436,17 @@ function TestCodebase() { echo -e "${NC}${B[R]}${F[W]}ERROR:${NC} Linter CMD:[$LINTER_COMMAND $FILE]${NC}" # Increment the error count (("ERRORS_FOUND_$FILE_TYPE++")) - # Increment counter that check was ran - ((TESTS_RAN++)) else ########### # Success # ########### echo -e "${NC}${F[B]} - File:${F[W]}[$FILE_NAME]${F[B]} was linted with ${F[W]}[$LINTER_NAME]${F[B]} successfully${NC}" - # Increment counter that check was ran - ((TESTS_RAN++)) + fi + ####################################################### + # Store the linting as a temporary file in TAP format # + ####################################################### + if IsTAP ; then + echo "ok ${TESTS_RAN} - ${FILE_NAME}" >> "${TMPFILE}" fi else ####################################### @@ -449,19 +465,60 @@ function TestCodebase() { echo -e "${NC}${B[R]}${F[W]}ERROR:${NC} Linter CMD:[$LINTER_COMMAND $FILE]${NC}" # Increment the error count (("ERRORS_FOUND_$FILE_TYPE++")) - # Increment counter that check was ran - ((TESTS_RAN++)) else ########### # Success # ########### echo -e "${NC}${F[B]} - File:${F[W]}[$FILE_NAME]${F[B]} failed test case with ${F[W]}[$LINTER_NAME]${F[B]} successfully${NC}" - # Increment counter that check was ran - ((TESTS_RAN++)) + fi + ####################################################### + # Store the linting as a temporary file in TAP format # + ####################################################### + if IsTAP ; then + echo "not ok ${TESTS_RAN} - ${FILE_NAME}" >> "${TMPFILE}" + ########################################## + # Report the detailed message if enabled # + ########################################## + DETAILED_MSG=$(TransformTAPDetails "$LINT_CMD") + if [ -n "${DETAILED_MSG}" ] ; then + printf " ---\n message: %s\n ...\n" "${DETAILED_MSG}" >> "${TMPFILE}" + fi fi fi done + ########################################################################### + # Generate report in TAP format and validate with the expected TAP output # + ########################################################################### + if IsTAP && [ ${TESTS_RAN} -gt 0 ] ; then + printf "TAP version 13\n1..%s\n" "${TESTS_RAN}" > "${REPORT_OUTPUT_FILE}" + cat "${TMPFILE}" >> "${REPORT_OUTPUT_FILE}" + + ######################################################################## + # If expected TAP report exists then compare with the generated report # + ######################################################################## + EXPECTED_FILE="$GITHUB_WORKSPACE/$TEST_CASE_FOLDER/$INDVIDUAL_TEST_FOLDER/reports/expected-${FILE_TYPE}.tap" + if [ -e "$EXPECTED_FILE" ] ; then + TMPFILE=$(mktemp -q "/tmp/diff-${FILE_TYPE}.XXXXXX") + ## Ignore white spaces, case sensitive + if ! diff -a -w -i "${EXPECTED_FILE}" "${REPORT_OUTPUT_FILE}" > "${TMPFILE}" 2>&1; then + ############################################# + # We failed to compare the reporting output # + ############################################# + echo -e "${NC}${B[R]}${F[W]}ERROR!${NC} Failed to assert TAP output:[$LINTER_NAME]${NC}"! + echo "Please validate the asserts!" + cat "${TMPFILE}" + exit 1 + else + # Success + echo -e "${NC}${F[B]}Successfully validation in the expected TAP format for ${F[W]}[$LINTER_NAME]${NC}" + fi + else + echo -e "${NC}${F[Y]}WARN!${NC} No TAP expected file found at:[$EXPECTED_FILE]${NC}" + echo "skipping report assertions" + fi + fi + ############################## # Validate we ran some tests # ##############################