actions/lint
1
0
Fork 0
mirror of https://github.com/super-linter/super-linter.git synced 2024-11-25 07:31:05 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

[tests] validate TAP format

Browse source
This commit is contained in:
Victor Martinez 2020-07-14 14:48:28 +01:00
parent de92cc732e
commit 41ddd0e1d3
No known key found for this signature in database
GPG key ID: 4058B656AD58C4F5
19 changed files with 187 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.automation/test/ansible/reports/expected-ANSIBLE.tap Normal file
View file

@ -0,0 +1,7 @@
TAP version 13
1..2
not ok 1 - ansible_bad_1.yml
---
message: Traceback (most recent call last) \n File "/usr/bin/ansible-lint", line 11, in <module>\n load_entry_point('ansible-lint==4.2.0', 'console_scripts', 'ansible-lint')()\n File "/usr/lib/python3.8/site-packages/ansiblelint/__main__.py", line 187, in main\n matches.extend(runner.run())\n File "/usr/lib/python3.8/site-packages/ansiblelint/__init__.py", line 267, in run\n for child in ansiblelint.utils.find_children(arg, self.playbook_dir) \n File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 163, in find_children\n for child in play_children(basedir, item, playbook[1], playbook_dir) \n File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 215, in play_children\n return delegate_map[k](basedir, k, v, parent_type)\n File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 246, in _taskshandlers_children\n results.extend(_roles_children(basedir, k, [th['action'].get('name')],\n File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 285, in _roles_children\n results.extend(_look_for_role_files(basedir, role, main=main))\n File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 330, in _look_for_role_files\n role_path = _rolepath(basedir, role)\n File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 299, in _rolepath\n path_dwim(basedir, os.path.join('roles', role)),\n File "/usr/lib/python3.8/posixpath.py", line 90, in join\n genericpath._check_arg_types('join', a, *p)\n File "/usr/lib/python3.8/genericpath.py", line 152, in _check_arg_types\n raise TypeError(f'{funcname}() argument must be str, bytes, or '\nTypeError join() argument must be str, bytes, or os.PathLike object, not 'NoneType'\n
...
ok 2 - ansible_good_1.yml

7
.automation/test/coffeescript/reports/expected-COFFEESCRIPT.tap Normal file
View file

@ -0,0 +1,7 @@
TAP version 13
1..2
not ok 1 - coffeescript_bad_1.coffee
---
message: ✗ /tmp/lint/.automation/test/coffeescript/coffeescript_bad_1.coffee\n ✗ #39 [stdin] 39 29 error unmatched )\nmodule.exports = (robot) -> )\n ^. (coffeescript_error)\n\n✗ Lint! » 1 error and 0 warnings in 1 file\n
...
ok 2 - coffeescript_good_1.coffee

7
.automation/test/docker/reports/expected-DOCKER.tap Normal file
View file

@ -0,0 +1,7 @@
TAP version 13
1..2
not ok 1 - Dockerfile
---
message: \nFile /tmp/lint/.automation/test/docker/bad/Dockerfile\nIssues 6\n\nLine 1 from node latest\nIssue Category Title Description\n 1 Clarity Capitalize For clarity and readability, all instructions in a Dockerfile\n Dockerfile should be uppercase.\n Instructions This is a convention adopted by most of the official images and\n greatly improves readability in long Dockerfiles. For an example\n of\n why this makes a difference, check out the current [redis\n Dockerfile](https //github.com/docker-library/redis/blob/b375650fb6\n 9b7db819e90c0033433c705b28656e/3.0/Dockerfile)\n and you should be able to easily see the instructions used.\n 2 Clarity Base Image Latest Base images should not use the latest tag.\n Tag\n\nLine 4 run mkdir -p /usr/src/app\nIssue Category Title Description\n 3 Clarity Capitalize For clarity and readability, all instructions in a Dockerfile\n Dockerfile should be uppercase.\n Instructions This is a convention adopted by most of the official images and\n greatly improves readability in long Dockerfiles. For an example\n of\n why this makes a difference, check out the current [redis\n Dockerfile](https //github.com/docker-library/redis/blob/b375650fb6\n 9b7db819e90c0033433c705b28656e/3.0/Dockerfile)\n and you should be able to easily see the instructions used.\n\nLine 8 copy package.json /usr/src/app/ /here/there\nIssue Category Title Description\n 4 Clarity Capitalize For clarity and readability, all instructions in a Dockerfile\n Dockerfile should be uppercase.\n Instructions This is a convention adopted by most of the official images and\n greatly improves readability in long Dockerfiles. For an example\n of\n why this makes a difference, check out the current [redis\n Dockerfile](https //github.com/docker-library/redis/blob/b375650fb6\n 9b7db819e90c0033433c705b28656e/3.0/Dockerfile)\n and you should be able to easily see the instructions used.\n\nLine 9 RUN sudo npm install\nIssue Category Title Description\n 5 Possible Bug Use Of sudo Is Not Use of `sudo` is not allowed in a Dockerfile. From the official\n Allowed document [Best practices for writing\n Dockerfiles](https //docs.docker.com/engine/userguide/eng-image/doc\n kerfile_best-practices/) \n > You should avoid installing or using `sudo` since it has\n unpredictable TTY and signal-forwarding behavior that can cause\n more problems than it solves.\n > If you absolutely need functionality similar to `sudo` (e.g.,\n initializing the daemon as root but running it as non-root), you\n may be able to use `gosu`.\n\nLine 14 ENtrypoint /tmp/here.sh\nIssue Category Title Description\n 6 Clarity Capitalize For clarity and readability, all instructions in a Dockerfile\n Dockerfile should be uppercase.\n Instructions This is a convention adopted by most of the official images and\n greatly improves readability in long Dockerfiles. For an example\n of\n why this makes a difference, check out the current [redis\n Dockerfile](https //github.com/docker-library/redis/blob/b375650fb6\n 9b7db819e90c0033433c705b28656e/3.0/Dockerfile)\n and you should be able to easily see the instructions used.\n
...
ok 2 - Dockerfile

7
.automation/test/javascript/reports/expected-JAVASCRIPT_ES.tap Normal file
View file

@ -0,0 +1,7 @@
TAP version 13
1..2
not ok 1 - javascript_bad_1.js
---
message: \n/tmp/lint/.automation/test/javascript/javascript_bad_1.js\n 4 39 error Parsing error Unterminated regular expression literal\n\n✖ 1 problem (1 error, 0 warnings)\n
...
ok 2 - javascript_good_1.js

7
.automation/test/javascript/reports/expected-JAVASCRIPT_STANDARD.tap Normal file
View file

@ -0,0 +1,7 @@
TAP version 13
1..2
not ok 1 - javascript_bad_1.js
---
message: standard Use JavaScript Standard Style (https //standardjs.com)\n /tmp/lint/.automation/test/javascript/javascript_bad_1.js 4 40 Parsing error Unterminated regular expression\n
...
ok 2 - javascript_good_1.js

7
.automation/test/json/reports/expected-JSON.tap Normal file
View file

@ -0,0 +1,7 @@
TAP version 13
1..2
not ok 1 - json_bad_1.json
---
message: Error Parse error on line 6 \n...ng" { "level" 'ignore', "space\n----------------------^\nExpecting 'STRING', 'NUMBER', 'NULL', 'TRUE', 'FALSE', '{', '[', got 'undefined'\n at Object.parseError (/usr/lib/node_modules/jsonlint/lib/jsonlint.js 55 11)\n at Object.parse (/usr/lib/node_modules/jsonlint/lib/jsonlint.js 132 22)\n at parse (/usr/lib/node_modules/jsonlint/lib/cli.js 82 14)\n at main (/usr/lib/node_modules/jsonlint/lib/cli.js 135 14)\n at Object.<anonymous> (/usr/lib/node_modules/jsonlint/lib/cli.js 179 1)\n at Module._compile (internal/modules/cjs/loader.js 1138 30)\n at Object.Module._extensions..js (internal/modules/cjs/loader.js 1158 10)\n at Module.load (internal/modules/cjs/loader.js 986 32)\n at Function.Module._load (internal/modules/cjs/loader.js 879 14)\n at Function.executeUserEntryPoint [as runMain] (internal/modules/run_main.js 71 12)\n
...
ok 2 - json_good_1.json

7
.automation/test/kotlin/reports/expected-KOTLIN.tap Normal file
View file

@ -0,0 +1,7 @@
TAP version 13
1..2
not ok 1 - kotlin_bad_1.kt
---
message: /tmp/lint/.automation/test/kotlin/kotlin_bad_1.kt 1 1 File must end with a newline (\n)\n/tmp/lint/.automation/test/kotlin/kotlin_bad_1.kt 2 20 Unnecessary semicolon\n/tmp/lint/.automation/test/kotlin/kotlin_bad_1.kt 3 16 Redundant curly braces\n/tmp/lint/.automation/test/kotlin/kotlin_bad_1.kt 3 27 Unnecessary semicolon\n/tmp/lint/.automation/test/kotlin/kotlin_bad_1.kt 5 15 Unnecessary semicolon\n
...
ok 2 - kotlint_good_1.kt

8
.automation/test/markdown/reports/expected-MARKDOWN.tap Normal file
View file

@ -0,0 +1,8 @@
TAP version 13
1..3
ok 1 - README.md
not ok 2 - markdown_bad_1.md
---
message: /tmp/lint/.automation/test/markdown/markdown_bad_1.md 1 MD041/first-line-heading/first-line-h1 First line in file should be a top level heading [Context "## Bad Markdown"]\n/tmp/lint/.automation/test/markdown/markdown_bad_1.md 5 MD001/heading-increment/header-increment Heading levels should only increment by one level at a time [Expected h3; Actual h6]\n/tmp/lint/.automation/test/markdown/markdown_bad_1.md 11 1 MD007/ul-indent Unordered list indentation [Expected 4; Actual 6]\n/tmp/lint/.automation/test/markdown/markdown_bad_1.md 13 MD040/fenced-code-language Fenced code blocks should have a language specified [Context "```"]\n/tmp/lint/.automation/test/markdown/markdown_bad_1.md 20 16 MD034/no-bare-urls Bare URL used [Context "https //github.com"]\n
...
ok 3 - markdown_good_1.md

7
.automation/test/perl/reports/expected-PERL.tap Normal file
View file

@ -0,0 +1,7 @@
TAP version 13
1..2
not ok 1 - perl_bad_1.pl
---
message: Global symbol "$here" requires explicit package name (did you forget to declare "my $here"?) at /tmp/lint/.automation/test/perl/perl_bad_1.pl line 37.\n/tmp/lint/.automation/test/perl/perl_bad_1.pl had compilation errors.\n
...
ok 2 - perl_good_1.pl

7
.automation/test/php/reports/expected-PHP.tap Normal file
View file

@ -0,0 +1,7 @@
TAP version 13
1..2
not ok 1 - php_bad_1.php
---
message: PHP Parse error syntax error, unexpected 'pe98y' (T_STRING) in /tmp/lint/.automation/test/php/php_bad_1.php on line 3\nErrors parsing /tmp/lint/.automation/test/php/php_bad_1.php\n
...
ok 2 - php_good_1.php

7
.automation/test/python/reports/expected-PYTHON.tap Normal file
View file

@ -0,0 +1,7 @@
TAP version 13
1..2
not ok 1 - python_bad_1.py
---
message: ************* Module python_bad_1\npython/python_bad_1.py 15 24 E0001 invalid syntax (<unknown>, line 15) (syntax-error)\n
...
ok 2 - python_good_1.py

7
.automation/test/ruby/reports/expected-RUBY.tap Normal file
View file

@ -0,0 +1,7 @@
TAP version 13
1..2
not ok 1 - ruby_bad_1.rb
---
message: /usr/lib/ruby/gems/2.7.0/gems/rubocop-0.74.0/lib/rubocop/config_loader_resolver.rb 76 warning Using the last argument as keyword parameters is deprecated; maybe ** should be added to the call\n/usr/lib/ruby/gems/2.7.0/gems/rubocop-0.74.0/lib/rubocop/config_loader_resolver.rb 84 warning The called method `merge' is defined here\nInspecting 1 file\nC\n\nOffenses \n\nruby/ruby_bad_1.rb 1 1 C Style/FrozenStringLiteralComment Missing magic comment # frozen_string_literal true.\nruby/ruby_bad_1.rb 11 11 C Style/StringLiterals Prefer double-quoted strings unless you need single quotes to avoid extra backslashes for escaping.\nFile.open('/tmp/urls.txt', " w" ) do | file|\n ^^^^^^^^^^^^^^^\nruby/ruby_bad_1.rb 11 32 C Layout/SpaceInsideParens Space inside parentheses detected.\nFile.open('/tmp/urls.txt', " w" ) do | file|\n ^\nruby/ruby_bad_1.rb 11 39 C Layout/SpaceAroundBlockParameters Space before first block parameter detected.\nFile.open('/tmp/urls.txt', " w" ) do | file|\n ^\nruby/ruby_bad_1.rb 12 24 C Layout/SpaceAroundBlockParameters Space after last block parameter detected.\n Hook.active.map do |h |\n ^\nruby/ruby_bad_1.rb 13 13 C Layout/SpaceInsideArrayLiteralBrackets Do not use space inside array brackets.\n urls = [ ARRAY_OF_URLS_CALLING_INSTANCE]\n ^\n\n1 file inspected, 6 offenses detected\n
...
ok 2 - ruby_good_1.rb

7
.automation/test/shell/reports/expected-BASH.tap Normal file
View file

@ -0,0 +1,7 @@
TAP version 13
1..2
not ok 1 - shell_bad_1.sh
---
message: \nIn /tmp/lint/.automation/test/shell/shell_bad_1.sh line 10 \nif [ $ERROR_CODE -ne 0]; then\n^-- SC1009 The mentioned syntax error was in this if expression.\n ^-- SC1073 Couldn't parse this test expression. Fix to allow more checks.\n ^-- SC1020 You need a space before the ].\n ^-- SC1072 Missing space before ]. Fix any mentioned problems and try again.\n\nFor more information \n https //www.shellcheck.net/wiki/SC1020 -- You need a space before the ].\n https //www.shellcheck.net/wiki/SC1072 -- Missing space before ]. Fix any m...\n https //www.shellcheck.net/wiki/SC1073 -- Couldn't parse this test expressi...\n
...
ok 2 - shell_good_1.sh

7
.automation/test/typescript/reports/expected-TYPESCRIPT_ES.tap Normal file
View file

@ -0,0 +1,7 @@
TAP version 13
1..2
not ok 1 - typescript_bad_1.ts
---
message: \n/tmp/lint/.automation/test/typescript/typescript_bad_1.ts\n 5 39 error Parsing error Unterminated regular expression literal\n\n✖ 1 problem (1 error, 0 warnings)\n
...
ok 2 - typescript_good_1.ts

7
.automation/test/typescript/reports/expected-TYPESCRIPT_STANDARD.tap Normal file
View file

@ -0,0 +1,7 @@
TAP version 13
1..2
not ok 1 - typescript_bad_1.ts
---
message: standard Use JavaScript Standard Style (https //standardjs.com)\n /tmp/lint/.automation/test/typescript/typescript_bad_1.ts 5 39 Parsing error Unterminated regular expression literal.\n
...
ok 2 - typescript_good_1.ts

7
.automation/test/xml/reports/expected-XML.tap Normal file
View file

@ -0,0 +1,7 @@
TAP version 13
1..2
not ok 1 - xml_bad_1.xml
---
message: /tmp/lint/.automation/test/xml/xml_bad_1.xml 7 parser error EndTag '</' not found\n\n^\n
...
ok 2 - xml_good_1.xml

7
.automation/test/yml/reports/expected-YML.tap Normal file
View file

@ -0,0 +1,7 @@
TAP version 13
1..2
not ok 1 - yml_bad_1.yml
---
message: /tmp/lint/.automation/test/yml/yml_bad_1.yml\n 10 1 warning missing document start "---" (document-start)\n 11 16 error syntax error mapping values are not allowed here (syntax)\n
...
ok 2 - yml_good_1.yml

2
.github/workflows/deploy-DEV.yml vendored
View file

@ -55,4 +55,4 @@ jobs:
################################
- name: Run Test Cases
shell: bash
run: docker run -e RUN_LOCAL=true -e TEST_CASE_RUN=true -v ${GITHUB_WORKSPACE}:/tmp/lint github/super-linter:${GITHUB_SHA}
run: docker run -e RUN_LOCAL=true -e TEST_CASE_RUN=true -e OUTPUT_FORMAT=tap -e OUTPUT_FOLDER=${GITHUB_SHA} -e OUTPUT_DETAILS=detailed -v ${GITHUB_WORKSPACE}:/tmp/lint github/super-linter:${GITHUB_SHA}

75
lib/worker.sh
View file

@ -307,7 +307,16 @@ function TestCodebase() {
#################################
# Get list of all files to lint #
#################################
mapfile -t LIST_FILES < <(find "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER/$INDVIDUAL_TEST_FOLDER" -type f -regex "$FILE_EXTENSIONS" ! -path "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER/ansible/ghe-initialize/*" 2>&1)
mapfile -t LIST_FILES < <(find "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER/$INDVIDUAL_TEST_FOLDER" -type f -regex "$FILE_EXTENSIONS" ! -path "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER/ansible/ghe-initialize/*" | sort 2>&1)
########################################
# Prepare context if TAP output format #
########################################
if IsTAP ; then
TMPFILE=$(mktemp -q "/tmp/super-linter-${FILE_TYPE}.XXXXXX")
mkdir -p "${REPORT_OUTPUT_FOLDER}"
REPORT_OUTPUT_FILE="${REPORT_OUTPUT_FOLDER}/super-linter-${FILE_TYPE}.${OUTPUT_FORMAT}"
fi
##################
# Lint the files #
@ -406,6 +415,11 @@ function TestCodebase() {
#######################
ERROR_CODE=$?
########################################
# Increment counter that check was ran #
########################################
(("TESTS_RAN++"))
########################################
# Check for if it was supposed to pass #
########################################
@ -422,15 +436,17 @@ function TestCodebase() {
echo -e "${NC}${B[R]}${F[W]}ERROR:${NC} Linter CMD:[$LINTER_COMMAND $FILE]${NC}"
# Increment the error count
(("ERRORS_FOUND_$FILE_TYPE++"))
# Increment counter that check was ran
((TESTS_RAN++))
else
###########
# Success #
###########
echo -e "${NC}${F[B]} - File:${F[W]}[$FILE_NAME]${F[B]} was linted with ${F[W]}[$LINTER_NAME]${F[B]} successfully${NC}"
# Increment counter that check was ran
((TESTS_RAN++))
fi
#######################################################
# Store the linting as a temporary file in TAP format #
#######################################################
if IsTAP ; then
echo "ok ${TESTS_RAN} - ${FILE_NAME}" >> "${TMPFILE}"
fi
else
#######################################
@ -449,19 +465,60 @@ function TestCodebase() {
echo -e "${NC}${B[R]}${F[W]}ERROR:${NC} Linter CMD:[$LINTER_COMMAND $FILE]${NC}"
# Increment the error count
(("ERRORS_FOUND_$FILE_TYPE++"))
# Increment counter that check was ran
((TESTS_RAN++))
else
###########
# Success #
###########
echo -e "${NC}${F[B]} - File:${F[W]}[$FILE_NAME]${F[B]} failed test case with ${F[W]}[$LINTER_NAME]${F[B]} successfully${NC}"
# Increment counter that check was ran
((TESTS_RAN++))
fi
#######################################################
# Store the linting as a temporary file in TAP format #
#######################################################
if IsTAP ; then
echo "not ok ${TESTS_RAN} - ${FILE_NAME}" >> "${TMPFILE}"
##########################################
# Report the detailed message if enabled #
##########################################
DETAILED_MSG=$(TransformTAPDetails "$LINT_CMD")
if [ -n "${DETAILED_MSG}" ] ; then
printf " ---\n message: %s\n ...\n" "${DETAILED_MSG}" >> "${TMPFILE}"
fi
fi
fi
done
###########################################################################
# Generate report in TAP format and validate with the expected TAP output #
###########################################################################
if IsTAP && [ ${TESTS_RAN} -gt 0 ] ; then
printf "TAP version 13\n1..%s\n" "${TESTS_RAN}" > "${REPORT_OUTPUT_FILE}"
cat "${TMPFILE}" >> "${REPORT_OUTPUT_FILE}"
########################################################################
# If expected TAP report exists then compare with the generated report #
########################################################################
EXPECTED_FILE="$GITHUB_WORKSPACE/$TEST_CASE_FOLDER/$INDVIDUAL_TEST_FOLDER/reports/expected-${FILE_TYPE}.tap"
if [ -e "$EXPECTED_FILE" ] ; then
TMPFILE=$(mktemp -q "/tmp/diff-${FILE_TYPE}.XXXXXX")
## Ignore white spaces, case sensitive
if ! diff -a -w -i "${EXPECTED_FILE}" "${REPORT_OUTPUT_FILE}" > "${TMPFILE}" 2>&1; then
#############################################
# We failed to compare the reporting output #
#############################################
echo -e "${NC}${B[R]}${F[W]}ERROR!${NC} Failed to assert TAP output:[$LINTER_NAME]${NC}"!
echo "Please validate the asserts!"
cat "${TMPFILE}"
exit 1
else
# Success
echo -e "${NC}${F[B]}Successfully validation in the expected TAP format for ${F[W]}[$LINTER_NAME]${NC}"
fi
else
echo -e "${NC}${F[Y]}WARN!${NC} No TAP expected file found at:[$EXPECTED_FILE]${NC}"
echo "skipping report assertions"
fi
fi
##############################
# Validate we ran some tests #
##############################