From ab8780a58f056cc470449cf96a41982c742dcaf8 Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Wed, 27 Jan 2021 20:47:34 +0100 Subject: [PATCH] Install, configure, and run jscpd. Add missing tests (#1032) Co-authored-by: Gabo Co-authored-by: Matt Desmond Co-authored-by: Lukas Gravley --- .automation/test/ansible/ansible_good_1.yml | 4 +- .../defaults/main.yml | 0 .../files/ghe-initial-config.json | 0 .../files/ghe-license.ghl | 0 .../handlers/main.yml | 0 .../tasks/collectd-settings.yml | 0 .../tasks/ghe-api-config-apply.yml | 0 .../tasks/ghe-config-apply.yml | 0 .../tasks/ghe-initial-configuration.yml | 0 .../tasks/ghe-ldap-configuration.yml | 0 .../tasks/main.yml | 0 .../tasks/splunk-settings.yml | 0 .../templates/collectd-settings.json.j2 | 0 .../templates/forwarding.conf.j2 | 0 .../templates/ghe-config-apply.sh | 0 .../templates/ldap-settings.json.j2 | 0 .../templates/settings.json.j2 | 0 .../templates/splunk-settings.json.j2 | 0 .../test/ansible/reports/expected-ANSIBLE.tap | 2 +- .../javascript_standard/javascript_good_1.js | 106 ++------ .automation/test/jscpd/clojure_good_1.clj | 34 +++ .../test/jscpd/coffeescript_good_1.coffee | 84 +++++++ .automation/test/jscpd/csharp_good_01.cs | 12 + .automation/test/jscpd/css_good_01.css | 22 ++ .automation/test/jscpd/dart_good_1.dart | 5 + .../test/jscpd/gherkin_good_01.feature | 5 + .automation/test/jscpd/golang_good_01.go | 7 + .automation/test/jscpd/good/Dockerfile | 13 + .automation/test/jscpd/good/Dockerfile.dev | 13 + .automation/test/jscpd/groovy_good_01.groovy | 6 + .automation/test/jscpd/html_good_01.html | 13 + .automation/test/jscpd/java_good_1.java | 9 + .automation/test/jscpd/javascript_bad_1.js | 215 ++++++++++++++++ .automation/test/jscpd/json_good_1.json | 10 + .automation/test/jscpd/kotlint_good_1.kt | 6 + .automation/test/jscpd/latex_good_1.tex | 232 ++++++++++++++++++ .automation/test/jscpd/lua_good_1.lua | 12 + .automation/test/jscpd/markdown_good_1.md | 20 ++ .automation/test/jscpd/perl_good_1.pl | 38 +++ .automation/test/jscpd/php_good_1.php | 3 + .automation/test/jscpd/powershell_good_1.ps1 | 1 + .automation/test/jscpd/protobuf_good_1.proto | 18 ++ .automation/test/jscpd/python_bad_1.py | 195 +++++++++++++++ .automation/test/jscpd/r_good_1.r | 48 ++++ .../test/jscpd/reports/expected-JSCPD.tap | 40 +++ .automation/test/jscpd/ruby_good_1.rb | 23 ++ .automation/test/jscpd/shell_good_1.sh | 17 ++ .automation/test/jscpd/sql_good_1.sql | 1 + .../test/jscpd/typescript_good_1 copy.ts | 6 + .automation/test/jscpd/typescript_good_1.ts | 6 + .automation/test/jscpd/xml_good_1.xml | 6 + .automation/test/jscpd/yml_good_1.yml | 19 ++ .automation/test/jsx/jsx_bad_1.jsx | 1 + .automation/test/jsx/jsx_good_1.jsx | 1 + .automation/test/jsx/reports/expected-JSX.tap | 7 + ...api_bad_1.ymlopenapi => openapi_bad_1.yml} | 0 ...i_bad_2.jsonopenapi => openapi_bad_2.json} | 0 ...i_good_1.ymlopenapi => openapi_good_1.yml} | 0 ...good_2.jsonopenapi => openapi_good_2.json} | 0 .../test/openapi/reports/expected-OPENAPI.tap | 12 +- .automation/test/tsx/reports/expected-TSX.tap | 7 + .automation/test/tsx/tsx_bad_1.tsx | 1 + .automation/test/tsx/tsx_good_1.tsx | 1 + .github/workflows/deploy-DEV.yml | 2 +- README.md | 3 + TEMPLATES/.jscpd.json | 10 + dependencies/package.json | 1 + lib/functions/buildFileList.sh | 6 +- lib/functions/detectFiles.sh | 9 +- lib/functions/worker.sh | 22 +- lib/linter.sh | 6 +- 71 files changed, 1218 insertions(+), 122 deletions(-) rename .automation/test/ansible/{ghe-initialize => ghe_initialize}/defaults/main.yml (100%) rename .automation/test/ansible/{ghe-initialize => ghe_initialize}/files/ghe-initial-config.json (100%) rename .automation/test/ansible/{ghe-initialize => ghe_initialize}/files/ghe-license.ghl (100%) rename .automation/test/ansible/{ghe-initialize => ghe_initialize}/handlers/main.yml (100%) rename .automation/test/ansible/{ghe-initialize => ghe_initialize}/tasks/collectd-settings.yml (100%) rename .automation/test/ansible/{ghe-initialize => ghe_initialize}/tasks/ghe-api-config-apply.yml (100%) rename .automation/test/ansible/{ghe-initialize => ghe_initialize}/tasks/ghe-config-apply.yml (100%) rename .automation/test/ansible/{ghe-initialize => ghe_initialize}/tasks/ghe-initial-configuration.yml (100%) rename .automation/test/ansible/{ghe-initialize => ghe_initialize}/tasks/ghe-ldap-configuration.yml (100%) rename .automation/test/ansible/{ghe-initialize => ghe_initialize}/tasks/main.yml (100%) rename .automation/test/ansible/{ghe-initialize => ghe_initialize}/tasks/splunk-settings.yml (100%) rename .automation/test/ansible/{ghe-initialize => ghe_initialize}/templates/collectd-settings.json.j2 (100%) rename .automation/test/ansible/{ghe-initialize => ghe_initialize}/templates/forwarding.conf.j2 (100%) rename .automation/test/ansible/{ghe-initialize => ghe_initialize}/templates/ghe-config-apply.sh (100%) rename .automation/test/ansible/{ghe-initialize => ghe_initialize}/templates/ldap-settings.json.j2 (100%) rename .automation/test/ansible/{ghe-initialize => ghe_initialize}/templates/settings.json.j2 (100%) rename .automation/test/ansible/{ghe-initialize => ghe_initialize}/templates/splunk-settings.json.j2 (100%) create mode 100644 .automation/test/jscpd/clojure_good_1.clj create mode 100644 .automation/test/jscpd/coffeescript_good_1.coffee create mode 100644 .automation/test/jscpd/csharp_good_01.cs create mode 100644 .automation/test/jscpd/css_good_01.css create mode 100644 .automation/test/jscpd/dart_good_1.dart create mode 100644 .automation/test/jscpd/gherkin_good_01.feature create mode 100644 .automation/test/jscpd/golang_good_01.go create mode 100644 .automation/test/jscpd/good/Dockerfile create mode 100644 .automation/test/jscpd/good/Dockerfile.dev create mode 100644 .automation/test/jscpd/groovy_good_01.groovy create mode 100644 .automation/test/jscpd/html_good_01.html create mode 100644 .automation/test/jscpd/java_good_1.java create mode 100644 .automation/test/jscpd/javascript_bad_1.js create mode 100644 .automation/test/jscpd/json_good_1.json create mode 100644 .automation/test/jscpd/kotlint_good_1.kt create mode 100644 .automation/test/jscpd/latex_good_1.tex create mode 100644 .automation/test/jscpd/lua_good_1.lua create mode 100644 .automation/test/jscpd/markdown_good_1.md create mode 100644 .automation/test/jscpd/perl_good_1.pl create mode 100644 .automation/test/jscpd/php_good_1.php create mode 100644 .automation/test/jscpd/powershell_good_1.ps1 create mode 100644 .automation/test/jscpd/protobuf_good_1.proto create mode 100644 .automation/test/jscpd/python_bad_1.py create mode 100644 .automation/test/jscpd/r_good_1.r create mode 100644 .automation/test/jscpd/reports/expected-JSCPD.tap create mode 100644 .automation/test/jscpd/ruby_good_1.rb create mode 100755 .automation/test/jscpd/shell_good_1.sh create mode 100644 .automation/test/jscpd/sql_good_1.sql create mode 100644 .automation/test/jscpd/typescript_good_1 copy.ts create mode 100644 .automation/test/jscpd/typescript_good_1.ts create mode 100644 .automation/test/jscpd/xml_good_1.xml create mode 100644 .automation/test/jscpd/yml_good_1.yml create mode 100644 .automation/test/jsx/jsx_bad_1.jsx create mode 100644 .automation/test/jsx/jsx_good_1.jsx create mode 100644 .automation/test/jsx/reports/expected-JSX.tap rename .automation/test/openapi/{openapi_bad_1.ymlopenapi => openapi_bad_1.yml} (100%) rename .automation/test/openapi/{openapi_bad_2.jsonopenapi => openapi_bad_2.json} (100%) rename .automation/test/openapi/{openapi_good_1.ymlopenapi => openapi_good_1.yml} (100%) rename .automation/test/openapi/{openapi_good_2.jsonopenapi => openapi_good_2.json} (100%) create mode 100644 .automation/test/tsx/reports/expected-TSX.tap create mode 100644 .automation/test/tsx/tsx_bad_1.tsx create mode 100644 .automation/test/tsx/tsx_good_1.tsx create mode 100644 TEMPLATES/.jscpd.json diff --git a/.automation/test/ansible/ansible_good_1.yml b/.automation/test/ansible/ansible_good_1.yml index 08edef72..dc2db333 100644 --- a/.automation/test/ansible/ansible_good_1.yml +++ b/.automation/test/ansible/ansible_good_1.yml @@ -22,7 +22,7 @@ github_host: "{{ hostvars['github_primary'].ansible_host }}" probot_server_ip: "{{ hostvars['backup-utils'].ansible_host }}" roles: - - role: ghe-initialize + - role: ghe_initialize ########################################## ## Run ghe-config-apply for all changes ## @@ -36,7 +36,7 @@ - block: - name: GHE-Config-Apply include_role: - name: ghe-initialize + name: ghe_initialize tasks_from: ghe-config-apply.yml tags: - github diff --git a/.automation/test/ansible/ghe-initialize/defaults/main.yml b/.automation/test/ansible/ghe_initialize/defaults/main.yml similarity index 100% rename from .automation/test/ansible/ghe-initialize/defaults/main.yml rename to .automation/test/ansible/ghe_initialize/defaults/main.yml diff --git a/.automation/test/ansible/ghe-initialize/files/ghe-initial-config.json b/.automation/test/ansible/ghe_initialize/files/ghe-initial-config.json similarity index 100% rename from .automation/test/ansible/ghe-initialize/files/ghe-initial-config.json rename to .automation/test/ansible/ghe_initialize/files/ghe-initial-config.json diff --git a/.automation/test/ansible/ghe-initialize/files/ghe-license.ghl b/.automation/test/ansible/ghe_initialize/files/ghe-license.ghl similarity index 100% rename from .automation/test/ansible/ghe-initialize/files/ghe-license.ghl rename to .automation/test/ansible/ghe_initialize/files/ghe-license.ghl diff --git a/.automation/test/ansible/ghe-initialize/handlers/main.yml b/.automation/test/ansible/ghe_initialize/handlers/main.yml similarity index 100% rename from .automation/test/ansible/ghe-initialize/handlers/main.yml rename to .automation/test/ansible/ghe_initialize/handlers/main.yml diff --git a/.automation/test/ansible/ghe-initialize/tasks/collectd-settings.yml b/.automation/test/ansible/ghe_initialize/tasks/collectd-settings.yml similarity index 100% rename from .automation/test/ansible/ghe-initialize/tasks/collectd-settings.yml rename to .automation/test/ansible/ghe_initialize/tasks/collectd-settings.yml diff --git a/.automation/test/ansible/ghe-initialize/tasks/ghe-api-config-apply.yml b/.automation/test/ansible/ghe_initialize/tasks/ghe-api-config-apply.yml similarity index 100% rename from .automation/test/ansible/ghe-initialize/tasks/ghe-api-config-apply.yml rename to .automation/test/ansible/ghe_initialize/tasks/ghe-api-config-apply.yml diff --git a/.automation/test/ansible/ghe-initialize/tasks/ghe-config-apply.yml b/.automation/test/ansible/ghe_initialize/tasks/ghe-config-apply.yml similarity index 100% rename from .automation/test/ansible/ghe-initialize/tasks/ghe-config-apply.yml rename to .automation/test/ansible/ghe_initialize/tasks/ghe-config-apply.yml diff --git a/.automation/test/ansible/ghe-initialize/tasks/ghe-initial-configuration.yml b/.automation/test/ansible/ghe_initialize/tasks/ghe-initial-configuration.yml similarity index 100% rename from .automation/test/ansible/ghe-initialize/tasks/ghe-initial-configuration.yml rename to .automation/test/ansible/ghe_initialize/tasks/ghe-initial-configuration.yml diff --git a/.automation/test/ansible/ghe-initialize/tasks/ghe-ldap-configuration.yml b/.automation/test/ansible/ghe_initialize/tasks/ghe-ldap-configuration.yml similarity index 100% rename from .automation/test/ansible/ghe-initialize/tasks/ghe-ldap-configuration.yml rename to .automation/test/ansible/ghe_initialize/tasks/ghe-ldap-configuration.yml diff --git a/.automation/test/ansible/ghe-initialize/tasks/main.yml b/.automation/test/ansible/ghe_initialize/tasks/main.yml similarity index 100% rename from .automation/test/ansible/ghe-initialize/tasks/main.yml rename to .automation/test/ansible/ghe_initialize/tasks/main.yml diff --git a/.automation/test/ansible/ghe-initialize/tasks/splunk-settings.yml b/.automation/test/ansible/ghe_initialize/tasks/splunk-settings.yml similarity index 100% rename from .automation/test/ansible/ghe-initialize/tasks/splunk-settings.yml rename to .automation/test/ansible/ghe_initialize/tasks/splunk-settings.yml diff --git a/.automation/test/ansible/ghe-initialize/templates/collectd-settings.json.j2 b/.automation/test/ansible/ghe_initialize/templates/collectd-settings.json.j2 similarity index 100% rename from .automation/test/ansible/ghe-initialize/templates/collectd-settings.json.j2 rename to .automation/test/ansible/ghe_initialize/templates/collectd-settings.json.j2 diff --git a/.automation/test/ansible/ghe-initialize/templates/forwarding.conf.j2 b/.automation/test/ansible/ghe_initialize/templates/forwarding.conf.j2 similarity index 100% rename from .automation/test/ansible/ghe-initialize/templates/forwarding.conf.j2 rename to .automation/test/ansible/ghe_initialize/templates/forwarding.conf.j2 diff --git a/.automation/test/ansible/ghe-initialize/templates/ghe-config-apply.sh b/.automation/test/ansible/ghe_initialize/templates/ghe-config-apply.sh similarity index 100% rename from .automation/test/ansible/ghe-initialize/templates/ghe-config-apply.sh rename to .automation/test/ansible/ghe_initialize/templates/ghe-config-apply.sh diff --git a/.automation/test/ansible/ghe-initialize/templates/ldap-settings.json.j2 b/.automation/test/ansible/ghe_initialize/templates/ldap-settings.json.j2 similarity index 100% rename from .automation/test/ansible/ghe-initialize/templates/ldap-settings.json.j2 rename to .automation/test/ansible/ghe_initialize/templates/ldap-settings.json.j2 diff --git a/.automation/test/ansible/ghe-initialize/templates/settings.json.j2 b/.automation/test/ansible/ghe_initialize/templates/settings.json.j2 similarity index 100% rename from .automation/test/ansible/ghe-initialize/templates/settings.json.j2 rename to .automation/test/ansible/ghe_initialize/templates/settings.json.j2 diff --git a/.automation/test/ansible/ghe-initialize/templates/splunk-settings.json.j2 b/.automation/test/ansible/ghe_initialize/templates/splunk-settings.json.j2 similarity index 100% rename from .automation/test/ansible/ghe-initialize/templates/splunk-settings.json.j2 rename to .automation/test/ansible/ghe_initialize/templates/splunk-settings.json.j2 diff --git a/.automation/test/ansible/reports/expected-ANSIBLE.tap b/.automation/test/ansible/reports/expected-ANSIBLE.tap index 5f81e1ed..ddb13710 100644 --- a/.automation/test/ansible/reports/expected-ANSIBLE.tap +++ b/.automation/test/ansible/reports/expected-ANSIBLE.tap @@ -2,6 +2,6 @@ TAP version 13 1..2 not ok 1 - ansible_bad_1.yml --- - message: Traceback (most recent call last) \n File "/usr/bin/ansible-lint", line 11, in \n load_entry_point('ansible-lint==4.2.0', 'console_scripts', 'ansible-lint')()\n File "/usr/lib/python3.8/site-packages/ansiblelint/__main__.py", line 187, in main\n matches.extend(runner.run())\n File "/usr/lib/python3.8/site-packages/ansiblelint/__init__.py", line 267, in run\n for child in ansiblelint.utils.find_children(arg, self.playbook_dir) \n File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 163, in find_children\n for child in play_children(basedir, item, playbook[1], playbook_dir) \n File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 215, in play_children\n return delegate_map[k](basedir, k, v, parent_type)\n File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 246, in _taskshandlers_children\n results.extend(_roles_children(basedir, k, [th['action'].get('name')],\n File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 285, in _roles_children\n results.extend(_look_for_role_files(basedir, role, main=main))\n File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 330, in _look_for_role_files\n role_path = _rolepath(basedir, role)\n File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 299, in _rolepath\n path_dwim(basedir, os.path.join('roles', role)),\n File "/usr/lib/python3.8/posixpath.py", line 90, in join\n genericpath._check_arg_types('join', a, *p)\n File "/usr/lib/python3.8/genericpath.py", line 152, in _check_arg_types\n raise TypeError(f'{funcname}() argument must be str, bytes, or '\nTypeError join() argument must be str, bytes, or os.PathLike object, not 'NoneType'\n + message: DEBUG Logging initialized to level 10\nDEBUG Options Namespace(listrules=False, format='rich', quiet=True, parseable=True, parseable_severity=False, progressive=False, rulesdir=[], use_default_rules=True, display_relative_path=True, tags=[], listtags=False, verbosity=2, skip_list=['602', '204', '301', '303', '305', '503'], warn_list=['experimental'], colored=False, exclude_paths=[], config_file='/tmp/lint/.github/linters/.ansible-lint.yml', playbook=['/tmp/lint/.automation/test/ansible/ansible_bad_1.yml'])\nDEBUG Loading rules from /usr/local/lib/python3.9/site-packages/ansiblelint/rules\nDEBUG Examining ansible_bad_1.yml of type playbook\nTraceback (most recent call last) \n File "/usr/local/bin/ansible-lint", line 8, in \n sys.exit(main())\n File "/usr/local/lib/python3.9/site-packages/ansiblelint/__main__.py", line 158, in main\n matches = _get_matches(rules, options)\n File "/usr/local/lib/python3.9/site-packages/ansiblelint/__main__.py", line 240, in _get_matches\n matches.extend(runner.run())\n File "/usr/local/lib/python3.9/site-packages/ansiblelint/runner.py", line 91, in run\n self.rules.run(file, tags=set(self.tags),\n File "/usr/local/lib/python3.9/site-packages/ansiblelint/rules/__init__.py", line 236, in run\n matches.extend(rule.matchtasks(playbookfile, text))\n File "/usr/local/lib/python3.9/site-packages/ansiblelint/rules/__init__.py", line 88, in matchtasks\n yaml = append_skipped_rules(yaml, text, file['type'])\n File "/usr/local/lib/python3.9/site-packages/ansiblelint/skip_utils.py", line 62, in append_skipped_rules\n yaml_skip = _append_skipped_rules(pyyaml_data, file_text, file_type)\n File "/usr/local/lib/python3.9/site-packages/ansiblelint/skip_utils.py", line 112, in _append_skipped_rules\n for ruamel_task, pyyaml_task in zip(ruamel_tasks, pyyaml_tasks) \n File "/usr/local/lib/python3.9/site-packages/ansiblelint/skip_utils.py", line 160, in _get_tasks_from_blocks\n for sub_task in get_nested_tasks(task) \n File "/usr/local/lib/python3.9/site-packages/ansiblelint/skip_utils.py", line 156, in \n for subtask in task[k]\nTypeError 'NoneType' object is not iterable\n ... ok 2 - ansible_good_1.yml diff --git a/.automation/test/javascript_standard/javascript_good_1.js b/.automation/test/javascript_standard/javascript_good_1.js index ad32089f..58dc8bfb 100644 --- a/.automation/test/javascript_standard/javascript_good_1.js +++ b/.automation/test/javascript_standard/javascript_good_1.js @@ -1,17 +1,15 @@ -var http = require('http') -var createHandler = require('github-webhook-handler') -var handler = createHandler({ path: '/webhook', secret: (process.env.SECRET) }) +const http = require('http') +const createHandler = require('github-webhook-handler') +const handler = createHandler({ path: '/webhook', secret: (process.env.SECRET) }) -var userArray = ['user1'] +const userArray = ['user1'] -var teamDescription = 'Team of Robots' -var teamPrivacy = 'closed' // closed (visible) / secret (hidden) are options here +const teamDescription = 'Team of Robots' +const teamPrivacy = 'closed' // closed (visible) / secret (hidden) are options here -var teamName = process.env.GHES_TEAM_NAME -var teamAccess = 'pull' // pull,push,admin options here -var teamId = '' - -var orgRepos = [] +const teamName = process.env.GHES_TEAM_NAME +const teamAccess = 'pull' // pull,push,admin options here +const teamId = '' // var creator = "" @@ -31,61 +29,21 @@ handler.on('repository', function (event) { if (event.payload.action === 'created') { const repo = event.payload.repository.full_name console.log(repo) - const org = event.payload.repository.owner.login - getTeamID(org) setTimeout(checkTeamIDVariable, 1000) } }) handler.on('team', function (event) { -// TODO user events such as being removed from team or org + // TODO user events such as being removed from team or org if (event.payload.action === 'deleted') { // const name = event.payload.team.name - const org = event.payload.organization.login - getRepositories(org) setTimeout(checkReposVariable, 5000) } else if (event.payload.action === 'removed_from_repository') { - const org = event.payload.organization.login - getTeamID(org) // const repo = event.payload.repository.full_name setTimeout(checkTeamIDVariable, 1000) } }) -function getTeamID (org) { - const https = require('https') - - const options = { - hostname: (process.env.GHE_HOST), - port: 443, - path: '/api/v3/orgs/' + org + '/teams', - method: 'GET', - headers: { - Authorization: 'token ' + (process.env.GHE_TOKEN), - 'Content-Type': 'application/json' - } - } - let body = [] - const req = https.request(options, (res) => { - res.on('data', (chunk) => { - body.push(chunk) - }).on('end', () => { - body = JSON.parse(Buffer.concat(body)) - body.forEach(item => { - if (item.name === teamName) { - teamId = item.id - } - }) - }) - }) - - req.on('error', (error) => { - console.error(error) - }) - - req.end() -} - function checkTeamIDVariable (repo) { if (typeof teamId !== 'undefined') { addTeamToRepo(repo, teamId) @@ -94,9 +52,9 @@ function checkTeamIDVariable (repo) { function checkReposVariable (org) { if (typeof orgRepos !== 'undefined') { - // for(var repo of orgRepos) { - // addTeamToRepo(repo, teamId) - // } + // for(var repo of orgRepos) { + // addTeamToRepo(repo, teamId) + // } reCreateTeam(org) } } @@ -143,8 +101,7 @@ function reCreateTeam (org) { name: teamName, description: teamDescription, privacy: teamPrivacy, - maintainers: userArray, - repo_names: orgRepos + maintainers: userArray }) const options = { @@ -178,38 +135,3 @@ function reCreateTeam (org) { req.write(data) req.end() } - -function getRepositories (org) { - orgRepos = [] - - const https = require('https') - - const options = { - hostname: (process.env.GHE_HOST), - port: 443, - path: '/api/v3/orgs/' + org + '/repos', - method: 'GET', - headers: { - Authorization: 'token ' + (process.env.GHE_TOKEN), - 'Content-Type': 'application/json' - } - } - let body = [] - const req = https.request(options, (res) => { - res.on('data', (chunk) => { - body.push(chunk) - }).on('end', () => { - body = JSON.parse(Buffer.concat(body)) - body.forEach(item => { - orgRepos.push(item.full_name) - console.log(item.full_name) - }) - }) - }) - - req.on('error', (error) => { - console.error(error) - }) - - req.end() -} diff --git a/.automation/test/jscpd/clojure_good_1.clj b/.automation/test/jscpd/clojure_good_1.clj new file mode 100644 index 00000000..f10b9497 --- /dev/null +++ b/.automation/test/jscpd/clojure_good_1.clj @@ -0,0 +1,34 @@ +(ns foo + (:require + [clojure.string :as str])) + +(butlast [1 2 3]) + +(str/join "" "") + +(defn foo-fn [x] + (let [y (fn [] (inc x))] + (y))) + +(letfn + [(f [g] (h g)) + (h [i] (f i))]) + +(defn foo [] 1) +(inc (foo)) + +(Thread/sleep 1000 1) + +;; Here we switch to another namespace and require the previous: +(ns bar (:require [foo :as f])) + +(f/foo-fn 1) + +{:a 1 :b 2} +#{1 2} +{:a 1 :b 2} + +(ns bar-test (:require [clojure.test :as t])) + +(t/deftest my-tests + (t/is (odd? (inc 1)))) diff --git a/.automation/test/jscpd/coffeescript_good_1.coffee b/.automation/test/jscpd/coffeescript_good_1.coffee new file mode 100644 index 00000000..388f28d7 --- /dev/null +++ b/.automation/test/jscpd/coffeescript_good_1.coffee @@ -0,0 +1,84 @@ +# Description +# silly hubot scripts +# These were created to blow off steam +# +# Commands: +# `mona echo *` - repeats what you say +# +# Author: +# admiralawkbar@github.com + +############################### +# Drop Hammer array of images # +############################### +dropHammer = [ + "https://s1.yimg.com/uu/api/res/1.2/.kFQAfQ6KQmlf5ip8.UzNA--/dz0xMjMwO2g9NjkyO2FwcGlkPXl0YWNoeW9u/http://media.zenfs.com/en-US/video/video.snl.com/SNL_1554_08_Update_03_Harry_Caray.png", + "http://media.tumblr.com/d12ea80b3a86dfc5fe36d3f306254fe4/tumblr_inline_mq1r0tbBCb1qz4rgp.jpg", + "http://the-artifice.com/wp-content/uploads/2014/01/94309-160x160.png", + "http://25.media.tumblr.com/35826348f2215069835c1733c75b29aa/tumblr_muuxmmBaOI1rw3gqyo2_250.gif", + "http://data2.whicdn.com/images/78766805/large.jpg", + "http://filmfisher.com/wp-content/uploads/2014/11/hunt_for_red_october.jpg", + "http://cdn.meme.am/instances/500x/57495736.jpg", +] + +################### +# Thank you array # +################### +thanks = [ + "You're welcome! Piece of cake...", + "It was nothing..." + "De nada...", + "Danke...", + "Merci...", + "Bitte...", + "De rien..." + "Prego..." +] + +################################# +# Start the robot for listening # +################################# +module.exports = (robot) -> + + ############################## + # Show the adapter connected # + ############################## + robot.respond /ADAPTER$/i, (msg) -> + msg.send robot.adapterName + + ########################## + # Echo back the response # + ########################## + robot.respond /ECHO (.*)$/i, (msg) -> + msg.send msg.match[1] + + ################## + # Whats going on # + ################## + robot.respond /whats going on/i, (msg) -> + msg.send "not much... robot stuff..." + + ################### + # Drop the hammer # + ################### + robot.respond /drop the hammer/i, (msg) -> + msg.send "Commencing the hammer dropping..." + msg.send msg.random dropHammer + + ############### + # Vape Nation # + ############### + robot.respond /lets roll/i, (msg) -> + msg.send "First Class! Vape Nation!!! @beardofedu" + + ############## + # Hubot Ping # + ############## + robot.respond /PING$/i, (msg) -> + msg.send "PONG" + +####################### +####################### +## END OF THE SCRIPT ## +####################### +####################### diff --git a/.automation/test/jscpd/csharp_good_01.cs b/.automation/test/jscpd/csharp_good_01.cs new file mode 100644 index 00000000..b17954e5 --- /dev/null +++ b/.automation/test/jscpd/csharp_good_01.cs @@ -0,0 +1,12 @@ +using System; + +namespace HelloWorld +{ + class Program + { + static void Main(string[] args) + { + Console.WriteLine("Hello World!"); + } + } +} diff --git a/.automation/test/jscpd/css_good_01.css b/.automation/test/jscpd/css_good_01.css new file mode 100644 index 00000000..a126e5b2 --- /dev/null +++ b/.automation/test/jscpd/css_good_01.css @@ -0,0 +1,22 @@ +/** + * Multi-line comment + */ + +.selector-1, +.selector-2, +.selector-3[type="text"] { + background: linear-gradient(#fff, rgba(0, 0, 0, 0.8)); + box-sizing: border-box; + display: block; + color: #333; +} + +.selector-a, +.selector-b:not(:first-child) { + padding: 10px !important; + top: calc(calc(1em * 2) / 3); +} + +.selector-x { width: 10%; } +.selector-y { width: 20%; } +.selector-z { width: 30%; } diff --git a/.automation/test/jscpd/dart_good_1.dart b/.automation/test/jscpd/dart_good_1.dart new file mode 100644 index 00000000..ba9c09c5 --- /dev/null +++ b/.automation/test/jscpd/dart_good_1.dart @@ -0,0 +1,5 @@ +// empty_constructor_bodies good ; +class Point { + int x, y; + Point(this.x, this.y); +} diff --git a/.automation/test/jscpd/gherkin_good_01.feature b/.automation/test/jscpd/gherkin_good_01.feature new file mode 100644 index 00000000..b3dc9730 --- /dev/null +++ b/.automation/test/jscpd/gherkin_good_01.feature @@ -0,0 +1,5 @@ +Feature: Test for the no-dupe-scenario-names rule + +Scenario: This is a Scenario for no-dupe-scenario-names + Given I have 2 scenarios with the same name + Then I should see a no-dupe-scenario-names error diff --git a/.automation/test/jscpd/golang_good_01.go b/.automation/test/jscpd/golang_good_01.go new file mode 100644 index 00000000..c0481191 --- /dev/null +++ b/.automation/test/jscpd/golang_good_01.go @@ -0,0 +1,7 @@ +package main + +import "fmt" + +func main() { + fmt.Println("hello world") +} diff --git a/.automation/test/jscpd/good/Dockerfile b/.automation/test/jscpd/good/Dockerfile new file mode 100644 index 00000000..33ed48d0 --- /dev/null +++ b/.automation/test/jscpd/good/Dockerfile @@ -0,0 +1,13 @@ +FROM node:10 + +# Create app directory +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app + +# Install app dependencies +COPY package.json /usr/src/app/ +RUN npm install + +COPY server.js server.js +EXPOSE 3000 +CMD ["node", "server.js"] diff --git a/.automation/test/jscpd/good/Dockerfile.dev b/.automation/test/jscpd/good/Dockerfile.dev new file mode 100644 index 00000000..33ed48d0 --- /dev/null +++ b/.automation/test/jscpd/good/Dockerfile.dev @@ -0,0 +1,13 @@ +FROM node:10 + +# Create app directory +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app + +# Install app dependencies +COPY package.json /usr/src/app/ +RUN npm install + +COPY server.js server.js +EXPOSE 3000 +CMD ["node", "server.js"] diff --git a/.automation/test/jscpd/groovy_good_01.groovy b/.automation/test/jscpd/groovy_good_01.groovy new file mode 100644 index 00000000..90941461 --- /dev/null +++ b/.automation/test/jscpd/groovy_good_01.groovy @@ -0,0 +1,6 @@ +class Example { + static void main(String[] args) { + File file = new File("E:/Example.txt") + println "The file ${file.absolutePath} has ${file.length()} bytes" + } +} diff --git a/.automation/test/jscpd/html_good_01.html b/.automation/test/jscpd/html_good_01.html new file mode 100644 index 00000000..d3a039bf --- /dev/null +++ b/.automation/test/jscpd/html_good_01.html @@ -0,0 +1,13 @@ + + + + + + Document + + + + Good HTML! + + + diff --git a/.automation/test/jscpd/java_good_1.java b/.automation/test/jscpd/java_good_1.java new file mode 100644 index 00000000..70f50e7c --- /dev/null +++ b/.automation/test/jscpd/java_good_1.java @@ -0,0 +1,9 @@ +/** + * Represents a good Java file. +*/ +public static class JavaGood { + // Prints "Hello, World" to the terminal window. + private void helloWorld() { + System.out.println("Hello, World"); + } +} diff --git a/.automation/test/jscpd/javascript_bad_1.js b/.automation/test/jscpd/javascript_bad_1.js new file mode 100644 index 00000000..ad32089f --- /dev/null +++ b/.automation/test/jscpd/javascript_bad_1.js @@ -0,0 +1,215 @@ +var http = require('http') +var createHandler = require('github-webhook-handler') +var handler = createHandler({ path: '/webhook', secret: (process.env.SECRET) }) + +var userArray = ['user1'] + +var teamDescription = 'Team of Robots' +var teamPrivacy = 'closed' // closed (visible) / secret (hidden) are options here + +var teamName = process.env.GHES_TEAM_NAME +var teamAccess = 'pull' // pull,push,admin options here +var teamId = '' + +var orgRepos = [] + +// var creator = "" + +http.createServer(function (req, res) { + handler(req, res, function (err) { + console.log(err) + res.statusCode = 404 + res.end('no such location') + }) +}).listen(3000) + +handler.on('error', function (err) { + console.error('Error:', err.message) +}) + +handler.on('repository', function (event) { + if (event.payload.action === 'created') { + const repo = event.payload.repository.full_name + console.log(repo) + const org = event.payload.repository.owner.login + getTeamID(org) + setTimeout(checkTeamIDVariable, 1000) + } +}) + +handler.on('team', function (event) { +// TODO user events such as being removed from team or org + if (event.payload.action === 'deleted') { + // const name = event.payload.team.name + const org = event.payload.organization.login + getRepositories(org) + setTimeout(checkReposVariable, 5000) + } else if (event.payload.action === 'removed_from_repository') { + const org = event.payload.organization.login + getTeamID(org) + // const repo = event.payload.repository.full_name + setTimeout(checkTeamIDVariable, 1000) + } +}) + +function getTeamID (org) { + const https = require('https') + + const options = { + hostname: (process.env.GHE_HOST), + port: 443, + path: '/api/v3/orgs/' + org + '/teams', + method: 'GET', + headers: { + Authorization: 'token ' + (process.env.GHE_TOKEN), + 'Content-Type': 'application/json' + } + } + let body = [] + const req = https.request(options, (res) => { + res.on('data', (chunk) => { + body.push(chunk) + }).on('end', () => { + body = JSON.parse(Buffer.concat(body)) + body.forEach(item => { + if (item.name === teamName) { + teamId = item.id + } + }) + }) + }) + + req.on('error', (error) => { + console.error(error) + }) + + req.end() +} + +function checkTeamIDVariable (repo) { + if (typeof teamId !== 'undefined') { + addTeamToRepo(repo, teamId) + } +} + +function checkReposVariable (org) { + if (typeof orgRepos !== 'undefined') { + // for(var repo of orgRepos) { + // addTeamToRepo(repo, teamId) + // } + reCreateTeam(org) + } +} + +function addTeamToRepo (repo, teamId) { + const https = require('https') + const data = JSON.stringify({ + permission: teamAccess + }) + + const options = { + hostname: (process.env.GHE_HOST), + port: 443, + path: '/api/v3/teams/' + teamId + '/repos/' + repo, + method: 'PUT', + headers: { + Authorization: 'token ' + (process.env.GHE_TOKEN), + 'Content-Type': 'application/json', + 'Content-Length': data.length + } + } + let body = [] + const req = https.request(options, (res) => { + res.on('data', (chunk) => { + body.push(chunk) + }).on('end', () => { + body = Buffer.concat(body).toString() + console.log(res.statusCode) + console.log('added team to ' + repo) + }) + }) + + req.on('error', (error) => { + console.error(error) + }) + + req.write(data) + req.end() +} + +function reCreateTeam (org) { + const https = require('https') + const data = JSON.stringify({ + name: teamName, + description: teamDescription, + privacy: teamPrivacy, + maintainers: userArray, + repo_names: orgRepos + }) + + const options = { + hostname: (process.env.GHE_HOST), + port: 443, + path: '/api/v3/orgs/' + org + '/teams', + method: 'POST', + headers: { + Authorization: 'token ' + (process.env.GHE_TOKEN), + 'Content-Type': 'application/json', + 'Content-Length': data.length + } + } + // const body = [] + const req = https.request(options, (res) => { + if (res.statusCode !== 201) { + console.log('Status code: ' + res.statusCode) + console.log('Added ' + teamName + ' to ' + org + ' Failed') + res.on('data', function (chunk) { + console.log('BODY: ' + chunk) + }) + } else { + console.log('Added ' + teamName + ' to ' + org) + } + }) + + req.on('error', (error) => { + console.error(error) + }) + + req.write(data) + req.end() +} + +function getRepositories (org) { + orgRepos = [] + + const https = require('https') + + const options = { + hostname: (process.env.GHE_HOST), + port: 443, + path: '/api/v3/orgs/' + org + '/repos', + method: 'GET', + headers: { + Authorization: 'token ' + (process.env.GHE_TOKEN), + 'Content-Type': 'application/json' + } + } + let body = [] + const req = https.request(options, (res) => { + res.on('data', (chunk) => { + body.push(chunk) + }).on('end', () => { + body = JSON.parse(Buffer.concat(body)) + body.forEach(item => { + orgRepos.push(item.full_name) + console.log(item.full_name) + }) + }) + }) + + req.on('error', (error) => { + console.error(error) + }) + + req.end() +} diff --git a/.automation/test/jscpd/json_good_1.json b/.automation/test/jscpd/json_good_1.json new file mode 100644 index 00000000..de60584a --- /dev/null +++ b/.automation/test/jscpd/json_good_1.json @@ -0,0 +1,10 @@ +{ + "arrow_spacing": { + "level": "ignore" + }, + "braces_spacing": { + "level": "ignore", + "spaces": 0, + "empty_object_spaces": 0 + } +} diff --git a/.automation/test/jscpd/kotlint_good_1.kt b/.automation/test/jscpd/kotlint_good_1.kt new file mode 100644 index 00000000..fdda8156 --- /dev/null +++ b/.automation/test/jscpd/kotlint_good_1.kt @@ -0,0 +1,6 @@ +fun main() { + val n = "World" + val v = "Hello, $n!" + + println(v) +} diff --git a/.automation/test/jscpd/latex_good_1.tex b/.automation/test/jscpd/latex_good_1.tex new file mode 100644 index 00000000..5b7de792 --- /dev/null +++ b/.automation/test/jscpd/latex_good_1.tex @@ -0,0 +1,232 @@ +Note: This file was written with only two purposes in mind: + o To test the program upon it + o To show off some of the features + +Most of the file does thus consist of lots of pseudo-commands, which +are nonsense in a TeXnical manner. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% Warning 1 + +\foo\ This is an error. +So is this \foo\ +\smallskip This is a not. $\foo Neither$ is this. + +\startsection[title={Testing ConTeXt}] +These should now be an error. +\stopsection + +% Warning 2 + +%This is a faulty reference to~\ref{foo} +This is not a faulty reference to~\ref{foo} + +% Warning 3 + +%$[(ab)^{-1}]^{-2}$ is not beautiful +${{[{(ab)}^{-1}]}}^{-2}$ is beautiful + +% Warning 4-6, 28 + +%Testing {\it italic} in \/ this {\em sentence\/}, as {\em you \/ see\/}. +Testing {\it italic\/} in this {\em sentence,} as {\em you see}. + +% LaTeX2e + +%Testing \textem{italic} in \/ this \textit{sentence\/}, as \textem{you \/ see\/}. +Testing \textem{italic} in this \textit{sentence}, as \textem{you see}. + +% Warning 7 + +%This \'is a test of $\hat{j}$ accents. +This \'{\i}s a test of $\hat{\jmath}$ accents. + +% Warning 8 + +%It wasn't anything - just a 2---3 star--shots. +It wasn't anything --- just a 2--3 star-shots. +It's just a start-shot. +is also used to send cross-calls (xc) and cross-traps (xt) to other +% From Knuths TeXbook Chapter 14 +% "How TeX Breaks Paragraphs into Lines", fourth paragraph: +in plain TeX---are the key + +% Using DashExcpt +The Birch--Swinnerton-Dyer conjecture is correct. +%The Birch--Swinnerton--Dyer conjecture is not correct. +The Birch-Swinnerton-Dyer conjecture is not correct (but not caught). + +% Warning 9-10 + +% Brackets: + +%)}{[])} }}}]]]))) +{[]} ((([[[{{{}}}]]]))) + +% Envs: + +%\begin{quux} \begin{baz} \end{quux} \end{baz} \end{asoi} \begin{dobedo} + +\begin{foo} \begin{bar} \end{bar}\end{foo} + +% Warning 11 + +%Foo...bar. $1,...,3$. $1+...+3$. $1,\cdots,3$. $1\cdot\ldots\cdot3$. +Foo\dots bar. $1,\ldots,3$. $1+\cdots+3$. $1,\ldots,3$. $1\cdot\cdots\cdot3$. + +% Warning 12 + +%1st. Foo Inc. Ab.cd. foo ab.cd. Foo. bar baz., billy.; bob.: joe.! frank.? james.. george +1st.\ foo Inc.\ ab.cd.\ foo ab.cd.\ Foo.\ bar baz., billy.; bob.:\ joe.!\ frank.?\ james..\ george + +% Warning 13 + +%Look at THIS! It's an error. +Look at THIS\@! It's an error. D. E. Knuth. + +% Warning 14 + +%\hat +\hat{a} + +% Warning 18,19 + +%Is this an "example", or is it an �example�. +Is this an `example', or is it an `example'. + +% Warning 20 + +%That bug is \unknown\ to me. +% That bug is \unknown\ to me. + +% Warning 21 + +\LaTeX\ is an extension of \TeX\. Right? +\LaTeX\ is an extension of \TeX. Right? + +% Warning 23 + +%```Hello', I heard him said'', she remembered. +``\,`Hello', I heard him said'', she remembered. + +% Warning 24 + +%Indexing text \index{text} is fun! +Indexing text\index{text} is fun! +Indexing text% + \index{text} is fun! +%Indexing text +% \index{text} is fun! + +% Warning 25 + +%$5\cdot10^10$ +$5\cdot10^{10}$ + +% Warning 26 + +%Do you understand ? +Do you understand? + +% Warning 29 +%The program opens a screen sized 640x200 pixels +The program opens a screen sized $640\times200$ pixels + +% Warning 30 + +%White is a beautiful colour. +White is a beautiful colour. + +% Warning 31 +\begin{verbatim} +\this is +\end{verbatim} %foo bar + +% Warning 32-34 + +%This is either an 'example`, an ''example`` or an `"`example'`'. +This is either an `example', an ``example'' or an ``example''. + +% Warning 35 + +%$sin^2 + cos^2 = 1$ +$\sin^2 + \cos^2 = 1$ + +% Warning 36-37 + +%This( an example( Nuff said )), illustrates( ``my'' )point. +This (an example (Nuff said)), illustrates (``my'') point. + +% Warning 38 +%``An example,'' he said, ``would be great.'' +``An example'', he said, ``would be great''. + +% Warning 39 + +%For output codes, see table ~\ref{tab:fmtout}. +For output codes, see table~\ref{tab:fmtout}. + +% Warning 40 +%$\this,$ and $$this$$. +$\this$, and \[this.\] + +% Warning 41 +%foo \above\ qux +\frac{foo}{qux} + +% Warning 42 +%This is a footnote \footnote{foo}. +This is a footnote\footnote{foo}. + +% Warning 43 +%Here is a mistake $\left{x\right}$. +%This one triggers warning 22 $\left\{x\right\}$. +Here \chktex\ doesn't complain $\left\lbrace x\right\rbrace$. + +% Warning 44 -- user regex -- default message +%You should always write a good intro. +You should always write a good introduction. + +% Warning 44 -- user regex -- user message +%For every $p\not|n$ you have an ugly prime which doesn't divide $n$. +For every $p\nmid n$ you have a cute prime which doesn't divide $n$. + +% Math mode check +\ensuremath{\sin\ x\text{is not the same as \sin\ x, but is the same as $\sin x$}} +Also, $x(3)\text{ is not x (3) but it is $x(3)$}$ + +This is\\% a comment. Nothing here should be checked(right)? +a broken line. +But this is not a \% comment, so we should find this error (right)? + +Here(on this line only)is a warning $sin(x)$ suppressed. % chktex 36 chktex 35 +%Here (on this line only)is a warning $sin(x)$ suppressed. % CHKTEX 35 36 + +%In section~\ref{sec:3} we have a warning. +In section~\ref{sec:4} it is suppressed. % chktex -1 +% In section~\ref{sec:5} we don't have a warning. + +% \begin{tabular*}{1.0\linewidth}[h]{|c|cc|} +% a & b \\ +% \hline +% c & d +% \end{tabular*} + +% Verb check + +\verb@\this is )() lots of errors, etc. Or what?@ +%\verb# + +\begin{verbatim} +\this is +\end{verbatim} + +% Warning 16,15 + +\[\] + +% Local Variables: +% require-final-newline: nil +% End: +% There should be no newline at the end of this file to test bug #46539 diff --git a/.automation/test/jscpd/lua_good_1.lua b/.automation/test/jscpd/lua_good_1.lua new file mode 100644 index 00000000..4826fc02 --- /dev/null +++ b/.automation/test/jscpd/lua_good_1.lua @@ -0,0 +1,12 @@ +local embracer = {} + +local function helper() + -- NYI wontfix +end + +function embracer.embrace(opt) + opt = opt or "default" + return helper(opt.."?") +end + +return embracer diff --git a/.automation/test/jscpd/markdown_good_1.md b/.automation/test/jscpd/markdown_good_1.md new file mode 100644 index 00000000..7a0aef45 --- /dev/null +++ b/.automation/test/jscpd/markdown_good_1.md @@ -0,0 +1,20 @@ +# Good Markdown + +This is just standard good markdown. + +## Second level header + +This header follows the step down from `level 1`. + +- Here it *is* + - Some more **indention** + - why so much? + +```bash +ls -la +``` + +### Walk away + +We're all done **here**. +- [Link Action](https://github.com) diff --git a/.automation/test/jscpd/perl_good_1.pl b/.automation/test/jscpd/perl_good_1.pl new file mode 100644 index 00000000..caca0994 --- /dev/null +++ b/.automation/test/jscpd/perl_good_1.pl @@ -0,0 +1,38 @@ +#!/usr/bin/perl +################################################################################ +################################################################################ +######### Script action @admiralawkbar ######################################### +################################################################################ + +############# +# Load Libs # +############# +use strict; + +$|=1; + +################## +#### GLOBALS: #### +################## +my $state = undef; # State to return to GHE +my $exitCode = 0; # Code to exit with +my $description = "Here it is"; # Description of the build + +############### +#### MAIN: #### +############### +Header(); # Basic print statements + +####################################################################### +#################### SUB ROUTINES BELOW ONLY ########################## +####################################################################### +####################################################################### +#### SUB ROUTINE Header ############################################### +sub Header +{ + print "-------------------------------------------------------------------\n"; + print "State:\[$state\]\n"; + print "ExitCode:\[$exitCode\]\n"; + print "Description:\[$description\]\n"; + print "-------------------------------------------------------------------\n"; +} diff --git a/.automation/test/jscpd/php_good_1.php b/.automation/test/jscpd/php_good_1.php new file mode 100644 index 00000000..16898d93 --- /dev/null +++ b/.automation/test/jscpd/php_good_1.php @@ -0,0 +1,3 @@ + my_map = 4; +} diff --git a/.automation/test/jscpd/python_bad_1.py b/.automation/test/jscpd/python_bad_1.py new file mode 100644 index 00000000..b3b4e7d1 --- /dev/null +++ b/.automation/test/jscpd/python_bad_1.py @@ -0,0 +1,195 @@ +import json +import sys +from os import getenv, path +from pprint import pprint + +import click # pylint: disable=import-error +import requests # pylint: disable=import-error +from dotenv import load_dotenv # pylint: disable=import-error + +env = load_dotenv() +api_url = getenv("API_URL", default="https://api.github.com/graphql") +github_token = getenv("GITHUB_TOKEN", default=None) + +if github_token is None: + sys.exit( + "GitHub Token is not set." + + "Please set the GITHUB_TOKEN env variable in your system or " + + "the .env file of your project." + ) + +client_id = getenv("CLIENT_ID", default="copy_labels.py") +headers = { + "Authorization": "bearer {github_token}".format(github_token=github_token), + "Accept": "application/vnd.github.bane-preview+json", + "Content-Type": "application/json", +} + + +def create_label(repo_id, label): + """ + Create label in the supplied repo. + + :param repo_id: Unique ID that represents the repo in GitHub + :type repo_id: str + :param label: Object with label information. + :type label: dict + :return: GitHub API request response + """ + + query_variables = { + "createLabelInput": { + "color": label["color"], + "description": label["description"], + "name": label["name"], + "repositoryId": repo_id, + } + } + + with open( + path.join(path.dirname(__file__), "queries/create_label.gql"), "r" + ) as query_file: + query = "".join(query_file.readlines()) + + payload = {"query": query, "variables": query_variables} + response = requests.post(api_url, data=json.dumps(payload), headers=headers).json() + print("Created label {label}".format(label=label["name"])) + + return response + + +def get_labels(owner, repo): + """ + Gets a list of labels from the supplied repo. + :param owner: Repo owner GitHub login. + :type owner: str + :param repo: Repository name. + :type repo: str + :return: A tuple with the GitHub id for the repository and a list of labels defined in the repository + """ + + query_variables = { + "owner": owner, + "name": repo, + } + + with open( + path.join(path.dirname(__file__), "queries/get_repo_data.gql"), "r" + ) as query_file: + query = "".join(query_file.readlines()) + + payload = {"query": query, "variables": query_variables} + response = requests.post(api_url, data=json.dumps(payload), headers=headers) + + status_code = response.status_code + result = response.json() + + if status_code >= 200 and status_code <= 300: + repo_id = result["data"]["repository"]["id"] + labels = result["data"]["repository"]["labels"]["nodes"] + + return repo_id, labels + else: + raise Exception( + "[ERROR] getting issue labels. Status Code: {status_code} - Message: {result}".format( + status_code=status_code, result=result["message"] + ) + ) + + +def delete_label(label_id): + """ + Delete the specified label + :param label_id: Label's node id. + :type label_id: str + :return: GitHub API request response. + """ + + query_variables = { + "deleteLabelInput": {"clientMutationId": client_id, "id": label_id} + } + + with open( + path.join(path.dirname(__file__), "queries/delete_label.gql"), "r" + ) as query_file: + query = "".join(query_file.readlines()) + + payload = {"query": query, "variables": query_variables} + result = requests.post(api_url, data=json.dumps(payload), headers=headers).json() + + return result + + +@click.command() +@click.option("--dry", is_flag=True) +@click.argument("source_repo") +@click.argument("target_repo") +def copy_labels(source_repo, target_repo, dry): + """ + Copy labels from the source repository to the target repository. + \f + :param source: The full name of a GitHub repo from where the labels will be copied from. Eg. github/opensourcefriday + :type source: str + :param target: The full name of a GitHub repo to where the labels will be copied. Eg. github/opensourcefriday + :type target: str + :return: + """ + source_owner, source_repo_name = source_repo.split("/") + target_owner, target_repo_name = target_repo.split("/") + + try: + print( + "Fetching labels for {source_repo_name} repo.".format( + source_repo_name=source_repo_name + ) + ) + _, source_repo_labels = get_labels(source_owner, source_repo_name) + print( + "Fetched labels for {source_repo_name}".format( + source_repo_name=source_repo_name + ) + ) + + print( + "Fetching labels for {target_repo_name} repo.".format( + target_repo_name=target_repo_name + ) + ) + target_repo_id, target_repo_labels = get_labels(target_owner, target_repo_name) + print( + "Fetched labels for {target_repo_name}".format( + target_repo_name=target_repo_name + ) + ) + + filtered_labels = list( + filter(lambda x: x not in target_repo_labels, source_repo_labels) + ) + + if dry: + print("This is just a dry run. No labels will be copied/created.") + print( + "{label_count} labels would have been created.".format( + label_count=len(filtered_labels) + ) + ) + pprint(filtered_labels, indent=4) + else: + print( + "Preparing to created {label_count} labels in {target_repo}".format( + label_count=len(filtered_labels), target_repo=target_repo + ) + ) + + for label in filtered_labels: + create_label(target_repo_id, label) + except Exception as error: + sys.exit(error) + + print("Done") + + +if __name__ == "__main__": + # Pylint doesn't know that @click.command takes care of injecting the + # function parameters. Disabling Pylint error. + copy_labels() # pylint: disable=no-value-for-parameter diff --git a/.automation/test/jscpd/r_good_1.r b/.automation/test/jscpd/r_good_1.r new file mode 100644 index 00000000..8bd91f2b --- /dev/null +++ b/.automation/test/jscpd/r_good_1.r @@ -0,0 +1,48 @@ +# Each of the default linters should throw at least one lint on this file + +# assignment +# function_left_parentheses +# closed_curly +# commas +# paren_brace +f <- function(x, y = 1) { + +} + +# commented_code + +# cyclocomp +# equals_na +# infix_spaces +# line_length +# object_length +# object_name +# object_usage +# open_curly +short_snake <- function(x) { + y <- 1 + y <- y^2 + if (1 > 2 && 5 * 10 > 6 && is.na(x)) { + TRUE + } else { + FALSE + } +} + +# pipe_continuation +# seq_linter +# spaces_inside +x <- 1:10 +x[2] +seq_len(x) %>% + lapply(function(x) x * 2) %>% + head() + +# single_quotes +message("single_quotes") + +# spaces_left_parentheses +# trailing_whitespace +y <- 2 + (1:10) + +# trailing_blank_lines diff --git a/.automation/test/jscpd/reports/expected-JSCPD.tap b/.automation/test/jscpd/reports/expected-JSCPD.tap new file mode 100644 index 00000000..9e0f2471 --- /dev/null +++ b/.automation/test/jscpd/reports/expected-JSCPD.tap @@ -0,0 +1,40 @@ +TAP version 13 +1..32 +ok 1 - clojure_good_1.clj +ok 2 - coffeescript_good_1.coffee +ok 3 - csharp_good_01.cs +ok 4 - css_good_01.css +ok 5 - dart_good_1.dart +ok 6 - gherkin_good_01.feature +ok 7 - golang_good_01.go +ok 8 - Dockerfile +ok 9 - Dockerfile.dev +ok 10 - groovy_good_01.groovy +ok 11 - html_good_01.html +ok 12 - java_good_1.java +not ok 13 - javascript_bad_1.js + --- + message: Clone found (javascript) \n - /tmp/lint/.automation/test/jscpd/javascript_bad_1.js [119 5 - 126 20] (7 lines, 82 tokens)\n /tmp/lint/.automation/test/jscpd/javascript_bad_1.js [66 5 - 73 18]\n\nClone found (javascript) \n - /tmp/lint/.automation/test/jscpd/javascript_bad_1.js [190 43 - 204 17] (14 lines, 145 tokens)\n /tmp/lint/.automation/test/jscpd/javascript_bad_1.js [61 43 - 75 11]\n\nClone found (javascript) \n - /tmp/lint/.automation/test/jscpd/javascript_bad_1.js [119 5 - 126 20] (7 lines, 82 tokens)\n /tmp/lint/.automation/test/jscpd/javascript_bad_1.js [66 5 - 73 18]\n\n 119 │ 66 │ } \n 120 │ 67 │ } \n 121 │ 68 │ let body = [] \n 122 │ 69 │ const req = https.request(options, (res) => { \n 123 │ 70 │ res.on('data', (chunk) => { \n 124 │ 71 │ body.push(chunk) \n 125 │ 72 │ }).on('end', () => { \n 126 │ 73 │ body = Buffer \n\nClone found (javascript) \n - /tmp/lint/.automation/test/jscpd/javascript_bad_1.js [190 43 - 204 17] (14 lines, 145 tokens)\n /tmp/lint/.automation/test/jscpd/javascript_bad_1.js [61 43 - 75 11]\n\n 190 │ 61 │ , \n 191 │ 62 │ method 'GET', \n 192 │ 63 │ headers { \n 193 │ 64 │ Authorization 'token ' + (process.env.GHE_TOKEN), \n 194 │ 65 │ 'Content-Type' 'application/json' \n 195 │ 66 │ } \n 196 │ 67 │ } \n 197 │ 68 │ let body = [] \n 198 │ 69 │ const req = https.request(options, (res) => { \n 199 │ 70 │ res.on('data', (chunk) => { \n 200 │ 71 │ body.push(chunk) \n 201 │ 72 │ }).on('end', () => { \n 202 │ 73 │ body = JSON.parse(Buffer.concat(body)) \n 203 │ 74 │ body.forEach(item => { \n 204 │ 75 │ orgRepos \n\nFound 2 clones.\nERROR jscpd found too many duplicates (9.81%) over threshold (0%)\nError ERROR jscpd found too many duplicates (9.81%) over threshold (0%)\n at ThresholdReporter.report (/node_modules/@jscpd/finder/dist/reporters/threshold.js)\n at /node_modules/@jscpd/finder/dist/in-files-detector.js\n at Array.forEach ()\n at /node_modules/@jscpd/finder/dist/in-files-detector.js\n + ... +ok 14 - json_good_1.json +ok 15 - kotlint_good_1.kt +ok 16 - latex_good_1.tex +ok 17 - lua_good_1.lua +ok 18 - markdown_good_1.md +ok 19 - perl_good_1.pl +ok 20 - php_good_1.php +ok 21 - powershell_good_1.ps1 +ok 22 - protobuf_good_1.proto +not ok 23 - python_bad_1.py + --- + message: Clone found (python) \n - /tmp/lint/.automation/test/jscpd/python_bad_1.py [77 70 - 84 16] (7 lines, 77 tokens)\n /tmp/lint/.automation/test/jscpd/python_bad_1.py [50 69 - 55 82]\n\nClone found (python) \n - /tmp/lint/.automation/test/jscpd/python_bad_1.py [77 70 - 84 16] (7 lines, 77 tokens)\n /tmp/lint/.automation/test/jscpd/python_bad_1.py [50 69 - 55 82]\n\n 77 │ 50 │ ), "r" \n 78 │ 51 │ ) as query_file \n 79 │ 52 │ query = "".join(query_file.readlines()) \n 80 │ 53 │ \n 81 │ 54 │ payload = {"query" query, "variables" query_variables} \n 82 │ 55 │ response = requests.post(api_url, data=json.dumps(payload), headers=headers) \n 83 │ 56 │ \n 84 │ 57 │ status_code \n\nFound 1 clones.\nERROR jscpd found too many duplicates (3.61%) over threshold (0%)\nError ERROR jscpd found too many duplicates (3.61%) over threshold (0%)\n at ThresholdReporter.report (/node_modules/@jscpd/finder/dist/reporters/threshold.js)\n at /node_modules/@jscpd/finder/dist/in-files-detector.js\n at Array.forEach ()\n at /node_modules/@jscpd/finder/dist/in-files-detector.js\n + ... +ok 24 - r_good_1.r +ok 25 - expected-JSCPD.tap +ok 26 - ruby_good_1.rb +ok 27 - shell_good_1.sh +ok 28 - sql_good_1.sql +ok 29 - typescript_good_1 copy.ts +ok 30 - typescript_good_1.ts +ok 31 - xml_good_1.xml +ok 32 - yml_good_1.yml diff --git a/.automation/test/jscpd/ruby_good_1.rb b/.automation/test/jscpd/ruby_good_1.rb new file mode 100644 index 00000000..0971f071 --- /dev/null +++ b/.automation/test/jscpd/ruby_good_1.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +# Rails Console only +# This script will output all active webhooks currently being processed by an instance. +# Replace ARRAY_OF_URLS_CALLING_INSTANCE and GHES_URL with the appropriate values before running + +# Prior to running this script, compile a list of the top URLs containing the phrase webhook +# This should be ran prior to entering the Rails Console with the command: +# grep -B1 --no-group-separator 'Faraday::TimeoutError' hookshot-logs/resqued.log | sed -n 1~2p | +# \ grep -v 'Faraday::TimeoutError: request timed out' | sort | uniq -c |sort -rn | head -n 20 + +File.open("/tmp/urls.txt", "w") do |file| + Hook.active.map do |h| + urls = [ARRAY_OF_URLS_CALLING_INSTANCE] + next if urls.include? h.url + + begin + file.puts "https://GHES_URL/api/v3/repos/#{h.installation_target.full_name}/hooks/#{h.id}" + rescue StandardError => e + puts e.message + end + end +end diff --git a/.automation/test/jscpd/shell_good_1.sh b/.automation/test/jscpd/shell_good_1.sh new file mode 100755 index 00000000..ee5435e5 --- /dev/null +++ b/.automation/test/jscpd/shell_good_1.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# CMD +HELLO_WORLD=$(echo "Hello World" | cut -f1 -d' ' 2>&1) + +# Load the error code +ERROR_CODE=$? + +# Check the shell +if [ ${ERROR_CODE} -ne 0 ]; then + echo "We did it!" + exit 0 +else + echo "We done goofed it..." + echo "${HELLO_WORLD}" + exit 1 +fi diff --git a/.automation/test/jscpd/sql_good_1.sql b/.automation/test/jscpd/sql_good_1.sql new file mode 100644 index 00000000..2e54affa --- /dev/null +++ b/.automation/test/jscpd/sql_good_1.sql @@ -0,0 +1 @@ +DELETE from person WHERE 1=1; diff --git a/.automation/test/jscpd/typescript_good_1 copy.ts b/.automation/test/jscpd/typescript_good_1 copy.ts new file mode 100644 index 00000000..68ef03e0 --- /dev/null +++ b/.automation/test/jscpd/typescript_good_1 copy.ts @@ -0,0 +1,6 @@ +const spiderman = (person) => { + return 'Hello, ' + person +} + +const user = 'Peter Parker' +console.log(spiderman(user)) diff --git a/.automation/test/jscpd/typescript_good_1.ts b/.automation/test/jscpd/typescript_good_1.ts new file mode 100644 index 00000000..68ef03e0 --- /dev/null +++ b/.automation/test/jscpd/typescript_good_1.ts @@ -0,0 +1,6 @@ +const spiderman = (person) => { + return 'Hello, ' + person +} + +const user = 'Peter Parker' +console.log(spiderman(user)) diff --git a/.automation/test/jscpd/xml_good_1.xml b/.automation/test/jscpd/xml_good_1.xml new file mode 100644 index 00000000..97de6219 --- /dev/null +++ b/.automation/test/jscpd/xml_good_1.xml @@ -0,0 +1,6 @@ + + Tove + Jani + Reminder + Don't forget me this weekend! + diff --git a/.automation/test/jscpd/yml_good_1.yml b/.automation/test/jscpd/yml_good_1.yml new file mode 100644 index 00000000..81221f0f --- /dev/null +++ b/.automation/test/jscpd/yml_good_1.yml @@ -0,0 +1,19 @@ +--- + +##################### +##################### +## Heres some vars ## +##################### +##################### + +############ +# Env Vars # +############ +env: + browser: true + es6: true + jest: true + +Here: there + +something: "For Nothing" diff --git a/.automation/test/jsx/jsx_bad_1.jsx b/.automation/test/jsx/jsx_bad_1.jsx new file mode 100644 index 00000000..cb96f0fe --- /dev/null +++ b/.automation/test/jsx/jsx_bad_1.jsx @@ -0,0 +1 @@ +const element =

Hello, world! diff --git a/.automation/test/jsx/jsx_good_1.jsx b/.automation/test/jsx/jsx_good_1.jsx new file mode 100644 index 00000000..f19f0a8f --- /dev/null +++ b/.automation/test/jsx/jsx_good_1.jsx @@ -0,0 +1 @@ +const element =

Hello, world!

; diff --git a/.automation/test/jsx/reports/expected-JSX.tap b/.automation/test/jsx/reports/expected-JSX.tap new file mode 100644 index 00000000..a5bb88c2 --- /dev/null +++ b/.automation/test/jsx/reports/expected-JSX.tap @@ -0,0 +1,7 @@ +TAP version 13 +1..2 +not ok 1 - jsx_bad_1.jsx + --- + message: \n/tmp/lint/.automation/test/jsx/jsx_bad_1.jsx\n 1 17 error Parsing error JSX element 'h1' has no corresponding closing tag\n\n✖ 1 problem (1 error, 0 warnings)\n + ... +ok 2 - jsx_good_1.jsx diff --git a/.automation/test/openapi/openapi_bad_1.ymlopenapi b/.automation/test/openapi/openapi_bad_1.yml similarity index 100% rename from .automation/test/openapi/openapi_bad_1.ymlopenapi rename to .automation/test/openapi/openapi_bad_1.yml diff --git a/.automation/test/openapi/openapi_bad_2.jsonopenapi b/.automation/test/openapi/openapi_bad_2.json similarity index 100% rename from .automation/test/openapi/openapi_bad_2.jsonopenapi rename to .automation/test/openapi/openapi_bad_2.json diff --git a/.automation/test/openapi/openapi_good_1.ymlopenapi b/.automation/test/openapi/openapi_good_1.yml similarity index 100% rename from .automation/test/openapi/openapi_good_1.ymlopenapi rename to .automation/test/openapi/openapi_good_1.yml diff --git a/.automation/test/openapi/openapi_good_2.jsonopenapi b/.automation/test/openapi/openapi_good_2.json similarity index 100% rename from .automation/test/openapi/openapi_good_2.jsonopenapi rename to .automation/test/openapi/openapi_good_2.json diff --git a/.automation/test/openapi/reports/expected-OPENAPI.tap b/.automation/test/openapi/reports/expected-OPENAPI.tap index a57d14bc..68c5e127 100644 --- a/.automation/test/openapi/reports/expected-OPENAPI.tap +++ b/.automation/test/openapi/reports/expected-OPENAPI.tap @@ -1,12 +1,12 @@ TAP version 13 1..4 -not ok 1 - openapi_bad_1.ymlopenapi +not ok 1 - openapi_bad_1.yml --- - message: OpenAPI 3.x detected\n\n/tmp/lint/.automation/test/openapi/openapi_bad_1.ymlopenapi\n 1 1 warning info-contact Info object should contain `contact` object.\n 1 1 warning info-description OpenAPI object info `description` must be present and non-empty string.\n 1 1 warning oas3-api-servers OpenAPI `servers` must be present and non-empty array.\n 1 1 error oas3-schema Object should have required property `info`.\n 1 1 warning openapi-tags OpenAPI object should have non-empty `tags` array.\n\n✖ 5 problems (1 error, 4 warnings, 0 infos, 0 hints)\n + message: OpenAPI 3.x detected\n\n/tmp/lint/.automation/test/openapi/openapi_bad_1.yml\n 1 1 warning info-contact Info object should contain `contact` object.\n 1 1 warning info-description OpenAPI object info `description` must be present and non-empty string.\n 1 1 warning oas3-api-servers OpenAPI `servers` must be present and non-empty array.\n 1 1 error oas3-schema Object should have required property `info`.\n 1 1 warning openapi-tags OpenAPI object should have non-empty `tags` array.\n\n✖ 5 problems (1 error, 4 warnings, 0 infos, 0 hints)\n ... -not ok 2 - openapi_bad_2.jsonopenapi +not ok 2 - openapi_bad_2.json --- - message: OpenAPI 3.x detected\n\n/tmp/lint/.automation/test/openapi/openapi_bad_2.jsonopenapi\n 1 1 warning info-contact Info object should contain `contact` object.\n 1 1 warning info-description OpenAPI object info `description` must be present and non-empty string.\n 1 1 warning oas3-api-servers OpenAPI `servers` must be present and non-empty array.\n 1 1 error oas3-schema Object should have required property `info`.\n 1 1 warning openapi-tags OpenAPI object should have non-empty `tags` array.\n\n✖ 5 problems (1 error, 4 warnings, 0 infos, 0 hints)\n + message: OpenAPI 3.x detected\n\n/tmp/lint/.automation/test/openapi/openapi_bad_2.json\n 1 1 warning info-contact Info object should contain `contact` object.\n 1 1 warning info-description OpenAPI object info `description` must be present and non-empty string.\n 1 1 warning oas3-api-servers OpenAPI `servers` must be present and non-empty array.\n 1 1 error oas3-schema Object should have required property `info`.\n 1 1 warning openapi-tags OpenAPI object should have non-empty `tags` array.\n\n✖ 5 problems (1 error, 4 warnings, 0 infos, 0 hints)\n ... -ok 3 - openapi_good_1.ymlopenapi -ok 4 - openapi_good_2.jsonopenapi +ok 3 - openapi_good_1.yml +ok 4 - openapi_good_2.json diff --git a/.automation/test/tsx/reports/expected-TSX.tap b/.automation/test/tsx/reports/expected-TSX.tap new file mode 100644 index 00000000..50dd9a79 --- /dev/null +++ b/.automation/test/tsx/reports/expected-TSX.tap @@ -0,0 +1,7 @@ +TAP version 13 +1..2 +not ok 1 - tsx_bad_1.tsx + --- + message: \n/tmp/lint/.automation/test/tsx/tsx_bad_1.tsx\n 1 16 error Parsing error Type expected\n\n✖ 1 problem (1 error, 0 warnings)\n + ... +ok 2 - tsx_good_1.tsx diff --git a/.automation/test/tsx/tsx_bad_1.tsx b/.automation/test/tsx/tsx_bad_1.tsx new file mode 100644 index 00000000..a6f6738f --- /dev/null +++ b/.automation/test/tsx/tsx_bad_1.tsx @@ -0,0 +1 @@ +var foo = bar as diff --git a/.automation/test/tsx/tsx_good_1.tsx b/.automation/test/tsx/tsx_good_1.tsx new file mode 100644 index 00000000..ac5b8f4f --- /dev/null +++ b/.automation/test/tsx/tsx_good_1.tsx @@ -0,0 +1 @@ +var foo = bar as foo; diff --git a/.github/workflows/deploy-DEV.yml b/.github/workflows/deploy-DEV.yml index 77dd0a28..32fa60a3 100644 --- a/.github/workflows/deploy-DEV.yml +++ b/.github/workflows/deploy-DEV.yml @@ -76,7 +76,7 @@ jobs: ##################################### - name: Run Test Cases shell: bash - run: docker run -e RUN_LOCAL=true -e TEST_CASE_RUN=true -e OUTPUT_FORMAT=tap -e OUTPUT_FOLDER=${GITHUB_SHA} -e OUTPUT_DETAILS=detailed -e ACTIONS_RUNNER_DEBUG=true -e ERROR_ON_MISSING_EXEC_BIT=true -v ${GITHUB_WORKSPACE}:/tmp/lint ghcr.io/github/super-linter:${GITHUB_SHA} + run: docker run -e RUN_LOCAL=true -e TEST_CASE_RUN=true -e ANSIBLE_DIRECTORY=.automation/test/ansible -e OUTPUT_FORMAT=tap -e OUTPUT_FOLDER=${GITHUB_SHA} -e OUTPUT_DETAILS=detailed -e ACTIONS_RUNNER_DEBUG=true -e ERROR_ON_MISSING_EXEC_BIT=true -v ${GITHUB_WORKSPACE}:/tmp/lint ghcr.io/github/super-linter:${GITHUB_SHA} ######################################### # Clean code base to run against it all # diff --git a/README.md b/README.md index acb24746..928e1da7 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ Developers on **GitHub** can call the **GitHub Action** to lint their code base | **CSS** | [stylelint](https://stylelint.io/) | | **Clojure** | [clj-kondo](https://github.com/borkdude/clj-kondo) | | **CoffeeScript** | [coffeelint](https://coffeelint.github.io/) | +| **Copy/paste detection** | [jscpd](https://github.com/kucherenko/jscpd) | | **Dart** | [dartanalyzer](https://dart.dev/guides/language/analysis-options) | | **Dockerfile** | [dockerfilelint](https://github.com/replicatedhq/dockerfilelint.git) / [hadolint](https://github.com/hadolint/hadolint) | | **EDITORCONFIG** | [editorconfig-checker](https://github.com/editorconfig-checker/editorconfig-checker) | @@ -218,6 +219,7 @@ But if you wish to select or exclude specific linters, we give you full control | **FILTER_REGEX_INCLUDE** | `all` | Regular expression defining which files will be processed by linters (ex: `.*src/.*`) | | **JAVASCRIPT_ES_CONFIG_FILE** | `.eslintrc.yml` | Filename for [eslint configuration](https://eslint.org/docs/user-guide/configuring#configuration-file-formats) (ex: `.eslintrc.yml`, `.eslintrc.json`) | | **JAVASCRIPT_DEFAULT_STYLE** | `standard` | Flag to set the default style of javascript. Available options: **standard**/**prettier** | +| **JSCPD_CONFIG_FILE** | `.jscpd.json` | Filename for JSCPD configuration | | **LINTER_RULES_PATH** | `.github/linters` | Directory for all linter configuration rules. | | **LOG_FILE** | `super-linter.log` | The file name for outputting logs. All output is sent to the log file regardless of `LOG_LEVEL`. | | **LOG_LEVEL** | `VERBOSE` | How much output the script will generate to the console. One of `ERROR`, `WARN`, `NOTICE`, `VERBOSE`, `DEBUG` or `TRACE`. | @@ -256,6 +258,7 @@ But if you wish to select or exclude specific linters, we give you full control | **VALIDATE_JAVA** | `true` | Flag to enable or disable the linting process of the language. | | **VALIDATE_JAVASCRIPT_ES** | `true` | Flag to enable or disable the linting process of the Javascript language. (Utilizing: eslint) | | **VALIDATE_JAVASCRIPT_STANDARD** | `true` | Flag to enable or disable the linting process of the Javascript language. (Utilizing: standard) | +| **VALIDATE_JSCPD** | `true` | Flag to enable or disable the JSCPD. | | **VALIDATE_JSON** | `true` | Flag to enable or disable the linting process of the JSON language. | | **VALIDATE_JSX** | `true` | Flag to enable or disable the linting process for jsx files (Utilizing: eslint) | | **VALIDATE_KOTLIN** | `true` | Flag to enable or disable the linting process of the Kotlin language. | diff --git a/TEMPLATES/.jscpd.json b/TEMPLATES/.jscpd.json new file mode 100644 index 00000000..e8969011 --- /dev/null +++ b/TEMPLATES/.jscpd.json @@ -0,0 +1,10 @@ +{ + "threshold": 0, + "reporters": [ + "consoleFull" + ], + "ignore": [ + "**/__snapshots__/**" + ], + "absolute": true +} diff --git a/dependencies/package.json b/dependencies/package.json index b428b880..0455a570 100644 --- a/dependencies/package.json +++ b/dependencies/package.json @@ -15,6 +15,7 @@ "eslint-plugin-prettier": "^3.3.1", "gherkin-lint": "^4.1.3", "htmlhint": "^0.14.2", + "jscpd": "^3.3.21", "jsonlint": "^1.6.3", "markdownlint-cli": "^0.26.0", "npm-groovy-lint": "^8.1.0", diff --git a/lib/functions/buildFileList.sh b/lib/functions/buildFileList.sh index 2660b228..5c5d2c30 100755 --- a/lib/functions/buildFileList.sh +++ b/lib/functions/buildFileList.sh @@ -223,6 +223,8 @@ function BuildFileList() { # Editorconfig-checker should check every file FILE_ARRAY_EDITORCONFIG+=("${FILE}") + # jscpd also runs an all files + FILE_ARRAY_JSCPD+=("${FILE}") ####################### # Get the shell files # @@ -514,9 +516,9 @@ function BuildFileList() { # Append the file to the array # ################################ FILE_ARRAY_PYTHON_BLACK+=("${FILE}") - FILE_ARRAY_PYTHON_PYLINT+=("${FILE}") FILE_ARRAY_PYTHON_FLAKE8+=("${FILE}") FILE_ARRAY_PYTHON_ISORT+=("${FILE}") + FILE_ARRAY_PYTHON_PYLINT+=("${FILE}") ###################### # Get the RAKU files # @@ -563,7 +565,7 @@ function BuildFileList() { elif [ "${FILE_TYPE}" == "sql" ]; then ################################ # Append the file to the array # - ##############################p## + ################################ FILE_ARRAY_SQL+=("${FILE}") ########################### diff --git a/lib/functions/detectFiles.sh b/lib/functions/detectFiles.sh index 1766a564..aafaa395 100755 --- a/lib/functions/detectFiles.sh +++ b/lib/functions/detectFiles.sh @@ -49,14 +49,10 @@ DetectOpenAPIFile() { # Check the shell for errors # ############################## if [ ${ERROR_CODE} -eq 0 ]; then - ######################## - # Found string in file # - ######################## + debug "${FILE} is an OpenAPI descriptor" return 0 else - ################### - # No string match # - ################### + debug "${FILE} is NOT an OpenAPI descriptor" return 1 fi } @@ -268,6 +264,7 @@ function CheckFileType() { ################################ # Append the file to the array # ################################ + FILE_ARRAY_JSCPD+=("${FILE}") FILE_ARRAY_RUBY+=("${FILE}") else ############################ diff --git a/lib/functions/worker.sh b/lib/functions/worker.sh index 24285bc2..f1a1d95a 100755 --- a/lib/functions/worker.sh +++ b/lib/functions/worker.sh @@ -32,6 +32,7 @@ function LintCodebase() { # Set the flag # ################ SKIP_FLAG=0 + INDEX=0 ############################################################ # Check to see if we need to go through array or all files # @@ -75,7 +76,6 @@ function LintCodebase() { ######################################## if IsTAP; then TMPFILE=$(mktemp -q "/tmp/super-linter-${FILE_TYPE}.XXXXXX") - INDEX=0 mkdir -p "${REPORT_OUTPUT_FOLDER}" REPORT_OUTPUT_FILE="${REPORT_OUTPUT_FOLDER}/super-linter-${FILE_TYPE}.${OUTPUT_FORMAT}" fi @@ -371,16 +371,16 @@ function LintCodebase() { fi fi fi + fi - ############################## - # Validate we ran some tests # - ############################## - if [ "${TEST_CASE_RUN}" = "true" ] && [ "${INDEX}" -eq 0 ]; then - ################################################# - # We failed to find files and no tests were ran # - ################################################# - error "Failed to find any tests ran for the Linter:[${LINTER_NAME}]"! - fatal "Please validate logic or that tests exist!" - fi + ############################## + # Validate we ran some tests # + ############################## + if [ "${TEST_CASE_RUN}" = "true" ] && [ "${INDEX}" -eq 0 ]; then + ################################################# + # We failed to find files and no tests were ran # + ################################################# + error "Failed to find any tests ran for the Linter:[${LINTER_NAME}]"! + fatal "Please validate logic or that tests exist!" fi } diff --git a/lib/linter.sh b/lib/linter.sh index 03698848..c40943c1 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -109,6 +109,8 @@ JAVASCRIPT_STYLE='' # Variable for the style # shellcheck disable=SC2034 # Variable is referenced indirectly JAVASCRIPT_STANDARD_FILE_NAME="${JAVASCRIPT_ES_CONFIG_FILE:-.eslintrc.yml}" # shellcheck disable=SC2034 # Variable is referenced indirectly +JSCPD_FILE_NAME="${JSCPD_CONFIG_FILE:-.jscpd.json}" +# shellcheck disable=SC2034 # Variable is referenced indirectly JSX_FILE_NAME="${JAVASCRIPT_ES_CONFIG_FILE:-.eslintrc.yml}" # shellcheck disable=SC2034 # Variable is referenced indirectly LATEX_FILE_NAME=".chktexrc" @@ -180,7 +182,7 @@ fi ################## LANGUAGE_ARRAY=('ANSIBLE' 'ARM' 'BASH' 'BASH_EXEC' 'CLOUDFORMATION' 'CLOJURE' 'COFFEESCRIPT' 'CSHARP' 'CSS' 'DART' 'DOCKERFILE' 'DOCKERFILE_HADOLINT' 'EDITORCONFIG' 'ENV' 'GHERKIN' 'GO' 'GROOVY' 'HTML' - 'JAVA' 'JAVASCRIPT_ES' "${JAVASCRIPT_STYLE_NAME}" 'JSON' 'JSX' 'KUBERNETES_KUBEVAL' 'KOTLIN' 'LATEX' 'LUA' 'MARKDOWN' + 'JAVA' 'JAVASCRIPT_ES' "${JAVASCRIPT_STYLE_NAME}" 'JSCPD' 'JSON' 'JSX' 'KUBERNETES_KUBEVAL' 'KOTLIN' 'LATEX' 'LUA' 'MARKDOWN' 'OPENAPI' 'PERL' 'PHP_BUILTIN' 'PHP_PHPCS' 'PHP_PHPSTAN' 'PHP_PSALM' 'POWERSHELL' 'PROTOBUF' 'PYTHON_BLACK' 'PYTHON_PYLINT' 'PYTHON_FLAKE8' 'PYTHON_ISORT' 'R' 'RAKU' 'RUBY' 'SHELL_SHFMT' 'SNAKEMAKE_LINT' 'SNAKEMAKE_SNAKEFMT' 'STATES' 'SQL' 'TEKTON' 'TERRAFORM' 'TERRAFORM_TERRASCAN' 'TERRAGRUNT' 'TSX' 'TYPESCRIPT_ES' 'TYPESCRIPT_STANDARD' 'XML' 'YAML') @@ -210,6 +212,7 @@ LINTER_NAMES_ARRAY['HTML']="htmlhint" LINTER_NAMES_ARRAY['JAVA']="checkstyle" LINTER_NAMES_ARRAY['JAVASCRIPT_ES']="eslint" LINTER_NAMES_ARRAY["${JAVASCRIPT_STYLE_NAME}"]="${JAVASCRIPT_STYLE}" +LINTER_NAMES_ARRAY['JSCPD']="jscpd" LINTER_NAMES_ARRAY['JSON']="jsonlint" LINTER_NAMES_ARRAY['JSX']="eslint" LINTER_NAMES_ARRAY['KOTLIN']="ktlint" @@ -773,6 +776,7 @@ LINTER_COMMANDS_ARRAY['JAVA']="java -jar /usr/bin/checkstyle -c ${JAVA_LINTER_RU LINTER_COMMANDS_ARRAY['JAVASCRIPT_ES']="eslint --no-eslintrc -c ${JAVASCRIPT_ES_LINTER_RULES}" LINTER_COMMANDS_ARRAY['JAVASCRIPT_STANDARD']="standard ${JAVASCRIPT_STANDARD_LINTER_RULES}" LINTER_COMMANDS_ARRAY['JAVASCRIPT_PRETTIER']="prettier --check" +LINTER_COMMANDS_ARRAY['JSCPD']="jscpd --config ${JSCPD_LINTER_RULES}" LINTER_COMMANDS_ARRAY['JSON']="jsonlint" LINTER_COMMANDS_ARRAY['JSX']="eslint --no-eslintrc -c ${JSX_LINTER_RULES}" LINTER_COMMANDS_ARRAY['KOTLIN']="ktlint"