Install, configure, and run jscpd. Add missing tests (#1032)

Co-authored-by: Gabo <gabo.fdc@gmail.com>
Co-authored-by: Matt Desmond <beardofedu@github.com>
Co-authored-by: Lukas Gravley <admiralawkbar@github.com>

.automation/test/ansible/ansible_good_1.yml

@@ -22,7 +22,7 @@
     github_host: "{{ hostvars['github_primary'].ansible_host }}"
     probot_server_ip: "{{ hostvars['backup-utils'].ansible_host }}"
   roles:
-    - role: ghe-initialize
+    - role: ghe_initialize
 
 ##########################################
 ## Run ghe-config-apply for all changes ##
@@ -36,7 +36,7 @@
     - block:
         - name: GHE-Config-Apply
           include_role:
-            name: ghe-initialize
+            name: ghe_initialize
             tasks_from: ghe-config-apply.yml
       tags:
         - github

.automation/test/ansible/reports/expected-ANSIBLE.tap

@@ -2,6 +2,6 @@ TAP version 13
 1..2
 not ok 1 - ansible_bad_1.yml
   ---
-  message: Traceback (most recent call last) \n  File "/usr/bin/ansible-lint", line 11, in <module>\n    load_entry_point('ansible-lint==4.2.0', 'console_scripts', 'ansible-lint')()\n  File "/usr/lib/python3.8/site-packages/ansiblelint/__main__.py", line 187, in main\n    matches.extend(runner.run())\n  File "/usr/lib/python3.8/site-packages/ansiblelint/__init__.py", line 267, in run\n    for child in ansiblelint.utils.find_children(arg, self.playbook_dir) \n  File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 163, in find_children\n    for child in play_children(basedir, item, playbook[1], playbook_dir) \n  File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 215, in play_children\n    return delegate_map[k](basedir, k, v, parent_type)\n  File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 246, in _taskshandlers_children\n    results.extend(_roles_children(basedir, k, [th['action'].get('name')],\n  File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 285, in _roles_children\n    results.extend(_look_for_role_files(basedir, role, main=main))\n  File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 330, in _look_for_role_files\n    role_path = _rolepath(basedir, role)\n  File "/usr/lib/python3.8/site-packages/ansiblelint/utils.py", line 299, in _rolepath\n    path_dwim(basedir, os.path.join('roles', role)),\n  File "/usr/lib/python3.8/posixpath.py", line 90, in join\n    genericpath._check_arg_types('join', a, *p)\n  File "/usr/lib/python3.8/genericpath.py", line 152, in _check_arg_types\n    raise TypeError(f'{funcname}() argument must be str, bytes, or '\nTypeError  join() argument must be str, bytes, or os.PathLike object, not 'NoneType'\n
+  message: DEBUG    Logging initialized to level 10\nDEBUG    Options  Namespace(listrules=False, format='rich', quiet=True, parseable=True, parseable_severity=False, progressive=False, rulesdir=[], use_default_rules=True, display_relative_path=True, tags=[], listtags=False, verbosity=2, skip_list=['602', '204', '301', '303', '305', '503'], warn_list=['experimental'], colored=False, exclude_paths=[], config_file='/tmp/lint/.github/linters/.ansible-lint.yml', playbook=['/tmp/lint/.automation/test/ansible/ansible_bad_1.yml'])\nDEBUG    Loading rules from /usr/local/lib/python3.9/site-packages/ansiblelint/rules\nDEBUG    Examining ansible_bad_1.yml of type playbook\nTraceback (most recent call last) \n  File "/usr/local/bin/ansible-lint", line 8, in <module>\n    sys.exit(main())\n  File "/usr/local/lib/python3.9/site-packages/ansiblelint/__main__.py", line 158, in main\n    matches = _get_matches(rules, options)\n  File "/usr/local/lib/python3.9/site-packages/ansiblelint/__main__.py", line 240, in _get_matches\n    matches.extend(runner.run())\n  File "/usr/local/lib/python3.9/site-packages/ansiblelint/runner.py", line 91, in run\n    self.rules.run(file, tags=set(self.tags),\n  File "/usr/local/lib/python3.9/site-packages/ansiblelint/rules/__init__.py", line 236, in run\n    matches.extend(rule.matchtasks(playbookfile, text))\n  File "/usr/local/lib/python3.9/site-packages/ansiblelint/rules/__init__.py", line 88, in matchtasks\n    yaml = append_skipped_rules(yaml, text, file['type'])\n  File "/usr/local/lib/python3.9/site-packages/ansiblelint/skip_utils.py", line 62, in append_skipped_rules\n    yaml_skip = _append_skipped_rules(pyyaml_data, file_text, file_type)\n  File "/usr/local/lib/python3.9/site-packages/ansiblelint/skip_utils.py", line 112, in _append_skipped_rules\n    for ruamel_task, pyyaml_task in zip(ruamel_tasks, pyyaml_tasks) \n  File "/usr/local/lib/python3.9/site-packages/ansiblelint/skip_utils.py", line 160, in _get_tasks_from_blocks\n    for sub_task in get_nested_tasks(task) \n  File "/usr/local/lib/python3.9/site-packages/ansiblelint/skip_utils.py", line 156, in <genexpr>\n    for subtask in task[k]\nTypeError  'NoneType' object is not iterable\n
   ...
 ok 2 - ansible_good_1.yml

.automation/test/javascript_standard/javascript_good_1.js

@@ -1,17 +1,15 @@
-var http = require('http')
+const http = require('http')
-var createHandler = require('github-webhook-handler')
+const createHandler = require('github-webhook-handler')
-var handler = createHandler({ path: '/webhook', secret: (process.env.SECRET) })
+const handler = createHandler({ path: '/webhook', secret: (process.env.SECRET) })
 
-var userArray = ['user1']
+const userArray = ['user1']
 
-var teamDescription = 'Team of Robots'
+const teamDescription = 'Team of Robots'
-var teamPrivacy = 'closed' // closed (visible) / secret (hidden) are options here
+const teamPrivacy = 'closed' // closed (visible) / secret (hidden) are options here
 
-var teamName = process.env.GHES_TEAM_NAME
+const teamName = process.env.GHES_TEAM_NAME
-var teamAccess = 'pull' // pull,push,admin options here
+const teamAccess = 'pull' // pull,push,admin options here
-var teamId = ''
+const teamId = ''
-
-var orgRepos = []
 
 // var creator = ""
 
@@ -31,61 +29,21 @@ handler.on('repository', function (event) {
   if (event.payload.action === 'created') {
     const repo = event.payload.repository.full_name
     console.log(repo)
-    const org = event.payload.repository.owner.login
-    getTeamID(org)
     setTimeout(checkTeamIDVariable, 1000)
   }
 })
 
 handler.on('team', function (event) {
-// TODO user events such as being removed from team or org
+  // TODO user events such as being removed from team or org
   if (event.payload.action === 'deleted') {
     // const name = event.payload.team.name
-    const org = event.payload.organization.login
-    getRepositories(org)
     setTimeout(checkReposVariable, 5000)
   } else if (event.payload.action === 'removed_from_repository') {
-    const org = event.payload.organization.login
-    getTeamID(org)
     // const repo = event.payload.repository.full_name
     setTimeout(checkTeamIDVariable, 1000)
   }
 })
 
-function getTeamID (org) {
-  const https = require('https')
-
-  const options = {
-    hostname: (process.env.GHE_HOST),
-    port: 443,
-    path: '/api/v3/orgs/' + org + '/teams',
-    method: 'GET',
-    headers: {
-      Authorization: 'token ' + (process.env.GHE_TOKEN),
-      'Content-Type': 'application/json'
-    }
-  }
-  let body = []
-  const req = https.request(options, (res) => {
-    res.on('data', (chunk) => {
-      body.push(chunk)
-    }).on('end', () => {
-      body = JSON.parse(Buffer.concat(body))
-      body.forEach(item => {
-        if (item.name === teamName) {
-          teamId = item.id
-        }
-      })
-    })
-  })
-
-  req.on('error', (error) => {
-    console.error(error)
-  })
-
-  req.end()
-}
-
 function checkTeamIDVariable (repo) {
   if (typeof teamId !== 'undefined') {
     addTeamToRepo(repo, teamId)
@@ -143,8 +101,7 @@ function reCreateTeam (org) {
     name: teamName,
     description: teamDescription,
     privacy: teamPrivacy,
-    maintainers: userArray,
+    maintainers: userArray
-    repo_names: orgRepos
   })
 
   const options = {
@@ -178,38 +135,3 @@ function reCreateTeam (org) {
   req.write(data)
   req.end()
 }
-
-function getRepositories (org) {
-  orgRepos = []
-
-  const https = require('https')
-
-  const options = {
-    hostname: (process.env.GHE_HOST),
-    port: 443,
-    path: '/api/v3/orgs/' + org + '/repos',
-    method: 'GET',
-    headers: {
-      Authorization: 'token ' + (process.env.GHE_TOKEN),
-      'Content-Type': 'application/json'
-    }
-  }
-  let body = []
-  const req = https.request(options, (res) => {
-    res.on('data', (chunk) => {
-      body.push(chunk)
-    }).on('end', () => {
-      body = JSON.parse(Buffer.concat(body))
-      body.forEach(item => {
-        orgRepos.push(item.full_name)
-        console.log(item.full_name)
-      })
-    })
-  })
-
-  req.on('error', (error) => {
-    console.error(error)
-  })
-
-  req.end()
-}

.automation/test/jscpd/clojure_good_1.clj

@@ -0,0 +1,34 @@
+(ns foo
+  (:require
+    [clojure.string :as str]))
+
+(butlast [1 2 3])
+
+(str/join "" "")
+
+(defn foo-fn [x]
+    (let [y (fn [] (inc x))]
+        (y)))
+
+(letfn
+  [(f [g] (h g))
+    (h [i] (f i))])
+
+(defn foo [] 1)
+(inc (foo))
+
+(Thread/sleep 1000 1)
+
+;; Here we switch to another namespace and require the previous:
+(ns bar (:require [foo :as f]))
+
+(f/foo-fn 1)
+
+{:a 1 :b 2}
+#{1 2}
+{:a 1 :b 2}
+
+(ns bar-test (:require [clojure.test :as t]))
+
+(t/deftest my-tests
+  (t/is (odd? (inc 1))))

.automation/test/jscpd/coffeescript_good_1.coffee

@@ -0,0 +1,84 @@
+# Description
+#   silly hubot scripts
+#	These were created to blow off steam
+#
+# Commands:
+#   `mona echo *` - repeats what you say
+#
+# Author:
+#   admiralawkbar@github.com
+
+###############################
+# Drop Hammer array of images #
+###############################
+dropHammer = [
+  "https://s1.yimg.com/uu/api/res/1.2/.kFQAfQ6KQmlf5ip8.UzNA--/dz0xMjMwO2g9NjkyO2FwcGlkPXl0YWNoeW9u/http://media.zenfs.com/en-US/video/video.snl.com/SNL_1554_08_Update_03_Harry_Caray.png",
+  "http://media.tumblr.com/d12ea80b3a86dfc5fe36d3f306254fe4/tumblr_inline_mq1r0tbBCb1qz4rgp.jpg",
+  "http://the-artifice.com/wp-content/uploads/2014/01/94309-160x160.png",
+  "http://25.media.tumblr.com/35826348f2215069835c1733c75b29aa/tumblr_muuxmmBaOI1rw3gqyo2_250.gif",
+  "http://data2.whicdn.com/images/78766805/large.jpg",
+  "http://filmfisher.com/wp-content/uploads/2014/11/hunt_for_red_october.jpg",
+  "http://cdn.meme.am/instances/500x/57495736.jpg",
+]
+
+###################
+# Thank you array #
+###################
+thanks = [
+  "You're welcome! Piece of cake...",
+  "It was nothing..."
+  "De nada...",
+  "Danke...",
+  "Merci...",
+  "Bitte...",
+  "De rien..."
+  "Prego..."
+]
+
+#################################
+# Start the robot for listening #
+#################################
+module.exports = (robot) ->
+
+  ##############################
+  # Show the adapter connected #
+  ##############################
+  robot.respond /ADAPTER$/i, (msg) ->
+    msg.send robot.adapterName
+
+  ##########################
+  # Echo back the response #
+  ##########################
+  robot.respond /ECHO (.*)$/i, (msg) ->
+    msg.send msg.match[1]
+
+  ##################
+  # Whats going on #
+  ##################
+  robot.respond /whats going on/i, (msg) ->
+    msg.send "not much... robot stuff..."
+
+  ###################
+  # Drop the hammer #
+  ###################
+  robot.respond /drop the hammer/i, (msg) ->
+    msg.send "Commencing the hammer dropping..."
+    msg.send msg.random dropHammer
+
+  ###############
+  # Vape Nation #
+  ###############
+  robot.respond /lets roll/i, (msg) ->
+    msg.send "First Class! Vape Nation!!! @beardofedu"
+
+  ##############
+  # Hubot Ping #
+  ##############
+  robot.respond /PING$/i, (msg) ->
+    msg.send "PONG"
+
+#######################
+#######################
+## END OF THE SCRIPT ##
+#######################
+#######################

.automation/test/jscpd/csharp_good_01.cs

@@ -0,0 +1,12 @@
+using System;
+
+namespace HelloWorld
+{
+  class Program
+  {
+    static void Main(string[] args)
+    {
+      Console.WriteLine("Hello World!");
+    }
+  }
+}

.automation/test/jscpd/css_good_01.css

@@ -0,0 +1,22 @@
+/**
+ * Multi-line comment
+ */
+
+.selector-1,
+.selector-2,
+.selector-3[type="text"] {
+  background: linear-gradient(#fff, rgba(0, 0, 0, 0.8));
+  box-sizing: border-box;
+  display: block;
+  color: #333;
+}
+
+.selector-a,
+.selector-b:not(:first-child) {
+  padding: 10px !important;
+  top: calc(calc(1em * 2) / 3);
+}
+
+.selector-x { width: 10%; }
+.selector-y { width: 20%; }
+.selector-z { width: 30%; }

.automation/test/jscpd/dart_good_1.dart

@@ -0,0 +1,5 @@
+// empty_constructor_bodies good ;
+class Point {
+  int x, y;
+  Point(this.x, this.y);
+}

.automation/test/jscpd/gherkin_good_01.feature

@@ -0,0 +1,5 @@
+Feature: Test for the no-dupe-scenario-names rule
+
+Scenario: This is a Scenario for no-dupe-scenario-names
+  Given I have 2 scenarios with the same name
+  Then I should see a no-dupe-scenario-names error

.automation/test/jscpd/golang_good_01.go

@@ -0,0 +1,7 @@
+package main
+
+import "fmt"
+
+func main() {
+	fmt.Println("hello world")
+}

.automation/test/jscpd/good/Dockerfile

@@ -0,0 +1,13 @@
+FROM node:10
+
+# Create app directory
+RUN mkdir -p /usr/src/app
+WORKDIR /usr/src/app
+
+# Install app dependencies
+COPY package.json /usr/src/app/
+RUN npm install
+
+COPY server.js server.js
+EXPOSE 3000
+CMD ["node", "server.js"]

.automation/test/jscpd/good/Dockerfile.dev

@@ -0,0 +1,13 @@
+FROM node:10
+
+# Create app directory
+RUN mkdir -p /usr/src/app
+WORKDIR /usr/src/app
+
+# Install app dependencies
+COPY package.json /usr/src/app/
+RUN npm install
+
+COPY server.js server.js
+EXPOSE 3000
+CMD ["node", "server.js"]

.automation/test/jscpd/groovy_good_01.groovy

@@ -0,0 +1,6 @@
+class Example {
+    static void main(String[] args) {
+        File file = new File("E:/Example.txt")
+        println "The file ${file.absolutePath} has ${file.length()} bytes"
+    }
+}

.automation/test/jscpd/html_good_01.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+
+  <head>
+    <meta charset="UTF-8">
+    <title>Document</title>
+  </head>
+
+  <body>
+    Good HTML!
+  </body>
+
+</html>

.automation/test/jscpd/java_good_1.java

@@ -0,0 +1,9 @@
+/**
+ * Represents a good Java file.
+*/
+public static class JavaGood {
+    // Prints "Hello, World" to the terminal window.
+    private void helloWorld() {
+        System.out.println("Hello, World");
+    }
+}

.automation/test/jscpd/javascript_bad_1.js

@@ -0,0 +1,215 @@
+var http = require('http')
+var createHandler = require('github-webhook-handler')
+var handler = createHandler({ path: '/webhook', secret: (process.env.SECRET) })
+
+var userArray = ['user1']
+
+var teamDescription = 'Team of Robots'
+var teamPrivacy = 'closed' // closed (visible) / secret (hidden) are options here
+
+var teamName = process.env.GHES_TEAM_NAME
+var teamAccess = 'pull' // pull,push,admin options here
+var teamId = ''
+
+var orgRepos = []
+
+// var creator = ""
+
+http.createServer(function (req, res) {
+  handler(req, res, function (err) {
+    console.log(err)
+    res.statusCode = 404
+    res.end('no such location')
+  })
+}).listen(3000)
+
+handler.on('error', function (err) {
+  console.error('Error:', err.message)
+})
+
+handler.on('repository', function (event) {
+  if (event.payload.action === 'created') {
+    const repo = event.payload.repository.full_name
+    console.log(repo)
+    const org = event.payload.repository.owner.login
+    getTeamID(org)
+    setTimeout(checkTeamIDVariable, 1000)
+  }
+})
+
+handler.on('team', function (event) {
+// TODO user events such as being removed from team or org
+  if (event.payload.action === 'deleted') {
+    // const name = event.payload.team.name
+    const org = event.payload.organization.login
+    getRepositories(org)
+    setTimeout(checkReposVariable, 5000)
+  } else if (event.payload.action === 'removed_from_repository') {
+    const org = event.payload.organization.login
+    getTeamID(org)
+    // const repo = event.payload.repository.full_name
+    setTimeout(checkTeamIDVariable, 1000)
+  }
+})
+
+function getTeamID (org) {
+  const https = require('https')
+
+  const options = {
+    hostname: (process.env.GHE_HOST),
+    port: 443,
+    path: '/api/v3/orgs/' + org + '/teams',
+    method: 'GET',
+    headers: {
+      Authorization: 'token ' + (process.env.GHE_TOKEN),
+      'Content-Type': 'application/json'
+    }
+  }
+  let body = []
+  const req = https.request(options, (res) => {
+    res.on('data', (chunk) => {
+      body.push(chunk)
+    }).on('end', () => {
+      body = JSON.parse(Buffer.concat(body))
+      body.forEach(item => {
+        if (item.name === teamName) {
+          teamId = item.id
+        }
+      })
+    })
+  })
+
+  req.on('error', (error) => {
+    console.error(error)
+  })
+
+  req.end()
+}
+
+function checkTeamIDVariable (repo) {
+  if (typeof teamId !== 'undefined') {
+    addTeamToRepo(repo, teamId)
+  }
+}
+
+function checkReposVariable (org) {
+  if (typeof orgRepos !== 'undefined') {
+  //      for(var repo of orgRepos) {
+  //        addTeamToRepo(repo, teamId)
+  // }
+    reCreateTeam(org)
+  }
+}
+
+function addTeamToRepo (repo, teamId) {
+  const https = require('https')
+  const data = JSON.stringify({
+    permission: teamAccess
+  })
+
+  const options = {
+    hostname: (process.env.GHE_HOST),
+    port: 443,
+    path: '/api/v3/teams/' + teamId + '/repos/' + repo,
+    method: 'PUT',
+    headers: {
+      Authorization: 'token ' + (process.env.GHE_TOKEN),
+      'Content-Type': 'application/json',
+      'Content-Length': data.length
+    }
+  }
+  let body = []
+  const req = https.request(options, (res) => {
+    res.on('data', (chunk) => {
+      body.push(chunk)
+    }).on('end', () => {
+      body = Buffer.concat(body).toString()
+      console.log(res.statusCode)
+      console.log('added team to ' + repo)
+    })
+  })
+
+  req.on('error', (error) => {
+    console.error(error)
+  })
+
+  req.write(data)
+  req.end()
+}
+
+function reCreateTeam (org) {
+  const https = require('https')
+  const data = JSON.stringify({
+    name: teamName,
+    description: teamDescription,
+    privacy: teamPrivacy,
+    maintainers: userArray,
+    repo_names: orgRepos
+  })
+
+  const options = {
+    hostname: (process.env.GHE_HOST),
+    port: 443,
+    path: '/api/v3/orgs/' + org + '/teams',
+    method: 'POST',
+    headers: {
+      Authorization: 'token ' + (process.env.GHE_TOKEN),
+      'Content-Type': 'application/json',
+      'Content-Length': data.length
+    }
+  }
+  // const body = []
+  const req = https.request(options, (res) => {
+    if (res.statusCode !== 201) {
+      console.log('Status code: ' + res.statusCode)
+      console.log('Added ' + teamName + ' to ' + org + ' Failed')
+      res.on('data', function (chunk) {
+        console.log('BODY: ' + chunk)
+      })
+    } else {
+      console.log('Added ' + teamName + ' to ' + org)
+    }
+  })
+
+  req.on('error', (error) => {
+    console.error(error)
+  })
+
+  req.write(data)
+  req.end()
+}
+
+function getRepositories (org) {
+  orgRepos = []
+
+  const https = require('https')
+
+  const options = {
+    hostname: (process.env.GHE_HOST),
+    port: 443,
+    path: '/api/v3/orgs/' + org + '/repos',
+    method: 'GET',
+    headers: {
+      Authorization: 'token ' + (process.env.GHE_TOKEN),
+      'Content-Type': 'application/json'
+    }
+  }
+  let body = []
+  const req = https.request(options, (res) => {
+    res.on('data', (chunk) => {
+      body.push(chunk)
+    }).on('end', () => {
+      body = JSON.parse(Buffer.concat(body))
+      body.forEach(item => {
+        orgRepos.push(item.full_name)
+        console.log(item.full_name)
+      })
+    })
+  })
+
+  req.on('error', (error) => {
+    console.error(error)
+  })
+
+  req.end()
+}

.automation/test/jscpd/json_good_1.json

@@ -0,0 +1,10 @@
+{
+  "arrow_spacing": {
+    "level": "ignore"
+  },
+  "braces_spacing": {
+    "level": "ignore",
+    "spaces": 0,
+    "empty_object_spaces": 0
+  }
+}

.automation/test/jscpd/kotlint_good_1.kt

@@ -0,0 +1,6 @@
+fun main() {
+  val n = "World"
+  val v = "Hello, $n!"
+
+  println(v)
+}

.automation/test/jscpd/latex_good_1.tex

@@ -0,0 +1,232 @@
+Note: This file was written with only two purposes in mind:
+    o To test the program upon it
+    o To show off some of the features
+
+Most of the file does thus consist of lots of pseudo-commands, which
+are nonsense in a TeXnical manner.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+% Warning 1
+
+\foo\ This is an error.
+So is this \foo\
+\smallskip This is a not. $\foo Neither$ is this.
+
+\startsection[title={Testing ConTeXt}]
+These should now be an error.
+\stopsection
+
+% Warning 2
+
+%This is a faulty reference to~\ref{foo}
+This is not a faulty reference to~\ref{foo}
+
+% Warning 3
+
+%$[(ab)^{-1}]^{-2}$  is not beautiful
+${{[{(ab)}^{-1}]}}^{-2}$ is beautiful
+
+% Warning 4-6, 28
+
+%Testing {\it italic} in \/ this {\em sentence\/}, as {\em you \/ see\/}.
+Testing {\it italic\/} in this {\em sentence,} as {\em you see}.
+
+% LaTeX2e
+
+%Testing \textem{italic} in \/ this \textit{sentence\/}, as \textem{you \/ see\/}.
+Testing \textem{italic} in this \textit{sentence}, as \textem{you see}.
+
+% Warning 7
+
+%This \'is a test of $\hat{j}$ accents.
+This \'{\i}s a test of $\hat{\jmath}$ accents.
+
+% Warning 8
+
+%It wasn't anything - just a 2---3 star--shots.
+It wasn't anything --- just a 2--3 star-shots.
+It's just a start-shot.
+is also used to send cross-calls (xc) and cross-traps (xt) to other
+% From Knuths TeXbook Chapter 14
+% "How TeX Breaks Paragraphs into Lines", fourth paragraph:
+in plain TeX---are the key
+
+% Using DashExcpt
+The Birch--Swinnerton-Dyer conjecture is correct.
+%The Birch--Swinnerton--Dyer conjecture is not correct.
+The Birch-Swinnerton-Dyer conjecture is not correct (but not caught).
+
+% Warning 9-10
+
+% Brackets:
+
+%)}{[])} }}}]]])))
+{[]} ((([[[{{{}}}]]])))
+
+% Envs:
+
+%\begin{quux} \begin{baz} \end{quux} \end{baz} \end{asoi} \begin{dobedo}
+
+\begin{foo} \begin{bar} \end{bar}\end{foo}
+
+% Warning 11
+
+%Foo...bar. $1,...,3$. $1+...+3$. $1,\cdots,3$. $1\cdot\ldots\cdot3$.
+Foo\dots bar. $1,\ldots,3$. $1+\cdots+3$. $1,\ldots,3$. $1\cdot\cdots\cdot3$.
+
+% Warning 12
+
+%1st. Foo Inc. Ab.cd. foo ab.cd. Foo. bar baz., billy.; bob.: joe.! frank.? james.. george
+1st.\ foo Inc.\ ab.cd.\ foo ab.cd.\ Foo.\ bar baz., billy.; bob.:\ joe.!\ frank.?\ james..\ george
+
+% Warning 13
+
+%Look at THIS! It's an error.
+Look at THIS\@! It's an error. D. E. Knuth.
+
+% Warning 14
+
+%\hat
+\hat{a}
+
+% Warning 18,19
+
+%Is this an "example", or is it an <20>example<6C>.
+Is this an `example', or is it an `example'.
+
+% Warning 20
+
+%That bug is \unknown\ to me.
+% That bug is \unknown\ to me.
+
+% Warning 21
+
+\LaTeX\ is an extension of \TeX\. Right?
+\LaTeX\ is an extension of \TeX. Right?
+
+% Warning 23
+
+%```Hello', I heard him said'', she remembered.
+``\,`Hello', I heard him said'', she remembered.
+
+% Warning 24
+
+%Indexing text \index{text} is fun!
+Indexing text\index{text} is fun!
+Indexing text%
+    \index{text} is fun!
+%Indexing text
+%     \index{text} is fun!
+
+% Warning 25
+
+%$5\cdot10^10$
+$5\cdot10^{10}$
+
+% Warning 26
+
+%Do you understand ?
+Do you understand?
+
+% Warning 29
+%The program opens a screen sized 640x200 pixels
+The program opens a screen sized $640\times200$ pixels
+
+% Warning 30
+
+%White           is a beautiful colour.
+White is a beautiful colour.
+
+% Warning 31
+\begin{verbatim}
+\this is
+\end{verbatim} %foo bar
+
+% Warning 32-34
+
+%This is either an 'example`, an ''example`` or an `"`example'`'.
+This is either an `example', an ``example'' or an ``example''.
+
+% Warning 35
+
+%$sin^2 + cos^2 = 1$
+$\sin^2 + \cos^2 = 1$
+
+% Warning 36-37
+
+%This( an example( Nuff said )), illustrates( ``my'' )point.
+This (an example (Nuff said)), illustrates (``my'') point.
+
+% Warning 38
+%``An example,'' he said, ``would be great.''
+``An example'', he said, ``would be great''.
+
+% Warning 39
+
+%For output codes, see table ~\ref{tab:fmtout}.
+For output codes, see table~\ref{tab:fmtout}.
+
+% Warning 40
+%$\this,$ and $$this$$.
+$\this$, and \[this.\]
+
+% Warning 41
+%foo \above\ qux
+\frac{foo}{qux}
+
+% Warning 42
+%This is a footnote \footnote{foo}.
+This is a footnote\footnote{foo}.
+
+% Warning 43
+%Here is a mistake $\left{x\right}$.
+%This one triggers warning 22 $\left\{x\right\}$.
+Here \chktex\ doesn't complain $\left\lbrace x\right\rbrace$.
+
+% Warning 44 -- user regex -- default message
+%You should always write a good intro.
+You should always write a good introduction.
+
+% Warning 44 -- user regex -- user message
+%For every $p\not|n$ you have an ugly prime which doesn't divide $n$.
+For every $p\nmid n$ you have a cute prime which doesn't divide $n$.
+
+% Math mode check
+\ensuremath{\sin\ x\text{is not the same as \sin\ x, but is the same as $\sin x$}}
+Also, $x(3)\text{ is not x (3) but it is $x(3)$}$
+
+This is\\% a comment. Nothing here should be checked(right)?
+a broken line.
+But this is not a \% comment, so we should find this error (right)?
+
+Here(on this line only)is a warning $sin(x)$ suppressed. % chktex 36 chktex 35
+%Here (on this line only)is a warning $sin(x)$ suppressed. % CHKTEX 35 36
+
+%In section~\ref{sec:3} we have a warning.
+In section~\ref{sec:4} it is suppressed. % chktex -1
+% In section~\ref{sec:5} we don't have a warning.
+
+% \begin{tabular*}{1.0\linewidth}[h]{|c|cc|}
+%   a & b \\
+%   \hline
+%   c & d
+% \end{tabular*}
+
+% Verb check
+
+\verb@\this is )() lots of errors, etc. Or what?@
+%\verb#
+
+\begin{verbatim}
+\this is
+\end{verbatim}
+
+% Warning 16,15
+
+\[\]
+
+% Local Variables:
+% require-final-newline: nil
+% End:
+% There should be no newline at the end of this file to test bug #46539

.automation/test/jscpd/lua_good_1.lua

@@ -0,0 +1,12 @@
+local embracer = {}
+
+local function helper()
+    -- NYI wontfix
+end
+
+function embracer.embrace(opt)
+    opt = opt or "default"
+    return helper(opt.."?")
+end
+
+return embracer

.automation/test/jscpd/markdown_good_1.md

@@ -0,0 +1,20 @@
+# Good Markdown
+
+This is just standard good markdown.
+
+## Second level header
+
+This header follows the step down from `level 1`.
+
+- Here it *is*
+  - Some more **indention**
+    - why so much?
+
+```bash
+ls -la
+```
+
+### Walk away
+
+We're all done **here**.
+- [Link Action](https://github.com)

.automation/test/jscpd/perl_good_1.pl

@@ -0,0 +1,38 @@
+#!/usr/bin/perl
+################################################################################
+################################################################################
+######### Script action @admiralawkbar #########################################
+################################################################################
+
+#############
+# Load Libs #
+#############
+use strict;
+
+$|=1;
+
+##################
+#### GLOBALS: ####
+##################
+my $state = undef;              # State to return to GHE
+my $exitCode = 0;               # Code to exit with
+my $description = "Here it is"; # Description of the build
+
+###############
+#### MAIN: ####
+###############
+Header();   # Basic print statements
+
+#######################################################################
+#################### SUB ROUTINES BELOW ONLY ##########################
+#######################################################################
+#######################################################################
+#### SUB ROUTINE Header ###############################################
+sub Header
+{
+  print "-------------------------------------------------------------------\n";
+  print "State:\[$state\]\n";
+  print "ExitCode:\[$exitCode\]\n";
+  print "Description:\[$description\]\n";
+  print "-------------------------------------------------------------------\n";
+}

.automation/test/jscpd/php_good_1.php

@@ -0,0 +1,3 @@
+<?php
+
+echo "Hello World!", PHP_EOL;

.automation/test/jscpd/powershell_good_1.ps1

@@ -0,0 +1 @@
+Write-Output "hello world!"

.automation/test/jscpd/protobuf_good_1.proto

@@ -0,0 +1,18 @@
+syntax = "proto3";
+import public "other.proto";
+option java_package = "com.example.foo";
+enum EnumAllowingAlias {
+  option allow_alias = true;
+  ALLOWING_UNSPECIFIED = 0;
+  STARTED = 1;
+  RUNNING = 2 [(custom_option) = "hello world"];
+}
+message Outer {
+  option (my_option).a = true;
+  message Inner {   // Level 2
+    int64 ival = 1;
+  }
+  inner inner_message = 2;
+  EnumAllowingAlias enum_field =3;
+  map<int32, string> my_map = 4;
+}

.automation/test/jscpd/python_bad_1.py

@@ -0,0 +1,195 @@
+import json
+import sys
+from os import getenv, path
+from pprint import pprint
+
+import click  # pylint: disable=import-error
+import requests  # pylint: disable=import-error
+from dotenv import load_dotenv  # pylint: disable=import-error
+
+env = load_dotenv()
+api_url = getenv("API_URL", default="https://api.github.com/graphql")
+github_token = getenv("GITHUB_TOKEN", default=None)
+
+if github_token is None:
+    sys.exit(
+        "GitHub Token is not set."
+        + "Please set the GITHUB_TOKEN env variable in your system or "
+        + "the .env file of your project."
+    )
+
+client_id = getenv("CLIENT_ID", default="copy_labels.py")
+headers = {
+    "Authorization": "bearer {github_token}".format(github_token=github_token),
+    "Accept": "application/vnd.github.bane-preview+json",
+    "Content-Type": "application/json",
+}
+
+
+def create_label(repo_id, label):
+    """
+    Create label in the supplied repo.
+
+    :param repo_id: Unique ID that represents the repo in GitHub
+    :type repo_id: str
+    :param label: Object with label information.
+    :type label: dict
+    :return: GitHub API request response
+    """
+
+    query_variables = {
+        "createLabelInput": {
+            "color": label["color"],
+            "description": label["description"],
+            "name": label["name"],
+            "repositoryId": repo_id,
+        }
+    }
+
+    with open(
+        path.join(path.dirname(__file__), "queries/create_label.gql"), "r"
+    ) as query_file:
+        query = "".join(query_file.readlines())
+
+    payload = {"query": query, "variables": query_variables}
+    response = requests.post(api_url, data=json.dumps(payload), headers=headers).json()
+    print("Created label {label}".format(label=label["name"]))
+
+    return response
+
+
+def get_labels(owner, repo):
+    """
+    Gets a list of labels from the supplied repo.
+    :param owner: Repo owner GitHub login.
+    :type owner: str
+    :param repo: Repository name.
+    :type repo: str
+    :return: A tuple with the GitHub id for the repository and a list of labels defined in the repository
+    """
+
+    query_variables = {
+        "owner": owner,
+        "name": repo,
+    }
+
+    with open(
+        path.join(path.dirname(__file__), "queries/get_repo_data.gql"), "r"
+    ) as query_file:
+        query = "".join(query_file.readlines())
+
+    payload = {"query": query, "variables": query_variables}
+    response = requests.post(api_url, data=json.dumps(payload), headers=headers)
+
+    status_code = response.status_code
+    result = response.json()
+
+    if status_code >= 200 and status_code <= 300:
+        repo_id = result["data"]["repository"]["id"]
+        labels = result["data"]["repository"]["labels"]["nodes"]
+
+        return repo_id, labels
+    else:
+        raise Exception(
+            "[ERROR] getting issue labels. Status Code: {status_code} - Message: {result}".format(
+                status_code=status_code, result=result["message"]
+            )
+        )
+
+
+def delete_label(label_id):
+    """
+    Delete the specified label
+    :param label_id: Label's node id.
+    :type label_id: str
+    :return: GitHub API request response.
+    """
+
+    query_variables = {
+        "deleteLabelInput": {"clientMutationId": client_id, "id": label_id}
+    }
+
+    with open(
+        path.join(path.dirname(__file__), "queries/delete_label.gql"), "r"
+    ) as query_file:
+        query = "".join(query_file.readlines())
+
+    payload = {"query": query, "variables": query_variables}
+    result = requests.post(api_url, data=json.dumps(payload), headers=headers).json()
+
+    return result
+
+
+@click.command()
+@click.option("--dry", is_flag=True)
+@click.argument("source_repo")
+@click.argument("target_repo")
+def copy_labels(source_repo, target_repo, dry):
+    """
+    Copy labels from the source repository to the target repository.
+    \f
+    :param source: The full name of a GitHub repo from where the labels will be copied from. Eg. github/opensourcefriday
+    :type source: str
+    :param target: The full name of a GitHub repo to where the labels will be copied. Eg. github/opensourcefriday
+    :type target: str
+    :return:
+    """
+    source_owner, source_repo_name = source_repo.split("/")
+    target_owner, target_repo_name = target_repo.split("/")
+
+    try:
+        print(
+            "Fetching labels for {source_repo_name} repo.".format(
+                source_repo_name=source_repo_name
+            )
+        )
+        _, source_repo_labels = get_labels(source_owner, source_repo_name)
+        print(
+            "Fetched labels for {source_repo_name}".format(
+                source_repo_name=source_repo_name
+            )
+        )
+
+        print(
+            "Fetching labels for {target_repo_name} repo.".format(
+                target_repo_name=target_repo_name
+            )
+        )
+        target_repo_id, target_repo_labels = get_labels(target_owner, target_repo_name)
+        print(
+            "Fetched labels for {target_repo_name}".format(
+                target_repo_name=target_repo_name
+            )
+        )
+
+        filtered_labels = list(
+            filter(lambda x: x not in target_repo_labels, source_repo_labels)
+        )
+
+        if dry:
+            print("This is just a dry run. No labels will be copied/created.")
+            print(
+                "{label_count} labels would have been created.".format(
+                    label_count=len(filtered_labels)
+                )
+            )
+            pprint(filtered_labels, indent=4)
+        else:
+            print(
+                "Preparing to created {label_count} labels in {target_repo}".format(
+                    label_count=len(filtered_labels), target_repo=target_repo
+                )
+            )
+
+            for label in filtered_labels:
+                create_label(target_repo_id, label)
+    except Exception as error:
+        sys.exit(error)
+
+    print("Done")
+
+
+if __name__ == "__main__":
+    # Pylint doesn't know that @click.command takes care of injecting the
+    # function parameters. Disabling Pylint error.
+    copy_labels()  # pylint: disable=no-value-for-parameter

.automation/test/jscpd/r_good_1.r

@@ -0,0 +1,48 @@
+# Each of the default linters should throw at least one lint on this file
+
+# assignment
+# function_left_parentheses
+# closed_curly
+# commas
+# paren_brace
+f <- function(x, y = 1) {
+
+}
+
+# commented_code
+
+# cyclocomp
+# equals_na
+# infix_spaces
+# line_length
+# object_length
+# object_name
+# object_usage
+# open_curly
+short_snake <- function(x) {
+  y <- 1
+  y <- y^2
+  if (1 > 2 && 5 * 10 > 6 && is.na(x)) {
+    TRUE
+  } else {
+    FALSE
+  }
+}
+
+# pipe_continuation
+# seq_linter
+# spaces_inside
+x <- 1:10
+x[2]
+seq_len(x) %>%
+  lapply(function(x) x * 2) %>%
+  head()
+
+# single_quotes
+message("single_quotes")
+
+# spaces_left_parentheses
+# trailing_whitespace
+y <- 2 + (1:10)
+
+# trailing_blank_lines

.automation/test/jscpd/reports/expected-JSCPD.tap

@@ -0,0 +1,40 @@
+TAP version 13
+1..32
+ok 1 - clojure_good_1.clj
+ok 2 - coffeescript_good_1.coffee
+ok 3 - csharp_good_01.cs
+ok 4 - css_good_01.css
+ok 5 - dart_good_1.dart
+ok 6 - gherkin_good_01.feature
+ok 7 - golang_good_01.go
+ok 8 - Dockerfile
+ok 9 - Dockerfile.dev
+ok 10 - groovy_good_01.groovy
+ok 11 - html_good_01.html
+ok 12 - java_good_1.java
+not ok 13 - javascript_bad_1.js
+  ---
+  message: Clone found (javascript) \n - /tmp/lint/.automation/test/jscpd/javascript_bad_1.js [119 5 - 126 20] (7 lines, 82 tokens)\n   /tmp/lint/.automation/test/jscpd/javascript_bad_1.js [66 5 - 73 18]\n\nClone found (javascript) \n - /tmp/lint/.automation/test/jscpd/javascript_bad_1.js [190 43 - 204 17] (14 lines, 145 tokens)\n   /tmp/lint/.automation/test/jscpd/javascript_bad_1.js [61 43 - 75 11]\n\nClone found (javascript) \n - /tmp/lint/.automation/test/jscpd/javascript_bad_1.js [119 5 - 126 20] (7 lines, 82 tokens)\n   /tmp/lint/.automation/test/jscpd/javascript_bad_1.js [66 5 - 73 18]\n\n 119 │ 66 │ }                                               \n 120 │ 67 │   }                                             \n 121 │ 68 │   let body = []                                 \n 122 │ 69 │   const req = https.request(options, (res) => { \n 123 │ 70 │     res.on('data', (chunk) => {                 \n 124 │ 71 │       body.push(chunk)                          \n 125 │ 72 │     }).on('end', () => {                        \n 126 │ 73 │       body = Buffer                             \n\nClone found (javascript) \n - /tmp/lint/.automation/test/jscpd/javascript_bad_1.js [190 43 - 204 17] (14 lines, 145 tokens)\n   /tmp/lint/.automation/test/jscpd/javascript_bad_1.js [61 43 - 75 11]\n\n 190 │ 61 │ ,                                                        \n 191 │ 62 │     method  'GET',                                       \n 192 │ 63 │     headers  {                                           \n 193 │ 64 │       Authorization  'token ' + (process.env.GHE_TOKEN), \n 194 │ 65 │       'Content-Type'  'application/json'                 \n 195 │ 66 │     }                                                    \n 196 │ 67 │   }                                                      \n 197 │ 68 │   let body = []                                          \n 198 │ 69 │   const req = https.request(options, (res) => {          \n 199 │ 70 │     res.on('data', (chunk) => {                          \n 200 │ 71 │       body.push(chunk)                                   \n 201 │ 72 │     }).on('end', () => {                                 \n 202 │ 73 │       body = JSON.parse(Buffer.concat(body))             \n 203 │ 74 │       body.forEach(item => {                             \n 204 │ 75 │         orgRepos                                         \n\nFound 2 clones.\nERROR  jscpd found too many duplicates (9.81%) over threshold (0%)\nError  ERROR  jscpd found too many duplicates (9.81%) over threshold (0%)\n    at ThresholdReporter.report (/node_modules/@jscpd/finder/dist/reporters/threshold.js)\n    at /node_modules/@jscpd/finder/dist/in-files-detector.js\n    at Array.forEach (<anonymous>)\n    at /node_modules/@jscpd/finder/dist/in-files-detector.js\n
+  ...
+ok 14 - json_good_1.json
+ok 15 - kotlint_good_1.kt
+ok 16 - latex_good_1.tex
+ok 17 - lua_good_1.lua
+ok 18 - markdown_good_1.md
+ok 19 - perl_good_1.pl
+ok 20 - php_good_1.php
+ok 21 - powershell_good_1.ps1
+ok 22 - protobuf_good_1.proto
+not ok 23 - python_bad_1.py
+  ---
+  message: Clone found (python) \n - /tmp/lint/.automation/test/jscpd/python_bad_1.py [77 70 - 84 16] (7 lines, 77 tokens)\n   /tmp/lint/.automation/test/jscpd/python_bad_1.py [50 69 - 55 82]\n\nClone found (python) \n - /tmp/lint/.automation/test/jscpd/python_bad_1.py [77 70 - 84 16] (7 lines, 77 tokens)\n   /tmp/lint/.automation/test/jscpd/python_bad_1.py [50 69 - 55 82]\n\n 77 │ 50 │ ), "r"                                                                           \n 78 │ 51 │     ) as query_file                                                              \n 79 │ 52 │         query = "".join(query_file.readlines())                                  \n 80 │ 53 │                                                                                  \n 81 │ 54 │     payload = {"query"  query, "variables"  query_variables}                     \n 82 │ 55 │     response = requests.post(api_url, data=json.dumps(payload), headers=headers) \n 83 │ 56 │                                                                                  \n 84 │ 57 │     status_code                                                                  \n\nFound 1 clones.\nERROR  jscpd found too many duplicates (3.61%) over threshold (0%)\nError  ERROR  jscpd found too many duplicates (3.61%) over threshold (0%)\n    at ThresholdReporter.report (/node_modules/@jscpd/finder/dist/reporters/threshold.js)\n    at /node_modules/@jscpd/finder/dist/in-files-detector.js\n    at Array.forEach (<anonymous>)\n    at /node_modules/@jscpd/finder/dist/in-files-detector.js\n
+  ...
+ok 24 - r_good_1.r
+ok 25 - expected-JSCPD.tap
+ok 26 - ruby_good_1.rb
+ok 27 - shell_good_1.sh
+ok 28 - sql_good_1.sql
+ok 29 - typescript_good_1 copy.ts
+ok 30 - typescript_good_1.ts
+ok 31 - xml_good_1.xml
+ok 32 - yml_good_1.yml

.automation/test/jscpd/ruby_good_1.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# Rails Console only
+# This script will output all active webhooks currently being processed by an instance.
+# Replace ARRAY_OF_URLS_CALLING_INSTANCE and GHES_URL with the appropriate values before running
+
+# Prior to running this script, compile a list of the top URLs containing the phrase webhook
+# This should be ran prior to entering the Rails Console with the command:
+# grep -B1 --no-group-separator 'Faraday::TimeoutError' hookshot-logs/resqued.log | sed -n 1~2p |
+# \ grep -v 'Faraday::TimeoutError: request timed out' | sort | uniq -c |sort -rn | head -n 20
+
+File.open("/tmp/urls.txt", "w") do |file|
+  Hook.active.map do |h|
+    urls = [ARRAY_OF_URLS_CALLING_INSTANCE]
+    next if urls.include? h.url
+
+    begin
+      file.puts "https://GHES_URL/api/v3/repos/#{h.installation_target.full_name}/hooks/#{h.id}"
+    rescue StandardError => e
+      puts e.message
+    end
+  end
+end

.automation/test/jscpd/shell_good_1.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# CMD
+HELLO_WORLD=$(echo "Hello World" | cut -f1 -d' ' 2>&1)
+
+# Load the error code
+ERROR_CODE=$?
+
+# Check the shell
+if [ ${ERROR_CODE} -ne 0 ]; then
+  echo "We did it!"
+  exit 0
+else
+  echo "We done goofed it..."
+  echo "${HELLO_WORLD}"
+  exit 1
+fi

.automation/test/jscpd/sql_good_1.sql

@@ -0,0 +1 @@
+DELETE from person WHERE 1=1;

.automation/test/jscpd/typescript_good_1 copy.ts

@@ -0,0 +1,6 @@
+const spiderman = (person) => {
+  return 'Hello, ' + person
+}
+
+const user = 'Peter Parker'
+console.log(spiderman(user))

.automation/test/jscpd/typescript_good_1.ts

@@ -0,0 +1,6 @@
+const spiderman = (person) => {
+  return 'Hello, ' + person
+}
+
+const user = 'Peter Parker'
+console.log(spiderman(user))

.automation/test/jscpd/xml_good_1.xml

@@ -0,0 +1,6 @@
+<note>
+  <to>Tove</to>
+  <from>Jani</from>
+  <heading>Reminder</heading>
+  <body>Don't forget me this weekend!</body>
+</note>

.automation/test/jscpd/yml_good_1.yml

@@ -0,0 +1,19 @@
+---
+
+#####################
+#####################
+## Heres some vars ##
+#####################
+#####################
+
+############
+# Env Vars #
+############
+env:
+  browser: true
+  es6: true
+  jest: true
+
+Here: there
+
+something: "For Nothing"

.automation/test/jsx/jsx_bad_1.jsx

@@ -0,0 +1 @@
+const element = <h1>Hello, world!

.automation/test/jsx/jsx_good_1.jsx

@@ -0,0 +1 @@
+const element = <h1>Hello, world!</h1>;

.automation/test/jsx/reports/expected-JSX.tap

@@ -0,0 +1,7 @@
+TAP version 13
+1..2
+not ok 1 - jsx_bad_1.jsx
+  ---
+  message: \n/tmp/lint/.automation/test/jsx/jsx_bad_1.jsx\n  1 17  error  Parsing error  JSX element 'h1' has no corresponding closing tag\n\n✖ 1 problem (1 error, 0 warnings)\n
+  ...
+ok 2 - jsx_good_1.jsx

.automation/test/openapi/reports/expected-OPENAPI.tap

@@ -1,12 +1,12 @@
 TAP version 13
 1..4
-not ok 1 - openapi_bad_1.ymlopenapi
+not ok 1 - openapi_bad_1.yml
   ---
-  message: OpenAPI 3.x detected\n\n/tmp/lint/.automation/test/openapi/openapi_bad_1.ymlopenapi\n 1 1  warning  info-contact      Info object should contain `contact` object.\n 1 1  warning  info-description  OpenAPI object info `description` must be present and non-empty string.\n 1 1  warning  oas3-api-servers  OpenAPI `servers` must be present and non-empty array.\n 1 1    error  oas3-schema       Object should have required property `info`.\n 1 1  warning  openapi-tags      OpenAPI object should have non-empty `tags` array.\n\n✖ 5 problems (1 error, 4 warnings, 0 infos, 0 hints)\n
+  message: OpenAPI 3.x detected\n\n/tmp/lint/.automation/test/openapi/openapi_bad_1.yml\n 1 1  warning  info-contact      Info object should contain `contact` object.\n 1 1  warning  info-description  OpenAPI object info `description` must be present and non-empty string.\n 1 1  warning  oas3-api-servers  OpenAPI `servers` must be present and non-empty array.\n 1 1    error  oas3-schema       Object should have required property `info`.\n 1 1  warning  openapi-tags      OpenAPI object should have non-empty `tags` array.\n\n✖ 5 problems (1 error, 4 warnings, 0 infos, 0 hints)\n
   ...
-not ok 2 - openapi_bad_2.jsonopenapi
+not ok 2 - openapi_bad_2.json
   ---
-  message: OpenAPI 3.x detected\n\n/tmp/lint/.automation/test/openapi/openapi_bad_2.jsonopenapi\n 1 1  warning  info-contact      Info object should contain `contact` object.\n 1 1  warning  info-description  OpenAPI object info `description` must be present and non-empty string.\n 1 1  warning  oas3-api-servers  OpenAPI `servers` must be present and non-empty array.\n 1 1    error  oas3-schema       Object should have required property `info`.\n 1 1  warning  openapi-tags      OpenAPI object should have non-empty `tags` array.\n\n✖ 5 problems (1 error, 4 warnings, 0 infos, 0 hints)\n
+  message: OpenAPI 3.x detected\n\n/tmp/lint/.automation/test/openapi/openapi_bad_2.json\n 1 1  warning  info-contact      Info object should contain `contact` object.\n 1 1  warning  info-description  OpenAPI object info `description` must be present and non-empty string.\n 1 1  warning  oas3-api-servers  OpenAPI `servers` must be present and non-empty array.\n 1 1    error  oas3-schema       Object should have required property `info`.\n 1 1  warning  openapi-tags      OpenAPI object should have non-empty `tags` array.\n\n✖ 5 problems (1 error, 4 warnings, 0 infos, 0 hints)\n
   ...
-ok 3 - openapi_good_1.ymlopenapi
+ok 3 - openapi_good_1.yml
-ok 4 - openapi_good_2.jsonopenapi
+ok 4 - openapi_good_2.json

.automation/test/tsx/reports/expected-TSX.tap

@@ -0,0 +1,7 @@
+TAP version 13
+1..2
+not ok 1 - tsx_bad_1.tsx
+  ---
+  message: \n/tmp/lint/.automation/test/tsx/tsx_bad_1.tsx\n  1 16  error  Parsing error  Type expected\n\n✖ 1 problem (1 error, 0 warnings)\n
+  ...
+ok 2 - tsx_good_1.tsx

.automation/test/tsx/tsx_bad_1.tsx

@@ -0,0 +1 @@
+var foo = bar as

.automation/test/tsx/tsx_good_1.tsx

@@ -0,0 +1 @@
+var foo = bar as foo;

.github/workflows/deploy-DEV.yml

@@ -76,7 +76,7 @@ jobs:
       #####################################
       - name: Run Test Cases
         shell: bash
-        run: docker run -e RUN_LOCAL=true -e TEST_CASE_RUN=true -e OUTPUT_FORMAT=tap -e OUTPUT_FOLDER=${GITHUB_SHA} -e OUTPUT_DETAILS=detailed -e ACTIONS_RUNNER_DEBUG=true -e ERROR_ON_MISSING_EXEC_BIT=true -v ${GITHUB_WORKSPACE}:/tmp/lint ghcr.io/github/super-linter:${GITHUB_SHA}
+        run: docker run -e RUN_LOCAL=true -e TEST_CASE_RUN=true -e ANSIBLE_DIRECTORY=.automation/test/ansible -e OUTPUT_FORMAT=tap -e OUTPUT_FOLDER=${GITHUB_SHA} -e OUTPUT_DETAILS=detailed -e ACTIONS_RUNNER_DEBUG=true -e ERROR_ON_MISSING_EXEC_BIT=true -v ${GITHUB_WORKSPACE}:/tmp/lint ghcr.io/github/super-linter:${GITHUB_SHA}
 
       #########################################
       # Clean code base to run against it all #

README.md

@@ -53,6 +53,7 @@ Developers on **GitHub** can call the **GitHub Action** to lint their code base
 | **CSS**                          | [stylelint](https://stylelint.io/)                                                                                                                                            |
 | **Clojure**                      | [clj-kondo](https://github.com/borkdude/clj-kondo)                                                                                                                            |
 | **CoffeeScript**                 | [coffeelint](https://coffeelint.github.io/)                                                                                                                                   |
+| **Copy/paste detection**         | [jscpd](https://github.com/kucherenko/jscpd)                                                                                                                                  |
 | **Dart**                         | [dartanalyzer](https://dart.dev/guides/language/analysis-options)                                                                                                             |
 | **Dockerfile**                   | [dockerfilelint](https://github.com/replicatedhq/dockerfilelint.git) / [hadolint](https://github.com/hadolint/hadolint)                                                       |
 | **EDITORCONFIG**                 | [editorconfig-checker](https://github.com/editorconfig-checker/editorconfig-checker)                                                                                          |
@@ -218,6 +219,7 @@ But if you wish to select or exclude specific linters, we give you full control
 | **FILTER_REGEX_INCLUDE**           | `all`                 | Regular expression defining which files will be processed by linters (ex: `.*src/.*`)                                                                                            |
 | **JAVASCRIPT_ES_CONFIG_FILE**      | `.eslintrc.yml`       | Filename for [eslint configuration](https://eslint.org/docs/user-guide/configuring#configuration-file-formats) (ex: `.eslintrc.yml`, `.eslintrc.json`)                           |
 | **JAVASCRIPT_DEFAULT_STYLE**       | `standard`            | Flag to set the default style of javascript. Available options: **standard**/**prettier**                                                                                        |
+| **JSCPD_CONFIG_FILE**              | `.jscpd.json`         | Filename for JSCPD configuration                                                                                                                                                 |
 | **LINTER_RULES_PATH**              | `.github/linters`     | Directory for all linter configuration rules.                                                                                                                                    |
 | **LOG_FILE**                       | `super-linter.log`    | The file name for outputting logs. All output is sent to the log file regardless of `LOG_LEVEL`.                                                                                 |
 | **LOG_LEVEL**                      | `VERBOSE`             | How much output the script will generate to the console. One of `ERROR`, `WARN`, `NOTICE`, `VERBOSE`, `DEBUG` or `TRACE`.                                                        |
@@ -256,6 +258,7 @@ But if you wish to select or exclude specific linters, we give you full control
 | **VALIDATE_JAVA**                  | `true`                | Flag to enable or disable the linting process of the language.                                                                                                                   |
 | **VALIDATE_JAVASCRIPT_ES**         | `true`                | Flag to enable or disable the linting process of the Javascript language. (Utilizing: eslint)                                                                                    |
 | **VALIDATE_JAVASCRIPT_STANDARD**   | `true`                | Flag to enable or disable the linting process of the Javascript language. (Utilizing: standard)                                                                                  |
+| **VALIDATE_JSCPD**                 | `true`                | Flag to enable or disable the JSCPD.                                                                                                                                             |
 | **VALIDATE_JSON**                  | `true`                | Flag to enable or disable the linting process of the JSON language.                                                                                                              |
 | **VALIDATE_JSX**                   | `true`                | Flag to enable or disable the linting process for jsx files (Utilizing: eslint)                                                                                                  |
 | **VALIDATE_KOTLIN**                | `true`                | Flag to enable or disable the linting process of the Kotlin language.                                                                                                            |

TEMPLATES/.jscpd.json

@@ -0,0 +1,10 @@
+{
+  "threshold": 0,
+  "reporters": [
+    "consoleFull"
+  ],
+  "ignore": [
+    "**/__snapshots__/**"
+  ],
+  "absolute": true
+}

dependencies/package.json

@@ -15,6 +15,7 @@
     "eslint-plugin-prettier": "^3.3.1",
     "gherkin-lint": "^4.1.3",
     "htmlhint": "^0.14.2",
+    "jscpd": "^3.3.21",
     "jsonlint": "^1.6.3",
     "markdownlint-cli": "^0.26.0",
     "npm-groovy-lint": "^8.1.0",

lib/functions/buildFileList.sh

@@ -223,6 +223,8 @@ function BuildFileList() {
 
     # Editorconfig-checker should check every file
     FILE_ARRAY_EDITORCONFIG+=("${FILE}")
+    # jscpd also runs an all files
+    FILE_ARRAY_JSCPD+=("${FILE}")
 
     #######################
     # Get the shell files #
@@ -514,9 +516,9 @@ function BuildFileList() {
       # Append the file to the array #
       ################################
       FILE_ARRAY_PYTHON_BLACK+=("${FILE}")
-      FILE_ARRAY_PYTHON_PYLINT+=("${FILE}")
       FILE_ARRAY_PYTHON_FLAKE8+=("${FILE}")
       FILE_ARRAY_PYTHON_ISORT+=("${FILE}")
+      FILE_ARRAY_PYTHON_PYLINT+=("${FILE}")
 
     ######################
     # Get the RAKU files #
@@ -563,7 +565,7 @@ function BuildFileList() {
     elif [ "${FILE_TYPE}" == "sql" ]; then
       ################################
       # Append the file to the array #
-      ##############################p##
+      ################################
       FILE_ARRAY_SQL+=("${FILE}")
 
     ###########################

lib/functions/detectFiles.sh

@@ -49,14 +49,10 @@ DetectOpenAPIFile() {
   # Check the shell for errors #
   ##############################
   if [ ${ERROR_CODE} -eq 0 ]; then
-    ########################
+    debug "${FILE} is an OpenAPI descriptor"
-    # Found string in file #
-    ########################
     return 0
   else
-    ###################
+    debug "${FILE} is NOT an OpenAPI descriptor"
-    # No string match #
-    ###################
     return 1
   fi
 }
@@ -268,6 +264,7 @@ function CheckFileType() {
     ################################
     # Append the file to the array #
     ################################
+    FILE_ARRAY_JSCPD+=("${FILE}")
     FILE_ARRAY_RUBY+=("${FILE}")
   else
     ############################

lib/functions/worker.sh

@@ -32,6 +32,7 @@ function LintCodebase() {
   # Set the flag #
   ################
   SKIP_FLAG=0
+  INDEX=0
 
   ############################################################
   # Check to see if we need to go through array or all files #
@@ -75,7 +76,6 @@ function LintCodebase() {
     ########################################
     if IsTAP; then
       TMPFILE=$(mktemp -q "/tmp/super-linter-${FILE_TYPE}.XXXXXX")
-      INDEX=0
       mkdir -p "${REPORT_OUTPUT_FOLDER}"
       REPORT_OUTPUT_FILE="${REPORT_OUTPUT_FOLDER}/super-linter-${FILE_TYPE}.${OUTPUT_FORMAT}"
     fi
@@ -371,6 +371,7 @@ function LintCodebase() {
         fi
       fi
     fi
+  fi
 
   ##############################
   # Validate we ran some tests #
@@ -382,5 +383,4 @@ function LintCodebase() {
     error "Failed to find any tests ran for the Linter:[${LINTER_NAME}]"!
     fatal "Please validate logic or that tests exist!"
   fi
-  fi
 }

lib/linter.sh

@@ -109,6 +109,8 @@ JAVASCRIPT_STYLE=''      # Variable for the style
 # shellcheck disable=SC2034  # Variable is referenced indirectly
 JAVASCRIPT_STANDARD_FILE_NAME="${JAVASCRIPT_ES_CONFIG_FILE:-.eslintrc.yml}"
 # shellcheck disable=SC2034  # Variable is referenced indirectly
+JSCPD_FILE_NAME="${JSCPD_CONFIG_FILE:-.jscpd.json}"
+# shellcheck disable=SC2034  # Variable is referenced indirectly
 JSX_FILE_NAME="${JAVASCRIPT_ES_CONFIG_FILE:-.eslintrc.yml}"
 # shellcheck disable=SC2034  # Variable is referenced indirectly
 LATEX_FILE_NAME=".chktexrc"
@@ -180,7 +182,7 @@ fi
 ##################
 LANGUAGE_ARRAY=('ANSIBLE' 'ARM' 'BASH' 'BASH_EXEC' 'CLOUDFORMATION' 'CLOJURE' 'COFFEESCRIPT' 'CSHARP' 'CSS'
   'DART' 'DOCKERFILE' 'DOCKERFILE_HADOLINT' 'EDITORCONFIG' 'ENV' 'GHERKIN' 'GO' 'GROOVY' 'HTML'
-  'JAVA' 'JAVASCRIPT_ES' "${JAVASCRIPT_STYLE_NAME}" 'JSON' 'JSX' 'KUBERNETES_KUBEVAL' 'KOTLIN' 'LATEX' 'LUA' 'MARKDOWN'
+  'JAVA' 'JAVASCRIPT_ES' "${JAVASCRIPT_STYLE_NAME}" 'JSCPD' 'JSON' 'JSX' 'KUBERNETES_KUBEVAL' 'KOTLIN' 'LATEX' 'LUA' 'MARKDOWN'
   'OPENAPI' 'PERL' 'PHP_BUILTIN' 'PHP_PHPCS' 'PHP_PHPSTAN' 'PHP_PSALM' 'POWERSHELL'
   'PROTOBUF' 'PYTHON_BLACK' 'PYTHON_PYLINT' 'PYTHON_FLAKE8' 'PYTHON_ISORT' 'R' 'RAKU' 'RUBY' 'SHELL_SHFMT' 'SNAKEMAKE_LINT' 'SNAKEMAKE_SNAKEFMT' 'STATES' 'SQL'
   'TEKTON' 'TERRAFORM' 'TERRAFORM_TERRASCAN' 'TERRAGRUNT' 'TSX' 'TYPESCRIPT_ES' 'TYPESCRIPT_STANDARD' 'XML' 'YAML')
@@ -210,6 +212,7 @@ LINTER_NAMES_ARRAY['HTML']="htmlhint"
 LINTER_NAMES_ARRAY['JAVA']="checkstyle"
 LINTER_NAMES_ARRAY['JAVASCRIPT_ES']="eslint"
 LINTER_NAMES_ARRAY["${JAVASCRIPT_STYLE_NAME}"]="${JAVASCRIPT_STYLE}"
+LINTER_NAMES_ARRAY['JSCPD']="jscpd"
 LINTER_NAMES_ARRAY['JSON']="jsonlint"
 LINTER_NAMES_ARRAY['JSX']="eslint"
 LINTER_NAMES_ARRAY['KOTLIN']="ktlint"
@@ -773,6 +776,7 @@ LINTER_COMMANDS_ARRAY['JAVA']="java -jar /usr/bin/checkstyle -c ${JAVA_LINTER_RU
 LINTER_COMMANDS_ARRAY['JAVASCRIPT_ES']="eslint --no-eslintrc -c ${JAVASCRIPT_ES_LINTER_RULES}"
 LINTER_COMMANDS_ARRAY['JAVASCRIPT_STANDARD']="standard ${JAVASCRIPT_STANDARD_LINTER_RULES}"
 LINTER_COMMANDS_ARRAY['JAVASCRIPT_PRETTIER']="prettier --check"
+LINTER_COMMANDS_ARRAY['JSCPD']="jscpd --config ${JSCPD_LINTER_RULES}"
 LINTER_COMMANDS_ARRAY['JSON']="jsonlint"
 LINTER_COMMANDS_ARRAY['JSX']="eslint --no-eslintrc -c ${JSX_LINTER_RULES}"
 LINTER_COMMANDS_ARRAY['KOTLIN']="ktlint"