Add find-unicode-control2 (#2286)

* Add find-unicode-control2

* make version easier

* cleaner version

Co-authored-by: Lukas Gravley <admiralawkbar@github.com>

.automation/test/unicode_control/README.md

@@ -0,0 +1,19 @@
+# Unicode control Test Cases
+
+This folder holds the test cases for **Unicode control**.
+
+## Additional Docs
+
+No Additional information is needed for this test case.
+
+## Good Test Cases
+
+The test cases denoted: `LANGUAGE_good_FILE.EXTENSION` are all valid, and should pass successfully when linted.
+
+- **Note:** They are linted utilizing the default linter rules.
+
+## Bad Test Cases
+
+The test cases denoted: `LANGUAGE_bad_FILE.EXTENSION` are **NOT** valid, and should trigger errors when linted.
+
+- **Note:** They are linted utilizing the default linter rules.

.automation/test/unicode_control/unicode_control_bad_1.cs

@@ -0,0 +1,5 @@
+string access_level = "user";
+if (access_level != "user"⁦) //Check if admin 
+{    
+  Console.WriteLine("You are an admin."); 
+}

.automation/test/unicode_control/unicode_control_good_1.cs

@@ -0,0 +1,5 @@
+string access_level = "user";
+if (access_level != "user") //Check if admin
+{    
+  Console.WriteLine("You are an admin."); 
+}

Dockerfile

@@ -38,6 +38,8 @@ FROM python:3.10.1-alpine as base_image
 ARG DART_VERSION='2.8.4'
 ## install alpine-pkg-glibc (glibc compatibility layer package for Alpine Linux)
 ARG GLIBC_VERSION='2.31-r0'
+# Unicode version info
+ARG UNICODE_VERSION='2021-11-01-1136'
 
 ####################
 # Run APK installs #
@@ -89,6 +91,9 @@ RUN pip3 install --no-cache-dir pipenv \
     # Bug in hadolint thinks pipenv is pip
     # hadolint ignore=DL3042
     && pipenv install --clear --system \
+    && wget --tries=5 -q https://access.redhat.com/sites/default/files/find_unicode_control2--${UNICODE_VERSION}.zip -O - -q | unzip -q - \
+    && mv find_unicode_control2.py /usr/local/bin/find_unicode_control2.py \
+    && chmod +x /usr/local/bin/find_unicode_control2.py \
     ####################
     # Run NPM Installs #
     ####################

README.md

@@ -100,6 +100,7 @@ Developers on **GitHub** can call the **GitHub Action** to lint their codebase w
 | **Terraform**                    | [tflint](https://github.com/terraform-linters/tflint) / [terrascan](https://github.com/accurics/terrascan)                                                                    |
 | **Terragrunt**                   | [terragrunt](https://github.com/gruntwork-io/terragrunt)                                                                                                                      |
 | **TypeScript**                   | [ESLint](https://eslint.org/) / [standard js](https://standardjs.com/)                                                                                                        |
+| **Unicode Control**              | [find-unicode-control2.py](https://access.redhat.com/security/vulnerabilities/RHSB-2021-007#diagnostic-tools:)                                                                |
 | **XML**                          | [LibXML](http://xmlsoft.org/)                                                                                                                                                 |
 | **YAML**                         | [YamlLint](https://github.com/adrienverge/yamllint)                                                                                                                           |
 
@@ -305,7 +306,7 @@ But if you wish to select or exclude specific linters, we give you full control
 | **MARKDOWN_CUSTOM_RULE_GLOBS**     | `.markdown-lint/rules,rules/**` | Comma-separated list of [file globs](https://github.com/igorshubovych/markdownlint-cli#globbing) matching [custom Markdownlint rule files](https://github.com/DavidAnson/markdownlint/blob/main/doc/CustomRules.md). |
 | **NATURAL_LANGUAGE_CONFIG_FILE**   | `.textlintrc`                   | Filename for [textlint configuration](https://textlint.github.io/docs/getting-started.html#configuration) (ex: `.textlintrc`)                                                                                        |
 | **PHP_CONFIG_FILE**                | `php.ini`                       | Filename for [PHP Configuration](https://www.php.net/manual/en/configuration.file.php) (ex: `php.ini`)                                                                                                               |
-| **PROTOBUF_CONFIG_FILE**           | `.protolintrc.yml`              | Filename for [protolint configuration](https://github.com/yoheimuta/protolint/blob/master/_example/config/.protolint.yaml) (ex: `.protolintrc.yml`)                                                                                                                                                    |
+| **PROTOBUF_CONFIG_FILE**           | `.protolintrc.yml`              | Filename for [protolint configuration](https://github.com/yoheimuta/protolint/blob/master/_example/config/.protolint.yaml) (ex: `.protolintrc.yml`)                                                                  |
 | **PYTHON_BLACK_CONFIG_FILE**       | `.python-black`                 | Filename for [black configuration](https://github.com/psf/black/blob/main/docs/guides/using_black_with_other_tools.md#black-compatible-configurations) (ex: `.isort.cfg`, `pyproject.toml`)                          |
 | **PYTHON_FLAKE8_CONFIG_FILE**      | `.flake8`                       | Filename for [flake8 configuration](https://flake8.pycqa.org/en/latest/user/configuration.html) (ex: `.flake8`, `tox.ini`)                                                                                           |
 | **PYTHON_ISORT_CONFIG_FILE**       | `.isort.cfg`                    | Filename for [isort configuration](https://pycqa.github.io/isort/docs/configuration/config_files.html) (ex: `.isort.cfg`, `pyproject.toml`)                                                                          |
@@ -322,6 +323,7 @@ But if you wish to select or exclude specific linters, we give you full control
 | **TERRAFORM_TERRASCAN_CONFIG_FILE**| `terrascan.toml`                | Filename for [terrascan configuration](https://github.com/accurics/terrascan) (ex: `terrascan.toml`)                                                                                                                 |
 | **TYPESCRIPT_DEFAULT_STYLE**       | `standard`                      | Flag to set the default style of TypeScript. Available options: **standard**/**prettier**                                                                                                                            |
 | **TYPESCRIPT_ES_CONFIG_FILE**      | `.eslintrc.yml`                 | Filename for [ESLint configuration](https://eslint.org/docs/user-guide/configuring#configuration-file-formats) (ex: `.eslintrc.yml`, `.eslintrc.json`)                                                               |
+| **UNICODE_CONTROL_CONFIG_FILE**    | `unicode_control_config.py`     | Filename for [find-unicode-control2.py configuration](https://access.redhat.com/security/vulnerabilities/RHSB-2021-007#diagnostic-tools:) (ex: `unicode_control_config.py`)                                          |
 | **USE_FIND_ALGORITHM**             | `false`                         | By default, we use `git diff` to find all files in the workspace and what has been updated, this would enable the Linux `find` method instead to find all files to lint                                              |
 | **VALIDATE_ALL_CODEBASE**          | `true`                          | Will parse the entire repository and find all files to validate across all types. **NOTE:** When set to `false`, only **new** or **edited** files will be parsed for validation.                                     |
 | **VALIDATE_JSCPD_ALL_CODEBASE**    | `false`                         | If set to `true`, will lint the whole codebase with JSCPD, in addition to linting files with JSCPD one by one. If set to `false`, JSCPD will only lint files one by one.                                             |
@@ -395,6 +397,7 @@ But if you wish to select or exclude specific linters, we give you full control
 | **VALIDATE_TSX**                   | `true`                          | Flag to enable or disable the linting process for tsx files (Utilizing: eslint)                                                                                                                                      |
 | **VALIDATE_TYPESCRIPT_ES**         | `true`                          | Flag to enable or disable the linting process of the TypeScript language. (Utilizing: eslint)                                                                                                                        |
 | **VALIDATE_TYPESCRIPT_STANDARD**   | `true`                          | Flag to enable or disable the linting process of the TypeScript language. (Utilizing: standard)                                                                                                                      |
+| **VALIDATE_UNICODE_CONTROL**       | `true`                          | Flag to enable or disable the linting process of the non-printable unicode characters.                                                                                                                               |
 | **VALIDATE_XML**                   | `true`                          | Flag to enable or disable the linting process of the XML language.                                                                                                                                                   |
 | **VALIDATE_YAML**                  | `true`                          | Flag to enable or disable the linting process of the YAML language.                                                                                                                                                  |
 | **YAML_CONFIG_FILE**               | `.yaml-lint.yml`                | Filename for [Yamllint configuration](https://yamllint.readthedocs.io/en/stable/configuration.html) (ex: `.yaml-lint.yml`, `.yamllint.yml`)                                                                          |

TEMPLATES/unicode_control_config.py

@@ -0,0 +1,10 @@
+scan_exclude = [
+    # Iconv test data
+    r"/iconvdata/testdata/",
+    # Test case data
+    r"libio/tst-widetext.input$",
+    # Test script.  This is to silence the warning:
+    # 'utf-8' codec can't decode byte 0xe9 in position 2118: invalid continuation byte
+    # since the script tests mixed encoding characters.
+    r"localedata/tst-langinfo.sh$",
+]

lib/functions/buildFileList.sh

@@ -318,6 +318,8 @@ function BuildFileList() {
     FILE_ARRAY_EDITORCONFIG+=("${FILE}")
     # jscpd also runs an all files
     FILE_ARRAY_JSCPD+=("${FILE}")
+    # unicode control also runs an all files
+    FILE_ARRAY_UNICODE_CONTROL+=("${FILE}")
     # Need to make sure we dont check the secrets paterns
     # for secrets, as it will pop!
     if [ "${BASE_FILE}" != ".gitleaks.toml" ]; then

lib/functions/linterVersions.sh

@@ -72,7 +72,7 @@ BuildLinterVersions() {
       if [[ ${LINTER} == "arm-ttk" ]]; then
         # Need specific command for ARM
         GET_VERSION_CMD="$(grep -iE 'version' "/usr/bin/arm-ttk" | xargs 2>&1)"
-      elif [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]] || [[ ${LINTER} == "gitleaks" ]]; then
+      elif [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]] || [[ ${LINTER} == "gitleaks" ]] || [[ ${LINTER} == "find_unicode_control2.py" ]]; then
         # Need specific command for Protolint and editorconfig-checker
         GET_VERSION_CMD="$(echo "--version not supported")"
       elif [[ ${LINTER} == "lintr" ]]; then

lib/functions/worker.sh

@@ -271,6 +271,10 @@ function LintCodebase() {
       #######################
       ERROR_CODE=$?
 
+      if [[ ${FILE_TYPE} == "UNICODE_CONTROL" ]] && [ -n "${LINT_CMD}" ]; then
+        ERROR_CODE=1
+      fi
+
       ########################################
       # Check for if it was supposed to pass #
       ########################################

lib/linter.sh

@@ -188,6 +188,8 @@ TYPESCRIPT_ES_FILE_NAME="${TYPESCRIPT_ES_CONFIG_FILE:-.eslintrc.yml}"
 # shellcheck disable=SC2034  # Variable is referenced indirectly
 TYPESCRIPT_STANDARD_FILE_NAME="${TYPESCRIPT_ES_CONFIG_FILE:-.eslintrc.yml}"
 # shellcheck disable=SC2034  # Variable is referenced indirectly
+UNICODE_CONTROL_FILE_NAME="${UNICODE_CONTROL_CONFIG_FILE:-unicode_control_config.py}"
+# shellcheck disable=SC2034  # Variable is referenced indirectly
 USE_FIND_ALGORITHM="${USE_FIND_ALGORITHM:-false}"
 # shellcheck disable=SC2034  # Variable is referenced indirectly
 YAML_FILE_NAME="${YAML_CONFIG_FILE:-.yaml-lint.yml}"
@@ -242,7 +244,7 @@ LANGUAGE_ARRAY=('ANSIBLE' 'ARM' 'BASH' 'BASH_EXEC' 'CLANG_FORMAT'
   'PYTHON_ISORT' 'PYTHON_MYPY' 'R' 'RAKU' 'RUBY' 'RUST_2015' 'RUST_2018'
   'RUST_2021' 'RUST_CLIPPY' 'SCALAFMT' 'SHELL_SHFMT' 'SNAKEMAKE_LINT'
   'SNAKEMAKE_SNAKEFMT' 'STATES' 'SQL' 'SQLFLUFF' 'TEKTON' 'TERRAFORM_TFLINT'
-  'TERRAFORM_TERRASCAN' 'TERRAGRUNT' 'TSX' 'TYPESCRIPT_ES'
+  'TERRAFORM_TERRASCAN' 'TERRAGRUNT' 'TSX' 'TYPESCRIPT_ES' 'UNICODE_CONTROL'
   "${TYPESCRIPT_STYLE_NAME}" 'XML' 'YAML')
 
 ##############################
@@ -318,6 +320,7 @@ LINTER_NAMES_ARRAY['TERRAGRUNT']="terragrunt"
 LINTER_NAMES_ARRAY['TSX']="eslint"
 LINTER_NAMES_ARRAY['TYPESCRIPT_ES']="eslint"
 LINTER_NAMES_ARRAY["${TYPESCRIPT_STYLE_NAME}"]="${TYPESCRIPT_STYLE}"
+LINTER_NAMES_ARRAY['UNICODE_CONTROL']="find_unicode_control2.py"
 LINTER_NAMES_ARRAY['XML']="xmllint"
 LINTER_NAMES_ARRAY['YAML']="yamllint"
 
@@ -938,6 +941,7 @@ LINTER_COMMANDS_ARRAY['TSX']="eslint --no-eslintrc -c ${TSX_LINTER_RULES}"
 LINTER_COMMANDS_ARRAY['TYPESCRIPT_ES']="eslint --no-eslintrc -c ${TYPESCRIPT_ES_LINTER_RULES}"
 LINTER_COMMANDS_ARRAY['TYPESCRIPT_STANDARD']="standard --parser @typescript-eslint/parser --plugin @typescript-eslint/eslint-plugin ${TYPESCRIPT_STANDARD_LINTER_RULES}"
 LINTER_COMMANDS_ARRAY['TYPESCRIPT_PRETTIER']="prettier --check"
+LINTER_COMMANDS_ARRAY['UNICODE_CONTROL']="find_unicode_control2.py -c ${UNICODE_CONTROL_LINTER_RULES}"
 LINTER_COMMANDS_ARRAY['XML']="xmllint"
 LINTER_COMMANDS_ARRAY['YAML']="yamllint -c ${YAML_LINTER_RULES} -f parsable"