diff --git a/.automation/test/unicode_control/README.md b/.automation/test/unicode_control/README.md new file mode 100644 index 00000000..f329c5fe --- /dev/null +++ b/.automation/test/unicode_control/README.md @@ -0,0 +1,19 @@ +# Unicode control Test Cases + +This folder holds the test cases for **Unicode control**. + +## Additional Docs + +No Additional information is needed for this test case. + +## Good Test Cases + +The test cases denoted: `LANGUAGE_good_FILE.EXTENSION` are all valid, and should pass successfully when linted. + +- **Note:** They are linted utilizing the default linter rules. + +## Bad Test Cases + +The test cases denoted: `LANGUAGE_bad_FILE.EXTENSION` are **NOT** valid, and should trigger errors when linted. + +- **Note:** They are linted utilizing the default linter rules. diff --git a/.automation/test/unicode_control/unicode_control_bad_1.cs b/.automation/test/unicode_control/unicode_control_bad_1.cs new file mode 100644 index 00000000..5987b848 --- /dev/null +++ b/.automation/test/unicode_control/unicode_control_bad_1.cs @@ -0,0 +1,5 @@ +string access_level = "user"; +if (access_level != "user"⁦) //Check if admin +{ + Console.WriteLine("You are an admin."); +} diff --git a/.automation/test/unicode_control/unicode_control_good_1.cs b/.automation/test/unicode_control/unicode_control_good_1.cs new file mode 100644 index 00000000..2758c07b --- /dev/null +++ b/.automation/test/unicode_control/unicode_control_good_1.cs @@ -0,0 +1,5 @@ +string access_level = "user"; +if (access_level != "user") //Check if admin +{ + Console.WriteLine("You are an admin."); +} diff --git a/Dockerfile b/Dockerfile index e290db84..adbe2d67 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,6 +38,8 @@ FROM python:3.10.1-alpine as base_image ARG DART_VERSION='2.8.4' ## install alpine-pkg-glibc (glibc compatibility layer package for Alpine Linux) ARG GLIBC_VERSION='2.31-r0' +# Unicode version info +ARG UNICODE_VERSION='2021-11-01-1136' #################### # Run APK installs # @@ -89,6 +91,9 @@ RUN pip3 install --no-cache-dir pipenv \ # Bug in hadolint thinks pipenv is pip # hadolint ignore=DL3042 && pipenv install --clear --system \ + && wget --tries=5 -q https://access.redhat.com/sites/default/files/find_unicode_control2--${UNICODE_VERSION}.zip -O - -q | unzip -q - \ + && mv find_unicode_control2.py /usr/local/bin/find_unicode_control2.py \ + && chmod +x /usr/local/bin/find_unicode_control2.py \ #################### # Run NPM Installs # #################### diff --git a/README.md b/README.md index 407da7e4..9f8b88f4 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ Developers on **GitHub** can call the **GitHub Action** to lint their codebase w | **Terraform** | [tflint](https://github.com/terraform-linters/tflint) / [terrascan](https://github.com/accurics/terrascan) | | **Terragrunt** | [terragrunt](https://github.com/gruntwork-io/terragrunt) | | **TypeScript** | [ESLint](https://eslint.org/) / [standard js](https://standardjs.com/) | +| **Unicode Control** | [find-unicode-control2.py](https://access.redhat.com/security/vulnerabilities/RHSB-2021-007#diagnostic-tools:) | | **XML** | [LibXML](http://xmlsoft.org/) | | **YAML** | [YamlLint](https://github.com/adrienverge/yamllint) | @@ -305,7 +306,7 @@ But if you wish to select or exclude specific linters, we give you full control | **MARKDOWN_CUSTOM_RULE_GLOBS** | `.markdown-lint/rules,rules/**` | Comma-separated list of [file globs](https://github.com/igorshubovych/markdownlint-cli#globbing) matching [custom Markdownlint rule files](https://github.com/DavidAnson/markdownlint/blob/main/doc/CustomRules.md). | | **NATURAL_LANGUAGE_CONFIG_FILE** | `.textlintrc` | Filename for [textlint configuration](https://textlint.github.io/docs/getting-started.html#configuration) (ex: `.textlintrc`) | | **PHP_CONFIG_FILE** | `php.ini` | Filename for [PHP Configuration](https://www.php.net/manual/en/configuration.file.php) (ex: `php.ini`) | -| **PROTOBUF_CONFIG_FILE** | `.protolintrc.yml` | Filename for [protolint configuration](https://github.com/yoheimuta/protolint/blob/master/_example/config/.protolint.yaml) (ex: `.protolintrc.yml`) | +| **PROTOBUF_CONFIG_FILE** | `.protolintrc.yml` | Filename for [protolint configuration](https://github.com/yoheimuta/protolint/blob/master/_example/config/.protolint.yaml) (ex: `.protolintrc.yml`) | | **PYTHON_BLACK_CONFIG_FILE** | `.python-black` | Filename for [black configuration](https://github.com/psf/black/blob/main/docs/guides/using_black_with_other_tools.md#black-compatible-configurations) (ex: `.isort.cfg`, `pyproject.toml`) | | **PYTHON_FLAKE8_CONFIG_FILE** | `.flake8` | Filename for [flake8 configuration](https://flake8.pycqa.org/en/latest/user/configuration.html) (ex: `.flake8`, `tox.ini`) | | **PYTHON_ISORT_CONFIG_FILE** | `.isort.cfg` | Filename for [isort configuration](https://pycqa.github.io/isort/docs/configuration/config_files.html) (ex: `.isort.cfg`, `pyproject.toml`) | @@ -322,6 +323,7 @@ But if you wish to select or exclude specific linters, we give you full control | **TERRAFORM_TERRASCAN_CONFIG_FILE**| `terrascan.toml` | Filename for [terrascan configuration](https://github.com/accurics/terrascan) (ex: `terrascan.toml`) | | **TYPESCRIPT_DEFAULT_STYLE** | `standard` | Flag to set the default style of TypeScript. Available options: **standard**/**prettier** | | **TYPESCRIPT_ES_CONFIG_FILE** | `.eslintrc.yml` | Filename for [ESLint configuration](https://eslint.org/docs/user-guide/configuring#configuration-file-formats) (ex: `.eslintrc.yml`, `.eslintrc.json`) | +| **UNICODE_CONTROL_CONFIG_FILE** | `unicode_control_config.py` | Filename for [find-unicode-control2.py configuration](https://access.redhat.com/security/vulnerabilities/RHSB-2021-007#diagnostic-tools:) (ex: `unicode_control_config.py`) | | **USE_FIND_ALGORITHM** | `false` | By default, we use `git diff` to find all files in the workspace and what has been updated, this would enable the Linux `find` method instead to find all files to lint | | **VALIDATE_ALL_CODEBASE** | `true` | Will parse the entire repository and find all files to validate across all types. **NOTE:** When set to `false`, only **new** or **edited** files will be parsed for validation. | | **VALIDATE_JSCPD_ALL_CODEBASE** | `false` | If set to `true`, will lint the whole codebase with JSCPD, in addition to linting files with JSCPD one by one. If set to `false`, JSCPD will only lint files one by one. | @@ -395,6 +397,7 @@ But if you wish to select or exclude specific linters, we give you full control | **VALIDATE_TSX** | `true` | Flag to enable or disable the linting process for tsx files (Utilizing: eslint) | | **VALIDATE_TYPESCRIPT_ES** | `true` | Flag to enable or disable the linting process of the TypeScript language. (Utilizing: eslint) | | **VALIDATE_TYPESCRIPT_STANDARD** | `true` | Flag to enable or disable the linting process of the TypeScript language. (Utilizing: standard) | +| **VALIDATE_UNICODE_CONTROL** | `true` | Flag to enable or disable the linting process of the non-printable unicode characters. | | **VALIDATE_XML** | `true` | Flag to enable or disable the linting process of the XML language. | | **VALIDATE_YAML** | `true` | Flag to enable or disable the linting process of the YAML language. | | **YAML_CONFIG_FILE** | `.yaml-lint.yml` | Filename for [Yamllint configuration](https://yamllint.readthedocs.io/en/stable/configuration.html) (ex: `.yaml-lint.yml`, `.yamllint.yml`) | diff --git a/TEMPLATES/unicode_control_config.py b/TEMPLATES/unicode_control_config.py new file mode 100644 index 00000000..5d41c68a --- /dev/null +++ b/TEMPLATES/unicode_control_config.py @@ -0,0 +1,10 @@ +scan_exclude = [ + # Iconv test data + r"/iconvdata/testdata/", + # Test case data + r"libio/tst-widetext.input$", + # Test script. This is to silence the warning: + # 'utf-8' codec can't decode byte 0xe9 in position 2118: invalid continuation byte + # since the script tests mixed encoding characters. + r"localedata/tst-langinfo.sh$", +] diff --git a/lib/functions/buildFileList.sh b/lib/functions/buildFileList.sh index 9ff8ad87..0550bb7c 100755 --- a/lib/functions/buildFileList.sh +++ b/lib/functions/buildFileList.sh @@ -318,6 +318,8 @@ function BuildFileList() { FILE_ARRAY_EDITORCONFIG+=("${FILE}") # jscpd also runs an all files FILE_ARRAY_JSCPD+=("${FILE}") + # unicode control also runs an all files + FILE_ARRAY_UNICODE_CONTROL+=("${FILE}") # Need to make sure we dont check the secrets paterns # for secrets, as it will pop! if [ "${BASE_FILE}" != ".gitleaks.toml" ]; then diff --git a/lib/functions/linterVersions.sh b/lib/functions/linterVersions.sh index acdfb458..7bebf445 100755 --- a/lib/functions/linterVersions.sh +++ b/lib/functions/linterVersions.sh @@ -72,7 +72,7 @@ BuildLinterVersions() { if [[ ${LINTER} == "arm-ttk" ]]; then # Need specific command for ARM GET_VERSION_CMD="$(grep -iE 'version' "/usr/bin/arm-ttk" | xargs 2>&1)" - elif [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]] || [[ ${LINTER} == "gitleaks" ]]; then + elif [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]] || [[ ${LINTER} == "gitleaks" ]] || [[ ${LINTER} == "find_unicode_control2.py" ]]; then # Need specific command for Protolint and editorconfig-checker GET_VERSION_CMD="$(echo "--version not supported")" elif [[ ${LINTER} == "lintr" ]]; then diff --git a/lib/functions/worker.sh b/lib/functions/worker.sh index 96c81205..58770408 100755 --- a/lib/functions/worker.sh +++ b/lib/functions/worker.sh @@ -271,6 +271,10 @@ function LintCodebase() { ####################### ERROR_CODE=$? + if [[ ${FILE_TYPE} == "UNICODE_CONTROL" ]] && [ -n "${LINT_CMD}" ]; then + ERROR_CODE=1 + fi + ######################################## # Check for if it was supposed to pass # ######################################## diff --git a/lib/linter.sh b/lib/linter.sh index 8b91bf11..b6a65fd2 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -188,6 +188,8 @@ TYPESCRIPT_ES_FILE_NAME="${TYPESCRIPT_ES_CONFIG_FILE:-.eslintrc.yml}" # shellcheck disable=SC2034 # Variable is referenced indirectly TYPESCRIPT_STANDARD_FILE_NAME="${TYPESCRIPT_ES_CONFIG_FILE:-.eslintrc.yml}" # shellcheck disable=SC2034 # Variable is referenced indirectly +UNICODE_CONTROL_FILE_NAME="${UNICODE_CONTROL_CONFIG_FILE:-unicode_control_config.py}" +# shellcheck disable=SC2034 # Variable is referenced indirectly USE_FIND_ALGORITHM="${USE_FIND_ALGORITHM:-false}" # shellcheck disable=SC2034 # Variable is referenced indirectly YAML_FILE_NAME="${YAML_CONFIG_FILE:-.yaml-lint.yml}" @@ -242,7 +244,7 @@ LANGUAGE_ARRAY=('ANSIBLE' 'ARM' 'BASH' 'BASH_EXEC' 'CLANG_FORMAT' 'PYTHON_ISORT' 'PYTHON_MYPY' 'R' 'RAKU' 'RUBY' 'RUST_2015' 'RUST_2018' 'RUST_2021' 'RUST_CLIPPY' 'SCALAFMT' 'SHELL_SHFMT' 'SNAKEMAKE_LINT' 'SNAKEMAKE_SNAKEFMT' 'STATES' 'SQL' 'SQLFLUFF' 'TEKTON' 'TERRAFORM_TFLINT' - 'TERRAFORM_TERRASCAN' 'TERRAGRUNT' 'TSX' 'TYPESCRIPT_ES' + 'TERRAFORM_TERRASCAN' 'TERRAGRUNT' 'TSX' 'TYPESCRIPT_ES' 'UNICODE_CONTROL' "${TYPESCRIPT_STYLE_NAME}" 'XML' 'YAML') ############################## @@ -318,6 +320,7 @@ LINTER_NAMES_ARRAY['TERRAGRUNT']="terragrunt" LINTER_NAMES_ARRAY['TSX']="eslint" LINTER_NAMES_ARRAY['TYPESCRIPT_ES']="eslint" LINTER_NAMES_ARRAY["${TYPESCRIPT_STYLE_NAME}"]="${TYPESCRIPT_STYLE}" +LINTER_NAMES_ARRAY['UNICODE_CONTROL']="find_unicode_control2.py" LINTER_NAMES_ARRAY['XML']="xmllint" LINTER_NAMES_ARRAY['YAML']="yamllint" @@ -938,6 +941,7 @@ LINTER_COMMANDS_ARRAY['TSX']="eslint --no-eslintrc -c ${TSX_LINTER_RULES}" LINTER_COMMANDS_ARRAY['TYPESCRIPT_ES']="eslint --no-eslintrc -c ${TYPESCRIPT_ES_LINTER_RULES}" LINTER_COMMANDS_ARRAY['TYPESCRIPT_STANDARD']="standard --parser @typescript-eslint/parser --plugin @typescript-eslint/eslint-plugin ${TYPESCRIPT_STANDARD_LINTER_RULES}" LINTER_COMMANDS_ARRAY['TYPESCRIPT_PRETTIER']="prettier --check" +LINTER_COMMANDS_ARRAY['UNICODE_CONTROL']="find_unicode_control2.py -c ${UNICODE_CONTROL_LINTER_RULES}" LINTER_COMMANDS_ARRAY['XML']="xmllint" LINTER_COMMANDS_ARRAY['YAML']="yamllint -c ${YAML_LINTER_RULES} -f parsable"