diff --git a/.automation/test/sql/sql_good_1.sql b/.automation/test/sql/sql_good_1.sql index 2e54affa..c1f23fb3 100644 --- a/.automation/test/sql/sql_good_1.sql +++ b/.automation/test/sql/sql_good_1.sql @@ -1 +1 @@ -DELETE from person WHERE 1=1; +DELETE FROM person WHERE 1 = 1; diff --git a/.automation/test/sqlfluff/README.md b/.automation/test/sqlfluff/README.md new file mode 100644 index 00000000..443f1eb0 --- /dev/null +++ b/.automation/test/sqlfluff/README.md @@ -0,0 +1,19 @@ +# SQL Fluff Test Cases + +This folder holds the test cases for **SQL**. + +## Additional Docs + +No Additional information is needed for this test case. + +## Good Test Cases + +The test cases denoted: `LANGUAGE_good_FILE.EXTENSION` are all valid, and should pass successfully when linted. + +- **Note:** They are linted utilizing the default linter rules. + +## Bad Test Cases + +The test cases denoted: `LANGUAGE_bad_FILE.EXTENSION` are **NOT** valid, and should trigger errors when linted. + +- **Note:** They are linted utilizing the default linter rules. diff --git a/.automation/test/sqlfluff/sqlfluff_bad_1.sql b/.automation/test/sqlfluff/sqlfluff_bad_1.sql new file mode 100644 index 00000000..82e1f3eb --- /dev/null +++ b/.automation/test/sqlfluff/sqlfluff_bad_1.sql @@ -0,0 +1 @@ +DELETE from person; diff --git a/.automation/test/sqlfluff/sqlfluff_good_1.sql b/.automation/test/sqlfluff/sqlfluff_good_1.sql new file mode 100644 index 00000000..c1f23fb3 --- /dev/null +++ b/.automation/test/sqlfluff/sqlfluff_good_1.sql @@ -0,0 +1 @@ +DELETE FROM person WHERE 1 = 1; diff --git a/README.md b/README.md index 5f131015..5e283039 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ Developers on **GitHub** can call the **GitHub Action** to lint their code base | **Rust** | [Rustfmt](https://github.com/rust-lang/rustfmt) / [Clippy](https://github.com/rust-lang/rust-clippy) | | **Shell** | [Shellcheck](https://github.com/koalaman/shellcheck) / [executable bit check] / [shfmt](https://github.com/mvdan/sh) | | **Snakemake** | [snakefmt](https://github.com/snakemake/snakefmt/) / [snakemake --lint](https://snakemake.readthedocs.io/en/stable/snakefiles/writing_snakefiles.html#best-practices) | -| **SQL** | [sql-lint](https://github.com/joereynolds/sql-lint) | +| **SQL** | [sql-lint](https://github.com/joereynolds/sql-lint) / [slqfluff](https://github.com/sqlfluff/sqlfluff) | | **Tekton** | [tekton-lint](https://github.com/IBM/tekton-lint) | | **Terraform** | [tflint](https://github.com/terraform-linters/tflint) / [terrascan](https://github.com/accurics/terrascan) | | **Terragrunt** | [terragrunt](https://github.com/gruntwork-io/terragrunt) | @@ -372,6 +372,7 @@ But if you wish to select or exclude specific linters, we give you full control | **VALIDATE_SNAKEMAKE_SNAKEFMT** | `true` | Flag to enable or disable the linting process of Snakefiles. (Utilizing: snakefmt) | | **VALIDATE_STATES** | `true` | Flag to enable or disable the linting process for AWS States Language. | | **VALIDATE_SQL** | `true` | Flag to enable or disable the linting process of the SQL language. | +| **VALIDATE_SQLFLUFF** | `true` | Flag to enable or disable the linting process of the SQL language. (Utilizing: sqlfuff) | | **VALIDATE_TEKTON** | `true` | Flag to enable or disable the linting process of the Tekton language. | | **VALIDATE_TERRAFORM** | `true` | Flag to enable or disable the linting process of the Terraform language. | | **VALIDATE_TERRAFORM_TERRASCAN** | `true` | Flag to enable or disable the linting process of the Terraform language for security related issues. | diff --git a/dependencies/Pipfile b/dependencies/Pipfile index f7877f76..183672ef 100644 --- a/dependencies/Pipfile +++ b/dependencies/Pipfile @@ -18,6 +18,7 @@ pylint = "*" pybind11 = "*" snakefmt = "*" snakemake = "*" +sqlfluff = "*" typing_extensions = "*" types-requests = "*" yamllint = "*" diff --git a/dependencies/Pipfile.lock b/dependencies/Pipfile.lock index a8299bf9..eeff0de9 100644 --- a/dependencies/Pipfile.lock +++ b/dependencies/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "db64d6989773d3dd9861e899160306153e8e952c1a58c6c124cad033d4b5911c" + "sha256": "041b763f22ac63892f4e3b138c4457fd235609c3bec9f27bfbe027c61c6e8fae" }, "pipfile-spec": 6, "requires": { @@ -58,11 +58,11 @@ }, "astroid": { "hashes": [ - "sha256:7b963d1c590d490f60d2973e57437115978d3a2529843f160b5003b721e1e925", - "sha256:83e494b02d75d07d4e347b27c066fd791c0c74fc96c613d1ea3de0c82c48168f" + "sha256:3975a0bd5373bdce166e60c851cfcbaf21ee96de80ec518c1f4cb3e94c3fb334", + "sha256:ab7f36e8a78b8e54a62028ba6beef7561db4cdb6f2a5009ecc44a6f42b5697ef" ], "markers": "python_version ~= '3.6'", - "version": "==2.6.5" + "version": "==2.6.6" }, "attrs": { "hashes": [ @@ -89,19 +89,19 @@ }, "boto3": { "hashes": [ - "sha256:596fb9df00a816780db8620d9f62982eb783b3eb63a75947e172101d0785e6aa", - "sha256:e5abbb2b5ebe5ad1157a3af8f28c5c944e9c6eff0dd3e778008894e018bc7e09" + "sha256:20516c8d0cb13d3d99496828d4b5dce82c254ce4c0163254c89d825ba9c65d30", + "sha256:611037785a35517d2a65339ad7531d7a223ce41a5c9a3968e2c14e64f6224f65" ], "markers": "python_version >= '3.6'", - "version": "==1.18.12" + "version": "==1.18.14" }, "botocore": { "hashes": [ - "sha256:7b205f96bf0e2e1017301339e4fba9fd6dfdf54680196eb43e60e60581d7d5cb", - "sha256:8710d03b9de3e3d94ed410f3e83809ca02050b091100d68c22ff7bf986f29fb6" + "sha256:45656938d7fb043f06cc1a013f276caa0193ab7cc4d71874a32c18910c1f8f8d", + "sha256:7f64424def270f3bb8e29da8c9120153a7f3ff5a32a517cc64a66f53d70ecec5" ], "markers": "python_version >= '3.6'", - "version": "==1.21.12" + "version": "==1.21.14" }, "bracex": { "hashes": [ @@ -111,6 +111,13 @@ "markers": "python_version >= '3.6'", "version": "==2.1.1" }, + "cached-property": { + "hashes": [ + "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130", + "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0" + ], + "version": "==1.5.2" + }, "certifi": { "hashes": [ "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee", @@ -176,6 +183,14 @@ "index": "pypi", "version": "==0.53.0" }, + "chardet": { + "hashes": [ + "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa", + "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==4.0.0" + }, "charset-normalizer": { "hashes": [ "sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b", @@ -215,6 +230,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==1.5.1" }, + "configparser": { + "hashes": [ + "sha256:85d5de102cfe6d14a5172676f09d19c465ce63d6019cf0a4ef13385fc535e828", + "sha256:af59f2cdd7efbdd5d111c1976ecd0b82db9066653362f0962d7bf1d3ab89a1fa" + ], + "markers": "python_version >= '3.6'", + "version": "==5.0.2" + }, "connection-pool": { "hashes": [ "sha256:bf429e7aef65921c69b4ed48f3d48d3eac1383b05d2df91884705842d974d0dc" @@ -323,6 +346,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.8.2" }, + "diff-cover": { + "hashes": [ + "sha256:b22fe97d118fadb2528bbc0a1f9fc370491b29b1b3fe2e2f1cdb94bf028814c7", + "sha256:cfe6333c9b83a28f91214e06e3a86108aeeb6a9a31233944ce8ef236121ba899" + ], + "markers": "python_version >= '3.6'", + "version": "==6.2.1" + }, "docutils": { "hashes": [ "sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125", @@ -386,6 +417,21 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==1.7.0" }, + "inflect": { + "hashes": [ + "sha256:41a23f6788962e9775e40e2ecfb1d6455d02de315022afeedd3c5dc070019d73", + "sha256:42560be16af702a21d43d59427f276b5aed79efb1ded9b713468c081f4353d10" + ], + "markers": "python_version >= '3.6'", + "version": "==5.3.0" + }, + "iniconfig": { + "hashes": [ + "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3", + "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32" + ], + "version": "==1.1.1" + }, "ipython-genutils": { "hashes": [ "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8", @@ -409,6 +455,13 @@ "markers": "python_version >= '3.6'", "version": "==3.0.1" }, + "jinja2-pluralize": { + "hashes": [ + "sha256:4fec874a591014774d4c66cb7f65314390731bfc57db4c27119db61aa93b2bc4", + "sha256:df5c2d5017b9b54c0a66cb790cca9fc08945837c3dbfc323589203f1ffb73c1c" + ], + "version": "==0.3.0" + }, "jmespath": { "hashes": [ "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", @@ -619,6 +672,13 @@ "markers": "python_version >= '3.5'", "version": "==2.6.2" }, + "oyaml": { + "hashes": [ + "sha256:3a378747b7fb2425533d1ce41962d6921cda075d46bb480a158d45242d156323", + "sha256:ed8fc096811f4763e1907dce29c35895d6d5936c4d0400fe843a91133d4744ed" + ], + "version": "==1.0" + }, "packaging": { "hashes": [ "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7", @@ -634,6 +694,14 @@ ], "version": "==0.9.0" }, + "pluggy": { + "hashes": [ + "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", + "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.13.1" + }, "psutil": { "hashes": [ "sha256:0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64", @@ -675,13 +743,21 @@ ], "version": "==2.4" }, + "py": { + "hashes": [ + "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3", + "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.10.0" + }, "pybind11": { "hashes": [ - "sha256:3e2a9a94396fbb27e75acf28d3de26e029576be1d4b38acc846ae08ef0eb3033", - "sha256:71dfd6e61f6aef3e24eda3b9770a0d53072346871f9f5a0510598ec86b5f9dc2" + "sha256:34663b2a16e7ac6ae8b77fef13e2b135e9fbc5ec13d2505d34bd35b3a41b9d82", + "sha256:8950aac5e5f4d505f7a0f067c5cb3893dcf098ff29cedfcb4ccf1e9e44d0bd9a" ], "index": "pypi", - "version": "==2.7.0" + "version": "==2.7.1" }, "pycodestyle": { "hashes": [ @@ -758,6 +834,14 @@ "markers": "python_version >= '3.6'", "version": "==0.18.0" }, + "pytest": { + "hashes": [ + "sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b", + "sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890" + ], + "markers": "python_version >= '3.6'", + "version": "==6.2.4" + }, "python-dateutil": { "hashes": [ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", @@ -944,6 +1028,14 @@ "index": "pypi", "version": "==6.6.1" }, + "sqlfluff": { + "hashes": [ + "sha256:3070181751a093d83fdb802b65d8163d765e5d5b6c01f2440d702899f7c51734", + "sha256:84eaa5257448eba36cbab06616de7a8e84c883ba03bfc984ea048e374299f64a" + ], + "index": "pypi", + "version": "==0.6.2" + }, "stopit": { "hashes": [ "sha256:f7f39c583fd92027bd9d06127b259aee7a5b7945c1f1fa56263811e1e766996d" @@ -957,6 +1049,14 @@ ], "version": "==0.8.9" }, + "tblib": { + "hashes": [ + "sha256:059bd77306ea7b419d4f76016aef6d7027cc8a0785579b5aad198803435f882c", + "sha256:289fa7359e580950e7d9743eab36b0691f0310fce64dee7d9c31065b8f723e23" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.7.0" + }, "tenacity": { "hashes": [ "sha256:43242a20e3e73291a28bcbcacfd6e000b02d3857a9a9fff56b297a27afdc932f", diff --git a/lib/functions/buildFileList.sh b/lib/functions/buildFileList.sh index c31e8a0f..b181a751 100755 --- a/lib/functions/buildFileList.sh +++ b/lib/functions/buildFileList.sh @@ -701,6 +701,7 @@ function BuildFileList() { # Append the file to the array # ################################ FILE_ARRAY_SQL+=("${FILE}") + FILE_ARRAY_SQLFLUFF+=("${FILE}") ########################### # Get the Terraform files # diff --git a/lib/linter.sh b/lib/linter.sh index 4a2d1d1d..9ca37937 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -204,8 +204,9 @@ LANGUAGE_ARRAY=('ANSIBLE' 'ARM' 'BASH' 'BASH_EXEC' 'CLANG_FORMAT' 'PHP_PHPSTAN' 'PHP_PSALM' 'POWERSHELL' 'PROTOBUF' 'PYTHON_BLACK' 'PYTHON_PYLINT' 'PYTHON_FLAKE8' 'PYTHON_ISORT' 'PYTHON_MYPY' 'R' 'RAKU' 'RUBY' 'RUST_2015' 'RUST_2018' 'RUST_CLIPPY' 'SHELL_SHFMT' 'SNAKEMAKE_LINT' - 'SNAKEMAKE_SNAKEFMT' 'STATES' 'SQL' 'TEKTON' 'TERRAFORM' 'TERRAFORM_TERRASCAN' - 'TERRAGRUNT' 'TSX' 'TYPESCRIPT_ES' 'TYPESCRIPT_STANDARD' 'XML' 'YAML') + 'SNAKEMAKE_SNAKEFMT' 'STATES' 'SQL' 'SQLFLUFF' 'TEKTON' 'TERRAFORM' + 'TERRAFORM_TERRASCAN' 'TERRAGRUNT' 'TSX' 'TYPESCRIPT_ES' 'TYPESCRIPT_STANDARD' + 'XML' 'YAML') ############################## # Linter command names array # @@ -268,6 +269,7 @@ LINTER_NAMES_ARRAY['SNAKEMAKE_LINT']="snakemake" LINTER_NAMES_ARRAY['SNAKEMAKE_SNAKEFMT']="snakefmt" LINTER_NAMES_ARRAY['STATES']="asl-validator" LINTER_NAMES_ARRAY['SQL']="sql-lint" +LINTER_NAMES_ARRAY['SQLFLUFF']="sqlfluff" LINTER_NAMES_ARRAY['TEKTON']="tekton-lint" LINTER_NAMES_ARRAY['TERRAFORM']="tflint" LINTER_NAMES_ARRAY['TERRAFORM_TERRASCAN']="terrascan" @@ -882,6 +884,7 @@ LINTER_COMMANDS_ARRAY['SNAKEMAKE_LINT']="snakemake --lint -s" LINTER_COMMANDS_ARRAY['SNAKEMAKE_SNAKEFMT']="snakefmt --config ${SNAKEMAKE_SNAKEFMT_LINTER_RULES} --check --compact-diff" LINTER_COMMANDS_ARRAY['STATES']="asl-validator --json-path" LINTER_COMMANDS_ARRAY['SQL']="sql-lint --config ${SQL_LINTER_RULES}" +LINTER_COMMANDS_ARRAY['SQLFLUFF']="sqlfluff lint" LINTER_COMMANDS_ARRAY['TEKTON']="tekton-lint" LINTER_COMMANDS_ARRAY['TERRAFORM']="tflint -c ${TERRAFORM_LINTER_RULES}" LINTER_COMMANDS_ARRAY['TERRAFORM_TERRASCAN']="terrascan scan -i terraform -t all -c ${TERRAFORM_TERRASCAN_LINTER_RULES} -f" diff --git a/test/inspec/super-linter/controls/super_linter.rb b/test/inspec/super-linter/controls/super_linter.rb index 5a24938a..b4586dfa 100644 --- a/test/inspec/super-linter/controls/super_linter.rb +++ b/test/inspec/super-linter/controls/super_linter.rb @@ -142,6 +142,7 @@ control "super-linter-installed-commands" do { linter_name: "snakemake"}, { linter_name: "spectral"}, { linter_name: "sql-lint"}, + { linter_name: "sqlfluff"}, { linter_name: "standard"}, { linter_name: "stylelint"}, { linter_name: "tekton-lint"}, @@ -260,7 +261,6 @@ control "super-linter-installed-pip-packages" do packages = [ "ansible-lint", - "black", "cfn-lint", "cpplint", "cython", @@ -270,6 +270,7 @@ control "super-linter-installed-pip-packages" do "pylint", "snakefmt", "snakemake", + "sqlfluff", "typing_extensions", "yamllint", "yq"