From 79286a45041274ee4f668e54e7748343ff0505f2 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 14:10:13 +0000 Subject: [PATCH 01/38] Fix: Add .gitattributes file to make development Windows-Friendly --- .gitattributes | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..5dc46e6b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +* text=auto eol=lf +*.{cmd,[cC][mM][dD]} text eol=crlf +*.{bat,[bB][aA][tT]} text eol=crlf \ No newline at end of file From 66983a8706d0a7e389b61ff45be51f89da383d78 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 14:10:28 +0000 Subject: [PATCH 02/38] Fix: Add devcontainer.json to allow vscode users to develop in container directly from repository --- .devcontainer/devcontainer.json | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..b470e0aa --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,34 @@ +// For format details, see https://aka.ms/vscode-remote/devcontainer.json or this file's README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.123.0/containers/docker-existing-dockerfile +{ + "name": "SUPER-LINTER", + + // Sets the run context to one level up instead of the .devcontainer folder. + "context": "..", + + // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. + "dockerFile": "..\\Dockerfile", + + // Set *default* container specific settings.json values on container create. + "settings": { + "terminal.integrated.shell.linux": null + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [] + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Uncomment the next line to run commands after the container is created - for example installing curl. + // "postCreateCommand": "apt-get update && apt-get install -y curl", + + // Uncomment when using a ptrace-based debugger like C++, Go, and Rust + // "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], + + // Uncomment to use the Docker CLI from inside the container. See https://aka.ms/vscode-remote/samples/docker-from-docker. + // "mounts": [ "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind" ], + + // Uncomment to connect as a non-root user. See https://aka.ms/vscode-remote/containers/non-root. + // "remoteUser": "vscode" +} From 7d95b48a05b60d7807d40090fae30a18e848f3de Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 15:09:52 +0000 Subject: [PATCH 03/38] Fix: Add proper shell and bash debugging extension --- .devcontainer/devcontainer.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b470e0aa..1d430c89 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -11,11 +11,13 @@ // Set *default* container specific settings.json values on container create. "settings": { - "terminal.integrated.shell.linux": null + "terminal.integrated.shell.linux": "/bin/ash" }, // Add the IDs of extensions you want installed when the container is created. - "extensions": [] + "extensions": [ + "rogalmic.bash-debug" + ] // Use 'forwardPorts' to make a list of ports inside the container available locally. // "forwardPorts": [], From a2cf5c643727fcefa88513fb90ffd8c2235e76f6 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 16:13:32 +0000 Subject: [PATCH 04/38] Feat: Add Launch.json for bash debugging --- .vscode/launch.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..744b2145 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,14 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "bashdb", + "request": "launch", + "name": "Bash-Debug (simplest configuration)", + "program": "${file}" + } + ] +} \ No newline at end of file From 9ce962cf6990997008a825274a55964dfa3f04f8 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 16:53:52 +0000 Subject: [PATCH 05/38] Feat: Automatically link automation test cases to /tmp/lint by default --- .devcontainer/devcontainer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 1d430c89..48a0b66a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -17,13 +17,13 @@ // Add the IDs of extensions you want installed when the container is created. "extensions": [ "rogalmic.bash-debug" - ] + ], // Use 'forwardPorts' to make a list of ports inside the container available locally. // "forwardPorts": [], // Uncomment the next line to run commands after the container is created - for example installing curl. - // "postCreateCommand": "apt-get update && apt-get install -y curl", + "postCreateCommand": "ln -s /workspaces/super-linter/.automation/test /tmp/lint", // Uncomment when using a ptrace-based debugger like C++, Go, and Rust // "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], From 8b47945eea851f9f1e55df5e8bfb65268923f098 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 16:54:23 +0000 Subject: [PATCH 06/38] Updated Launch.json --- .vscode/launch.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 744b2145..7e128dc6 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -7,8 +7,12 @@ { "type": "bashdb", "request": "launch", - "name": "Bash-Debug (simplest configuration)", - "program": "${file}" + "name": "Test Linter", + "program": ".vscode/testlinter.sh", + "internalConsoleOptions": "openOnSessionStart", + "presentation": { + "group": "aLinter" + } } ] } \ No newline at end of file From 7f23223ed5178db913b53a8c60e90069f933816a Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 16:54:36 +0000 Subject: [PATCH 07/38] Fix: Add testlinter file to force run_local --- .vscode/testlinter.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .vscode/testlinter.sh diff --git a/.vscode/testlinter.sh b/.vscode/testlinter.sh new file mode 100644 index 00000000..464c420e --- /dev/null +++ b/.vscode/testlinter.sh @@ -0,0 +1,3 @@ +#!/bin/bash +export RUN_LOCAL=true +/workspaces/super-linter/lib/linter.sh \ No newline at end of file From 90a1f525b04a6ffde616805c902e8f6a37217e38 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 17:31:53 +0000 Subject: [PATCH 08/38] Feat: Make GITHUB_WORKSPACE overridable in RUN_LOCAL Mode --- lib/linter.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/linter.sh b/lib/linter.sh index 473f53cd..6a81206f 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -579,6 +579,14 @@ GetGitHubVars() ########################################## echo "NOTE: ENV VAR [RUN_LOCAL] has been set to:[true]" echo "bypassing GitHub Actions variables..." + + ############################ + # Set the GITHUB_WORKSPACE # + ############################ + if [ -z "$GITHUB_WORKSPACE" ]; then + GITHUB_WORKSPACE="$DEFAULT_WORKSPACE" + fi + echo "Linting all files in mapped directory:[$DEFAULT_WORKSPACE]" # No need to touch or set the GITHUB_SHA @@ -586,10 +594,6 @@ GetGitHubVars() # No need to touch or set the GITHUB_ORG # No need to touch or set the GITHUB_REPO - ############################ - # Set the GITHUB_WORKSPACE # - ############################ - GITHUB_WORKSPACE="$DEFAULT_WORKSPACE" ################################# # Set the VALIDATE_ALL_CODEBASE # From 9bce2fecb9052370d918bed33b16ea40583a8803 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 17:32:09 +0000 Subject: [PATCH 09/38] Minor refinements --- .devcontainer/devcontainer.json | 2 +- .vscode/testlinter.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 48a0b66a..b3259b1f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -23,7 +23,7 @@ // "forwardPorts": [], // Uncomment the next line to run commands after the container is created - for example installing curl. - "postCreateCommand": "ln -s /workspaces/super-linter/.automation/test /tmp/lint", + //"postCreateCommand": "ln -s /workspaces/super-linter/.automation/test /tmp/lint", // Uncomment when using a ptrace-based debugger like C++, Go, and Rust // "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], diff --git a/.vscode/testlinter.sh b/.vscode/testlinter.sh index 464c420e..1bc56ad8 100644 --- a/.vscode/testlinter.sh +++ b/.vscode/testlinter.sh @@ -1,3 +1,5 @@ #!/bin/bash +#At this point you can use the debug console to add export GITHUB_WORKSPACE=/path to test only a specific folder +#You can also use ln -s /path /tmp/lint as an alternative export RUN_LOCAL=true /workspaces/super-linter/lib/linter.sh \ No newline at end of file From fdc003eb9101a6c1e3dd5d51169b0cbfacde567f Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 17:48:17 +0000 Subject: [PATCH 10/38] Feat: Add Github Pull Requests Extension --- .devcontainer/devcontainer.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b3259b1f..03dde16a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -16,6 +16,7 @@ // Add the IDs of extensions you want installed when the container is created. "extensions": [ + "github.vscode-pull-request-github", "rogalmic.bash-debug" ], From 5064b0c67aa4fb981fe4693d573ce8258ad75af4 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 17:48:39 +0000 Subject: [PATCH 11/38] Fix: Default Linter to automation tests --- .vscode/launch.json | 1 + .vscode/testlinter.sh | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 7e128dc6..a02e962c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,6 +9,7 @@ "request": "launch", "name": "Test Linter", "program": ".vscode/testlinter.sh", + "cwd": "${workspaceFolder}", "internalConsoleOptions": "openOnSessionStart", "presentation": { "group": "aLinter" diff --git a/.vscode/testlinter.sh b/.vscode/testlinter.sh index 1bc56ad8..40502987 100644 --- a/.vscode/testlinter.sh +++ b/.vscode/testlinter.sh @@ -1,5 +1,11 @@ #!/bin/bash #At this point you can use the debug console to add export GITHUB_WORKSPACE=/path to test only a specific folder #You can also use ln -s /path /tmp/lint as an alternative +#If you do neither, this will default to running against the test automation files +tmppath=/tmp/lint +if [ ! -L tmppath ]; then + ln -s $PWD/.automation/test $tmppath +fi + export RUN_LOCAL=true -/workspaces/super-linter/lib/linter.sh \ No newline at end of file +source $PWD/lib/linter.sh \ No newline at end of file From 211a22c329eaea54e13e9757e40c9d6bf1813262 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 12:03:30 -0700 Subject: [PATCH 12/38] Fix: Change ash to bash for default Terminal Co-authored-by: Antoine Leblanc --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 03dde16a..72190b10 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -11,7 +11,7 @@ // Set *default* container specific settings.json values on container create. "settings": { - "terminal.integrated.shell.linux": "/bin/ash" + "terminal.integrated.shell.linux": "/bin/bash" }, // Add the IDs of extensions you want installed when the container is created. From 524ebed9cde62baf1a2cfbcf3fba39c708878964 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 21:14:17 +0000 Subject: [PATCH 13/38] Fix tmppath detection --- .vscode/testlinter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/testlinter.sh b/.vscode/testlinter.sh index 40502987..f7e7eb79 100644 --- a/.vscode/testlinter.sh +++ b/.vscode/testlinter.sh @@ -3,7 +3,7 @@ #You can also use ln -s /path /tmp/lint as an alternative #If you do neither, this will default to running against the test automation files tmppath=/tmp/lint -if [ ! -L tmppath ]; then +if [ ! -L $tmppath ]; then ln -s $PWD/.automation/test $tmppath fi From 6705cf967602f96f859fce0cfc75d8de77f98c66 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Wed, 24 Jun 2020 14:19:24 -0700 Subject: [PATCH 14/38] Docs: Add note about vscode --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index f1dfb044..a768514a 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,13 @@ Below are a list of the known limitations for the **GitHub Super-Linter**: ## How to contribute If you would like to help contribute to this **GitHub** Action, please see [CONTRIBUTING](https://github.com/github/super-linter/blob/master/.github/CONTRIBUTING.md) +### Visual Studio Code +You can checkout this repository using [Container Remote Development](https://code.visualstudio.com/docs/remote/containers), and debug the linter using the `Test Linter` task. +![Example](https://user-images.githubusercontent.com/15258962/85165778-2d2ce700-b21b-11ea-803e-3f6709d8e609.gif) + +We will also support [Github Codespaces](https://github.com/features/codespaces/) once it becomes available + + -------------------------------------------------------------------------------- ### License From 172e29dad3e4d7768590e191201522ece5b384f9 Mon Sep 17 00:00:00 2001 From: Gabo Date: Thu, 25 Jun 2020 10:03:39 -0500 Subject: [PATCH 15/38] Fix bash lint errors --- .vscode/testlinter.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.vscode/testlinter.sh b/.vscode/testlinter.sh index f7e7eb79..c6a54f00 100644 --- a/.vscode/testlinter.sh +++ b/.vscode/testlinter.sh @@ -4,8 +4,9 @@ #If you do neither, this will default to running against the test automation files tmppath=/tmp/lint if [ ! -L $tmppath ]; then - ln -s $PWD/.automation/test $tmppath + ln -s "$PWD"/.automation/test $tmppath fi export RUN_LOCAL=true -source $PWD/lib/linter.sh \ No newline at end of file +# shellcheck source=lib/linter.sh +source "$PWD"/lib/linter.sh \ No newline at end of file From b8171a9bfbd7702465c8fe38657aec632d2f0c12 Mon Sep 17 00:00:00 2001 From: Gabo Date: Thu, 25 Jun 2020 10:04:02 -0500 Subject: [PATCH 16/38] Fix markdown lint errors --- README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 38a83bf8..fe4627be 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Super-Linter -This repository is for the **GitHub Action** to run a **Super-Linter**. +This repository is for the **GitHub Action** to run a **Super-Linter**. It is a simple combination of various linters, written in `bash`, to help validate your source code. The end goal of this tool: @@ -112,9 +112,9 @@ jobs: ## Environment variables The super-linter allows you to pass the following `ENV` variables to be able to trigger different functionality. -*Note:* All the `VALIDATE_[LANGUAGE]` variables behave in a specific way. -If none of them are passed, then they all default to true. -However if any one of the variables are set, we default to leaving any unset variable to false. +*Note:* All the `VALIDATE_[LANGUAGE]` variables behave in a specific way. +If none of them are passed, then they all default to true. +However if any one of the variables are set, we default to leaving any unset variable to false. This means that if you run the linter "out of the box", all languages will be checked. But if you wish to select specific linters, we give you full control to choose which linters are run, and won't run anything unexpected. @@ -186,12 +186,11 @@ Below are a list of the known limitations for the **GitHub Super-Linter**: If you would like to help contribute to this **GitHub** Action, please see [CONTRIBUTING](https://github.com/github/super-linter/blob/master/.github/CONTRIBUTING.md) ### Visual Studio Code -You can checkout this repository using [Container Remote Development](https://code.visualstudio.com/docs/remote/containers), and debug the linter using the `Test Linter` task. +You can checkout this repository using [Container Remote Development](https://code.visualstudio.com/docs/remote/containers), and debug the linter using the `Test Linter` task. ![Example](https://user-images.githubusercontent.com/15258962/85165778-2d2ce700-b21b-11ea-803e-3f6709d8e609.gif) We will also support [Github Codespaces](https://github.com/features/codespaces/) once it becomes available - -------------------------------------------------------------------------------- ### License From a51265c9f0925b7ee09a7d4c8baaba0836b611fd Mon Sep 17 00:00:00 2001 From: Gabo Date: Thu, 25 Jun 2020 10:25:56 -0500 Subject: [PATCH 17/38] Fix slashes --- .devcontainer/devcontainer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 72190b10..e8990ac0 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -7,10 +7,10 @@ "context": "..", // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. - "dockerFile": "..\\Dockerfile", + "dockerFile": "..//Dockerfile", // Set *default* container specific settings.json values on container create. - "settings": { + "settings": { "terminal.integrated.shell.linux": "/bin/bash" }, From 26f4d56449e0991a7c70201cadcb8daa03565255 Mon Sep 17 00:00:00 2001 From: Gabo Date: Fri, 26 Jun 2020 13:55:11 -0500 Subject: [PATCH 18/38] Move json comments to readme --- .devcontainer/README.md | 10 ++++++++++ .devcontainer/devcontainer.json | 27 +-------------------------- .vscode/launch.json | 3 --- 3 files changed, 11 insertions(+), 29 deletions(-) create mode 100644 .devcontainer/README.md diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 00000000..75057d9f --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,10 @@ +# Devcontainer +This file specifies to vscode how to run the container + +For format details, see [documentation](https://aka.ms/vscode-remote/devcontainer.json) or this file's [README](https://github.com/microsoft/vscode-dev-containers/tree/v0.123.0/containers/docker-existing-dockerfile) + + +context: Sets the run context to one level up instead of the .devcontainer folder. +dockerFile: Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. +settings: Set *default* container specific settings.json values on container create. +extensions: Add the IDs of extensions you want installed when the container is created. \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e8990ac0..ac0ac8e1 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,37 +1,12 @@ -// For format details, see https://aka.ms/vscode-remote/devcontainer.json or this file's README at: -// https://github.com/microsoft/vscode-dev-containers/tree/v0.123.0/containers/docker-existing-dockerfile { "name": "SUPER-LINTER", - - // Sets the run context to one level up instead of the .devcontainer folder. "context": "..", - - // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. "dockerFile": "..//Dockerfile", - - // Set *default* container specific settings.json values on container create. "settings": { "terminal.integrated.shell.linux": "/bin/bash" }, - - // Add the IDs of extensions you want installed when the container is created. "extensions": [ "github.vscode-pull-request-github", "rogalmic.bash-debug" - ], - - // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [], - - // Uncomment the next line to run commands after the container is created - for example installing curl. - //"postCreateCommand": "ln -s /workspaces/super-linter/.automation/test /tmp/lint", - - // Uncomment when using a ptrace-based debugger like C++, Go, and Rust - // "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], - - // Uncomment to use the Docker CLI from inside the container. See https://aka.ms/vscode-remote/samples/docker-from-docker. - // "mounts": [ "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind" ], - - // Uncomment to connect as a non-root user. See https://aka.ms/vscode-remote/containers/non-root. - // "remoteUser": "vscode" + ] } diff --git a/.vscode/launch.json b/.vscode/launch.json index a02e962c..77e1ced2 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,7 +1,4 @@ { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ { From d155ff2a3c4d2034e77ad66aa9bec5ab590d26ab Mon Sep 17 00:00:00 2001 From: Drew Ayling Date: Fri, 26 Jun 2020 16:16:22 -0600 Subject: [PATCH 19/38] Fix honoring pylint config file The pylint config file is correctly being applied to the pylint command, however, the `-E` flag is actually not honoring any of the disabled messages in the `.python-lint` file. By making this change to the default lint file only the errors are output as expected but honoring the disabled messages. Also, disable the score as the -E flag also does. --- .github/linters/.python-lint | 3 ++- TEMPLATES/.python-lint | 3 ++- lib/linter.sh | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/linters/.python-lint b/.github/linters/.python-lint index 8e9cc00e..e3d9f5b5 100644 --- a/.github/linters/.python-lint +++ b/.github/linters/.python-lint @@ -1,4 +1,5 @@ [MASTER] +errors-only= # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may @@ -157,7 +158,7 @@ output-format=text reports=no # Activate the evaluation score. -score=yes +score=no [REFACTORING] diff --git a/TEMPLATES/.python-lint b/TEMPLATES/.python-lint index 8e9cc00e..e3d9f5b5 100644 --- a/TEMPLATES/.python-lint +++ b/TEMPLATES/.python-lint @@ -1,4 +1,5 @@ [MASTER] +errors-only= # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may @@ -157,7 +158,7 @@ output-format=text reports=no # Activate the evaluation score. -score=yes +score=no [REFACTORING] diff --git a/lib/linter.sh b/lib/linter.sh index 6226455c..6cf8d46d 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -2300,7 +2300,7 @@ RunTestCases() TestCodebase "XML" "xmllint" "xmllint" ".*\.\(xml\)\$" TestCodebase "MARKDOWN" "markdownlint" "markdownlint -c $MD_LINTER_RULES" ".*\.\(md\)\$" TestCodebase "BASH" "shellcheck" "shellcheck" ".*\.\(sh\)\$" - TestCodebase "PYTHON" "pylint" "pylint --rcfile $PYTHON_LINTER_RULES -E" ".*\.\(py\)\$" + TestCodebase "PYTHON" "pylint" "pylint --rcfile $PYTHON_LINTER_RULES" ".*\.\(py\)\$" TestCodebase "PERL" "perl" "perl -Mstrict -cw" ".*\.\(pl\)\$" TestCodebase "PHP" "php" "php -l" ".*\.\(php\)\$" TestCodebase "RUBY" "rubocop" "rubocop -c $RUBY_LINTER_RULES" ".*\.\(rb\)\$" @@ -2472,7 +2472,7 @@ if [ "$VALIDATE_PYTHON" == "true" ]; then # Lint the python files # ######################### # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY" - LintCodebase "PYTHON" "pylint" "pylint --rcfile $PYTHON_LINTER_RULES -E" ".*\.\(py\)\$" "${FILE_ARRAY_PYTHON[@]}" + LintCodebase "PYTHON" "pylint" "pylint --rcfile $PYTHON_LINTER_RULES" ".*\.\(py\)\$" "${FILE_ARRAY_PYTHON[@]}" fi ################ From fb90d7952412d1d50a85d9f597b82a2c4f3f34ad Mon Sep 17 00:00:00 2001 From: yoheimuta Date: Sun, 28 Jun 2020 20:07:51 +0900 Subject: [PATCH 20/38] Add protolint support to super-linter --- .automation/test/protobuf/README.md | 19 +++++++ .../test/protobuf/protobuf_bad_1.proto | 48 ++++++++++++++++ .../test/protobuf/protobuf_good_1.proto | 18 ++++++ .github/linters/.protolintrc.yml | 7 +++ Dockerfile | 9 +++ README.md | 2 + TEMPLATES/.protolintrc.yml | 7 +++ docs/disabling-linters.md | 48 ++++++++++++++++ lib/linter.sh | 55 ++++++++++++++++++- 9 files changed, 211 insertions(+), 2 deletions(-) create mode 100644 .automation/test/protobuf/README.md create mode 100644 .automation/test/protobuf/protobuf_bad_1.proto create mode 100644 .automation/test/protobuf/protobuf_good_1.proto create mode 100644 .github/linters/.protolintrc.yml create mode 100644 TEMPLATES/.protolintrc.yml diff --git a/.automation/test/protobuf/README.md b/.automation/test/protobuf/README.md new file mode 100644 index 00000000..7048213f --- /dev/null +++ b/.automation/test/protobuf/README.md @@ -0,0 +1,19 @@ +# Protobuf Test Cases + +This folder holds the test cases for **Protobuf**. + +## Additional Docs + +No Additional information is needed for this test case. + +## Good Test Cases + +The test cases denoted: `LANGUAGE_good_FILE.EXTENSION` are all valid, and should pass successfully when linted. + +- **Note:** They are linted utilizing the default linter rules. + +## Bad Test Cases + +The test cases denoted: `LANGUAGE_bad_FILE.EXTENSION` are **NOT** valid, and should trigger errors when linted. + +- **Note:** They are linted utilizing the default linter rules. diff --git a/.automation/test/protobuf/protobuf_bad_1.proto b/.automation/test/protobuf/protobuf_bad_1.proto new file mode 100644 index 00000000..b42980ba --- /dev/null +++ b/.automation/test/protobuf/protobuf_bad_1.proto @@ -0,0 +1,48 @@ +syntax = "proto3"; +// A broken example of the official reference +// See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#proto_file +package examplePb; + +option java_package = "com.example.foo"; + +import "other.proto"; +import public "new.proto"; + +import "google/protobuf/empty.proto"; +import "google/protobuf/timestamp.proto"; + +import "myproject/other_protos.proto"; +import "myproject/main_protos.proto"; + +enum enumAllowingAlias { + option allow_alias = true; + UNKNOWN = 0; + STARTED = 1; + RUNNING = 2 [(custom_option) = "hello world"]; +} +message outer { + option (my_option).a = true; + // inner is an inner message. + message inner { // Level 2 + int64 ival = 1; + } + repeated inner inner_message = 2; + EnumAllowingAlias enum_field =3; + map my_map = 4; + string reason_for_error = 5; + string end_of_support_version= 6; + message AccountForAdmin {} + message SpecialEndOfSupport {} + required inner inner_message = 7; + group Result = 8 { + string url = 9; + } + repeated group Result = 10 { + } + repeated inner paper = 11; + repeated group Regular = 12 { + } +} +service SearchApi { + rpc search (SearchRequest) returns (SearchResponse) {}; +}; diff --git a/.automation/test/protobuf/protobuf_good_1.proto b/.automation/test/protobuf/protobuf_good_1.proto new file mode 100644 index 00000000..76022990 --- /dev/null +++ b/.automation/test/protobuf/protobuf_good_1.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; +import public "other.proto"; +option java_package = "com.example.foo"; +enum EnumAllowingAlias { + option allow_alias = true; + ALLOWING_UNSPECIFIED = 0; + STARTED = 1; + RUNNING = 2 [(custom_option) = "hello world"]; +} +message Outer { + option (my_option).a = true; + message Inner { // Level 2 + int64 ival = 1; + } + inner inner_message = 2; + EnumAllowingAlias enum_field =3; + map my_map = 4; +} diff --git a/.github/linters/.protolintrc.yml b/.github/linters/.protolintrc.yml new file mode 100644 index 00000000..7bd3e0ce --- /dev/null +++ b/.github/linters/.protolintrc.yml @@ -0,0 +1,7 @@ +# Lint directives. +lint: + # Linter rules. + # Run `protolint list` to see all available rules. + rules: + # Set the default to all linters. + all_default: false diff --git a/Dockerfile b/Dockerfile index d0c3a70b..46e6a686 100644 --- a/Dockerfile +++ b/Dockerfile @@ -126,6 +126,14 @@ RUN wget -O- -nvq https://raw.githubusercontent.com/golangci/golangci-lint/maste RUN curl -Ls "$(curl -Ls https://api.github.com/repos/terraform-linters/tflint/releases/latest | grep -o -E "https://.+?_linux_amd64.zip")" -o tflint.zip && unzip tflint.zip && rm tflint.zip \ && mv "tflint" /usr/bin/ +###################### +# Install protolint # +###################### +RUN curl -LsS "$(curl -Ls https://api.github.com/repos/yoheimuta/protolint/releases/latest | grep -o -E "https://.+?_Linux_x86_64.tar.gz")" -o protolint.tar.gz \ + && tar -xzf protolint.tar.gz \ + && rm protolint.tar.gz \ + && mv "protolint" /usr/bin/ + ######################### # Install dotenv-linter # ######################### @@ -180,6 +188,7 @@ ENV GITHUB_SHA=${GITHUB_SHA} \ VALIDATE_KOTLIN=${VALIDATE_KOTLIN} \ VALIDATE_POWERSHELL=${VALIDATE_POWERSHELL} \ VALIDATE_OPENAPI=${VALIDATE_OPENAPI} \ + VALIDATE_PROTOBUF=${VALIDATE_PROTOBUF} \ ANSIBLE_DIRECTORY=${ANSIBLE_DIRECTORY} \ RUN_LOCAL=${RUN_LOCAL} \ TEST_CASE_RUN=${TEST_CASE_RUN} \ diff --git a/README.md b/README.md index 1b9b7e09..ab9efb85 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ Developers on **GitHub** can call the **GitHub Action** to lint their code base | **ENV** | [dotenv-linter](https://github.com/dotenv-linter/dotenv-linter) | | **Kotlin** | [ktlint](https://github.com/pinterest/ktlint) | | **OpenAPI** | [spectral](https://github.com/stoplightio/spectral) | +| **Protocol Buffers** | [protolint](https://github.com/yoheimuta/protolint) | ## How to use To use this **GitHub** Action you will need to complete the following: @@ -167,6 +168,7 @@ and won't run anything unexpected. | **VALIDATE_CLOJURE** | `true` | Flag to enable or disable the linting process of the language. | | **VALIDATE_KOTLIN** | `true` | Flag to enable or disable the linting process of the language. | | **VALIDATE_OPENAPI** | `true` | Flag to enable or disable the linting process of the language. | +| **VALIDATE_PROTOBUF** | `true` | Flag to enable or disable the linting process of the language. | | **ANSIBLE_DIRECTORY** | `/ansible` | Flag to set the root directory for Ansible file location(s). | | **ACTIONS_RUNNER_DEBUG** | `false` | Flag to enable additional information about the linter, versions, and additional output. | | **DISABLE_ERRORS** | `false` | Flag to have the linter complete with exit code 0 even if errors were detected. | diff --git a/TEMPLATES/.protolintrc.yml b/TEMPLATES/.protolintrc.yml new file mode 100644 index 00000000..7bd3e0ce --- /dev/null +++ b/TEMPLATES/.protolintrc.yml @@ -0,0 +1,7 @@ +# Lint directives. +lint: + # Linter rules. + # Run `protolint list` to see all available rules. + rules: + # Set the default to all linters. + all_default: false diff --git a/docs/disabling-linters.md b/docs/disabling-linters.md index 408eba19..f08fd22b 100644 --- a/docs/disabling-linters.md +++ b/docs/disabling-linters.md @@ -25,6 +25,7 @@ Below are examples and documentation for each language and the various methods t - [ENV](#dotenv-linter) - [Kotlin](#kotlin) - [OpenAPI](#openapi) +- [Protocol Buffers](#protocol-buffers) @@ -632,6 +633,53 @@ import package.b.* -------------------------------------------------------------------------------- +## Protocol Buffers + +- [protolint](https://github.com/yoheimuta/protolint) + +### protolint Config file + +- `.github/linters/.protolintrc.yml` +- You can add, extend, and disable rules +- Documentation at [Rules](https://github.com/yoheimuta/protolint#rules) and [Configuring](https://github.com/yoheimuta/protolint#configuring) + +### protolint disable single line + +```protobuf +enum Foo { + // protolint:disable:next ENUM_FIELD_NAMES_UPPER_SNAKE_CASE + firstValue = 0; + second_value = 1; // protolint:disable:this ENUM_FIELD_NAMES_UPPER_SNAKE_CASE + THIRD_VALUE = 2; +} +``` + +### protolint disable code block + +```protobuf +// protolint:disable ENUM_FIELD_NAMES_UPPER_SNAKE_CASE +enum Foo { + firstValue = 0; + second_value = 1; + THIRD_VALUE = 2; +} +// protolint:enable ENUM_FIELD_NAMES_UPPER_SNAKE_CASE +``` + +### protolint disable entire file + +- You can disable entire files with the `lint.files.exclude` property in `.protolintrc.yml` + +```yaml +# Lint directives. +lint: + # Linter files to walk. + files: + # The specific files to exclude. + exclude: + - path/to/file +``` + ## Clojure - [clj-kondo](https://github.com/borkdude/clj-kondo) - Since clj-kondo approaches static analysis in a very Clojure way, it is advised to read the [configuration docs](https://github.com/borkdude/clj-kondo/blob/master/doc/config.md) diff --git a/lib/linter.sh b/lib/linter.sh index 6226455c..69523ee4 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -57,6 +57,9 @@ CSS_LINTER_RULES="$DEFAULT_RULES_LOCATION/$CSS_FILE_NAME" # Path to th # OpenAPI Vars OPENAPI_FILE_NAME='.openapirc.yml' # Name of the file OPENAPI_LINTER_RULES="$DEFAULT_RULES_LOCATION/$OPENAPI_FILE_NAME" # Path to the OpenAPI lint rules +# Protocol Buffers Vars +PROTOBUF_FILE_NAME='.protolintrc.yml' # Name of the file +PROTOBUF_LINTER_RULES="$DEFAULT_RULES_LOCATION/$PROTOBUF_FILE_NAME" # Path to the Protocol Buffers lint rules # Clojure Vars CLOJURE_FILE_NAME='.clj-kondo/config.edn' CLOJURE_LINTER_RULES="$DEFAULT_RULES_LOCATION/$CLOJURE_FILE_NAME" @@ -67,7 +70,7 @@ CLOJURE_LINTER_RULES="$DEFAULT_RULES_LOCATION/$CLOJURE_FILE_NAME" LINTER_ARRAY=("jsonlint" "yamllint" "xmllint" "markdownlint" "shellcheck" "pylint" "perl" "rubocop" "coffeelint" "eslint" "standard" "ansible-lint" "/dockerfilelint/bin/dockerfilelint" "golangci-lint" "tflint" - "stylelint" "dotenv-linter" "powershell" "ktlint" "clj-kondo" "spectral") + "stylelint" "dotenv-linter" "powershell" "ktlint" "protolint" "clj-kondo" "spectral") ############################# # Language array for prints # @@ -75,7 +78,7 @@ LINTER_ARRAY=("jsonlint" "yamllint" "xmllint" "markdownlint" "shellcheck" LANGUAGE_ARRAY=('YML' 'JSON' 'XML' 'MARKDOWN' 'BASH' 'PERL' 'PHP' 'RUBY' 'PYTHON' 'COFFEESCRIPT' 'ANSIBLE' 'JAVASCRIPT_STANDARD' 'JAVASCRIPT_ES' 'TYPESCRIPT_STANDARD' 'TYPESCRIPT_ES' 'DOCKER' 'GO' 'TERRAFORM' - 'CSS' 'ENV' 'POWERSHELL' 'KOTLIN' 'CLOJURE' 'OPENAPI') + 'CSS' 'ENV' 'POWERSHELL' 'KOTLIN' 'PROTO' 'CLOJURE' 'OPENAPI') ################### # GitHub ENV Vars # @@ -159,6 +162,7 @@ FILE_ARRAY_CSS=() # Array of files to check FILE_ARRAY_ENV=() # Array of files to check FILE_ARRAY_CLOJURE=() # Array of files to check FILE_ARRAY_KOTLIN=() # Array of files to check +FILE_ARRAY_PROTOBUF=() # Array of files to check FILE_ARRAY_OPENAPI=() # Array of files to check ############ @@ -187,6 +191,7 @@ ERRORS_FOUND_CSS=0 # Count of errors found ERRORS_FOUND_ENV=0 # Count of errors found ERRORS_FOUND_CLOJURE=0 # Count of errors found ERRORS_FOUND_KOTLIN=0 # Count of errors found +ERRORS_FOUND_PROTOBUF=0 # Count of errors found ERRORS_FOUND_OPENAPI=0 # Count of errors found ################################################################################ @@ -806,6 +811,7 @@ GetValidationInfo() VALIDATE_ENV=$(echo "$VALIDATE_ENV" | awk '{print tolower($0)}') VALIDATE_CLOJURE=$(echo "$VALIDATE_CLOJURE" | awk '{print tolower($0)') VALIDATE_KOTLIN=$(echo "$VALIDATE_KOTLIN" | awk '{print tolower($0)}') + VALIDATE_PROTOBUF=$(echo "$VALIDATE_PROTOBUF" | awk '{print tolower($0)}') VALIDATE_OPENAPI=$(echo "$VALIDATE_OPENAPI" | awk '{print tolower($0)}') ################################################ @@ -834,6 +840,7 @@ GetValidationInfo() -n "$VALIDATE_CSS" || \ -n "$VALIDATE_ENV" || \ -n "$VALIDATE_CLOJURE" || \ + -n "$VALIDATE_PROTOBUF" || \ -n "$VALIDATE_OPENAPI" || \ -n "$VALIDATE_KOTLIN" ]]; then ANY_SET="true" @@ -1161,6 +1168,20 @@ GetValidationInfo() VALIDATE_OPENAPI="true" fi + ####################################### + # Validate if we should check PROTOBUF # + ####################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_PROTOBUF" ]]; then + # PROTOBUF flag was not set - default to false + VALIDATE_PROTOBUF="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_PROTOBUF="true" + fi + ####################################### # Validate if we should check Clojure # ####################################### @@ -1298,6 +1319,11 @@ GetValidationInfo() else PRINT_ARRAY+=("- Excluding [OPENAPI] files in code base...") fi + if [[ "$VALIDATE_PROTOBUF" == "true" ]]; then + PRINT_ARRAY+=("- Validating [PROTOBUF] files in code base...") + else + PRINT_ARRAY+=("- Excluding [PROTOBUF] files in code base...") + fi ############################## # Validate Ansible Directory # @@ -1686,6 +1712,18 @@ BuildFileList() # Set the READ_ONLY_CHANGE_FLAG since this could be exec # ########################################################## READ_ONLY_CHANGE_FLAG=1 + ############################ + # Get the Protocol Buffers files # + ############################ + elif [ "$FILE_TYPE" == "proto" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_PROTOBUF+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 elif [ "$FILE" == "Dockerfile" ]; then ################################ # Append the file to the array # @@ -2254,6 +2292,7 @@ Footer() [ "$ERRORS_FOUND_CSS" -ne 0 ] || \ [ "$ERRORS_FOUND_ENV" -ne 0 ] || \ [ "$ERRORS_FOUND_OPENAPI" -ne 0 ] || \ + [ "$ERRORS_FOUND_PROTOBUF" -ne 0 ] || \ [ "$ERRORS_FOUND_CLOJURE" -ne 0 ] || \ [ "$ERRORS_FOUND_KOTLIN" -ne 0 ]; then # Failed exit @@ -2318,6 +2357,7 @@ RunTestCases() TestCodebase "ENV" "dotenv-linter" "dotenv-linter" ".*\.\(env\)\$" TestCodebase "CLOJURE" "clj-kondo" "clj-kondo --config $CLOJURE_LINTER_RULES --lint" ".*\.\(clj\|cljs\|cljc\|edn\)\$" TestCodebase "KOTLIN" "ktlint" "ktlint" ".*\.\(kt\|kts\)\$" + TestCodebase "PROTOBUF" "protolint" "protolint lint --config_path $PROTOBUF_LINTER_RULES" ".*\.\(proto\)\$" TestCodebase "OPENAPI" "spectral" "spectral lint -r $OPENAPI_LINTER_RULES" ".*\.\(ymlopenapi\|jsonopenapi\)\$" ################# @@ -2664,6 +2704,17 @@ if [ "$VALIDATE_CLOJURE" == "true" ]; then LintCodebase "CLOJURE" "clj-kondo" "clj-kondo --config $CLOJURE_LINTER_RULES --lint" ".*\.\(clj\|cljs\|cljc\|edn\)\$" "${FILE_ARRAY_CLOJURE[@]}" fi +################## +# PROTOBUF LINTING # +################## +if [ "$VALIDATE_PROTOBUF" == "true" ]; then + ####################### + # Lint the Protocol Buffers files # + ####################### + # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY" + LintCodebase "PROTOBUF" "protolint" "protolint lint --config_path $PROTOBUF_LINTER_RULES" ".*\.\(proto\)\$" "${FILE_ARRAY_PROTOBUF[@]}" +fi + ###################### # POWERSHELL LINTING # ###################### From 0482ce0b8ff5e4d8d05fef4035ce8641cbf4d72d Mon Sep 17 00:00:00 2001 From: Thomas Hughes Date: Sun, 28 Jun 2020 07:39:31 -0500 Subject: [PATCH 21/38] Delete welcome_new_people.yml --- .github/workflows/welcome_new_people.yml | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 .github/workflows/welcome_new_people.yml diff --git a/.github/workflows/welcome_new_people.yml b/.github/workflows/welcome_new_people.yml deleted file mode 100644 index 985ee57e..00000000 --- a/.github/workflows/welcome_new_people.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Greetings - -on: [pull_request, issues] - -jobs: - greeting: - runs-on: ubuntu-latest - steps: - - uses: actions/first-interaction@v1 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - issue-message: 'Thanks for opening your first issue in the **super-linter** repo! :tada:' - pr-message: 'Thanks for opening your first pr in the **super-linter** repo! :tada: This project is built on contributions just like this.' From a4fa97234ad9dfa3a749d7bdddd329ed17d458a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Knudsen?= Date: Mon, 29 Jun 2020 09:07:40 +0200 Subject: [PATCH 22/38] Read Clojure filetypes from $FILE_TYPE, not $FILE --- lib/linter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/linter.sh b/lib/linter.sh index 6226455c..d7e025a9 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -1695,7 +1695,7 @@ BuildFileList() # Set the READ_ONLY_CHANGE_FLAG since this could be exec # ########################################################## READ_ONLY_CHANGE_FLAG=1 - elif [ "$FILE" == "clj" ] || [ "$FILE" == "cljs" ] || [ "$FILE" == "cljc" ] || [ "$FILE" == "edn" ]; then + elif [ "$FILE_TYPE" == "clj" ] || [ "$FILE_TYPE" == "cljs" ] || [ "$FILE_TYPE" == "cljc" ] || [ "$FILE_TYPE" == "edn" ]; then ################################ # Append the file to the array # ################################ From 3975731d07ad4e849c08aa2c8bbf8d1fbb2f56cf Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 09:08:04 -0500 Subject: [PATCH 23/38] cleanup --- .vscode/testlinter.sh | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/.vscode/testlinter.sh b/.vscode/testlinter.sh index c6a54f00..e27c6ac2 100644 --- a/.vscode/testlinter.sh +++ b/.vscode/testlinter.sh @@ -1,12 +1,26 @@ -#!/bin/bash -#At this point you can use the debug console to add export GITHUB_WORKSPACE=/path to test only a specific folder -#You can also use ln -s /path /tmp/lint as an alternative -#If you do neither, this will default to running against the test automation files -tmppath=/tmp/lint -if [ ! -L $tmppath ]; then - ln -s "$PWD"/.automation/test $tmppath +#!/usr/bin/env bash + +# At this point you can use the debug console to add export GITHUB_WORKSPACE=/path to test only a specific folder +# You can also use ln -s /path /tmp/lint as an alternative +# If you do neither, this will default to running against the test automation files + +########### +# GLOBALS # +########### +CODE_PATH='/tmp/lint' # Path to code base + +################## +# Check the path # +################## +if [ ! -L $CODE_PATH ]; then + # Create symbolic link + ln -s "$PWD"/.automation/test $CODE_PATH fi +######################### +# Export to run locally # +######################### export RUN_LOCAL=true -# shellcheck source=lib/linter.sh -source "$PWD"/lib/linter.sh \ No newline at end of file + +# # shellcheck disable=SC1091 +source "$PWD"/lib/linter.sh From a578c20faa765916e7a72f2325efb456df398d07 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 09:13:04 -0500 Subject: [PATCH 24/38] fixing linkage --- .vscode/testlinter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/testlinter.sh b/.vscode/testlinter.sh index e27c6ac2..dc45a319 100644 --- a/.vscode/testlinter.sh +++ b/.vscode/testlinter.sh @@ -22,5 +22,5 @@ fi ######################### export RUN_LOCAL=true -# # shellcheck disable=SC1091 +# # shellcheck disable=SC1091,SC1090 source "$PWD"/lib/linter.sh From 529df42adb2aa1c10d8dfef320ac7e507ce5a1fe Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 09:16:17 -0500 Subject: [PATCH 25/38] fixed double pound --- .vscode/testlinter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/testlinter.sh b/.vscode/testlinter.sh index dc45a319..8d778d16 100644 --- a/.vscode/testlinter.sh +++ b/.vscode/testlinter.sh @@ -22,5 +22,5 @@ fi ######################### export RUN_LOCAL=true -# # shellcheck disable=SC1091,SC1090 +# shellcheck disable=SC1091,SC1090 source "$PWD"/lib/linter.sh From 701abcc065b486a089028429514e466052c6fc24 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 09:55:59 -0500 Subject: [PATCH 26/38] creating files --- lib/buildFileList.sh | 420 ++++++++++ lib/linter.sh | 1729 +----------------------------------------- lib/validation.sh | 669 ++++++++++++++++ lib/worker.sh | 658 ++++++++++++++++ 4 files changed, 1754 insertions(+), 1722 deletions(-) create mode 100755 lib/buildFileList.sh create mode 100755 lib/validation.sh create mode 100755 lib/worker.sh diff --git a/lib/buildFileList.sh b/lib/buildFileList.sh new file mode 100755 index 00000000..2ed2ce60 --- /dev/null +++ b/lib/buildFileList.sh @@ -0,0 +1,420 @@ +#!/usr/bin/env bash + +################################################################################ +################################################################################ +########### Super-Linter Build File List Functions @admiralawkbar ############## +################################################################################ +################################################################################ +########################## FUNCTION CALLS BELOW ################################ +################################################################################ +################################################################################ +#### Function BuildFileList #################################################### +function BuildFileList() +{ + # Need to build a list of all files changed + # This can be pulled from the GITHUB_EVENT_PATH payload + + ################ + # print header # + ################ + if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then + echo "" + echo "----------------------------------------------" + echo "Pulling in code history and branches..." + fi + + ################################################################################# + # Switch codebase back to the default branch to get a list of all files changed # + ################################################################################# + SWITCH_CMD=$(cd "$GITHUB_WORKSPACE" || exit; git pull --quiet; git checkout "$DEFAULT_BRANCH" 2>&1) + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + # Error + echo "Failed to switch to $DEFAULT_BRANCH branch to get files changed!" + echo "ERROR:[$SWITCH_CMD]" + exit 1 + fi + + ################ + # print header # + ################ + if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then + echo "" + echo "----------------------------------------------" + echo "Generating Diff with:[git diff --name-only '$DEFAULT_BRANCH..$GITHUB_SHA' --diff-filter=d]" + fi + + ################################################# + # Get the Array of files changed in the commits # + ################################################# + # shellcheck disable=SC2207 + RAW_FILE_ARRAY=($(cd "$GITHUB_WORKSPACE" || exit; git diff --name-only "$DEFAULT_BRANCH..$GITHUB_SHA" --diff-filter=d 2>&1)) + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + # Error + echo "ERROR! Failed to gain a list of all files changed!" + echo "ERROR:[${RAW_FILE_ARRAY[*]}]" + exit 1 + fi + + ################################################ + # Iterate through the array of all files found # + ################################################ + echo "" + echo "----------------------------------------------" + echo "Files that have been modified in the commit(s):" + for FILE in "${RAW_FILE_ARRAY[@]}" + do + ############## + # Print file # + ############## + echo "File:[$FILE]" + + ########################### + # Get the files extension # + ########################### + # Extract just the file and extension, reverse it, cut off extension, + # reverse it back, substitute to lowercase + FILE_TYPE=$(basename "$FILE" | rev | cut -f1 -d'.' | rev | awk '{print tolower($0)}') + + ######### + # DEBUG # + ######### + #echo "FILE_TYPE:[$FILE_TYPE]" + + ##################### + # Get the YML files # + ##################### + if [ "$FILE_TYPE" == "yml" ] || [ "$FILE_TYPE" == "yaml" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_YML+=("$FILE") + ############################ + # Check if file is OpenAPI # + ############################ + if DetectOpenAPIFile "$FILE"; then + FILE_ARRAY_OPENAPI+=("$FILE") + fi + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ###################### + # Get the JSON files # + ###################### + elif [ "$FILE_TYPE" == "json" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_JSON+=("$FILE") + ############################ + # Check if file is OpenAPI # + ############################ + if DetectOpenAPIFile "$FILE"; then + FILE_ARRAY_OPENAPI+=("$FILE") + fi + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ##################### + # Get the XML files # + ##################### + elif [ "$FILE_TYPE" == "xml" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_XML+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ########################## + # Get the MARKDOWN files # + ########################## + elif [ "$FILE_TYPE" == "md" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_MD+=("$FILE") + ###################### + # Get the BASH files # + ###################### + elif [ "$FILE_TYPE" == "sh" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_BASH+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ###################### + # Get the PERL files # + ###################### + elif [ "$FILE_TYPE" == "pl" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_PERL+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ###################### + # Get the PHP files # + ###################### + elif [ "$FILE_TYPE" == "php" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_PHP+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ###################### + # Get the RUBY files # + ###################### + elif [ "$FILE_TYPE" == "rb" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_RUBY+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ######################## + # Get the PYTHON files # + ######################## + elif [ "$FILE_TYPE" == "py" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_PYTHON+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ######################## + # Get the COFFEE files # + ######################## + elif [ "$FILE_TYPE" == "coffee" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_COFFEESCRIPT+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ############################ + # Get the JavaScript files # + ############################ + elif [ "$FILE_TYPE" == "js" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_JAVASCRIPT_ES+=("$FILE") + FILE_ARRAY_JAVASCRIPT_STANDARD+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ############################ + # Get the TypeScript files # + ############################ + elif [ "$FILE_TYPE" == "ts" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_TYPESCRIPT_ES+=("$FILE") + FILE_ARRAY_TYPESCRIPT_STANDARD+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ######################## + # Get the Golang files # + ######################## + elif [ "$FILE_TYPE" == "go" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_GO+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ########################### + # Get the Terraform files # + ########################### + elif [ "$FILE_TYPE" == "tf" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_TERRAFORM+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ########################### + # Get the Powershell files # + ########################### + elif [ "$FILE_TYPE" == "ps1" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_POWERSHELL+=("$FILE") + elif [ "$FILE_TYPE" == "css" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_CSS+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + elif [ "$FILE_TYPE" == "env" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_ENV+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + elif [ "$FILE_TYPE" == "kt" ] || [ "$FILE_TYPE" == "kts" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_KOTLIN+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ############################ + # Get the Protocol Buffers files # + ############################ + elif [ "$FILE_TYPE" == "proto" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_PROTOBUF+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + elif [ "$FILE" == "Dockerfile" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_DOCKER+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + elif [ "$FILE_TYPE" == "clj" ] || [ "$FILE_TYPE" == "cljs" ] || [ "$FILE_TYPE" == "cljc" ] || [ "$FILE_TYPE" == "edn" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_CLOJURE+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + else + ############################################## + # Use file to see if we can parse what it is # + ############################################## + GET_FILE_TYPE_CMD=$(file "$FILE" 2>&1) + + ################# + # Check if bash # + ################# + if [[ "$GET_FILE_TYPE_CMD" == *"Bourne-Again shell script"* ]]; then + ####################### + # It is a bash script # + ####################### + echo "WARN! Found bash script without extension:[.sh]" + echo "Please update file with proper extensions." + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_BASH+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + elif [[ "$GET_FILE_TYPE_CMD" == *"Ruby script"* ]]; then + ####################### + # It is a Ruby script # + ####################### + echo "WARN! Found ruby script without extension:[.rb]" + echo "Please update file with proper extensions." + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_RUBY+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + else + ############################ + # Extension was not found! # + ############################ + echo " - WARN! Failed to get filetype for:[$FILE]!" + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + fi + fi + done + + ######################################### + # Need to switch back to branch of code # + ######################################### + SWITCH2_CMD=$(cd "$GITHUB_WORKSPACE" || exit; git checkout --progress --force "$GITHUB_SHA" 2>&1) + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + # Error + echo "Failed to switch back to branch!" + echo "ERROR:[$SWITCH2_CMD]" + exit 1 + fi + + ################ + # Footer print # + ################ + echo "" + echo "----------------------------------------------" + echo "Successfully gathered list of files..." +} diff --git a/lib/linter.sh b/lib/linter.sh index 8b3bc141..f1dad6ec 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -7,6 +7,12 @@ ################################################################################ ################################################################################ +######################### +# Source Function Files # +######################### +source ./validation.sh # Source the main validation function script +source ./buildFileList.sh # Source the main build file list function script + ########### # GLOBALS # ########### @@ -393,188 +399,6 @@ GetStandardRules() TYPESCRIPT_STANDARD_LINTER_RULES="$(echo -e "${ENV_STRING}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" fi } -################################################################################ -#### Function LintAnsibleFiles ################################################# -LintAnsibleFiles() -{ - ###################### - # Create Print Array # - ###################### - PRINT_ARRAY=() - - ################ - # print header # - ################ - PRINT_ARRAY+=("") - PRINT_ARRAY+=("----------------------------------------------") - PRINT_ARRAY+=("----------------------------------------------") - PRINT_ARRAY+=("Linting [Ansible] files...") - PRINT_ARRAY+=("----------------------------------------------") - PRINT_ARRAY+=("----------------------------------------------") - - ###################### - # Name of the linter # - ###################### - LINTER_NAME="ansible-lint" - - ########################################### - # Validate we have ansible-lint installed # - ########################################### - # shellcheck disable=SC2230 - VALIDATE_INSTALL_CMD=$(command -v "$LINTER_NAME" 2>&1) - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ $ERROR_CODE -ne 0 ]; then - # Failed - echo "ERROR! Failed to find $LINTER_NAME in system!" - echo "ERROR:[$VALIDATE_INSTALL_CMD]" - exit 1 - else - # Success - if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then - # Success - echo "Successfully found binary in system" - echo "Location:[$VALIDATE_INSTALL_CMD]" - fi - fi - - ########################## - # Initialize empty Array # - ########################## - LIST_FILES=() - - ####################### - # Create flag to skip # - ####################### - SKIP_FLAG=0 - - ###################################################### - # Only go into ansible linter if we have base folder # - ###################################################### - if [ -d "$ANSIBLE_DIRECTORY" ]; then - - ############################################################ - # Check to see if we need to go through array or all files # - ############################################################ - if [ "$VALIDATE_ALL_CODEBASE" == "false" ]; then - # We need to only check the ansible playbooks that have updates - #LIST_FILES=("${ANSIBLE_ARRAY[@]}") - # shellcheck disable=SC2164,SC2010,SC2207 - LIST_FILES=($(cd "$ANSIBLE_DIRECTORY"; ls | grep ".yml" 2>&1)) - else - ################################# - # Get list of all files to lint # - ################################# - # shellcheck disable=SC2164,SC2010,SC2207 - LIST_FILES=($(cd "$ANSIBLE_DIRECTORY"; ls | grep ".yml" 2>&1)) - fi - - ############################################################### - # Set the list to empty if only MD and TXT files were changed # - ############################################################### - # No need to run the full ansible checks on read only file changes - if [ "$READ_ONLY_CHANGE_FLAG" -eq 0 ]; then - ########################## - # Set the array to empty # - ########################## - LIST_FILES=() - ################################### - # Send message that were skipping # - ################################### - #echo "- Skipping Ansible lint run as file(s) that were modified were read only..." - ############################ - # Create flag to skip loop # - ############################ - SKIP_FLAG=1 - fi - - #################################### - # Check if we have data to look at # - #################################### - if [ $SKIP_FLAG -eq 0 ]; then - for LINE in "${PRINT_ARRAY[@]}" - do - ######################### - # Print the header line # - ######################### - echo "$LINE" - done - fi - - ################## - # Lint the files # - ################## - for FILE in "${LIST_FILES[@]}" - do - - ######################################## - # Make sure we dont lint certain files # - ######################################## - if [[ $FILE == *"vault.yml"* ]] || [[ $FILE == *"galaxy.yml"* ]]; then - # This is a file we dont look at - continue - fi - - #################### - # Get the filename # - #################### - FILE_NAME=$(basename "$ANSIBLE_DIRECTORY/$FILE" 2>&1) - - ############## - # File print # - ############## - echo "---------------------------" - echo "File:[$FILE]" - - ################################ - # Lint the file with the rules # - ################################ - LINT_CMD=$("$LINTER_NAME" -v -c "$ANSIBLE_LINTER_RULES" "$ANSIBLE_DIRECTORY/$FILE" 2>&1) - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ $ERROR_CODE -ne 0 ]; then - ######### - # Error # - ######### - echo "ERROR! Found errors in [$LINTER_NAME] linter!" - echo "ERROR:[$LINT_CMD]" - # Increment error count - ((ERRORS_FOUND_ANSIBLE++)) - else - ########### - # Success # - ########### - echo " - File:[$FILE_NAME] was linted with [$LINTER_NAME] successfully" - fi - done - else # No ansible directory found in path - ############################### - # Check to see if debug is on # - ############################### - if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then - ######################## - # No Ansible dir found # - ######################## - echo "WARN! No Ansible base directory found at:[$ANSIBLE_DIRECTORY]" - echo "skipping ansible lint" - fi - fi -} - ################################################################################ #### Function DetectOpenAPIFile ################################################ DetectOpenAPIFile() @@ -666,7 +490,7 @@ GetGitHubVars() if [ -z "$GITHUB_WORKSPACE" ]; then GITHUB_WORKSPACE="$DEFAULT_WORKSPACE" fi - + echo "Linting all files in mapped directory:[$DEFAULT_WORKSPACE]" # No need to touch or set the GITHUB_SHA @@ -753,1488 +577,6 @@ GetGitHubVars() fi } ################################################################################ -#### Function GetValidationInfo ################################################ -GetValidationInfo() -{ - ############################################ - # Print headers for user provided env vars # - ############################################ - echo "" - echo "--------------------------------------------" - echo "Gathering user validation information..." - - ########################################### - # Skip validation if were running locally # - ########################################### - if [[ "$RUN_LOCAL" != "true" ]]; then - ############################### - # Convert string to lowercase # - ############################### - VALIDATE_ALL_CODEBASE=$(echo "$VALIDATE_ALL_CODEBASE" | awk '{print tolower($0)}') - ###################################### - # Validate we should check all files # - ###################################### - if [[ "$VALIDATE_ALL_CODEBASE" != "false" ]]; then - # Set to true - VALIDATE_ALL_CODEBASE="$DEFAULT_VALIDATE_ALL_CODEBASE" - echo "- Validating ALL files in code base..." - else - # Its false - echo "- Only validating [new], or [edited] files in code base..." - fi - fi - - ###################### - # Create Print Array # - ###################### - PRINT_ARRAY=() - - ################################ - # Convert strings to lowercase # - ################################ - VALIDATE_YAML=$(echo "$VALIDATE_YAML" | awk '{print tolower($0)}') - VALIDATE_JSON=$(echo "$VALIDATE_JSON" | awk '{print tolower($0)}') - VALIDATE_XML=$(echo "$VALIDATE_XML" | awk '{print tolower($0)}') - VALIDATE_MD=$(echo "$VALIDATE_MD" | awk '{print tolower($0)}') - VALIDATE_BASH=$(echo "$VALIDATE_BASH" | awk '{print tolower($0)}') - VALIDATE_PERL=$(echo "$VALIDATE_PERL" | awk '{print tolower($0)}') - VALIDATE_PHP=$(echo "$VALIDATE_PHP" | awk '{print tolower($0)}') - VALIDATE_PYTHON=$(echo "$VALIDATE_PYTHON" | awk '{print tolower($0)}') - VALIDATE_RUBY=$(echo "$VALIDATE_RUBY" | awk '{print tolower($0)}') - VALIDATE_COFFEE=$(echo "$VALIDATE_COFFEE" | awk '{print tolower($0)}') - VALIDATE_ANSIBLE=$(echo "$VALIDATE_ANSIBLE" | awk '{print tolower($0)}') - VALIDATE_JAVASCRIPT_ES=$(echo "$VALIDATE_JAVASCRIPT_ES" | awk '{print tolower($0)}') - VALIDATE_JAVASCRIPT_STANDARD=$(echo "$VALIDATE_JAVASCRIPT_STANDARD" | awk '{print tolower($0)}') - VALIDATE_TYPESCRIPT_ES=$(echo "$VALIDATE_TYPESCRIPT_ES" | awk '{print tolower($0)}') - VALIDATE_TYPESCRIPT_STANDARD=$(echo "$VALIDATE_TYPESCRIPT_STANDARD" | awk '{print tolower($0)}') - VALIDATE_DOCKER=$(echo "$VALIDATE_DOCKER" | awk '{print tolower($0)}') - VALIDATE_GO=$(echo "$VALIDATE_GO" | awk '{print tolower($0)}') - VALIDATE_TERRAFORM=$(echo "$VALIDATE_TERRAFORM" | awk '{print tolower($0)}') - VALIDATE_POWERSHELL=$(echo "$VALIDATE_POWERSHELL" | awk '{print tolower($0)}') - VALIDATE_CSS=$(echo "$VALIDATE_CSS" | awk '{print tolower($0)}') - VALIDATE_ENV=$(echo "$VALIDATE_ENV" | awk '{print tolower($0)}') - VALIDATE_CLOJURE=$(echo "$VALIDATE_CLOJURE" | awk '{print tolower($0)') - VALIDATE_KOTLIN=$(echo "$VALIDATE_KOTLIN" | awk '{print tolower($0)}') - VALIDATE_PROTOBUF=$(echo "$VALIDATE_PROTOBUF" | awk '{print tolower($0)}') - VALIDATE_OPENAPI=$(echo "$VALIDATE_OPENAPI" | awk '{print tolower($0)}') - - ################################################ - # Determine if any linters were explicitly set # - ################################################ - ANY_SET="false" - if [[ -n "$VALIDATE_YAML" || \ - -n "$VALIDATE_JSON" || \ - -n "$VALIDATE_XML" || \ - -n "$VALIDATE_MD" || \ - -n "$VALIDATE_BASH" || \ - -n "$VALIDATE_PERL" || \ - -n "$VALIDATE_PHP" || \ - -n "$VALIDATE_PYTHON" || \ - -n "$VALIDATE_RUBY" || \ - -n "$VALIDATE_COFFEE" || \ - -n "$VALIDATE_ANSIBLE" || \ - -n "$VALIDATE_JAVASCRIPT_ES" || \ - -n "$VALIDATE_JAVASCRIPT_STANDARD" || \ - -n "$VALIDATE_TYPESCRIPT_ES" || \ - -n "$VALIDATE_TYPESCRIPT_STANDARD" || \ - -n "$VALIDATE_DOCKER" || \ - -n "$VALIDATE_GO" || \ - -n "$VALIDATE_TERRAFORM" || \ - -n "$VALIDATE_POWERSHELL" || \ - -n "$VALIDATE_CSS" || \ - -n "$VALIDATE_ENV" || \ - -n "$VALIDATE_CLOJURE" || \ - -n "$VALIDATE_PROTOBUF" || \ - -n "$VALIDATE_OPENAPI" || \ - -n "$VALIDATE_KOTLIN" ]]; then - ANY_SET="true" - fi - - #################################### - # Validate if we should check YAML # - #################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_YAML" ]]; then - # YAML flag was not set - default to false - VALIDATE_YAML="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_YAML="true" - fi - - #################################### - # Validate if we should check JSON # - #################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_JSON" ]]; then - # JSON flag was not set - default to false - VALIDATE_JSON="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_JSON="true" - fi - - ################################### - # Validate if we should check XML # - ################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_XML" ]]; then - # XML flag was not set - default to false - VALIDATE_XML="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_XML="true" - fi - - ######################################## - # Validate if we should check MARKDOWN # - ######################################## - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_MD" ]]; then - # MD flag was not set - default to false - VALIDATE_MD="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_MD="true" - fi - - #################################### - # Validate if we should check BASH # - #################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_BASH" ]]; then - # BASH flag was not set - default to false - VALIDATE_BASH="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_BASH="true" - fi - - #################################### - # Validate if we should check PERL # - #################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_PERL" ]]; then - # PERL flag was not set - default to false - VALIDATE_PERL="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_PERL="true" - fi - - #################################### - # Validate if we should check PHP # - #################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_PHP" ]]; then - # PHP flag was not set - default to false - VALIDATE_PHP="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_PHP="true" - fi - - ###################################### - # Validate if we should check PYTHON # - ###################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_PYTHON" ]]; then - # PYTHON flag was not set - default to false - VALIDATE_PYTHON="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_PYTHON="true" - fi - - #################################### - # Validate if we should check RUBY # - #################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_RUBY" ]]; then - # RUBY flag was not set - default to false - VALIDATE_RUBY="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_RUBY="true" - fi - - ###################################### - # Validate if we should check COFFEE # - ###################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_COFFEE" ]]; then - # COFFEE flag was not set - default to false - VALIDATE_COFFEE="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_COFFEE="true" - fi - - ####################################### - # Validate if we should check ANSIBLE # - ####################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_ANSIBLE" ]]; then - # ANSIBLE flag was not set - default to false - VALIDATE_ANSIBLE="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_ANSIBLE="true" - fi - - ############################################# - # Validate if we should check JAVASCRIPT_ES # - ############################################# - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_JAVASCRIPT_ES" ]]; then - # JAVASCRIPT_ES flag was not set - default to false - VALIDATE_JAVASCRIPT_ES="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_JAVASCRIPT_ES="true" - fi - - ################################################### - # Validate if we should check JAVASCRIPT_STANDARD # - ################################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_JAVASCRIPT_STANDARD" ]]; then - # JAVASCRIPT_STANDARD flag was not set - default to false - VALIDATE_JAVASCRIPT_STANDARD="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_JAVASCRIPT_STANDARD="true" - fi - - ############################################# - # Validate if we should check TYPESCRIPT_ES # - ############################################# - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_TYPESCRIPT_ES" ]]; then - # TYPESCRIPT_ES flag was not set - default to false - VALIDATE_TYPESCRIPT_ES="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_TYPESCRIPT_ES="true" - fi - - ################################################### - # Validate if we should check TYPESCRIPT_STANDARD # - ################################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_TYPESCRIPT_STANDARD" ]]; then - # TYPESCRIPT_STANDARD flag was not set - default to false - VALIDATE_TYPESCRIPT_STANDARD="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_TYPESCRIPT_STANDARD="true" - fi - - ###################################### - # Validate if we should check DOCKER # - ###################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_DOCKER" ]]; then - # DOCKER flag was not set - default to false - VALIDATE_DOCKER="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_DOCKER="true" - fi - - ################################## - # Validate if we should check GO # - ################################## - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_GO" ]]; then - # GO flag was not set - default to false - VALIDATE_GO="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_GO="true" - fi - - ######################################### - # Validate if we should check TERRAFORM # - ######################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_TERRAFORM" ]]; then - # TERRAFORM flag was not set - default to false - VALIDATE_TERRAFORM="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_TERRAFORM="true" - fi - - ######################################### - # Validate if we should check POWERSHELL # - ######################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_POWERSHELL" ]]; then - # POWERSHELL flag was not set - default to false - VALIDATE_POWERSHELL="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_POWERSHELL="true" - fi - - ################################### - # Validate if we should check CSS # - ################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_CSS" ]]; then - # CSS flag was not set - default to false - VALIDATE_CSS="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_CSS="true" - fi - - ################################### - # Validate if we should check ENV # - ################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_ENV" ]]; then - # ENV flag was not set - default to false - VALIDATE_ENV="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_ENV="true" - fi - - ###################################### - # Validate if we should check KOTLIN # - ###################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_KOTLIN" ]]; then - # ENV flag was not set - default to false - VALIDATE_KOTLIN="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_KOTLIN="true" - fi - - ####################################### - # Validate if we should check OPENAPI # - ####################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_OPENAPI" ]]; then - # OPENAPI flag was not set - default to false - VALIDATE_OPENAPI="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_OPENAPI="true" - fi - - ####################################### - # Validate if we should check PROTOBUF # - ####################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_PROTOBUF" ]]; then - # PROTOBUF flag was not set - default to false - VALIDATE_PROTOBUF="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_PROTOBUF="true" - fi - - ####################################### - # Validate if we should check Clojure # - ####################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_CLOJURE" ]]; then - # Clojure flag was not set - default to false - VALIDATE_CLOJURE="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_CLOJURE="true" - fi - - ####################################### - # Print which linters we are enabling # - ####################################### - if [[ "$VALIDATE_YAML" == "true" ]]; then - PRINT_ARRAY+=("- Validating [YAML] files in code base...") - else - PRINT_ARRAY+=("- Excluding [YAML] files in code base...") - fi - if [[ "$VALIDATE_JSON" == "true" ]]; then - PRINT_ARRAY+=("- Validating [JSON] files in code base...") - else - PRINT_ARRAY+=("- Excluding [JSON] files in code base...") - fi - if [[ "$VALIDATE_XML" == "true" ]]; then - PRINT_ARRAY+=("- Validating [XML] files in code base...") - else - PRINT_ARRAY+=("- Excluding [XML] files in code base...") - fi - if [[ "$VALIDATE_MD" == "true" ]]; then - PRINT_ARRAY+=("- Validating [MARKDOWN] files in code base...") - else - PRINT_ARRAY+=("- Excluding [MARKDOWN] files in code base...") - fi - if [[ "$VALIDATE_BASH" == "true" ]]; then - PRINT_ARRAY+=("- Validating [BASH] files in code base...") - else - PRINT_ARRAY+=("- Excluding [BASH] files in code base...") - fi - if [[ "$VALIDATE_PERL" == "true" ]]; then - PRINT_ARRAY+=("- Validating [PERL] files in code base...") - else - PRINT_ARRAY+=("- Excluding [PERL] files in code base...") - fi - if [[ "$VALIDATE_PHP" == "true" ]]; then - PRINT_ARRAY+=("- Validating [PHP] files in code base...") - else - PRINT_ARRAY+=("- Excluding [PHP] files in code base...") - fi - if [[ "$VALIDATE_PYTHON" == "true" ]]; then - PRINT_ARRAY+=("- Validating [PYTHON] files in code base...") - else - PRINT_ARRAY+=("- Excluding [PYTHON] files in code base...") - fi - if [[ "$VALIDATE_RUBY" == "true" ]]; then - PRINT_ARRAY+=("- Validating [RUBY] files in code base...") - else - PRINT_ARRAY+=("- Excluding [RUBY] files in code base...") - fi - if [[ "$VALIDATE_COFFEE" == "true" ]]; then - PRINT_ARRAY+=("- Validating [COFFEE] files in code base...") - else - PRINT_ARRAY+=("- Excluding [COFFEE] files in code base...") - fi - if [[ "$VALIDATE_ANSIBLE" == "true" ]]; then - PRINT_ARRAY+=("- Validating [ANSIBLE] files in code base...") - else - PRINT_ARRAY+=("- Excluding [ANSIBLE] files in code base...") - fi - if [[ "$VALIDATE_JAVASCRIPT_ES" == "true" ]]; then - PRINT_ARRAY+=("- Validating [JAVASCRIPT(eslint)] files in code base...") - else - PRINT_ARRAY+=("- Excluding [JAVASCRIPT(eslint)] files in code base...") - fi - if [[ "$VALIDATE_JAVASCRIPT_STANDARD" == "true" ]]; then - PRINT_ARRAY+=("- Validating [JAVASCRIPT(standard)] files in code base...") - else - PRINT_ARRAY+=("- Excluding [JAVASCRIPT(standard)] files in code base...") - fi - if [[ "$VALIDATE_TYPESCRIPT_ES" == "true" ]]; then - PRINT_ARRAY+=("- Validating [TYPESCRIPT(eslint)] files in code base...") - else - PRINT_ARRAY+=("- Excluding [TYPESCRIPT(eslint)] files in code base...") - fi - if [[ "$VALIDATE_TYPESCRIPT_STANDARD" == "true" ]]; then - PRINT_ARRAY+=("- Validating [TYPESCRIPT(standard)] files in code base...") - else - PRINT_ARRAY+=("- Excluding [TYPESCRIPT(standard)] files in code base...") - fi - if [[ "$VALIDATE_DOCKER" == "true" ]]; then - PRINT_ARRAY+=("- Validating [DOCKER] files in code base...") - else - PRINT_ARRAY+=("- Excluding [DOCKER] files in code base...") - fi - if [[ "$VALIDATE_GO" == "true" ]]; then - PRINT_ARRAY+=("- Validating [GOLANG] files in code base...") - else - PRINT_ARRAY+=("- Excluding [GOLANG] files in code base...") - fi - if [[ "$VALIDATE_TERRAFORM" == "true" ]]; then - PRINT_ARRAY+=("- Validating [TERRAFORM] files in code base...") - else - PRINT_ARRAY+=("- Excluding [TERRAFORM] files in code base...") - fi - if [[ "$VALIDATE_POWERSHELL" == "true" ]]; then - PRINT_ARRAY+=("- Validating [POWERSHELL] files in code base...") - else - PRINT_ARRAY+=("- Excluding [POWERSHELL] files in code base...") - fi - if [[ "$VALIDATE_CSS" == "true" ]]; then - PRINT_ARRAY+=("- Validating [CSS] files in code base...") - else - PRINT_ARRAY+=("- Excluding [CSS] files in code base...") - fi - if [[ "$VALIDATE_CLOJURE" == "true" ]]; then - PRINT_ARRAY+=("- Validating [CLOJURE] files in code base...") - else - PRINT_ARRAY+=("- Excluding [CLOJURE] files in code base...") - fi - if [[ "$VALIDATE_ENV" == "true" ]]; then - PRINT_ARRAY+=("- Validating [ENV] files in code base...") - else - PRINT_ARRAY+=("- Excluding [ENV] files in code base...") - fi - if [[ "$VALIDATE_KOTLIN" == "true" ]]; then - PRINT_ARRAY+=("- Validating [KOTLIN] files in code base...") - else - PRINT_ARRAY+=("- Excluding [KOTLIN] files in code base...") - fi - if [[ "$VALIDATE_OPENAPI" == "true" ]]; then - PRINT_ARRAY+=("- Validating [OPENAPI] files in code base...") - else - PRINT_ARRAY+=("- Excluding [OPENAPI] files in code base...") - fi - if [[ "$VALIDATE_PROTOBUF" == "true" ]]; then - PRINT_ARRAY+=("- Validating [PROTOBUF] files in code base...") - else - PRINT_ARRAY+=("- Excluding [PROTOBUF] files in code base...") - fi - - ############################## - # Validate Ansible Directory # - ############################## - if [ -z "$ANSIBLE_DIRECTORY" ]; then - # No Value, need to default - ANSIBLE_DIRECTORY="$DEFAULT_ANSIBLE_DIRECTORY" - else - # Check if first char is '/' - if [[ ${ANSIBLE_DIRECTORY:0:1} == "/" ]]; then - # Remove first char - ANSIBLE_DIRECTORY="${ANSIBLE_DIRECTORY:1}" - fi - # Need to give it full path - TEMP_ANSIBLE_DIRECTORY="$GITHUB_WORKSPACE/$ANSIBLE_DIRECTORY" - # Set the value - ANSIBLE_DIRECTORY="$TEMP_ANSIBLE_DIRECTORY" - fi - - ############################### - # Get the disable errors flag # - ############################### - if [ -z "$DISABLE_ERRORS" ]; then - ################################## - # No flag passed, set to default # - ################################## - DISABLE_ERRORS="$DEFAULT_DISABLE_ERRORS" - fi - - ############################### - # Convert string to lowercase # - ############################### - DISABLE_ERRORS=$(echo "$DISABLE_ERRORS" | awk '{print tolower($0)}') - - ############################ - # Set to false if not true # - ############################ - if [ "$DISABLE_ERRORS" != "true" ]; then - DISABLE_ERRORS="false" - fi - - ############################ - # Get the run verbose flag # - ############################ - if [ -z "$ACTIONS_RUNNER_DEBUG" ]; then - ################################## - # No flag passed, set to default # - ################################## - ACTIONS_RUNNER_DEBUG="$DEFAULT_ACTIONS_RUNNER_DEBUG" - fi - - ############################### - # Convert string to lowercase # - ############################### - ACTIONS_RUNNER_DEBUG=$(echo "$ACTIONS_RUNNER_DEBUG" | awk '{print tolower($0)}') - - ############################ - # Set to true if not false # - ############################ - if [ "$ACTIONS_RUNNER_DEBUG" != "false" ]; then - ACTIONS_RUNNER_DEBUG="true" - fi - - ################### - # Debug on runner # - ################### - if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then - ########################### - # Print the validate info # - ########################### - for LINE in "${PRINT_ARRAY[@]}" - do - echo "$LINE" - done - - echo "--- DEBUG INFO ---" - echo "---------------------------------------------" - RUNNER=$(whoami) - echo "Runner:[$RUNNER]" - echo "ENV:" - printenv - echo "---------------------------------------------" - fi -} -################################################################################ -#### Function BuildFileList #################################################### -BuildFileList() -{ - # Need to build a list of all files changed - # This can be pulled from the GITHUB_EVENT_PATH payload - - ################ - # print header # - ################ - if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then - echo "" - echo "----------------------------------------------" - echo "Pulling in code history and branches..." - fi - - ################################################################################# - # Switch codebase back to the default branch to get a list of all files changed # - ################################################################################# - SWITCH_CMD=$(cd "$GITHUB_WORKSPACE" || exit; git pull --quiet; git checkout "$DEFAULT_BRANCH" 2>&1) - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ $ERROR_CODE -ne 0 ]; then - # Error - echo "Failed to switch to $DEFAULT_BRANCH branch to get files changed!" - echo "ERROR:[$SWITCH_CMD]" - exit 1 - fi - - ################ - # print header # - ################ - if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then - echo "" - echo "----------------------------------------------" - echo "Generating Diff with:[git diff --name-only '$DEFAULT_BRANCH..$GITHUB_SHA' --diff-filter=d]" - fi - - ################################################# - # Get the Array of files changed in the commits # - ################################################# - # shellcheck disable=SC2207 - RAW_FILE_ARRAY=($(cd "$GITHUB_WORKSPACE" || exit; git diff --name-only "$DEFAULT_BRANCH..$GITHUB_SHA" --diff-filter=d 2>&1)) - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ $ERROR_CODE -ne 0 ]; then - # Error - echo "ERROR! Failed to gain a list of all files changed!" - echo "ERROR:[${RAW_FILE_ARRAY[*]}]" - exit 1 - fi - - ################################################ - # Iterate through the array of all files found # - ################################################ - echo "" - echo "----------------------------------------------" - echo "Files that have been modified in the commit(s):" - for FILE in "${RAW_FILE_ARRAY[@]}" - do - ############## - # Print file # - ############## - echo "File:[$FILE]" - - ########################### - # Get the files extension # - ########################### - # Extract just the file and extension, reverse it, cut off extension, - # reverse it back, substitute to lowercase - FILE_TYPE=$(basename "$FILE" | rev | cut -f1 -d'.' | rev | awk '{print tolower($0)}') - - ######### - # DEBUG # - ######### - #echo "FILE_TYPE:[$FILE_TYPE]" - - ##################### - # Get the YML files # - ##################### - if [ "$FILE_TYPE" == "yml" ] || [ "$FILE_TYPE" == "yaml" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_YML+=("$FILE") - ############################ - # Check if file is OpenAPI # - ############################ - if DetectOpenAPIFile "$FILE"; then - FILE_ARRAY_OPENAPI+=("$FILE") - fi - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ###################### - # Get the JSON files # - ###################### - elif [ "$FILE_TYPE" == "json" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_JSON+=("$FILE") - ############################ - # Check if file is OpenAPI # - ############################ - if DetectOpenAPIFile "$FILE"; then - FILE_ARRAY_OPENAPI+=("$FILE") - fi - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ##################### - # Get the XML files # - ##################### - elif [ "$FILE_TYPE" == "xml" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_XML+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ########################## - # Get the MARKDOWN files # - ########################## - elif [ "$FILE_TYPE" == "md" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_MD+=("$FILE") - ###################### - # Get the BASH files # - ###################### - elif [ "$FILE_TYPE" == "sh" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_BASH+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ###################### - # Get the PERL files # - ###################### - elif [ "$FILE_TYPE" == "pl" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_PERL+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ###################### - # Get the PHP files # - ###################### - elif [ "$FILE_TYPE" == "php" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_PHP+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ###################### - # Get the RUBY files # - ###################### - elif [ "$FILE_TYPE" == "rb" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_RUBY+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ######################## - # Get the PYTHON files # - ######################## - elif [ "$FILE_TYPE" == "py" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_PYTHON+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ######################## - # Get the COFFEE files # - ######################## - elif [ "$FILE_TYPE" == "coffee" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_COFFEESCRIPT+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ############################ - # Get the JavaScript files # - ############################ - elif [ "$FILE_TYPE" == "js" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_JAVASCRIPT_ES+=("$FILE") - FILE_ARRAY_JAVASCRIPT_STANDARD+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ############################ - # Get the TypeScript files # - ############################ - elif [ "$FILE_TYPE" == "ts" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_TYPESCRIPT_ES+=("$FILE") - FILE_ARRAY_TYPESCRIPT_STANDARD+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ######################## - # Get the Golang files # - ######################## - elif [ "$FILE_TYPE" == "go" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_GO+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ########################### - # Get the Terraform files # - ########################### - elif [ "$FILE_TYPE" == "tf" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_TERRAFORM+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ########################### - # Get the Powershell files # - ########################### - elif [ "$FILE_TYPE" == "ps1" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_POWERSHELL+=("$FILE") - elif [ "$FILE_TYPE" == "css" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_CSS+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - elif [ "$FILE_TYPE" == "env" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_ENV+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - elif [ "$FILE_TYPE" == "kt" ] || [ "$FILE_TYPE" == "kts" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_KOTLIN+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ############################ - # Get the Protocol Buffers files # - ############################ - elif [ "$FILE_TYPE" == "proto" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_PROTOBUF+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - elif [ "$FILE" == "Dockerfile" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_DOCKER+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - elif [ "$FILE_TYPE" == "clj" ] || [ "$FILE_TYPE" == "cljs" ] || [ "$FILE_TYPE" == "cljc" ] || [ "$FILE_TYPE" == "edn" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_CLOJURE+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - else - ############################################## - # Use file to see if we can parse what it is # - ############################################## - GET_FILE_TYPE_CMD=$(file "$FILE" 2>&1) - - ################# - # Check if bash # - ################# - if [[ "$GET_FILE_TYPE_CMD" == *"Bourne-Again shell script"* ]]; then - ####################### - # It is a bash script # - ####################### - echo "WARN! Found bash script without extension:[.sh]" - echo "Please update file with proper extensions." - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_BASH+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - elif [[ "$GET_FILE_TYPE_CMD" == *"Ruby script"* ]]; then - ####################### - # It is a Ruby script # - ####################### - echo "WARN! Found ruby script without extension:[.rb]" - echo "Please update file with proper extensions." - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_RUBY+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - else - ############################ - # Extension was not found! # - ############################ - echo " - WARN! Failed to get filetype for:[$FILE]!" - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - fi - fi - done - - ######################################### - # Need to switch back to branch of code # - ######################################### - SWITCH2_CMD=$(cd "$GITHUB_WORKSPACE" || exit; git checkout --progress --force "$GITHUB_SHA" 2>&1) - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ $ERROR_CODE -ne 0 ]; then - # Error - echo "Failed to switch back to branch!" - echo "ERROR:[$SWITCH2_CMD]" - exit 1 - fi - - ################ - # Footer print # - ################ - echo "" - echo "----------------------------------------------" - echo "Successfully gathered list of files..." -} -################################################################################ -#### Function LintCodebase ##################################################### -LintCodebase() -{ - #################### - # Pull in the vars # - #################### - FILE_TYPE="$1" && shift # Pull the variable and remove from array path (Example: JSON) - LINTER_NAME="$1" && shift # Pull the variable and remove from array path (Example: jsonlint) - LINTER_COMMAND="$1" && shift # Pull the variable and remove from array path (Example: jsonlint -c ConfigFile /path/to/file) - FILE_EXTENSIONS="$1" && shift # Pull the variable and remove from array path (Example: *.json) - FILE_ARRAY=("$@") # Array of files to validate (Example: $FILE_ARRAY_JSON) - - ###################### - # Create Print Array # - ###################### - PRINT_ARRAY=() - - ################ - # print header # - ################ - PRINT_ARRAY+=("") - PRINT_ARRAY+=("----------------------------------------------") - PRINT_ARRAY+=("----------------------------------------------") - PRINT_ARRAY+=("Linting [$FILE_TYPE] files...") - PRINT_ARRAY+=("----------------------------------------------") - PRINT_ARRAY+=("----------------------------------------------") - - ####################################### - # Validate we have jsonlint installed # - ####################################### - # shellcheck disable=SC2230 - VALIDATE_INSTALL_CMD=$(command -v "$LINTER_NAME" 2>&1) - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ $ERROR_CODE -ne 0 ]; then - # Failed - echo "ERROR! Failed to find [$LINTER_NAME] in system!" - echo "ERROR:[$VALIDATE_INSTALL_CMD]" - exit 1 - else - # Success - if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then - echo "Successfully found binary in system" - echo "Location:[$VALIDATE_INSTALL_CMD]" - fi - fi - - ########################## - # Initialize empty Array # - ########################## - LIST_FILES=() - - ################ - # Set the flag # - ################ - SKIP_FLAG=0 - - ############################################################ - # Check to see if we need to go through array or all files # - ############################################################ - if [ ${#FILE_ARRAY[@]} -eq 0 ] && [ "$VALIDATE_ALL_CODEBASE" == "false" ]; then - # No files found in commit and user has asked to not validate code base - SKIP_FLAG=1 - # echo " - No files found in changeset to lint for language:[$FILE_TYPE]" - elif [ ${#FILE_ARRAY[@]} -ne 0 ]; then - # We have files added to array of files to check - LIST_FILES=("${FILE_ARRAY[@]}") # Copy the array into list - else - ############################################################################### - # Set the file seperator to newline to allow for grabbing objects with spaces # - ############################################################################### - IFS=$'\n' - - ################################# - # Get list of all files to lint # - ################################# - # shellcheck disable=SC2207,SC2086 - LIST_FILES=($(cd "$GITHUB_WORKSPACE" || exit; find . -type f -regex "$FILE_EXTENSIONS" 2>&1)) - - ########################### - # Set IFS back to default # - ########################### - IFS="$DEFAULT_IFS" - - ############################################################ - # Set it back to empty if loaded with blanks from scanning # - ############################################################ - if [ ${#LIST_FILES[@]} -lt 1 ]; then - ###################### - # Set to empty array # - ###################### - LIST_FILES=() - ############################# - # Skip as we found no files # - ############################# - SKIP_FLAG=1 - fi - fi - - ############################### - # Check if any data was found # - ############################### - if [ $SKIP_FLAG -eq 0 ]; then - ###################### - # Print Header array # - ###################### - for LINE in "${PRINT_ARRAY[@]}" - do - ######################### - # Print the header info # - ######################### - echo "$LINE" - done - - ################## - # Lint the files # - ################## - for FILE in "${LIST_FILES[@]}" - do - ##################### - # Get the file name # - ##################### - FILE_NAME=$(basename "$FILE" 2>&1) - - ##################################################### - # Make sure we dont lint node modules or test cases # - ##################################################### - if [[ $FILE == *"node_modules"* ]]; then - # This is a node modules file - continue - elif [[ $FILE == *"$TEST_CASE_FOLDER"* ]]; then - # This is the test cases, we should always skip - continue - fi - - ############## - # File print # - ############## - echo "---------------------------" - echo "File:[$FILE]" - - #################### - # Set the base Var # - #################### - LINT_CMD='' - - ####################################### - # Corner case for Powershell subshell # - ####################################### - if [[ "$FILE_TYPE" == "POWERSHELL" ]]; then - ################################ - # Lint the file with the rules # - ################################ - # Need to append "'" to make the pwsh call syntax correct, also exit with exit code from inner subshell - LINT_CMD=$(cd "$GITHUB_WORKSPACE" || exit; $LINTER_COMMAND "$FILE"; exit $? 2>&1) - else - ################################ - # Lint the file with the rules # - ################################ - LINT_CMD=$(cd "$GITHUB_WORKSPACE" || exit; $LINTER_COMMAND "$FILE" 2>&1) - fi - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ $ERROR_CODE -ne 0 ]; then - ######### - # Error # - ######### - echo "ERROR! Found errors in [$LINTER_NAME] linter!" - echo "ERROR:[$LINT_CMD]" - # Increment the error count - (("ERRORS_FOUND_$FILE_TYPE++")) - else - ########### - # Success # - ########### - echo " - File:[$FILE_NAME] was linted with [$LINTER_NAME] successfully" - fi - done - fi -} -################################################################################ -#### Function TestCodebase ##################################################### -TestCodebase() -{ - #################### - # Pull in the vars # - #################### - FILE_TYPE="$1" # Pull the variable and remove from array path (Example: JSON) - LINTER_NAME="$2" # Pull the variable and remove from array path (Example: jsonlint) - LINTER_COMMAND="$3" # Pull the variable and remove from array path (Example: jsonlint -c ConfigFile /path/to/file) - FILE_EXTENSIONS="$4" # Pull the variable and remove from array path (Example: *.json) - - ################ - # print header # - ################ - echo "" - echo "----------------------------------------------" - echo "----------------------------------------------" - echo "Testing Codebase [$FILE_TYPE] files..." - echo "----------------------------------------------" - echo "----------------------------------------------" - echo "" - - ##################################### - # Validate we have linter installed # - ##################################### - # shellcheck disable=SC2230 - VALIDATE_INSTALL_CMD=$(command -v "$LINTER_NAME" 2>&1) - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ $ERROR_CODE -ne 0 ]; then - # Failed - echo "ERROR! Failed to find [$LINTER_NAME] in system!" - echo "ERROR:[$VALIDATE_INSTALL_CMD]" - exit 1 - else - # Success - echo "Successfully found binary in system" - echo "Location:[$VALIDATE_INSTALL_CMD]" - fi - - ########################## - # Initialize empty Array # - ########################## - LIST_FILES=() - - ############################################ - # Check if its ansible, as its the outlier # - ############################################ - if [[ "$FILE_TYPE" == "ANSIBLE" ]]; then - ################################# - # Get list of all files to lint # - ################################# - # shellcheck disable=SC2207,SC2086,SC2010 - LIST_FILES=($(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER" || exit; ls ansible/ | grep ".yml" 2>&1)) - else - ############################################################################### - # Set the file seperator to newline to allow for grabbing objects with spaces # - ############################################################################### - IFS=$'\n' - - ################################# - # Get list of all files to lint # - ################################# - # shellcheck disable=SC2207,SC2086 - LIST_FILES=($(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER" || exit; find . -type f -regex "$FILE_EXTENSIONS" ! -path "*./ansible*" 2>&1)) - - ########################### - # Set IFS back to default # - ########################### - IFS="$DEFAULT_IFS" - fi - - ################## - # Lint the files # - ################## - for FILE in "${LIST_FILES[@]}" - do - ##################### - # Get the file name # - ##################### - FILE_NAME=$(basename "$FILE" 2>&1) - - ############################ - # Get the file pass status # - ############################ - # Example: markdown_good_1.md -> good - FILE_STATUS=$(echo "$FILE_NAME" |cut -f2 -d'_') - - ######################################################### - # If not found, assume it should be linted successfully # - ######################################################### - if [ -z "$FILE_STATUS" ] || [[ "$FILE" == *"README"* ]]; then - ################################## - # Set to good for proper linting # - ################################## - FILE_STATUS="good" - fi - - ############## - # File print # - ############## - echo "---------------------------" - echo "File:[$FILE]" - - ######################## - # Set the lint command # - ######################## - LINT_CMD='' - - ####################################### - # Check if docker and get folder name # - ####################################### - if [[ "$FILE_TYPE" == "DOCKER" ]]; then - if [[ "$FILE" == *"good"* ]]; then - ############# - # Good file # - ############# - FILE_STATUS='good' - else - ############ - # Bad file # - ############ - FILE_STATUS='bad' - fi - fi - - ##################### - # Check for ansible # - ##################### - if [[ "$FILE_TYPE" == "ANSIBLE" ]]; then - ######################################## - # Make sure we dont lint certain files # - ######################################## - if [[ $FILE == *"vault.yml"* ]] || [[ $FILE == *"galaxy.yml"* ]]; then - # This is a file we dont look at - continue - fi - - ################################ - # Lint the file with the rules # - ################################ - LINT_CMD=$(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER/ansible" || exit; $LINTER_COMMAND "$FILE" 2>&1) - elif [[ "$FILE_TYPE" == "POWERSHELL" ]]; then - ################################ - # Lint the file with the rules # - ################################ - # Need to append "'" to make the pwsh call syntax correct, also exit with exit code from inner subshell - LINT_CMD=$(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER" || exit; $LINTER_COMMAND "$FILE"; exit $? 2>&1) - else - ################################ - # Lint the file with the rules # - ################################ - LINT_CMD=$(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER" || exit; $LINTER_COMMAND "$FILE" 2>&1) - fi - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ######################################## - # Check for if it was supposed to pass # - ######################################## - if [[ "$FILE_STATUS" == "good" ]]; then - ############################## - # Check the shell for errors # - ############################## - if [ $ERROR_CODE -ne 0 ]; then - ######### - # Error # - ######### - echo "ERROR! Found errors in [$LINTER_NAME] linter!" - echo "ERROR:[$LINT_CMD]" - echo "ERROR: Linter CMD:[$LINTER_COMMAND $FILE]" - # Increment the error count - (("ERRORS_FOUND_$FILE_TYPE++")) - else - ########### - # Success # - ########### - echo " - File:[$FILE_NAME] was linted with [$LINTER_NAME] successfully" - fi - else - ####################################### - # File status = bad, this should fail # - ####################################### - ############################## - # Check the shell for errors # - ############################## - if [ $ERROR_CODE -eq 0 ]; then - ######### - # Error # - ######### - echo "ERROR! Found errors in [$LINTER_NAME] linter!" - echo "ERROR! This file should have failed test case!" - echo "ERROR:[$LINT_CMD]" - echo "ERROR: Linter CMD:[$LINTER_COMMAND $FILE]" - # Increment the error count - (("ERRORS_FOUND_$FILE_TYPE++")) - else - ########### - # Success # - ########### - echo " - File:[$FILE_NAME] failed test case with [$LINTER_NAME] successfully" - fi - fi - done -} -################################################################################ #### Function Footer ########################################################### Footer() { @@ -2315,63 +657,6 @@ Footer() fi } ################################################################################ -#### Function RunTestCases ##################################################### -RunTestCases() -{ - # This loop will run the test cases and exclude user code - # This is called from the automation process to validate new code - # When a PR is opened, the new code is validated with the default branch - # version of linter.sh, and a new container is built with the latest codebase - # for testing. That container is spun up, and ran, - # with the flag: TEST_CASE_RUN=true - # So that the new code can be validated against the test cases - - ################# - # Header prints # - ################# - echo "" - echo "----------------------------------------------" - echo "-------------- TEST CASE RUN -----------------" - echo "----------------------------------------------" - echo "" - - ####################### - # Test case languages # - ####################### - TestCodebase "YML" "yamllint" "yamllint -c $YAML_LINTER_RULES" ".*\.\(yml\|yaml\)\$" - TestCodebase "JSON" "jsonlint" "jsonlint" ".*\.\(json\)\$" - TestCodebase "XML" "xmllint" "xmllint" ".*\.\(xml\)\$" - TestCodebase "MARKDOWN" "markdownlint" "markdownlint -c $MD_LINTER_RULES" ".*\.\(md\)\$" - TestCodebase "BASH" "shellcheck" "shellcheck" ".*\.\(sh\)\$" - TestCodebase "PYTHON" "pylint" "pylint --rcfile $PYTHON_LINTER_RULES" ".*\.\(py\)\$" - TestCodebase "PERL" "perl" "perl -Mstrict -cw" ".*\.\(pl\)\$" - TestCodebase "PHP" "php" "php -l" ".*\.\(php\)\$" - TestCodebase "RUBY" "rubocop" "rubocop -c $RUBY_LINTER_RULES" ".*\.\(rb\)\$" - TestCodebase "GO" "golangci-lint" "golangci-lint run -c $GO_LINTER_RULES" ".*\.\(go\)\$" - TestCodebase "COFFEESCRIPT" "coffeelint" "coffeelint -f $COFFEESCRIPT_LINTER_RULES" ".*\.\(coffee\)\$" - TestCodebase "JAVASCRIPT_ES" "eslint" "eslint --no-eslintrc -c $JAVASCRIPT_LINTER_RULES" ".*\.\(js\)\$" - TestCodebase "JAVASCRIPT_STANDARD" "standard" "standard $JAVASCRIPT_STANDARD_LINTER_RULES" ".*\.\(js\)\$" - TestCodebase "TYPESCRIPT_ES" "eslint" "eslint --no-eslintrc -c $TYPESCRIPT_LINTER_RULES" ".*\.\(ts\)\$" - TestCodebase "TYPESCRIPT_STANDARD" "standard" "standard --parser @typescript-eslint/parser --plugin @typescript-eslint/eslint-plugin $TYPESCRIPT_STANDARD_LINTER_RULES" ".*\.\(ts\)\$" - TestCodebase "DOCKER" "/dockerfilelint/bin/dockerfilelint" "/dockerfilelint/bin/dockerfilelint -c $DOCKER_LINTER_RULES" ".*\(Dockerfile\)\$" - TestCodebase "ANSIBLE" "ansible-lint" "ansible-lint -v -c $ANSIBLE_LINTER_RULES" "ansible-lint" - TestCodebase "TERRAFORM" "tflint" "tflint -c $TERRAFORM_LINTER_RULES" ".*\.\(tf\)\$" - TestCodebase "POWERSHELL" "pwsh" "pwsh -c Invoke-ScriptAnalyzer -EnableExit -Settings $POWERSHELL_LINTER_RULES -Path" ".*\.\(ps1\|psm1\|psd1\|ps1xml\|pssc\|psrc\|cdxml\)\$" - TestCodebase "CSS" "stylelint" "stylelint --config $CSS_LINTER_RULES" ".*\.\(css\)\$" - TestCodebase "ENV" "dotenv-linter" "dotenv-linter" ".*\.\(env\)\$" - TestCodebase "CLOJURE" "clj-kondo" "clj-kondo --config $CLOJURE_LINTER_RULES --lint" ".*\.\(clj\|cljs\|cljc\|edn\)\$" - TestCodebase "KOTLIN" "ktlint" "ktlint" ".*\.\(kt\|kts\)\$" - TestCodebase "PROTOBUF" "protolint" "protolint lint --config_path $PROTOBUF_LINTER_RULES" ".*\.\(proto\)\$" - TestCodebase "OPENAPI" "spectral" "spectral lint -r $OPENAPI_LINTER_RULES" ".*\.\(ymlopenapi\|jsonopenapi\)\$" - - ################# - # Footer prints # - ################# - # Call the footer to display run information - # and exit with error code - Footer -} -################################################################################ ############################### MAIN ########################################### ################################################################################ diff --git a/lib/validation.sh b/lib/validation.sh new file mode 100755 index 00000000..97f85a38 --- /dev/null +++ b/lib/validation.sh @@ -0,0 +1,669 @@ +#!/usr/bin/env bash + +################################################################################ +################################################################################ +########### Super-Linter Validation Functions @admiralawkbar ################### +################################################################################ +################################################################################ +########################## FUNCTION CALLS BELOW ################################ +################################################################################ +################################################################################ +#### Function GetValidationInfo ################################################ +function GetValidationInfo() +{ + ############################################ + # Print headers for user provided env vars # + ############################################ + echo "" + echo "--------------------------------------------" + echo "Gathering user validation information..." + + ########################################### + # Skip validation if were running locally # + ########################################### + if [[ "$RUN_LOCAL" != "true" ]]; then + ############################### + # Convert string to lowercase # + ############################### + VALIDATE_ALL_CODEBASE=$(echo "$VALIDATE_ALL_CODEBASE" | awk '{print tolower($0)}') + ###################################### + # Validate we should check all files # + ###################################### + if [[ "$VALIDATE_ALL_CODEBASE" != "false" ]]; then + # Set to true + VALIDATE_ALL_CODEBASE="$DEFAULT_VALIDATE_ALL_CODEBASE" + echo "- Validating ALL files in code base..." + else + # Its false + echo "- Only validating [new], or [edited] files in code base..." + fi + fi + + ###################### + # Create Print Array # + ###################### + PRINT_ARRAY=() + + ################################ + # Convert strings to lowercase # + ################################ + VALIDATE_YAML=$(echo "$VALIDATE_YAML" | awk '{print tolower($0)}') + VALIDATE_JSON=$(echo "$VALIDATE_JSON" | awk '{print tolower($0)}') + VALIDATE_XML=$(echo "$VALIDATE_XML" | awk '{print tolower($0)}') + VALIDATE_MD=$(echo "$VALIDATE_MD" | awk '{print tolower($0)}') + VALIDATE_BASH=$(echo "$VALIDATE_BASH" | awk '{print tolower($0)}') + VALIDATE_PERL=$(echo "$VALIDATE_PERL" | awk '{print tolower($0)}') + VALIDATE_PHP=$(echo "$VALIDATE_PHP" | awk '{print tolower($0)}') + VALIDATE_PYTHON=$(echo "$VALIDATE_PYTHON" | awk '{print tolower($0)}') + VALIDATE_RUBY=$(echo "$VALIDATE_RUBY" | awk '{print tolower($0)}') + VALIDATE_COFFEE=$(echo "$VALIDATE_COFFEE" | awk '{print tolower($0)}') + VALIDATE_ANSIBLE=$(echo "$VALIDATE_ANSIBLE" | awk '{print tolower($0)}') + VALIDATE_JAVASCRIPT_ES=$(echo "$VALIDATE_JAVASCRIPT_ES" | awk '{print tolower($0)}') + VALIDATE_JAVASCRIPT_STANDARD=$(echo "$VALIDATE_JAVASCRIPT_STANDARD" | awk '{print tolower($0)}') + VALIDATE_TYPESCRIPT_ES=$(echo "$VALIDATE_TYPESCRIPT_ES" | awk '{print tolower($0)}') + VALIDATE_TYPESCRIPT_STANDARD=$(echo "$VALIDATE_TYPESCRIPT_STANDARD" | awk '{print tolower($0)}') + VALIDATE_DOCKER=$(echo "$VALIDATE_DOCKER" | awk '{print tolower($0)}') + VALIDATE_GO=$(echo "$VALIDATE_GO" | awk '{print tolower($0)}') + VALIDATE_TERRAFORM=$(echo "$VALIDATE_TERRAFORM" | awk '{print tolower($0)}') + VALIDATE_POWERSHELL=$(echo "$VALIDATE_POWERSHELL" | awk '{print tolower($0)}') + VALIDATE_CSS=$(echo "$VALIDATE_CSS" | awk '{print tolower($0)}') + VALIDATE_ENV=$(echo "$VALIDATE_ENV" | awk '{print tolower($0)}') + VALIDATE_CLOJURE=$(echo "$VALIDATE_CLOJURE" | awk '{print tolower($0)') + VALIDATE_KOTLIN=$(echo "$VALIDATE_KOTLIN" | awk '{print tolower($0)}') + VALIDATE_PROTOBUF=$(echo "$VALIDATE_PROTOBUF" | awk '{print tolower($0)}') + VALIDATE_OPENAPI=$(echo "$VALIDATE_OPENAPI" | awk '{print tolower($0)}') + + ################################################ + # Determine if any linters were explicitly set # + ################################################ + ANY_SET="false" + if [[ -n "$VALIDATE_YAML" || \ + -n "$VALIDATE_JSON" || \ + -n "$VALIDATE_XML" || \ + -n "$VALIDATE_MD" || \ + -n "$VALIDATE_BASH" || \ + -n "$VALIDATE_PERL" || \ + -n "$VALIDATE_PHP" || \ + -n "$VALIDATE_PYTHON" || \ + -n "$VALIDATE_RUBY" || \ + -n "$VALIDATE_COFFEE" || \ + -n "$VALIDATE_ANSIBLE" || \ + -n "$VALIDATE_JAVASCRIPT_ES" || \ + -n "$VALIDATE_JAVASCRIPT_STANDARD" || \ + -n "$VALIDATE_TYPESCRIPT_ES" || \ + -n "$VALIDATE_TYPESCRIPT_STANDARD" || \ + -n "$VALIDATE_DOCKER" || \ + -n "$VALIDATE_GO" || \ + -n "$VALIDATE_TERRAFORM" || \ + -n "$VALIDATE_POWERSHELL" || \ + -n "$VALIDATE_CSS" || \ + -n "$VALIDATE_ENV" || \ + -n "$VALIDATE_CLOJURE" || \ + -n "$VALIDATE_PROTOBUF" || \ + -n "$VALIDATE_OPENAPI" || \ + -n "$VALIDATE_KOTLIN" ]]; then + ANY_SET="true" + fi + + #################################### + # Validate if we should check YAML # + #################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_YAML" ]]; then + # YAML flag was not set - default to false + VALIDATE_YAML="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_YAML="true" + fi + + #################################### + # Validate if we should check JSON # + #################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_JSON" ]]; then + # JSON flag was not set - default to false + VALIDATE_JSON="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_JSON="true" + fi + + ################################### + # Validate if we should check XML # + ################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_XML" ]]; then + # XML flag was not set - default to false + VALIDATE_XML="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_XML="true" + fi + + ######################################## + # Validate if we should check MARKDOWN # + ######################################## + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_MD" ]]; then + # MD flag was not set - default to false + VALIDATE_MD="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_MD="true" + fi + + #################################### + # Validate if we should check BASH # + #################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_BASH" ]]; then + # BASH flag was not set - default to false + VALIDATE_BASH="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_BASH="true" + fi + + #################################### + # Validate if we should check PERL # + #################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_PERL" ]]; then + # PERL flag was not set - default to false + VALIDATE_PERL="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_PERL="true" + fi + + #################################### + # Validate if we should check PHP # + #################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_PHP" ]]; then + # PHP flag was not set - default to false + VALIDATE_PHP="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_PHP="true" + fi + + ###################################### + # Validate if we should check PYTHON # + ###################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_PYTHON" ]]; then + # PYTHON flag was not set - default to false + VALIDATE_PYTHON="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_PYTHON="true" + fi + + #################################### + # Validate if we should check RUBY # + #################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_RUBY" ]]; then + # RUBY flag was not set - default to false + VALIDATE_RUBY="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_RUBY="true" + fi + + ###################################### + # Validate if we should check COFFEE # + ###################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_COFFEE" ]]; then + # COFFEE flag was not set - default to false + VALIDATE_COFFEE="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_COFFEE="true" + fi + + ####################################### + # Validate if we should check ANSIBLE # + ####################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_ANSIBLE" ]]; then + # ANSIBLE flag was not set - default to false + VALIDATE_ANSIBLE="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_ANSIBLE="true" + fi + + ############################################# + # Validate if we should check JAVASCRIPT_ES # + ############################################# + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_JAVASCRIPT_ES" ]]; then + # JAVASCRIPT_ES flag was not set - default to false + VALIDATE_JAVASCRIPT_ES="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_JAVASCRIPT_ES="true" + fi + + ################################################### + # Validate if we should check JAVASCRIPT_STANDARD # + ################################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_JAVASCRIPT_STANDARD" ]]; then + # JAVASCRIPT_STANDARD flag was not set - default to false + VALIDATE_JAVASCRIPT_STANDARD="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_JAVASCRIPT_STANDARD="true" + fi + + ############################################# + # Validate if we should check TYPESCRIPT_ES # + ############################################# + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_TYPESCRIPT_ES" ]]; then + # TYPESCRIPT_ES flag was not set - default to false + VALIDATE_TYPESCRIPT_ES="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_TYPESCRIPT_ES="true" + fi + + ################################################### + # Validate if we should check TYPESCRIPT_STANDARD # + ################################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_TYPESCRIPT_STANDARD" ]]; then + # TYPESCRIPT_STANDARD flag was not set - default to false + VALIDATE_TYPESCRIPT_STANDARD="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_TYPESCRIPT_STANDARD="true" + fi + + ###################################### + # Validate if we should check DOCKER # + ###################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_DOCKER" ]]; then + # DOCKER flag was not set - default to false + VALIDATE_DOCKER="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_DOCKER="true" + fi + + ################################## + # Validate if we should check GO # + ################################## + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_GO" ]]; then + # GO flag was not set - default to false + VALIDATE_GO="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_GO="true" + fi + + ######################################### + # Validate if we should check TERRAFORM # + ######################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_TERRAFORM" ]]; then + # TERRAFORM flag was not set - default to false + VALIDATE_TERRAFORM="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_TERRAFORM="true" + fi + + ######################################### + # Validate if we should check POWERSHELL # + ######################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_POWERSHELL" ]]; then + # POWERSHELL flag was not set - default to false + VALIDATE_POWERSHELL="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_POWERSHELL="true" + fi + + ################################### + # Validate if we should check CSS # + ################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_CSS" ]]; then + # CSS flag was not set - default to false + VALIDATE_CSS="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_CSS="true" + fi + + ################################### + # Validate if we should check ENV # + ################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_ENV" ]]; then + # ENV flag was not set - default to false + VALIDATE_ENV="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_ENV="true" + fi + + ###################################### + # Validate if we should check KOTLIN # + ###################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_KOTLIN" ]]; then + # ENV flag was not set - default to false + VALIDATE_KOTLIN="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_KOTLIN="true" + fi + + ####################################### + # Validate if we should check OPENAPI # + ####################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_OPENAPI" ]]; then + # OPENAPI flag was not set - default to false + VALIDATE_OPENAPI="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_OPENAPI="true" + fi + + ####################################### + # Validate if we should check PROTOBUF # + ####################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_PROTOBUF" ]]; then + # PROTOBUF flag was not set - default to false + VALIDATE_PROTOBUF="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_PROTOBUF="true" + fi + + ####################################### + # Validate if we should check Clojure # + ####################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_CLOJURE" ]]; then + # Clojure flag was not set - default to false + VALIDATE_CLOJURE="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_CLOJURE="true" + fi + + ####################################### + # Print which linters we are enabling # + ####################################### + if [[ "$VALIDATE_YAML" == "true" ]]; then + PRINT_ARRAY+=("- Validating [YAML] files in code base...") + else + PRINT_ARRAY+=("- Excluding [YAML] files in code base...") + fi + if [[ "$VALIDATE_JSON" == "true" ]]; then + PRINT_ARRAY+=("- Validating [JSON] files in code base...") + else + PRINT_ARRAY+=("- Excluding [JSON] files in code base...") + fi + if [[ "$VALIDATE_XML" == "true" ]]; then + PRINT_ARRAY+=("- Validating [XML] files in code base...") + else + PRINT_ARRAY+=("- Excluding [XML] files in code base...") + fi + if [[ "$VALIDATE_MD" == "true" ]]; then + PRINT_ARRAY+=("- Validating [MARKDOWN] files in code base...") + else + PRINT_ARRAY+=("- Excluding [MARKDOWN] files in code base...") + fi + if [[ "$VALIDATE_BASH" == "true" ]]; then + PRINT_ARRAY+=("- Validating [BASH] files in code base...") + else + PRINT_ARRAY+=("- Excluding [BASH] files in code base...") + fi + if [[ "$VALIDATE_PERL" == "true" ]]; then + PRINT_ARRAY+=("- Validating [PERL] files in code base...") + else + PRINT_ARRAY+=("- Excluding [PERL] files in code base...") + fi + if [[ "$VALIDATE_PHP" == "true" ]]; then + PRINT_ARRAY+=("- Validating [PHP] files in code base...") + else + PRINT_ARRAY+=("- Excluding [PHP] files in code base...") + fi + if [[ "$VALIDATE_PYTHON" == "true" ]]; then + PRINT_ARRAY+=("- Validating [PYTHON] files in code base...") + else + PRINT_ARRAY+=("- Excluding [PYTHON] files in code base...") + fi + if [[ "$VALIDATE_RUBY" == "true" ]]; then + PRINT_ARRAY+=("- Validating [RUBY] files in code base...") + else + PRINT_ARRAY+=("- Excluding [RUBY] files in code base...") + fi + if [[ "$VALIDATE_COFFEE" == "true" ]]; then + PRINT_ARRAY+=("- Validating [COFFEE] files in code base...") + else + PRINT_ARRAY+=("- Excluding [COFFEE] files in code base...") + fi + if [[ "$VALIDATE_ANSIBLE" == "true" ]]; then + PRINT_ARRAY+=("- Validating [ANSIBLE] files in code base...") + else + PRINT_ARRAY+=("- Excluding [ANSIBLE] files in code base...") + fi + if [[ "$VALIDATE_JAVASCRIPT_ES" == "true" ]]; then + PRINT_ARRAY+=("- Validating [JAVASCRIPT(eslint)] files in code base...") + else + PRINT_ARRAY+=("- Excluding [JAVASCRIPT(eslint)] files in code base...") + fi + if [[ "$VALIDATE_JAVASCRIPT_STANDARD" == "true" ]]; then + PRINT_ARRAY+=("- Validating [JAVASCRIPT(standard)] files in code base...") + else + PRINT_ARRAY+=("- Excluding [JAVASCRIPT(standard)] files in code base...") + fi + if [[ "$VALIDATE_TYPESCRIPT_ES" == "true" ]]; then + PRINT_ARRAY+=("- Validating [TYPESCRIPT(eslint)] files in code base...") + else + PRINT_ARRAY+=("- Excluding [TYPESCRIPT(eslint)] files in code base...") + fi + if [[ "$VALIDATE_TYPESCRIPT_STANDARD" == "true" ]]; then + PRINT_ARRAY+=("- Validating [TYPESCRIPT(standard)] files in code base...") + else + PRINT_ARRAY+=("- Excluding [TYPESCRIPT(standard)] files in code base...") + fi + if [[ "$VALIDATE_DOCKER" == "true" ]]; then + PRINT_ARRAY+=("- Validating [DOCKER] files in code base...") + else + PRINT_ARRAY+=("- Excluding [DOCKER] files in code base...") + fi + if [[ "$VALIDATE_GO" == "true" ]]; then + PRINT_ARRAY+=("- Validating [GOLANG] files in code base...") + else + PRINT_ARRAY+=("- Excluding [GOLANG] files in code base...") + fi + if [[ "$VALIDATE_TERRAFORM" == "true" ]]; then + PRINT_ARRAY+=("- Validating [TERRAFORM] files in code base...") + else + PRINT_ARRAY+=("- Excluding [TERRAFORM] files in code base...") + fi + if [[ "$VALIDATE_POWERSHELL" == "true" ]]; then + PRINT_ARRAY+=("- Validating [POWERSHELL] files in code base...") + else + PRINT_ARRAY+=("- Excluding [POWERSHELL] files in code base...") + fi + if [[ "$VALIDATE_CSS" == "true" ]]; then + PRINT_ARRAY+=("- Validating [CSS] files in code base...") + else + PRINT_ARRAY+=("- Excluding [CSS] files in code base...") + fi + if [[ "$VALIDATE_CLOJURE" == "true" ]]; then + PRINT_ARRAY+=("- Validating [CLOJURE] files in code base...") + else + PRINT_ARRAY+=("- Excluding [CLOJURE] files in code base...") + fi + if [[ "$VALIDATE_ENV" == "true" ]]; then + PRINT_ARRAY+=("- Validating [ENV] files in code base...") + else + PRINT_ARRAY+=("- Excluding [ENV] files in code base...") + fi + if [[ "$VALIDATE_KOTLIN" == "true" ]]; then + PRINT_ARRAY+=("- Validating [KOTLIN] files in code base...") + else + PRINT_ARRAY+=("- Excluding [KOTLIN] files in code base...") + fi + if [[ "$VALIDATE_OPENAPI" == "true" ]]; then + PRINT_ARRAY+=("- Validating [OPENAPI] files in code base...") + else + PRINT_ARRAY+=("- Excluding [OPENAPI] files in code base...") + fi + if [[ "$VALIDATE_PROTOBUF" == "true" ]]; then + PRINT_ARRAY+=("- Validating [PROTOBUF] files in code base...") + else + PRINT_ARRAY+=("- Excluding [PROTOBUF] files in code base...") + fi + + ############################## + # Validate Ansible Directory # + ############################## + if [ -z "$ANSIBLE_DIRECTORY" ]; then + # No Value, need to default + ANSIBLE_DIRECTORY="$DEFAULT_ANSIBLE_DIRECTORY" + else + # Check if first char is '/' + if [[ ${ANSIBLE_DIRECTORY:0:1} == "/" ]]; then + # Remove first char + ANSIBLE_DIRECTORY="${ANSIBLE_DIRECTORY:1}" + fi + # Need to give it full path + TEMP_ANSIBLE_DIRECTORY="$GITHUB_WORKSPACE/$ANSIBLE_DIRECTORY" + # Set the value + ANSIBLE_DIRECTORY="$TEMP_ANSIBLE_DIRECTORY" + fi + + ############################### + # Get the disable errors flag # + ############################### + if [ -z "$DISABLE_ERRORS" ]; then + ################################## + # No flag passed, set to default # + ################################## + DISABLE_ERRORS="$DEFAULT_DISABLE_ERRORS" + fi + + ############################### + # Convert string to lowercase # + ############################### + DISABLE_ERRORS=$(echo "$DISABLE_ERRORS" | awk '{print tolower($0)}') + + ############################ + # Set to false if not true # + ############################ + if [ "$DISABLE_ERRORS" != "true" ]; then + DISABLE_ERRORS="false" + fi + + ############################ + # Get the run verbose flag # + ############################ + if [ -z "$ACTIONS_RUNNER_DEBUG" ]; then + ################################## + # No flag passed, set to default # + ################################## + ACTIONS_RUNNER_DEBUG="$DEFAULT_ACTIONS_RUNNER_DEBUG" + fi + + ############################### + # Convert string to lowercase # + ############################### + ACTIONS_RUNNER_DEBUG=$(echo "$ACTIONS_RUNNER_DEBUG" | awk '{print tolower($0)}') + + ############################ + # Set to true if not false # + ############################ + if [ "$ACTIONS_RUNNER_DEBUG" != "false" ]; then + ACTIONS_RUNNER_DEBUG="true" + fi + + ################### + # Debug on runner # + ################### + if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then + ########################### + # Print the validate info # + ########################### + for LINE in "${PRINT_ARRAY[@]}" + do + echo "$LINE" + done + + echo "--- DEBUG INFO ---" + echo "---------------------------------------------" + RUNNER=$(whoami) + echo "Runner:[$RUNNER]" + echo "ENV:" + printenv + echo "---------------------------------------------" + fi +} diff --git a/lib/worker.sh b/lib/worker.sh new file mode 100755 index 00000000..1b879768 --- /dev/null +++ b/lib/worker.sh @@ -0,0 +1,658 @@ +#!/usr/bin/env bash + +################################################################################ +################################################################################ +########### Super-Linter linting Functions @admiralawkbar ###################### +################################################################################ +################################################################################ +########################## FUNCTION CALLS BELOW ################################ +################################################################################ +################################################################################ +#### Function LintCodebase ##################################################### +function LintCodebase() +{ + #################### + # Pull in the vars # + #################### + FILE_TYPE="$1" && shift # Pull the variable and remove from array path (Example: JSON) + LINTER_NAME="$1" && shift # Pull the variable and remove from array path (Example: jsonlint) + LINTER_COMMAND="$1" && shift # Pull the variable and remove from array path (Example: jsonlint -c ConfigFile /path/to/file) + FILE_EXTENSIONS="$1" && shift # Pull the variable and remove from array path (Example: *.json) + FILE_ARRAY=("$@") # Array of files to validate (Example: $FILE_ARRAY_JSON) + + ###################### + # Create Print Array # + ###################### + PRINT_ARRAY=() + + ################ + # print header # + ################ + PRINT_ARRAY+=("") + PRINT_ARRAY+=("----------------------------------------------") + PRINT_ARRAY+=("----------------------------------------------") + PRINT_ARRAY+=("Linting [$FILE_TYPE] files...") + PRINT_ARRAY+=("----------------------------------------------") + PRINT_ARRAY+=("----------------------------------------------") + + ####################################### + # Validate we have jsonlint installed # + ####################################### + # shellcheck disable=SC2230 + VALIDATE_INSTALL_CMD=$(command -v "$LINTER_NAME" 2>&1) + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + # Failed + echo "ERROR! Failed to find [$LINTER_NAME] in system!" + echo "ERROR:[$VALIDATE_INSTALL_CMD]" + exit 1 + else + # Success + if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then + echo "Successfully found binary in system" + echo "Location:[$VALIDATE_INSTALL_CMD]" + fi + fi + + ########################## + # Initialize empty Array # + ########################## + LIST_FILES=() + + ################ + # Set the flag # + ################ + SKIP_FLAG=0 + + ############################################################ + # Check to see if we need to go through array or all files # + ############################################################ + if [ ${#FILE_ARRAY[@]} -eq 0 ] && [ "$VALIDATE_ALL_CODEBASE" == "false" ]; then + # No files found in commit and user has asked to not validate code base + SKIP_FLAG=1 + # echo " - No files found in changeset to lint for language:[$FILE_TYPE]" + elif [ ${#FILE_ARRAY[@]} -ne 0 ]; then + # We have files added to array of files to check + LIST_FILES=("${FILE_ARRAY[@]}") # Copy the array into list + else + ############################################################################### + # Set the file seperator to newline to allow for grabbing objects with spaces # + ############################################################################### + IFS=$'\n' + + ################################# + # Get list of all files to lint # + ################################# + # shellcheck disable=SC2207,SC2086 + LIST_FILES=($(cd "$GITHUB_WORKSPACE" || exit; find . -type f -regex "$FILE_EXTENSIONS" 2>&1)) + + ########################### + # Set IFS back to default # + ########################### + IFS="$DEFAULT_IFS" + + ############################################################ + # Set it back to empty if loaded with blanks from scanning # + ############################################################ + if [ ${#LIST_FILES[@]} -lt 1 ]; then + ###################### + # Set to empty array # + ###################### + LIST_FILES=() + ############################# + # Skip as we found no files # + ############################# + SKIP_FLAG=1 + fi + fi + + ############################### + # Check if any data was found # + ############################### + if [ $SKIP_FLAG -eq 0 ]; then + ###################### + # Print Header array # + ###################### + for LINE in "${PRINT_ARRAY[@]}" + do + ######################### + # Print the header info # + ######################### + echo "$LINE" + done + + ################## + # Lint the files # + ################## + for FILE in "${LIST_FILES[@]}" + do + ##################### + # Get the file name # + ##################### + FILE_NAME=$(basename "$FILE" 2>&1) + + ##################################################### + # Make sure we dont lint node modules or test cases # + ##################################################### + if [[ $FILE == *"node_modules"* ]]; then + # This is a node modules file + continue + elif [[ $FILE == *"$TEST_CASE_FOLDER"* ]]; then + # This is the test cases, we should always skip + continue + fi + + ############## + # File print # + ############## + echo "---------------------------" + echo "File:[$FILE]" + + #################### + # Set the base Var # + #################### + LINT_CMD='' + + ####################################### + # Corner case for Powershell subshell # + ####################################### + if [[ "$FILE_TYPE" == "POWERSHELL" ]]; then + ################################ + # Lint the file with the rules # + ################################ + # Need to append "'" to make the pwsh call syntax correct, also exit with exit code from inner subshell + LINT_CMD=$(cd "$GITHUB_WORKSPACE" || exit; $LINTER_COMMAND "$FILE"; exit $? 2>&1) + else + ################################ + # Lint the file with the rules # + ################################ + LINT_CMD=$(cd "$GITHUB_WORKSPACE" || exit; $LINTER_COMMAND "$FILE" 2>&1) + fi + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + ######### + # Error # + ######### + echo "ERROR! Found errors in [$LINTER_NAME] linter!" + echo "ERROR:[$LINT_CMD]" + # Increment the error count + (("ERRORS_FOUND_$FILE_TYPE++")) + else + ########### + # Success # + ########### + echo " - File:[$FILE_NAME] was linted with [$LINTER_NAME] successfully" + fi + done + fi +} +################################################################################ +#### Function TestCodebase ##################################################### +function TestCodebase() +{ + #################### + # Pull in the vars # + #################### + FILE_TYPE="$1" # Pull the variable and remove from array path (Example: JSON) + LINTER_NAME="$2" # Pull the variable and remove from array path (Example: jsonlint) + LINTER_COMMAND="$3" # Pull the variable and remove from array path (Example: jsonlint -c ConfigFile /path/to/file) + FILE_EXTENSIONS="$4" # Pull the variable and remove from array path (Example: *.json) + + ################ + # print header # + ################ + echo "" + echo "----------------------------------------------" + echo "----------------------------------------------" + echo "Testing Codebase [$FILE_TYPE] files..." + echo "----------------------------------------------" + echo "----------------------------------------------" + echo "" + + ##################################### + # Validate we have linter installed # + ##################################### + # shellcheck disable=SC2230 + VALIDATE_INSTALL_CMD=$(command -v "$LINTER_NAME" 2>&1) + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + # Failed + echo "ERROR! Failed to find [$LINTER_NAME] in system!" + echo "ERROR:[$VALIDATE_INSTALL_CMD]" + exit 1 + else + # Success + echo "Successfully found binary in system" + echo "Location:[$VALIDATE_INSTALL_CMD]" + fi + + ########################## + # Initialize empty Array # + ########################## + LIST_FILES=() + + ############################################ + # Check if its ansible, as its the outlier # + ############################################ + if [[ "$FILE_TYPE" == "ANSIBLE" ]]; then + ################################# + # Get list of all files to lint # + ################################# + # shellcheck disable=SC2207,SC2086,SC2010 + LIST_FILES=($(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER" || exit; ls ansible/ | grep ".yml" 2>&1)) + else + ############################################################################### + # Set the file seperator to newline to allow for grabbing objects with spaces # + ############################################################################### + IFS=$'\n' + + ################################# + # Get list of all files to lint # + ################################# + # shellcheck disable=SC2207,SC2086 + LIST_FILES=($(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER" || exit; find . -type f -regex "$FILE_EXTENSIONS" ! -path "*./ansible*" 2>&1)) + + ########################### + # Set IFS back to default # + ########################### + IFS="$DEFAULT_IFS" + fi + + ################## + # Lint the files # + ################## + for FILE in "${LIST_FILES[@]}" + do + ##################### + # Get the file name # + ##################### + FILE_NAME=$(basename "$FILE" 2>&1) + + ############################ + # Get the file pass status # + ############################ + # Example: markdown_good_1.md -> good + FILE_STATUS=$(echo "$FILE_NAME" |cut -f2 -d'_') + + ######################################################### + # If not found, assume it should be linted successfully # + ######################################################### + if [ -z "$FILE_STATUS" ] || [[ "$FILE" == *"README"* ]]; then + ################################## + # Set to good for proper linting # + ################################## + FILE_STATUS="good" + fi + + ############## + # File print # + ############## + echo "---------------------------" + echo "File:[$FILE]" + + ######################## + # Set the lint command # + ######################## + LINT_CMD='' + + ####################################### + # Check if docker and get folder name # + ####################################### + if [[ "$FILE_TYPE" == "DOCKER" ]]; then + if [[ "$FILE" == *"good"* ]]; then + ############# + # Good file # + ############# + FILE_STATUS='good' + else + ############ + # Bad file # + ############ + FILE_STATUS='bad' + fi + fi + + ##################### + # Check for ansible # + ##################### + if [[ "$FILE_TYPE" == "ANSIBLE" ]]; then + ######################################## + # Make sure we dont lint certain files # + ######################################## + if [[ $FILE == *"vault.yml"* ]] || [[ $FILE == *"galaxy.yml"* ]]; then + # This is a file we dont look at + continue + fi + + ################################ + # Lint the file with the rules # + ################################ + LINT_CMD=$(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER/ansible" || exit; $LINTER_COMMAND "$FILE" 2>&1) + elif [[ "$FILE_TYPE" == "POWERSHELL" ]]; then + ################################ + # Lint the file with the rules # + ################################ + # Need to append "'" to make the pwsh call syntax correct, also exit with exit code from inner subshell + LINT_CMD=$(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER" || exit; $LINTER_COMMAND "$FILE"; exit $? 2>&1) + else + ################################ + # Lint the file with the rules # + ################################ + LINT_CMD=$(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER" || exit; $LINTER_COMMAND "$FILE" 2>&1) + fi + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ######################################## + # Check for if it was supposed to pass # + ######################################## + if [[ "$FILE_STATUS" == "good" ]]; then + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + ######### + # Error # + ######### + echo "ERROR! Found errors in [$LINTER_NAME] linter!" + echo "ERROR:[$LINT_CMD]" + echo "ERROR: Linter CMD:[$LINTER_COMMAND $FILE]" + # Increment the error count + (("ERRORS_FOUND_$FILE_TYPE++")) + else + ########### + # Success # + ########### + echo " - File:[$FILE_NAME] was linted with [$LINTER_NAME] successfully" + fi + else + ####################################### + # File status = bad, this should fail # + ####################################### + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -eq 0 ]; then + ######### + # Error # + ######### + echo "ERROR! Found errors in [$LINTER_NAME] linter!" + echo "ERROR! This file should have failed test case!" + echo "ERROR:[$LINT_CMD]" + echo "ERROR: Linter CMD:[$LINTER_COMMAND $FILE]" + # Increment the error count + (("ERRORS_FOUND_$FILE_TYPE++")) + else + ########### + # Success # + ########### + echo " - File:[$FILE_NAME] failed test case with [$LINTER_NAME] successfully" + fi + fi + done +} +################################################################################ +#### Function RunTestCases ##################################################### +function RunTestCases() +{ + # This loop will run the test cases and exclude user code + # This is called from the automation process to validate new code + # When a PR is opened, the new code is validated with the default branch + # version of linter.sh, and a new container is built with the latest codebase + # for testing. That container is spun up, and ran, + # with the flag: TEST_CASE_RUN=true + # So that the new code can be validated against the test cases + + ################# + # Header prints # + ################# + echo "" + echo "----------------------------------------------" + echo "-------------- TEST CASE RUN -----------------" + echo "----------------------------------------------" + echo "" + + ####################### + # Test case languages # + ####################### + TestCodebase "YML" "yamllint" "yamllint -c $YAML_LINTER_RULES" ".*\.\(yml\|yaml\)\$" + TestCodebase "JSON" "jsonlint" "jsonlint" ".*\.\(json\)\$" + TestCodebase "XML" "xmllint" "xmllint" ".*\.\(xml\)\$" + TestCodebase "MARKDOWN" "markdownlint" "markdownlint -c $MD_LINTER_RULES" ".*\.\(md\)\$" + TestCodebase "BASH" "shellcheck" "shellcheck" ".*\.\(sh\)\$" + TestCodebase "PYTHON" "pylint" "pylint --rcfile $PYTHON_LINTER_RULES" ".*\.\(py\)\$" + TestCodebase "PERL" "perl" "perl -Mstrict -cw" ".*\.\(pl\)\$" + TestCodebase "PHP" "php" "php -l" ".*\.\(php\)\$" + TestCodebase "RUBY" "rubocop" "rubocop -c $RUBY_LINTER_RULES" ".*\.\(rb\)\$" + TestCodebase "GO" "golangci-lint" "golangci-lint run -c $GO_LINTER_RULES" ".*\.\(go\)\$" + TestCodebase "COFFEESCRIPT" "coffeelint" "coffeelint -f $COFFEESCRIPT_LINTER_RULES" ".*\.\(coffee\)\$" + TestCodebase "JAVASCRIPT_ES" "eslint" "eslint --no-eslintrc -c $JAVASCRIPT_LINTER_RULES" ".*\.\(js\)\$" + TestCodebase "JAVASCRIPT_STANDARD" "standard" "standard $JAVASCRIPT_STANDARD_LINTER_RULES" ".*\.\(js\)\$" + TestCodebase "TYPESCRIPT_ES" "eslint" "eslint --no-eslintrc -c $TYPESCRIPT_LINTER_RULES" ".*\.\(ts\)\$" + TestCodebase "TYPESCRIPT_STANDARD" "standard" "standard --parser @typescript-eslint/parser --plugin @typescript-eslint/eslint-plugin $TYPESCRIPT_STANDARD_LINTER_RULES" ".*\.\(ts\)\$" + TestCodebase "DOCKER" "/dockerfilelint/bin/dockerfilelint" "/dockerfilelint/bin/dockerfilelint -c $DOCKER_LINTER_RULES" ".*\(Dockerfile\)\$" + TestCodebase "ANSIBLE" "ansible-lint" "ansible-lint -v -c $ANSIBLE_LINTER_RULES" "ansible-lint" + TestCodebase "TERRAFORM" "tflint" "tflint -c $TERRAFORM_LINTER_RULES" ".*\.\(tf\)\$" + TestCodebase "POWERSHELL" "pwsh" "pwsh -c Invoke-ScriptAnalyzer -EnableExit -Settings $POWERSHELL_LINTER_RULES -Path" ".*\.\(ps1\|psm1\|psd1\|ps1xml\|pssc\|psrc\|cdxml\)\$" + TestCodebase "CSS" "stylelint" "stylelint --config $CSS_LINTER_RULES" ".*\.\(css\)\$" + TestCodebase "ENV" "dotenv-linter" "dotenv-linter" ".*\.\(env\)\$" + TestCodebase "CLOJURE" "clj-kondo" "clj-kondo --config $CLOJURE_LINTER_RULES --lint" ".*\.\(clj\|cljs\|cljc\|edn\)\$" + TestCodebase "KOTLIN" "ktlint" "ktlint" ".*\.\(kt\|kts\)\$" + TestCodebase "PROTOBUF" "protolint" "protolint lint --config_path $PROTOBUF_LINTER_RULES" ".*\.\(proto\)\$" + TestCodebase "OPENAPI" "spectral" "spectral lint -r $OPENAPI_LINTER_RULES" ".*\.\(ymlopenapi\|jsonopenapi\)\$" + + ################# + # Footer prints # + ################# + # Call the footer to display run information + # and exit with error code + Footer +} +################################################################################ +#### Function LintAnsibleFiles ################################################# +function LintAnsibleFiles() +{ + ###################### + # Create Print Array # + ###################### + PRINT_ARRAY=() + + ################ + # print header # + ################ + PRINT_ARRAY+=("") + PRINT_ARRAY+=("----------------------------------------------") + PRINT_ARRAY+=("----------------------------------------------") + PRINT_ARRAY+=("Linting [Ansible] files...") + PRINT_ARRAY+=("----------------------------------------------") + PRINT_ARRAY+=("----------------------------------------------") + + ###################### + # Name of the linter # + ###################### + LINTER_NAME="ansible-lint" + + ########################################### + # Validate we have ansible-lint installed # + ########################################### + # shellcheck disable=SC2230 + VALIDATE_INSTALL_CMD=$(command -v "$LINTER_NAME" 2>&1) + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + # Failed + echo "ERROR! Failed to find $LINTER_NAME in system!" + echo "ERROR:[$VALIDATE_INSTALL_CMD]" + exit 1 + else + # Success + if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then + # Success + echo "Successfully found binary in system" + echo "Location:[$VALIDATE_INSTALL_CMD]" + fi + fi + + ########################## + # Initialize empty Array # + ########################## + LIST_FILES=() + + ####################### + # Create flag to skip # + ####################### + SKIP_FLAG=0 + + ###################################################### + # Only go into ansible linter if we have base folder # + ###################################################### + if [ -d "$ANSIBLE_DIRECTORY" ]; then + + ############################################################ + # Check to see if we need to go through array or all files # + ############################################################ + if [ "$VALIDATE_ALL_CODEBASE" == "false" ]; then + # We need to only check the ansible playbooks that have updates + #LIST_FILES=("${ANSIBLE_ARRAY[@]}") + # shellcheck disable=SC2164,SC2010,SC2207 + LIST_FILES=($(cd "$ANSIBLE_DIRECTORY"; ls | grep ".yml" 2>&1)) + else + ################################# + # Get list of all files to lint # + ################################# + # shellcheck disable=SC2164,SC2010,SC2207 + LIST_FILES=($(cd "$ANSIBLE_DIRECTORY"; ls | grep ".yml" 2>&1)) + fi + + ############################################################### + # Set the list to empty if only MD and TXT files were changed # + ############################################################### + # No need to run the full ansible checks on read only file changes + if [ "$READ_ONLY_CHANGE_FLAG" -eq 0 ]; then + ########################## + # Set the array to empty # + ########################## + LIST_FILES=() + ################################### + # Send message that were skipping # + ################################### + #echo "- Skipping Ansible lint run as file(s) that were modified were read only..." + ############################ + # Create flag to skip loop # + ############################ + SKIP_FLAG=1 + fi + + #################################### + # Check if we have data to look at # + #################################### + if [ $SKIP_FLAG -eq 0 ]; then + for LINE in "${PRINT_ARRAY[@]}" + do + ######################### + # Print the header line # + ######################### + echo "$LINE" + done + fi + + ################## + # Lint the files # + ################## + for FILE in "${LIST_FILES[@]}" + do + + ######################################## + # Make sure we dont lint certain files # + ######################################## + if [[ $FILE == *"vault.yml"* ]] || [[ $FILE == *"galaxy.yml"* ]]; then + # This is a file we dont look at + continue + fi + + #################### + # Get the filename # + #################### + FILE_NAME=$(basename "$ANSIBLE_DIRECTORY/$FILE" 2>&1) + + ############## + # File print # + ############## + echo "---------------------------" + echo "File:[$FILE]" + + ################################ + # Lint the file with the rules # + ################################ + LINT_CMD=$("$LINTER_NAME" -v -c "$ANSIBLE_LINTER_RULES" "$ANSIBLE_DIRECTORY/$FILE" 2>&1) + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + ######### + # Error # + ######### + echo "ERROR! Found errors in [$LINTER_NAME] linter!" + echo "ERROR:[$LINT_CMD]" + # Increment error count + ((ERRORS_FOUND_ANSIBLE++)) + else + ########### + # Success # + ########### + echo " - File:[$FILE_NAME] was linted with [$LINTER_NAME] successfully" + fi + done + else # No ansible directory found in path + ############################### + # Check to see if debug is on # + ############################### + if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then + ######################## + # No Ansible dir found # + ######################## + echo "WARN! No Ansible base directory found at:[$ANSIBLE_DIRECTORY]" + echo "skipping ansible lint" + fi + fi +} From b90310f2a81c7f30922aa223175d139459da28a6 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 10:06:12 -0500 Subject: [PATCH 27/38] fix path --- lib/linter.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/linter.sh b/lib/linter.sh index f1dad6ec..d7644d38 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -10,8 +10,7 @@ ######################### # Source Function Files # ######################### -source ./validation.sh # Source the main validation function script -source ./buildFileList.sh # Source the main build file list function script +source /action/lib/*.sh # Source the function script(s) ########### # GLOBALS # From 679ecf3b1ae9138b2ef4f50e381be61b5d2deb17 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 10:15:39 -0500 Subject: [PATCH 28/38] lets be literal --- lib/linter.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/linter.sh b/lib/linter.sh index d7644d38..6b4e6761 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -10,7 +10,9 @@ ######################### # Source Function Files # ######################### -source /action/lib/*.sh # Source the function script(s) +source /action/lib/buildFileList.sh # Source the function script(s) +source /action/lib/validation.sh # Source the function script(s) +source /action/lib/worker.sh # Source the function script(s) ########### # GLOBALS # From 4194a7a611ec2c8e5fba1014e22e56c5045fb261 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 10:34:23 -0500 Subject: [PATCH 29/38] cleaner --- lib/buildFileList.sh | 1 + lib/linter.sh | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/buildFileList.sh b/lib/buildFileList.sh index 2ed2ce60..d39f7a64 100755 --- a/lib/buildFileList.sh +++ b/lib/buildFileList.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck disable=SC2034 ################################################################################ ################################################################################ diff --git a/lib/linter.sh b/lib/linter.sh index 6b4e6761..b69ee0b7 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# shellcheck disable=SC1003,SC2016 +# shellcheck disable=SC1003,SC2016,SC1091 ################################################################################ ################################################################################ @@ -126,23 +126,27 @@ DISABLE_ERRORS="${DISABLE_ERRORS}" # Boolean to enable warnin ############## # Debug Vars # ############## -RUN_LOCAL="${RUN_LOCAL}" # Boolean to see if we are running locally -ACTIONS_RUNNER_DEBUG="${ACTIONS_RUNNER_DEBUG}" # Boolean to see even more info (debug) +RUN_LOCAL="${RUN_LOCAL}" # Boolean to see if we are running locally +ACTIONS_RUNNER_DEBUG="${ACTIONS_RUNNER_DEBUG:-false}" # Boolean to see even more info (debug) ################ # Default Vars # ################ DEFAULT_VALIDATE_ALL_CODEBASE='true' # Default value for validate all files DEFAULT_WORKSPACE="${DEFAULT_WORKSPACE:-/tmp/lint}" # Default workspace if running locally -DEFAULT_ANSIBLE_DIRECTORY="$GITHUB_WORKSPACE/ansible" # Default Ansible Directory DEFAULT_RUN_LOCAL='false' # Default value for debugging locally DEFAULT_TEST_CASE_RUN='false' # Flag to tell code to run only test cases -DEFAULT_ACTIONS_RUNNER_DEBUG='false' # Default value for debugging output +DEFAULT_IFS="$IFS" # Get the Default IFS for updating + +############################################################### +# Default Vars that are called in Subs and need to be ignored # +############################################################### +# shellcheck disable=SC2034 +DEFAULT_DISABLE_ERRORS='false' # Default to enabling errors RAW_FILE_ARRAY=() # Array of all files that were changed READ_ONLY_CHANGE_FLAG=0 # Flag set to 1 if files changed are not txt or md TEST_CASE_FOLDER='.automation/test' # Folder for test cases we should always ignore -DEFAULT_DISABLE_ERRORS='false' # Default to enabling errors -DEFAULT_IFS="$IFS" # Get the Default IFS for updating +DEFAULT_ANSIBLE_DIRECTORY="$GITHUB_WORKSPACE/ansible" # Default Ansible Directory ########################## # Array of changed files # @@ -966,7 +970,7 @@ if [ "$VALIDATE_KOTLIN" == "true" ]; then # Lint the Kotlin files # ####################### # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY" - LintCodebase "KOTLIN" "ktlint" "ktlint" ".*\.\(kt\|kts\)\$" "${FILE_ARRAY_ENV[@]}" + LintCodebase "KOTLIN" "ktlint" "ktlint" ".*\.\(kt\|kts\)\$" "${FILE_ARRAY_KOTLIN[@]}" fi ################## From 06e237d19eb09fd468648379da4f836be09817b4 Mon Sep 17 00:00:00 2001 From: Thomas Hughes Date: Mon, 29 Jun 2020 10:39:09 -0500 Subject: [PATCH 30/38] Add actions-tagger workflow to automate version tags --- .github/workflows/versioning.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/workflows/versioning.yml diff --git a/.github/workflows/versioning.yml b/.github/workflows/versioning.yml new file mode 100644 index 00000000..f25ee032 --- /dev/null +++ b/.github/workflows/versioning.yml @@ -0,0 +1,15 @@ +name: Keep the versions up-to-date + +on: + release: + types: [published, edited] + +jobs: + actions-tagger: + runs-on: windows-latest + steps: + - uses: Actions-R-Us/actions-tagger@latest + env: + GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" + with: + - publish_latest_tag: true From 4ccd6b35c969a717613e027707c606b537d1a271 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 10:43:52 -0500 Subject: [PATCH 31/38] fixed bug --- lib/linter.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/linter.sh b/lib/linter.sh index b69ee0b7..dea50fab 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -143,9 +143,13 @@ DEFAULT_IFS="$IFS" # Get the Default IFS for updating ############################################################### # shellcheck disable=SC2034 DEFAULT_DISABLE_ERRORS='false' # Default to enabling errors +# shellcheck disable=SC2034 RAW_FILE_ARRAY=() # Array of all files that were changed +# shellcheck disable=SC2034 READ_ONLY_CHANGE_FLAG=0 # Flag set to 1 if files changed are not txt or md +# shellcheck disable=SC2034 TEST_CASE_FOLDER='.automation/test' # Folder for test cases we should always ignore +# shellcheck disable=SC2034 DEFAULT_ANSIBLE_DIRECTORY="$GITHUB_WORKSPACE/ansible" # Default Ansible Directory ########################## @@ -603,6 +607,11 @@ Footer() ########################### ERROR_COUNTER="ERRORS_FOUND_$LANGUAGE" + ################################# + # Fix Integer Expression errors # + ################################# + ERROR_COUNTER="${ERROR_COUNTER/.*}" + ################## # Print if not 0 # ################## From 37a37f09300e356952821bf7308475231ed81019 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 11:17:25 -0500 Subject: [PATCH 32/38] adding readme --- lib/README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 lib/README.md diff --git a/lib/README.md b/lib/README.md new file mode 100644 index 00000000..500e2142 --- /dev/null +++ b/lib/README.md @@ -0,0 +1,15 @@ +# Super-Linter Library + +## Main script +The file `linter.sh` is the main script that is called for the process and loads all other scripts as functions. + +## Functions +The additional files in the folder are functions to help streamline the main build process and allow for easier maintenance. +- `possum.sh` + - Official mascot of the **Super-Linter** +- `buildFileList.sh` + - Functions to help find files that were modified, or need to be scanned +- `validation.sh` + - Logic to see what linters are enabled +- `worker.sh` + - Calls to the various linters and the test cases for each From 2067571f1f37af8ee2a584498377b01715505fa0 Mon Sep 17 00:00:00 2001 From: Thomas Hughes Date: Mon, 29 Jun 2020 11:36:18 -0500 Subject: [PATCH 33/38] Fix versioning.yml --- .github/workflows/versioning.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/versioning.yml b/.github/workflows/versioning.yml index f25ee032..01039a30 100644 --- a/.github/workflows/versioning.yml +++ b/.github/workflows/versioning.yml @@ -8,8 +8,9 @@ jobs: actions-tagger: runs-on: windows-latest steps: + - uses: actions/checkout@v2 - uses: Actions-R-Us/actions-tagger@latest + with: + publish_latest_tag: true env: GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" - with: - - publish_latest_tag: true From 42124d07366541aa6f9eb50a9a21f4b3bbad16b8 Mon Sep 17 00:00:00 2001 From: Thomas Hughes Date: Mon, 29 Jun 2020 11:42:18 -0500 Subject: [PATCH 34/38] Update to v3 --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index aaaf494f..57f97859 100644 --- a/README.md +++ b/README.md @@ -111,20 +111,20 @@ jobs: # Checkout the code base # ########################## - name: Checkout Code - uses: actions/checkout@v2 + uses: actions/checkout@v3 ################################ # Run Linter against code base # ################################ - name: Lint Code Base - uses: docker://github/super-linter:v2.2.0 + uses: docker://github/super-linter:v3 env: VALIDATE_ALL_CODEBASE: false VALIDATE_ANSIBLE: false ... ``` -**NOTE:** Using the line:`uses: docker://github/super-linter:v2.2.0` will pull the image down from **DockerHub** and run the **GitHub Super-Linter**. Using the line: `uses: github/super-linter@v2.2.0` will build and compile the **GitHub Super-Linter** at build time. This can be far more costly in time... +**NOTE:** Using the line:`uses: docker://github/super-linter:v3` will pull the image down from **DockerHub** and run the **GitHub Super-Linter**. Using the line: `uses: github/super-linter@v3` will build and compile the **GitHub Super-Linter** at build time. This can be far more costly in time... ## Environment variables The super-linter allows you to pass the following `ENV` variables to be able to trigger different functionality. From cbf576073b5df7ff886f3051a942460121b0de8a Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 11:48:07 -0500 Subject: [PATCH 35/38] fixed typo --- .automation/upload-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.automation/upload-docker.sh b/.automation/upload-docker.sh index 12c15635..fe1414d3 100755 --- a/.automation/upload-docker.sh +++ b/.automation/upload-docker.sh @@ -149,7 +149,7 @@ ValidateInput() # Need to see if GPR registry and update name # ############################################### if [[ "$REGISTRY" == "GPR" ]]; then - NAME="docker.pkg.github/$IMAGE_REPO" + NAME="docker.pkg.github.com/$IMAGE_REPO" IMAGE_REPO="$NAME" echo "Updated [IMAGE_REPO] to:[$IMAGE_REPO] for GPR" fi From 4671142e0651b5f33747d6bde65cba6f3cb8cbb5 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 12:39:10 -0500 Subject: [PATCH 36/38] adding logic to deploy major tag versions as well Co-authored-by: Thomas Hughes Co-authored-by: John Wiebalk --- .automation/upload-docker.sh | 81 +++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/.automation/upload-docker.sh b/.automation/upload-docker.sh index fe1414d3..eefb5c50 100755 --- a/.automation/upload-docker.sh +++ b/.automation/upload-docker.sh @@ -26,6 +26,8 @@ REGISTRY="${REGISTRY}" # What registry to upload | or &1 + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + # ERROR + echo "ERROR! failed to [tag] Dockerfile!" + exit 1 + else + # SUCCESS + echo "Successfully tagged image!" + fi fi } ################################################################################ @@ -328,7 +378,7 @@ UploadImage() exit 1 else # SUCCESS - echo "Successfully Uploaded Docker image to $REGISTRY!" + echo "Successfully Uploaded Docker image:[$IMAGE_VERSION] to $REGISTRY!" fi ######################### @@ -371,6 +421,33 @@ UploadImage() echo "Size:[$SIZE]" echo "----------------------------------------------" fi + + ############################################################### + # Check if we need to upload the major tagged version as well # + ############################################################### + if [ $UPDATE_MAJOR_TAG -eq 1 ]; then + ############################################ + # Upload the docker image that was created # + ############################################ + docker push "$IMAGE_REPO:$MAJOR_TAG" 2>&1 + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + # ERROR + echo "ERROR! failed to [upload] MAJOR_TAG:[$MAJOR_TAG] Dockerfile!" + exit 1 + else + # SUCCESS + echo "Successfully Uploaded TAGOR_TAG:[$MAJOR_TAG] Docker image to $REGISTRY!" + fi + fi } ################################################################################ #### Function Footer ########################################################### From efb491d6cf17d956aed94dfa66e726e8e82305f2 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 12:52:36 -0500 Subject: [PATCH 37/38] fixed protobuf var Co-authored-by: Thomas Hughes Co-authored-by: John Wiebalk --- lib/linter.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/linter.sh b/lib/linter.sh index dea50fab..1981e71a 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -85,7 +85,7 @@ LINTER_ARRAY=("jsonlint" "yamllint" "xmllint" "markdownlint" "shellcheck" LANGUAGE_ARRAY=('YML' 'JSON' 'XML' 'MARKDOWN' 'BASH' 'PERL' 'PHP' 'RUBY' 'PYTHON' 'COFFEESCRIPT' 'ANSIBLE' 'JAVASCRIPT_STANDARD' 'JAVASCRIPT_ES' 'TYPESCRIPT_STANDARD' 'TYPESCRIPT_ES' 'DOCKER' 'GO' 'TERRAFORM' - 'CSS' 'ENV' 'POWERSHELL' 'KOTLIN' 'PROTO' 'CLOJURE' 'OPENAPI') + 'CSS' 'ENV' 'POWERSHELL' 'KOTLIN' 'PROTOBUF' 'CLOJURE' 'OPENAPI') ################### # GitHub ENV Vars # @@ -607,11 +607,6 @@ Footer() ########################### ERROR_COUNTER="ERRORS_FOUND_$LANGUAGE" - ################################# - # Fix Integer Expression errors # - ################################# - ERROR_COUNTER="${ERROR_COUNTER/.*}" - ################## # Print if not 0 # ################## From 03b6b69c40a665df5c6085fc7124244f9c1bcdb6 Mon Sep 17 00:00:00 2001 From: Mark Bond <34947848+bondms@users.noreply.github.com> Date: Mon, 29 Jun 2020 19:13:22 +0100 Subject: [PATCH 38/38] Fix checkout action version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 57f97859..330cfa7a 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ jobs: # Checkout the code base # ########################## - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@v2 ################################ # Run Linter against code base #