From 79286a45041274ee4f668e54e7748343ff0505f2 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 14:10:13 +0000 Subject: [PATCH 01/65] Fix: Add .gitattributes file to make development Windows-Friendly --- .gitattributes | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..5dc46e6b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +* text=auto eol=lf +*.{cmd,[cC][mM][dD]} text eol=crlf +*.{bat,[bB][aA][tT]} text eol=crlf \ No newline at end of file From 66983a8706d0a7e389b61ff45be51f89da383d78 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 14:10:28 +0000 Subject: [PATCH 02/65] Fix: Add devcontainer.json to allow vscode users to develop in container directly from repository --- .devcontainer/devcontainer.json | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..b470e0aa --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,34 @@ +// For format details, see https://aka.ms/vscode-remote/devcontainer.json or this file's README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.123.0/containers/docker-existing-dockerfile +{ + "name": "SUPER-LINTER", + + // Sets the run context to one level up instead of the .devcontainer folder. + "context": "..", + + // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. + "dockerFile": "..\\Dockerfile", + + // Set *default* container specific settings.json values on container create. + "settings": { + "terminal.integrated.shell.linux": null + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [] + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Uncomment the next line to run commands after the container is created - for example installing curl. + // "postCreateCommand": "apt-get update && apt-get install -y curl", + + // Uncomment when using a ptrace-based debugger like C++, Go, and Rust + // "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], + + // Uncomment to use the Docker CLI from inside the container. See https://aka.ms/vscode-remote/samples/docker-from-docker. + // "mounts": [ "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind" ], + + // Uncomment to connect as a non-root user. See https://aka.ms/vscode-remote/containers/non-root. + // "remoteUser": "vscode" +} From 7d95b48a05b60d7807d40090fae30a18e848f3de Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 15:09:52 +0000 Subject: [PATCH 03/65] Fix: Add proper shell and bash debugging extension --- .devcontainer/devcontainer.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b470e0aa..1d430c89 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -11,11 +11,13 @@ // Set *default* container specific settings.json values on container create. "settings": { - "terminal.integrated.shell.linux": null + "terminal.integrated.shell.linux": "/bin/ash" }, // Add the IDs of extensions you want installed when the container is created. - "extensions": [] + "extensions": [ + "rogalmic.bash-debug" + ] // Use 'forwardPorts' to make a list of ports inside the container available locally. // "forwardPorts": [], From a2cf5c643727fcefa88513fb90ffd8c2235e76f6 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 16:13:32 +0000 Subject: [PATCH 04/65] Feat: Add Launch.json for bash debugging --- .vscode/launch.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..744b2145 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,14 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "bashdb", + "request": "launch", + "name": "Bash-Debug (simplest configuration)", + "program": "${file}" + } + ] +} \ No newline at end of file From 9ce962cf6990997008a825274a55964dfa3f04f8 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 16:53:52 +0000 Subject: [PATCH 05/65] Feat: Automatically link automation test cases to /tmp/lint by default --- .devcontainer/devcontainer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 1d430c89..48a0b66a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -17,13 +17,13 @@ // Add the IDs of extensions you want installed when the container is created. "extensions": [ "rogalmic.bash-debug" - ] + ], // Use 'forwardPorts' to make a list of ports inside the container available locally. // "forwardPorts": [], // Uncomment the next line to run commands after the container is created - for example installing curl. - // "postCreateCommand": "apt-get update && apt-get install -y curl", + "postCreateCommand": "ln -s /workspaces/super-linter/.automation/test /tmp/lint", // Uncomment when using a ptrace-based debugger like C++, Go, and Rust // "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], From 8b47945eea851f9f1e55df5e8bfb65268923f098 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 16:54:23 +0000 Subject: [PATCH 06/65] Updated Launch.json --- .vscode/launch.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 744b2145..7e128dc6 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -7,8 +7,12 @@ { "type": "bashdb", "request": "launch", - "name": "Bash-Debug (simplest configuration)", - "program": "${file}" + "name": "Test Linter", + "program": ".vscode/testlinter.sh", + "internalConsoleOptions": "openOnSessionStart", + "presentation": { + "group": "aLinter" + } } ] } \ No newline at end of file From 7f23223ed5178db913b53a8c60e90069f933816a Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 16:54:36 +0000 Subject: [PATCH 07/65] Fix: Add testlinter file to force run_local --- .vscode/testlinter.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .vscode/testlinter.sh diff --git a/.vscode/testlinter.sh b/.vscode/testlinter.sh new file mode 100644 index 00000000..464c420e --- /dev/null +++ b/.vscode/testlinter.sh @@ -0,0 +1,3 @@ +#!/bin/bash +export RUN_LOCAL=true +/workspaces/super-linter/lib/linter.sh \ No newline at end of file From 90a1f525b04a6ffde616805c902e8f6a37217e38 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 17:31:53 +0000 Subject: [PATCH 08/65] Feat: Make GITHUB_WORKSPACE overridable in RUN_LOCAL Mode --- lib/linter.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/linter.sh b/lib/linter.sh index 473f53cd..6a81206f 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -579,6 +579,14 @@ GetGitHubVars() ########################################## echo "NOTE: ENV VAR [RUN_LOCAL] has been set to:[true]" echo "bypassing GitHub Actions variables..." + + ############################ + # Set the GITHUB_WORKSPACE # + ############################ + if [ -z "$GITHUB_WORKSPACE" ]; then + GITHUB_WORKSPACE="$DEFAULT_WORKSPACE" + fi + echo "Linting all files in mapped directory:[$DEFAULT_WORKSPACE]" # No need to touch or set the GITHUB_SHA @@ -586,10 +594,6 @@ GetGitHubVars() # No need to touch or set the GITHUB_ORG # No need to touch or set the GITHUB_REPO - ############################ - # Set the GITHUB_WORKSPACE # - ############################ - GITHUB_WORKSPACE="$DEFAULT_WORKSPACE" ################################# # Set the VALIDATE_ALL_CODEBASE # From 9bce2fecb9052370d918bed33b16ea40583a8803 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 17:32:09 +0000 Subject: [PATCH 09/65] Minor refinements --- .devcontainer/devcontainer.json | 2 +- .vscode/testlinter.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 48a0b66a..b3259b1f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -23,7 +23,7 @@ // "forwardPorts": [], // Uncomment the next line to run commands after the container is created - for example installing curl. - "postCreateCommand": "ln -s /workspaces/super-linter/.automation/test /tmp/lint", + //"postCreateCommand": "ln -s /workspaces/super-linter/.automation/test /tmp/lint", // Uncomment when using a ptrace-based debugger like C++, Go, and Rust // "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], diff --git a/.vscode/testlinter.sh b/.vscode/testlinter.sh index 464c420e..1bc56ad8 100644 --- a/.vscode/testlinter.sh +++ b/.vscode/testlinter.sh @@ -1,3 +1,5 @@ #!/bin/bash +#At this point you can use the debug console to add export GITHUB_WORKSPACE=/path to test only a specific folder +#You can also use ln -s /path /tmp/lint as an alternative export RUN_LOCAL=true /workspaces/super-linter/lib/linter.sh \ No newline at end of file From fdc003eb9101a6c1e3dd5d51169b0cbfacde567f Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 17:48:17 +0000 Subject: [PATCH 10/65] Feat: Add Github Pull Requests Extension --- .devcontainer/devcontainer.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b3259b1f..03dde16a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -16,6 +16,7 @@ // Add the IDs of extensions you want installed when the container is created. "extensions": [ + "github.vscode-pull-request-github", "rogalmic.bash-debug" ], From 5064b0c67aa4fb981fe4693d573ce8258ad75af4 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 17:48:39 +0000 Subject: [PATCH 11/65] Fix: Default Linter to automation tests --- .vscode/launch.json | 1 + .vscode/testlinter.sh | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 7e128dc6..a02e962c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,6 +9,7 @@ "request": "launch", "name": "Test Linter", "program": ".vscode/testlinter.sh", + "cwd": "${workspaceFolder}", "internalConsoleOptions": "openOnSessionStart", "presentation": { "group": "aLinter" diff --git a/.vscode/testlinter.sh b/.vscode/testlinter.sh index 1bc56ad8..40502987 100644 --- a/.vscode/testlinter.sh +++ b/.vscode/testlinter.sh @@ -1,5 +1,11 @@ #!/bin/bash #At this point you can use the debug console to add export GITHUB_WORKSPACE=/path to test only a specific folder #You can also use ln -s /path /tmp/lint as an alternative +#If you do neither, this will default to running against the test automation files +tmppath=/tmp/lint +if [ ! -L tmppath ]; then + ln -s $PWD/.automation/test $tmppath +fi + export RUN_LOCAL=true -/workspaces/super-linter/lib/linter.sh \ No newline at end of file +source $PWD/lib/linter.sh \ No newline at end of file From 574bef0b3cf0a707a4b44008242d7d939b709aad Mon Sep 17 00:00:00 2001 From: goedelsoup Date: Fri, 19 Jun 2020 14:30:28 -0400 Subject: [PATCH 12/65] Add support for Clojure with clj-kondo --- .automation/test/clojure/README.md | 13 +++++ .automation/test/clojure/clojure_bad_1.clj | 64 +++++++++++++++++++++ .automation/test/clojure/clojure_good_1.clj | 35 +++++++++++ Dockerfile | 9 +++ README.md | 2 + lib/linter.sh | 58 ++++++++++++++++++- 6 files changed, 178 insertions(+), 3 deletions(-) create mode 100644 .automation/test/clojure/README.md create mode 100644 .automation/test/clojure/clojure_bad_1.clj create mode 100644 .automation/test/clojure/clojure_good_1.clj diff --git a/.automation/test/clojure/README.md b/.automation/test/clojure/README.md new file mode 100644 index 00000000..6606ef5b --- /dev/null +++ b/.automation/test/clojure/README.md @@ -0,0 +1,13 @@ +# Clojure Test Cases +This folder holds the test cases for **Clojure**. + +## Additional Docs +No Additional information is needed for this test case. + +## Good Test Cases +The test cases denoted: `LANGUAGE_good_FILE.EXTENSION` are all valid, and should pass successfully when linted. +- **Note:** They are linted utilizing the default linter rules. + +## Bad Test Cases +The test cases denoted: `LANGUAGE_bad_FILE.EXTENSION` are **NOT** valid, and should trigger errors when linted. +- **Note:** They are linted utilizing the default linter rules. diff --git a/.automation/test/clojure/clojure_bad_1.clj b/.automation/test/clojure/clojure_bad_1.clj new file mode 100644 index 00000000..c8366066 --- /dev/null +++ b/.automation/test/clojure/clojure_bad_1.clj @@ -0,0 +1,64 @@ +(ns foo + (:require + [clojure.string :as str] + ;; We're never using this namespace. Also, the namespaces aren't sorted. + [clojure.set :as set])) + +;; Here we made a typo, so the symbol is unresolved: +(but-last [1 2 3]) + +;; Clj-kondo knows about arities of clojure namespaces, but you can also teach +;; it about your libraries or own namespaces +(str/join) + +;; foo has an arity of 2, but we're not actually using y +(defn foo-fn [x y] + ;; this do is redundant: + (do + ;; this is handy for debugging, but please remove it before pushing your code: + (def tmp_x x) + (let [y (fn [] (inc x))] + ;; the next let can be squashed together with the previous: + (let [z y] + ;; whoopsy, calling a local function with an incorrect number of args: + (y x) + ;; also wrong: + (recur))))) + +(letfn + [(f [] (h 1)) + (h [] (f 1))]) + +(defn- private-fn []) +;; redefining it... +(defn- private-fn []) + +(defn foo [] :foo) +;; Type error, because foo doesn't return a number! +(inc (foo)) + +;; I'm tired now, let's sleep... +;; Oops, not happening because of wrong amount of args: +(Thread/sleep 1000 1 2) + +;; Here we switch to another namespace and require the previous: +(ns bar (:require [foo :as f])) + +;; Wrong arity when calling a function from the previous namespace: +(f/foo-fn) + +;; private: +(f/private-fn) + +;; this won't pass the reader: +{:a 1 :a 2} +;; and neither will this: +#{1 1} +;; nor this: +{:a 1 :b} + +(ns bar-test (:require [clojure.test :as t])) + +(t/deftest my-tests + ;; you're not actually testing something here: + (odd? (inc 1))) \ No newline at end of file diff --git a/.automation/test/clojure/clojure_good_1.clj b/.automation/test/clojure/clojure_good_1.clj new file mode 100644 index 00000000..7ea739e1 --- /dev/null +++ b/.automation/test/clojure/clojure_good_1.clj @@ -0,0 +1,35 @@ +(ns foo + (:require + [clojure.string :as str])) + +(butlast [1 2 3]) + +(str/join "" "") + +(defn foo-fn [x] + (let [y (fn [] (inc x)) + z y] + (y))) + +(letfn + [(f [g] (h g)) + (h [i] (f i))]) + +(defn foo [] 1) +(inc (foo)) + +(Thread/sleep 1000 1) + +;; Here we switch to another namespace and require the previous: +(ns bar (:require [foo :as f])) + +(f/foo-fn 1) + +{:a 1 :b 2} +#{1 2} +{:a 1 :b 2} + +(ns bar-test (:require [clojure.test :as t])) + +(t/deftest my-tests + (t/is (odd? (inc 1)))) \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 7f655c31..f66fb596 100644 --- a/Dockerfile +++ b/Dockerfile @@ -102,6 +102,14 @@ RUN wget -O- -nvq https://raw.githubusercontent.com/golangci/golangci-lint/maste RUN curl -Ls "$(curl -Ls https://api.github.com/repos/terraform-linters/tflint/releases/latest | grep -o -E "https://.+?_linux_amd64.zip")" -o tflint.zip && unzip tflint.zip && rm tflint.zip \ && mv "tflint" /usr/bin/ +##################### +# Install clj-kondo # +##################### +RUN curl -sLO https://github.com/borkdude/clj-kondo/releases/download/v2020.06.12/clj-kondo-2020.06.12-linux-static-amd64.zip \ + && unzip clj-kondo-2020.06.12-linux-static-amd64.zip \ + && rm clj-kondo-2020.06.12-linux-static-amd64.zip \ + && mv clj-kondo /usr/bin/ + ########################################### # Load GitHub Env Vars for GitHub Actions # ########################################### @@ -128,6 +136,7 @@ ENV GITHUB_SHA=${GITHUB_SHA} \ VALIDATE_GO=${VALIDATE_GO} \ VALIDATE_TERRAFORM=${VALIDATE_TERRAFORM} \ VALIDATE_CSS=${VALIDATE_CSS} \ + VALIDATE_CLOJURE=${VALIDATE_CLOJURE} \ ANSIBLE_DIRECTORY=${ANSIBLE_DIRECTORY} \ RUN_LOCAL=${RUN_LOCAL} \ TEST_CASE_RUN=${TEST_CASE_RUN} \ diff --git a/README.md b/README.md index b7f5455d..4a001385 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Developers on **GitHub** can call the **GitHub Action** to lint their code base | --- | --- | | **Ansible** | [ansible-lint](https://github.com/ansible/ansible-lint) | | **CSS** | [stylelint](https://stylelint.io/) | +| **Clojure** | [clj-kondo](https://github.com/borkdude/clj-kondo) | | **CoffeeScript** | [coffeelint](https://coffeelint.github.io/) | | **Dockerfile** | [dockerfilelint](https://github.com/replicatedhq/dockerfilelint.git) | | **Golang** | [golangci-lint](https://github.com/golangci/golangci-lint) | @@ -135,6 +136,7 @@ and won't run anything unexpected. | **VALIDATE_GO** | `true` | Flag to enable or disable the linting process of the language. | | **VALIDATE_TERRAFORM** | `true` | Flag to enable or disable the linting process of the language. | | **VALIDATE_CSS** | `true` | Flag to enable or disable the linting process of the language. | +| **VALIDATE_CLOJURE** | `true` | Flag to enable or disable the linting process of the language. | | **ANSIBLE_DIRECTORY** | `/ansible` | Flag to set the root directory for Ansible file location(s). | | **ACTIONS_RUNNER_DEBUG** | `false` | Flag to enable additional information about the linter, versions, and additional output. | | **DEFAULT_WORKSPACE** | `/tmp/lint` | The location containing files to lint if you are running locally. | diff --git a/lib/linter.sh b/lib/linter.sh index b96f44ec..c1c46b7a 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -51,6 +51,9 @@ TERRAFORM_LINTER_RULES="$DEFAULT_RULES_LOCATION/$TERRAFORM_FILE_NAME" # Path # CSS Vars CSS_FILE_NAME='.stylelintrc.json' # Name of the file CSS_LINTER_RULES="$DEFAULT_RULES_LOCATION/$CSS_FILE_NAME" # Path to the CSS lint rules +# Clojure Vars +CLOJURE_FILE_NAME='.clj-kondo/config.edn' +CLOJURE_LINTER_RULES="$DEFAULT_RULES_LOCATION/$CLOJURE_FILE_NAME" ####################################### @@ -59,14 +62,15 @@ CSS_LINTER_RULES="$DEFAULT_RULES_LOCATION/$CSS_FILE_NAME" # Path to th LINTER_ARRAY=("jsonlint" "yamllint" "xmllint" "markdownlint" "shellcheck" "pylint" "perl" "rubocop" "coffeelint" "eslint" "standard" "ansible-lint" "/dockerfilelint/bin/dockerfilelint" "golangci-lint" "tflint" - "stylelint") + "stylelint" "clj-kondo") ############################# # Language array for prints # ############################# LANGUAGE_ARRAY=('YML' 'JSON' 'XML' 'MARKDOWN' 'BASH' 'PERL' 'RUBY' 'PYTHON' 'COFFEESCRIPT' 'ANSIBLE' 'JAVASCRIPT_STANDARD' 'JAVASCRIPT_ES' - 'TYPESCRIPT_STANDARD' 'TYPESCRIPT_ES' 'DOCKER' 'GO' 'TERRAFORM' 'CSS') + 'TYPESCRIPT_STANDARD' 'TYPESCRIPT_ES' 'DOCKER' 'GO' 'TERRAFORM' 'CSS' + 'CLOJURE') ################### # GitHub ENV Vars # @@ -95,6 +99,7 @@ VALIDATE_DOCKER="${VALIDATE_DOCKER}" # Boolean to validate lang VALIDATE_GO="${VALIDATE_GO}" # Boolean to validate language VALIDATE_TERRAFORM="${VALIDATE_TERRAFORM}" # Boolean to validate language VALIDATE_CSS="${VALIDATE_CSS}" # Boolean to validate language +VALIDATE_CLOJURE="${VALIDATE_CLOJURE}" # Boolean to validate language TEST_CASE_RUN="${TEST_CASE_RUN}" # Boolean to validate only test cases ############## @@ -136,6 +141,7 @@ FILE_ARRAY_DOCKER=() # Array of files to check FILE_ARRAY_GO=() # Array of files to check FILE_ARRAY_TERRAFORM=() # Array of files to check FILE_ARRAY_CSS=() # Array of files to check +FILE_ARRAAY_CLOJURE=() # Array of files to check ############ # Counters # @@ -158,6 +164,7 @@ ERRORS_FOUND_DOCKER=0 # Count of errors found ERRORS_FOUND_GO=0 # Count of errors found ERRORS_FOUND_TERRAFORM=0 # Count of errors found ERRORS_FOUND_CSS=0 # Count of errors found +ERRORS_FOUND_CLOJURE=0 # Count of errors found ################################################################################ ########################## FUNCTIONS BELOW ##################################### @@ -735,6 +742,7 @@ GetValidationInfo() VALIDATE_GO=$(echo "$VALIDATE_GO" | awk '{print tolower($0)}') VALIDATE_TERRAFORM=$(echo "$VALIDATE_TERRAFORM" | awk '{print tolower($0)}') VALIDATE_CSS=$(echo "$VALIDATE_CSS" | awk '{print tolower($0)') + VALIDATE_CLOJURE=$(echo "$VALIDATE_CLOJURE" | awk '{print tolower($0)') ################################################ # Determine if any linters were explicitly set # @@ -757,7 +765,8 @@ GetValidationInfo() -n "$VALIDATE_DOCKER" || \ -n "$VALIDATE_GO" || \ -n "$VALIDATE_TERRAFORM" || \ - -n "$VALIDATE_CSS" ]]; then + -n "$VALIDATE_CSS" || \ + -n "$VALIDATE_CLOJURE" ]]; then ANY_SET="true" fi @@ -1013,6 +1022,20 @@ GetValidationInfo() VALIDATE_CSS="true" fi + ####################################### + # Validate if we should check Clojure # + ####################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_CLOJURE" ]]; then + # Clojure flag was not set - default to false + VALIDATE_CLOJURE="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_CLOJURE="true" + fi + ####################################### # Print which linters we are enabling # ####################################### @@ -1106,6 +1129,11 @@ GetValidationInfo() else PRINT_ARRAY+=("- Excluding [CSS] files in code base...") fi + if [[ "$VALIDATE_CLOJURE" == "true" ]]; then + PRINT_ARRAY+=("- Validating [CLOJURE] files in code base...") + else + PRINT_ARRAY+=("- Excluding [CLOJURE] files in code base...") + fi ############################## # Validate Ansible Directory # @@ -1431,6 +1459,15 @@ BuildFileList() # Set the READ_ONLY_CHANGE_FLAG since this could be exec # ########################################################## READ_ONLY_CHANGE_FLAG=1 + elif [ "$FILE" == "clj" ] || [ "$FILE" == "cljs" ] || [ "$FILE" == "cljc" ] || [ "$FILE" == "edn" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_CLOJURE+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 else ############################################## # Use file to see if we can parse what it is # @@ -1986,6 +2023,7 @@ RunTestCases() TestCodebase "ANSIBLE" "ansible-lint" "ansible-lint -v -c $ANSIBLE_LINTER_RULES" "ansible-lint" TestCodebase "TERRAFORM" "tflint" "tflint -c $TERRAFORM_LINTER_RULES" ".*\.\(tf\)\$" TestCodebase "CSS" "stylelint" "stylelint --config $CSS_LINTER_RULES" ".*\.\(css\)\$" + TestCodebase "CLOJURE" "clj-kondo" "clj-kondo --config $CLOJURE_LINTER_RULES" ".*\.\(clj\)\$" ################# # Footer prints # @@ -2282,6 +2320,20 @@ if [ "$VALIDATE_DOCKER" == "true" ]; then LintCodebase "DOCKER" "/dockerfilelint/bin/dockerfilelint" "/dockerfilelint/bin/dockerfilelint" ".*\(Dockerfile\)\$" "${FILE_ARRAY_DOCKER[@]}" fi +################### +# CLOJURE LINTING # +################### +if [ "$VALIDATE_CLOJURE" == "true" ]; then + ################################# + # Get Clojure standard rules # + ################################# + GetStandardRules "clj-kondo" + ######################### + # Lint the Clojure files # + ######################### + LintCodebase "CLOJURE" "clj-kondo" "clj-kondo --config $CLOJURE_LINTER_RULES" ".*\.\(clj\)\$" "${FILE_ARRAY_CLOJURE[@]}" +fi + ########## # Footer # ########## From 69b98486a3645a62d769dd34389072e0338e660d Mon Sep 17 00:00:00 2001 From: goedelsoup Date: Fri, 19 Jun 2020 14:31:50 -0400 Subject: [PATCH 13/65] fix typo in variable name --- lib/linter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/linter.sh b/lib/linter.sh index c1c46b7a..53828113 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -141,7 +141,7 @@ FILE_ARRAY_DOCKER=() # Array of files to check FILE_ARRAY_GO=() # Array of files to check FILE_ARRAY_TERRAFORM=() # Array of files to check FILE_ARRAY_CSS=() # Array of files to check -FILE_ARRAAY_CLOJURE=() # Array of files to check +FILE_ARRAY_CLOJURE=() # Array of files to check ############ # Counters # From 8a5be8d448427c01a2eb38aec156fab7333fa7cf Mon Sep 17 00:00:00 2001 From: goedelsoup Date: Fri, 19 Jun 2020 14:34:53 -0400 Subject: [PATCH 14/65] make clj-kondo version a build arg --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index f66fb596..2529279a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -105,9 +105,10 @@ RUN curl -Ls "$(curl -Ls https://api.github.com/repos/terraform-linters/tflint/r ##################### # Install clj-kondo # ##################### -RUN curl -sLO https://github.com/borkdude/clj-kondo/releases/download/v2020.06.12/clj-kondo-2020.06.12-linux-static-amd64.zip \ - && unzip clj-kondo-2020.06.12-linux-static-amd64.zip \ - && rm clj-kondo-2020.06.12-linux-static-amd64.zip \ +ARG CLJ_KONDO_VERSION='2020.06.12' +RUN curl -sLO https://github.com/borkdude/clj-kondo/releases/download/v${CLJ_KONDO_VERSION}/clj-kondo-${CLJ_KONDO_VERSION}-linux-static-amd64.zip \ + && unzip clj-kondo-${CLJ_KONDO_VERSION}-linux-static-amd64.zip \ + && rm clj-kondo-${CLJ_KONDO_VERSION}-linux-static-amd64.zip \ && mv clj-kondo /usr/bin/ ########################################### From 211a22c329eaea54e13e9757e40c9d6bf1813262 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 12:03:30 -0700 Subject: [PATCH 15/65] Fix: Change ash to bash for default Terminal Co-authored-by: Antoine Leblanc --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 03dde16a..72190b10 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -11,7 +11,7 @@ // Set *default* container specific settings.json values on container create. "settings": { - "terminal.integrated.shell.linux": "/bin/ash" + "terminal.integrated.shell.linux": "/bin/bash" }, // Add the IDs of extensions you want installed when the container is created. From 524ebed9cde62baf1a2cfbcf3fba39c708878964 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Fri, 19 Jun 2020 21:14:17 +0000 Subject: [PATCH 16/65] Fix tmppath detection --- .vscode/testlinter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/testlinter.sh b/.vscode/testlinter.sh index 40502987..f7e7eb79 100644 --- a/.vscode/testlinter.sh +++ b/.vscode/testlinter.sh @@ -3,7 +3,7 @@ #You can also use ln -s /path /tmp/lint as an alternative #If you do neither, this will default to running against the test automation files tmppath=/tmp/lint -if [ ! -L tmppath ]; then +if [ ! -L $tmppath ]; then ln -s $PWD/.automation/test $tmppath fi From 5063d7cb6c03e3aa8c7031b5b7e3bdb46e63e4f7 Mon Sep 17 00:00:00 2001 From: goedelsoup Date: Mon, 22 Jun 2020 08:39:32 -0400 Subject: [PATCH 17/65] fix clojure file regex --- lib/linter.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/linter.sh b/lib/linter.sh index 82a3ebaf..e4c7d218 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -2088,7 +2088,7 @@ RunTestCases() TestCodebase "TERRAFORM" "tflint" "tflint -c $TERRAFORM_LINTER_RULES" ".*\.\(tf\)\$" TestCodebase "CSS" "stylelint" "stylelint --config $CSS_LINTER_RULES" ".*\.\(css\)\$" TestCodebase "ENV" "dotenv-linter" "dotenv-linter" ".*\.\(env\)\$" - TestCodebase "CLOJURE" "clj-kondo" "clj-kondo --config $CLOJURE_LINTER_RULES" ".*\.\(clj\)\$" + TestCodebase "CLOJURE" "clj-kondo" "clj-kondo --config $CLOJURE_LINTER_RULES" ".*\.\(clj\|cljs\|cljc\|edn\)\$" ################# # Footer prints # @@ -2407,7 +2407,7 @@ if [ "$VALIDATE_CLOJURE" == "true" ]; then ######################### # Lint the Clojure files # ######################### - LintCodebase "CLOJURE" "clj-kondo" "clj-kondo --config $CLOJURE_LINTER_RULES" ".*\.\(clj\)\$" "${FILE_ARRAY_CLOJURE[@]}" + LintCodebase "CLOJURE" "clj-kondo" "clj-kondo --config $CLOJURE_LINTER_RULES" ".*\.\(clj\|cljs\|cljc\|edn\)\$" "${FILE_ARRAY_CLOJURE[@]}" fi ########## From 7a298710713bbbe52efc450161aa7770ccc59fb8 Mon Sep 17 00:00:00 2001 From: goedelsoup Date: Tue, 23 Jun 2020 09:43:29 -0400 Subject: [PATCH 18/65] add documentation to configuration --- .github/linters/.clj-kondo/config.edn | 2 ++ .gitignore | 3 +++ docs/disabling-linters.md | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+) create mode 100644 .github/linters/.clj-kondo/config.edn diff --git a/.github/linters/.clj-kondo/config.edn b/.github/linters/.clj-kondo/config.edn new file mode 100644 index 00000000..30cae4c6 --- /dev/null +++ b/.github/linters/.clj-kondo/config.edn @@ -0,0 +1,2 @@ +{:linters {:unresolved-symbol {:exclude [(compojure.api.sweet/defroutes)]} + :refer-all {:exclude [clj-time.jdbc]}}} diff --git a/.gitignore b/.gitignore index ad46b308..ae755b39 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,6 @@ typings/ # next.js build output .next + +# clj-kondo cache +.cache \ No newline at end of file diff --git a/docs/disabling-linters.md b/docs/disabling-linters.md index 0efb4d27..45fa6957 100644 --- a/docs/disabling-linters.md +++ b/docs/disabling-linters.md @@ -607,3 +607,22 @@ import package.b.* ### ktlint disable entire file - There is currently **No** way to disable rules inline of the file(s) + +-------------------------------------------------------------------------------- + +## Clojure +- [clj-kondo](https://github.com/borkdude/clj-kondo) + +### clj-kondo standard Config file +- `.github/linters/.clj-kondo/config.edn` + +### clj-kondo disable single line +- There is currently **No** way to disable rules in a single line + +### clj-kondo disable code block +- There is currently **No** way to disable rules in a code block + +### clj-kondo disable entire file +```clojure +{:output {:exclude-files ["path/to/file"]}} +``` From a52fd6a5fe44eb9dc13b729519847081665fcb4e Mon Sep 17 00:00:00 2001 From: goedelsoup Date: Tue, 23 Jun 2020 09:46:15 -0400 Subject: [PATCH 19/65] link to clj-kondo docs --- docs/disabling-linters.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/disabling-linters.md b/docs/disabling-linters.md index 45fa6957..5b11ee51 100644 --- a/docs/disabling-linters.md +++ b/docs/disabling-linters.md @@ -612,6 +612,7 @@ import package.b.* ## Clojure - [clj-kondo](https://github.com/borkdude/clj-kondo) +- Since clj-kondo approaches static analysis in a very Clojure way, it is advised to read the [configuration docs](https://github.com/borkdude/clj-kondo/blob/master/doc/config.md) ### clj-kondo standard Config file - `.github/linters/.clj-kondo/config.edn` From b922ece707b9a57c7ad42c9159fdd6bdbab7c217 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Tue, 23 Jun 2020 12:45:57 -0500 Subject: [PATCH 20/65] found missing var --- lib/linter.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/linter.sh b/lib/linter.sh index 5c8042bd..5f7af0c9 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -150,7 +150,7 @@ FILE_ARRAY_TYPESCRIPT_STANDARD=() # Array of files to check FILE_ARRAY_DOCKER=() # Array of files to check FILE_ARRAY_GO=() # Array of files to check FILE_ARRAY_TERRAFORM=() # Array of files to check -FILE_ARRAY_POWERSHELL=() # Array of files to check +FILE_ARRAY_POWERSHELL=() # Array of files to check FILE_ARRAY_CSS=() # Array of files to check FILE_ARRAY_ENV=() # Array of files to check FILE_ARRAY_CLOJURE=() # Array of files to check @@ -2159,6 +2159,7 @@ Footer() [ "$ERRORS_FOUND_RUBY" -ne 0 ] || \ [ "$ERRORS_FOUND_CSS" -ne 0 ] || \ [ "$ERRORS_FOUND_ENV" -ne 0 ] || \ + [ "$ERRORS_FOUND_CLOJURE" -ne 0 ] || \ [ "$ERRORS_FOUND_KOTLIN" -ne 0 ]; then # Failed exit echo "Exiting with errors found!" From e2eaf633277bc11514231f62dc9541fe47861740 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Wed, 24 Jun 2020 08:35:56 -0500 Subject: [PATCH 21/65] Adding config to templates --- TEMPLATES/.clj-kondo/config.edn | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 TEMPLATES/.clj-kondo/config.edn diff --git a/TEMPLATES/.clj-kondo/config.edn b/TEMPLATES/.clj-kondo/config.edn new file mode 100644 index 00000000..30cae4c6 --- /dev/null +++ b/TEMPLATES/.clj-kondo/config.edn @@ -0,0 +1,2 @@ +{:linters {:unresolved-symbol {:exclude [(compojure.api.sweet/defroutes)]} + :refer-all {:exclude [clj-time.jdbc]}}} From b26f7eda6d9c5f88cce367f34bad8bad8a78a8c4 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Wed, 24 Jun 2020 09:51:16 -0500 Subject: [PATCH 22/65] fixed command --- lib/linter.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/linter.sh b/lib/linter.sh index 74ea060a..39921b04 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -2221,7 +2221,7 @@ RunTestCases() TestCodebase "POWERSHELL" "pwsh" "pwsh -c Invoke-ScriptAnalyzer -EnableExit -Settings $POWERSHELL_LINTER_RULES -Path" ".*\.\(ps1\|psm1\|psd1\|ps1xml\|pssc\|psrc\|cdxml\)\$" TestCodebase "CSS" "stylelint" "stylelint --config $CSS_LINTER_RULES" ".*\.\(css\)\$" TestCodebase "ENV" "dotenv-linter" "dotenv-linter" ".*\.\(env\)\$" - TestCodebase "CLOJURE" "clj-kondo" "clj-kondo --config $CLOJURE_LINTER_RULES" ".*\.\(clj\|cljs\|cljc\|edn\)\$" + TestCodebase "CLOJURE" "clj-kondo" "clj-kondo --config $CLOJURE_LINTER_RULES --lint" ".*\.\(clj\|cljs\|cljc\|edn\)\$" TestCodebase "KOTLIN" "ktlint" "ktlint" ".*\.\(kt\|kts\)\$" ################# @@ -2565,7 +2565,7 @@ if [ "$VALIDATE_CLOJURE" == "true" ]; then ######################### # Lint the Clojure files # ######################### - LintCodebase "CLOJURE" "clj-kondo" "clj-kondo --config $CLOJURE_LINTER_RULES" ".*\.\(clj\|cljs\|cljc\|edn\)\$" "${FILE_ARRAY_CLOJURE[@]}" + LintCodebase "CLOJURE" "clj-kondo" "clj-kondo --config $CLOJURE_LINTER_RULES --lint" ".*\.\(clj\|cljs\|cljc\|edn\)\$" "${FILE_ARRAY_CLOJURE[@]}" fi ###################### From 6705cf967602f96f859fce0cfc75d8de77f98c66 Mon Sep 17 00:00:00 2001 From: Justin Grote Date: Wed, 24 Jun 2020 14:19:24 -0700 Subject: [PATCH 23/65] Docs: Add note about vscode --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index f1dfb044..a768514a 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,13 @@ Below are a list of the known limitations for the **GitHub Super-Linter**: ## How to contribute If you would like to help contribute to this **GitHub** Action, please see [CONTRIBUTING](https://github.com/github/super-linter/blob/master/.github/CONTRIBUTING.md) +### Visual Studio Code +You can checkout this repository using [Container Remote Development](https://code.visualstudio.com/docs/remote/containers), and debug the linter using the `Test Linter` task. +![Example](https://user-images.githubusercontent.com/15258962/85165778-2d2ce700-b21b-11ea-803e-3f6709d8e609.gif) + +We will also support [Github Codespaces](https://github.com/features/codespaces/) once it becomes available + + -------------------------------------------------------------------------------- ### License From eac560b35b45c3850f4aa1b6d9ed0e52a37ee9d4 Mon Sep 17 00:00:00 2001 From: Justin Kalland Date: Thu, 25 Jun 2020 08:13:19 +0200 Subject: [PATCH 24/65] Add support for OpenAPI --- .automation/test/openapi/README.md | 13 +++ .automation/test/openapi/openapi_bad_1.yml | 1 + .automation/test/openapi/openapi_bad_2.json | 3 + .automation/test/openapi/openapi_good_1.yml | 13 +++ .automation/test/openapi/openapi_good_2.json | 23 ++++++ Dockerfile | 4 +- README.md | 2 + TEMPLATES/.openapirc.yml | 9 +++ lib/linter.sh | 83 +++++++++++++++++++- 9 files changed, 146 insertions(+), 5 deletions(-) create mode 100644 .automation/test/openapi/README.md create mode 100644 .automation/test/openapi/openapi_bad_1.yml create mode 100644 .automation/test/openapi/openapi_bad_2.json create mode 100644 .automation/test/openapi/openapi_good_1.yml create mode 100644 .automation/test/openapi/openapi_good_2.json create mode 100644 TEMPLATES/.openapirc.yml diff --git a/.automation/test/openapi/README.md b/.automation/test/openapi/README.md new file mode 100644 index 00000000..8e77fdbf --- /dev/null +++ b/.automation/test/openapi/README.md @@ -0,0 +1,13 @@ +# OpenAPI Test Cases +This folder holds the test cases for **OpenAPI**. + +## Additional Docs +No Additional information is needed for this test case. + +## Good Test Cases +The test cases denoted: `LANGUAGE_good_FILE.EXTENSION` are all valid, and should pass successfully when linted. +- **Note:** They are linted utilizing the default linter rules. + +## Bad Test Cases +The test cases denoted: `LANGUAGE_bad_FILE.EXTENSION` are **NOT** valid, and should trigger errors when linted. +- **Note:** They are linted utilizing the default linter rules. diff --git a/.automation/test/openapi/openapi_bad_1.yml b/.automation/test/openapi/openapi_bad_1.yml new file mode 100644 index 00000000..6c86b1b4 --- /dev/null +++ b/.automation/test/openapi/openapi_bad_1.yml @@ -0,0 +1 @@ +openapi: '3.0.0' diff --git a/.automation/test/openapi/openapi_bad_2.json b/.automation/test/openapi/openapi_bad_2.json new file mode 100644 index 00000000..b0b97ddb --- /dev/null +++ b/.automation/test/openapi/openapi_bad_2.json @@ -0,0 +1,3 @@ +{ + "openapi": "3.0.0" +} diff --git a/.automation/test/openapi/openapi_good_1.yml b/.automation/test/openapi/openapi_good_1.yml new file mode 100644 index 00000000..eb4924a1 --- /dev/null +++ b/.automation/test/openapi/openapi_good_1.yml @@ -0,0 +1,13 @@ +openapi: 3.0.0 +info: + title: Example + version: '1.0' + contact: + name: Justin Kalland + email: justin@kalland.com + description: Test for super-linter +servers: + - url: 'http://localhost:3000' +paths: {} +tags: + - name: example diff --git a/.automation/test/openapi/openapi_good_2.json b/.automation/test/openapi/openapi_good_2.json new file mode 100644 index 00000000..93f59635 --- /dev/null +++ b/.automation/test/openapi/openapi_good_2.json @@ -0,0 +1,23 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "Example", + "version": "1.0", + "contact": { + "name": "Justin Kalland", + "email": "justin@kalland.com" + }, + "description": "Test for super-linter" + }, + "servers": [ + { + "url": "http://localhost:3000" + } + ], + "paths": {}, + "tags": [ + { + "name": "example" + } + ] +} diff --git a/Dockerfile b/Dockerfile index 49a5dbb1..b25a7b1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ RUN apk add --no-cache \ libxml2-utils perl \ ruby ruby-dev ruby-bundler ruby-rdoc make \ py3-setuptools ansible-lint \ - go \ + go \ openjdk8-jre \ php7 \ ca-certificates less ncurses-terminfo-base \ @@ -72,6 +72,7 @@ RUN npm config set package-lock false \ eslint-plugin-jest \ stylelint \ stylelint-config-standard \ + @stoplight/spectral \ && npm --no-cache install \ markdownlint-cli \ jsonlint prettyjson \ @@ -168,6 +169,7 @@ ENV GITHUB_SHA=${GITHUB_SHA} \ VALIDATE_ENV=${VALIDATE_ENV} \ VALIDATE_KOTLIN=${VALIDATE_KOTLIN} \ VALIDATE_POWERSHELL=${VALIDATE_POWERSHELL} \ + VALIDATE_OPENAPI=${VALIDATE_OPENAPI} \ ANSIBLE_DIRECTORY=${ANSIBLE_DIRECTORY} \ RUN_LOCAL=${RUN_LOCAL} \ TEST_CASE_RUN=${TEST_CASE_RUN} \ diff --git a/README.md b/README.md index 218a30d5..638ed28b 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ Developers on **GitHub** can call the **GitHub Action** to lint their code base | **PowerShell** | [PSScriptAnalyzer](https://github.com/PowerShell/Psscriptanalyzer) | | **ENV** | [dotenv-linter](https://github.com/dotenv-linter/dotenv-linter) | | **Kotlin** | [ktlint](https://github.com/pinterest/ktlint) | +| **OpenAPI** | [spectral](https://github.com/stoplightio/spectral) | ## How to use To use this **GitHub** Action you will need to complete the following: @@ -144,6 +145,7 @@ and won't run anything unexpected. | **VALIDATE_CSS** | `true` | Flag to enable or disable the linting process of the language. | | **VALIDATE_ENV** | `true` | Flag to enable or disable the linting process of the language. | | **VALIDATE_KOTLIN** | `true` | Flag to enable or disable the linting process of the language. | +| **VALIDATE_OPENAPI** | `true` | Flag to enable or disable the linting process of the language. | | **ANSIBLE_DIRECTORY** | `/ansible` | Flag to set the root directory for Ansible file location(s). | | **ACTIONS_RUNNER_DEBUG** | `false` | Flag to enable additional information about the linter, versions, and additional output. | | **DISABLE_ERRORS** | `false` | Flag to have the linter complete with exit code 0 even if errors were detected. | diff --git a/TEMPLATES/.openapirc.yml b/TEMPLATES/.openapirc.yml new file mode 100644 index 00000000..fdf641e1 --- /dev/null +++ b/TEMPLATES/.openapirc.yml @@ -0,0 +1,9 @@ +--- + +########################## +########################## +## OpenAPI Linter rules ## +########################## +########################## + +extends: spectral:oas diff --git a/lib/linter.sh b/lib/linter.sh index 9bad8c8b..7b90ae32 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -54,6 +54,9 @@ POWERSHELL_LINTER_RULES="$DEFAULT_RULES_LOCATION/$POWERSHELL_FILE_NAME" # Pat # CSS Vars CSS_FILE_NAME='.stylelintrc.json' # Name of the file CSS_LINTER_RULES="$DEFAULT_RULES_LOCATION/$CSS_FILE_NAME" # Path to the CSS lint rules +# OpenAPI Vars +OPENAPI_FILE_NAME='.openapirc.yml' # Name of the file +OPENAPI_LINTER_RULES="$DEFAULT_RULES_LOCATION/$OPENAPI_FILE_NAME" # Path to the OpenAPI lint rules ####################################### # Linter array for information prints # @@ -61,7 +64,7 @@ CSS_LINTER_RULES="$DEFAULT_RULES_LOCATION/$CSS_FILE_NAME" # Path to th LINTER_ARRAY=("jsonlint" "yamllint" "xmllint" "markdownlint" "shellcheck" "pylint" "perl" "rubocop" "coffeelint" "eslint" "standard" "ansible-lint" "/dockerfilelint/bin/dockerfilelint" "golangci-lint" "tflint" - "stylelint" "dotenv-linter" "powershell" "ktlint") + "stylelint" "dotenv-linter" "powershell" "ktlint" "spectral") ############################# # Language array for prints # @@ -69,7 +72,7 @@ LINTER_ARRAY=("jsonlint" "yamllint" "xmllint" "markdownlint" "shellcheck" LANGUAGE_ARRAY=('YML' 'JSON' 'XML' 'MARKDOWN' 'BASH' 'PERL' 'PHP' 'RUBY' 'PYTHON' 'COFFEESCRIPT' 'ANSIBLE' 'JAVASCRIPT_STANDARD' 'JAVASCRIPT_ES' 'TYPESCRIPT_STANDARD' 'TYPESCRIPT_ES' 'DOCKER' 'GO' 'TERRAFORM' - 'ENV' 'POWERSHELL' 'KOTLIN') + 'ENV' 'POWERSHELL' 'KOTLIN' 'OPENAPI') ################### # GitHub ENV Vars # @@ -102,6 +105,7 @@ VALIDATE_ENV="${VALIDATE_ENV}" # Boolean to validate lang VALIDATE_TERRAFORM="${VALIDATE_TERRAFORM}" # Boolean to validate language VALIDATE_POWERSHELL="${VALIDATE_POWERSHELL}" # Boolean to validate language VALIDATE_KOTLIN="${VALIDATE_KOTLIN}" # Boolean to validate language +VALIDATE_OPENAPI="${VALIDATE_OPENAPI}" # Boolean to validate language TEST_CASE_RUN="${TEST_CASE_RUN}" # Boolean to validate only test cases DISABLE_ERRORS="${DISABLE_ERRORS}" # Boolean to enable warning-only output without throwing errors @@ -149,6 +153,7 @@ FILE_ARRAY_POWERSHELL=() # Array of files to check FILE_ARRAY_CSS=() # Array of files to check FILE_ARRAY_ENV=() # Array of files to check FILE_ARRAY_KOTLIN=() # Array of files to check +FILE_ARRAY_OPENAPI=() # Array of files to check ############ # Counters # @@ -175,6 +180,7 @@ ERRORS_FOUND_POWERSHELL=0 # Count of errors found ERRORS_FOUND_CSS=0 # Count of errors found ERRORS_FOUND_ENV=0 # Count of errors found ERRORS_FOUND_KOTLIN=0 # Count of errors found +ERRORS_FOUND_OPENAPI=0 # Count of errors found ################################################################################ ########################## FUNCTIONS BELOW ##################################### @@ -556,6 +562,19 @@ LintAnsibleFiles() fi fi } + +################################################################################ +#### Function DetectOpenAPIFile ################################################ +DetectOpenAPIFile() +{ + egrep '"openapi":|"swagger":|^openapi:|^swagger:' $GITHUB_WORKSPACE/$1 > /dev/null + if [ $? -eq 0 ]; then + return 0 + else + return 1 + fi +} + ################################################################################ #### Function GetGitHubVars #################################################### GetGitHubVars() @@ -756,6 +775,7 @@ GetValidationInfo() VALIDATE_CSS=$(echo "$VALIDATE_CSS" | awk '{print tolower($0)}') VALIDATE_ENV=$(echo "$VALIDATE_ENV" | awk '{print tolower($0)}') VALIDATE_KOTLIN=$(echo "$VALIDATE_KOTLIN" | awk '{print tolower($0)}') + VALIDATE_OPENAPI=$(echo "$VALIDATE_OPENAPI" | awk '{print tolower($0)}') ################################################ # Determine if any linters were explicitly set # @@ -782,7 +802,8 @@ GetValidationInfo() -n "$VALIDATE_POWERSHELL" || \ -n "$VALIDATE_CSS" || \ -n "$VALIDATE_ENV" || \ - -n "$VALIDATE_KOTLIN" ]]; then + -n "$VALIDATE_KOTLIN" || \ + -n "$VALIDATE_OPENAPI" ]]; then ANY_SET="true" fi @@ -1094,6 +1115,19 @@ GetValidationInfo() VALIDATE_KOTLIN="true" fi + ####################################### + # Validate if we should check OPENAPI # + ####################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_OPENAPI" ]]; then + # OPENAPI flag was not set - default to false + VALIDATE_OPENAPI="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_OPENAPI="true" + fi ####################################### # Print which linters we are enabling # @@ -1208,6 +1242,11 @@ GetValidationInfo() else PRINT_ARRAY+=("- Excluding [KOTLIN] files in code base...") fi + if [[ "$VALIDATE_OPENAPI" == "true" ]]; then + PRINT_ARRAY+=("- Validating [OPENAPI] files in code base...") + else + PRINT_ARRAY+=("- Excluding [OPENAPI] files in code base...") + fi ############################## # Validate Ansible Directory # @@ -1391,6 +1430,12 @@ BuildFileList() # Append the file to the array # ################################ FILE_ARRAY_YML+=("$FILE") + ############################ + # Check if file is OpenAPI # + ############################ + if DetectOpenAPIFile $FILE; then + FILE_ARRAY_OPENAPI+=("$FILE") + fi ########################################################## # Set the READ_ONLY_CHANGE_FLAG since this could be exec # ########################################################## @@ -1403,6 +1448,12 @@ BuildFileList() # Append the file to the array # ################################ FILE_ARRAY_JSON+=("$FILE") + ############################ + # Check if file is OpenAPI # + ############################ + if DetectOpenAPIFile $FILE; then + FILE_ARRAY_OPENAPI+=("$FILE") + fi ########################################################## # Set the READ_ONLY_CHANGE_FLAG since this could be exec # ########################################################## @@ -2122,7 +2173,8 @@ Footer() [ "$ERRORS_FOUND_RUBY" -ne 0 ] || \ [ "$ERRORS_FOUND_CSS" -ne 0 ] || \ [ "$ERRORS_FOUND_ENV" -ne 0 ] || \ - [ "$ERRORS_FOUND_KOTLIN" -ne 0 ]; then + [ "$ERRORS_FOUND_KOTLIN" -ne 0 ] || \ + [ "$ERRORS_FOUND_OPENAPI" -ne 0 ]; then # Failed exit echo "Exiting with errors found!" exit 1 @@ -2184,6 +2236,7 @@ RunTestCases() TestCodebase "CSS" "stylelint" "stylelint --config $CSS_LINTER_RULES" ".*\.\(css\)\$" TestCodebase "ENV" "dotenv-linter" "dotenv-linter" ".*\.\(env\)\$" TestCodebase "KOTLIN" "ktlint" "ktlint" ".*\.\(kt\|kts\)\$" + TestCodebase "OPENAPI" "spectral" "spectral lint -r $OPENAPI_LINTER_RULES" ".*\.\(yml\|yaml\|json\)\$" ################# # Footer prints # @@ -2526,6 +2579,28 @@ if [ "$VALIDATE_POWERSHELL" == "true" ]; then LintCodebase "POWERSHELL" "pwsh" "pwsh -c Invoke-ScriptAnalyzer -EnableExit -Settings $POWERSHELL_LINTER_RULES -Path" ".*\.\(ps1\|psm1\|psd1\|ps1xml\|pssc\|psrc\|cdxml\)\$" "${FILE_ARRAY_POWERSHELL[@]}" fi +################### +# OPENAPI LINTING # +################### +if [ "$VALIDATE_OPENAPI" == "true" ]; then + # If we are validating all codebase we need to build file list because not every yml/json file is an OpenAPI file + if [ "$VALIDATE_ALL_CODEBASE" == "true" ]; then + LIST_FILES=($(cd "$GITHUB_WORKSPACE" || exit; find . -type f -regex ".*\.\(yml\|yaml\|json\)\$" 2>&1)) + for FILE in "${LIST_FILES[@]}" + do + if DetectOpenAPIFile $FILE; then + FILE_ARRAY_OPENAPI+=("$FILE") + fi + done + fi + + ########################## + # Lint the OpenAPI files # + ########################## + # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY" + LintCodebase "OPENAPI" "spectral" "spectral lint -r $OPENAPI_LINTER_RULES" "disabledfileext" "${FILE_ARRAY_OPENAPI[@]}" +fi + ########## # Footer # ########## From b80ea7e335ae4c3a18007c3372e6d9b91a87e09d Mon Sep 17 00:00:00 2001 From: danielcompton Date: Thu, 25 Jun 2020 20:06:44 +1200 Subject: [PATCH 25/65] Remove unused binding from good Clojure test file Prevents clj-kondo from warning about an unused binding. --- .automation/test/clojure/clojure_good_1.clj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.automation/test/clojure/clojure_good_1.clj b/.automation/test/clojure/clojure_good_1.clj index 7ea739e1..391f4688 100644 --- a/.automation/test/clojure/clojure_good_1.clj +++ b/.automation/test/clojure/clojure_good_1.clj @@ -7,8 +7,7 @@ (str/join "" "") (defn foo-fn [x] - (let [y (fn [] (inc x)) - z y] + (let [y (fn [] (inc x))] (y))) (letfn From 37f6828d1ecbb30ef3090caf46cfeb4f1fd3ab61 Mon Sep 17 00:00:00 2001 From: Justin Kalland Date: Thu, 25 Jun 2020 11:49:09 +0200 Subject: [PATCH 26/65] Fix OpenAPI tests being run by JSON & YAML Tests are run by file extension. The [JSON] and [YAML] tests were running the _bad_ [OPENAPI] tests and expecting them to fail. For [OPENAPI] the _bad_ tests are syntactically good JSON/YAML but bad OAS. --- .automation/test/openapi/README.md | 3 ++- .../openapi/{openapi_bad_1.yml => openapi_bad_1.ymlopenapi} | 0 .../openapi/{openapi_bad_2.json => openapi_bad_2.jsonopenapi} | 0 .../openapi/{openapi_good_1.yml => openapi_good_1.ymlopenapi} | 0 .../{openapi_good_2.json => openapi_good_2.jsonopenapi} | 0 lib/linter.sh | 2 +- 6 files changed, 3 insertions(+), 2 deletions(-) rename .automation/test/openapi/{openapi_bad_1.yml => openapi_bad_1.ymlopenapi} (100%) rename .automation/test/openapi/{openapi_bad_2.json => openapi_bad_2.jsonopenapi} (100%) rename .automation/test/openapi/{openapi_good_1.yml => openapi_good_1.ymlopenapi} (100%) rename .automation/test/openapi/{openapi_good_2.json => openapi_good_2.jsonopenapi} (100%) diff --git a/.automation/test/openapi/README.md b/.automation/test/openapi/README.md index 8e77fdbf..6f5d2c24 100644 --- a/.automation/test/openapi/README.md +++ b/.automation/test/openapi/README.md @@ -2,7 +2,8 @@ This folder holds the test cases for **OpenAPI**. ## Additional Docs -No Additional information is needed for this test case. +The `_bad_` tests are valid `.yml`/`.json` but invalid OpenAPI specs. +The test extensions used are `.ymlopenapi`/`.jsonopenapi` instead of `.yml`/`.json`. This is to prevent the [YAML] and [JSON] tests from picking them up. ## Good Test Cases The test cases denoted: `LANGUAGE_good_FILE.EXTENSION` are all valid, and should pass successfully when linted. diff --git a/.automation/test/openapi/openapi_bad_1.yml b/.automation/test/openapi/openapi_bad_1.ymlopenapi similarity index 100% rename from .automation/test/openapi/openapi_bad_1.yml rename to .automation/test/openapi/openapi_bad_1.ymlopenapi diff --git a/.automation/test/openapi/openapi_bad_2.json b/.automation/test/openapi/openapi_bad_2.jsonopenapi similarity index 100% rename from .automation/test/openapi/openapi_bad_2.json rename to .automation/test/openapi/openapi_bad_2.jsonopenapi diff --git a/.automation/test/openapi/openapi_good_1.yml b/.automation/test/openapi/openapi_good_1.ymlopenapi similarity index 100% rename from .automation/test/openapi/openapi_good_1.yml rename to .automation/test/openapi/openapi_good_1.ymlopenapi diff --git a/.automation/test/openapi/openapi_good_2.json b/.automation/test/openapi/openapi_good_2.jsonopenapi similarity index 100% rename from .automation/test/openapi/openapi_good_2.json rename to .automation/test/openapi/openapi_good_2.jsonopenapi diff --git a/lib/linter.sh b/lib/linter.sh index 7b90ae32..b23b96e6 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -2236,7 +2236,7 @@ RunTestCases() TestCodebase "CSS" "stylelint" "stylelint --config $CSS_LINTER_RULES" ".*\.\(css\)\$" TestCodebase "ENV" "dotenv-linter" "dotenv-linter" ".*\.\(env\)\$" TestCodebase "KOTLIN" "ktlint" "ktlint" ".*\.\(kt\|kts\)\$" - TestCodebase "OPENAPI" "spectral" "spectral lint -r $OPENAPI_LINTER_RULES" ".*\.\(yml\|yaml\|json\)\$" + TestCodebase "OPENAPI" "spectral" "spectral lint -r $OPENAPI_LINTER_RULES" ".*\.\(ymlopenapi\|jsonopenapi\)\$" ################# # Footer prints # From 79671724a0b0142b773e1387c5ddf75fef2f7638 Mon Sep 17 00:00:00 2001 From: Justin Kalland Date: Thu, 25 Jun 2020 12:21:11 +0200 Subject: [PATCH 27/65] Add OpenAPI to disabling-linters doc --- .github/linters/.openapirc.yml | 9 +++++++++ docs/disabling-linters.md | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 .github/linters/.openapirc.yml diff --git a/.github/linters/.openapirc.yml b/.github/linters/.openapirc.yml new file mode 100644 index 00000000..fdf641e1 --- /dev/null +++ b/.github/linters/.openapirc.yml @@ -0,0 +1,9 @@ +--- + +########################## +########################## +## OpenAPI Linter rules ## +########################## +########################## + +extends: spectral:oas diff --git a/docs/disabling-linters.md b/docs/disabling-linters.md index c738bdbd..2f36d22d 100644 --- a/docs/disabling-linters.md +++ b/docs/disabling-linters.md @@ -24,6 +24,7 @@ Below are examples and documentation for each language and the various methods t - [CSS](#stylelint) - [ENV](#dotenv-linter) - [Kotlin](#kotlin) +- [OpenAPI](#openapi) @@ -607,3 +608,24 @@ import package.b.* ### ktlint disable entire file - There is currently **No** way to disable rules inline of the file(s) + +-------------------------------------------------------------------------------- + +## OpenAPI +- [spectral](https://github.com/stoplightio/spectral) + +### OpenAPI Config file +- `.github/linters/.openapirc.yml` +- You can add, extend, and disable rules +- Documentation at [Spectral Custom Rulesets](https://stoplight.io/p/docs/gh/stoplightio/spectral/docs/guides/4-custom-rulesets.md) +- File should be located at: `.github/linters/.openapirc.yml` + +### OpenAPI disable single line +- There is currently **No** way to disable rules inline of the file(s) + +### OpenAPI disable code block +- There is currently **No** way to disable rules inline of the file(s) + +### OpenAPI disable entire file +- There is currently **No** way to disable rules inline of the file(s) +- However, you can make [rule exceptions](https://stoplight.io/p/docs/gh/stoplightio/spectral/docs/guides/6-exceptions.md?srn=gh/stoplightio/spectral/docs/guides/6-exceptions.md) in the config for individual file(s). From 172e29dad3e4d7768590e191201522ece5b384f9 Mon Sep 17 00:00:00 2001 From: Gabo Date: Thu, 25 Jun 2020 10:03:39 -0500 Subject: [PATCH 28/65] Fix bash lint errors --- .vscode/testlinter.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.vscode/testlinter.sh b/.vscode/testlinter.sh index f7e7eb79..c6a54f00 100644 --- a/.vscode/testlinter.sh +++ b/.vscode/testlinter.sh @@ -4,8 +4,9 @@ #If you do neither, this will default to running against the test automation files tmppath=/tmp/lint if [ ! -L $tmppath ]; then - ln -s $PWD/.automation/test $tmppath + ln -s "$PWD"/.automation/test $tmppath fi export RUN_LOCAL=true -source $PWD/lib/linter.sh \ No newline at end of file +# shellcheck source=lib/linter.sh +source "$PWD"/lib/linter.sh \ No newline at end of file From b8171a9bfbd7702465c8fe38657aec632d2f0c12 Mon Sep 17 00:00:00 2001 From: Gabo Date: Thu, 25 Jun 2020 10:04:02 -0500 Subject: [PATCH 29/65] Fix markdown lint errors --- README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 38a83bf8..fe4627be 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Super-Linter -This repository is for the **GitHub Action** to run a **Super-Linter**. +This repository is for the **GitHub Action** to run a **Super-Linter**. It is a simple combination of various linters, written in `bash`, to help validate your source code. The end goal of this tool: @@ -112,9 +112,9 @@ jobs: ## Environment variables The super-linter allows you to pass the following `ENV` variables to be able to trigger different functionality. -*Note:* All the `VALIDATE_[LANGUAGE]` variables behave in a specific way. -If none of them are passed, then they all default to true. -However if any one of the variables are set, we default to leaving any unset variable to false. +*Note:* All the `VALIDATE_[LANGUAGE]` variables behave in a specific way. +If none of them are passed, then they all default to true. +However if any one of the variables are set, we default to leaving any unset variable to false. This means that if you run the linter "out of the box", all languages will be checked. But if you wish to select specific linters, we give you full control to choose which linters are run, and won't run anything unexpected. @@ -186,12 +186,11 @@ Below are a list of the known limitations for the **GitHub Super-Linter**: If you would like to help contribute to this **GitHub** Action, please see [CONTRIBUTING](https://github.com/github/super-linter/blob/master/.github/CONTRIBUTING.md) ### Visual Studio Code -You can checkout this repository using [Container Remote Development](https://code.visualstudio.com/docs/remote/containers), and debug the linter using the `Test Linter` task. +You can checkout this repository using [Container Remote Development](https://code.visualstudio.com/docs/remote/containers), and debug the linter using the `Test Linter` task. ![Example](https://user-images.githubusercontent.com/15258962/85165778-2d2ce700-b21b-11ea-803e-3f6709d8e609.gif) We will also support [Github Codespaces](https://github.com/features/codespaces/) once it becomes available - -------------------------------------------------------------------------------- ### License From a51265c9f0925b7ee09a7d4c8baaba0836b611fd Mon Sep 17 00:00:00 2001 From: Gabo Date: Thu, 25 Jun 2020 10:25:56 -0500 Subject: [PATCH 30/65] Fix slashes --- .devcontainer/devcontainer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 72190b10..e8990ac0 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -7,10 +7,10 @@ "context": "..", // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. - "dockerFile": "..\\Dockerfile", + "dockerFile": "..//Dockerfile", // Set *default* container specific settings.json values on container create. - "settings": { + "settings": { "terminal.integrated.shell.linux": "/bin/bash" }, From ce9b8526f85bd248fa4fe0167ccbdce89557f94e Mon Sep 17 00:00:00 2001 From: Josh Soref Date: Thu, 25 Jun 2020 16:56:06 -0400 Subject: [PATCH 31/65] Proactively add GitHub markdown emojis --- .github/actions/spelling/allow/emoji.txt | 48 ++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .github/actions/spelling/allow/emoji.txt diff --git a/.github/actions/spelling/allow/emoji.txt b/.github/actions/spelling/allow/emoji.txt new file mode 100644 index 00000000..5b4b2b08 --- /dev/null +++ b/.github/actions/spelling/allow/emoji.txt @@ -0,0 +1,48 @@ +abcd +bangbang +bento +bullettrain +busstop +cn +couplekiss +dango +dvd +facepunch +feelsgood +finnadie +fuelpump +gb +goberserk +godmode +gua +hankey +heartpulse +hocho +hurtrealbad +icecream +inbox +iphone +izakaya +jp +keycap +mega +minidisc +moyai +neckbeard +octocat +oden +ramen +ru +scorpius +shipit +snowboarder +tada +tanabata +thumbsdown +thumbsup +tophat +trollface +tshirt +uk +vhs +zzz From aba407fff349a76dead6af5c60b45c26c57edcae Mon Sep 17 00:00:00 2001 From: powerOFMAX Date: Thu, 25 Jun 2020 21:08:05 -0300 Subject: [PATCH 32/65] Add Table of Contents to README --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 6f4799f2..7b89a709 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,20 @@ The end goal of this tool: - Build guidelines for code layout and format - Automate the process to help streamline code reviews + +## Table of Contents + +* [How it works](#how-it-works) +* [Supported linters](#supported-linters) +* [Usage](#how-to-use) +* [Environment variables](#environment-variables) +* [Disable rules](#disabling-rules) +* [Docker Hub](#docker-hub) +* [Run Super-Linter locally](#running-super-linter-locally-troubleshootingdebuggingenhancements) + * [CI / CT/ CD](#cictcd) +* [Limitations](#limitations) + + ## How it Works The super-linter finds issues and reports them to the console output. Fixes are suggested in the console output but not automatically fixed, and a status check will show up as failed on the pull request. From 04e7807d4e7ff7203650d421ca3d6564f0cf0253 Mon Sep 17 00:00:00 2001 From: Thomas Hughes Date: Thu, 25 Jun 2020 19:19:12 -0500 Subject: [PATCH 33/65] Add contributing to ToC --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b89a709..6aabfb51 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ The end goal of this tool: * [Run Super-Linter locally](#running-super-linter-locally-troubleshootingdebuggingenhancements) * [CI / CT/ CD](#cictcd) * [Limitations](#limitations) - +* [Contributing](#how-to-contribute) ## How it Works From b1a578b42c4c5469d47c453545cbdde7331eda64 Mon Sep 17 00:00:00 2001 From: Thomas Hughes Date: Thu, 25 Jun 2020 19:22:10 -0500 Subject: [PATCH 34/65] Change * for - for consistency --- README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 6aabfb51..d21b666b 100644 --- a/README.md +++ b/README.md @@ -11,16 +11,16 @@ The end goal of this tool: ## Table of Contents -* [How it works](#how-it-works) -* [Supported linters](#supported-linters) -* [Usage](#how-to-use) -* [Environment variables](#environment-variables) -* [Disable rules](#disabling-rules) -* [Docker Hub](#docker-hub) -* [Run Super-Linter locally](#running-super-linter-locally-troubleshootingdebuggingenhancements) - * [CI / CT/ CD](#cictcd) -* [Limitations](#limitations) -* [Contributing](#how-to-contribute) +- [How it works](#how-it-works) +- [Supported linters](#supported-linters) +- [Usage](#how-to-use) +- [Environment variables](#environment-variables) +- [Disable rules](#disabling-rules) +- [Docker Hub](#docker-hub) +- [Run Super-Linter locally](#running-super-linter-locally-troubleshootingdebuggingenhancements) + - [CI / CT/ CD](#cictcd) +- [Limitations](#limitations) +- [Contributing](#how-to-contribute) ## How it Works From 70f51ccd8fafdd647fa7846f8197efe0379100e9 Mon Sep 17 00:00:00 2001 From: past-due <30942300+past-due@users.noreply.github.com> Date: Thu, 25 Jun 2020 22:54:56 -0400 Subject: [PATCH 35/65] Add JAVASCRIPT_ES_CONFIG_FILE to specify eslintrc filename See: https://eslint.org/docs/user-guide/configuring#configuration-file-formats --- README.md | 1 + lib/linter.sh | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/README.md b/README.md index d21b666b..19a816cd 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,7 @@ and won't run anything unexpected. | **VALIDATE_COFFEE** | `true` | Flag to enable or disable the linting process of the language . | | **VALIDATE_ANSIBLE** | `true` | Flag to enable or disable the linting process of the language. | | **VALIDATE_JAVASCRIPT_ES** | `true` | Flag to enable or disable the linting process of the language. (Utilizing: eslint) | +| **JAVASCRIPT_ES_CONFIG_FILE** | `.eslintrc.yml` | Filename for [eslint configuration](https://eslint.org/docs/user-guide/configuring#configuration-file-formats) (ex: `.eslintrc.yml`, `.eslintrc.json`)| | **VALIDATE_JAVASCRIPT_STANDARD** | `true` | Flag to enable or disable the linting process of the language. (Utilizing: standard) | | **VALIDATE_TYPESCRIPT_ES** | `true` | Flag to enable or disable the linting process of the language. (Utilizing: eslint) | | **VALIDATE_TYPESCRIPT_STANDARD** | `true` | Flag to enable or disable the linting process of the language. (Utilizing: standard) | diff --git a/lib/linter.sh b/lib/linter.sh index e422ad9d..41d36428 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -92,6 +92,7 @@ VALIDATE_RUBY="${VALIDATE_RUBY}" # Boolean to validate lang VALIDATE_COFFEE="${VALIDATE_COFFEE}" # Boolean to validate language VALIDATE_ANSIBLE="${VALIDATE_ANSIBLE}" # Boolean to validate language VALIDATE_JAVASCRIPT_ES="${VALIDATE_JAVASCRIPT_ES}" # Boolean to validate language +JAVASCRIPT_ES_CONFIG_FILE="${JAVASCRIPT_ES_CONFIG_FILE}" # Filename for eslint configuration (ex: .eslintrc.yml, .eslintrc.json) VALIDATE_JAVASCRIPT_STANDARD="${VALIDATE_JAVASCRIPT_STANDARD}" # Boolean to validate language VALIDATE_TYPESCRIPT_ES="${VALIDATE_TYPESCRIPT_ES}" # Boolean to validate language VALIDATE_TYPESCRIPT_STANDARD="${VALIDATE_TYPESCRIPT_STANDARD}" # Boolean to validate language @@ -117,6 +118,7 @@ ACTIONS_RUNNER_DEBUG="${ACTIONS_RUNNER_DEBUG}" # Boolean to see even more info DEFAULT_VALIDATE_ALL_CODEBASE='true' # Default value for validate all files DEFAULT_WORKSPACE="${DEFAULT_WORKSPACE:-/tmp/lint}" # Default workspace if running locally DEFAULT_ANSIBLE_DIRECTORY="$GITHUB_WORKSPACE/ansible" # Default Ansible Directory +DEFAULT_JAVASCRIPT_ES_CONFIG_FILE=".eslintrc.yml" # Default eslint configuration filename DEFAULT_RUN_LOCAL='false' # Default value for debugging locally DEFAULT_TEST_CASE_RUN='false' # Flag to tell code to run only test cases DEFAULT_ACTIONS_RUNNER_DEBUG='false' # Default value for debugging output @@ -1226,6 +1228,19 @@ GetValidationInfo() # Set the value ANSIBLE_DIRECTORY="$TEMP_ANSIBLE_DIRECTORY" fi + + ######################################### + # Get the eslint configuration filename # + ######################################### + if [ -z "$JAVASCRIPT_ES_CONFIG_FILE" ]; then + ###################################### + # No filename passed, set to default # + ###################################### + JAVASCRIPT_ES_CONFIG_FILE="$DEFAULT_JAVASCRIPT_ES_CONFIG_FILE" + fi + # Set Javascript Vars based on JAVASCRIPT_ES_CONFIG_FILE + JAVASCRIPT_FILE_NAME='$JAVASCRIPT_ES_CONFIG_FILE' # Name of the file + JAVASCRIPT_LINTER_RULES="$DEFAULT_RULES_LOCATION/$JAVASCRIPT_FILE_NAME" # Path to the Javascript lint rules ############################### # Get the disable errors flag # From 116698b03237fed27fa86ab9e4ebc1e4ac1266e0 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Fri, 26 Jun 2020 08:26:36 -0500 Subject: [PATCH 36/65] fixed linting errors --- lib/linter.sh | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/lib/linter.sh b/lib/linter.sh index a89bb3fe..06bc0022 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -573,10 +573,33 @@ LintAnsibleFiles() #### Function DetectOpenAPIFile ################################################ DetectOpenAPIFile() { - egrep '"openapi":|"swagger":|^openapi:|^swagger:' $GITHUB_WORKSPACE/$1 > /dev/null - if [ $? -eq 0 ]; then + ################ + # Pull in vars # + ################ + FILE="$1" + + ############################### + # Check the file for keywords # + ############################### + egrep '"openapi":|"swagger":|^openapi:|^swagger:' "$GITHUB_WORKSPACE/$FILE" > /dev/null + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -eq 0 ]; then + ######################## + # Found string in file # + ######################## return 0 else + ################### + # No string match # + ################### return 1 fi } @@ -1460,7 +1483,7 @@ BuildFileList() ############################ # Check if file is OpenAPI # ############################ - if DetectOpenAPIFile $FILE; then + if DetectOpenAPIFile "$FILE"; then FILE_ARRAY_OPENAPI+=("$FILE") fi ########################################################## @@ -1478,7 +1501,7 @@ BuildFileList() ############################ # Check if file is OpenAPI # ############################ - if DetectOpenAPIFile $FILE; then + if DetectOpenAPIFile "$FILE"; then FILE_ARRAY_OPENAPI+=("$FILE") fi ########################################################## @@ -2640,7 +2663,7 @@ if [ "$VALIDATE_OPENAPI" == "true" ]; then LIST_FILES=($(cd "$GITHUB_WORKSPACE" || exit; find . -type f -regex ".*\.\(yml\|yaml\|json\)\$" 2>&1)) for FILE in "${LIST_FILES[@]}" do - if DetectOpenAPIFile $FILE; then + if DetectOpenAPIFile "$FILE"; then FILE_ARRAY_OPENAPI+=("$FILE") fi done From 063e48d9c4926a8353973adc9703bd2a143bfc7e Mon Sep 17 00:00:00 2001 From: Thomas Hughes Date: Fri, 26 Jun 2020 08:35:36 -0500 Subject: [PATCH 37/65] Remove spell-checker --- .github/actions/spelling/allow/Dockerfile.txt | 7 - .../spelling/allow/cleanup-docker.sh.txt | 1 - .../spelling/allow/disabling-linters.md.txt | 3 - .github/actions/spelling/allow/emoji.txt | 48 ------ .../allow/ghe-api-config-apply.yml.txt | 2 - .../spelling/allow/ghe-config-apply.sh.txt | 2 - .../allow/ghe-initial-configuration.yml.txt | 1 - .../allow/ghe-ldap-configuration.yml.txt | 2 - .github/actions/spelling/allow/gitignore.txt | 8 - .github/actions/spelling/allow/linter.sh.txt | 7 - .github/actions/spelling/allow/main.yml.txt | 8 - .../spelling/allow/settings.json.j2.txt | 4 - .github/actions/spelling/allow/words.txt | 158 ------------------ .github/actions/spelling/excludes.txt | 2 - .github/actions/spelling/expect.txt | 39 ----- .github/actions/spelling/patterns.txt | 7 - .github/workflows/check-spelling.yml | 48 ------ 17 files changed, 347 deletions(-) delete mode 100644 .github/actions/spelling/allow/Dockerfile.txt delete mode 100644 .github/actions/spelling/allow/cleanup-docker.sh.txt delete mode 100644 .github/actions/spelling/allow/disabling-linters.md.txt delete mode 100644 .github/actions/spelling/allow/emoji.txt delete mode 100644 .github/actions/spelling/allow/ghe-api-config-apply.yml.txt delete mode 100644 .github/actions/spelling/allow/ghe-config-apply.sh.txt delete mode 100644 .github/actions/spelling/allow/ghe-initial-configuration.yml.txt delete mode 100644 .github/actions/spelling/allow/ghe-ldap-configuration.yml.txt delete mode 100644 .github/actions/spelling/allow/gitignore.txt delete mode 100644 .github/actions/spelling/allow/linter.sh.txt delete mode 100644 .github/actions/spelling/allow/main.yml.txt delete mode 100644 .github/actions/spelling/allow/settings.json.j2.txt delete mode 100644 .github/actions/spelling/allow/words.txt delete mode 100644 .github/actions/spelling/excludes.txt delete mode 100644 .github/actions/spelling/expect.txt delete mode 100644 .github/actions/spelling/patterns.txt delete mode 100644 .github/workflows/check-spelling.yml diff --git a/.github/actions/spelling/allow/Dockerfile.txt b/.github/actions/spelling/allow/Dockerfile.txt deleted file mode 100644 index bb490a92..00000000 --- a/.github/actions/spelling/allow/Dockerfile.txt +++ /dev/null @@ -1,7 +0,0 @@ -musl -nvq -rdoc -setuptools -uninstall -wget -WORKDIR diff --git a/.github/actions/spelling/allow/cleanup-docker.sh.txt b/.github/actions/spelling/allow/cleanup-docker.sh.txt deleted file mode 100644 index 1efe4ad7..00000000 --- a/.github/actions/spelling/allow/cleanup-docker.sh.txt +++ /dev/null @@ -1 +0,0 @@ -alnum diff --git a/.github/actions/spelling/allow/disabling-linters.md.txt b/.github/actions/spelling/allow/disabling-linters.md.txt deleted file mode 100644 index fe1303c8..00000000 --- a/.github/actions/spelling/allow/disabling-linters.md.txt +++ /dev/null @@ -1,3 +0,0 @@ -noqa -toc -todo diff --git a/.github/actions/spelling/allow/emoji.txt b/.github/actions/spelling/allow/emoji.txt deleted file mode 100644 index 5b4b2b08..00000000 --- a/.github/actions/spelling/allow/emoji.txt +++ /dev/null @@ -1,48 +0,0 @@ -abcd -bangbang -bento -bullettrain -busstop -cn -couplekiss -dango -dvd -facepunch -feelsgood -finnadie -fuelpump -gb -goberserk -godmode -gua -hankey -heartpulse -hocho -hurtrealbad -icecream -inbox -iphone -izakaya -jp -keycap -mega -minidisc -moyai -neckbeard -octocat -oden -ramen -ru -scorpius -shipit -snowboarder -tada -tanabata -thumbsdown -thumbsup -tophat -trollface -tshirt -uk -vhs -zzz diff --git a/.github/actions/spelling/allow/ghe-api-config-apply.yml.txt b/.github/actions/spelling/allow/ghe-api-config-apply.yml.txt deleted file mode 100644 index 8d39ee0b..00000000 --- a/.github/actions/spelling/allow/ghe-api-config-apply.yml.txt +++ /dev/null @@ -1,2 +0,0 @@ -configcheck -nohup diff --git a/.github/actions/spelling/allow/ghe-config-apply.sh.txt b/.github/actions/spelling/allow/ghe-config-apply.sh.txt deleted file mode 100644 index 0570567b..00000000 --- a/.github/actions/spelling/allow/ghe-config-apply.sh.txt +++ /dev/null @@ -1,2 +0,0 @@ -args -pid diff --git a/.github/actions/spelling/allow/ghe-initial-configuration.yml.txt b/.github/actions/spelling/allow/ghe-initial-configuration.yml.txt deleted file mode 100644 index a09f0564..00000000 --- a/.github/actions/spelling/allow/ghe-initial-configuration.yml.txt +++ /dev/null @@ -1 +0,0 @@ -initialconfig diff --git a/.github/actions/spelling/allow/ghe-ldap-configuration.yml.txt b/.github/actions/spelling/allow/ghe-ldap-configuration.yml.txt deleted file mode 100644 index 04b1c12f..00000000 --- a/.github/actions/spelling/allow/ghe-ldap-configuration.yml.txt +++ /dev/null @@ -1,2 +0,0 @@ -ldapconfig -openldap diff --git a/.github/actions/spelling/allow/gitignore.txt b/.github/actions/spelling/allow/gitignore.txt deleted file mode 100644 index 6fcb9a9c..00000000 --- a/.github/actions/spelling/allow/gitignore.txt +++ /dev/null @@ -1,8 +0,0 @@ -cov -eslintcache -jscoverage -jspm -nyc -tgz -typings -wscript diff --git a/.github/actions/spelling/allow/linter.sh.txt b/.github/actions/spelling/allow/linter.sh.txt deleted file mode 100644 index 9bd25b2f..00000000 --- a/.github/actions/spelling/allow/linter.sh.txt +++ /dev/null @@ -1,7 +0,0 @@ -cw -Mstrict -printenv -rcfile -tf -tolower -whoami diff --git a/.github/actions/spelling/allow/main.yml.txt b/.github/actions/spelling/allow/main.yml.txt deleted file mode 100644 index 877e71bb..00000000 --- a/.github/actions/spelling/allow/main.yml.txt +++ /dev/null @@ -1,8 +0,0 @@ -Autobots -basemap -cas -cn -crt -rsa -tlsv -tmp diff --git a/.github/actions/spelling/allow/settings.json.j2.txt b/.github/actions/spelling/allow/settings.json.j2.txt deleted file mode 100644 index 7f01269b..00000000 --- a/.github/actions/spelling/allow/settings.json.j2.txt +++ /dev/null @@ -1,4 +0,0 @@ -dotcom -identicons -oauth -timezone diff --git a/.github/actions/spelling/allow/words.txt b/.github/actions/spelling/allow/words.txt deleted file mode 100644 index 0e91754b..00000000 --- a/.github/actions/spelling/allow/words.txt +++ /dev/null @@ -1,158 +0,0 @@ -admiralawkbar -ansible -api -apk -aws -baz -beardofedu -certs -changeset -codebase -CODEOWNERS -coffeelint -coffeescript -collectd -concat -config -configs -css -dest -devops -dirname -dockerfile -dockerfilelint -dockerfilelintrc -dotenv -elif -emails -entrypoint -Errorf -eslint -eslintrc -filetype -func -gcc -getenv -ghe -GHES -ghl -github -globals -golang -golangci -Google -gpg -gql -Grafana -graphql -grep -homepage -hookshot -hostname -hostvars -http -https -hubot -idp -ip -Jani -javascript -jq -json -jsonlint -jwiebalk -JWT -ldap -len -lfs -libxml -linted -linting -loadbalancer -localhost -loglevel -markdownlint -millis -mkdir -nodejs -NONINFRINGEMENT -noproxy -noreply -npm -ntp -opensource -opensourcefriday -perl -plugin -posix -pprint -Prego -prettyjson -Println -probot -px -py -pylint -rb -readlines -README -regex -regexp -resqued -rgba -rien -Rubo -rubocop -saml -screenshots -shellcheck -signup -smtp -snmp -socio -splunk -src -ssh -ssl -sso -stackoverflow -stacktraces -standardjs -stringify -stylelint -stylelintrc -subdomain -subfolders -sudo -sys -syslog -taz -terraform -tflint -tileserver -tls -typeof -ubuntu -udp -uid -undef -uniq -uri -url -urlencode -username -usr -utils -Vape -vnd -webhook -wiki -wildcards -workflow -xml -xmllint -yaml -yamllint -yml -yq -zkoppert diff --git a/.github/actions/spelling/excludes.txt b/.github/actions/spelling/excludes.txt deleted file mode 100644 index 549b89c5..00000000 --- a/.github/actions/spelling/excludes.txt +++ /dev/null @@ -1,2 +0,0 @@ -^\.github/linters/ -^TEMPLATES/\.\S* diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt deleted file mode 100644 index 64d9960d..00000000 --- a/.github/actions/spelling/expect.txt +++ /dev/null @@ -1,39 +0,0 @@ -cdxml -changelog -chmod -Dockerhub -EOL -GPR -IAm -icu -jre -kotlin -krb -ktlint -libgcc -libintl -libssl -libstdc -linux -lttng -microsoft -ncurses -nq -openjdk -php -powershell -println -psd -psm -psrc -pssc -psscriptanalyzer -pwsh -rcu -rhc -ry -terminfo -tzdata -userspace -xargs -zlib diff --git a/.github/actions/spelling/patterns.txt b/.github/actions/spelling/patterns.txt deleted file mode 100644 index 5c595e9b..00000000 --- a/.github/actions/spelling/patterns.txt +++ /dev/null @@ -1,7 +0,0 @@ -https?:\S* -# ignore long runs of a single character: -([A-Za-z])\1{3,} -# Any CLI args (-xzf -aef) -\ -\w+\b -# Hex colors (dummy group to not be treated as comment) -(?:#)([A-Fa-f0-9]{6}|[A-Fa-f0-9]{3}) diff --git a/.github/workflows/check-spelling.yml b/.github/workflows/check-spelling.yml deleted file mode 100644 index b99524b0..00000000 --- a/.github/workflows/check-spelling.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- -#################### -## Check spelling ## -#################### - -# -# Documentation: -# https://help.github.com/en/articles/workflow-syntax-for-github-actions -# - -name: Spell checking -############################# -# Start the job on all push # -############################# -on: - push: - branches-ignore: - - 'master' - -############### -# Set the Job # -############### -jobs: - build: - # Name the Job - name: Spell checking - # Set the agent to run on - runs-on: ubuntu-latest - ################## - # Load all steps # - ################## - steps: - ########################## - # Checkout the code base # - ########################## - - name: Checkout Code - uses: actions/checkout@v2 - - ############################# - # Run check spelling action # - ############################# - - name: Check spelling - uses: check-spelling/check-spelling@0.0.16-alpha - with: - bucket: .github/actions - project: spelling - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 8fdce000bf0a0f18721d169b8dc79503c1c633bd Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Fri, 26 Jun 2020 08:44:27 -0500 Subject: [PATCH 38/65] got to catch them all --- lib/linter.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/linter.sh b/lib/linter.sh index 06bc0022..ca352e6e 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -581,7 +581,7 @@ DetectOpenAPIFile() ############################### # Check the file for keywords # ############################### - egrep '"openapi":|"swagger":|^openapi:|^swagger:' "$GITHUB_WORKSPACE/$FILE" > /dev/null + grep -E '"openapi":|"swagger":|^openapi:|^swagger:' "$GITHUB_WORKSPACE/$FILE" > /dev/null ####################### # Load the error code # @@ -2232,7 +2232,7 @@ Footer() [ "$ERRORS_FOUND_RUBY" -ne 0 ] || \ [ "$ERRORS_FOUND_CSS" -ne 0 ] || \ [ "$ERRORS_FOUND_ENV" -ne 0 ] || \ - [ "$ERRORS_FOUND_OPENAPI" -ne 0 ] \ + [ "$ERRORS_FOUND_OPENAPI" -ne 0 ] || \ [ "$ERRORS_FOUND_CLOJURE" -ne 0 ] || \ [ "$ERRORS_FOUND_KOTLIN" -ne 0 ]; then # Failed exit @@ -2660,6 +2660,7 @@ fi if [ "$VALIDATE_OPENAPI" == "true" ]; then # If we are validating all codebase we need to build file list because not every yml/json file is an OpenAPI file if [ "$VALIDATE_ALL_CODEBASE" == "true" ]; then + # shellcheck disable=SC2207 LIST_FILES=($(cd "$GITHUB_WORKSPACE" || exit; find . -type f -regex ".*\.\(yml\|yaml\|json\)\$" 2>&1)) for FILE in "${LIST_FILES[@]}" do From f087775d99b15bf6632bbde557d17e18cab00d8b Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Fri, 26 Jun 2020 09:00:28 -0500 Subject: [PATCH 39/65] typo --- lib/linter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/linter.sh b/lib/linter.sh index ca352e6e..65b973a1 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -833,7 +833,7 @@ GetValidationInfo() -n "$VALIDATE_CSS" || \ -n "$VALIDATE_ENV" || \ -n "$VALIDATE_CLOJURE" || \ - -n "$VALIDATE_OPENAPI" ]] \ + -n "$VALIDATE_OPENAPI" || \ -n "$VALIDATE_KOTLIN" ]]; then ANY_SET="true" fi From 24234cc73b0f2e4b1362d9faca72a19860ea1a90 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Fri, 26 Jun 2020 09:38:41 -0500 Subject: [PATCH 40/65] fixed the open issue --- lib/linter.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/linter.sh b/lib/linter.sh index 65b973a1..a9ebf15e 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -2289,7 +2289,7 @@ RunTestCases() TestCodebase "JAVASCRIPT_STANDARD" "standard" "standard $JAVASCRIPT_STANDARD_LINTER_RULES" ".*\.\(js\)\$" TestCodebase "TYPESCRIPT_ES" "eslint" "eslint --no-eslintrc -c $TYPESCRIPT_LINTER_RULES" ".*\.\(ts\)\$" TestCodebase "TYPESCRIPT_STANDARD" "standard" "standard --parser @typescript-eslint/parser --plugin @typescript-eslint/eslint-plugin $TYPESCRIPT_STANDARD_LINTER_RULES" ".*\.\(ts\)\$" - TestCodebase "DOCKER" "/dockerfilelint/bin/dockerfilelint" "/dockerfilelint/bin/dockerfilelint" ".*\(Dockerfile\)\$" + TestCodebase "DOCKER" "/dockerfilelint/bin/dockerfilelint" "/dockerfilelint/bin/dockerfilelint -c $DOCKER_LINTER_RULES" ".*\(Dockerfile\)\$" TestCodebase "ANSIBLE" "ansible-lint" "ansible-lint -v -c $ANSIBLE_LINTER_RULES" "ansible-lint" TestCodebase "TERRAFORM" "tflint" "tflint -c $TERRAFORM_LINTER_RULES" ".*\.\(tf\)\$" TestCodebase "POWERSHELL" "pwsh" "pwsh -c Invoke-ScriptAnalyzer -EnableExit -Settings $POWERSHELL_LINTER_RULES -Path" ".*\.\(ps1\|psm1\|psd1\|ps1xml\|pssc\|psrc\|cdxml\)\$" @@ -2626,7 +2626,7 @@ if [ "$VALIDATE_DOCKER" == "true" ]; then # Lint the docker files # ######################### # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY" - LintCodebase "DOCKER" "/dockerfilelint/bin/dockerfilelint" "/dockerfilelint/bin/dockerfilelint" ".*\(Dockerfile\)\$" "${FILE_ARRAY_DOCKER[@]}" + LintCodebase "DOCKER" "/dockerfilelint/bin/dockerfilelint" "/dockerfilelint/bin/dockerfilelint -c $DOCKER_LINTER_RULES" ".*\(Dockerfile\)\$" "${FILE_ARRAY_DOCKER[@]}" fi ################### From d5724472870a0536250f8821bf8d0c921448b9f8 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Fri, 26 Jun 2020 10:38:02 -0500 Subject: [PATCH 41/65] Adding space support --- lib/linter.sh | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/lib/linter.sh b/lib/linter.sh index 65b973a1..df3241fa 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -132,6 +132,7 @@ RAW_FILE_ARRAY=() # Array of all files that were changed READ_ONLY_CHANGE_FLAG=0 # Flag set to 1 if files changed are not txt or md TEST_CASE_FOLDER='.automation/test' # Folder for test cases we should always ignore DEFAULT_DISABLE_ERRORS='false' # Default to enabling errors +DEFAULT_IFS="$IFS" # Get the Default IFS for updating ########################## # Array of changed files # @@ -1856,12 +1857,22 @@ LintCodebase() # We have files added to array of files to check LIST_FILES=("${FILE_ARRAY[@]}") # Copy the array into list else + ############################################################################### + # Set the file seperator to newline to allow for grabbing objects with spaces # + ############################################################################### + IFS=$'\n' + ################################# # Get list of all files to lint # ################################# # shellcheck disable=SC2207,SC2086 LIST_FILES=($(cd "$GITHUB_WORKSPACE" || exit; find . -type f -regex "$FILE_EXTENSIONS" 2>&1)) + ########################### + # Set IFS back to default # + ########################### + IFS="$DEFAULT_IFS" + ############################################################ # Set it back to empty if loaded with blanks from scanning # ############################################################ @@ -2028,11 +2039,21 @@ TestCodebase() # shellcheck disable=SC2207,SC2086,SC2010 LIST_FILES=($(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER" || exit; ls ansible/ | grep ".yml" 2>&1)) else + ############################################################################### + # Set the file seperator to newline to allow for grabbing objects with spaces # + ############################################################################### + IFS=$'\n' + ################################# # Get list of all files to lint # ################################# # shellcheck disable=SC2207,SC2086 LIST_FILES=($(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER" || exit; find . -type f -regex "$FILE_EXTENSIONS" ! -path "*./ansible*" 2>&1)) + + ########################### + # Set IFS back to default # + ########################### + IFS="$DEFAULT_IFS" fi ################## @@ -2289,7 +2310,7 @@ RunTestCases() TestCodebase "JAVASCRIPT_STANDARD" "standard" "standard $JAVASCRIPT_STANDARD_LINTER_RULES" ".*\.\(js\)\$" TestCodebase "TYPESCRIPT_ES" "eslint" "eslint --no-eslintrc -c $TYPESCRIPT_LINTER_RULES" ".*\.\(ts\)\$" TestCodebase "TYPESCRIPT_STANDARD" "standard" "standard --parser @typescript-eslint/parser --plugin @typescript-eslint/eslint-plugin $TYPESCRIPT_STANDARD_LINTER_RULES" ".*\.\(ts\)\$" - TestCodebase "DOCKER" "/dockerfilelint/bin/dockerfilelint" "/dockerfilelint/bin/dockerfilelint" ".*\(Dockerfile\)\$" + TestCodebase "DOCKER" "/dockerfilelint/bin/dockerfilelint" "/dockerfilelint/bin/dockerfilelint -c $DOCKER_LINTER_RULES" ".*\(Dockerfile\)\$" TestCodebase "ANSIBLE" "ansible-lint" "ansible-lint -v -c $ANSIBLE_LINTER_RULES" "ansible-lint" TestCodebase "TERRAFORM" "tflint" "tflint -c $TERRAFORM_LINTER_RULES" ".*\.\(tf\)\$" TestCodebase "POWERSHELL" "pwsh" "pwsh -c Invoke-ScriptAnalyzer -EnableExit -Settings $POWERSHELL_LINTER_RULES -Path" ".*\.\(ps1\|psm1\|psd1\|ps1xml\|pssc\|psrc\|cdxml\)\$" @@ -2626,7 +2647,7 @@ if [ "$VALIDATE_DOCKER" == "true" ]; then # Lint the docker files # ######################### # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY" - LintCodebase "DOCKER" "/dockerfilelint/bin/dockerfilelint" "/dockerfilelint/bin/dockerfilelint" ".*\(Dockerfile\)\$" "${FILE_ARRAY_DOCKER[@]}" + LintCodebase "DOCKER" "/dockerfilelint/bin/dockerfilelint" "/dockerfilelint/bin/dockerfilelint -c $DOCKER_LINTER_RULES" ".*\(Dockerfile\)\$" "${FILE_ARRAY_DOCKER[@]}" fi ################### @@ -2660,6 +2681,11 @@ fi if [ "$VALIDATE_OPENAPI" == "true" ]; then # If we are validating all codebase we need to build file list because not every yml/json file is an OpenAPI file if [ "$VALIDATE_ALL_CODEBASE" == "true" ]; then + ############################################################################### + # Set the file seperator to newline to allow for grabbing objects with spaces # + ############################################################################### + IFS=$'\n' + # shellcheck disable=SC2207 LIST_FILES=($(cd "$GITHUB_WORKSPACE" || exit; find . -type f -regex ".*\.\(yml\|yaml\|json\)\$" 2>&1)) for FILE in "${LIST_FILES[@]}" @@ -2668,6 +2694,11 @@ if [ "$VALIDATE_OPENAPI" == "true" ]; then FILE_ARRAY_OPENAPI+=("$FILE") fi done + + ########################### + # Set IFS back to default # + ########################### + IFS="$DEFAULT_IFS" fi ########################## From 1495c3b22cda396ee6e18bf34c75eb1d0c8b6e1e Mon Sep 17 00:00:00 2001 From: Thomas Hughes Date: Fri, 26 Jun 2020 10:45:43 -0500 Subject: [PATCH 42/65] Add step to ensure release is on marketplace --- .github/CONTRIBUTING.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 253b1933..86d05b97 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -33,9 +33,10 @@ Draft pull requests are also welcome to get feedback early on, or if there is so If you are the current maintainer of this action: 1. Update `README.md` and the wiki to reflect new version number in the example workflow file sections 2. Draft [Release](https://help.github.com/en/github/administering-a-repository/managing-releases-in-a-repository) with a summarized changelog -3. A GitHub Action will Publish the Docker image to GitHub Package Registry once a Release is created -4. A GitHub Action will Publish the Docker image to Docker Hub once a Release is created -5. Look for approval from [CODEOWNERS](https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners) +3. Ensure you check the box for [publishing to the marketplace](https://help.github.com/en/actions/creating-actions/publishing-actions-in-github-marketplace#publishing-an-action) +4. A GitHub Action will Publish the Docker image to GitHub Package Registry once a Release is created +5. A GitHub Action will Publish the Docker image to Docker Hub once a Release is created +6. Look for approval from [CODEOWNERS](https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners) ## Resources - [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/) From 10a7c9c4798321343a1f38a4617fb5f15c819f87 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Fri, 26 Jun 2020 11:49:41 -0500 Subject: [PATCH 43/65] Adding cleaner update --- README.md | 1 + lib/linter.sh | 19 ++----------------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 567bb466..42fbcbc2 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,7 @@ and won't run anything unexpected. | **JAVASCRIPT_ES_CONFIG_FILE** | `.eslintrc.yml` | Filename for [eslint configuration](https://eslint.org/docs/user-guide/configuring#configuration-file-formats) (ex: `.eslintrc.yml`, `.eslintrc.json`)| | **VALIDATE_JAVASCRIPT_STANDARD** | `true` | Flag to enable or disable the linting process of the language. (Utilizing: standard) | | **VALIDATE_TYPESCRIPT_ES** | `true` | Flag to enable or disable the linting process of the language. (Utilizing: eslint) | +| **TYPESCRIPT_ES_CONFIG_FILE** | `.eslintrc.yml` | Filename for [eslint configuration](https://eslint.org/docs/user-guide/configuring#configuration-file-formats) (ex: `.eslintrc.yml`, `.eslintrc.json`)| | **VALIDATE_TYPESCRIPT_STANDARD** | `true` | Flag to enable or disable the linting process of the language. (Utilizing: standard) | | **VALIDATE_DOCKER** | `true` | Flag to enable or disable the linting process of the language. | | **VALIDATE_GO** | `true` | Flag to enable or disable the linting process of the language. | diff --git a/lib/linter.sh b/lib/linter.sh index 2dd5d3cb..da97c9f2 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -29,11 +29,11 @@ RUBY_LINTER_RULES="$DEFAULT_RULES_LOCATION/$RUBY_FILE_NAME" # Path to th COFFEE_FILE_NAME='.coffee-lint.json' # Name of the file COFFEESCRIPT_LINTER_RULES="$DEFAULT_RULES_LOCATION/$COFFEE_FILE_NAME" # Path to the coffeescript lint rules # Javascript Vars -JAVASCRIPT_FILE_NAME='.eslintrc.yml' # Name of the file +JAVASCRIPT_FILE_NAME="${JAVASCRIPT_ES_CONFIG_FILE:-.eslintrc.yml}" # Name of the file JAVASCRIPT_LINTER_RULES="$DEFAULT_RULES_LOCATION/$JAVASCRIPT_FILE_NAME" # Path to the Javascript lint rules JAVASCRIPT_STANDARD_LINTER_RULES='' # ENV string to pass when running js standard # Typescript Vars -TYPESCRIPT_FILE_NAME='.eslintrc.yml' # Name of the file +TYPESCRIPT_FILE_NAME="${TYPESCRIPT_ES_CONFIG_FILE:-.eslintrc.yml}" # Name of the file TYPESCRIPT_LINTER_RULES="$DEFAULT_RULES_LOCATION/$TYPESCRIPT_FILE_NAME" # Path to the Typescript lint rules TYPESCRIPT_STANDARD_LINTER_RULES='' # ENV string to pass when running js standard # Ansible Vars @@ -98,7 +98,6 @@ VALIDATE_RUBY="${VALIDATE_RUBY}" # Boolean to validate lang VALIDATE_COFFEE="${VALIDATE_COFFEE}" # Boolean to validate language VALIDATE_ANSIBLE="${VALIDATE_ANSIBLE}" # Boolean to validate language VALIDATE_JAVASCRIPT_ES="${VALIDATE_JAVASCRIPT_ES}" # Boolean to validate language -JAVASCRIPT_ES_CONFIG_FILE="${JAVASCRIPT_ES_CONFIG_FILE}" # Filename for eslint configuration (ex: .eslintrc.yml, .eslintrc.json) VALIDATE_JAVASCRIPT_STANDARD="${VALIDATE_JAVASCRIPT_STANDARD}" # Boolean to validate language VALIDATE_TYPESCRIPT_ES="${VALIDATE_TYPESCRIPT_ES}" # Boolean to validate language VALIDATE_TYPESCRIPT_STANDARD="${VALIDATE_TYPESCRIPT_STANDARD}" # Boolean to validate language @@ -126,7 +125,6 @@ ACTIONS_RUNNER_DEBUG="${ACTIONS_RUNNER_DEBUG}" # Boolean to see even more info DEFAULT_VALIDATE_ALL_CODEBASE='true' # Default value for validate all files DEFAULT_WORKSPACE="${DEFAULT_WORKSPACE:-/tmp/lint}" # Default workspace if running locally DEFAULT_ANSIBLE_DIRECTORY="$GITHUB_WORKSPACE/ansible" # Default Ansible Directory -DEFAULT_JAVASCRIPT_ES_CONFIG_FILE=".eslintrc.yml" # Default eslint configuration filename DEFAULT_RUN_LOCAL='false' # Default value for debugging locally DEFAULT_TEST_CASE_RUN='false' # Flag to tell code to run only test cases DEFAULT_ACTIONS_RUNNER_DEBUG='false' # Default value for debugging output @@ -1318,19 +1316,6 @@ GetValidationInfo() # Set the value ANSIBLE_DIRECTORY="$TEMP_ANSIBLE_DIRECTORY" fi - - ######################################### - # Get the eslint configuration filename # - ######################################### - if [ -z "$JAVASCRIPT_ES_CONFIG_FILE" ]; then - ###################################### - # No filename passed, set to default # - ###################################### - JAVASCRIPT_ES_CONFIG_FILE="$DEFAULT_JAVASCRIPT_ES_CONFIG_FILE" - fi - # Set Javascript Vars based on JAVASCRIPT_ES_CONFIG_FILE - JAVASCRIPT_FILE_NAME='$JAVASCRIPT_ES_CONFIG_FILE' # Name of the file - JAVASCRIPT_LINTER_RULES="$DEFAULT_RULES_LOCATION/$JAVASCRIPT_FILE_NAME" # Path to the Javascript lint rules ############################### # Get the disable errors flag # From 26f4d56449e0991a7c70201cadcb8daa03565255 Mon Sep 17 00:00:00 2001 From: Gabo Date: Fri, 26 Jun 2020 13:55:11 -0500 Subject: [PATCH 44/65] Move json comments to readme --- .devcontainer/README.md | 10 ++++++++++ .devcontainer/devcontainer.json | 27 +-------------------------- .vscode/launch.json | 3 --- 3 files changed, 11 insertions(+), 29 deletions(-) create mode 100644 .devcontainer/README.md diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 00000000..75057d9f --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,10 @@ +# Devcontainer +This file specifies to vscode how to run the container + +For format details, see [documentation](https://aka.ms/vscode-remote/devcontainer.json) or this file's [README](https://github.com/microsoft/vscode-dev-containers/tree/v0.123.0/containers/docker-existing-dockerfile) + + +context: Sets the run context to one level up instead of the .devcontainer folder. +dockerFile: Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. +settings: Set *default* container specific settings.json values on container create. +extensions: Add the IDs of extensions you want installed when the container is created. \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e8990ac0..ac0ac8e1 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,37 +1,12 @@ -// For format details, see https://aka.ms/vscode-remote/devcontainer.json or this file's README at: -// https://github.com/microsoft/vscode-dev-containers/tree/v0.123.0/containers/docker-existing-dockerfile { "name": "SUPER-LINTER", - - // Sets the run context to one level up instead of the .devcontainer folder. "context": "..", - - // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. "dockerFile": "..//Dockerfile", - - // Set *default* container specific settings.json values on container create. "settings": { "terminal.integrated.shell.linux": "/bin/bash" }, - - // Add the IDs of extensions you want installed when the container is created. "extensions": [ "github.vscode-pull-request-github", "rogalmic.bash-debug" - ], - - // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [], - - // Uncomment the next line to run commands after the container is created - for example installing curl. - //"postCreateCommand": "ln -s /workspaces/super-linter/.automation/test /tmp/lint", - - // Uncomment when using a ptrace-based debugger like C++, Go, and Rust - // "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], - - // Uncomment to use the Docker CLI from inside the container. See https://aka.ms/vscode-remote/samples/docker-from-docker. - // "mounts": [ "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind" ], - - // Uncomment to connect as a non-root user. See https://aka.ms/vscode-remote/containers/non-root. - // "remoteUser": "vscode" + ] } diff --git a/.vscode/launch.json b/.vscode/launch.json index a02e962c..77e1ced2 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,7 +1,4 @@ { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ { From d8608a6cf128cc6dfe3223275b98a6dc98459729 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Fri, 26 Jun 2020 14:21:37 -0500 Subject: [PATCH 45/65] solve name --- README.md | 1 + lib/linter.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 42fbcbc2..1b9b7e09 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,7 @@ and won't run anything unexpected. | **VALIDATE_PHP** | `true` | Flag to enable or disable the linting process of the language. | | **VALIDATE_PYTHON** | `true` | Flag to enable or disable the linting process of the language. | | **VALIDATE_RUBY** | `true` | Flag to enable or disable the linting process of the language. | +| **RUBY_CONFIG_FILE** | `.ruby-lint.yml` | Filename for [rubocop configuration](https://docs.rubocop.org/rubocop/configuration.html) (ex: `.ruby-lint.yml`, `.rubocop.yml`)| | **VALIDATE_COFFEE** | `true` | Flag to enable or disable the linting process of the language . | | **VALIDATE_ANSIBLE** | `true` | Flag to enable or disable the linting process of the language. | | **VALIDATE_JAVASCRIPT_ES** | `true` | Flag to enable or disable the linting process of the language. (Utilizing: eslint) | diff --git a/lib/linter.sh b/lib/linter.sh index da97c9f2..6226455c 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -23,7 +23,7 @@ MD_LINTER_RULES="$DEFAULT_RULES_LOCATION/$MD_FILE_NAME" # Path to th PYTHON_FILE_NAME='.python-lint' # Name of the file PYTHON_LINTER_RULES="$DEFAULT_RULES_LOCATION/$PYTHON_FILE_NAME" # Path to the python lint rules # Ruby Vars -RUBY_FILE_NAME='.ruby-lint.yml' # Name of the file +RUBY_FILE_NAME="${RUBY_CONFIG_FILE:-.ruby-lint.yml}" # Name of the file RUBY_LINTER_RULES="$DEFAULT_RULES_LOCATION/$RUBY_FILE_NAME" # Path to the ruby lint rules # Coffee Vars COFFEE_FILE_NAME='.coffee-lint.json' # Name of the file From d155ff2a3c4d2034e77ad66aa9bec5ab590d26ab Mon Sep 17 00:00:00 2001 From: Drew Ayling Date: Fri, 26 Jun 2020 16:16:22 -0600 Subject: [PATCH 46/65] Fix honoring pylint config file The pylint config file is correctly being applied to the pylint command, however, the `-E` flag is actually not honoring any of the disabled messages in the `.python-lint` file. By making this change to the default lint file only the errors are output as expected but honoring the disabled messages. Also, disable the score as the -E flag also does. --- .github/linters/.python-lint | 3 ++- TEMPLATES/.python-lint | 3 ++- lib/linter.sh | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/linters/.python-lint b/.github/linters/.python-lint index 8e9cc00e..e3d9f5b5 100644 --- a/.github/linters/.python-lint +++ b/.github/linters/.python-lint @@ -1,4 +1,5 @@ [MASTER] +errors-only= # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may @@ -157,7 +158,7 @@ output-format=text reports=no # Activate the evaluation score. -score=yes +score=no [REFACTORING] diff --git a/TEMPLATES/.python-lint b/TEMPLATES/.python-lint index 8e9cc00e..e3d9f5b5 100644 --- a/TEMPLATES/.python-lint +++ b/TEMPLATES/.python-lint @@ -1,4 +1,5 @@ [MASTER] +errors-only= # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may @@ -157,7 +158,7 @@ output-format=text reports=no # Activate the evaluation score. -score=yes +score=no [REFACTORING] diff --git a/lib/linter.sh b/lib/linter.sh index 6226455c..6cf8d46d 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -2300,7 +2300,7 @@ RunTestCases() TestCodebase "XML" "xmllint" "xmllint" ".*\.\(xml\)\$" TestCodebase "MARKDOWN" "markdownlint" "markdownlint -c $MD_LINTER_RULES" ".*\.\(md\)\$" TestCodebase "BASH" "shellcheck" "shellcheck" ".*\.\(sh\)\$" - TestCodebase "PYTHON" "pylint" "pylint --rcfile $PYTHON_LINTER_RULES -E" ".*\.\(py\)\$" + TestCodebase "PYTHON" "pylint" "pylint --rcfile $PYTHON_LINTER_RULES" ".*\.\(py\)\$" TestCodebase "PERL" "perl" "perl -Mstrict -cw" ".*\.\(pl\)\$" TestCodebase "PHP" "php" "php -l" ".*\.\(php\)\$" TestCodebase "RUBY" "rubocop" "rubocop -c $RUBY_LINTER_RULES" ".*\.\(rb\)\$" @@ -2472,7 +2472,7 @@ if [ "$VALIDATE_PYTHON" == "true" ]; then # Lint the python files # ######################### # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY" - LintCodebase "PYTHON" "pylint" "pylint --rcfile $PYTHON_LINTER_RULES -E" ".*\.\(py\)\$" "${FILE_ARRAY_PYTHON[@]}" + LintCodebase "PYTHON" "pylint" "pylint --rcfile $PYTHON_LINTER_RULES" ".*\.\(py\)\$" "${FILE_ARRAY_PYTHON[@]}" fi ################ From fb90d7952412d1d50a85d9f597b82a2c4f3f34ad Mon Sep 17 00:00:00 2001 From: yoheimuta Date: Sun, 28 Jun 2020 20:07:51 +0900 Subject: [PATCH 47/65] Add protolint support to super-linter --- .automation/test/protobuf/README.md | 19 +++++++ .../test/protobuf/protobuf_bad_1.proto | 48 ++++++++++++++++ .../test/protobuf/protobuf_good_1.proto | 18 ++++++ .github/linters/.protolintrc.yml | 7 +++ Dockerfile | 9 +++ README.md | 2 + TEMPLATES/.protolintrc.yml | 7 +++ docs/disabling-linters.md | 48 ++++++++++++++++ lib/linter.sh | 55 ++++++++++++++++++- 9 files changed, 211 insertions(+), 2 deletions(-) create mode 100644 .automation/test/protobuf/README.md create mode 100644 .automation/test/protobuf/protobuf_bad_1.proto create mode 100644 .automation/test/protobuf/protobuf_good_1.proto create mode 100644 .github/linters/.protolintrc.yml create mode 100644 TEMPLATES/.protolintrc.yml diff --git a/.automation/test/protobuf/README.md b/.automation/test/protobuf/README.md new file mode 100644 index 00000000..7048213f --- /dev/null +++ b/.automation/test/protobuf/README.md @@ -0,0 +1,19 @@ +# Protobuf Test Cases + +This folder holds the test cases for **Protobuf**. + +## Additional Docs + +No Additional information is needed for this test case. + +## Good Test Cases + +The test cases denoted: `LANGUAGE_good_FILE.EXTENSION` are all valid, and should pass successfully when linted. + +- **Note:** They are linted utilizing the default linter rules. + +## Bad Test Cases + +The test cases denoted: `LANGUAGE_bad_FILE.EXTENSION` are **NOT** valid, and should trigger errors when linted. + +- **Note:** They are linted utilizing the default linter rules. diff --git a/.automation/test/protobuf/protobuf_bad_1.proto b/.automation/test/protobuf/protobuf_bad_1.proto new file mode 100644 index 00000000..b42980ba --- /dev/null +++ b/.automation/test/protobuf/protobuf_bad_1.proto @@ -0,0 +1,48 @@ +syntax = "proto3"; +// A broken example of the official reference +// See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#proto_file +package examplePb; + +option java_package = "com.example.foo"; + +import "other.proto"; +import public "new.proto"; + +import "google/protobuf/empty.proto"; +import "google/protobuf/timestamp.proto"; + +import "myproject/other_protos.proto"; +import "myproject/main_protos.proto"; + +enum enumAllowingAlias { + option allow_alias = true; + UNKNOWN = 0; + STARTED = 1; + RUNNING = 2 [(custom_option) = "hello world"]; +} +message outer { + option (my_option).a = true; + // inner is an inner message. + message inner { // Level 2 + int64 ival = 1; + } + repeated inner inner_message = 2; + EnumAllowingAlias enum_field =3; + map my_map = 4; + string reason_for_error = 5; + string end_of_support_version= 6; + message AccountForAdmin {} + message SpecialEndOfSupport {} + required inner inner_message = 7; + group Result = 8 { + string url = 9; + } + repeated group Result = 10 { + } + repeated inner paper = 11; + repeated group Regular = 12 { + } +} +service SearchApi { + rpc search (SearchRequest) returns (SearchResponse) {}; +}; diff --git a/.automation/test/protobuf/protobuf_good_1.proto b/.automation/test/protobuf/protobuf_good_1.proto new file mode 100644 index 00000000..76022990 --- /dev/null +++ b/.automation/test/protobuf/protobuf_good_1.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; +import public "other.proto"; +option java_package = "com.example.foo"; +enum EnumAllowingAlias { + option allow_alias = true; + ALLOWING_UNSPECIFIED = 0; + STARTED = 1; + RUNNING = 2 [(custom_option) = "hello world"]; +} +message Outer { + option (my_option).a = true; + message Inner { // Level 2 + int64 ival = 1; + } + inner inner_message = 2; + EnumAllowingAlias enum_field =3; + map my_map = 4; +} diff --git a/.github/linters/.protolintrc.yml b/.github/linters/.protolintrc.yml new file mode 100644 index 00000000..7bd3e0ce --- /dev/null +++ b/.github/linters/.protolintrc.yml @@ -0,0 +1,7 @@ +# Lint directives. +lint: + # Linter rules. + # Run `protolint list` to see all available rules. + rules: + # Set the default to all linters. + all_default: false diff --git a/Dockerfile b/Dockerfile index d0c3a70b..46e6a686 100644 --- a/Dockerfile +++ b/Dockerfile @@ -126,6 +126,14 @@ RUN wget -O- -nvq https://raw.githubusercontent.com/golangci/golangci-lint/maste RUN curl -Ls "$(curl -Ls https://api.github.com/repos/terraform-linters/tflint/releases/latest | grep -o -E "https://.+?_linux_amd64.zip")" -o tflint.zip && unzip tflint.zip && rm tflint.zip \ && mv "tflint" /usr/bin/ +###################### +# Install protolint # +###################### +RUN curl -LsS "$(curl -Ls https://api.github.com/repos/yoheimuta/protolint/releases/latest | grep -o -E "https://.+?_Linux_x86_64.tar.gz")" -o protolint.tar.gz \ + && tar -xzf protolint.tar.gz \ + && rm protolint.tar.gz \ + && mv "protolint" /usr/bin/ + ######################### # Install dotenv-linter # ######################### @@ -180,6 +188,7 @@ ENV GITHUB_SHA=${GITHUB_SHA} \ VALIDATE_KOTLIN=${VALIDATE_KOTLIN} \ VALIDATE_POWERSHELL=${VALIDATE_POWERSHELL} \ VALIDATE_OPENAPI=${VALIDATE_OPENAPI} \ + VALIDATE_PROTOBUF=${VALIDATE_PROTOBUF} \ ANSIBLE_DIRECTORY=${ANSIBLE_DIRECTORY} \ RUN_LOCAL=${RUN_LOCAL} \ TEST_CASE_RUN=${TEST_CASE_RUN} \ diff --git a/README.md b/README.md index 1b9b7e09..ab9efb85 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ Developers on **GitHub** can call the **GitHub Action** to lint their code base | **ENV** | [dotenv-linter](https://github.com/dotenv-linter/dotenv-linter) | | **Kotlin** | [ktlint](https://github.com/pinterest/ktlint) | | **OpenAPI** | [spectral](https://github.com/stoplightio/spectral) | +| **Protocol Buffers** | [protolint](https://github.com/yoheimuta/protolint) | ## How to use To use this **GitHub** Action you will need to complete the following: @@ -167,6 +168,7 @@ and won't run anything unexpected. | **VALIDATE_CLOJURE** | `true` | Flag to enable or disable the linting process of the language. | | **VALIDATE_KOTLIN** | `true` | Flag to enable or disable the linting process of the language. | | **VALIDATE_OPENAPI** | `true` | Flag to enable or disable the linting process of the language. | +| **VALIDATE_PROTOBUF** | `true` | Flag to enable or disable the linting process of the language. | | **ANSIBLE_DIRECTORY** | `/ansible` | Flag to set the root directory for Ansible file location(s). | | **ACTIONS_RUNNER_DEBUG** | `false` | Flag to enable additional information about the linter, versions, and additional output. | | **DISABLE_ERRORS** | `false` | Flag to have the linter complete with exit code 0 even if errors were detected. | diff --git a/TEMPLATES/.protolintrc.yml b/TEMPLATES/.protolintrc.yml new file mode 100644 index 00000000..7bd3e0ce --- /dev/null +++ b/TEMPLATES/.protolintrc.yml @@ -0,0 +1,7 @@ +# Lint directives. +lint: + # Linter rules. + # Run `protolint list` to see all available rules. + rules: + # Set the default to all linters. + all_default: false diff --git a/docs/disabling-linters.md b/docs/disabling-linters.md index 408eba19..f08fd22b 100644 --- a/docs/disabling-linters.md +++ b/docs/disabling-linters.md @@ -25,6 +25,7 @@ Below are examples and documentation for each language and the various methods t - [ENV](#dotenv-linter) - [Kotlin](#kotlin) - [OpenAPI](#openapi) +- [Protocol Buffers](#protocol-buffers) @@ -632,6 +633,53 @@ import package.b.* -------------------------------------------------------------------------------- +## Protocol Buffers + +- [protolint](https://github.com/yoheimuta/protolint) + +### protolint Config file + +- `.github/linters/.protolintrc.yml` +- You can add, extend, and disable rules +- Documentation at [Rules](https://github.com/yoheimuta/protolint#rules) and [Configuring](https://github.com/yoheimuta/protolint#configuring) + +### protolint disable single line + +```protobuf +enum Foo { + // protolint:disable:next ENUM_FIELD_NAMES_UPPER_SNAKE_CASE + firstValue = 0; + second_value = 1; // protolint:disable:this ENUM_FIELD_NAMES_UPPER_SNAKE_CASE + THIRD_VALUE = 2; +} +``` + +### protolint disable code block + +```protobuf +// protolint:disable ENUM_FIELD_NAMES_UPPER_SNAKE_CASE +enum Foo { + firstValue = 0; + second_value = 1; + THIRD_VALUE = 2; +} +// protolint:enable ENUM_FIELD_NAMES_UPPER_SNAKE_CASE +``` + +### protolint disable entire file + +- You can disable entire files with the `lint.files.exclude` property in `.protolintrc.yml` + +```yaml +# Lint directives. +lint: + # Linter files to walk. + files: + # The specific files to exclude. + exclude: + - path/to/file +``` + ## Clojure - [clj-kondo](https://github.com/borkdude/clj-kondo) - Since clj-kondo approaches static analysis in a very Clojure way, it is advised to read the [configuration docs](https://github.com/borkdude/clj-kondo/blob/master/doc/config.md) diff --git a/lib/linter.sh b/lib/linter.sh index 6226455c..69523ee4 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -57,6 +57,9 @@ CSS_LINTER_RULES="$DEFAULT_RULES_LOCATION/$CSS_FILE_NAME" # Path to th # OpenAPI Vars OPENAPI_FILE_NAME='.openapirc.yml' # Name of the file OPENAPI_LINTER_RULES="$DEFAULT_RULES_LOCATION/$OPENAPI_FILE_NAME" # Path to the OpenAPI lint rules +# Protocol Buffers Vars +PROTOBUF_FILE_NAME='.protolintrc.yml' # Name of the file +PROTOBUF_LINTER_RULES="$DEFAULT_RULES_LOCATION/$PROTOBUF_FILE_NAME" # Path to the Protocol Buffers lint rules # Clojure Vars CLOJURE_FILE_NAME='.clj-kondo/config.edn' CLOJURE_LINTER_RULES="$DEFAULT_RULES_LOCATION/$CLOJURE_FILE_NAME" @@ -67,7 +70,7 @@ CLOJURE_LINTER_RULES="$DEFAULT_RULES_LOCATION/$CLOJURE_FILE_NAME" LINTER_ARRAY=("jsonlint" "yamllint" "xmllint" "markdownlint" "shellcheck" "pylint" "perl" "rubocop" "coffeelint" "eslint" "standard" "ansible-lint" "/dockerfilelint/bin/dockerfilelint" "golangci-lint" "tflint" - "stylelint" "dotenv-linter" "powershell" "ktlint" "clj-kondo" "spectral") + "stylelint" "dotenv-linter" "powershell" "ktlint" "protolint" "clj-kondo" "spectral") ############################# # Language array for prints # @@ -75,7 +78,7 @@ LINTER_ARRAY=("jsonlint" "yamllint" "xmllint" "markdownlint" "shellcheck" LANGUAGE_ARRAY=('YML' 'JSON' 'XML' 'MARKDOWN' 'BASH' 'PERL' 'PHP' 'RUBY' 'PYTHON' 'COFFEESCRIPT' 'ANSIBLE' 'JAVASCRIPT_STANDARD' 'JAVASCRIPT_ES' 'TYPESCRIPT_STANDARD' 'TYPESCRIPT_ES' 'DOCKER' 'GO' 'TERRAFORM' - 'CSS' 'ENV' 'POWERSHELL' 'KOTLIN' 'CLOJURE' 'OPENAPI') + 'CSS' 'ENV' 'POWERSHELL' 'KOTLIN' 'PROTO' 'CLOJURE' 'OPENAPI') ################### # GitHub ENV Vars # @@ -159,6 +162,7 @@ FILE_ARRAY_CSS=() # Array of files to check FILE_ARRAY_ENV=() # Array of files to check FILE_ARRAY_CLOJURE=() # Array of files to check FILE_ARRAY_KOTLIN=() # Array of files to check +FILE_ARRAY_PROTOBUF=() # Array of files to check FILE_ARRAY_OPENAPI=() # Array of files to check ############ @@ -187,6 +191,7 @@ ERRORS_FOUND_CSS=0 # Count of errors found ERRORS_FOUND_ENV=0 # Count of errors found ERRORS_FOUND_CLOJURE=0 # Count of errors found ERRORS_FOUND_KOTLIN=0 # Count of errors found +ERRORS_FOUND_PROTOBUF=0 # Count of errors found ERRORS_FOUND_OPENAPI=0 # Count of errors found ################################################################################ @@ -806,6 +811,7 @@ GetValidationInfo() VALIDATE_ENV=$(echo "$VALIDATE_ENV" | awk '{print tolower($0)}') VALIDATE_CLOJURE=$(echo "$VALIDATE_CLOJURE" | awk '{print tolower($0)') VALIDATE_KOTLIN=$(echo "$VALIDATE_KOTLIN" | awk '{print tolower($0)}') + VALIDATE_PROTOBUF=$(echo "$VALIDATE_PROTOBUF" | awk '{print tolower($0)}') VALIDATE_OPENAPI=$(echo "$VALIDATE_OPENAPI" | awk '{print tolower($0)}') ################################################ @@ -834,6 +840,7 @@ GetValidationInfo() -n "$VALIDATE_CSS" || \ -n "$VALIDATE_ENV" || \ -n "$VALIDATE_CLOJURE" || \ + -n "$VALIDATE_PROTOBUF" || \ -n "$VALIDATE_OPENAPI" || \ -n "$VALIDATE_KOTLIN" ]]; then ANY_SET="true" @@ -1161,6 +1168,20 @@ GetValidationInfo() VALIDATE_OPENAPI="true" fi + ####################################### + # Validate if we should check PROTOBUF # + ####################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_PROTOBUF" ]]; then + # PROTOBUF flag was not set - default to false + VALIDATE_PROTOBUF="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_PROTOBUF="true" + fi + ####################################### # Validate if we should check Clojure # ####################################### @@ -1298,6 +1319,11 @@ GetValidationInfo() else PRINT_ARRAY+=("- Excluding [OPENAPI] files in code base...") fi + if [[ "$VALIDATE_PROTOBUF" == "true" ]]; then + PRINT_ARRAY+=("- Validating [PROTOBUF] files in code base...") + else + PRINT_ARRAY+=("- Excluding [PROTOBUF] files in code base...") + fi ############################## # Validate Ansible Directory # @@ -1686,6 +1712,18 @@ BuildFileList() # Set the READ_ONLY_CHANGE_FLAG since this could be exec # ########################################################## READ_ONLY_CHANGE_FLAG=1 + ############################ + # Get the Protocol Buffers files # + ############################ + elif [ "$FILE_TYPE" == "proto" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_PROTOBUF+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 elif [ "$FILE" == "Dockerfile" ]; then ################################ # Append the file to the array # @@ -2254,6 +2292,7 @@ Footer() [ "$ERRORS_FOUND_CSS" -ne 0 ] || \ [ "$ERRORS_FOUND_ENV" -ne 0 ] || \ [ "$ERRORS_FOUND_OPENAPI" -ne 0 ] || \ + [ "$ERRORS_FOUND_PROTOBUF" -ne 0 ] || \ [ "$ERRORS_FOUND_CLOJURE" -ne 0 ] || \ [ "$ERRORS_FOUND_KOTLIN" -ne 0 ]; then # Failed exit @@ -2318,6 +2357,7 @@ RunTestCases() TestCodebase "ENV" "dotenv-linter" "dotenv-linter" ".*\.\(env\)\$" TestCodebase "CLOJURE" "clj-kondo" "clj-kondo --config $CLOJURE_LINTER_RULES --lint" ".*\.\(clj\|cljs\|cljc\|edn\)\$" TestCodebase "KOTLIN" "ktlint" "ktlint" ".*\.\(kt\|kts\)\$" + TestCodebase "PROTOBUF" "protolint" "protolint lint --config_path $PROTOBUF_LINTER_RULES" ".*\.\(proto\)\$" TestCodebase "OPENAPI" "spectral" "spectral lint -r $OPENAPI_LINTER_RULES" ".*\.\(ymlopenapi\|jsonopenapi\)\$" ################# @@ -2664,6 +2704,17 @@ if [ "$VALIDATE_CLOJURE" == "true" ]; then LintCodebase "CLOJURE" "clj-kondo" "clj-kondo --config $CLOJURE_LINTER_RULES --lint" ".*\.\(clj\|cljs\|cljc\|edn\)\$" "${FILE_ARRAY_CLOJURE[@]}" fi +################## +# PROTOBUF LINTING # +################## +if [ "$VALIDATE_PROTOBUF" == "true" ]; then + ####################### + # Lint the Protocol Buffers files # + ####################### + # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY" + LintCodebase "PROTOBUF" "protolint" "protolint lint --config_path $PROTOBUF_LINTER_RULES" ".*\.\(proto\)\$" "${FILE_ARRAY_PROTOBUF[@]}" +fi + ###################### # POWERSHELL LINTING # ###################### From 0482ce0b8ff5e4d8d05fef4035ce8641cbf4d72d Mon Sep 17 00:00:00 2001 From: Thomas Hughes Date: Sun, 28 Jun 2020 07:39:31 -0500 Subject: [PATCH 48/65] Delete welcome_new_people.yml --- .github/workflows/welcome_new_people.yml | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 .github/workflows/welcome_new_people.yml diff --git a/.github/workflows/welcome_new_people.yml b/.github/workflows/welcome_new_people.yml deleted file mode 100644 index 985ee57e..00000000 --- a/.github/workflows/welcome_new_people.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Greetings - -on: [pull_request, issues] - -jobs: - greeting: - runs-on: ubuntu-latest - steps: - - uses: actions/first-interaction@v1 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - issue-message: 'Thanks for opening your first issue in the **super-linter** repo! :tada:' - pr-message: 'Thanks for opening your first pr in the **super-linter** repo! :tada: This project is built on contributions just like this.' From a4fa97234ad9dfa3a749d7bdddd329ed17d458a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Knudsen?= Date: Mon, 29 Jun 2020 09:07:40 +0200 Subject: [PATCH 49/65] Read Clojure filetypes from $FILE_TYPE, not $FILE --- lib/linter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/linter.sh b/lib/linter.sh index 6226455c..d7e025a9 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -1695,7 +1695,7 @@ BuildFileList() # Set the READ_ONLY_CHANGE_FLAG since this could be exec # ########################################################## READ_ONLY_CHANGE_FLAG=1 - elif [ "$FILE" == "clj" ] || [ "$FILE" == "cljs" ] || [ "$FILE" == "cljc" ] || [ "$FILE" == "edn" ]; then + elif [ "$FILE_TYPE" == "clj" ] || [ "$FILE_TYPE" == "cljs" ] || [ "$FILE_TYPE" == "cljc" ] || [ "$FILE_TYPE" == "edn" ]; then ################################ # Append the file to the array # ################################ From 3975731d07ad4e849c08aa2c8bbf8d1fbb2f56cf Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 09:08:04 -0500 Subject: [PATCH 50/65] cleanup --- .vscode/testlinter.sh | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/.vscode/testlinter.sh b/.vscode/testlinter.sh index c6a54f00..e27c6ac2 100644 --- a/.vscode/testlinter.sh +++ b/.vscode/testlinter.sh @@ -1,12 +1,26 @@ -#!/bin/bash -#At this point you can use the debug console to add export GITHUB_WORKSPACE=/path to test only a specific folder -#You can also use ln -s /path /tmp/lint as an alternative -#If you do neither, this will default to running against the test automation files -tmppath=/tmp/lint -if [ ! -L $tmppath ]; then - ln -s "$PWD"/.automation/test $tmppath +#!/usr/bin/env bash + +# At this point you can use the debug console to add export GITHUB_WORKSPACE=/path to test only a specific folder +# You can also use ln -s /path /tmp/lint as an alternative +# If you do neither, this will default to running against the test automation files + +########### +# GLOBALS # +########### +CODE_PATH='/tmp/lint' # Path to code base + +################## +# Check the path # +################## +if [ ! -L $CODE_PATH ]; then + # Create symbolic link + ln -s "$PWD"/.automation/test $CODE_PATH fi +######################### +# Export to run locally # +######################### export RUN_LOCAL=true -# shellcheck source=lib/linter.sh -source "$PWD"/lib/linter.sh \ No newline at end of file + +# # shellcheck disable=SC1091 +source "$PWD"/lib/linter.sh From a578c20faa765916e7a72f2325efb456df398d07 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 09:13:04 -0500 Subject: [PATCH 51/65] fixing linkage --- .vscode/testlinter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/testlinter.sh b/.vscode/testlinter.sh index e27c6ac2..dc45a319 100644 --- a/.vscode/testlinter.sh +++ b/.vscode/testlinter.sh @@ -22,5 +22,5 @@ fi ######################### export RUN_LOCAL=true -# # shellcheck disable=SC1091 +# # shellcheck disable=SC1091,SC1090 source "$PWD"/lib/linter.sh From 529df42adb2aa1c10d8dfef320ac7e507ce5a1fe Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 09:16:17 -0500 Subject: [PATCH 52/65] fixed double pound --- .vscode/testlinter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/testlinter.sh b/.vscode/testlinter.sh index dc45a319..8d778d16 100644 --- a/.vscode/testlinter.sh +++ b/.vscode/testlinter.sh @@ -22,5 +22,5 @@ fi ######################### export RUN_LOCAL=true -# # shellcheck disable=SC1091,SC1090 +# shellcheck disable=SC1091,SC1090 source "$PWD"/lib/linter.sh From 701abcc065b486a089028429514e466052c6fc24 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 09:55:59 -0500 Subject: [PATCH 53/65] creating files --- lib/buildFileList.sh | 420 ++++++++++ lib/linter.sh | 1729 +----------------------------------------- lib/validation.sh | 669 ++++++++++++++++ lib/worker.sh | 658 ++++++++++++++++ 4 files changed, 1754 insertions(+), 1722 deletions(-) create mode 100755 lib/buildFileList.sh create mode 100755 lib/validation.sh create mode 100755 lib/worker.sh diff --git a/lib/buildFileList.sh b/lib/buildFileList.sh new file mode 100755 index 00000000..2ed2ce60 --- /dev/null +++ b/lib/buildFileList.sh @@ -0,0 +1,420 @@ +#!/usr/bin/env bash + +################################################################################ +################################################################################ +########### Super-Linter Build File List Functions @admiralawkbar ############## +################################################################################ +################################################################################ +########################## FUNCTION CALLS BELOW ################################ +################################################################################ +################################################################################ +#### Function BuildFileList #################################################### +function BuildFileList() +{ + # Need to build a list of all files changed + # This can be pulled from the GITHUB_EVENT_PATH payload + + ################ + # print header # + ################ + if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then + echo "" + echo "----------------------------------------------" + echo "Pulling in code history and branches..." + fi + + ################################################################################# + # Switch codebase back to the default branch to get a list of all files changed # + ################################################################################# + SWITCH_CMD=$(cd "$GITHUB_WORKSPACE" || exit; git pull --quiet; git checkout "$DEFAULT_BRANCH" 2>&1) + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + # Error + echo "Failed to switch to $DEFAULT_BRANCH branch to get files changed!" + echo "ERROR:[$SWITCH_CMD]" + exit 1 + fi + + ################ + # print header # + ################ + if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then + echo "" + echo "----------------------------------------------" + echo "Generating Diff with:[git diff --name-only '$DEFAULT_BRANCH..$GITHUB_SHA' --diff-filter=d]" + fi + + ################################################# + # Get the Array of files changed in the commits # + ################################################# + # shellcheck disable=SC2207 + RAW_FILE_ARRAY=($(cd "$GITHUB_WORKSPACE" || exit; git diff --name-only "$DEFAULT_BRANCH..$GITHUB_SHA" --diff-filter=d 2>&1)) + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + # Error + echo "ERROR! Failed to gain a list of all files changed!" + echo "ERROR:[${RAW_FILE_ARRAY[*]}]" + exit 1 + fi + + ################################################ + # Iterate through the array of all files found # + ################################################ + echo "" + echo "----------------------------------------------" + echo "Files that have been modified in the commit(s):" + for FILE in "${RAW_FILE_ARRAY[@]}" + do + ############## + # Print file # + ############## + echo "File:[$FILE]" + + ########################### + # Get the files extension # + ########################### + # Extract just the file and extension, reverse it, cut off extension, + # reverse it back, substitute to lowercase + FILE_TYPE=$(basename "$FILE" | rev | cut -f1 -d'.' | rev | awk '{print tolower($0)}') + + ######### + # DEBUG # + ######### + #echo "FILE_TYPE:[$FILE_TYPE]" + + ##################### + # Get the YML files # + ##################### + if [ "$FILE_TYPE" == "yml" ] || [ "$FILE_TYPE" == "yaml" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_YML+=("$FILE") + ############################ + # Check if file is OpenAPI # + ############################ + if DetectOpenAPIFile "$FILE"; then + FILE_ARRAY_OPENAPI+=("$FILE") + fi + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ###################### + # Get the JSON files # + ###################### + elif [ "$FILE_TYPE" == "json" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_JSON+=("$FILE") + ############################ + # Check if file is OpenAPI # + ############################ + if DetectOpenAPIFile "$FILE"; then + FILE_ARRAY_OPENAPI+=("$FILE") + fi + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ##################### + # Get the XML files # + ##################### + elif [ "$FILE_TYPE" == "xml" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_XML+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ########################## + # Get the MARKDOWN files # + ########################## + elif [ "$FILE_TYPE" == "md" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_MD+=("$FILE") + ###################### + # Get the BASH files # + ###################### + elif [ "$FILE_TYPE" == "sh" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_BASH+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ###################### + # Get the PERL files # + ###################### + elif [ "$FILE_TYPE" == "pl" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_PERL+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ###################### + # Get the PHP files # + ###################### + elif [ "$FILE_TYPE" == "php" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_PHP+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ###################### + # Get the RUBY files # + ###################### + elif [ "$FILE_TYPE" == "rb" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_RUBY+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ######################## + # Get the PYTHON files # + ######################## + elif [ "$FILE_TYPE" == "py" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_PYTHON+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ######################## + # Get the COFFEE files # + ######################## + elif [ "$FILE_TYPE" == "coffee" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_COFFEESCRIPT+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ############################ + # Get the JavaScript files # + ############################ + elif [ "$FILE_TYPE" == "js" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_JAVASCRIPT_ES+=("$FILE") + FILE_ARRAY_JAVASCRIPT_STANDARD+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ############################ + # Get the TypeScript files # + ############################ + elif [ "$FILE_TYPE" == "ts" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_TYPESCRIPT_ES+=("$FILE") + FILE_ARRAY_TYPESCRIPT_STANDARD+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ######################## + # Get the Golang files # + ######################## + elif [ "$FILE_TYPE" == "go" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_GO+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ########################### + # Get the Terraform files # + ########################### + elif [ "$FILE_TYPE" == "tf" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_TERRAFORM+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ########################### + # Get the Powershell files # + ########################### + elif [ "$FILE_TYPE" == "ps1" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_POWERSHELL+=("$FILE") + elif [ "$FILE_TYPE" == "css" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_CSS+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + elif [ "$FILE_TYPE" == "env" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_ENV+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + elif [ "$FILE_TYPE" == "kt" ] || [ "$FILE_TYPE" == "kts" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_KOTLIN+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + ############################ + # Get the Protocol Buffers files # + ############################ + elif [ "$FILE_TYPE" == "proto" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_PROTOBUF+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + elif [ "$FILE" == "Dockerfile" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_DOCKER+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + elif [ "$FILE_TYPE" == "clj" ] || [ "$FILE_TYPE" == "cljs" ] || [ "$FILE_TYPE" == "cljc" ] || [ "$FILE_TYPE" == "edn" ]; then + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_CLOJURE+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + else + ############################################## + # Use file to see if we can parse what it is # + ############################################## + GET_FILE_TYPE_CMD=$(file "$FILE" 2>&1) + + ################# + # Check if bash # + ################# + if [[ "$GET_FILE_TYPE_CMD" == *"Bourne-Again shell script"* ]]; then + ####################### + # It is a bash script # + ####################### + echo "WARN! Found bash script without extension:[.sh]" + echo "Please update file with proper extensions." + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_BASH+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + elif [[ "$GET_FILE_TYPE_CMD" == *"Ruby script"* ]]; then + ####################### + # It is a Ruby script # + ####################### + echo "WARN! Found ruby script without extension:[.rb]" + echo "Please update file with proper extensions." + ################################ + # Append the file to the array # + ################################ + FILE_ARRAY_RUBY+=("$FILE") + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + else + ############################ + # Extension was not found! # + ############################ + echo " - WARN! Failed to get filetype for:[$FILE]!" + ########################################################## + # Set the READ_ONLY_CHANGE_FLAG since this could be exec # + ########################################################## + READ_ONLY_CHANGE_FLAG=1 + fi + fi + done + + ######################################### + # Need to switch back to branch of code # + ######################################### + SWITCH2_CMD=$(cd "$GITHUB_WORKSPACE" || exit; git checkout --progress --force "$GITHUB_SHA" 2>&1) + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + # Error + echo "Failed to switch back to branch!" + echo "ERROR:[$SWITCH2_CMD]" + exit 1 + fi + + ################ + # Footer print # + ################ + echo "" + echo "----------------------------------------------" + echo "Successfully gathered list of files..." +} diff --git a/lib/linter.sh b/lib/linter.sh index 8b3bc141..f1dad6ec 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -7,6 +7,12 @@ ################################################################################ ################################################################################ +######################### +# Source Function Files # +######################### +source ./validation.sh # Source the main validation function script +source ./buildFileList.sh # Source the main build file list function script + ########### # GLOBALS # ########### @@ -393,188 +399,6 @@ GetStandardRules() TYPESCRIPT_STANDARD_LINTER_RULES="$(echo -e "${ENV_STRING}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" fi } -################################################################################ -#### Function LintAnsibleFiles ################################################# -LintAnsibleFiles() -{ - ###################### - # Create Print Array # - ###################### - PRINT_ARRAY=() - - ################ - # print header # - ################ - PRINT_ARRAY+=("") - PRINT_ARRAY+=("----------------------------------------------") - PRINT_ARRAY+=("----------------------------------------------") - PRINT_ARRAY+=("Linting [Ansible] files...") - PRINT_ARRAY+=("----------------------------------------------") - PRINT_ARRAY+=("----------------------------------------------") - - ###################### - # Name of the linter # - ###################### - LINTER_NAME="ansible-lint" - - ########################################### - # Validate we have ansible-lint installed # - ########################################### - # shellcheck disable=SC2230 - VALIDATE_INSTALL_CMD=$(command -v "$LINTER_NAME" 2>&1) - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ $ERROR_CODE -ne 0 ]; then - # Failed - echo "ERROR! Failed to find $LINTER_NAME in system!" - echo "ERROR:[$VALIDATE_INSTALL_CMD]" - exit 1 - else - # Success - if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then - # Success - echo "Successfully found binary in system" - echo "Location:[$VALIDATE_INSTALL_CMD]" - fi - fi - - ########################## - # Initialize empty Array # - ########################## - LIST_FILES=() - - ####################### - # Create flag to skip # - ####################### - SKIP_FLAG=0 - - ###################################################### - # Only go into ansible linter if we have base folder # - ###################################################### - if [ -d "$ANSIBLE_DIRECTORY" ]; then - - ############################################################ - # Check to see if we need to go through array or all files # - ############################################################ - if [ "$VALIDATE_ALL_CODEBASE" == "false" ]; then - # We need to only check the ansible playbooks that have updates - #LIST_FILES=("${ANSIBLE_ARRAY[@]}") - # shellcheck disable=SC2164,SC2010,SC2207 - LIST_FILES=($(cd "$ANSIBLE_DIRECTORY"; ls | grep ".yml" 2>&1)) - else - ################################# - # Get list of all files to lint # - ################################# - # shellcheck disable=SC2164,SC2010,SC2207 - LIST_FILES=($(cd "$ANSIBLE_DIRECTORY"; ls | grep ".yml" 2>&1)) - fi - - ############################################################### - # Set the list to empty if only MD and TXT files were changed # - ############################################################### - # No need to run the full ansible checks on read only file changes - if [ "$READ_ONLY_CHANGE_FLAG" -eq 0 ]; then - ########################## - # Set the array to empty # - ########################## - LIST_FILES=() - ################################### - # Send message that were skipping # - ################################### - #echo "- Skipping Ansible lint run as file(s) that were modified were read only..." - ############################ - # Create flag to skip loop # - ############################ - SKIP_FLAG=1 - fi - - #################################### - # Check if we have data to look at # - #################################### - if [ $SKIP_FLAG -eq 0 ]; then - for LINE in "${PRINT_ARRAY[@]}" - do - ######################### - # Print the header line # - ######################### - echo "$LINE" - done - fi - - ################## - # Lint the files # - ################## - for FILE in "${LIST_FILES[@]}" - do - - ######################################## - # Make sure we dont lint certain files # - ######################################## - if [[ $FILE == *"vault.yml"* ]] || [[ $FILE == *"galaxy.yml"* ]]; then - # This is a file we dont look at - continue - fi - - #################### - # Get the filename # - #################### - FILE_NAME=$(basename "$ANSIBLE_DIRECTORY/$FILE" 2>&1) - - ############## - # File print # - ############## - echo "---------------------------" - echo "File:[$FILE]" - - ################################ - # Lint the file with the rules # - ################################ - LINT_CMD=$("$LINTER_NAME" -v -c "$ANSIBLE_LINTER_RULES" "$ANSIBLE_DIRECTORY/$FILE" 2>&1) - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ $ERROR_CODE -ne 0 ]; then - ######### - # Error # - ######### - echo "ERROR! Found errors in [$LINTER_NAME] linter!" - echo "ERROR:[$LINT_CMD]" - # Increment error count - ((ERRORS_FOUND_ANSIBLE++)) - else - ########### - # Success # - ########### - echo " - File:[$FILE_NAME] was linted with [$LINTER_NAME] successfully" - fi - done - else # No ansible directory found in path - ############################### - # Check to see if debug is on # - ############################### - if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then - ######################## - # No Ansible dir found # - ######################## - echo "WARN! No Ansible base directory found at:[$ANSIBLE_DIRECTORY]" - echo "skipping ansible lint" - fi - fi -} - ################################################################################ #### Function DetectOpenAPIFile ################################################ DetectOpenAPIFile() @@ -666,7 +490,7 @@ GetGitHubVars() if [ -z "$GITHUB_WORKSPACE" ]; then GITHUB_WORKSPACE="$DEFAULT_WORKSPACE" fi - + echo "Linting all files in mapped directory:[$DEFAULT_WORKSPACE]" # No need to touch or set the GITHUB_SHA @@ -753,1488 +577,6 @@ GetGitHubVars() fi } ################################################################################ -#### Function GetValidationInfo ################################################ -GetValidationInfo() -{ - ############################################ - # Print headers for user provided env vars # - ############################################ - echo "" - echo "--------------------------------------------" - echo "Gathering user validation information..." - - ########################################### - # Skip validation if were running locally # - ########################################### - if [[ "$RUN_LOCAL" != "true" ]]; then - ############################### - # Convert string to lowercase # - ############################### - VALIDATE_ALL_CODEBASE=$(echo "$VALIDATE_ALL_CODEBASE" | awk '{print tolower($0)}') - ###################################### - # Validate we should check all files # - ###################################### - if [[ "$VALIDATE_ALL_CODEBASE" != "false" ]]; then - # Set to true - VALIDATE_ALL_CODEBASE="$DEFAULT_VALIDATE_ALL_CODEBASE" - echo "- Validating ALL files in code base..." - else - # Its false - echo "- Only validating [new], or [edited] files in code base..." - fi - fi - - ###################### - # Create Print Array # - ###################### - PRINT_ARRAY=() - - ################################ - # Convert strings to lowercase # - ################################ - VALIDATE_YAML=$(echo "$VALIDATE_YAML" | awk '{print tolower($0)}') - VALIDATE_JSON=$(echo "$VALIDATE_JSON" | awk '{print tolower($0)}') - VALIDATE_XML=$(echo "$VALIDATE_XML" | awk '{print tolower($0)}') - VALIDATE_MD=$(echo "$VALIDATE_MD" | awk '{print tolower($0)}') - VALIDATE_BASH=$(echo "$VALIDATE_BASH" | awk '{print tolower($0)}') - VALIDATE_PERL=$(echo "$VALIDATE_PERL" | awk '{print tolower($0)}') - VALIDATE_PHP=$(echo "$VALIDATE_PHP" | awk '{print tolower($0)}') - VALIDATE_PYTHON=$(echo "$VALIDATE_PYTHON" | awk '{print tolower($0)}') - VALIDATE_RUBY=$(echo "$VALIDATE_RUBY" | awk '{print tolower($0)}') - VALIDATE_COFFEE=$(echo "$VALIDATE_COFFEE" | awk '{print tolower($0)}') - VALIDATE_ANSIBLE=$(echo "$VALIDATE_ANSIBLE" | awk '{print tolower($0)}') - VALIDATE_JAVASCRIPT_ES=$(echo "$VALIDATE_JAVASCRIPT_ES" | awk '{print tolower($0)}') - VALIDATE_JAVASCRIPT_STANDARD=$(echo "$VALIDATE_JAVASCRIPT_STANDARD" | awk '{print tolower($0)}') - VALIDATE_TYPESCRIPT_ES=$(echo "$VALIDATE_TYPESCRIPT_ES" | awk '{print tolower($0)}') - VALIDATE_TYPESCRIPT_STANDARD=$(echo "$VALIDATE_TYPESCRIPT_STANDARD" | awk '{print tolower($0)}') - VALIDATE_DOCKER=$(echo "$VALIDATE_DOCKER" | awk '{print tolower($0)}') - VALIDATE_GO=$(echo "$VALIDATE_GO" | awk '{print tolower($0)}') - VALIDATE_TERRAFORM=$(echo "$VALIDATE_TERRAFORM" | awk '{print tolower($0)}') - VALIDATE_POWERSHELL=$(echo "$VALIDATE_POWERSHELL" | awk '{print tolower($0)}') - VALIDATE_CSS=$(echo "$VALIDATE_CSS" | awk '{print tolower($0)}') - VALIDATE_ENV=$(echo "$VALIDATE_ENV" | awk '{print tolower($0)}') - VALIDATE_CLOJURE=$(echo "$VALIDATE_CLOJURE" | awk '{print tolower($0)') - VALIDATE_KOTLIN=$(echo "$VALIDATE_KOTLIN" | awk '{print tolower($0)}') - VALIDATE_PROTOBUF=$(echo "$VALIDATE_PROTOBUF" | awk '{print tolower($0)}') - VALIDATE_OPENAPI=$(echo "$VALIDATE_OPENAPI" | awk '{print tolower($0)}') - - ################################################ - # Determine if any linters were explicitly set # - ################################################ - ANY_SET="false" - if [[ -n "$VALIDATE_YAML" || \ - -n "$VALIDATE_JSON" || \ - -n "$VALIDATE_XML" || \ - -n "$VALIDATE_MD" || \ - -n "$VALIDATE_BASH" || \ - -n "$VALIDATE_PERL" || \ - -n "$VALIDATE_PHP" || \ - -n "$VALIDATE_PYTHON" || \ - -n "$VALIDATE_RUBY" || \ - -n "$VALIDATE_COFFEE" || \ - -n "$VALIDATE_ANSIBLE" || \ - -n "$VALIDATE_JAVASCRIPT_ES" || \ - -n "$VALIDATE_JAVASCRIPT_STANDARD" || \ - -n "$VALIDATE_TYPESCRIPT_ES" || \ - -n "$VALIDATE_TYPESCRIPT_STANDARD" || \ - -n "$VALIDATE_DOCKER" || \ - -n "$VALIDATE_GO" || \ - -n "$VALIDATE_TERRAFORM" || \ - -n "$VALIDATE_POWERSHELL" || \ - -n "$VALIDATE_CSS" || \ - -n "$VALIDATE_ENV" || \ - -n "$VALIDATE_CLOJURE" || \ - -n "$VALIDATE_PROTOBUF" || \ - -n "$VALIDATE_OPENAPI" || \ - -n "$VALIDATE_KOTLIN" ]]; then - ANY_SET="true" - fi - - #################################### - # Validate if we should check YAML # - #################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_YAML" ]]; then - # YAML flag was not set - default to false - VALIDATE_YAML="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_YAML="true" - fi - - #################################### - # Validate if we should check JSON # - #################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_JSON" ]]; then - # JSON flag was not set - default to false - VALIDATE_JSON="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_JSON="true" - fi - - ################################### - # Validate if we should check XML # - ################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_XML" ]]; then - # XML flag was not set - default to false - VALIDATE_XML="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_XML="true" - fi - - ######################################## - # Validate if we should check MARKDOWN # - ######################################## - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_MD" ]]; then - # MD flag was not set - default to false - VALIDATE_MD="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_MD="true" - fi - - #################################### - # Validate if we should check BASH # - #################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_BASH" ]]; then - # BASH flag was not set - default to false - VALIDATE_BASH="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_BASH="true" - fi - - #################################### - # Validate if we should check PERL # - #################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_PERL" ]]; then - # PERL flag was not set - default to false - VALIDATE_PERL="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_PERL="true" - fi - - #################################### - # Validate if we should check PHP # - #################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_PHP" ]]; then - # PHP flag was not set - default to false - VALIDATE_PHP="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_PHP="true" - fi - - ###################################### - # Validate if we should check PYTHON # - ###################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_PYTHON" ]]; then - # PYTHON flag was not set - default to false - VALIDATE_PYTHON="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_PYTHON="true" - fi - - #################################### - # Validate if we should check RUBY # - #################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_RUBY" ]]; then - # RUBY flag was not set - default to false - VALIDATE_RUBY="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_RUBY="true" - fi - - ###################################### - # Validate if we should check COFFEE # - ###################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_COFFEE" ]]; then - # COFFEE flag was not set - default to false - VALIDATE_COFFEE="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_COFFEE="true" - fi - - ####################################### - # Validate if we should check ANSIBLE # - ####################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_ANSIBLE" ]]; then - # ANSIBLE flag was not set - default to false - VALIDATE_ANSIBLE="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_ANSIBLE="true" - fi - - ############################################# - # Validate if we should check JAVASCRIPT_ES # - ############################################# - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_JAVASCRIPT_ES" ]]; then - # JAVASCRIPT_ES flag was not set - default to false - VALIDATE_JAVASCRIPT_ES="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_JAVASCRIPT_ES="true" - fi - - ################################################### - # Validate if we should check JAVASCRIPT_STANDARD # - ################################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_JAVASCRIPT_STANDARD" ]]; then - # JAVASCRIPT_STANDARD flag was not set - default to false - VALIDATE_JAVASCRIPT_STANDARD="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_JAVASCRIPT_STANDARD="true" - fi - - ############################################# - # Validate if we should check TYPESCRIPT_ES # - ############################################# - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_TYPESCRIPT_ES" ]]; then - # TYPESCRIPT_ES flag was not set - default to false - VALIDATE_TYPESCRIPT_ES="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_TYPESCRIPT_ES="true" - fi - - ################################################### - # Validate if we should check TYPESCRIPT_STANDARD # - ################################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_TYPESCRIPT_STANDARD" ]]; then - # TYPESCRIPT_STANDARD flag was not set - default to false - VALIDATE_TYPESCRIPT_STANDARD="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_TYPESCRIPT_STANDARD="true" - fi - - ###################################### - # Validate if we should check DOCKER # - ###################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_DOCKER" ]]; then - # DOCKER flag was not set - default to false - VALIDATE_DOCKER="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_DOCKER="true" - fi - - ################################## - # Validate if we should check GO # - ################################## - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_GO" ]]; then - # GO flag was not set - default to false - VALIDATE_GO="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_GO="true" - fi - - ######################################### - # Validate if we should check TERRAFORM # - ######################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_TERRAFORM" ]]; then - # TERRAFORM flag was not set - default to false - VALIDATE_TERRAFORM="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_TERRAFORM="true" - fi - - ######################################### - # Validate if we should check POWERSHELL # - ######################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_POWERSHELL" ]]; then - # POWERSHELL flag was not set - default to false - VALIDATE_POWERSHELL="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_POWERSHELL="true" - fi - - ################################### - # Validate if we should check CSS # - ################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_CSS" ]]; then - # CSS flag was not set - default to false - VALIDATE_CSS="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_CSS="true" - fi - - ################################### - # Validate if we should check ENV # - ################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_ENV" ]]; then - # ENV flag was not set - default to false - VALIDATE_ENV="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_ENV="true" - fi - - ###################################### - # Validate if we should check KOTLIN # - ###################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_KOTLIN" ]]; then - # ENV flag was not set - default to false - VALIDATE_KOTLIN="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_KOTLIN="true" - fi - - ####################################### - # Validate if we should check OPENAPI # - ####################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_OPENAPI" ]]; then - # OPENAPI flag was not set - default to false - VALIDATE_OPENAPI="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_OPENAPI="true" - fi - - ####################################### - # Validate if we should check PROTOBUF # - ####################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_PROTOBUF" ]]; then - # PROTOBUF flag was not set - default to false - VALIDATE_PROTOBUF="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_PROTOBUF="true" - fi - - ####################################### - # Validate if we should check Clojure # - ####################################### - if [[ "$ANY_SET" == "true" ]]; then - # Some linter flags were set - only run those set to true - if [[ -z "$VALIDATE_CLOJURE" ]]; then - # Clojure flag was not set - default to false - VALIDATE_CLOJURE="false" - fi - else - # No linter flags were set - default all to true - VALIDATE_CLOJURE="true" - fi - - ####################################### - # Print which linters we are enabling # - ####################################### - if [[ "$VALIDATE_YAML" == "true" ]]; then - PRINT_ARRAY+=("- Validating [YAML] files in code base...") - else - PRINT_ARRAY+=("- Excluding [YAML] files in code base...") - fi - if [[ "$VALIDATE_JSON" == "true" ]]; then - PRINT_ARRAY+=("- Validating [JSON] files in code base...") - else - PRINT_ARRAY+=("- Excluding [JSON] files in code base...") - fi - if [[ "$VALIDATE_XML" == "true" ]]; then - PRINT_ARRAY+=("- Validating [XML] files in code base...") - else - PRINT_ARRAY+=("- Excluding [XML] files in code base...") - fi - if [[ "$VALIDATE_MD" == "true" ]]; then - PRINT_ARRAY+=("- Validating [MARKDOWN] files in code base...") - else - PRINT_ARRAY+=("- Excluding [MARKDOWN] files in code base...") - fi - if [[ "$VALIDATE_BASH" == "true" ]]; then - PRINT_ARRAY+=("- Validating [BASH] files in code base...") - else - PRINT_ARRAY+=("- Excluding [BASH] files in code base...") - fi - if [[ "$VALIDATE_PERL" == "true" ]]; then - PRINT_ARRAY+=("- Validating [PERL] files in code base...") - else - PRINT_ARRAY+=("- Excluding [PERL] files in code base...") - fi - if [[ "$VALIDATE_PHP" == "true" ]]; then - PRINT_ARRAY+=("- Validating [PHP] files in code base...") - else - PRINT_ARRAY+=("- Excluding [PHP] files in code base...") - fi - if [[ "$VALIDATE_PYTHON" == "true" ]]; then - PRINT_ARRAY+=("- Validating [PYTHON] files in code base...") - else - PRINT_ARRAY+=("- Excluding [PYTHON] files in code base...") - fi - if [[ "$VALIDATE_RUBY" == "true" ]]; then - PRINT_ARRAY+=("- Validating [RUBY] files in code base...") - else - PRINT_ARRAY+=("- Excluding [RUBY] files in code base...") - fi - if [[ "$VALIDATE_COFFEE" == "true" ]]; then - PRINT_ARRAY+=("- Validating [COFFEE] files in code base...") - else - PRINT_ARRAY+=("- Excluding [COFFEE] files in code base...") - fi - if [[ "$VALIDATE_ANSIBLE" == "true" ]]; then - PRINT_ARRAY+=("- Validating [ANSIBLE] files in code base...") - else - PRINT_ARRAY+=("- Excluding [ANSIBLE] files in code base...") - fi - if [[ "$VALIDATE_JAVASCRIPT_ES" == "true" ]]; then - PRINT_ARRAY+=("- Validating [JAVASCRIPT(eslint)] files in code base...") - else - PRINT_ARRAY+=("- Excluding [JAVASCRIPT(eslint)] files in code base...") - fi - if [[ "$VALIDATE_JAVASCRIPT_STANDARD" == "true" ]]; then - PRINT_ARRAY+=("- Validating [JAVASCRIPT(standard)] files in code base...") - else - PRINT_ARRAY+=("- Excluding [JAVASCRIPT(standard)] files in code base...") - fi - if [[ "$VALIDATE_TYPESCRIPT_ES" == "true" ]]; then - PRINT_ARRAY+=("- Validating [TYPESCRIPT(eslint)] files in code base...") - else - PRINT_ARRAY+=("- Excluding [TYPESCRIPT(eslint)] files in code base...") - fi - if [[ "$VALIDATE_TYPESCRIPT_STANDARD" == "true" ]]; then - PRINT_ARRAY+=("- Validating [TYPESCRIPT(standard)] files in code base...") - else - PRINT_ARRAY+=("- Excluding [TYPESCRIPT(standard)] files in code base...") - fi - if [[ "$VALIDATE_DOCKER" == "true" ]]; then - PRINT_ARRAY+=("- Validating [DOCKER] files in code base...") - else - PRINT_ARRAY+=("- Excluding [DOCKER] files in code base...") - fi - if [[ "$VALIDATE_GO" == "true" ]]; then - PRINT_ARRAY+=("- Validating [GOLANG] files in code base...") - else - PRINT_ARRAY+=("- Excluding [GOLANG] files in code base...") - fi - if [[ "$VALIDATE_TERRAFORM" == "true" ]]; then - PRINT_ARRAY+=("- Validating [TERRAFORM] files in code base...") - else - PRINT_ARRAY+=("- Excluding [TERRAFORM] files in code base...") - fi - if [[ "$VALIDATE_POWERSHELL" == "true" ]]; then - PRINT_ARRAY+=("- Validating [POWERSHELL] files in code base...") - else - PRINT_ARRAY+=("- Excluding [POWERSHELL] files in code base...") - fi - if [[ "$VALIDATE_CSS" == "true" ]]; then - PRINT_ARRAY+=("- Validating [CSS] files in code base...") - else - PRINT_ARRAY+=("- Excluding [CSS] files in code base...") - fi - if [[ "$VALIDATE_CLOJURE" == "true" ]]; then - PRINT_ARRAY+=("- Validating [CLOJURE] files in code base...") - else - PRINT_ARRAY+=("- Excluding [CLOJURE] files in code base...") - fi - if [[ "$VALIDATE_ENV" == "true" ]]; then - PRINT_ARRAY+=("- Validating [ENV] files in code base...") - else - PRINT_ARRAY+=("- Excluding [ENV] files in code base...") - fi - if [[ "$VALIDATE_KOTLIN" == "true" ]]; then - PRINT_ARRAY+=("- Validating [KOTLIN] files in code base...") - else - PRINT_ARRAY+=("- Excluding [KOTLIN] files in code base...") - fi - if [[ "$VALIDATE_OPENAPI" == "true" ]]; then - PRINT_ARRAY+=("- Validating [OPENAPI] files in code base...") - else - PRINT_ARRAY+=("- Excluding [OPENAPI] files in code base...") - fi - if [[ "$VALIDATE_PROTOBUF" == "true" ]]; then - PRINT_ARRAY+=("- Validating [PROTOBUF] files in code base...") - else - PRINT_ARRAY+=("- Excluding [PROTOBUF] files in code base...") - fi - - ############################## - # Validate Ansible Directory # - ############################## - if [ -z "$ANSIBLE_DIRECTORY" ]; then - # No Value, need to default - ANSIBLE_DIRECTORY="$DEFAULT_ANSIBLE_DIRECTORY" - else - # Check if first char is '/' - if [[ ${ANSIBLE_DIRECTORY:0:1} == "/" ]]; then - # Remove first char - ANSIBLE_DIRECTORY="${ANSIBLE_DIRECTORY:1}" - fi - # Need to give it full path - TEMP_ANSIBLE_DIRECTORY="$GITHUB_WORKSPACE/$ANSIBLE_DIRECTORY" - # Set the value - ANSIBLE_DIRECTORY="$TEMP_ANSIBLE_DIRECTORY" - fi - - ############################### - # Get the disable errors flag # - ############################### - if [ -z "$DISABLE_ERRORS" ]; then - ################################## - # No flag passed, set to default # - ################################## - DISABLE_ERRORS="$DEFAULT_DISABLE_ERRORS" - fi - - ############################### - # Convert string to lowercase # - ############################### - DISABLE_ERRORS=$(echo "$DISABLE_ERRORS" | awk '{print tolower($0)}') - - ############################ - # Set to false if not true # - ############################ - if [ "$DISABLE_ERRORS" != "true" ]; then - DISABLE_ERRORS="false" - fi - - ############################ - # Get the run verbose flag # - ############################ - if [ -z "$ACTIONS_RUNNER_DEBUG" ]; then - ################################## - # No flag passed, set to default # - ################################## - ACTIONS_RUNNER_DEBUG="$DEFAULT_ACTIONS_RUNNER_DEBUG" - fi - - ############################### - # Convert string to lowercase # - ############################### - ACTIONS_RUNNER_DEBUG=$(echo "$ACTIONS_RUNNER_DEBUG" | awk '{print tolower($0)}') - - ############################ - # Set to true if not false # - ############################ - if [ "$ACTIONS_RUNNER_DEBUG" != "false" ]; then - ACTIONS_RUNNER_DEBUG="true" - fi - - ################### - # Debug on runner # - ################### - if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then - ########################### - # Print the validate info # - ########################### - for LINE in "${PRINT_ARRAY[@]}" - do - echo "$LINE" - done - - echo "--- DEBUG INFO ---" - echo "---------------------------------------------" - RUNNER=$(whoami) - echo "Runner:[$RUNNER]" - echo "ENV:" - printenv - echo "---------------------------------------------" - fi -} -################################################################################ -#### Function BuildFileList #################################################### -BuildFileList() -{ - # Need to build a list of all files changed - # This can be pulled from the GITHUB_EVENT_PATH payload - - ################ - # print header # - ################ - if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then - echo "" - echo "----------------------------------------------" - echo "Pulling in code history and branches..." - fi - - ################################################################################# - # Switch codebase back to the default branch to get a list of all files changed # - ################################################################################# - SWITCH_CMD=$(cd "$GITHUB_WORKSPACE" || exit; git pull --quiet; git checkout "$DEFAULT_BRANCH" 2>&1) - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ $ERROR_CODE -ne 0 ]; then - # Error - echo "Failed to switch to $DEFAULT_BRANCH branch to get files changed!" - echo "ERROR:[$SWITCH_CMD]" - exit 1 - fi - - ################ - # print header # - ################ - if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then - echo "" - echo "----------------------------------------------" - echo "Generating Diff with:[git diff --name-only '$DEFAULT_BRANCH..$GITHUB_SHA' --diff-filter=d]" - fi - - ################################################# - # Get the Array of files changed in the commits # - ################################################# - # shellcheck disable=SC2207 - RAW_FILE_ARRAY=($(cd "$GITHUB_WORKSPACE" || exit; git diff --name-only "$DEFAULT_BRANCH..$GITHUB_SHA" --diff-filter=d 2>&1)) - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ $ERROR_CODE -ne 0 ]; then - # Error - echo "ERROR! Failed to gain a list of all files changed!" - echo "ERROR:[${RAW_FILE_ARRAY[*]}]" - exit 1 - fi - - ################################################ - # Iterate through the array of all files found # - ################################################ - echo "" - echo "----------------------------------------------" - echo "Files that have been modified in the commit(s):" - for FILE in "${RAW_FILE_ARRAY[@]}" - do - ############## - # Print file # - ############## - echo "File:[$FILE]" - - ########################### - # Get the files extension # - ########################### - # Extract just the file and extension, reverse it, cut off extension, - # reverse it back, substitute to lowercase - FILE_TYPE=$(basename "$FILE" | rev | cut -f1 -d'.' | rev | awk '{print tolower($0)}') - - ######### - # DEBUG # - ######### - #echo "FILE_TYPE:[$FILE_TYPE]" - - ##################### - # Get the YML files # - ##################### - if [ "$FILE_TYPE" == "yml" ] || [ "$FILE_TYPE" == "yaml" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_YML+=("$FILE") - ############################ - # Check if file is OpenAPI # - ############################ - if DetectOpenAPIFile "$FILE"; then - FILE_ARRAY_OPENAPI+=("$FILE") - fi - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ###################### - # Get the JSON files # - ###################### - elif [ "$FILE_TYPE" == "json" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_JSON+=("$FILE") - ############################ - # Check if file is OpenAPI # - ############################ - if DetectOpenAPIFile "$FILE"; then - FILE_ARRAY_OPENAPI+=("$FILE") - fi - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ##################### - # Get the XML files # - ##################### - elif [ "$FILE_TYPE" == "xml" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_XML+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ########################## - # Get the MARKDOWN files # - ########################## - elif [ "$FILE_TYPE" == "md" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_MD+=("$FILE") - ###################### - # Get the BASH files # - ###################### - elif [ "$FILE_TYPE" == "sh" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_BASH+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ###################### - # Get the PERL files # - ###################### - elif [ "$FILE_TYPE" == "pl" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_PERL+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ###################### - # Get the PHP files # - ###################### - elif [ "$FILE_TYPE" == "php" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_PHP+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ###################### - # Get the RUBY files # - ###################### - elif [ "$FILE_TYPE" == "rb" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_RUBY+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ######################## - # Get the PYTHON files # - ######################## - elif [ "$FILE_TYPE" == "py" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_PYTHON+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ######################## - # Get the COFFEE files # - ######################## - elif [ "$FILE_TYPE" == "coffee" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_COFFEESCRIPT+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ############################ - # Get the JavaScript files # - ############################ - elif [ "$FILE_TYPE" == "js" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_JAVASCRIPT_ES+=("$FILE") - FILE_ARRAY_JAVASCRIPT_STANDARD+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ############################ - # Get the TypeScript files # - ############################ - elif [ "$FILE_TYPE" == "ts" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_TYPESCRIPT_ES+=("$FILE") - FILE_ARRAY_TYPESCRIPT_STANDARD+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ######################## - # Get the Golang files # - ######################## - elif [ "$FILE_TYPE" == "go" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_GO+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ########################### - # Get the Terraform files # - ########################### - elif [ "$FILE_TYPE" == "tf" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_TERRAFORM+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ########################### - # Get the Powershell files # - ########################### - elif [ "$FILE_TYPE" == "ps1" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_POWERSHELL+=("$FILE") - elif [ "$FILE_TYPE" == "css" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_CSS+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - elif [ "$FILE_TYPE" == "env" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_ENV+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - elif [ "$FILE_TYPE" == "kt" ] || [ "$FILE_TYPE" == "kts" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_KOTLIN+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - ############################ - # Get the Protocol Buffers files # - ############################ - elif [ "$FILE_TYPE" == "proto" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_PROTOBUF+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - elif [ "$FILE" == "Dockerfile" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_DOCKER+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - elif [ "$FILE_TYPE" == "clj" ] || [ "$FILE_TYPE" == "cljs" ] || [ "$FILE_TYPE" == "cljc" ] || [ "$FILE_TYPE" == "edn" ]; then - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_CLOJURE+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - else - ############################################## - # Use file to see if we can parse what it is # - ############################################## - GET_FILE_TYPE_CMD=$(file "$FILE" 2>&1) - - ################# - # Check if bash # - ################# - if [[ "$GET_FILE_TYPE_CMD" == *"Bourne-Again shell script"* ]]; then - ####################### - # It is a bash script # - ####################### - echo "WARN! Found bash script without extension:[.sh]" - echo "Please update file with proper extensions." - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_BASH+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - elif [[ "$GET_FILE_TYPE_CMD" == *"Ruby script"* ]]; then - ####################### - # It is a Ruby script # - ####################### - echo "WARN! Found ruby script without extension:[.rb]" - echo "Please update file with proper extensions." - ################################ - # Append the file to the array # - ################################ - FILE_ARRAY_RUBY+=("$FILE") - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - else - ############################ - # Extension was not found! # - ############################ - echo " - WARN! Failed to get filetype for:[$FILE]!" - ########################################################## - # Set the READ_ONLY_CHANGE_FLAG since this could be exec # - ########################################################## - READ_ONLY_CHANGE_FLAG=1 - fi - fi - done - - ######################################### - # Need to switch back to branch of code # - ######################################### - SWITCH2_CMD=$(cd "$GITHUB_WORKSPACE" || exit; git checkout --progress --force "$GITHUB_SHA" 2>&1) - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ $ERROR_CODE -ne 0 ]; then - # Error - echo "Failed to switch back to branch!" - echo "ERROR:[$SWITCH2_CMD]" - exit 1 - fi - - ################ - # Footer print # - ################ - echo "" - echo "----------------------------------------------" - echo "Successfully gathered list of files..." -} -################################################################################ -#### Function LintCodebase ##################################################### -LintCodebase() -{ - #################### - # Pull in the vars # - #################### - FILE_TYPE="$1" && shift # Pull the variable and remove from array path (Example: JSON) - LINTER_NAME="$1" && shift # Pull the variable and remove from array path (Example: jsonlint) - LINTER_COMMAND="$1" && shift # Pull the variable and remove from array path (Example: jsonlint -c ConfigFile /path/to/file) - FILE_EXTENSIONS="$1" && shift # Pull the variable and remove from array path (Example: *.json) - FILE_ARRAY=("$@") # Array of files to validate (Example: $FILE_ARRAY_JSON) - - ###################### - # Create Print Array # - ###################### - PRINT_ARRAY=() - - ################ - # print header # - ################ - PRINT_ARRAY+=("") - PRINT_ARRAY+=("----------------------------------------------") - PRINT_ARRAY+=("----------------------------------------------") - PRINT_ARRAY+=("Linting [$FILE_TYPE] files...") - PRINT_ARRAY+=("----------------------------------------------") - PRINT_ARRAY+=("----------------------------------------------") - - ####################################### - # Validate we have jsonlint installed # - ####################################### - # shellcheck disable=SC2230 - VALIDATE_INSTALL_CMD=$(command -v "$LINTER_NAME" 2>&1) - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ $ERROR_CODE -ne 0 ]; then - # Failed - echo "ERROR! Failed to find [$LINTER_NAME] in system!" - echo "ERROR:[$VALIDATE_INSTALL_CMD]" - exit 1 - else - # Success - if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then - echo "Successfully found binary in system" - echo "Location:[$VALIDATE_INSTALL_CMD]" - fi - fi - - ########################## - # Initialize empty Array # - ########################## - LIST_FILES=() - - ################ - # Set the flag # - ################ - SKIP_FLAG=0 - - ############################################################ - # Check to see if we need to go through array or all files # - ############################################################ - if [ ${#FILE_ARRAY[@]} -eq 0 ] && [ "$VALIDATE_ALL_CODEBASE" == "false" ]; then - # No files found in commit and user has asked to not validate code base - SKIP_FLAG=1 - # echo " - No files found in changeset to lint for language:[$FILE_TYPE]" - elif [ ${#FILE_ARRAY[@]} -ne 0 ]; then - # We have files added to array of files to check - LIST_FILES=("${FILE_ARRAY[@]}") # Copy the array into list - else - ############################################################################### - # Set the file seperator to newline to allow for grabbing objects with spaces # - ############################################################################### - IFS=$'\n' - - ################################# - # Get list of all files to lint # - ################################# - # shellcheck disable=SC2207,SC2086 - LIST_FILES=($(cd "$GITHUB_WORKSPACE" || exit; find . -type f -regex "$FILE_EXTENSIONS" 2>&1)) - - ########################### - # Set IFS back to default # - ########################### - IFS="$DEFAULT_IFS" - - ############################################################ - # Set it back to empty if loaded with blanks from scanning # - ############################################################ - if [ ${#LIST_FILES[@]} -lt 1 ]; then - ###################### - # Set to empty array # - ###################### - LIST_FILES=() - ############################# - # Skip as we found no files # - ############################# - SKIP_FLAG=1 - fi - fi - - ############################### - # Check if any data was found # - ############################### - if [ $SKIP_FLAG -eq 0 ]; then - ###################### - # Print Header array # - ###################### - for LINE in "${PRINT_ARRAY[@]}" - do - ######################### - # Print the header info # - ######################### - echo "$LINE" - done - - ################## - # Lint the files # - ################## - for FILE in "${LIST_FILES[@]}" - do - ##################### - # Get the file name # - ##################### - FILE_NAME=$(basename "$FILE" 2>&1) - - ##################################################### - # Make sure we dont lint node modules or test cases # - ##################################################### - if [[ $FILE == *"node_modules"* ]]; then - # This is a node modules file - continue - elif [[ $FILE == *"$TEST_CASE_FOLDER"* ]]; then - # This is the test cases, we should always skip - continue - fi - - ############## - # File print # - ############## - echo "---------------------------" - echo "File:[$FILE]" - - #################### - # Set the base Var # - #################### - LINT_CMD='' - - ####################################### - # Corner case for Powershell subshell # - ####################################### - if [[ "$FILE_TYPE" == "POWERSHELL" ]]; then - ################################ - # Lint the file with the rules # - ################################ - # Need to append "'" to make the pwsh call syntax correct, also exit with exit code from inner subshell - LINT_CMD=$(cd "$GITHUB_WORKSPACE" || exit; $LINTER_COMMAND "$FILE"; exit $? 2>&1) - else - ################################ - # Lint the file with the rules # - ################################ - LINT_CMD=$(cd "$GITHUB_WORKSPACE" || exit; $LINTER_COMMAND "$FILE" 2>&1) - fi - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ $ERROR_CODE -ne 0 ]; then - ######### - # Error # - ######### - echo "ERROR! Found errors in [$LINTER_NAME] linter!" - echo "ERROR:[$LINT_CMD]" - # Increment the error count - (("ERRORS_FOUND_$FILE_TYPE++")) - else - ########### - # Success # - ########### - echo " - File:[$FILE_NAME] was linted with [$LINTER_NAME] successfully" - fi - done - fi -} -################################################################################ -#### Function TestCodebase ##################################################### -TestCodebase() -{ - #################### - # Pull in the vars # - #################### - FILE_TYPE="$1" # Pull the variable and remove from array path (Example: JSON) - LINTER_NAME="$2" # Pull the variable and remove from array path (Example: jsonlint) - LINTER_COMMAND="$3" # Pull the variable and remove from array path (Example: jsonlint -c ConfigFile /path/to/file) - FILE_EXTENSIONS="$4" # Pull the variable and remove from array path (Example: *.json) - - ################ - # print header # - ################ - echo "" - echo "----------------------------------------------" - echo "----------------------------------------------" - echo "Testing Codebase [$FILE_TYPE] files..." - echo "----------------------------------------------" - echo "----------------------------------------------" - echo "" - - ##################################### - # Validate we have linter installed # - ##################################### - # shellcheck disable=SC2230 - VALIDATE_INSTALL_CMD=$(command -v "$LINTER_NAME" 2>&1) - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ############################## - # Check the shell for errors # - ############################## - if [ $ERROR_CODE -ne 0 ]; then - # Failed - echo "ERROR! Failed to find [$LINTER_NAME] in system!" - echo "ERROR:[$VALIDATE_INSTALL_CMD]" - exit 1 - else - # Success - echo "Successfully found binary in system" - echo "Location:[$VALIDATE_INSTALL_CMD]" - fi - - ########################## - # Initialize empty Array # - ########################## - LIST_FILES=() - - ############################################ - # Check if its ansible, as its the outlier # - ############################################ - if [[ "$FILE_TYPE" == "ANSIBLE" ]]; then - ################################# - # Get list of all files to lint # - ################################# - # shellcheck disable=SC2207,SC2086,SC2010 - LIST_FILES=($(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER" || exit; ls ansible/ | grep ".yml" 2>&1)) - else - ############################################################################### - # Set the file seperator to newline to allow for grabbing objects with spaces # - ############################################################################### - IFS=$'\n' - - ################################# - # Get list of all files to lint # - ################################# - # shellcheck disable=SC2207,SC2086 - LIST_FILES=($(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER" || exit; find . -type f -regex "$FILE_EXTENSIONS" ! -path "*./ansible*" 2>&1)) - - ########################### - # Set IFS back to default # - ########################### - IFS="$DEFAULT_IFS" - fi - - ################## - # Lint the files # - ################## - for FILE in "${LIST_FILES[@]}" - do - ##################### - # Get the file name # - ##################### - FILE_NAME=$(basename "$FILE" 2>&1) - - ############################ - # Get the file pass status # - ############################ - # Example: markdown_good_1.md -> good - FILE_STATUS=$(echo "$FILE_NAME" |cut -f2 -d'_') - - ######################################################### - # If not found, assume it should be linted successfully # - ######################################################### - if [ -z "$FILE_STATUS" ] || [[ "$FILE" == *"README"* ]]; then - ################################## - # Set to good for proper linting # - ################################## - FILE_STATUS="good" - fi - - ############## - # File print # - ############## - echo "---------------------------" - echo "File:[$FILE]" - - ######################## - # Set the lint command # - ######################## - LINT_CMD='' - - ####################################### - # Check if docker and get folder name # - ####################################### - if [[ "$FILE_TYPE" == "DOCKER" ]]; then - if [[ "$FILE" == *"good"* ]]; then - ############# - # Good file # - ############# - FILE_STATUS='good' - else - ############ - # Bad file # - ############ - FILE_STATUS='bad' - fi - fi - - ##################### - # Check for ansible # - ##################### - if [[ "$FILE_TYPE" == "ANSIBLE" ]]; then - ######################################## - # Make sure we dont lint certain files # - ######################################## - if [[ $FILE == *"vault.yml"* ]] || [[ $FILE == *"galaxy.yml"* ]]; then - # This is a file we dont look at - continue - fi - - ################################ - # Lint the file with the rules # - ################################ - LINT_CMD=$(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER/ansible" || exit; $LINTER_COMMAND "$FILE" 2>&1) - elif [[ "$FILE_TYPE" == "POWERSHELL" ]]; then - ################################ - # Lint the file with the rules # - ################################ - # Need to append "'" to make the pwsh call syntax correct, also exit with exit code from inner subshell - LINT_CMD=$(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER" || exit; $LINTER_COMMAND "$FILE"; exit $? 2>&1) - else - ################################ - # Lint the file with the rules # - ################################ - LINT_CMD=$(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER" || exit; $LINTER_COMMAND "$FILE" 2>&1) - fi - - ####################### - # Load the error code # - ####################### - ERROR_CODE=$? - - ######################################## - # Check for if it was supposed to pass # - ######################################## - if [[ "$FILE_STATUS" == "good" ]]; then - ############################## - # Check the shell for errors # - ############################## - if [ $ERROR_CODE -ne 0 ]; then - ######### - # Error # - ######### - echo "ERROR! Found errors in [$LINTER_NAME] linter!" - echo "ERROR:[$LINT_CMD]" - echo "ERROR: Linter CMD:[$LINTER_COMMAND $FILE]" - # Increment the error count - (("ERRORS_FOUND_$FILE_TYPE++")) - else - ########### - # Success # - ########### - echo " - File:[$FILE_NAME] was linted with [$LINTER_NAME] successfully" - fi - else - ####################################### - # File status = bad, this should fail # - ####################################### - ############################## - # Check the shell for errors # - ############################## - if [ $ERROR_CODE -eq 0 ]; then - ######### - # Error # - ######### - echo "ERROR! Found errors in [$LINTER_NAME] linter!" - echo "ERROR! This file should have failed test case!" - echo "ERROR:[$LINT_CMD]" - echo "ERROR: Linter CMD:[$LINTER_COMMAND $FILE]" - # Increment the error count - (("ERRORS_FOUND_$FILE_TYPE++")) - else - ########### - # Success # - ########### - echo " - File:[$FILE_NAME] failed test case with [$LINTER_NAME] successfully" - fi - fi - done -} -################################################################################ #### Function Footer ########################################################### Footer() { @@ -2315,63 +657,6 @@ Footer() fi } ################################################################################ -#### Function RunTestCases ##################################################### -RunTestCases() -{ - # This loop will run the test cases and exclude user code - # This is called from the automation process to validate new code - # When a PR is opened, the new code is validated with the default branch - # version of linter.sh, and a new container is built with the latest codebase - # for testing. That container is spun up, and ran, - # with the flag: TEST_CASE_RUN=true - # So that the new code can be validated against the test cases - - ################# - # Header prints # - ################# - echo "" - echo "----------------------------------------------" - echo "-------------- TEST CASE RUN -----------------" - echo "----------------------------------------------" - echo "" - - ####################### - # Test case languages # - ####################### - TestCodebase "YML" "yamllint" "yamllint -c $YAML_LINTER_RULES" ".*\.\(yml\|yaml\)\$" - TestCodebase "JSON" "jsonlint" "jsonlint" ".*\.\(json\)\$" - TestCodebase "XML" "xmllint" "xmllint" ".*\.\(xml\)\$" - TestCodebase "MARKDOWN" "markdownlint" "markdownlint -c $MD_LINTER_RULES" ".*\.\(md\)\$" - TestCodebase "BASH" "shellcheck" "shellcheck" ".*\.\(sh\)\$" - TestCodebase "PYTHON" "pylint" "pylint --rcfile $PYTHON_LINTER_RULES" ".*\.\(py\)\$" - TestCodebase "PERL" "perl" "perl -Mstrict -cw" ".*\.\(pl\)\$" - TestCodebase "PHP" "php" "php -l" ".*\.\(php\)\$" - TestCodebase "RUBY" "rubocop" "rubocop -c $RUBY_LINTER_RULES" ".*\.\(rb\)\$" - TestCodebase "GO" "golangci-lint" "golangci-lint run -c $GO_LINTER_RULES" ".*\.\(go\)\$" - TestCodebase "COFFEESCRIPT" "coffeelint" "coffeelint -f $COFFEESCRIPT_LINTER_RULES" ".*\.\(coffee\)\$" - TestCodebase "JAVASCRIPT_ES" "eslint" "eslint --no-eslintrc -c $JAVASCRIPT_LINTER_RULES" ".*\.\(js\)\$" - TestCodebase "JAVASCRIPT_STANDARD" "standard" "standard $JAVASCRIPT_STANDARD_LINTER_RULES" ".*\.\(js\)\$" - TestCodebase "TYPESCRIPT_ES" "eslint" "eslint --no-eslintrc -c $TYPESCRIPT_LINTER_RULES" ".*\.\(ts\)\$" - TestCodebase "TYPESCRIPT_STANDARD" "standard" "standard --parser @typescript-eslint/parser --plugin @typescript-eslint/eslint-plugin $TYPESCRIPT_STANDARD_LINTER_RULES" ".*\.\(ts\)\$" - TestCodebase "DOCKER" "/dockerfilelint/bin/dockerfilelint" "/dockerfilelint/bin/dockerfilelint -c $DOCKER_LINTER_RULES" ".*\(Dockerfile\)\$" - TestCodebase "ANSIBLE" "ansible-lint" "ansible-lint -v -c $ANSIBLE_LINTER_RULES" "ansible-lint" - TestCodebase "TERRAFORM" "tflint" "tflint -c $TERRAFORM_LINTER_RULES" ".*\.\(tf\)\$" - TestCodebase "POWERSHELL" "pwsh" "pwsh -c Invoke-ScriptAnalyzer -EnableExit -Settings $POWERSHELL_LINTER_RULES -Path" ".*\.\(ps1\|psm1\|psd1\|ps1xml\|pssc\|psrc\|cdxml\)\$" - TestCodebase "CSS" "stylelint" "stylelint --config $CSS_LINTER_RULES" ".*\.\(css\)\$" - TestCodebase "ENV" "dotenv-linter" "dotenv-linter" ".*\.\(env\)\$" - TestCodebase "CLOJURE" "clj-kondo" "clj-kondo --config $CLOJURE_LINTER_RULES --lint" ".*\.\(clj\|cljs\|cljc\|edn\)\$" - TestCodebase "KOTLIN" "ktlint" "ktlint" ".*\.\(kt\|kts\)\$" - TestCodebase "PROTOBUF" "protolint" "protolint lint --config_path $PROTOBUF_LINTER_RULES" ".*\.\(proto\)\$" - TestCodebase "OPENAPI" "spectral" "spectral lint -r $OPENAPI_LINTER_RULES" ".*\.\(ymlopenapi\|jsonopenapi\)\$" - - ################# - # Footer prints # - ################# - # Call the footer to display run information - # and exit with error code - Footer -} -################################################################################ ############################### MAIN ########################################### ################################################################################ diff --git a/lib/validation.sh b/lib/validation.sh new file mode 100755 index 00000000..97f85a38 --- /dev/null +++ b/lib/validation.sh @@ -0,0 +1,669 @@ +#!/usr/bin/env bash + +################################################################################ +################################################################################ +########### Super-Linter Validation Functions @admiralawkbar ################### +################################################################################ +################################################################################ +########################## FUNCTION CALLS BELOW ################################ +################################################################################ +################################################################################ +#### Function GetValidationInfo ################################################ +function GetValidationInfo() +{ + ############################################ + # Print headers for user provided env vars # + ############################################ + echo "" + echo "--------------------------------------------" + echo "Gathering user validation information..." + + ########################################### + # Skip validation if were running locally # + ########################################### + if [[ "$RUN_LOCAL" != "true" ]]; then + ############################### + # Convert string to lowercase # + ############################### + VALIDATE_ALL_CODEBASE=$(echo "$VALIDATE_ALL_CODEBASE" | awk '{print tolower($0)}') + ###################################### + # Validate we should check all files # + ###################################### + if [[ "$VALIDATE_ALL_CODEBASE" != "false" ]]; then + # Set to true + VALIDATE_ALL_CODEBASE="$DEFAULT_VALIDATE_ALL_CODEBASE" + echo "- Validating ALL files in code base..." + else + # Its false + echo "- Only validating [new], or [edited] files in code base..." + fi + fi + + ###################### + # Create Print Array # + ###################### + PRINT_ARRAY=() + + ################################ + # Convert strings to lowercase # + ################################ + VALIDATE_YAML=$(echo "$VALIDATE_YAML" | awk '{print tolower($0)}') + VALIDATE_JSON=$(echo "$VALIDATE_JSON" | awk '{print tolower($0)}') + VALIDATE_XML=$(echo "$VALIDATE_XML" | awk '{print tolower($0)}') + VALIDATE_MD=$(echo "$VALIDATE_MD" | awk '{print tolower($0)}') + VALIDATE_BASH=$(echo "$VALIDATE_BASH" | awk '{print tolower($0)}') + VALIDATE_PERL=$(echo "$VALIDATE_PERL" | awk '{print tolower($0)}') + VALIDATE_PHP=$(echo "$VALIDATE_PHP" | awk '{print tolower($0)}') + VALIDATE_PYTHON=$(echo "$VALIDATE_PYTHON" | awk '{print tolower($0)}') + VALIDATE_RUBY=$(echo "$VALIDATE_RUBY" | awk '{print tolower($0)}') + VALIDATE_COFFEE=$(echo "$VALIDATE_COFFEE" | awk '{print tolower($0)}') + VALIDATE_ANSIBLE=$(echo "$VALIDATE_ANSIBLE" | awk '{print tolower($0)}') + VALIDATE_JAVASCRIPT_ES=$(echo "$VALIDATE_JAVASCRIPT_ES" | awk '{print tolower($0)}') + VALIDATE_JAVASCRIPT_STANDARD=$(echo "$VALIDATE_JAVASCRIPT_STANDARD" | awk '{print tolower($0)}') + VALIDATE_TYPESCRIPT_ES=$(echo "$VALIDATE_TYPESCRIPT_ES" | awk '{print tolower($0)}') + VALIDATE_TYPESCRIPT_STANDARD=$(echo "$VALIDATE_TYPESCRIPT_STANDARD" | awk '{print tolower($0)}') + VALIDATE_DOCKER=$(echo "$VALIDATE_DOCKER" | awk '{print tolower($0)}') + VALIDATE_GO=$(echo "$VALIDATE_GO" | awk '{print tolower($0)}') + VALIDATE_TERRAFORM=$(echo "$VALIDATE_TERRAFORM" | awk '{print tolower($0)}') + VALIDATE_POWERSHELL=$(echo "$VALIDATE_POWERSHELL" | awk '{print tolower($0)}') + VALIDATE_CSS=$(echo "$VALIDATE_CSS" | awk '{print tolower($0)}') + VALIDATE_ENV=$(echo "$VALIDATE_ENV" | awk '{print tolower($0)}') + VALIDATE_CLOJURE=$(echo "$VALIDATE_CLOJURE" | awk '{print tolower($0)') + VALIDATE_KOTLIN=$(echo "$VALIDATE_KOTLIN" | awk '{print tolower($0)}') + VALIDATE_PROTOBUF=$(echo "$VALIDATE_PROTOBUF" | awk '{print tolower($0)}') + VALIDATE_OPENAPI=$(echo "$VALIDATE_OPENAPI" | awk '{print tolower($0)}') + + ################################################ + # Determine if any linters were explicitly set # + ################################################ + ANY_SET="false" + if [[ -n "$VALIDATE_YAML" || \ + -n "$VALIDATE_JSON" || \ + -n "$VALIDATE_XML" || \ + -n "$VALIDATE_MD" || \ + -n "$VALIDATE_BASH" || \ + -n "$VALIDATE_PERL" || \ + -n "$VALIDATE_PHP" || \ + -n "$VALIDATE_PYTHON" || \ + -n "$VALIDATE_RUBY" || \ + -n "$VALIDATE_COFFEE" || \ + -n "$VALIDATE_ANSIBLE" || \ + -n "$VALIDATE_JAVASCRIPT_ES" || \ + -n "$VALIDATE_JAVASCRIPT_STANDARD" || \ + -n "$VALIDATE_TYPESCRIPT_ES" || \ + -n "$VALIDATE_TYPESCRIPT_STANDARD" || \ + -n "$VALIDATE_DOCKER" || \ + -n "$VALIDATE_GO" || \ + -n "$VALIDATE_TERRAFORM" || \ + -n "$VALIDATE_POWERSHELL" || \ + -n "$VALIDATE_CSS" || \ + -n "$VALIDATE_ENV" || \ + -n "$VALIDATE_CLOJURE" || \ + -n "$VALIDATE_PROTOBUF" || \ + -n "$VALIDATE_OPENAPI" || \ + -n "$VALIDATE_KOTLIN" ]]; then + ANY_SET="true" + fi + + #################################### + # Validate if we should check YAML # + #################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_YAML" ]]; then + # YAML flag was not set - default to false + VALIDATE_YAML="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_YAML="true" + fi + + #################################### + # Validate if we should check JSON # + #################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_JSON" ]]; then + # JSON flag was not set - default to false + VALIDATE_JSON="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_JSON="true" + fi + + ################################### + # Validate if we should check XML # + ################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_XML" ]]; then + # XML flag was not set - default to false + VALIDATE_XML="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_XML="true" + fi + + ######################################## + # Validate if we should check MARKDOWN # + ######################################## + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_MD" ]]; then + # MD flag was not set - default to false + VALIDATE_MD="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_MD="true" + fi + + #################################### + # Validate if we should check BASH # + #################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_BASH" ]]; then + # BASH flag was not set - default to false + VALIDATE_BASH="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_BASH="true" + fi + + #################################### + # Validate if we should check PERL # + #################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_PERL" ]]; then + # PERL flag was not set - default to false + VALIDATE_PERL="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_PERL="true" + fi + + #################################### + # Validate if we should check PHP # + #################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_PHP" ]]; then + # PHP flag was not set - default to false + VALIDATE_PHP="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_PHP="true" + fi + + ###################################### + # Validate if we should check PYTHON # + ###################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_PYTHON" ]]; then + # PYTHON flag was not set - default to false + VALIDATE_PYTHON="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_PYTHON="true" + fi + + #################################### + # Validate if we should check RUBY # + #################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_RUBY" ]]; then + # RUBY flag was not set - default to false + VALIDATE_RUBY="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_RUBY="true" + fi + + ###################################### + # Validate if we should check COFFEE # + ###################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_COFFEE" ]]; then + # COFFEE flag was not set - default to false + VALIDATE_COFFEE="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_COFFEE="true" + fi + + ####################################### + # Validate if we should check ANSIBLE # + ####################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_ANSIBLE" ]]; then + # ANSIBLE flag was not set - default to false + VALIDATE_ANSIBLE="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_ANSIBLE="true" + fi + + ############################################# + # Validate if we should check JAVASCRIPT_ES # + ############################################# + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_JAVASCRIPT_ES" ]]; then + # JAVASCRIPT_ES flag was not set - default to false + VALIDATE_JAVASCRIPT_ES="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_JAVASCRIPT_ES="true" + fi + + ################################################### + # Validate if we should check JAVASCRIPT_STANDARD # + ################################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_JAVASCRIPT_STANDARD" ]]; then + # JAVASCRIPT_STANDARD flag was not set - default to false + VALIDATE_JAVASCRIPT_STANDARD="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_JAVASCRIPT_STANDARD="true" + fi + + ############################################# + # Validate if we should check TYPESCRIPT_ES # + ############################################# + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_TYPESCRIPT_ES" ]]; then + # TYPESCRIPT_ES flag was not set - default to false + VALIDATE_TYPESCRIPT_ES="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_TYPESCRIPT_ES="true" + fi + + ################################################### + # Validate if we should check TYPESCRIPT_STANDARD # + ################################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_TYPESCRIPT_STANDARD" ]]; then + # TYPESCRIPT_STANDARD flag was not set - default to false + VALIDATE_TYPESCRIPT_STANDARD="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_TYPESCRIPT_STANDARD="true" + fi + + ###################################### + # Validate if we should check DOCKER # + ###################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_DOCKER" ]]; then + # DOCKER flag was not set - default to false + VALIDATE_DOCKER="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_DOCKER="true" + fi + + ################################## + # Validate if we should check GO # + ################################## + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_GO" ]]; then + # GO flag was not set - default to false + VALIDATE_GO="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_GO="true" + fi + + ######################################### + # Validate if we should check TERRAFORM # + ######################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_TERRAFORM" ]]; then + # TERRAFORM flag was not set - default to false + VALIDATE_TERRAFORM="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_TERRAFORM="true" + fi + + ######################################### + # Validate if we should check POWERSHELL # + ######################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_POWERSHELL" ]]; then + # POWERSHELL flag was not set - default to false + VALIDATE_POWERSHELL="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_POWERSHELL="true" + fi + + ################################### + # Validate if we should check CSS # + ################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_CSS" ]]; then + # CSS flag was not set - default to false + VALIDATE_CSS="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_CSS="true" + fi + + ################################### + # Validate if we should check ENV # + ################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_ENV" ]]; then + # ENV flag was not set - default to false + VALIDATE_ENV="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_ENV="true" + fi + + ###################################### + # Validate if we should check KOTLIN # + ###################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_KOTLIN" ]]; then + # ENV flag was not set - default to false + VALIDATE_KOTLIN="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_KOTLIN="true" + fi + + ####################################### + # Validate if we should check OPENAPI # + ####################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_OPENAPI" ]]; then + # OPENAPI flag was not set - default to false + VALIDATE_OPENAPI="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_OPENAPI="true" + fi + + ####################################### + # Validate if we should check PROTOBUF # + ####################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_PROTOBUF" ]]; then + # PROTOBUF flag was not set - default to false + VALIDATE_PROTOBUF="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_PROTOBUF="true" + fi + + ####################################### + # Validate if we should check Clojure # + ####################################### + if [[ "$ANY_SET" == "true" ]]; then + # Some linter flags were set - only run those set to true + if [[ -z "$VALIDATE_CLOJURE" ]]; then + # Clojure flag was not set - default to false + VALIDATE_CLOJURE="false" + fi + else + # No linter flags were set - default all to true + VALIDATE_CLOJURE="true" + fi + + ####################################### + # Print which linters we are enabling # + ####################################### + if [[ "$VALIDATE_YAML" == "true" ]]; then + PRINT_ARRAY+=("- Validating [YAML] files in code base...") + else + PRINT_ARRAY+=("- Excluding [YAML] files in code base...") + fi + if [[ "$VALIDATE_JSON" == "true" ]]; then + PRINT_ARRAY+=("- Validating [JSON] files in code base...") + else + PRINT_ARRAY+=("- Excluding [JSON] files in code base...") + fi + if [[ "$VALIDATE_XML" == "true" ]]; then + PRINT_ARRAY+=("- Validating [XML] files in code base...") + else + PRINT_ARRAY+=("- Excluding [XML] files in code base...") + fi + if [[ "$VALIDATE_MD" == "true" ]]; then + PRINT_ARRAY+=("- Validating [MARKDOWN] files in code base...") + else + PRINT_ARRAY+=("- Excluding [MARKDOWN] files in code base...") + fi + if [[ "$VALIDATE_BASH" == "true" ]]; then + PRINT_ARRAY+=("- Validating [BASH] files in code base...") + else + PRINT_ARRAY+=("- Excluding [BASH] files in code base...") + fi + if [[ "$VALIDATE_PERL" == "true" ]]; then + PRINT_ARRAY+=("- Validating [PERL] files in code base...") + else + PRINT_ARRAY+=("- Excluding [PERL] files in code base...") + fi + if [[ "$VALIDATE_PHP" == "true" ]]; then + PRINT_ARRAY+=("- Validating [PHP] files in code base...") + else + PRINT_ARRAY+=("- Excluding [PHP] files in code base...") + fi + if [[ "$VALIDATE_PYTHON" == "true" ]]; then + PRINT_ARRAY+=("- Validating [PYTHON] files in code base...") + else + PRINT_ARRAY+=("- Excluding [PYTHON] files in code base...") + fi + if [[ "$VALIDATE_RUBY" == "true" ]]; then + PRINT_ARRAY+=("- Validating [RUBY] files in code base...") + else + PRINT_ARRAY+=("- Excluding [RUBY] files in code base...") + fi + if [[ "$VALIDATE_COFFEE" == "true" ]]; then + PRINT_ARRAY+=("- Validating [COFFEE] files in code base...") + else + PRINT_ARRAY+=("- Excluding [COFFEE] files in code base...") + fi + if [[ "$VALIDATE_ANSIBLE" == "true" ]]; then + PRINT_ARRAY+=("- Validating [ANSIBLE] files in code base...") + else + PRINT_ARRAY+=("- Excluding [ANSIBLE] files in code base...") + fi + if [[ "$VALIDATE_JAVASCRIPT_ES" == "true" ]]; then + PRINT_ARRAY+=("- Validating [JAVASCRIPT(eslint)] files in code base...") + else + PRINT_ARRAY+=("- Excluding [JAVASCRIPT(eslint)] files in code base...") + fi + if [[ "$VALIDATE_JAVASCRIPT_STANDARD" == "true" ]]; then + PRINT_ARRAY+=("- Validating [JAVASCRIPT(standard)] files in code base...") + else + PRINT_ARRAY+=("- Excluding [JAVASCRIPT(standard)] files in code base...") + fi + if [[ "$VALIDATE_TYPESCRIPT_ES" == "true" ]]; then + PRINT_ARRAY+=("- Validating [TYPESCRIPT(eslint)] files in code base...") + else + PRINT_ARRAY+=("- Excluding [TYPESCRIPT(eslint)] files in code base...") + fi + if [[ "$VALIDATE_TYPESCRIPT_STANDARD" == "true" ]]; then + PRINT_ARRAY+=("- Validating [TYPESCRIPT(standard)] files in code base...") + else + PRINT_ARRAY+=("- Excluding [TYPESCRIPT(standard)] files in code base...") + fi + if [[ "$VALIDATE_DOCKER" == "true" ]]; then + PRINT_ARRAY+=("- Validating [DOCKER] files in code base...") + else + PRINT_ARRAY+=("- Excluding [DOCKER] files in code base...") + fi + if [[ "$VALIDATE_GO" == "true" ]]; then + PRINT_ARRAY+=("- Validating [GOLANG] files in code base...") + else + PRINT_ARRAY+=("- Excluding [GOLANG] files in code base...") + fi + if [[ "$VALIDATE_TERRAFORM" == "true" ]]; then + PRINT_ARRAY+=("- Validating [TERRAFORM] files in code base...") + else + PRINT_ARRAY+=("- Excluding [TERRAFORM] files in code base...") + fi + if [[ "$VALIDATE_POWERSHELL" == "true" ]]; then + PRINT_ARRAY+=("- Validating [POWERSHELL] files in code base...") + else + PRINT_ARRAY+=("- Excluding [POWERSHELL] files in code base...") + fi + if [[ "$VALIDATE_CSS" == "true" ]]; then + PRINT_ARRAY+=("- Validating [CSS] files in code base...") + else + PRINT_ARRAY+=("- Excluding [CSS] files in code base...") + fi + if [[ "$VALIDATE_CLOJURE" == "true" ]]; then + PRINT_ARRAY+=("- Validating [CLOJURE] files in code base...") + else + PRINT_ARRAY+=("- Excluding [CLOJURE] files in code base...") + fi + if [[ "$VALIDATE_ENV" == "true" ]]; then + PRINT_ARRAY+=("- Validating [ENV] files in code base...") + else + PRINT_ARRAY+=("- Excluding [ENV] files in code base...") + fi + if [[ "$VALIDATE_KOTLIN" == "true" ]]; then + PRINT_ARRAY+=("- Validating [KOTLIN] files in code base...") + else + PRINT_ARRAY+=("- Excluding [KOTLIN] files in code base...") + fi + if [[ "$VALIDATE_OPENAPI" == "true" ]]; then + PRINT_ARRAY+=("- Validating [OPENAPI] files in code base...") + else + PRINT_ARRAY+=("- Excluding [OPENAPI] files in code base...") + fi + if [[ "$VALIDATE_PROTOBUF" == "true" ]]; then + PRINT_ARRAY+=("- Validating [PROTOBUF] files in code base...") + else + PRINT_ARRAY+=("- Excluding [PROTOBUF] files in code base...") + fi + + ############################## + # Validate Ansible Directory # + ############################## + if [ -z "$ANSIBLE_DIRECTORY" ]; then + # No Value, need to default + ANSIBLE_DIRECTORY="$DEFAULT_ANSIBLE_DIRECTORY" + else + # Check if first char is '/' + if [[ ${ANSIBLE_DIRECTORY:0:1} == "/" ]]; then + # Remove first char + ANSIBLE_DIRECTORY="${ANSIBLE_DIRECTORY:1}" + fi + # Need to give it full path + TEMP_ANSIBLE_DIRECTORY="$GITHUB_WORKSPACE/$ANSIBLE_DIRECTORY" + # Set the value + ANSIBLE_DIRECTORY="$TEMP_ANSIBLE_DIRECTORY" + fi + + ############################### + # Get the disable errors flag # + ############################### + if [ -z "$DISABLE_ERRORS" ]; then + ################################## + # No flag passed, set to default # + ################################## + DISABLE_ERRORS="$DEFAULT_DISABLE_ERRORS" + fi + + ############################### + # Convert string to lowercase # + ############################### + DISABLE_ERRORS=$(echo "$DISABLE_ERRORS" | awk '{print tolower($0)}') + + ############################ + # Set to false if not true # + ############################ + if [ "$DISABLE_ERRORS" != "true" ]; then + DISABLE_ERRORS="false" + fi + + ############################ + # Get the run verbose flag # + ############################ + if [ -z "$ACTIONS_RUNNER_DEBUG" ]; then + ################################## + # No flag passed, set to default # + ################################## + ACTIONS_RUNNER_DEBUG="$DEFAULT_ACTIONS_RUNNER_DEBUG" + fi + + ############################### + # Convert string to lowercase # + ############################### + ACTIONS_RUNNER_DEBUG=$(echo "$ACTIONS_RUNNER_DEBUG" | awk '{print tolower($0)}') + + ############################ + # Set to true if not false # + ############################ + if [ "$ACTIONS_RUNNER_DEBUG" != "false" ]; then + ACTIONS_RUNNER_DEBUG="true" + fi + + ################### + # Debug on runner # + ################### + if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then + ########################### + # Print the validate info # + ########################### + for LINE in "${PRINT_ARRAY[@]}" + do + echo "$LINE" + done + + echo "--- DEBUG INFO ---" + echo "---------------------------------------------" + RUNNER=$(whoami) + echo "Runner:[$RUNNER]" + echo "ENV:" + printenv + echo "---------------------------------------------" + fi +} diff --git a/lib/worker.sh b/lib/worker.sh new file mode 100755 index 00000000..1b879768 --- /dev/null +++ b/lib/worker.sh @@ -0,0 +1,658 @@ +#!/usr/bin/env bash + +################################################################################ +################################################################################ +########### Super-Linter linting Functions @admiralawkbar ###################### +################################################################################ +################################################################################ +########################## FUNCTION CALLS BELOW ################################ +################################################################################ +################################################################################ +#### Function LintCodebase ##################################################### +function LintCodebase() +{ + #################### + # Pull in the vars # + #################### + FILE_TYPE="$1" && shift # Pull the variable and remove from array path (Example: JSON) + LINTER_NAME="$1" && shift # Pull the variable and remove from array path (Example: jsonlint) + LINTER_COMMAND="$1" && shift # Pull the variable and remove from array path (Example: jsonlint -c ConfigFile /path/to/file) + FILE_EXTENSIONS="$1" && shift # Pull the variable and remove from array path (Example: *.json) + FILE_ARRAY=("$@") # Array of files to validate (Example: $FILE_ARRAY_JSON) + + ###################### + # Create Print Array # + ###################### + PRINT_ARRAY=() + + ################ + # print header # + ################ + PRINT_ARRAY+=("") + PRINT_ARRAY+=("----------------------------------------------") + PRINT_ARRAY+=("----------------------------------------------") + PRINT_ARRAY+=("Linting [$FILE_TYPE] files...") + PRINT_ARRAY+=("----------------------------------------------") + PRINT_ARRAY+=("----------------------------------------------") + + ####################################### + # Validate we have jsonlint installed # + ####################################### + # shellcheck disable=SC2230 + VALIDATE_INSTALL_CMD=$(command -v "$LINTER_NAME" 2>&1) + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + # Failed + echo "ERROR! Failed to find [$LINTER_NAME] in system!" + echo "ERROR:[$VALIDATE_INSTALL_CMD]" + exit 1 + else + # Success + if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then + echo "Successfully found binary in system" + echo "Location:[$VALIDATE_INSTALL_CMD]" + fi + fi + + ########################## + # Initialize empty Array # + ########################## + LIST_FILES=() + + ################ + # Set the flag # + ################ + SKIP_FLAG=0 + + ############################################################ + # Check to see if we need to go through array or all files # + ############################################################ + if [ ${#FILE_ARRAY[@]} -eq 0 ] && [ "$VALIDATE_ALL_CODEBASE" == "false" ]; then + # No files found in commit and user has asked to not validate code base + SKIP_FLAG=1 + # echo " - No files found in changeset to lint for language:[$FILE_TYPE]" + elif [ ${#FILE_ARRAY[@]} -ne 0 ]; then + # We have files added to array of files to check + LIST_FILES=("${FILE_ARRAY[@]}") # Copy the array into list + else + ############################################################################### + # Set the file seperator to newline to allow for grabbing objects with spaces # + ############################################################################### + IFS=$'\n' + + ################################# + # Get list of all files to lint # + ################################# + # shellcheck disable=SC2207,SC2086 + LIST_FILES=($(cd "$GITHUB_WORKSPACE" || exit; find . -type f -regex "$FILE_EXTENSIONS" 2>&1)) + + ########################### + # Set IFS back to default # + ########################### + IFS="$DEFAULT_IFS" + + ############################################################ + # Set it back to empty if loaded with blanks from scanning # + ############################################################ + if [ ${#LIST_FILES[@]} -lt 1 ]; then + ###################### + # Set to empty array # + ###################### + LIST_FILES=() + ############################# + # Skip as we found no files # + ############################# + SKIP_FLAG=1 + fi + fi + + ############################### + # Check if any data was found # + ############################### + if [ $SKIP_FLAG -eq 0 ]; then + ###################### + # Print Header array # + ###################### + for LINE in "${PRINT_ARRAY[@]}" + do + ######################### + # Print the header info # + ######################### + echo "$LINE" + done + + ################## + # Lint the files # + ################## + for FILE in "${LIST_FILES[@]}" + do + ##################### + # Get the file name # + ##################### + FILE_NAME=$(basename "$FILE" 2>&1) + + ##################################################### + # Make sure we dont lint node modules or test cases # + ##################################################### + if [[ $FILE == *"node_modules"* ]]; then + # This is a node modules file + continue + elif [[ $FILE == *"$TEST_CASE_FOLDER"* ]]; then + # This is the test cases, we should always skip + continue + fi + + ############## + # File print # + ############## + echo "---------------------------" + echo "File:[$FILE]" + + #################### + # Set the base Var # + #################### + LINT_CMD='' + + ####################################### + # Corner case for Powershell subshell # + ####################################### + if [[ "$FILE_TYPE" == "POWERSHELL" ]]; then + ################################ + # Lint the file with the rules # + ################################ + # Need to append "'" to make the pwsh call syntax correct, also exit with exit code from inner subshell + LINT_CMD=$(cd "$GITHUB_WORKSPACE" || exit; $LINTER_COMMAND "$FILE"; exit $? 2>&1) + else + ################################ + # Lint the file with the rules # + ################################ + LINT_CMD=$(cd "$GITHUB_WORKSPACE" || exit; $LINTER_COMMAND "$FILE" 2>&1) + fi + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + ######### + # Error # + ######### + echo "ERROR! Found errors in [$LINTER_NAME] linter!" + echo "ERROR:[$LINT_CMD]" + # Increment the error count + (("ERRORS_FOUND_$FILE_TYPE++")) + else + ########### + # Success # + ########### + echo " - File:[$FILE_NAME] was linted with [$LINTER_NAME] successfully" + fi + done + fi +} +################################################################################ +#### Function TestCodebase ##################################################### +function TestCodebase() +{ + #################### + # Pull in the vars # + #################### + FILE_TYPE="$1" # Pull the variable and remove from array path (Example: JSON) + LINTER_NAME="$2" # Pull the variable and remove from array path (Example: jsonlint) + LINTER_COMMAND="$3" # Pull the variable and remove from array path (Example: jsonlint -c ConfigFile /path/to/file) + FILE_EXTENSIONS="$4" # Pull the variable and remove from array path (Example: *.json) + + ################ + # print header # + ################ + echo "" + echo "----------------------------------------------" + echo "----------------------------------------------" + echo "Testing Codebase [$FILE_TYPE] files..." + echo "----------------------------------------------" + echo "----------------------------------------------" + echo "" + + ##################################### + # Validate we have linter installed # + ##################################### + # shellcheck disable=SC2230 + VALIDATE_INSTALL_CMD=$(command -v "$LINTER_NAME" 2>&1) + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + # Failed + echo "ERROR! Failed to find [$LINTER_NAME] in system!" + echo "ERROR:[$VALIDATE_INSTALL_CMD]" + exit 1 + else + # Success + echo "Successfully found binary in system" + echo "Location:[$VALIDATE_INSTALL_CMD]" + fi + + ########################## + # Initialize empty Array # + ########################## + LIST_FILES=() + + ############################################ + # Check if its ansible, as its the outlier # + ############################################ + if [[ "$FILE_TYPE" == "ANSIBLE" ]]; then + ################################# + # Get list of all files to lint # + ################################# + # shellcheck disable=SC2207,SC2086,SC2010 + LIST_FILES=($(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER" || exit; ls ansible/ | grep ".yml" 2>&1)) + else + ############################################################################### + # Set the file seperator to newline to allow for grabbing objects with spaces # + ############################################################################### + IFS=$'\n' + + ################################# + # Get list of all files to lint # + ################################# + # shellcheck disable=SC2207,SC2086 + LIST_FILES=($(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER" || exit; find . -type f -regex "$FILE_EXTENSIONS" ! -path "*./ansible*" 2>&1)) + + ########################### + # Set IFS back to default # + ########################### + IFS="$DEFAULT_IFS" + fi + + ################## + # Lint the files # + ################## + for FILE in "${LIST_FILES[@]}" + do + ##################### + # Get the file name # + ##################### + FILE_NAME=$(basename "$FILE" 2>&1) + + ############################ + # Get the file pass status # + ############################ + # Example: markdown_good_1.md -> good + FILE_STATUS=$(echo "$FILE_NAME" |cut -f2 -d'_') + + ######################################################### + # If not found, assume it should be linted successfully # + ######################################################### + if [ -z "$FILE_STATUS" ] || [[ "$FILE" == *"README"* ]]; then + ################################## + # Set to good for proper linting # + ################################## + FILE_STATUS="good" + fi + + ############## + # File print # + ############## + echo "---------------------------" + echo "File:[$FILE]" + + ######################## + # Set the lint command # + ######################## + LINT_CMD='' + + ####################################### + # Check if docker and get folder name # + ####################################### + if [[ "$FILE_TYPE" == "DOCKER" ]]; then + if [[ "$FILE" == *"good"* ]]; then + ############# + # Good file # + ############# + FILE_STATUS='good' + else + ############ + # Bad file # + ############ + FILE_STATUS='bad' + fi + fi + + ##################### + # Check for ansible # + ##################### + if [[ "$FILE_TYPE" == "ANSIBLE" ]]; then + ######################################## + # Make sure we dont lint certain files # + ######################################## + if [[ $FILE == *"vault.yml"* ]] || [[ $FILE == *"galaxy.yml"* ]]; then + # This is a file we dont look at + continue + fi + + ################################ + # Lint the file with the rules # + ################################ + LINT_CMD=$(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER/ansible" || exit; $LINTER_COMMAND "$FILE" 2>&1) + elif [[ "$FILE_TYPE" == "POWERSHELL" ]]; then + ################################ + # Lint the file with the rules # + ################################ + # Need to append "'" to make the pwsh call syntax correct, also exit with exit code from inner subshell + LINT_CMD=$(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER" || exit; $LINTER_COMMAND "$FILE"; exit $? 2>&1) + else + ################################ + # Lint the file with the rules # + ################################ + LINT_CMD=$(cd "$GITHUB_WORKSPACE/$TEST_CASE_FOLDER" || exit; $LINTER_COMMAND "$FILE" 2>&1) + fi + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ######################################## + # Check for if it was supposed to pass # + ######################################## + if [[ "$FILE_STATUS" == "good" ]]; then + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + ######### + # Error # + ######### + echo "ERROR! Found errors in [$LINTER_NAME] linter!" + echo "ERROR:[$LINT_CMD]" + echo "ERROR: Linter CMD:[$LINTER_COMMAND $FILE]" + # Increment the error count + (("ERRORS_FOUND_$FILE_TYPE++")) + else + ########### + # Success # + ########### + echo " - File:[$FILE_NAME] was linted with [$LINTER_NAME] successfully" + fi + else + ####################################### + # File status = bad, this should fail # + ####################################### + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -eq 0 ]; then + ######### + # Error # + ######### + echo "ERROR! Found errors in [$LINTER_NAME] linter!" + echo "ERROR! This file should have failed test case!" + echo "ERROR:[$LINT_CMD]" + echo "ERROR: Linter CMD:[$LINTER_COMMAND $FILE]" + # Increment the error count + (("ERRORS_FOUND_$FILE_TYPE++")) + else + ########### + # Success # + ########### + echo " - File:[$FILE_NAME] failed test case with [$LINTER_NAME] successfully" + fi + fi + done +} +################################################################################ +#### Function RunTestCases ##################################################### +function RunTestCases() +{ + # This loop will run the test cases and exclude user code + # This is called from the automation process to validate new code + # When a PR is opened, the new code is validated with the default branch + # version of linter.sh, and a new container is built with the latest codebase + # for testing. That container is spun up, and ran, + # with the flag: TEST_CASE_RUN=true + # So that the new code can be validated against the test cases + + ################# + # Header prints # + ################# + echo "" + echo "----------------------------------------------" + echo "-------------- TEST CASE RUN -----------------" + echo "----------------------------------------------" + echo "" + + ####################### + # Test case languages # + ####################### + TestCodebase "YML" "yamllint" "yamllint -c $YAML_LINTER_RULES" ".*\.\(yml\|yaml\)\$" + TestCodebase "JSON" "jsonlint" "jsonlint" ".*\.\(json\)\$" + TestCodebase "XML" "xmllint" "xmllint" ".*\.\(xml\)\$" + TestCodebase "MARKDOWN" "markdownlint" "markdownlint -c $MD_LINTER_RULES" ".*\.\(md\)\$" + TestCodebase "BASH" "shellcheck" "shellcheck" ".*\.\(sh\)\$" + TestCodebase "PYTHON" "pylint" "pylint --rcfile $PYTHON_LINTER_RULES" ".*\.\(py\)\$" + TestCodebase "PERL" "perl" "perl -Mstrict -cw" ".*\.\(pl\)\$" + TestCodebase "PHP" "php" "php -l" ".*\.\(php\)\$" + TestCodebase "RUBY" "rubocop" "rubocop -c $RUBY_LINTER_RULES" ".*\.\(rb\)\$" + TestCodebase "GO" "golangci-lint" "golangci-lint run -c $GO_LINTER_RULES" ".*\.\(go\)\$" + TestCodebase "COFFEESCRIPT" "coffeelint" "coffeelint -f $COFFEESCRIPT_LINTER_RULES" ".*\.\(coffee\)\$" + TestCodebase "JAVASCRIPT_ES" "eslint" "eslint --no-eslintrc -c $JAVASCRIPT_LINTER_RULES" ".*\.\(js\)\$" + TestCodebase "JAVASCRIPT_STANDARD" "standard" "standard $JAVASCRIPT_STANDARD_LINTER_RULES" ".*\.\(js\)\$" + TestCodebase "TYPESCRIPT_ES" "eslint" "eslint --no-eslintrc -c $TYPESCRIPT_LINTER_RULES" ".*\.\(ts\)\$" + TestCodebase "TYPESCRIPT_STANDARD" "standard" "standard --parser @typescript-eslint/parser --plugin @typescript-eslint/eslint-plugin $TYPESCRIPT_STANDARD_LINTER_RULES" ".*\.\(ts\)\$" + TestCodebase "DOCKER" "/dockerfilelint/bin/dockerfilelint" "/dockerfilelint/bin/dockerfilelint -c $DOCKER_LINTER_RULES" ".*\(Dockerfile\)\$" + TestCodebase "ANSIBLE" "ansible-lint" "ansible-lint -v -c $ANSIBLE_LINTER_RULES" "ansible-lint" + TestCodebase "TERRAFORM" "tflint" "tflint -c $TERRAFORM_LINTER_RULES" ".*\.\(tf\)\$" + TestCodebase "POWERSHELL" "pwsh" "pwsh -c Invoke-ScriptAnalyzer -EnableExit -Settings $POWERSHELL_LINTER_RULES -Path" ".*\.\(ps1\|psm1\|psd1\|ps1xml\|pssc\|psrc\|cdxml\)\$" + TestCodebase "CSS" "stylelint" "stylelint --config $CSS_LINTER_RULES" ".*\.\(css\)\$" + TestCodebase "ENV" "dotenv-linter" "dotenv-linter" ".*\.\(env\)\$" + TestCodebase "CLOJURE" "clj-kondo" "clj-kondo --config $CLOJURE_LINTER_RULES --lint" ".*\.\(clj\|cljs\|cljc\|edn\)\$" + TestCodebase "KOTLIN" "ktlint" "ktlint" ".*\.\(kt\|kts\)\$" + TestCodebase "PROTOBUF" "protolint" "protolint lint --config_path $PROTOBUF_LINTER_RULES" ".*\.\(proto\)\$" + TestCodebase "OPENAPI" "spectral" "spectral lint -r $OPENAPI_LINTER_RULES" ".*\.\(ymlopenapi\|jsonopenapi\)\$" + + ################# + # Footer prints # + ################# + # Call the footer to display run information + # and exit with error code + Footer +} +################################################################################ +#### Function LintAnsibleFiles ################################################# +function LintAnsibleFiles() +{ + ###################### + # Create Print Array # + ###################### + PRINT_ARRAY=() + + ################ + # print header # + ################ + PRINT_ARRAY+=("") + PRINT_ARRAY+=("----------------------------------------------") + PRINT_ARRAY+=("----------------------------------------------") + PRINT_ARRAY+=("Linting [Ansible] files...") + PRINT_ARRAY+=("----------------------------------------------") + PRINT_ARRAY+=("----------------------------------------------") + + ###################### + # Name of the linter # + ###################### + LINTER_NAME="ansible-lint" + + ########################################### + # Validate we have ansible-lint installed # + ########################################### + # shellcheck disable=SC2230 + VALIDATE_INSTALL_CMD=$(command -v "$LINTER_NAME" 2>&1) + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + # Failed + echo "ERROR! Failed to find $LINTER_NAME in system!" + echo "ERROR:[$VALIDATE_INSTALL_CMD]" + exit 1 + else + # Success + if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then + # Success + echo "Successfully found binary in system" + echo "Location:[$VALIDATE_INSTALL_CMD]" + fi + fi + + ########################## + # Initialize empty Array # + ########################## + LIST_FILES=() + + ####################### + # Create flag to skip # + ####################### + SKIP_FLAG=0 + + ###################################################### + # Only go into ansible linter if we have base folder # + ###################################################### + if [ -d "$ANSIBLE_DIRECTORY" ]; then + + ############################################################ + # Check to see if we need to go through array or all files # + ############################################################ + if [ "$VALIDATE_ALL_CODEBASE" == "false" ]; then + # We need to only check the ansible playbooks that have updates + #LIST_FILES=("${ANSIBLE_ARRAY[@]}") + # shellcheck disable=SC2164,SC2010,SC2207 + LIST_FILES=($(cd "$ANSIBLE_DIRECTORY"; ls | grep ".yml" 2>&1)) + else + ################################# + # Get list of all files to lint # + ################################# + # shellcheck disable=SC2164,SC2010,SC2207 + LIST_FILES=($(cd "$ANSIBLE_DIRECTORY"; ls | grep ".yml" 2>&1)) + fi + + ############################################################### + # Set the list to empty if only MD and TXT files were changed # + ############################################################### + # No need to run the full ansible checks on read only file changes + if [ "$READ_ONLY_CHANGE_FLAG" -eq 0 ]; then + ########################## + # Set the array to empty # + ########################## + LIST_FILES=() + ################################### + # Send message that were skipping # + ################################### + #echo "- Skipping Ansible lint run as file(s) that were modified were read only..." + ############################ + # Create flag to skip loop # + ############################ + SKIP_FLAG=1 + fi + + #################################### + # Check if we have data to look at # + #################################### + if [ $SKIP_FLAG -eq 0 ]; then + for LINE in "${PRINT_ARRAY[@]}" + do + ######################### + # Print the header line # + ######################### + echo "$LINE" + done + fi + + ################## + # Lint the files # + ################## + for FILE in "${LIST_FILES[@]}" + do + + ######################################## + # Make sure we dont lint certain files # + ######################################## + if [[ $FILE == *"vault.yml"* ]] || [[ $FILE == *"galaxy.yml"* ]]; then + # This is a file we dont look at + continue + fi + + #################### + # Get the filename # + #################### + FILE_NAME=$(basename "$ANSIBLE_DIRECTORY/$FILE" 2>&1) + + ############## + # File print # + ############## + echo "---------------------------" + echo "File:[$FILE]" + + ################################ + # Lint the file with the rules # + ################################ + LINT_CMD=$("$LINTER_NAME" -v -c "$ANSIBLE_LINTER_RULES" "$ANSIBLE_DIRECTORY/$FILE" 2>&1) + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + ######### + # Error # + ######### + echo "ERROR! Found errors in [$LINTER_NAME] linter!" + echo "ERROR:[$LINT_CMD]" + # Increment error count + ((ERRORS_FOUND_ANSIBLE++)) + else + ########### + # Success # + ########### + echo " - File:[$FILE_NAME] was linted with [$LINTER_NAME] successfully" + fi + done + else # No ansible directory found in path + ############################### + # Check to see if debug is on # + ############################### + if [[ "$ACTIONS_RUNNER_DEBUG" == "true" ]]; then + ######################## + # No Ansible dir found # + ######################## + echo "WARN! No Ansible base directory found at:[$ANSIBLE_DIRECTORY]" + echo "skipping ansible lint" + fi + fi +} From b90310f2a81c7f30922aa223175d139459da28a6 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 10:06:12 -0500 Subject: [PATCH 54/65] fix path --- lib/linter.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/linter.sh b/lib/linter.sh index f1dad6ec..d7644d38 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -10,8 +10,7 @@ ######################### # Source Function Files # ######################### -source ./validation.sh # Source the main validation function script -source ./buildFileList.sh # Source the main build file list function script +source /action/lib/*.sh # Source the function script(s) ########### # GLOBALS # From 679ecf3b1ae9138b2ef4f50e381be61b5d2deb17 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 10:15:39 -0500 Subject: [PATCH 55/65] lets be literal --- lib/linter.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/linter.sh b/lib/linter.sh index d7644d38..6b4e6761 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -10,7 +10,9 @@ ######################### # Source Function Files # ######################### -source /action/lib/*.sh # Source the function script(s) +source /action/lib/buildFileList.sh # Source the function script(s) +source /action/lib/validation.sh # Source the function script(s) +source /action/lib/worker.sh # Source the function script(s) ########### # GLOBALS # From 4194a7a611ec2c8e5fba1014e22e56c5045fb261 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 10:34:23 -0500 Subject: [PATCH 56/65] cleaner --- lib/buildFileList.sh | 1 + lib/linter.sh | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/buildFileList.sh b/lib/buildFileList.sh index 2ed2ce60..d39f7a64 100755 --- a/lib/buildFileList.sh +++ b/lib/buildFileList.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck disable=SC2034 ################################################################################ ################################################################################ diff --git a/lib/linter.sh b/lib/linter.sh index 6b4e6761..b69ee0b7 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# shellcheck disable=SC1003,SC2016 +# shellcheck disable=SC1003,SC2016,SC1091 ################################################################################ ################################################################################ @@ -126,23 +126,27 @@ DISABLE_ERRORS="${DISABLE_ERRORS}" # Boolean to enable warnin ############## # Debug Vars # ############## -RUN_LOCAL="${RUN_LOCAL}" # Boolean to see if we are running locally -ACTIONS_RUNNER_DEBUG="${ACTIONS_RUNNER_DEBUG}" # Boolean to see even more info (debug) +RUN_LOCAL="${RUN_LOCAL}" # Boolean to see if we are running locally +ACTIONS_RUNNER_DEBUG="${ACTIONS_RUNNER_DEBUG:-false}" # Boolean to see even more info (debug) ################ # Default Vars # ################ DEFAULT_VALIDATE_ALL_CODEBASE='true' # Default value for validate all files DEFAULT_WORKSPACE="${DEFAULT_WORKSPACE:-/tmp/lint}" # Default workspace if running locally -DEFAULT_ANSIBLE_DIRECTORY="$GITHUB_WORKSPACE/ansible" # Default Ansible Directory DEFAULT_RUN_LOCAL='false' # Default value for debugging locally DEFAULT_TEST_CASE_RUN='false' # Flag to tell code to run only test cases -DEFAULT_ACTIONS_RUNNER_DEBUG='false' # Default value for debugging output +DEFAULT_IFS="$IFS" # Get the Default IFS for updating + +############################################################### +# Default Vars that are called in Subs and need to be ignored # +############################################################### +# shellcheck disable=SC2034 +DEFAULT_DISABLE_ERRORS='false' # Default to enabling errors RAW_FILE_ARRAY=() # Array of all files that were changed READ_ONLY_CHANGE_FLAG=0 # Flag set to 1 if files changed are not txt or md TEST_CASE_FOLDER='.automation/test' # Folder for test cases we should always ignore -DEFAULT_DISABLE_ERRORS='false' # Default to enabling errors -DEFAULT_IFS="$IFS" # Get the Default IFS for updating +DEFAULT_ANSIBLE_DIRECTORY="$GITHUB_WORKSPACE/ansible" # Default Ansible Directory ########################## # Array of changed files # @@ -966,7 +970,7 @@ if [ "$VALIDATE_KOTLIN" == "true" ]; then # Lint the Kotlin files # ####################### # LintCodebase "FILE_TYPE" "LINTER_NAME" "LINTER_CMD" "FILE_TYPES_REGEX" "FILE_ARRAY" - LintCodebase "KOTLIN" "ktlint" "ktlint" ".*\.\(kt\|kts\)\$" "${FILE_ARRAY_ENV[@]}" + LintCodebase "KOTLIN" "ktlint" "ktlint" ".*\.\(kt\|kts\)\$" "${FILE_ARRAY_KOTLIN[@]}" fi ################## From 06e237d19eb09fd468648379da4f836be09817b4 Mon Sep 17 00:00:00 2001 From: Thomas Hughes Date: Mon, 29 Jun 2020 10:39:09 -0500 Subject: [PATCH 57/65] Add actions-tagger workflow to automate version tags --- .github/workflows/versioning.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/workflows/versioning.yml diff --git a/.github/workflows/versioning.yml b/.github/workflows/versioning.yml new file mode 100644 index 00000000..f25ee032 --- /dev/null +++ b/.github/workflows/versioning.yml @@ -0,0 +1,15 @@ +name: Keep the versions up-to-date + +on: + release: + types: [published, edited] + +jobs: + actions-tagger: + runs-on: windows-latest + steps: + - uses: Actions-R-Us/actions-tagger@latest + env: + GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" + with: + - publish_latest_tag: true From 4ccd6b35c969a717613e027707c606b537d1a271 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 10:43:52 -0500 Subject: [PATCH 58/65] fixed bug --- lib/linter.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/linter.sh b/lib/linter.sh index b69ee0b7..dea50fab 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -143,9 +143,13 @@ DEFAULT_IFS="$IFS" # Get the Default IFS for updating ############################################################### # shellcheck disable=SC2034 DEFAULT_DISABLE_ERRORS='false' # Default to enabling errors +# shellcheck disable=SC2034 RAW_FILE_ARRAY=() # Array of all files that were changed +# shellcheck disable=SC2034 READ_ONLY_CHANGE_FLAG=0 # Flag set to 1 if files changed are not txt or md +# shellcheck disable=SC2034 TEST_CASE_FOLDER='.automation/test' # Folder for test cases we should always ignore +# shellcheck disable=SC2034 DEFAULT_ANSIBLE_DIRECTORY="$GITHUB_WORKSPACE/ansible" # Default Ansible Directory ########################## @@ -603,6 +607,11 @@ Footer() ########################### ERROR_COUNTER="ERRORS_FOUND_$LANGUAGE" + ################################# + # Fix Integer Expression errors # + ################################# + ERROR_COUNTER="${ERROR_COUNTER/.*}" + ################## # Print if not 0 # ################## From 37a37f09300e356952821bf7308475231ed81019 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 11:17:25 -0500 Subject: [PATCH 59/65] adding readme --- lib/README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 lib/README.md diff --git a/lib/README.md b/lib/README.md new file mode 100644 index 00000000..500e2142 --- /dev/null +++ b/lib/README.md @@ -0,0 +1,15 @@ +# Super-Linter Library + +## Main script +The file `linter.sh` is the main script that is called for the process and loads all other scripts as functions. + +## Functions +The additional files in the folder are functions to help streamline the main build process and allow for easier maintenance. +- `possum.sh` + - Official mascot of the **Super-Linter** +- `buildFileList.sh` + - Functions to help find files that were modified, or need to be scanned +- `validation.sh` + - Logic to see what linters are enabled +- `worker.sh` + - Calls to the various linters and the test cases for each From 2067571f1f37af8ee2a584498377b01715505fa0 Mon Sep 17 00:00:00 2001 From: Thomas Hughes Date: Mon, 29 Jun 2020 11:36:18 -0500 Subject: [PATCH 60/65] Fix versioning.yml --- .github/workflows/versioning.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/versioning.yml b/.github/workflows/versioning.yml index f25ee032..01039a30 100644 --- a/.github/workflows/versioning.yml +++ b/.github/workflows/versioning.yml @@ -8,8 +8,9 @@ jobs: actions-tagger: runs-on: windows-latest steps: + - uses: actions/checkout@v2 - uses: Actions-R-Us/actions-tagger@latest + with: + publish_latest_tag: true env: GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" - with: - - publish_latest_tag: true From 42124d07366541aa6f9eb50a9a21f4b3bbad16b8 Mon Sep 17 00:00:00 2001 From: Thomas Hughes Date: Mon, 29 Jun 2020 11:42:18 -0500 Subject: [PATCH 61/65] Update to v3 --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index aaaf494f..57f97859 100644 --- a/README.md +++ b/README.md @@ -111,20 +111,20 @@ jobs: # Checkout the code base # ########################## - name: Checkout Code - uses: actions/checkout@v2 + uses: actions/checkout@v3 ################################ # Run Linter against code base # ################################ - name: Lint Code Base - uses: docker://github/super-linter:v2.2.0 + uses: docker://github/super-linter:v3 env: VALIDATE_ALL_CODEBASE: false VALIDATE_ANSIBLE: false ... ``` -**NOTE:** Using the line:`uses: docker://github/super-linter:v2.2.0` will pull the image down from **DockerHub** and run the **GitHub Super-Linter**. Using the line: `uses: github/super-linter@v2.2.0` will build and compile the **GitHub Super-Linter** at build time. This can be far more costly in time... +**NOTE:** Using the line:`uses: docker://github/super-linter:v3` will pull the image down from **DockerHub** and run the **GitHub Super-Linter**. Using the line: `uses: github/super-linter@v3` will build and compile the **GitHub Super-Linter** at build time. This can be far more costly in time... ## Environment variables The super-linter allows you to pass the following `ENV` variables to be able to trigger different functionality. From cbf576073b5df7ff886f3051a942460121b0de8a Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 11:48:07 -0500 Subject: [PATCH 62/65] fixed typo --- .automation/upload-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.automation/upload-docker.sh b/.automation/upload-docker.sh index 12c15635..fe1414d3 100755 --- a/.automation/upload-docker.sh +++ b/.automation/upload-docker.sh @@ -149,7 +149,7 @@ ValidateInput() # Need to see if GPR registry and update name # ############################################### if [[ "$REGISTRY" == "GPR" ]]; then - NAME="docker.pkg.github/$IMAGE_REPO" + NAME="docker.pkg.github.com/$IMAGE_REPO" IMAGE_REPO="$NAME" echo "Updated [IMAGE_REPO] to:[$IMAGE_REPO] for GPR" fi From 4671142e0651b5f33747d6bde65cba6f3cb8cbb5 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 12:39:10 -0500 Subject: [PATCH 63/65] adding logic to deploy major tag versions as well Co-authored-by: Thomas Hughes Co-authored-by: John Wiebalk --- .automation/upload-docker.sh | 81 +++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/.automation/upload-docker.sh b/.automation/upload-docker.sh index fe1414d3..eefb5c50 100755 --- a/.automation/upload-docker.sh +++ b/.automation/upload-docker.sh @@ -26,6 +26,8 @@ REGISTRY="${REGISTRY}" # What registry to upload | or &1 + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + # ERROR + echo "ERROR! failed to [tag] Dockerfile!" + exit 1 + else + # SUCCESS + echo "Successfully tagged image!" + fi fi } ################################################################################ @@ -328,7 +378,7 @@ UploadImage() exit 1 else # SUCCESS - echo "Successfully Uploaded Docker image to $REGISTRY!" + echo "Successfully Uploaded Docker image:[$IMAGE_VERSION] to $REGISTRY!" fi ######################### @@ -371,6 +421,33 @@ UploadImage() echo "Size:[$SIZE]" echo "----------------------------------------------" fi + + ############################################################### + # Check if we need to upload the major tagged version as well # + ############################################################### + if [ $UPDATE_MAJOR_TAG -eq 1 ]; then + ############################################ + # Upload the docker image that was created # + ############################################ + docker push "$IMAGE_REPO:$MAJOR_TAG" 2>&1 + + ####################### + # Load the error code # + ####################### + ERROR_CODE=$? + + ############################## + # Check the shell for errors # + ############################## + if [ $ERROR_CODE -ne 0 ]; then + # ERROR + echo "ERROR! failed to [upload] MAJOR_TAG:[$MAJOR_TAG] Dockerfile!" + exit 1 + else + # SUCCESS + echo "Successfully Uploaded TAGOR_TAG:[$MAJOR_TAG] Docker image to $REGISTRY!" + fi + fi } ################################################################################ #### Function Footer ########################################################### From efb491d6cf17d956aed94dfa66e726e8e82305f2 Mon Sep 17 00:00:00 2001 From: Lucas Gravley <29484535+admiralAwkbar@users.noreply.github.com> Date: Mon, 29 Jun 2020 12:52:36 -0500 Subject: [PATCH 64/65] fixed protobuf var Co-authored-by: Thomas Hughes Co-authored-by: John Wiebalk --- lib/linter.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/linter.sh b/lib/linter.sh index dea50fab..1981e71a 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -85,7 +85,7 @@ LINTER_ARRAY=("jsonlint" "yamllint" "xmllint" "markdownlint" "shellcheck" LANGUAGE_ARRAY=('YML' 'JSON' 'XML' 'MARKDOWN' 'BASH' 'PERL' 'PHP' 'RUBY' 'PYTHON' 'COFFEESCRIPT' 'ANSIBLE' 'JAVASCRIPT_STANDARD' 'JAVASCRIPT_ES' 'TYPESCRIPT_STANDARD' 'TYPESCRIPT_ES' 'DOCKER' 'GO' 'TERRAFORM' - 'CSS' 'ENV' 'POWERSHELL' 'KOTLIN' 'PROTO' 'CLOJURE' 'OPENAPI') + 'CSS' 'ENV' 'POWERSHELL' 'KOTLIN' 'PROTOBUF' 'CLOJURE' 'OPENAPI') ################### # GitHub ENV Vars # @@ -607,11 +607,6 @@ Footer() ########################### ERROR_COUNTER="ERRORS_FOUND_$LANGUAGE" - ################################# - # Fix Integer Expression errors # - ################################# - ERROR_COUNTER="${ERROR_COUNTER/.*}" - ################## # Print if not 0 # ################## From 03b6b69c40a665df5c6085fc7124244f9c1bcdb6 Mon Sep 17 00:00:00 2001 From: Mark Bond <34947848+bondms@users.noreply.github.com> Date: Mon, 29 Jun 2020 19:13:22 +0100 Subject: [PATCH 65/65] Fix checkout action version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 57f97859..330cfa7a 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ jobs: # Checkout the code base # ########################## - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@v2 ################################ # Run Linter against code base #