From 0e1a1e50d69c725cec030c34693f26dd7d4f0f2b Mon Sep 17 00:00:00 2001 From: Masaya Suzuki <15100604+massongit@users.noreply.github.com> Date: Mon, 27 Sep 2021 23:32:18 +0900 Subject: [PATCH] Add Gitleaks for secret scanning (#1951) * Add actionlint * adding gitleaks * rm secretlint * Update .gitleaks.toml Co-authored-by: Admiral Awkbar --- .automation/test/gitleaks/README.md | 20 ++ .automation/test/gitleaks/gitleaks_bad_01.txt | 1 + .../test/gitleaks/gitleaks_good_01.txt | 1 + Dockerfile | 8 +- Dockerfile-slim | 7 +- README.md | 3 + TEMPLATES/.gitleaks.toml | 175 ++++++++++++++++++ lib/functions/buildFileList.sh | 2 + lib/functions/linterVersions.sh | 2 +- lib/linter.sh | 8 +- 10 files changed, 222 insertions(+), 5 deletions(-) create mode 100644 .automation/test/gitleaks/README.md create mode 100644 .automation/test/gitleaks/gitleaks_bad_01.txt create mode 100644 .automation/test/gitleaks/gitleaks_good_01.txt create mode 100644 TEMPLATES/.gitleaks.toml diff --git a/.automation/test/gitleaks/README.md b/.automation/test/gitleaks/README.md new file mode 100644 index 00000000..29721d1f --- /dev/null +++ b/.automation/test/gitleaks/README.md @@ -0,0 +1,20 @@ +# GitLeaks Test Cases + +This folder holds the test cases for **GitLeaks**. + +## Additional Docs + +No Additional information is needed for this test case. + +## Good Test Cases + +The test cases denoted: `LANGUAGE_good_FILE.EXTENSION` are all valid, and should pass successfully when linted. + +- **Note:** They are linted utilizing the default linter rules. + +## Bad Test Cases + +The test cases denoted: `LANGUAGE_bad_FILE.EXTENSION` are **NOT** valid, and should trigger errors when linted. + +- **Note:** They are linted utilizing the default linter rules. +{"mode":"full","isActive":false} diff --git a/.automation/test/gitleaks/gitleaks_bad_01.txt b/.automation/test/gitleaks/gitleaks_bad_01.txt new file mode 100644 index 00000000..f75b79cd --- /dev/null +++ b/.automation/test/gitleaks/gitleaks_bad_01.txt @@ -0,0 +1 @@ +AWS_SECRET_ACCESS_KEY = wJalrXUtnFEMI/K7MDENG/bPxRfiCYSECRETSKEY diff --git a/.automation/test/gitleaks/gitleaks_good_01.txt b/.automation/test/gitleaks/gitleaks_good_01.txt new file mode 100644 index 00000000..b68f854e --- /dev/null +++ b/.automation/test/gitleaks/gitleaks_good_01.txt @@ -0,0 +1 @@ +AWS_SECRET_ACCESS_KEY = diff --git a/Dockerfile b/Dockerfile index decd4338..4f235f99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -307,7 +307,13 @@ RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/community/" >> /etc/apk/repo && find /node_modules/ -type f -name 'LICENSE' -exec rm {} + \ && find /node_modules/ -type f -name '*.md' -exec rm {} + \ && find /node_modules/ -type f -name '*.txt' -exec rm {} + \ - && find /usr/ -type f -name '*.md' -exec rm {} + + && find /usr/ -type f -name '*.md' -exec rm {} + \ +#################### +# Install GitLeaks # +#################### + && GO111MODULE=on go get github.com/zricethezav/gitleaks/v7 \ + && mv /root/go/bin/gitleaks /usr/bin + ################################################################################ # Build the clang-format binary ################################################ diff --git a/Dockerfile-slim b/Dockerfile-slim index e926dcac..4c0a997c 100644 --- a/Dockerfile-slim +++ b/Dockerfile-slim @@ -245,7 +245,12 @@ RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/community/" >> /etc/apk/repo && find /node_modules/ -type f -name 'LICENSE' -exec rm {} + \ && find /node_modules/ -type f -name '*.md' -exec rm {} + \ && find /node_modules/ -type f -name '*.txt' -exec rm {} + \ - && find /usr/ -type f -name '*.md' -exec rm {} + + && find /usr/ -type f -name '*.md' -exec rm {} + \ +#################### +# Install GitLeaks # +#################### + && GO111MODULE=on go get github.com/zricethezav/gitleaks/v7 \ + && mv /root/go/bin/gitleaks /usr/bin ################################################################################ # Build the clang-format binary ################################################ diff --git a/README.md b/README.md index 8d5d1dd1..56dc88e9 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,7 @@ Developers on **GitHub** can call the **GitHub Action** to lint their code base | **Raku** | [Raku](https://raku.org) | | **Ruby** | [RuboCop](https://github.com/rubocop-hq/rubocop) | | **Rust** | [Rustfmt](https://github.com/rust-lang/rustfmt) / [Clippy](https://github.com/rust-lang/rust-clippy) | +| **Secrets** | [GitLeaks](https://github.com/zricethezav/gitleaks) | | **Shell** | [Shellcheck](https://github.com/koalaman/shellcheck) / [executable bit check] / [shfmt](https://github.com/mvdan/sh) | | **Snakemake** | [snakefmt](https://github.com/snakemake/snakefmt/) / [snakemake --lint](https://snakemake.readthedocs.io/en/stable/snakefiles/writing_snakefiles.html#best-practices) | | **SQL** | [sql-lint](https://github.com/joereynolds/sql-lint) / [sqlfluff](https://github.com/sqlfluff/sqlfluff) | @@ -285,6 +286,7 @@ But if you wish to select or exclude specific linters, we give you full control | **GITHUB_ACTIONS_CONFIG_FILE** | `actionlint.yml` | Filename for [Actionlint configuration](https://github.com/rhysd/actionlint/blob/main/docs/config.md) (ex: `actionlint.yml`) | | **GITHUB_DOMAIN** | `github.com` | Specify a custom Github domain in case Github Enterprise is used: e.g. `github.myenterprise.com` | | **GITHUB_CUSTOM_API_URL** | `api.github.com` | Specify a custom Github API URL in case Github Enterprise is used: e.g. `https://github.myenterprise.com/api/v3/` | +| **GITLEAKS_CONFIG_FILE** | `.gitleaks.toml` | Filename for [GitLeaks configuration](https://github.com/zricethezav/gitleaks#configuration) (ex: `.geatleaks.toml`) | | **IGNORE_GENERATED_FILES** | `false` | If set to `true`, super-linter will ignore all the files with `@generated` marker but without `@not-generated` marker. | | **IGNORE_GITIGNORED_FILES** | `false` | If set to `true`, super-linter will ignore all the files that are ignored by Git. | | **JAVA_FILE_NAME** | `sun-checks.xml` | Filename for [Checkstyle configuration](https://checkstyle.sourceforge.io/config.html) (ex: `checkstyle.xml`) | @@ -332,6 +334,7 @@ But if you wish to select or exclude specific linters, we give you full control | **VALIDATE_EDITORCONFIG** | `true` | Flag to enable or disable the linting process with the editorconfig. | | **VALIDATE_ENV** | `true` | Flag to enable or disable the linting process of the ENV language. | | **VALIDATE_GITHUB_ACTIONS** | `true` | Flag to enable or disable the linting process of the GitHub Actions. | +| **VALIDATE_GITLEAKS** | `true` | Flag to enable or disable the linting process of the secrets. | | **VALIDATE_GHERKIN** | `true` | Flag to enable or disable the linting process of the Gherkin language. | | **VALIDATE_GO** | `true` | Flag to enable or disable the linting process of the Golang language. | | **VALIDATE_GROOVY** | `true` | Flag to enable or disable the linting process of the language. | diff --git a/TEMPLATES/.gitleaks.toml b/TEMPLATES/.gitleaks.toml new file mode 100644 index 00000000..b46fc601 --- /dev/null +++ b/TEMPLATES/.gitleaks.toml @@ -0,0 +1,175 @@ + +title = "gitleaks config" + +[[rules]] + description = "AWS Access Key" + regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}''' + tags = ["key", "AWS"] + +[[rules]] + description = "AWS Secret Key" + regex = '''(?i)aws(.{0,20})?(?-i)['\"][0-9a-zA-Z\/+]{40}['\"]''' + tags = ["key", "AWS"] + +[[rules]] + description = "AWS MWS key" + regex = '''amzn\.mws\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}''' + tags = ["key", "AWS", "MWS"] + +[[rules]] + description = "Facebook Secret Key" + regex = '''(?i)(facebook|fb)(.{0,20})?(?-i)['\"][0-9a-f]{32}['\"]''' + tags = ["key", "Facebook"] + +[[rules]] + description = "Facebook Client ID" + regex = '''(?i)(facebook|fb)(.{0,20})?['\"][0-9]{13,17}['\"]''' + tags = ["key", "Facebook"] + +[[rules]] + description = "Twitter Secret Key" + regex = '''(?i)twitter(.{0,20})?['\"][0-9a-z]{35,44}['\"]''' + tags = ["key", "Twitter"] + +[[rules]] + description = "Twitter Client ID" + regex = '''(?i)twitter(.{0,20})?['\"][0-9a-z]{18,25}['\"]''' + tags = ["client", "Twitter"] + +[[rules]] + description = "Github Personal Access Token" + regex = '''ghp_[0-9a-zA-Z]{36}''' + tags = ["key", "Github"] +[[rules]] + description = "Github OAuth Access Token" + regex = '''gho_[0-9a-zA-Z]{36}''' + tags = ["key", "Github"] +[[rules]] + description = "Github App Token" + regex = '''(ghu|ghs)_[0-9a-zA-Z]{36}''' + tags = ["key", "Github"] +[[rules]] + description = "Github Refresh Token" + regex = '''ghr_[0-9a-zA-Z]{76}''' + tags = ["key", "Github"] + +[[rules]] + description = "LinkedIn Client ID" + regex = '''(?i)linkedin(.{0,20})?(?-i)[0-9a-z]{12}''' + tags = ["client", "LinkedIn"] + +[[rules]] + description = "LinkedIn Secret Key" + regex = '''(?i)linkedin(.{0,20})?[0-9a-z]{16}''' + tags = ["secret", "LinkedIn"] + +[[rules]] + description = "Slack" + regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?''' + tags = ["key", "Slack"] + +[[rules]] + description = "Asymmetric Private Key" + regex = '''-----BEGIN ((EC|PGP|DSA|RSA|OPENSSH) )?PRIVATE KEY( BLOCK)?-----''' + tags = ["key", "AsymmetricPrivateKey"] + +[[rules]] + description = "Google API key" + regex = '''AIza[0-9A-Za-z\\-_]{35}''' + tags = ["key", "Google"] + +[[rules]] + description = "Google (GCP) Service Account" + regex = '''"type": "service_account"''' + tags = ["key", "Google"] + +[[rules]] + description = "Heroku API key" + regex = '''(?i)heroku(.{0,20})?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}''' + tags = ["key", "Heroku"] + +[[rules]] + description = "MailChimp API key" + regex = '''(?i)(mailchimp|mc)(.{0,20})?[0-9a-f]{32}-us[0-9]{1,2}''' + tags = ["key", "Mailchimp"] + +[[rules]] + description = "Mailgun API key" + regex = '''((?i)(mailgun|mg)(.{0,20})?)?key-[0-9a-z]{32}''' + tags = ["key", "Mailgun"] + +[[rules]] + description = "PayPal Braintree access token" + regex = '''access_token\$production\$[0-9a-z]{16}\$[0-9a-f]{32}''' + tags = ["key", "Paypal"] + +[[rules]] + description = "Picatic API key" + regex = '''sk_live_[0-9a-z]{32}''' + tags = ["key", "Picatic"] + +[[rules]] + description = "SendGrid API Key" + regex = '''SG\.[\w_]{16,32}\.[\w_]{16,64}''' + tags = ["key", "SendGrid"] + +[[rules]] + description = "Slack Webhook" + regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8,12}/[a-zA-Z0-9_]{24}''' + tags = ["key", "slack"] + +[[rules]] + description = "Stripe API key" + regex = '''(?i)stripe(.{0,20})?[sr]k_live_[0-9a-zA-Z]{24}''' + tags = ["key", "Stripe"] + +[[rules]] + description = "Square access token" + regex = '''sq0atp-[0-9A-Za-z\-_]{22}''' + tags = ["key", "square"] + +[[rules]] + description = "Square OAuth secret" + regex = '''sq0csp-[0-9A-Za-z\\-_]{43}''' + tags = ["key", "square"] + +[[rules]] + description = "Twilio API key" + regex = '''(?i)twilio(.{0,20})?SK[0-9a-f]{32}''' + tags = ["key", "twilio"] + +[[rules]] + description = "Dynatrace ttoken" + regex = '''dt0[a-zA-Z]{1}[0-9]{2}\.[A-Z0-9]{24}\.[A-Z0-9]{64}''' + tags = ["key", "Dynatrace"] + +[[rules]] + description = "Shopify shared secret" + regex = '''shpss_[a-fA-F0-9]{32}''' + tags = ["key", "Shopify"] + +[[rules]] + description = "Shopify access token" + regex = '''shpat_[a-fA-F0-9]{32}''' + tags = ["key", "Shopify"] + +[[rules]] + description = "Shopify custom app access token" + regex = '''shpca_[a-fA-F0-9]{32}''' + tags = ["key", "Shopify"] + +[[rules]] + description = "Shopify private app access token" + regex = '''shppa_[a-fA-F0-9]{32}''' + tags = ["key", "Shopify"] + +[[rules]] + description = "PyPI upload token" + regex = '''pypi-AgEIcHlwaS5vcmc[A-Za-z0-9-_]{50,1000}''' + tags = ["key", "pypi"] + +[allowlist] + description = "Allowlisted files" + files = ['''^\.?gitleaks.toml$''', + '''(.*?)(png|jpg|gif|doc|docx|pdf|bin|xls|pyc|zip)$''', + '''(go.mod|go.sum)$'''] diff --git a/lib/functions/buildFileList.sh b/lib/functions/buildFileList.sh index a2cc6f7b..cb5ecf21 100755 --- a/lib/functions/buildFileList.sh +++ b/lib/functions/buildFileList.sh @@ -316,6 +316,8 @@ function BuildFileList() { FILE_ARRAY_EDITORCONFIG+=("${FILE}") # jscpd also runs an all files FILE_ARRAY_JSCPD+=("${FILE}") + # GitLeaks also runs an all files + FILE_ARRAY_GITLEAKS+=("${FILE}") ####################### # Get the shell files # diff --git a/lib/functions/linterVersions.sh b/lib/functions/linterVersions.sh index b9328749..b5668546 100755 --- a/lib/functions/linterVersions.sh +++ b/lib/functions/linterVersions.sh @@ -68,7 +68,7 @@ BuildLinterVersions() { if [[ ${LINTER} == "arm-ttk" ]]; then # Need specific command for ARM GET_VERSION_CMD="$(grep -iE 'version' "/usr/bin/arm-ttk" | xargs 2>&1)" - elif [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]]; then + elif [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]] || [[ ${LINTER} == "gitleaks" ]]; then # Need specific command for Protolint and editorconfig-checker GET_VERSION_CMD="$(echo "--version not supported")" elif [[ ${LINTER} == "lintr" ]]; then diff --git a/lib/linter.sh b/lib/linter.sh index 59c24ff3..87c05dcc 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -94,6 +94,8 @@ EDITORCONFIG_FILE_NAME="${EDITORCONFIG_FILE_NAME:-.ecrc}" # shellcheck disable=SC2034 # Variable is referenced indirectly GITHUB_ACTIONS_FILE_NAME="${GITHUB_ACTIONS_CONFIG_FILE:-actionlint.yml}" # shellcheck disable=SC2034 # Variable is referenced indirectly +GITLEAKS_FILE_NAME="${GITLEAKS_CONFIG_FILE:.gitleaks.toml}" +# shellcheck disable=SC2034 # Variable is referenced indirectly GHERKIN_FILE_NAME=".gherkin-lintrc" # shellcheck disable=SC2034 # Variable is referenced indirectly GO_FILE_NAME=".golangci.yml" @@ -200,12 +202,12 @@ fi LANGUAGE_ARRAY=('ANSIBLE' 'ARM' 'BASH' 'BASH_EXEC' 'CLANG_FORMAT' 'CLOUDFORMATION' 'CLOJURE' 'COFFEESCRIPT' 'CPP' 'CSHARP' 'CSS' 'DART' 'DOCKERFILE' 'DOCKERFILE_HADOLINT' 'EDITORCONFIG' 'ENV' 'GITHUB_ACTIONS' - 'GHERKIN' 'GO' 'GROOVY' 'HTML' 'JAVA' 'JAVASCRIPT_ES' + 'GITLEAKS' 'GHERKIN' 'GO' 'GROOVY' 'HTML' 'JAVA' 'JAVASCRIPT_ES' "${JAVASCRIPT_STYLE_NAME}" 'JSCPD' 'JSON' 'JSONC' 'JSX' 'KUBERNETES_KUBEVAL' 'KOTLIN' 'LATEX' 'LUA' 'MARKDOWN' 'OPENAPI' 'PERL' 'PHP_BUILTIN' 'PHP_PHPCS' 'PHP_PHPSTAN' 'PHP_PSALM' 'POWERSHELL' 'PROTOBUF' 'PYTHON_BLACK' 'PYTHON_PYLINT' 'PYTHON_FLAKE8' 'PYTHON_ISORT' 'PYTHON_MYPY' 'R' 'RAKU' 'RUBY' - 'RUST_2015' 'RUST_2018' 'RUST_CLIPPY' 'SHELL_SHFMT' 'SNAKEMAKE_LINT' + 'RUST_2015' 'RUST_2018' 'RUST_CLIPPY' 'SECRET' 'SHELL_SHFMT' 'SNAKEMAKE_LINT' 'SNAKEMAKE_SNAKEFMT' 'STATES' 'SQL' 'SQLFLUFF' 'TEKTON' 'TERRAFORM_TFLINT' 'TERRAFORM_TERRASCAN' 'TERRAGRUNT' 'TSX' 'TYPESCRIPT_ES' 'TYPESCRIPT_STANDARD' 'XML' 'YAML') @@ -231,6 +233,7 @@ LINTER_NAMES_ARRAY['DOCKERFILE_HADOLINT']="hadolint" LINTER_NAMES_ARRAY['EDITORCONFIG']="editorconfig-checker" LINTER_NAMES_ARRAY['ENV']="dotenv-linter" LINTER_NAMES_ARRAY['GITHUB_ACTIONS']="actionlint" +LINTER_NAMES_ARRAY['GITLEAKS']="gitleaks" LINTER_NAMES_ARRAY['GHERKIN']="gherkin-lint" LINTER_NAMES_ARRAY['GO']="golangci-lint" LINTER_NAMES_ARRAY['GROOVY']="npm-groovy-lint" @@ -831,6 +834,7 @@ LINTER_COMMANDS_ARRAY['DOCKERFILE_HADOLINT']="hadolint -c ${DOCKERFILE_HADOLINT_ LINTER_COMMANDS_ARRAY['EDITORCONFIG']="editorconfig-checker -config ${EDITORCONFIG_LINTER_RULES}" LINTER_COMMANDS_ARRAY['ENV']="dotenv-linter" LINTER_COMMANDS_ARRAY['GITHUB_ACTIONS']="actionlint -config-file ${GITHUB_ACTIONS_LINTER_RULES}" +LINTER_COMMANDS_ARRAY['GITLEAKS']="gitleaks -q -c ${GITLEAKS_LINTER_RULES} -p" LINTER_COMMANDS_ARRAY['GHERKIN']="gherkin-lint -c ${GHERKIN_LINTER_RULES}" LINTER_COMMANDS_ARRAY['GO']="golangci-lint run -c ${GO_LINTER_RULES}" LINTER_COMMANDS_ARRAY['GROOVY']="npm-groovy-lint -c ${GROOVY_LINTER_RULES} --failon warning"