From 6fd6830fb4a277ab1eefd57b10e2d05fd978d929 Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Fri, 9 Feb 2024 19:45:44 +0100 Subject: [PATCH] fix: initialize GitHub domain variable (#5216) - Initialize GITHUB_DOMAIN as documented in the README - Configure other URLs according to GITHUB_DOMAIN - Automatically fetch SSH key fingerprints for GITHUB_DOMAINS --- .github/workflows/cd.yml | 2 ++ .github/workflows/ci.yml | 2 ++ Makefile | 17 ++++++++++-- README.md | 2 +- lib/functions/setupSSH.sh | 33 +++++++++++++++++----- lib/linter.sh | 37 ++++++++++++++----------- test/lib/setupSSHTest.sh | 58 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 125 insertions(+), 26 deletions(-) create mode 100755 test/lib/setupSSHTest.sh diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 0ccaa213..bbf2472f 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -94,6 +94,8 @@ jobs: - name: Run Test Suite run: make test + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Login to GHCR uses: docker/login-action@v3.0.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b5fa3a7..115c8a4b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -130,6 +130,8 @@ jobs: - name: Run Test Suite run: make test + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} preview-release-notes: if: github.event_name == 'pull_request' diff --git a/Makefile b/Makefile index 0d3ddca9..6690e138 100644 --- a/Makefile +++ b/Makefile @@ -73,6 +73,10 @@ endif GITHUB_TOKEN_PATH := "$(CURDIR)/.github-personal-access-token" +ifeq ($(GITHUB_TOKEN),) +GITHUB_TOKEN="$(shell cat "${GITHUB_TOKEN_PATH}")" +endif + DEV_CONTAINER_URL := "super-linter/dev-container:latest" @@ -224,7 +228,7 @@ lint-subset-files-enable-expensive-io-checks: ## Lint a small subset of files in $(SUPER_LINTER_TEST_CONTAINER_URL) .phony: test-lib -test-lib: test-build-file-list test-github-event test-validation ## Test super-linter +test-lib: test-build-file-list test-github-event test-setup-ssh test-validation ## Test super-linter .phony: test-build-file-list test-build-file-list: ## Test buildFileList @@ -242,6 +246,15 @@ test-github-event: ## Test githubEvent --entrypoint /tmp/lint/test/lib/githubEventTest.sh \ $(SUPER_LINTER_TEST_CONTAINER_URL) +.phony: test-setup-ssh +test-setup-ssh: ## Test setupSSH + @docker run \ + -e GITHUB_TOKEN=${GITHUB_TOKEN} \ + -v "$(CURDIR):/tmp/lint" \ + -w /tmp/lint \ + --entrypoint /tmp/lint/test/lib/setupSSHTest.sh \ + $(SUPER_LINTER_TEST_CONTAINER_URL) + .phony: test-validation test-validation: ## Test validation docker run \ @@ -329,5 +342,5 @@ release-please-dry-run: build-dev-container-image check-github-token ## Run rele --manifest-file .github/release-please/.release-please-manifest.json \ --repo-url super-linter/super-linter \ --target-branch ${RELEASE_PLEASE_TARGET_BRANCH} \ - --token "$(shell cat "${GITHUB_TOKEN_PATH}")" \ + --token "${GITHUB_TOKEN}" \ --trace diff --git a/README.md b/README.md index 86744f17..ed1040fd 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ You can configure super-linter using the following environment variables: | **FILTER_REGEX_INCLUDE** | `all` | Regular expression defining which files will be processed by linters (ex: `.*src/.*`) | | **GITHUB_ACTIONS_CONFIG_FILE** | `actionlint.yml` | Filename for [Actionlint configuration](https://github.com/rhysd/actionlint/blob/main/docs/config.md) (ex: `actionlint.yml`) | | **GITHUB_ACTIONS_COMMAND_ARGS** | `null` | Additional arguments passed to `actionlint` command. Useful to [ignore some errors](https://github.com/rhysd/actionlint/blob/main/docs/usage.md#ignore-some-errors) | -| **GITHUB_CUSTOM_API_URL** | `https://api.github.com` | Specify a custom GitHub API URL in case GitHub Enterprise is used: e.g. `https://github.myenterprise.com/api/v3` | +| **GITHUB_CUSTOM_API_URL** | `https://api.${GITHUB_DOMAIN}` | Specify a custom GitHub API URL in case GitHub Enterprise is used: e.g. `https://github.myenterprise.com/api/v3` | | **GITHUB_DOMAIN** | `github.com` | Specify a custom GitHub domain in case GitHub Enterprise is used: e.g. `github.myenterprise.com` | | **GITLEAKS_CONFIG_FILE** | `.gitleaks.toml` | Filename for [GitLeaks configuration](https://github.com/zricethezav/gitleaks#configuration) (ex: `.gitleaks.toml`) | | **IGNORE_GENERATED_FILES** | `false` | If set to `true`, super-linter will ignore all the files with `@generated` marker but without `@not-generated` marker. | diff --git a/lib/functions/setupSSH.sh b/lib/functions/setupSSH.sh index 81e9609d..95d256c6 100755 --- a/lib/functions/setupSSH.sh +++ b/lib/functions/setupSSH.sh @@ -11,23 +11,42 @@ function SetupSshAgent() { fi } +function GetGitHubSshRsaKeyFingerprint() { + local GET_SSH_RSA_KEY_FINGERPRINT_CMD + if ! GET_SSH_RSA_KEY_FINGERPRINT_CMD=$( + curl -f -s --show-error -X GET \ + --url "${GITHUB_META_URL}" \ + -H 'Accept: application/vnd.github.v3+json' \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" 2>&1 + ); then + fatal "Failed to get GitHub RSA key fingerprint from ${GITHUB_META_URL}: ${GET_SSH_RSA_KEY_FINGERPRINT_CMD}" + fi + + local SSH_RSA_KEY_FINGERPRINT + SSH_RSA_KEY_FINGERPRINT="SHA256:$(jq -r '.ssh_key_fingerprints.SHA256_RSA' <<<"${GET_SSH_RSA_KEY_FINGERPRINT_CMD}")" + echo "${SSH_RSA_KEY_FINGERPRINT}" +} +export -f GetGitHubSshRsaKeyFingerprint + function SetupGithubComSshKeys() { if [[ -n "${SSH_KEY}" || "${SSH_SETUP_GITHUB}" == "true" ]]; then - info "Adding github.com SSH keys" + info "Adding ${GITHUB_DOMAIN} SSH keys" # Fetched out of band from # https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/githubs-ssh-key-fingerprints - GITHUB_RSA_FINGERPRINT="SHA256:uNiVztksCsDhcc0u9e8BujQXVUpKZIDTMczCvj3tD2s" - ssh-keyscan -t rsa github.com >/tmp/github.pub 2>/dev/null + GITHUB_RSA_FINGERPRINT="$(GetGitHubSshRsaKeyFingerprint)" + debug "${GITHUB_DOMAIN} key RSA key fingerprint: ${GITHUB_RSA_FINGERPRINT}" + ssh-keyscan -t rsa "${GITHUB_DOMAIN}" >/tmp/github.pub 2>/dev/null if [[ "${SSH_INSECURE_NO_VERIFY_GITHUB_KEY}" == "true" ]]; then - warn "Skipping github.com key verification and adding without checking fingerprint" + warn "Skipping ${GITHUB_DOMAIN} key verification and adding without checking fingerprint" mkdir -p ~/.ssh cat /tmp/github.pub >>~/.ssh/known_hosts - elif [[ "$(ssh-keygen -lf /tmp/github.pub)" == "2048 ${GITHUB_RSA_FINGERPRINT} github.com (RSA)" ]]; then - info "Successfully verified github.com key" + elif [[ "$(ssh-keygen -lf /tmp/github.pub)" == "3072 ${GITHUB_RSA_FINGERPRINT} ${GITHUB_DOMAIN} (RSA)" ]]; then + info "Successfully verified ${GITHUB_DOMAIN} key" mkdir -p ~/.ssh cat /tmp/github.pub >>~/.ssh/known_hosts else - error "Could not verify github.com key. SSH requests to github.com will likely fail." + error "Could not verify ${GITHUB_DOMAIN} key. SSH requests to ${GITHUB_DOMAIN} will likely fail." fi fi } diff --git a/lib/linter.sh b/lib/linter.sh index 31772f34..2b5d1f43 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -140,13 +140,14 @@ DEFAULT_SUPER_LINTER_WORKSPACE="/tmp/lint" # Fa DEFAULT_WORKSPACE="${DEFAULT_WORKSPACE:-${DEFAULT_SUPER_LINTER_WORKSPACE}}" # Default workspace if running locally FILTER_REGEX_INCLUDE="${FILTER_REGEX_INCLUDE:-""}" FILTER_REGEX_EXCLUDE="${FILTER_REGEX_EXCLUDE:-""}" +GITHUB_DOMAIN="${GITHUB_DOMAIN:-"github.com"}" +GITHUB_DOMAIN="${GITHUB_DOMAIN%/}" # Remove trailing slash if present # GitHub API root url -GITHUB_API_URL="${GITHUB_CUSTOM_API_URL:-"https://api.github.com"}" -# Remove trailing slash if present -GITHUB_API_URL="${GITHUB_API_URL%/}" -GITHUB_SERVER_URL="${GITHUB_DOMAIN:-"https://github.com"}" -# Extract domain name from URL -GITHUB_SERVER_URL=$(echo "$GITHUB_SERVER_URL" | cut -d '/' -f 3) +GITHUB_API_URL="${GITHUB_CUSTOM_API_URL:-"https://api.${GITHUB_DOMAIN}"}" +GITHUB_API_URL="${GITHUB_API_URL%/}" # Remove trailing slash if present +GITHUB_SERVER_URL="https://${GITHUB_DOMAIN}" +# shellcheck disable=SC2034 # Variable is referenced indirectly +GITHUB_META_URL="${GITHUB_API_URL}/meta" LINTER_RULES_PATH="${LINTER_RULES_PATH:-.github/linters}" # Linter rules directory # shellcheck disable=SC2034 # Variable is referenced in other scripts RAW_FILE_ARRAY=() # Array of all files that were changed @@ -532,6 +533,12 @@ GetGitHubVars() { else info "Successfully found GITHUB_RUN_ID ${GITHUB_RUN_ID}" fi + + GITHUB_STATUS_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/statuses/${GITHUB_SHA}" + debug "GitHub Status URL: ${GITHUB_STATUS_URL}" + + GITHUB_STATUS_TARGET_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + debug "GitHub Status target URL: ${GITHUB_STATUS_TARGET_URL}" else debug "Skip GITHUB_TOKEN, GITHUB_REPOSITORY, and GITHUB_RUN_ID validation because we don't need these variables for GitHub Actions status reports. MULTI_STATUS: ${MULTI_STATUS}" fi @@ -574,19 +581,17 @@ CallStatusAPI() { STATUS="success" fi - debug "Status URL: ${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/statuses/${GITHUB_SHA}" - ############################################## # Call the status API to create status check # ############################################## if ! SEND_STATUS_CMD=$( curl -f -s --show-error -X POST \ - --url "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/statuses/${GITHUB_SHA}" \ + --url "${GITHUB_STATUS_URL}" \ -H 'accept: application/vnd.github.v3+json' \ -H "authorization: Bearer ${GITHUB_TOKEN}" \ -H 'content-type: application/json' \ -d "{ \"state\": \"${STATUS}\", - \"target_url\": \"https://${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}\", + \"target_url\": \"${GITHUB_STATUS_TARGET_URL}\", \"description\": \"${MESSAGE}\", \"context\": \"--> Linted: ${LANGUAGE}\" }" 2>&1 ); then @@ -718,12 +723,6 @@ trap 'cleanup' 0 1 2 3 6 14 15 ########## Header -############################################ -# Create SSH agent and add key if provided # -############################################ -SetupSshAgent -SetupGithubComSshKeys - ################################################ # Need to update the loops for the image style # ################################################ @@ -744,6 +743,12 @@ GetGitHubVars # all cases when initializing variables ConfigureGitSafeDirectories +############################################ +# Create SSH agent and add key if provided # +############################################ +SetupSshAgent +SetupGithubComSshKeys + ######################################################## # Initialize variables that depend on GitHub variables # ######################################################## diff --git a/test/lib/setupSSHTest.sh b/test/lib/setupSSHTest.sh new file mode 100755 index 00000000..65138302 --- /dev/null +++ b/test/lib/setupSSHTest.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +# shellcheck disable=SC2034 +LOG_TRACE="true" +# shellcheck disable=SC2034 +LOG_DEBUG="true" +# shellcheck disable=SC2034 +LOG_VERBOSE="true" +# shellcheck disable=SC2034 +LOG_NOTICE="true" +# shellcheck disable=SC2034 +LOG_WARN="true" +# shellcheck disable=SC2034 +LOG_ERROR="true" + +# shellcheck source=/dev/null +source "lib/functions/log.sh" + +# shellcheck disable=SC2034 +CREATE_LOG_FILE=false + +GITHUB_DOMAIN="github.com" +# shellcheck disable=SC2034 +GITHUB_META_URL="https://api.${GITHUB_DOMAIN}/meta" + +# shellcheck source=/dev/null +source "lib/functions/setupSSH.sh" + +function GetGitHubSshRsaKeyFingerprintTest() { + local SSH_RSA_KEY_FINGERPRINT + SSH_RSA_KEY_FINGERPRINT=$(GetGitHubSshRsaKeyFingerprint) + + debug "SSH_RSA_KEY_FINGERPRINT: ${SSH_RSA_KEY_FINGERPRINT}" + local EXPECTED_GITHUB_RSA_KEY_FINGERPRINT + EXPECTED_GITHUB_RSA_KEY_FINGERPRINT="$(ssh-keygen -lf /dev/stdin <<<"$(ssh-keyscan -t rsa github.com)" | cut -d ' ' -f2)" + debug "Expected output: ${EXPECTED_GITHUB_RSA_KEY_FINGERPRINT}" + + if [ "${SSH_RSA_KEY_FINGERPRINT}" != "${EXPECTED_GITHUB_RSA_KEY_FINGERPRINT}" ]; then + fatal "SSH_RSA_KEY_FINGERPRINT is not equal to ${EXPECTED_GITHUB_RSA_KEY_FINGERPRINT}: ${SSH_RSA_KEY_FINGERPRINT}" + fi + + FUNCTION_NAME="${FUNCNAME[0]}" + notice "${FUNCTION_NAME} PASS" +} + +function SetupGithubComSshKeysTest() { + SSH_KEY="test_ssh_key" SSH_INSECURE_NO_VERIFY_GITHUB_KEY="false" SetupGithubComSshKeys + + FUNCTION_NAME="${FUNCNAME[0]}" + notice "${FUNCTION_NAME} PASS" +} + +GetGitHubSshRsaKeyFingerprintTest +SetupGithubComSshKeysTest