From 3041bf56c941b39c61721a86cd11f3bb1338122a Mon Sep 17 00:00:00 2001 From: Tobias Date: Mon, 25 Nov 2024 19:37:21 +0100 Subject: [PATCH] feat: fallback to "raw" endpoint for manifest when rate limit is reached (#496) * feat: fallback to "raw" endpoint for manifest when rate limit is reached * add information about raw access to the README * prettier * update cross-spawn to 7.0.6 to fix vulnerability --- README.md | 21 +++++++----------- __tests__/setup-go.test.ts | 23 ++++++++++++++++++++ dist/setup/index.js | 30 +++++++++++++++++++++++++- package-lock.json | 7 +++--- src/installer.ts | 44 ++++++++++++++++++++++++++++++++++++-- 5 files changed, 106 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 44726a6..b2e6de4 100644 --- a/README.md +++ b/README.md @@ -242,18 +242,14 @@ documentation. ## Using `setup-go` on GHES -`setup-go` comes pre-installed on the appliance with GHES if Actions is enabled. When dynamically downloading Go -distributions, `setup-go` downloads distributions from [`actions/go-versions`](https://github.com/actions/go-versions) -on github.com (outside of the appliance). These calls to `actions/go-versions` are made via unauthenticated requests, -which are limited -to [60 requests per hour per IP](https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting). If -more requests are made within the time frame, then you will start to see rate-limit errors during downloading that looks -like: `##[error]API rate limit exceeded for...`. After that error the action will try to download versions directly -from https://storage.googleapis.com/golang, but it also can have rate limit so it's better to put token. +`setup-go` comes pre-installed on the appliance with GHES if Actions is enabled. +When dynamically downloading Go distributions, `setup-go` downloads distributions from [`actions/go-versions`](https://github.com/actions/go-versions) on github.com (outside of the appliance). -To get a higher rate limit, you -can [generate a personal access token on github.com](https://github.com/settings/tokens/new) and pass it as the `token` -input for the action: +These calls to `actions/go-versions` are made via unauthenticated requests, which are limited to [60 requests per hour per IP](https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting). +If more requests are made within the time frame, then the action leverages the `raw API` to retrieve the version-manifest. This approach does not impose a rate limit and hence facilitates unrestricted consumption. This is particularly beneficial for GHES runners, which often share the same IP, to avoid the quick exhaustion of the unauthenticated rate limit. +If that fails as well the action will try to download versions directly from https://storage.googleapis.com/golang. + +If that fails as well you can get a higher rate limit with [generating a personal access token on github.com](https://github.com/settings/tokens/new) and passing it as the `token` input to the action: ```yaml uses: actions/setup-go@v5 @@ -262,8 +258,7 @@ with: go-version: '1.18' ``` -If the runner is not able to access github.com, any Go versions requested during a workflow run must come from the -runner's tool cache. +If the runner is not able to access github.com, any Go versions requested during a workflow run must come from the runner's tool cache. See "[Setting up the tool cache on self-hosted runners without internet access](https://docs.github.com/en/enterprise-server@3.2/admin/github-actions/managing-access-to-actions-from-githubcom/setting-up-the-tool-cache-on-self-hosted-runners-without-internet-access)" for more information. diff --git a/__tests__/setup-go.test.ts b/__tests__/setup-go.test.ts index 70f2166..f94f9ee 100644 --- a/__tests__/setup-go.test.ts +++ b/__tests__/setup-go.test.ts @@ -7,6 +7,7 @@ import osm, {type} from 'os'; import path from 'path'; import * as main from '../src/main'; import * as im from '../src/installer'; +import * as httpm from '@actions/http-client'; import goJsonData from './data/golang-dl.json'; import matchers from '../matchers.json'; @@ -46,6 +47,7 @@ describe('setup-go', () => { let execSpy: jest.SpyInstance; let getManifestSpy: jest.SpyInstance; let getAllVersionsSpy: jest.SpyInstance; + let httpmGetJsonSpy: jest.SpyInstance; beforeAll(async () => { process.env['GITHUB_ENV'] = ''; // Stub out Environment file functionality so we can verify it writes to standard out (toolkit is backwards compatible) @@ -90,6 +92,9 @@ describe('setup-go', () => { getManifestSpy = jest.spyOn(tc, 'getManifestFromRepo'); getAllVersionsSpy = jest.spyOn(im, 'getManifest'); + // httm + httpmGetJsonSpy = jest.spyOn(httpm.HttpClient.prototype, 'getJson'); + // io whichSpy = jest.spyOn(io, 'which'); existsSpy = jest.spyOn(fs, 'existsSync'); @@ -151,6 +156,21 @@ describe('setup-go', () => { ); }); + it('should return manifest from repo', async () => { + const manifest = await im.getManifest(undefined); + expect(manifest).toEqual(goTestManifest); + }); + + it('should return manifest from raw URL if repo fetch fails', async () => { + getManifestSpy.mockRejectedValue(new Error('Fetch failed')); + httpmGetJsonSpy.mockResolvedValue({ + result: goTestManifest + }); + const manifest = await im.getManifest(undefined); + expect(httpmGetJsonSpy).toHaveBeenCalled(); + expect(manifest).toEqual(goTestManifest); + }); + it('can find 1.9 from manifest on linux', async () => { os.platform = 'linux'; os.arch = 'x64'; @@ -790,6 +810,9 @@ describe('setup-go', () => { getManifestSpy.mockImplementation(() => { throw new Error('Unable to download manifest'); }); + httpmGetJsonSpy.mockRejectedValue( + new Error('Unable to download manifest from raw URL') + ); getAllVersionsSpy.mockImplementationOnce(() => undefined); dlSpy.mockImplementation(async () => '/some/temp/path'); diff --git a/dist/setup/index.js b/dist/setup/index.js index f21396a..d769775 100644 --- a/dist/setup/index.js +++ b/dist/setup/index.js @@ -88259,6 +88259,10 @@ const sys = __importStar(__nccwpck_require__(5632)); const fs_1 = __importDefault(__nccwpck_require__(7147)); const os_1 = __importDefault(__nccwpck_require__(2037)); const utils_1 = __nccwpck_require__(1314); +const MANIFEST_REPO_OWNER = 'actions'; +const MANIFEST_REPO_NAME = 'go-versions'; +const MANIFEST_REPO_BRANCH = 'main'; +const MANIFEST_URL = `https://raw.githubusercontent.com/${MANIFEST_REPO_OWNER}/${MANIFEST_REPO_NAME}/${MANIFEST_REPO_BRANCH}/versions-manifest.json`; function getGo(versionSpec_1, checkLatest_1, auth_1) { return __awaiter(this, arguments, void 0, function* (versionSpec, checkLatest, auth, arch = os_1.default.arch()) { var _a; @@ -88433,10 +88437,34 @@ function extractGoArchive(archivePath) { exports.extractGoArchive = extractGoArchive; function getManifest(auth) { return __awaiter(this, void 0, void 0, function* () { - return tc.getManifestFromRepo('actions', 'go-versions', auth, 'main'); + try { + return yield getManifestFromRepo(auth); + } + catch (err) { + core.debug('Fetching the manifest via the API failed.'); + if (err instanceof Error) { + core.debug(err.message); + } + } + return yield getManifestFromURL(); }); } exports.getManifest = getManifest; +function getManifestFromRepo(auth) { + core.debug(`Getting manifest from ${MANIFEST_REPO_OWNER}/${MANIFEST_REPO_NAME}@${MANIFEST_REPO_BRANCH}`); + return tc.getManifestFromRepo(MANIFEST_REPO_OWNER, MANIFEST_REPO_NAME, auth, MANIFEST_REPO_BRANCH); +} +function getManifestFromURL() { + return __awaiter(this, void 0, void 0, function* () { + core.debug('Falling back to fetching the manifest using raw URL.'); + const http = new httpm.HttpClient('tool-cache'); + const response = yield http.getJson(MANIFEST_URL); + if (!response.result) { + throw new Error(`Unable to get manifest from ${MANIFEST_URL}`); + } + return response.result; + }); +} function getInfoFromManifest(versionSpec_1, stable_1, auth_1) { return __awaiter(this, arguments, void 0, function* (versionSpec, stable, auth, arch = os_1.default.arch(), manifest) { let info = null; diff --git a/package-lock.json b/package-lock.json index c51f97c..8a06fd9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2504,10 +2504,11 @@ } }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, + "license": "MIT", "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", diff --git a/src/installer.ts b/src/installer.ts index 817c334..cc0f2dc 100644 --- a/src/installer.ts +++ b/src/installer.ts @@ -8,6 +8,11 @@ import fs from 'fs'; import os from 'os'; import {StableReleaseAlias} from './utils'; +const MANIFEST_REPO_OWNER = 'actions'; +const MANIFEST_REPO_NAME = 'go-versions'; +const MANIFEST_REPO_BRANCH = 'main'; +const MANIFEST_URL = `https://raw.githubusercontent.com/${MANIFEST_REPO_OWNER}/${MANIFEST_REPO_NAME}/${MANIFEST_REPO_BRANCH}/versions-manifest.json`; + type InstallationType = 'dist' | 'manifest'; export interface IGoVersionFile { @@ -274,8 +279,43 @@ export async function extractGoArchive(archivePath: string): Promise { return extPath; } -export async function getManifest(auth: string | undefined) { - return tc.getManifestFromRepo('actions', 'go-versions', auth, 'main'); +export async function getManifest( + auth: string | undefined +): Promise { + try { + return await getManifestFromRepo(auth); + } catch (err) { + core.debug('Fetching the manifest via the API failed.'); + if (err instanceof Error) { + core.debug(err.message); + } + } + return await getManifestFromURL(); +} + +function getManifestFromRepo( + auth: string | undefined +): Promise { + core.debug( + `Getting manifest from ${MANIFEST_REPO_OWNER}/${MANIFEST_REPO_NAME}@${MANIFEST_REPO_BRANCH}` + ); + return tc.getManifestFromRepo( + MANIFEST_REPO_OWNER, + MANIFEST_REPO_NAME, + auth, + MANIFEST_REPO_BRANCH + ); +} + +async function getManifestFromURL(): Promise { + core.debug('Falling back to fetching the manifest using raw URL.'); + + const http: httpm.HttpClient = new httpm.HttpClient('tool-cache'); + const response = await http.getJson(MANIFEST_URL); + if (!response.result) { + throw new Error(`Unable to get manifest from ${MANIFEST_URL}`); + } + return response.result; } export async function getInfoFromManifest(