Refactor: use a single .json file to describe all cached artifact bundles (#121)

This is a pure refactor, moving from a separate .cache file per bundle to a single cache-metadata.json file describing all bundles. Instead of storing cache metadata in a separate .cache file per artifact bundle, all of the metadata is now stored in a single `.json` file.

This will make it easier to implement more flexible artifact-caching strategies, such as caching each wrapper zip separately.

* Always include cache protocol version in cache key
* Store all cache metadata in a single JSON file
* Rename cache-metadata file and bump protocol version
* Polish and documentation

src/cache-base.ts

@@ -3,14 +3,47 @@ import * as cache from '@actions/cache'
 import * as github from '@actions/github'
 import {isCacheDebuggingEnabled, getCacheKeyPrefix, hashStrings, handleCacheFailure} from './cache-utils'
 
+const CACHE_PROTOCOL_VERSION = 'v5-'
 const JOB_CONTEXT_PARAMETER = 'workflow-job-context'
 
+/**
+ * Represents a key used to restore a cache entry.
+ * The Github Actions cache will first try for an exact match on the key.
+ * If that fails, it will try for a prefix match on any of the restoreKeys.
+ */
+class CacheKey {
+    key: string
+    restoreKeys: string[]
+
+    constructor(key: string, restoreKeys: string[]) {
+        this.key = key
+        this.restoreKeys = restoreKeys
+    }
+}
+
+/**
+ * Generates a cache key specific to the current job execution.
+ * The key is constructed from the following inputs:
+ * - A user-defined prefix (optional)
+ * - The cache protocol version
+ * - The name of the cache
+ * - The runner operating system
+ * - The name of the Job being executed
+ * - The matrix values for the Job being executed (job context)
+ * - The SHA of the commit being executed
+ *
+ * Caches are restored by trying to match the these key prefixes in order:
+ * - The full key with SHA
+ * - A previous key for this Job + matrix
+ * - Any previous key for this Job (any matrix)
+ * - Any previous key for this cache on the current OS
+ */
 function generateCacheKey(cacheName: string): CacheKey {
-    const cacheKeyPrefix = getCacheKeyPrefix()
+    const cacheKeyBase = `${getCacheKeyPrefix()}${CACHE_PROTOCOL_VERSION}${cacheName}`
 
     // At the most general level, share caches for all executions on the same OS
     const runnerOs = process.env['RUNNER_OS'] || ''
-    const cacheKeyForOs = `${cacheKeyPrefix}${cacheName}|${runnerOs}`
+    const cacheKeyForOs = `${cacheKeyBase}|${runnerOs}`
 
     // Prefer caches that run this job
     const cacheKeyForJob = `${cacheKeyForOs}|${github.context.job}`
@@ -26,20 +59,15 @@ function generateCacheKey(cacheName: string): CacheKey {
 
 function determineJobContext(): string {
     // By default, we hash the full `matrix` data for the run, to uniquely identify this job invocation
+    // The only way we can obtain the `matrix` data is via the `workflow-job-context` parameter in action.yml.
     const workflowJobContext = core.getInput(JOB_CONTEXT_PARAMETER)
     return hashStrings([workflowJobContext])
 }
 
-class CacheKey {
+/**
-    key: string
+ * Collects information on what entries were saved and restored during the action.
-    restoreKeys: string[]
+ * This information is used to generate a summary of the cache usage.
-
+ */
-    constructor(key: string, restoreKeys: string[]) {
-        this.key = key
-        this.restoreKeys = restoreKeys
-    }
-}
-
 export class CacheListener {
     cacheEntries: CacheEntryListener[] = []
 
@@ -74,6 +102,9 @@ export class CacheListener {
     }
 }
 
+/**
+ * Collects information on the state of a single cache entry.
+ */
 export class CacheEntryListener {
     entryName: string
     requestedKey: string | undefined
@@ -127,15 +158,18 @@ export abstract class AbstractCache {
         this.cacheDebuggingEnabled = isCacheDebuggingEnabled()
     }
 
+    /**
+     * Restores the cache entry, finding the closest match to the currently running job.
+     * If the target output already exists, caching will be skipped.
+     */
     async restore(listener: CacheListener): Promise<void> {
         if (this.cacheOutputExists()) {
             core.info(`${this.cacheDescription} already exists. Not restoring from cache.`)
             return
         }
+        const entryListener = listener.entry(this.cacheDescription)
 
         const cacheKey = this.prepareCacheKey()
-        const entryReport = listener.entry(this.cacheDescription)
-        entryReport.markRequested(cacheKey.key, cacheKey.restoreKeys)
 
         this.debug(
             `Requesting ${this.cacheDescription} with
@@ -144,6 +178,7 @@ export abstract class AbstractCache {
         )
 
         const cacheResult = await this.restoreCache(this.getCachePath(), cacheKey.key, cacheKey.restoreKeys)
+        entryListener.markRequested(cacheKey.key, cacheKey.restoreKeys)
 
         if (!cacheResult) {
             core.info(`${this.cacheDescription} cache not found. Will initialize empty.`)
@@ -151,7 +186,8 @@ export abstract class AbstractCache {
         }
 
         core.saveState(this.cacheResultStateKey, cacheResult.key)
-        entryReport.markRestored(cacheResult.key, cacheResult.size)
+        entryListener.markRestored(cacheResult.key, cacheResult.size)
+
         core.info(`Restored ${this.cacheDescription} from cache key: ${cacheResult.key}`)
 
         try {
@@ -163,7 +199,6 @@ export abstract class AbstractCache {
 
     prepareCacheKey(): CacheKey {
         const cacheKey = generateCacheKey(this.cacheName)
-
         core.saveState(this.cacheKeyStateKey, cacheKey.key)
         return cacheKey
     }
@@ -183,22 +218,31 @@ export abstract class AbstractCache {
 
     protected async afterRestore(_listener: CacheListener): Promise<void> {}
 
+    /**
+     * Saves the cache entry based on the current cache key, unless:
+     * - If the cache output existed before restore, then it is not saved.
+     * - If the cache was restored with the exact key, we cannot overwrite it.
+     *
+     * If the cache entry was restored with a partial match on a restore key, then
+     * it is saved with the exact key.
+     */
     async save(listener: CacheListener): Promise<void> {
         if (!this.cacheOutputExists()) {
             core.info(`No ${this.cacheDescription} to cache.`)
             return
         }
 
-        const cacheKey = core.getState(this.cacheKeyStateKey)
+        // Retrieve the state set in the previous 'restore' step.
-        const cacheResult = core.getState(this.cacheResultStateKey)
+        const cacheKeyFromRestore = core.getState(this.cacheKeyStateKey)
+        const cacheResultFromRestore = core.getState(this.cacheResultStateKey)
 
-        if (!cacheKey) {
+        if (!cacheKeyFromRestore) {
             core.info(`${this.cacheDescription} existed prior to cache restore. Not saving.`)
             return
         }
 
-        if (cacheResult && cacheKey === cacheResult) {
+        if (cacheResultFromRestore && cacheKeyFromRestore === cacheResultFromRestore) {
-            core.info(`Cache hit occurred on the cache key ${cacheKey}, not saving cache.`)
+            core.info(`Cache hit occurred on the cache key ${cacheKeyFromRestore}, not saving cache.`)
             return
         }
 
@@ -209,9 +253,9 @@ export abstract class AbstractCache {
             return
         }
 
-        core.info(`Caching ${this.cacheDescription} with cache key: ${cacheKey}`)
+        core.info(`Caching ${this.cacheDescription} with cache key: ${cacheKeyFromRestore}`)
         const cachePath = this.getCachePath()
-        const savedEntry = await this.saveCache(cachePath, cacheKey)
+        const savedEntry = await this.saveCache(cachePath, cacheKeyFromRestore)
 
         if (savedEntry) {
             listener.entry(this.cacheDescription).markSaved(savedEntry.key, savedEntry.size)

src/cache-gradle-user-home.ts

@@ -9,11 +9,31 @@ import {AbstractCache, CacheEntryListener, CacheListener} from './cache-base'
 import {getCacheKeyPrefix, hashFileNames, tryDelete} from './cache-utils'
 
 const META_FILE_DIR = '.gradle-build-action'
+const META_FILE = 'cache-metadata.json'
 
 const INCLUDE_PATHS_PARAMETER = 'gradle-home-cache-includes'
 const EXCLUDE_PATHS_PARAMETER = 'gradle-home-cache-excludes'
 const ARTIFACT_BUNDLES_PARAMETER = 'gradle-home-cache-artifact-bundles'
 
+/**
+ * Represents the result of attempting to load or store a cache bundle entry.
+ * An undefined cacheKey indicates that the operation did not succeed.
+ * The collected results are then used to populate the `cache-metadata.json` file for later use.
+ */
+class CacheBundleResult {
+    readonly bundle: string
+    readonly cacheKey: string | undefined
+
+    constructor(bundle: string, cacheKey: string | undefined) {
+        this.bundle = bundle
+        this.cacheKey = cacheKey
+    }
+}
+
+/**
+ * Caches and restores the entire Gradle User Home directory, extracting bundles of common artifacts
+ * for more efficient storage.
+ */
 export class GradleUserHomeCache extends AbstractCache {
     private gradleUserHome: string
 
@@ -27,31 +47,35 @@ export class GradleUserHomeCache extends AbstractCache {
         initializeGradleUserHome(this.gradleUserHome)
     }
 
+    /**
+     * Restore any artifact bundles after the main Gradle User Home entry is restored.
+     */
     async afterRestore(listener: CacheListener): Promise<void> {
-        await this.reportGradleUserHomeSize('as restored from cache')
+        await this.debugReportGradleUserHomeSize('as restored from cache')
         await this.restoreArtifactBundles(listener)
-        await this.reportGradleUserHomeSize('after restoring common artifacts')
+        await this.debugReportGradleUserHomeSize('after restoring common artifacts')
     }
 
+    /**
+     * Restores any artifacts that were cached separately, based on the information in the `cache-metadata.json` file.
+     * Each artifact bundle is restored in parallel, except when debugging is enabled.
+     */
     private async restoreArtifactBundles(listener: CacheListener): Promise<void> {
-        const processes: Promise<void>[] = []
+        const bundleMetadata = this.loadBundleMetadata()
+        const bundlePatterns = this.getArtifactBundleDefinitions()
 
-        const bundleMetaFiles = await this.getBundleMetaFiles()
+        const processes: Promise<CacheBundleResult>[] = []
-        const bundlePatterns = this.getArtifactBundles()
 
-        // Iterate over all bundle meta files and try to restore
+        for (const [bundle, cacheKey] of bundleMetadata) {
-        for (const bundleMetaFile of bundleMetaFiles) {
-            const bundle = path.basename(bundleMetaFile, '.cache')
             const entryListener = listener.entry(bundle)
             const bundlePattern = bundlePatterns.get(bundle)
 
             // Handle case where the 'artifactBundlePatterns' have been changed
             if (bundlePattern === undefined) {
-                core.info(`Found bundle metafile for ${bundle} but no such bundle defined`)
+                core.info(`Found bundle metadata for ${bundle} but no such bundle defined`)
                 entryListener.markRequested('BUNDLE_NOT_CONFIGURED')
-                tryDelete(bundleMetaFile)
             } else {
-                const p = this.restoreArtifactBundle(bundle, bundlePattern, bundleMetaFile, entryListener)
+                const p = this.restoreArtifactBundle(bundle, cacheKey, bundlePattern, entryListener)
                 // Run sequentially when debugging enabled
                 if (this.cacheDebuggingEnabled) {
                     await p
@@ -60,48 +84,45 @@ export class GradleUserHomeCache extends AbstractCache {
             }
         }
 
-        await Promise.all(processes)
+        const results = await Promise.all(processes)
+
+        this.saveMetadataForCacheResults(results)
     }
 
     private async restoreArtifactBundle(
         bundle: string,
+        cacheKey: string,
         bundlePattern: string,
-        bundleMetaFile: string,
         listener: CacheEntryListener
-    ): Promise<void> {
+    ): Promise<CacheBundleResult> {
-        const cacheKey = fs.readFileSync(bundleMetaFile, 'utf-8').trim()
         listener.markRequested(cacheKey)
 
         const restoredEntry = await this.restoreCache([bundlePattern], cacheKey)
         if (restoredEntry) {
             core.info(`Restored ${bundle} with key ${cacheKey} to ${bundlePattern}`)
             listener.markRestored(restoredEntry.key, restoredEntry.size)
+            return new CacheBundleResult(bundle, cacheKey)
         } else {
             core.info(`Did not restore ${bundle} with key ${cacheKey} to ${bundlePattern}`)
-            tryDelete(bundleMetaFile)
+            return new CacheBundleResult(bundle, undefined)
         }
     }
 
-    private getBundleMetaFile(name: string): string {
+    /**
-        return path.resolve(this.gradleUserHome, META_FILE_DIR, `${name}.cache`)
+     * Save and delete any artifact bundles prior to the main Gradle User Home entry being saved.
-    }
+     */
-
-    private async getBundleMetaFiles(): Promise<string[]> {
-        const metaFiles = path.resolve(this.gradleUserHome, META_FILE_DIR, '*.cache')
-        const globber = await glob.create(metaFiles)
-        const bundleFiles = await globber.glob()
-        return bundleFiles
-    }
-
     async beforeSave(listener: CacheListener): Promise<void> {
-        await this.reportGradleUserHomeSize('before saving common artifacts')
+        await this.debugReportGradleUserHomeSize('before saving common artifacts')
         this.removeExcludedPaths()
         await this.saveArtifactBundles(listener)
-        await this.reportGradleUserHomeSize(
+        await this.debugReportGradleUserHomeSize(
             "after saving common artifacts (only 'caches' and 'notifications' will be stored)"
         )
     }
 
+    /**
+     * Delete any file paths that are excluded by the `gradle-home-cache-excludes` parameter.
+     */
     private removeExcludedPaths(): void {
         const rawPaths: string[] = core.getMultilineInput(EXCLUDE_PATHS_PARAMETER)
         const resolvedPaths = rawPaths.map(x => path.resolve(this.gradleUserHome, x))
@@ -112,12 +133,19 @@ export class GradleUserHomeCache extends AbstractCache {
         }
     }
 
+    /**
+     * Saves any artifacts that are configured to be cached separately, based on the artifact bundle definitions.
+     * These definitions are normally fixed, but can be overridden by the `gradle-home-cache-artifact-bundles` parameter.
+     * Each artifact bundle is saved in parallel, except when debugging is enabled.
+     */
     private async saveArtifactBundles(listener: CacheListener): Promise<void> {
-        const processes: Promise<void>[] = []
+        const bundleMetadata = this.loadBundleMetadata()
-        for (const [bundle, pattern] of this.getArtifactBundles()) {
-            const entryListener = listener.entry(bundle)
 
-            const p = this.saveArtifactBundle(bundle, pattern, entryListener)
+        const processes: Promise<CacheBundleResult>[] = []
+        for (const [bundle, pattern] of this.getArtifactBundleDefinitions()) {
+            const entryListener = listener.entry(bundle)
+            const previouslyRestoredKey = bundleMetadata.get(bundle)
+            const p = this.saveArtifactBundle(bundle, pattern, previouslyRestoredKey, entryListener)
             // Run sequentially when debugging enabled
             if (this.cacheDebuggingEnabled) {
                 await p
@@ -125,16 +153,17 @@ export class GradleUserHomeCache extends AbstractCache {
             processes.push(p)
         }
 
-        await Promise.all(processes)
+        const results = await Promise.all(processes)
+
+        this.saveMetadataForCacheResults(results)
     }
 
     private async saveArtifactBundle(
         bundle: string,
         artifactPath: string,
+        previouslyRestoredKey: string | undefined,
         listener: CacheEntryListener
-    ): Promise<void> {
+    ): Promise<CacheBundleResult> {
-        const bundleMetaFile = this.getBundleMetaFile(bundle)
-
         const globber = await glob.create(artifactPath, {
             implicitDescendants: false,
             followSymbolicLinks: false
@@ -144,16 +173,10 @@ export class GradleUserHomeCache extends AbstractCache {
         // Handle no matching files
         if (bundleFiles.length === 0) {
             this.debug(`No files found to cache for ${bundle}`)
-            if (fs.existsSync(bundleMetaFile)) {
+            return new CacheBundleResult(bundle, undefined)
-                tryDelete(bundleMetaFile)
-            }
-            return
         }
 
-        const previouslyRestoredKey = fs.existsSync(bundleMetaFile)
+        const cacheKey = this.createCacheKeyForArtifacts(bundle, bundleFiles)
-            ? fs.readFileSync(bundleMetaFile, 'utf-8').trim()
-            : ''
-        const cacheKey = this.createCacheKey(bundle, bundleFiles)
 
         if (previouslyRestoredKey === cacheKey) {
             this.debug(`No change to previously restored ${bundle}. Not caching.`)
@@ -161,7 +184,6 @@ export class GradleUserHomeCache extends AbstractCache {
             core.info(`Caching ${bundle} with cache key: ${cacheKey}`)
             const savedEntry = await this.saveCache([artifactPath], cacheKey)
             if (savedEntry !== undefined) {
-                this.writeBundleMetaFile(bundleMetaFile, cacheKey)
                 listener.markSaved(savedEntry.key, savedEntry.size)
             }
         }
@@ -169,9 +191,11 @@ export class GradleUserHomeCache extends AbstractCache {
         for (const file of bundleFiles) {
             tryDelete(file)
         }
+
+        return new CacheBundleResult(bundle, cacheKey)
     }
 
-    protected createCacheKey(bundle: string, files: string[]): string {
+    protected createCacheKeyForArtifacts(bundle: string, files: string[]): string {
         const cacheKeyPrefix = getCacheKeyPrefix()
         const relativeFiles = files.map(x => path.relative(this.gradleUserHome, x))
         const key = hashFileNames(relativeFiles)
@@ -181,15 +205,39 @@ export class GradleUserHomeCache extends AbstractCache {
         return `${cacheKeyPrefix}${bundle}-${key}`
     }
 
-    private writeBundleMetaFile(metaFile: string, cacheKey: string): void {
+    /**
-        this.debug(`Writing bundle metafile: ${metaFile}`)
+     * Load information about the previously restored/saved artifact bundles from the 'cache-metadata.json' file.
-
+     */
-        const dirName = path.dirname(metaFile)
+    private loadBundleMetadata(): Map<string, string> {
-        if (!fs.existsSync(dirName)) {
+        const bundleMetaFile = path.resolve(this.gradleUserHome, META_FILE_DIR, META_FILE)
-            fs.mkdirSync(dirName)
+        if (!fs.existsSync(bundleMetaFile)) {
+            return new Map<string, string>()
         }
+        const filedata = fs.readFileSync(bundleMetaFile, 'utf-8')
+        core.debug(`Loaded bundle metadata: ${filedata}`)
+        return new Map(JSON.parse(filedata))
+    }
 
-        fs.writeFileSync(metaFile, cacheKey)
+    /**
+     * Saves information about the artifact bundle restore/save into the 'cache-metadata.json' file.
+     */
+    private saveMetadataForCacheResults(results: CacheBundleResult[]): void {
+        const metadata = new Map<string, string>()
+        for (const result of results) {
+            if (result.cacheKey !== undefined) {
+                metadata.set(result.bundle, result.cacheKey)
+            }
+        }
+        const filedata = JSON.stringify(Array.from(metadata))
+        core.debug(`Saving bundle metadata: ${filedata}`)
+
+        const bundleMetaDir = path.resolve(this.gradleUserHome, META_FILE_DIR)
+        const bundleMetaFile = path.resolve(bundleMetaDir, META_FILE)
+
+        if (!fs.existsSync(bundleMetaDir)) {
+            fs.mkdirSync(bundleMetaDir, {recursive: true})
+        }
+        fs.writeFileSync(bundleMetaFile, filedata, 'utf-8')
     }
 
     protected determineGradleUserHome(rootDir: string): string {
@@ -207,6 +255,11 @@ export class GradleUserHomeCache extends AbstractCache {
         return fs.existsSync(dir)
     }
 
+    /**
+     * Determines the paths within Gradle User Home to cache.
+     * By default, this is the 'caches' and 'notifications' directories,
+     * but this can be overridden by the `gradle-home-cache-includes` parameter.
+     */
     protected getCachePath(): string[] {
         const rawPaths: string[] = core.getMultilineInput(INCLUDE_PATHS_PARAMETER)
         rawPaths.push(META_FILE_DIR)
@@ -223,14 +276,23 @@ export class GradleUserHomeCache extends AbstractCache {
         return path.resolve(this.gradleUserHome, rawPath)
     }
 
-    private getArtifactBundles(): Map<string, string> {
+    /**
+     * Return the artifact bundle definitions, which determine which artifacts will be cached
+     * separately from the rest of the Gradle User Home cache entry.
+     * This is normally a fixed set, but can be overridden by the `gradle-home-cache-artifact-bundles` parameter.
+     */
+    private getArtifactBundleDefinitions(): Map<string, string> {
         const artifactBundleDefinition = core.getInput(ARTIFACT_BUNDLES_PARAMETER)
         this.debug(`Using artifact bundle definition: ${artifactBundleDefinition}`)
         const artifactBundles = JSON.parse(artifactBundleDefinition)
         return new Map(Array.from(artifactBundles, ([key, value]) => [key, path.resolve(this.gradleUserHome, value)]))
     }
 
-    private async reportGradleUserHomeSize(label: string): Promise<void> {
+    /**
+     * When cache debugging is enabled, this method will give a detailed report
+     * of the Gradle User Home contents.
+     */
+    private async debugReportGradleUserHomeSize(label: string): Promise<void> {
         if (!this.cacheDebuggingEnabled) {
             return
         }

src/cache-project-dot-gradle.ts

@@ -7,6 +7,9 @@ const PATHS_TO_CACHE = [
     'configuration-cache' // Only configuration-cache is stored at present
 ]
 
+/**
+ * A simple cache that saves and restores the '.gradle/configuration-cache' directory in the project root.
+ */
 export class ProjectDotGradleCache extends AbstractCache {
     private rootDir: string
     constructor(rootDir: string) {

src/cache-utils.ts

@@ -4,8 +4,6 @@ import * as crypto from 'crypto'
 import * as path from 'path'
 import * as fs from 'fs'
 
-const CACHE_PROTOCOL_VERSION = 'v4-'
-
 const CACHE_DISABLED_PARAMETER = 'cache-disabled'
 const CACHE_READONLY_PARAMETER = 'cache-read-only'
 const CACHE_DEBUG_VAR = 'GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED'
@@ -25,7 +23,7 @@ export function isCacheDebuggingEnabled(): boolean {
 
 export function getCacheKeyPrefix(): string {
     // Prefix can be used to force change all cache keys (defaults to cache protocol version)
-    return process.env[CACHE_PREFIX_VAR] || CACHE_PROTOCOL_VERSION
+    return process.env[CACHE_PREFIX_VAR] || ''
 }
 
 export function hashStrings(values: string[]): string {

src/main.ts

@@ -7,7 +7,9 @@ import * as execution from './execution'
 import * as gradlew from './gradlew'
 import * as provision from './provision'
 
-// Invoked by GitHub Actions
+/**
+ * The main entry point for the action, called by Github Actions for the step.
+ */
 export async function run(): Promise<void> {
     try {
         const workspaceDirectory = process.env[`GITHUB_WORKSPACE`] || ''

src/post.ts

@@ -6,7 +6,9 @@ import * as caches from './caches'
 // throw an uncaught exception.  Instead of failing this action, just warn.
 process.on('uncaughtException', e => handleFailure(e))
 
-// Invoked by GitHub Actions
+/**
+ * The post-execution entry point for the action, called by Github Actions after completing all steps for the Job.
+ */
 export async function run(): Promise<void> {
     try {
         await caches.save()