From 2df79913f54be72c2136eeff2170d6ec34d5d3e2 Mon Sep 17 00:00:00 2001 From: Bassem Dghaidi <568794+Link-@users.noreply.github.com> Date: Mon, 2 Dec 2024 02:34:19 -0800 Subject: [PATCH] Add progress tracking for blob uploads --- dist/restore-only/index.js | 115 ++++++++++++++++++++++++++++++++++--- dist/restore/index.js | 115 ++++++++++++++++++++++++++++++++++--- dist/save-only/index.js | 115 ++++++++++++++++++++++++++++++++++--- dist/save/index.js | 115 ++++++++++++++++++++++++++++++++++--- 4 files changed, 424 insertions(+), 36 deletions(-) diff --git a/dist/restore-only/index.js b/dist/restore-only/index.js index ea511da..92385b9 100644 --- a/dist/restore-only/index.js +++ b/dist/restore-only/index.js @@ -9734,26 +9734,123 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge }); }; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.uploadCacheArchiveSDK = void 0; +exports.uploadCacheArchiveSDK = exports.UploadProgress = void 0; const core = __importStar(__nccwpck_require__(4850)); const storage_blob_1 = __nccwpck_require__(3864); const errors_1 = __nccwpck_require__(6333); +/** + * Class for tracking the upload state and displaying stats. + */ +class UploadProgress { + constructor(contentLength) { + this.contentLength = contentLength; + this.sentBytes = 0; + this.displayedComplete = false; + this.startTime = Date.now(); + } + /** + * Sets the number of bytes sent + * + * @param sentBytes the number of bytes sent + */ + setSentBytes(sentBytes) { + this.sentBytes = sentBytes; + } + /** + * Returns the total number of bytes transferred. + */ + getTransferredBytes() { + return this.sentBytes; + } + /** + * Returns true if the upload is complete. + */ + isDone() { + return this.getTransferredBytes() === this.contentLength; + } + /** + * Prints the current upload stats. Once the upload completes, this will print one + * last line and then stop. + */ + display() { + if (this.displayedComplete) { + return; + } + const transferredBytes = this.sentBytes; + const percentage = (100 * (transferredBytes / this.contentLength)).toFixed(1); + const elapsedTime = Date.now() - this.startTime; + const uploadSpeed = (transferredBytes / + (1024 * 1024) / + (elapsedTime / 1000)).toFixed(1); + core.info(`Sent ${transferredBytes} of ${this.contentLength} (${percentage}%), ${uploadSpeed} MBs/sec`); + if (this.isDone()) { + this.displayedComplete = true; + } + } + /** + * Returns a function used to handle TransferProgressEvents. + */ + onProgress() { + return (progress) => { + this.setSentBytes(progress.loadedBytes); + }; + } + /** + * Starts the timer that displays the stats. + * + * @param delayInMs the delay between each write + */ + startDisplayTimer(delayInMs = 1000) { + const displayCallback = () => { + this.display(); + if (!this.isDone()) { + this.timeoutHandle = setTimeout(displayCallback, delayInMs); + } + }; + this.timeoutHandle = setTimeout(displayCallback, delayInMs); + } + /** + * Stops the timer that displays the stats. As this typically indicates the upload + * is complete, this will display one last line, unless the last line has already + * been written. + */ + stopDisplayTimer() { + if (this.timeoutHandle) { + clearTimeout(this.timeoutHandle); + this.timeoutHandle = undefined; + } + this.display(); + } +} +exports.UploadProgress = UploadProgress; function uploadCacheArchiveSDK(signedUploadURL, archivePath, options) { + var _a; return __awaiter(this, void 0, void 0, function* () { + const blobClient = new storage_blob_1.BlobClient(signedUploadURL); + const blockBlobClient = blobClient.getBlockBlobClient(); + const properties = yield blobClient.getProperties(); + const contentLength = (_a = properties.contentLength) !== null && _a !== void 0 ? _a : -1; + const uploadProgress = new UploadProgress(contentLength); // Specify data transfer options const uploadOptions = { blockSize: options === null || options === void 0 ? void 0 : options.uploadChunkSize, concurrency: options === null || options === void 0 ? void 0 : options.uploadConcurrency, - maxSingleShotSize: 128 * 1024 * 1024 // 128 MiB initial transfer size + maxSingleShotSize: 128 * 1024 * 1024, + onProgress: uploadProgress.onProgress() }; - const blobClient = new storage_blob_1.BlobClient(signedUploadURL); - const blockBlobClient = blobClient.getBlockBlobClient(); - core.debug(`BlobClient: ${blobClient.name}:${blobClient.accountName}:${blobClient.containerName}`); - const resp = yield blockBlobClient.uploadFile(archivePath, uploadOptions); - if (resp._response.status >= 400) { - throw new errors_1.InvalidResponseError(`Upload failed with status code ${resp._response.status}`); + try { + uploadProgress.startDisplayTimer(); + core.debug(`BlobClient: ${blobClient.name}:${blobClient.accountName}:${blobClient.containerName}`); + const response = yield blockBlobClient.uploadFile(archivePath, uploadOptions); + // TODO: better management of non-retryable errors + if (response._response.status >= 400) { + throw new errors_1.InvalidResponseError(`Upload failed with status code ${response._response.status}`); + } + return response; + } + finally { + uploadProgress.stopDisplayTimer(); } - return resp; }); } exports.uploadCacheArchiveSDK = uploadCacheArchiveSDK; diff --git a/dist/restore/index.js b/dist/restore/index.js index 888173e..5623f5f 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -9734,26 +9734,123 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge }); }; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.uploadCacheArchiveSDK = void 0; +exports.uploadCacheArchiveSDK = exports.UploadProgress = void 0; const core = __importStar(__nccwpck_require__(4850)); const storage_blob_1 = __nccwpck_require__(3864); const errors_1 = __nccwpck_require__(6333); +/** + * Class for tracking the upload state and displaying stats. + */ +class UploadProgress { + constructor(contentLength) { + this.contentLength = contentLength; + this.sentBytes = 0; + this.displayedComplete = false; + this.startTime = Date.now(); + } + /** + * Sets the number of bytes sent + * + * @param sentBytes the number of bytes sent + */ + setSentBytes(sentBytes) { + this.sentBytes = sentBytes; + } + /** + * Returns the total number of bytes transferred. + */ + getTransferredBytes() { + return this.sentBytes; + } + /** + * Returns true if the upload is complete. + */ + isDone() { + return this.getTransferredBytes() === this.contentLength; + } + /** + * Prints the current upload stats. Once the upload completes, this will print one + * last line and then stop. + */ + display() { + if (this.displayedComplete) { + return; + } + const transferredBytes = this.sentBytes; + const percentage = (100 * (transferredBytes / this.contentLength)).toFixed(1); + const elapsedTime = Date.now() - this.startTime; + const uploadSpeed = (transferredBytes / + (1024 * 1024) / + (elapsedTime / 1000)).toFixed(1); + core.info(`Sent ${transferredBytes} of ${this.contentLength} (${percentage}%), ${uploadSpeed} MBs/sec`); + if (this.isDone()) { + this.displayedComplete = true; + } + } + /** + * Returns a function used to handle TransferProgressEvents. + */ + onProgress() { + return (progress) => { + this.setSentBytes(progress.loadedBytes); + }; + } + /** + * Starts the timer that displays the stats. + * + * @param delayInMs the delay between each write + */ + startDisplayTimer(delayInMs = 1000) { + const displayCallback = () => { + this.display(); + if (!this.isDone()) { + this.timeoutHandle = setTimeout(displayCallback, delayInMs); + } + }; + this.timeoutHandle = setTimeout(displayCallback, delayInMs); + } + /** + * Stops the timer that displays the stats. As this typically indicates the upload + * is complete, this will display one last line, unless the last line has already + * been written. + */ + stopDisplayTimer() { + if (this.timeoutHandle) { + clearTimeout(this.timeoutHandle); + this.timeoutHandle = undefined; + } + this.display(); + } +} +exports.UploadProgress = UploadProgress; function uploadCacheArchiveSDK(signedUploadURL, archivePath, options) { + var _a; return __awaiter(this, void 0, void 0, function* () { + const blobClient = new storage_blob_1.BlobClient(signedUploadURL); + const blockBlobClient = blobClient.getBlockBlobClient(); + const properties = yield blobClient.getProperties(); + const contentLength = (_a = properties.contentLength) !== null && _a !== void 0 ? _a : -1; + const uploadProgress = new UploadProgress(contentLength); // Specify data transfer options const uploadOptions = { blockSize: options === null || options === void 0 ? void 0 : options.uploadChunkSize, concurrency: options === null || options === void 0 ? void 0 : options.uploadConcurrency, - maxSingleShotSize: 128 * 1024 * 1024 // 128 MiB initial transfer size + maxSingleShotSize: 128 * 1024 * 1024, + onProgress: uploadProgress.onProgress() }; - const blobClient = new storage_blob_1.BlobClient(signedUploadURL); - const blockBlobClient = blobClient.getBlockBlobClient(); - core.debug(`BlobClient: ${blobClient.name}:${blobClient.accountName}:${blobClient.containerName}`); - const resp = yield blockBlobClient.uploadFile(archivePath, uploadOptions); - if (resp._response.status >= 400) { - throw new errors_1.InvalidResponseError(`Upload failed with status code ${resp._response.status}`); + try { + uploadProgress.startDisplayTimer(); + core.debug(`BlobClient: ${blobClient.name}:${blobClient.accountName}:${blobClient.containerName}`); + const response = yield blockBlobClient.uploadFile(archivePath, uploadOptions); + // TODO: better management of non-retryable errors + if (response._response.status >= 400) { + throw new errors_1.InvalidResponseError(`Upload failed with status code ${response._response.status}`); + } + return response; + } + finally { + uploadProgress.stopDisplayTimer(); } - return resp; }); } exports.uploadCacheArchiveSDK = uploadCacheArchiveSDK; diff --git a/dist/save-only/index.js b/dist/save-only/index.js index 95c0e5e..8995e8f 100644 --- a/dist/save-only/index.js +++ b/dist/save-only/index.js @@ -9734,26 +9734,123 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge }); }; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.uploadCacheArchiveSDK = void 0; +exports.uploadCacheArchiveSDK = exports.UploadProgress = void 0; const core = __importStar(__nccwpck_require__(4850)); const storage_blob_1 = __nccwpck_require__(3864); const errors_1 = __nccwpck_require__(6333); +/** + * Class for tracking the upload state and displaying stats. + */ +class UploadProgress { + constructor(contentLength) { + this.contentLength = contentLength; + this.sentBytes = 0; + this.displayedComplete = false; + this.startTime = Date.now(); + } + /** + * Sets the number of bytes sent + * + * @param sentBytes the number of bytes sent + */ + setSentBytes(sentBytes) { + this.sentBytes = sentBytes; + } + /** + * Returns the total number of bytes transferred. + */ + getTransferredBytes() { + return this.sentBytes; + } + /** + * Returns true if the upload is complete. + */ + isDone() { + return this.getTransferredBytes() === this.contentLength; + } + /** + * Prints the current upload stats. Once the upload completes, this will print one + * last line and then stop. + */ + display() { + if (this.displayedComplete) { + return; + } + const transferredBytes = this.sentBytes; + const percentage = (100 * (transferredBytes / this.contentLength)).toFixed(1); + const elapsedTime = Date.now() - this.startTime; + const uploadSpeed = (transferredBytes / + (1024 * 1024) / + (elapsedTime / 1000)).toFixed(1); + core.info(`Sent ${transferredBytes} of ${this.contentLength} (${percentage}%), ${uploadSpeed} MBs/sec`); + if (this.isDone()) { + this.displayedComplete = true; + } + } + /** + * Returns a function used to handle TransferProgressEvents. + */ + onProgress() { + return (progress) => { + this.setSentBytes(progress.loadedBytes); + }; + } + /** + * Starts the timer that displays the stats. + * + * @param delayInMs the delay between each write + */ + startDisplayTimer(delayInMs = 1000) { + const displayCallback = () => { + this.display(); + if (!this.isDone()) { + this.timeoutHandle = setTimeout(displayCallback, delayInMs); + } + }; + this.timeoutHandle = setTimeout(displayCallback, delayInMs); + } + /** + * Stops the timer that displays the stats. As this typically indicates the upload + * is complete, this will display one last line, unless the last line has already + * been written. + */ + stopDisplayTimer() { + if (this.timeoutHandle) { + clearTimeout(this.timeoutHandle); + this.timeoutHandle = undefined; + } + this.display(); + } +} +exports.UploadProgress = UploadProgress; function uploadCacheArchiveSDK(signedUploadURL, archivePath, options) { + var _a; return __awaiter(this, void 0, void 0, function* () { + const blobClient = new storage_blob_1.BlobClient(signedUploadURL); + const blockBlobClient = blobClient.getBlockBlobClient(); + const properties = yield blobClient.getProperties(); + const contentLength = (_a = properties.contentLength) !== null && _a !== void 0 ? _a : -1; + const uploadProgress = new UploadProgress(contentLength); // Specify data transfer options const uploadOptions = { blockSize: options === null || options === void 0 ? void 0 : options.uploadChunkSize, concurrency: options === null || options === void 0 ? void 0 : options.uploadConcurrency, - maxSingleShotSize: 128 * 1024 * 1024 // 128 MiB initial transfer size + maxSingleShotSize: 128 * 1024 * 1024, + onProgress: uploadProgress.onProgress() }; - const blobClient = new storage_blob_1.BlobClient(signedUploadURL); - const blockBlobClient = blobClient.getBlockBlobClient(); - core.debug(`BlobClient: ${blobClient.name}:${blobClient.accountName}:${blobClient.containerName}`); - const resp = yield blockBlobClient.uploadFile(archivePath, uploadOptions); - if (resp._response.status >= 400) { - throw new errors_1.InvalidResponseError(`Upload failed with status code ${resp._response.status}`); + try { + uploadProgress.startDisplayTimer(); + core.debug(`BlobClient: ${blobClient.name}:${blobClient.accountName}:${blobClient.containerName}`); + const response = yield blockBlobClient.uploadFile(archivePath, uploadOptions); + // TODO: better management of non-retryable errors + if (response._response.status >= 400) { + throw new errors_1.InvalidResponseError(`Upload failed with status code ${response._response.status}`); + } + return response; + } + finally { + uploadProgress.stopDisplayTimer(); } - return resp; }); } exports.uploadCacheArchiveSDK = uploadCacheArchiveSDK; diff --git a/dist/save/index.js b/dist/save/index.js index fc50c36..5aee1c2 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -9734,26 +9734,123 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge }); }; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.uploadCacheArchiveSDK = void 0; +exports.uploadCacheArchiveSDK = exports.UploadProgress = void 0; const core = __importStar(__nccwpck_require__(4850)); const storage_blob_1 = __nccwpck_require__(3864); const errors_1 = __nccwpck_require__(6333); +/** + * Class for tracking the upload state and displaying stats. + */ +class UploadProgress { + constructor(contentLength) { + this.contentLength = contentLength; + this.sentBytes = 0; + this.displayedComplete = false; + this.startTime = Date.now(); + } + /** + * Sets the number of bytes sent + * + * @param sentBytes the number of bytes sent + */ + setSentBytes(sentBytes) { + this.sentBytes = sentBytes; + } + /** + * Returns the total number of bytes transferred. + */ + getTransferredBytes() { + return this.sentBytes; + } + /** + * Returns true if the upload is complete. + */ + isDone() { + return this.getTransferredBytes() === this.contentLength; + } + /** + * Prints the current upload stats. Once the upload completes, this will print one + * last line and then stop. + */ + display() { + if (this.displayedComplete) { + return; + } + const transferredBytes = this.sentBytes; + const percentage = (100 * (transferredBytes / this.contentLength)).toFixed(1); + const elapsedTime = Date.now() - this.startTime; + const uploadSpeed = (transferredBytes / + (1024 * 1024) / + (elapsedTime / 1000)).toFixed(1); + core.info(`Sent ${transferredBytes} of ${this.contentLength} (${percentage}%), ${uploadSpeed} MBs/sec`); + if (this.isDone()) { + this.displayedComplete = true; + } + } + /** + * Returns a function used to handle TransferProgressEvents. + */ + onProgress() { + return (progress) => { + this.setSentBytes(progress.loadedBytes); + }; + } + /** + * Starts the timer that displays the stats. + * + * @param delayInMs the delay between each write + */ + startDisplayTimer(delayInMs = 1000) { + const displayCallback = () => { + this.display(); + if (!this.isDone()) { + this.timeoutHandle = setTimeout(displayCallback, delayInMs); + } + }; + this.timeoutHandle = setTimeout(displayCallback, delayInMs); + } + /** + * Stops the timer that displays the stats. As this typically indicates the upload + * is complete, this will display one last line, unless the last line has already + * been written. + */ + stopDisplayTimer() { + if (this.timeoutHandle) { + clearTimeout(this.timeoutHandle); + this.timeoutHandle = undefined; + } + this.display(); + } +} +exports.UploadProgress = UploadProgress; function uploadCacheArchiveSDK(signedUploadURL, archivePath, options) { + var _a; return __awaiter(this, void 0, void 0, function* () { + const blobClient = new storage_blob_1.BlobClient(signedUploadURL); + const blockBlobClient = blobClient.getBlockBlobClient(); + const properties = yield blobClient.getProperties(); + const contentLength = (_a = properties.contentLength) !== null && _a !== void 0 ? _a : -1; + const uploadProgress = new UploadProgress(contentLength); // Specify data transfer options const uploadOptions = { blockSize: options === null || options === void 0 ? void 0 : options.uploadChunkSize, concurrency: options === null || options === void 0 ? void 0 : options.uploadConcurrency, - maxSingleShotSize: 128 * 1024 * 1024 // 128 MiB initial transfer size + maxSingleShotSize: 128 * 1024 * 1024, + onProgress: uploadProgress.onProgress() }; - const blobClient = new storage_blob_1.BlobClient(signedUploadURL); - const blockBlobClient = blobClient.getBlockBlobClient(); - core.debug(`BlobClient: ${blobClient.name}:${blobClient.accountName}:${blobClient.containerName}`); - const resp = yield blockBlobClient.uploadFile(archivePath, uploadOptions); - if (resp._response.status >= 400) { - throw new errors_1.InvalidResponseError(`Upload failed with status code ${resp._response.status}`); + try { + uploadProgress.startDisplayTimer(); + core.debug(`BlobClient: ${blobClient.name}:${blobClient.accountName}:${blobClient.containerName}`); + const response = yield blockBlobClient.uploadFile(archivePath, uploadOptions); + // TODO: better management of non-retryable errors + if (response._response.status >= 400) { + throw new errors_1.InvalidResponseError(`Upload failed with status code ${response._response.status}`); + } + return response; + } + finally { + uploadProgress.stopDisplayTimer(); } - return resp; }); } exports.uploadCacheArchiveSDK = uploadCacheArchiveSDK;