actions/cache
1
0
Fork 0
mirror of https://code.forgejo.org/actions/cache.git synced 2024-12-04 04:11:07 -05:00
Code Issues Projects Releases Packages Wiki Activity

Add progress tracking for blob uploads

Browse source
This commit is contained in:
Bassem Dghaidi 2024-12-02 02:34:19 -08:00 committed by GitHub
parent 18fe6e3304
commit 2df79913f5
4 changed files with 424 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

113
dist/restore-only/index.js vendored
View file

@ -9734,26 +9734,123 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
});
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.uploadCacheArchiveSDK = void 0;
exports.uploadCacheArchiveSDK = exports.UploadProgress = void 0;
const core = __importStar(__nccwpck_require__(4850));
const storage_blob_1 = __nccwpck_require__(3864);
const errors_1 = __nccwpck_require__(6333);
/**
* Class for tracking the upload state and displaying stats.
*/
class UploadProgress {
constructor(contentLength) {
this.contentLength = contentLength;
this.sentBytes = 0;
this.displayedComplete = false;
this.startTime = Date.now();
}
/**
* Sets the number of bytes sent
*
* @param sentBytes the number of bytes sent
*/
setSentBytes(sentBytes) {
this.sentBytes = sentBytes;
}
/**
* Returns the total number of bytes transferred.
*/
getTransferredBytes() {
return this.sentBytes;
}
/**
* Returns true if the upload is complete.
*/
isDone() {
return this.getTransferredBytes() === this.contentLength;
}
/**
* Prints the current upload stats. Once the upload completes, this will print one
* last line and then stop.
*/
display() {
if (this.displayedComplete) {
return;
}
const transferredBytes = this.sentBytes;
const percentage = (100 * (transferredBytes / this.contentLength)).toFixed(1);
const elapsedTime = Date.now() - this.startTime;
const uploadSpeed = (transferredBytes /
(1024 * 1024) /
(elapsedTime / 1000)).toFixed(1);
core.info(`Sent ${transferredBytes} of ${this.contentLength} (${percentage}%), ${uploadSpeed} MBs/sec`);
if (this.isDone()) {
this.displayedComplete = true;
}
}
/**
* Returns a function used to handle TransferProgressEvents.
*/
onProgress() {
return (progress) => {
this.setSentBytes(progress.loadedBytes);
};
}
/**
* Starts the timer that displays the stats.
*
* @param delayInMs the delay between each write
*/
startDisplayTimer(delayInMs = 1000) {
const displayCallback = () => {
this.display();
if (!this.isDone()) {
this.timeoutHandle = setTimeout(displayCallback, delayInMs);
}
};
this.timeoutHandle = setTimeout(displayCallback, delayInMs);
}
/**
* Stops the timer that displays the stats. As this typically indicates the upload
* is complete, this will display one last line, unless the last line has already
* been written.
*/
stopDisplayTimer() {
if (this.timeoutHandle) {
clearTimeout(this.timeoutHandle);
this.timeoutHandle = undefined;
}
this.display();
}
}
exports.UploadProgress = UploadProgress;
function uploadCacheArchiveSDK(signedUploadURL, archivePath, options) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
const blobClient = new storage_blob_1.BlobClient(signedUploadURL);
const blockBlobClient = blobClient.getBlockBlobClient();
const properties = yield blobClient.getProperties();
const contentLength = (_a = properties.contentLength) !== null && _a !== void 0 ? _a : -1;
const uploadProgress = new UploadProgress(contentLength);
// Specify data transfer options
const uploadOptions = {
blockSize: options === null || options === void 0 ? void 0 : options.uploadChunkSize,
concurrency: options === null || options === void 0 ? void 0 : options.uploadConcurrency,
maxSingleShotSize: 128 * 1024 * 1024 // 128 MiB initial transfer size
maxSingleShotSize: 128 * 1024 * 1024,
onProgress: uploadProgress.onProgress()
};
const blobClient = new storage_blob_1.BlobClient(signedUploadURL);
const blockBlobClient = blobClient.getBlockBlobClient();
try {
uploadProgress.startDisplayTimer();
core.debug(`BlobClient: ${blobClient.name}:${blobClient.accountName}:${blobClient.containerName}`);
const resp = yield blockBlobClient.uploadFile(archivePath, uploadOptions);
if (resp._response.status >= 400) {
throw new errors_1.InvalidResponseError(`Upload failed with status code ${resp._response.status}`);
const response = yield blockBlobClient.uploadFile(archivePath, uploadOptions);
// TODO: better management of non-retryable errors
if (response._response.status >= 400) {
throw new errors_1.InvalidResponseError(`Upload failed with status code ${response._response.status}`);
}
return response;
}
finally {
uploadProgress.stopDisplayTimer();
}
return resp;
});
}
exports.uploadCacheArchiveSDK = uploadCacheArchiveSDK;

113
dist/restore/index.js vendored
View file

@ -9734,26 +9734,123 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
});
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.uploadCacheArchiveSDK = void 0;
exports.uploadCacheArchiveSDK = exports.UploadProgress = void 0;
const core = __importStar(__nccwpck_require__(4850));
const storage_blob_1 = __nccwpck_require__(3864);
const errors_1 = __nccwpck_require__(6333);
/**
* Class for tracking the upload state and displaying stats.
*/
class UploadProgress {
constructor(contentLength) {
this.contentLength = contentLength;
this.sentBytes = 0;
this.displayedComplete = false;
this.startTime = Date.now();
}
/**
* Sets the number of bytes sent
*
* @param sentBytes the number of bytes sent
*/
setSentBytes(sentBytes) {
this.sentBytes = sentBytes;
}
/**
* Returns the total number of bytes transferred.
*/
getTransferredBytes() {
return this.sentBytes;
}
/**
* Returns true if the upload is complete.
*/
isDone() {
return this.getTransferredBytes() === this.contentLength;
}
/**
* Prints the current upload stats. Once the upload completes, this will print one
* last line and then stop.
*/
display() {
if (this.displayedComplete) {
return;
}
const transferredBytes = this.sentBytes;
const percentage = (100 * (transferredBytes / this.contentLength)).toFixed(1);
const elapsedTime = Date.now() - this.startTime;
const uploadSpeed = (transferredBytes /
(1024 * 1024) /
(elapsedTime / 1000)).toFixed(1);
core.info(`Sent ${transferredBytes} of ${this.contentLength} (${percentage}%), ${uploadSpeed} MBs/sec`);
if (this.isDone()) {
this.displayedComplete = true;
}
}
/**
* Returns a function used to handle TransferProgressEvents.
*/
onProgress() {
return (progress) => {
this.setSentBytes(progress.loadedBytes);
};
}
/**
* Starts the timer that displays the stats.
*
* @param delayInMs the delay between each write
*/
startDisplayTimer(delayInMs = 1000) {
const displayCallback = () => {
this.display();
if (!this.isDone()) {
this.timeoutHandle = setTimeout(displayCallback, delayInMs);
}
};
this.timeoutHandle = setTimeout(displayCallback, delayInMs);
}
/**
* Stops the timer that displays the stats. As this typically indicates the upload
* is complete, this will display one last line, unless the last line has already
* been written.
*/
stopDisplayTimer() {
if (this.timeoutHandle) {
clearTimeout(this.timeoutHandle);
this.timeoutHandle = undefined;
}
this.display();
}
}
exports.UploadProgress = UploadProgress;
function uploadCacheArchiveSDK(signedUploadURL, archivePath, options) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
const blobClient = new storage_blob_1.BlobClient(signedUploadURL);
const blockBlobClient = blobClient.getBlockBlobClient();
const properties = yield blobClient.getProperties();
const contentLength = (_a = properties.contentLength) !== null && _a !== void 0 ? _a : -1;
const uploadProgress = new UploadProgress(contentLength);
// Specify data transfer options
const uploadOptions = {
blockSize: options === null || options === void 0 ? void 0 : options.uploadChunkSize,
concurrency: options === null || options === void 0 ? void 0 : options.uploadConcurrency,
maxSingleShotSize: 128 * 1024 * 1024 // 128 MiB initial transfer size
maxSingleShotSize: 128 * 1024 * 1024,
onProgress: uploadProgress.onProgress()
};
const blobClient = new storage_blob_1.BlobClient(signedUploadURL);
const blockBlobClient = blobClient.getBlockBlobClient();
try {
uploadProgress.startDisplayTimer();
core.debug(`BlobClient: ${blobClient.name}:${blobClient.accountName}:${blobClient.containerName}`);
const resp = yield blockBlobClient.uploadFile(archivePath, uploadOptions);
if (resp._response.status >= 400) {
throw new errors_1.InvalidResponseError(`Upload failed with status code ${resp._response.status}`);
const response = yield blockBlobClient.uploadFile(archivePath, uploadOptions);
// TODO: better management of non-retryable errors
if (response._response.status >= 400) {
throw new errors_1.InvalidResponseError(`Upload failed with status code ${response._response.status}`);
}
return response;
}
finally {
uploadProgress.stopDisplayTimer();
}
return resp;
});
}
exports.uploadCacheArchiveSDK = uploadCacheArchiveSDK;

113
dist/save-only/index.js vendored
View file

@ -9734,26 +9734,123 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
});
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.uploadCacheArchiveSDK = void 0;
exports.uploadCacheArchiveSDK = exports.UploadProgress = void 0;
const core = __importStar(__nccwpck_require__(4850));
const storage_blob_1 = __nccwpck_require__(3864);
const errors_1 = __nccwpck_require__(6333);
/**
* Class for tracking the upload state and displaying stats.
*/
class UploadProgress {
constructor(contentLength) {
this.contentLength = contentLength;
this.sentBytes = 0;
this.displayedComplete = false;
this.startTime = Date.now();
}
/**
* Sets the number of bytes sent
*
* @param sentBytes the number of bytes sent
*/
setSentBytes(sentBytes) {
this.sentBytes = sentBytes;
}
/**
* Returns the total number of bytes transferred.
*/
getTransferredBytes() {
return this.sentBytes;
}
/**
* Returns true if the upload is complete.
*/
isDone() {
return this.getTransferredBytes() === this.contentLength;
}
/**
* Prints the current upload stats. Once the upload completes, this will print one
* last line and then stop.
*/
display() {
if (this.displayedComplete) {
return;
}
const transferredBytes = this.sentBytes;
const percentage = (100 * (transferredBytes / this.contentLength)).toFixed(1);
const elapsedTime = Date.now() - this.startTime;
const uploadSpeed = (transferredBytes /
(1024 * 1024) /
(elapsedTime / 1000)).toFixed(1);
core.info(`Sent ${transferredBytes} of ${this.contentLength} (${percentage}%), ${uploadSpeed} MBs/sec`);
if (this.isDone()) {
this.displayedComplete = true;
}
}
/**
* Returns a function used to handle TransferProgressEvents.
*/
onProgress() {
return (progress) => {
this.setSentBytes(progress.loadedBytes);
};
}
/**
* Starts the timer that displays the stats.
*
* @param delayInMs the delay between each write
*/
startDisplayTimer(delayInMs = 1000) {
const displayCallback = () => {
this.display();
if (!this.isDone()) {
this.timeoutHandle = setTimeout(displayCallback, delayInMs);
}
};
this.timeoutHandle = setTimeout(displayCallback, delayInMs);
}
/**
* Stops the timer that displays the stats. As this typically indicates the upload
* is complete, this will display one last line, unless the last line has already
* been written.
*/
stopDisplayTimer() {
if (this.timeoutHandle) {
clearTimeout(this.timeoutHandle);
this.timeoutHandle = undefined;
}
this.display();
}
}
exports.UploadProgress = UploadProgress;
function uploadCacheArchiveSDK(signedUploadURL, archivePath, options) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
const blobClient = new storage_blob_1.BlobClient(signedUploadURL);
const blockBlobClient = blobClient.getBlockBlobClient();
const properties = yield blobClient.getProperties();
const contentLength = (_a = properties.contentLength) !== null && _a !== void 0 ? _a : -1;
const uploadProgress = new UploadProgress(contentLength);
// Specify data transfer options
const uploadOptions = {
blockSize: options === null || options === void 0 ? void 0 : options.uploadChunkSize,
concurrency: options === null || options === void 0 ? void 0 : options.uploadConcurrency,
maxSingleShotSize: 128 * 1024 * 1024 // 128 MiB initial transfer size
maxSingleShotSize: 128 * 1024 * 1024,
onProgress: uploadProgress.onProgress()
};
const blobClient = new storage_blob_1.BlobClient(signedUploadURL);
const blockBlobClient = blobClient.getBlockBlobClient();
try {
uploadProgress.startDisplayTimer();
core.debug(`BlobClient: ${blobClient.name}:${blobClient.accountName}:${blobClient.containerName}`);
const resp = yield blockBlobClient.uploadFile(archivePath, uploadOptions);
if (resp._response.status >= 400) {
throw new errors_1.InvalidResponseError(`Upload failed with status code ${resp._response.status}`);
const response = yield blockBlobClient.uploadFile(archivePath, uploadOptions);
// TODO: better management of non-retryable errors
if (response._response.status >= 400) {
throw new errors_1.InvalidResponseError(`Upload failed with status code ${response._response.status}`);
}
return response;
}
finally {
uploadProgress.stopDisplayTimer();
}
return resp;
});
}
exports.uploadCacheArchiveSDK = uploadCacheArchiveSDK;

113
dist/save/index.js vendored
View file

@ -9734,26 +9734,123 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
});
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.uploadCacheArchiveSDK = void 0;
exports.uploadCacheArchiveSDK = exports.UploadProgress = void 0;
const core = __importStar(__nccwpck_require__(4850));
const storage_blob_1 = __nccwpck_require__(3864);
const errors_1 = __nccwpck_require__(6333);
/**
* Class for tracking the upload state and displaying stats.
*/
class UploadProgress {
constructor(contentLength) {
this.contentLength = contentLength;
this.sentBytes = 0;
this.displayedComplete = false;
this.startTime = Date.now();
}
/**
* Sets the number of bytes sent
*
* @param sentBytes the number of bytes sent
*/
setSentBytes(sentBytes) {
this.sentBytes = sentBytes;
}
/**
* Returns the total number of bytes transferred.
*/
getTransferredBytes() {
return this.sentBytes;
}
/**
* Returns true if the upload is complete.
*/
isDone() {
return this.getTransferredBytes() === this.contentLength;
}
/**
* Prints the current upload stats. Once the upload completes, this will print one
* last line and then stop.
*/
display() {
if (this.displayedComplete) {
return;
}
const transferredBytes = this.sentBytes;
const percentage = (100 * (transferredBytes / this.contentLength)).toFixed(1);
const elapsedTime = Date.now() - this.startTime;
const uploadSpeed = (transferredBytes /
(1024 * 1024) /
(elapsedTime / 1000)).toFixed(1);
core.info(`Sent ${transferredBytes} of ${this.contentLength} (${percentage}%), ${uploadSpeed} MBs/sec`);
if (this.isDone()) {
this.displayedComplete = true;
}
}
/**
* Returns a function used to handle TransferProgressEvents.
*/
onProgress() {
return (progress) => {
this.setSentBytes(progress.loadedBytes);
};
}
/**
* Starts the timer that displays the stats.
*
* @param delayInMs the delay between each write
*/
startDisplayTimer(delayInMs = 1000) {
const displayCallback = () => {
this.display();
if (!this.isDone()) {
this.timeoutHandle = setTimeout(displayCallback, delayInMs);
}
};
this.timeoutHandle = setTimeout(displayCallback, delayInMs);
}
/**
* Stops the timer that displays the stats. As this typically indicates the upload
* is complete, this will display one last line, unless the last line has already
* been written.
*/
stopDisplayTimer() {
if (this.timeoutHandle) {
clearTimeout(this.timeoutHandle);
this.timeoutHandle = undefined;
}
this.display();
}
}
exports.UploadProgress = UploadProgress;
function uploadCacheArchiveSDK(signedUploadURL, archivePath, options) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
const blobClient = new storage_blob_1.BlobClient(signedUploadURL);
const blockBlobClient = blobClient.getBlockBlobClient();
const properties = yield blobClient.getProperties();
const contentLength = (_a = properties.contentLength) !== null && _a !== void 0 ? _a : -1;
const uploadProgress = new UploadProgress(contentLength);
// Specify data transfer options
const uploadOptions = {
blockSize: options === null || options === void 0 ? void 0 : options.uploadChunkSize,
concurrency: options === null || options === void 0 ? void 0 : options.uploadConcurrency,
maxSingleShotSize: 128 * 1024 * 1024 // 128 MiB initial transfer size
maxSingleShotSize: 128 * 1024 * 1024,
onProgress: uploadProgress.onProgress()
};
const blobClient = new storage_blob_1.BlobClient(signedUploadURL);
const blockBlobClient = blobClient.getBlockBlobClient();
try {
uploadProgress.startDisplayTimer();
core.debug(`BlobClient: ${blobClient.name}:${blobClient.accountName}:${blobClient.containerName}`);
const resp = yield blockBlobClient.uploadFile(archivePath, uploadOptions);
if (resp._response.status >= 400) {
throw new errors_1.InvalidResponseError(`Upload failed with status code ${resp._response.status}`);
const response = yield blockBlobClient.uploadFile(archivePath, uploadOptions);
// TODO: better management of non-retryable errors
if (response._response.status >= 400) {
throw new errors_1.InvalidResponseError(`Upload failed with status code ${response._response.status}`);
}
return response;
}
finally {
uploadProgress.stopDisplayTimer();
}
return resp;
});
}
exports.uploadCacheArchiveSDK = uploadCacheArchiveSDK;