diff --git a/package-lock.json b/package-lock.json index 7a8be4d414..c6a731d013 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26327,7 +26327,7 @@ "@contentstack/cli-auth": "~1.3.20", "@contentstack/cli-cm-bootstrap": "~1.10.0", "@contentstack/cli-cm-branches": "~1.1.2", - "@contentstack/cli-cm-bulk-publish": "~1.4.8", + "@contentstack/cli-cm-bulk-publish": "~1.5.0", "@contentstack/cli-cm-clone": "~1.10.7", "@contentstack/cli-cm-export": "~1.11.7", "@contentstack/cli-cm-export-to-csv": "~1.7.2", @@ -26870,7 +26870,7 @@ }, "packages/contentstack-bulk-publish": { "name": "@contentstack/cli-cm-bulk-publish", - "version": "1.4.8", + "version": "1.5.0", "license": "MIT", "dependencies": { "@contentstack/cli-command": "~1.3.0", diff --git a/packages/contentstack-bulk-publish/README.md b/packages/contentstack-bulk-publish/README.md index 7780bea08b..b740c4937a 100644 --- a/packages/contentstack-bulk-publish/README.md +++ b/packages/contentstack-bulk-publish/README.md @@ -18,7 +18,7 @@ $ npm install -g @contentstack/cli-cm-bulk-publish $ csdx COMMAND running command... $ csdx (--version) -@contentstack/cli-cm-bulk-publish/1.4.8 darwin-arm64 node-v22.2.0 +@contentstack/cli-cm-bulk-publish/1.5.0 darwin-arm64 node-v22.2.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/packages/contentstack-bulk-publish/package.json b/packages/contentstack-bulk-publish/package.json index b227eb1048..bc41533b64 100644 --- a/packages/contentstack-bulk-publish/package.json +++ b/packages/contentstack-bulk-publish/package.json @@ -1,7 +1,7 @@ { "name": "@contentstack/cli-cm-bulk-publish", "description": "Contentstack CLI plugin for bulk publish actions", - "version": "1.4.8", + "version": "1.5.0", "author": "Contentstack", "bugs": "https://github.com/contentstack/cli/issues", "dependencies": { diff --git a/packages/contentstack-bulk-publish/src/producer/publish-assets.js b/packages/contentstack-bulk-publish/src/producer/publish-assets.js index 1846b13aff..054dd61424 100644 --- a/packages/contentstack-bulk-publish/src/producer/publish-assets.js +++ b/packages/contentstack-bulk-publish/src/producer/publish-assets.js @@ -6,6 +6,7 @@ const { performBulkPublish, publishAsset, initializeLogger } = require('../consu const retryFailedLogs = require('../util/retryfailed'); const { validateFile } = require('../util/fs'); const { isEmpty } = require('../util'); +const { fetchBulkPublishLimit } = require('../util/common-utility'); const queue = getQueue(); let logFileName; @@ -14,7 +15,7 @@ let filePath; /* eslint-disable no-param-reassign */ -async function getAssets(stack, folder, bulkPublish, environments, locale, apiVersion, skip = 0) { +async function getAssets(stack, folder, bulkPublish, environments, locale, apiVersion, bulkPublishLimit, skip = 0) { return new Promise((resolve, reject) => { let queryParams = { folder: folder, @@ -33,18 +34,27 @@ async function getAssets(stack, folder, bulkPublish, environments, locale, apiVe let assets = assetResponse.items; for (let index = 0; index < assetResponse.items.length; index++) { if (assets[index].is_dir === true) { - await getAssets(stack, assets[index].uid, bulkPublish, environments, locale, apiVersion, 0); + await getAssets( + stack, + assets[index].uid, + bulkPublish, + environments, + locale, + apiVersion, + bulkPublishLimit, + 0, + ); continue; } if (bulkPublish) { - if (bulkPublishSet.length < 10) { + if (bulkPublishSet.length < bulkPublishLimit) { bulkPublishSet.push({ uid: assets[index].uid, locale, publish_details: assets[index].publish_details || [], }); } - if (bulkPublishSet.length === 10) { + if (bulkPublishSet.length === bulkPublishLimit) { await queue.Enqueue({ assets: bulkPublishSet, Type: 'asset', @@ -56,7 +66,11 @@ async function getAssets(stack, folder, bulkPublish, environments, locale, apiVe bulkPublishSet = []; } - if (assetResponse.items.length - 1 === index && bulkPublishSet.length > 0 && bulkPublishSet.length < 10) { + if ( + assetResponse.items.length - 1 === index && + bulkPublishSet.length > 0 && + bulkPublishSet.length < bulkPublishLimit + ) { await queue.Enqueue({ assets: bulkPublishSet, Type: 'asset', @@ -81,7 +95,7 @@ async function getAssets(stack, folder, bulkPublish, environments, locale, apiVe if (skip === assetResponse.count) { return resolve(true); } - await getAssets(stack, folder, bulkPublish, environments, locale, apiVersion, skip); + await getAssets(stack, folder, bulkPublish, environments, locale, apiVersion, bulkPublishLimit, skip); return resolve(); } else { resolve(); @@ -133,8 +147,9 @@ async function start({ retryFailed, bulkPublish, environments, folderUid, locale } } else if (folderUid) { setConfig(config, bulkPublish); + const bulkPublishLimit = fetchBulkPublishLimit(stack?.org_uid); for (const element of locales) { - await getAssets(stack, folderUid, bulkPublish, environments, element, apiVersion); + await getAssets(stack, folderUid, bulkPublish, environments, element, apiVersion, bulkPublishLimit); } } } diff --git a/packages/contentstack-bulk-publish/src/producer/publish-entries.js b/packages/contentstack-bulk-publish/src/producer/publish-entries.js index 126260ce73..0f6cce121f 100644 --- a/packages/contentstack-bulk-publish/src/producer/publish-entries.js +++ b/packages/contentstack-bulk-publish/src/producer/publish-entries.js @@ -8,6 +8,7 @@ const { performBulkPublish, publishEntry, initializeLogger } = require('../consu const retryFailedLogs = require('../util/retryfailed'); const { validateFile } = require('../util/fs'); const { isEmpty } = require('../util'); +const { fetchBulkPublishLimit } = require('../util/common-utility'); const queue = getQueue(); @@ -18,7 +19,16 @@ let allContentTypes = []; let bulkPublishSet = []; let filePath; -async function getEntries(stack, contentType, locale, bulkPublish, environments, apiVersion, skip = 0) { +async function getEntries( + stack, + contentType, + locale, + bulkPublish, + environments, + apiVersion, + bulkPublishLimit, + skip = 0, +) { return new Promise((resolve, reject) => { skipCount = skip; @@ -39,7 +49,7 @@ async function getEntries(stack, contentType, locale, bulkPublish, environments, let entries = entriesResponse.items; for (let index = 0; index < entriesResponse.items.length; index++) { if (bulkPublish) { - if (bulkPublishSet.length < 10) { + if (bulkPublishSet.length < bulkPublishLimit) { bulkPublishSet.push({ uid: entries[index].uid, content_type: contentType, @@ -48,21 +58,21 @@ async function getEntries(stack, contentType, locale, bulkPublish, environments, }); } - if (bulkPublishSet.length === 10) { + if (bulkPublishSet.length === bulkPublishLimit) { await queue.Enqueue({ entries: bulkPublishSet, locale, Type: 'entry', environments: environments, stack: stack, - apiVersion + apiVersion, }); bulkPublishSet = []; } if ( index === entriesResponse.items.length - 1 && - bulkPublishSet.length <= 10 && + bulkPublishSet.length <= bulkPublishLimit && bulkPublishSet.length > 0 ) { await queue.Enqueue({ @@ -71,7 +81,7 @@ async function getEntries(stack, contentType, locale, bulkPublish, environments, Type: 'entry', environments: environments, stack: stack, - apiVersion + apiVersion, }); bulkPublishSet = []; } // bulkPublish @@ -92,7 +102,16 @@ async function getEntries(stack, contentType, locale, bulkPublish, environments, bulkPublishSet = []; return resolve(); } - await getEntries(stack, contentType, locale, bulkPublish, environments, apiVersion, skipCount); + await getEntries( + stack, + contentType, + locale, + bulkPublish, + environments, + apiVersion, + bulkPublishLimit, + skipCount, + ); return resolve(); }) .catch((error) => reject(error)); @@ -170,10 +189,19 @@ async function start( } else { allContentTypes = contentTypes; } + const bulkPublishLimit = fetchBulkPublishLimit(stack?.org_uid); for (let loc = 0; loc < locales.length; loc += 1) { for (let i = 0; i < allContentTypes.length; i += 1) { /* eslint-disable no-await-in-loop */ - await getEntries(stack, allContentTypes[i].uid || allContentTypes[i], locales[loc], bulkPublish, environments, apiVersion); + await getEntries( + stack, + allContentTypes[i].uid || allContentTypes[i], + locales[loc], + bulkPublish, + environments, + apiVersion, + bulkPublishLimit, + ); /* eslint-enable no-await-in-loop */ } } diff --git a/packages/contentstack-bulk-publish/src/util/client.js b/packages/contentstack-bulk-publish/src/util/client.js index 12ad2c4164..22a73bcca7 100644 --- a/packages/contentstack-bulk-publish/src/util/client.js +++ b/packages/contentstack-bulk-publish/src/util/client.js @@ -15,12 +15,18 @@ async function getStack(data) { stackOptions.api_key = tokenDetails.apiKey; } else if (data.stackApiKey) { if (!isAuthenticated()) { - throw new Error('Please login to proceed further. Or use `--alias` instead of `--stack-api-key` to proceed without logging in.') + throw new Error( + 'Please login to proceed further. Or use `--alias` instead of `--stack-api-key` to proceed without logging in.', + ); } stackOptions.api_key = data.stackApiKey; } const managementClient = await managementSDKClient(options); const stack = managementClient.stack(stackOptions); + if (data.stackApiKey && isAuthenticated()) { + const stackDetails = await stack.fetch(); + stack.org_uid = stackDetails.org_uid; + } stack.alias = data.alias; stack.host = data.host; return stack; diff --git a/packages/contentstack-bulk-publish/src/util/common-utility.js b/packages/contentstack-bulk-publish/src/util/common-utility.js new file mode 100644 index 0000000000..c51c0a40de --- /dev/null +++ b/packages/contentstack-bulk-publish/src/util/common-utility.js @@ -0,0 +1,30 @@ +const { configHandler, cliux } = require('@contentstack/cli-utilities'); + +function fetchBulkPublishLimit(orgUid) { + const plan = configHandler.get('rateLimit'); + let bulkPublishLimit = 1; // Default limit according to the default plan + + if (plan) { + const orgPlan = plan[orgUid]?.bulkLimit; + const defaultPlan = plan['default']?.bulkLimit; + + if (orgPlan?.value && orgPlan?.utilize) { + bulkPublishLimit = Math.ceil((orgPlan.value * orgPlan.utilize) / 100); + } else if (defaultPlan?.value && defaultPlan?.utilize) { + bulkPublishLimit = Math.ceil((defaultPlan.value * defaultPlan.utilize) / 100); + } + } else { + cliux.print( + 'Bulk publish limit not found in config. Using default limit. Please set the limit using $csdx config:set:rate-limit', + { color: 'yellow' }, + ); + // TODO: Update the link once the rate-limit documentation is ready + cliux.print( + 'Suggestions: To set the rate limit, visit https://www.contentstack.com/docs/developers/cli#get-started-with-contentstack-command-line-interface-cli', + { color: 'blue' }, + ); + } + return bulkPublishLimit; +} + +module.exports = { fetchBulkPublishLimit }; diff --git a/packages/contentstack/package.json b/packages/contentstack/package.json index 4d75b13a80..1773e90595 100755 --- a/packages/contentstack/package.json +++ b/packages/contentstack/package.json @@ -26,7 +26,7 @@ "@contentstack/cli-auth": "~1.3.20", "@contentstack/cli-cm-bootstrap": "~1.10.0", "@contentstack/cli-cm-branches": "~1.1.2", - "@contentstack/cli-cm-bulk-publish": "~1.4.8", + "@contentstack/cli-cm-bulk-publish": "~1.5.0", "@contentstack/cli-cm-export": "~1.11.7", "@contentstack/cli-cm-clone": "~1.10.7", "@contentstack/cli-cm-export-to-csv": "~1.7.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 919b3d6daf..d097ad81a5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,7 +14,7 @@ importers: '@contentstack/cli-auth': ~1.3.20 '@contentstack/cli-cm-bootstrap': ~1.10.0 '@contentstack/cli-cm-branches': ~1.1.2 - '@contentstack/cli-cm-bulk-publish': ~1.4.8 + '@contentstack/cli-cm-bulk-publish': ~1.5.0 '@contentstack/cli-cm-clone': ~1.10.7 '@contentstack/cli-cm-export': ~1.11.7 '@contentstack/cli-cm-export-to-csv': ~1.7.2