49 lines
1.2 KiB
JavaScript
49 lines
1.2 KiB
JavaScript
exports.processInBatches = async (arr, handler, batchSize = 10) => {
|
|
const result = [];
|
|
for (let i = 0; i < arr.length; i += batchSize) {
|
|
const batch = arr.slice(i, i + batchSize);
|
|
const batchResult = await Promise.all(batch.map(handler));
|
|
result.push(...batchResult);
|
|
}
|
|
return result;
|
|
};
|
|
|
|
exports.getAllData = async (fetchData, db, pageSize = 100) => {
|
|
let page = 1;
|
|
let allData = [];
|
|
while (true) {
|
|
let list = await fetchData(page, pageSize, db);
|
|
if (list.length > 0) {
|
|
allData.push(...list);
|
|
page++;
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
return allData;
|
|
};
|
|
|
|
exports.executeWithConcurrency = async (items, taskFn, maxConcurrency = 10) => {
|
|
let index = 0;
|
|
const results = [];
|
|
|
|
while (index < items.length) {
|
|
// 获取当前批次
|
|
const batch = items.slice(index, index + maxConcurrency);
|
|
|
|
// 映射当前批次的任务并执行
|
|
const promises = batch.map((item) => taskFn(item));
|
|
|
|
// 等待当前批次的任务完成
|
|
const batchResults = await Promise.all(promises);
|
|
|
|
// 收集结果
|
|
results.push(...batchResults);
|
|
|
|
// 更新索引
|
|
index += maxConcurrency;
|
|
}
|
|
|
|
return results;
|
|
};
|