exports.processInBatches = async (arr, handler, batchSize = 10) => {
  const result = [];
  for (let i = 0; i < arr.length; i += batchSize) {
    const batch = arr.slice(i, i + batchSize);
    const batchResult = await Promise.all(batch.map(handler));
    result.push(...batchResult);
  }
  return result;
};

exports.getAllData = async (fetchData, db, pageSize = 100) => {
  let page = 1;
  let allData = [];
  while (true) {
    let list = await fetchData(page, pageSize, db);
    if (list.length > 0) {
      allData.push(...list);
      page++;
    } else {
      break;
    }
  }
  return allData;
};

exports.executeWithConcurrency = async (items, taskFn, maxConcurrency = 10) => {
  let index = 0;
  const results = [];

  while (index < items.length) {
    // 获取当前批次
    const batch = items.slice(index, index + maxConcurrency);

    // 映射当前批次的任务并执行
    const promises = batch.map((item) => taskFn(item));

    // 等待当前批次的任务完成
    const batchResults = await Promise.all(promises);

    // 收集结果
    results.push(...batchResults);

    // 更新索引
    index += maxConcurrency;
  }

  return results;
};