exports.processInBatches = async (arr, handler, batchSize = 10) => {
  const result = [];
  for (let i = 0; i < arr.length; i += batchSize) {
    const batch = arr.slice(i, i + batchSize);
    const batchResult = await Promise.all(batch.map(handler));
    result.push(...batchResult);
  }
  return result;
};

exports.getAllData = async (fetchData, db, pageSize = 100) => {
  let page = 1;
  let allData = [];
  while (true) {
    let list = await fetchData(page, pageSize, db);
    if (list.length > 0) {
      allData.push(...list);
      page++;
    } else {
      break;
    }
  }
  return allData;
};