1. const fs = require('fs').promises;
  2. const path = require('path');
  4. /**
  5. * Splits a directory's data into smaller chunks for short-lived tasks.
  6. * @param {string} directoryPath The path to the directory.
  7. * @param {number} chunkSize The maximum size of each chunk in bytes.
  8. * @param {function} processChunk A function to process each chunk of data. Takes the chunk data as input.
  9. * @returns {Promise<void>} A promise that resolves when all chunks have been processed.
  10. * @throws {Error} If the directory does not exist.
  11. */
  12. async function splitDirectoryData(directoryPath, chunkSize, processChunk) {
  13. try {
  14. const files = await fs.readdir(directoryPath);
  16. for (const file of files) {
  17. const filePath = path.join(directoryPath, file);
  18. const stats = await fs.stat(filePath);
  20. if (stats.isFile()) {
  21. const fileContent = await fs.readFile(filePath, { encoding: 'utf-8' }); // Read file content
  22. const chunkLength = fileContent.length;
  24. if (chunkLength > 0) { // Avoid processing empty files
  25. for (let i = 0; i < chunkLength; i += chunkSize) {
  26. const chunk = fileContent.substring(i, i + chunkSize); // Extract chunk
  27. processChunk(chunk); // Process the chunk
  28. }
  29. }
  30. }
  31. }
  32. } catch (error) {
  33. if (error.code === 'ENOENT') {
  34. throw new Error(`Directory not found: ${directoryPath}`);
  35. }
  36. throw error; // Re-throw other errors
  37. }
  38. }
  40. module.exports = splitDirectoryData;

Add your comment