1. const fs = require('fs');
  2. const { Readable } = require('stream');
  4. /**
  5. * Streams data from a dataset file with rate limiting.
  6. * @param {string} filePath Path to the dataset file.
  7. * @param {number} rateLimit Tokens per second.
  8. * @returns {Readable} A Readable stream for the data.
  9. */
  10. function streamDataset(filePath, rateLimit) {
  11. return new Readable({
  12. read() {
  13. // Simulate reading data from a file. Replace with actual file reading.
  14. fs.readFile(filePath, 'utf8', (err, data) => {
  15. if (err) {
  16. console.error("Error reading file:", err);
  17. process.exit(1); // Exit on error
  18. }
  20. const chunk = data; //Read the whole file
  21. this.push(chunk);
  23. // Rate limiting: Simulate a delay to respect rate limit.
  24. const delay = 1000 / rateLimit; // milliseconds
  25. const timeout = new Promise(resolve => setTimeout(resolve, delay));
  26. timeout.then(() => {
  27. this.read(); // Request more data
  28. });
  29. });
  30. },
  31. close() {
  32. console.log("Stream closed.");
  33. }
  34. });
  35. }
  38. // Example Usage:
  39. // const filePath = 'your_dataset.txt'; // Replace with your file path
  40. // const rateLimit = 10; // 10 tokens per second
  42. // streamDataset(filePath, rateLimit)
  43. // .pipe(process.stdout); // Pipe to console or other destination

Add your comment