1. function tokenizeLogStream(logStream, regexPatterns) {
  2. const tokens = [];
  4. for (const line of logStream.split('\n')) {
  5. for (const pattern of regexPatterns) {
  6. const match = line.match(pattern);
  7. if (match) {
  8. tokens.push({
  9. type: pattern.source, // Store the regex pattern as token type
  10. value: match[0],
  11. // Add more fields as needed (e.g., captured groups)
  12. });
  13. }
  14. }
  15. }
  17. return tokens;
  18. }
  20. // Example usage:
  21. // Define regex patterns for tokenization. Important to compile these for performance
  22. const regexPatterns = [
  23. { source: /\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\]/g }, // Date/time
  24. { source: /ERROR: (.*)/g }, // Error messages
  25. { source: /WARN: (.*)/g }, // Warning messages
  26. { source: /INFO: (.*)/g }, // Info messages
  27. { source: /(\w+)=(\w+)/g } // Key-value pairs
  28. ];
  30. // Simulate a log stream (replace with your actual log data)
  31. const logStream = `
  32. 2023-10-27 10:00:00 [INFO] Application started
  33. 2023-10-27 10:00:05 ERROR: Failed to connect to database
  34. user=john
  35. password=secret
  36. 2023-10-27 10:00:10 [WARN] Low disk space
  37. `;
  39. const tokens = tokenizeLogStream(logStream, regexPatterns);
  41. console.log(JSON.stringify(tokens, null, 2)); // Output tokens as JSON

Add your comment