1. import pandas as pd
  2. import time
  3. import logging
  5. logging.basicConfig(level=logging.INFO)
  7. def diff_datasets(dataset1_path, dataset2_path, timeout_seconds=60):
  8. """
  9. Compares two datasets (Pandas DataFrames) and returns the differences.
  11. Args:
  12. dataset1_path (str): Path to the first dataset (e.g., CSV file).
  13. dataset2_path (str): Path to the second dataset.
  14. timeout_seconds (int): Timeout in seconds for the comparison.
  16. Returns:
  17. pandas.DataFrame or None: DataFrame containing the differences between the datasets,
  18. or None if an error occurs or the timeout is reached.
  19. """
  20. try:
  21. # Load datasets
  22. df1 = pd.read_csv(dataset1_path)
  23. df2 = pd.read_csv(dataset2_path)
  25. # Validate that the columns match
  26. if not df1.columns.equals(df2.columns):
  27. logging.error("Columns do not match between datasets.")
  28. return None
  30. # Start timer
  31. start_time = time.time()
  33. # Calculate differences
  34. diff = df1.compare(df2, align_axis=0)
  36. # Check for timeout
  37. if time.time() - start_time > timeout_seconds:
  38. logging.warning(f"Timeout reached ({timeout_seconds} seconds).")
  39. return None
  41. return diff
  43. except Exception as e:
  44. logging.error(f"An error occurred: {e}")
  45. return None
  48. if __name__ == '__main__':
  49. # Example usage:
  50. dataset1 = "dataset1.csv"
  51. dataset2 = "dataset2.csv"
  53. # Create dummy datasets for testing
  54. data1 = {'col1': [1, 2, 3, 4], 'col2': ['a', 'b', 'c', 'd']}
  55. data2 = {'col1': [1, 2, 5, 4], 'col2': ['a', 'b', 'c', 'e']}
  56. df1 = pd.DataFrame(data1)
  57. df2 = pd.DataFrame(data2)
  58. df1.to_csv(dataset1, index=False)
  59. df2.to_csv(dataset2, index=False)
  61. diff_df = diff_datasets(dataset1, dataset2, timeout_seconds=5)
  63. if diff_df is not None:
  64. print(diff_df)
  65. else:
  66. print("No differences found or an error occurred.")

Add your comment