1. import pandas as pd
  2. import logging
  4. # Configure logging
  5. logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
  7. def deduplicate_records(df, columns):
  8. """
  9. Deduplicates records in a Pandas DataFrame based on specified columns.
  11. Args:
  12. df (pd.DataFrame): The input DataFrame.
  13. columns (list): A list of column names to consider for deduplication.
  15. Returns:
  16. pd.DataFrame: The DataFrame with duplicate rows removed.
  17. """
  18. try:
  19. # Check if columns exist
  20. for col in columns:
  21. if col not in df.columns:
  22. raise ValueError(f"Column '{col}' not found in DataFrame.")
  24. # Drop duplicate rows based on the specified columns
  25. df_deduplicated = df.drop_duplicates(subset=columns, keep='first') # Keep the first occurrence
  27. logging.info(f"Deduplication completed successfully based on columns: {columns}")
  28. logging.info(f"Original DataFrame shape: {df.shape}")
  29. logging.info(f"Deduplicated DataFrame shape: {df_deduplicated.shape}")
  31. return df_deduplicated
  32. except ValueError as e:
  33. logging.error(f"Error during deduplication: {e}")
  34. return None
  35. except Exception as e:
  36. logging.error(f"An unexpected error occurred: {e}")
  37. return None
  40. if __name__ == '__main__':
  41. # Example Usage
  42. data = {'col1': [1, 2, 2, 3, 4, 4, 4],
  43. 'col2': ['A', 'B', 'B', 'C', 'D', 'D', 'D'],
  44. 'col3': [10, 20, 20, 30, 40, 40, 40]}
  45. df = pd.DataFrame(data)
  47. columns_to_deduplicate = ['col1', 'col2']
  48. deduplicated_df = deduplicate_records(df.copy(), columns_to_deduplicate) #Using .copy() to avoid modifying original
  50. if deduplicated_df is not None:
  51. print(deduplicated_df)

Add your comment