1. import json
  2. import logging
  4. def remove_duplicate_json_objects(json_list, output_file="output.json"):
  5. """
  6. Removes duplicate JSON objects from a list of JSON objects.
  8. Args:
  9. json_list (list): A list of JSON objects (represented as strings).
  10. output_file (str): The file to write the de-duplicated JSON objects to.
  11. Defaults to "output.json".
  13. Returns:
  14. list: A list of unique JSON objects. Returns an empty list on error.
  15. """
  17. logging.basicConfig(level=logging.INFO) #Basic logging
  19. unique_json_objects = []
  20. seen_json = set()
  22. try:
  23. for json_string in json_list:
  24. try:
  25. json_object = json.loads(json_string)
  26. # Convert JSON object to a tuple to make it hashable
  27. json_tuple = tuple(sorted(json_object.items()))
  28. if json_tuple not in seen_json:
  29. unique_json_objects.append(json_object)
  30. seen_json.add(json_tuple)
  31. except json.JSONDecodeError as e:
  32. logging.warning(f"Error decoding JSON: {e}. Skipping object.")
  33. except TypeError as e:
  34. logging.warning(f"Type error processing JSON: {e}. Skipping object.")
  35. except Exception as e:
  36. logging.error(f"Unexpected error processing JSON: {e}. Skipping object.")
  38. except Exception as e:
  39. logging.error(f"An unexpected error occurred: {e}")
  40. return [] #Return empty list on major error
  42. try:
  43. with open(output_file, 'w') as f:
  44. json.dump(unique_json_objects, f, indent=4)
  45. except Exception as e:
  46. logging.error(f"Error writing to file: {e}")
  47. return []
  49. return unique_json_objects

Add your comment