1. import json
  3. def deduplicate_json(json_data):
  4. """
  5. Deduplicates records in a JSON response based on a defined key.
  6. Performs basic sanity checks on the data.
  8. Args:
  9. json_data (str or list): JSON string or a list of dictionaries.
  11. Returns:
  12. list: A deduplicated list of dictionaries. Returns an empty list if input is invalid.
  13. """
  15. if isinstance(json_data, str):
  16. try:
  17. data = json.loads(json_data)
  18. except json.JSONDecodeError:
  19. print("Error: Invalid JSON string.")
  20. return []
  21. elif isinstance(json_data, list):
  22. data = json_data
  23. else:
  24. print("Error: Input must be a JSON string or a list of dictionaries.")
  25. return []
  27. if not isinstance(data, list):
  28. print("Error: JSON data must represent a list of dictionaries.")
  29. return []
  31. if not data: #handle empty list
  32. return []
  34. # Sanity check: Ensure all elements are dictionaries
  35. for item in data:
  36. if not isinstance(item, dict):
  37. print("Error: List elements must be dictionaries.")
  38. return []
  40. seen = set()
  41. deduplicated_data = []
  43. for record in data:
  44. # Define the key to check for duplicates. Adapt as needed.
  45. key_to_check = 'id' # Change this to your unique identifier key
  47. if key_to_check not in record:
  48. print(f"Warning: Record missing key '{key_to_check}'. Skipping.")
  49. continue
  51. value = record[key_to_check]
  52. if value not in seen:
  53. deduplicated_data.append(record)
  54. seen.add(value)
  55. else:
  56. print(f"Duplicate record found with key '{key_to_check}': {value}. Skipping.")
  58. return deduplicated_data
  60. if __name__ == '__main__':
  61. # Example Usage:
  62. json_string = """
  63. [
  64. {"id": 1, "name": "Alice", "age": 30},
  65. {"id": 2, "name": "Bob", "age": 25},
  66. {"id": 1, "name": "Alice", "age": 30},
  67. {"id": 3, "name": "Charlie", "age": 35},
  68. {"name": "David", "age": 40}
  69. ]
  70. """
  72. deduplicated_records = deduplicate_json(json_string)
  73. print(json.dumps(deduplicated_records, indent=4))
  75. #Example with a list of dictionaries:
  76. data = [
  77. {"id": 4, "name": "Eve", "age": 28},
  78. {"id": 5, "name": "Frank", "age": 42},
  79. {"id": 4, "name": "Eve", "age": 28}
  80. ]
  81. deduplicated_records = deduplicate_json(data)
  82. print(json.dumps(deduplicated_records, indent=4))
  84. #Example with invalid JSON:
  85. invalid_json = "{'name': 'John'}"
  86. deduplicated_records = deduplicate_json(invalid_json)
  87. print(deduplicated_records)

Add your comment