1. import requests
  2. from bs4 import BeautifulSoup
  3. import json
  4. import logging
  5. import time
  6. import random
  7. from urllib.parse import urljoin
  9. # Configure logging
  10. logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
  12. def batch_cookie_operations(url, cookie_data, operation_type, max_retries=3):
  13. """
  14. Performs batch operations on a given URL using provided cookies.
  16. Args:
  17. url (str): The URL to perform operations on.
  18. cookie_data (dict): A dictionary containing session cookies.
  19. operation_type (str): The type of operation (e.g., 'get_data', 'post_data').
  20. max_retries (int): Maximum number of retries for each operation.
  22. Returns:
  23. list: A list of results from the operations. Returns None if a critical error occurs.
  24. """
  25. results = []
  26. for attempt in range(max_retries):
  27. try:
  28. if operation_type == 'get_data':
  29. response = requests.get(url, cookies=cookie_data, timeout=10) # Add timeout
  30. response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx)
  31. results.append(response.text)
  32. logging.info(f"Successfully retrieved data from {url} (attempt {attempt + 1})")
  33. break # Exit retry loop on success
  34. elif operation_type == 'post_data':
  35. data = {'key1': 'value1', 'key2': 'value2'} #example data
  36. response = requests.post(url, cookies=cookie_data, data=data, timeout=10)
  37. response.raise_for_status()
  38. results.append(response.text)
  39. logging.info(f"Successfully posted data to {url} (attempt {attempt + 1})")
  40. break #Exit retry loop on success
  41. else:
  42. logging.error(f"Unsupported operation type: {operation_type}")
  43. return None
  45. except requests.exceptions.RequestException as e:
  46. logging.warning(f"Request failed for {url} (attempt {attempt + 1}): {e}")
  47. if attempt < max_retries - 1:
  48. sleep_time = random.uniform(1, 5) #random sleep to avoid overwhelming server
  49. logging.info(f"Retrying in {sleep_time:.2f} seconds...")
  50. time.sleep(sleep_time)
  51. else:
  52. logging.error(f"Request failed for {url} after {max_retries} attempts.")
  53. return None #Critical failure, stop further operations
  55. except Exception as e:
  56. logging.exception(f"An unexpected error occurred for {url} (attempt {attempt + 1}): {e}")
  57. return None #Critical failure, stop further operations
  60. return results
  64. def process_data_migration(migration_config):
  65. """
  66. Processes data migration operations based on the provided configuration.
  68. Args:
  69. migration_config (dict): A dictionary containing migration parameters (URLs, cookies, operation types).
  71. Returns:
  72. dict: A dictionary containing the results of the operations. Returns None if a critical error occurs.
  73. """
  74. all_results = {}
  75. for url, cookie_data, operation_type in migration_config.items():
  76. logging.info(f"Processing {url} with {operation_type} operation.")
  77. results = batch_cookie_operations(url, cookie_data, operation_type)
  79. if results is None:
  80. logging.error(f"Data migration failed for {url}. Aborting.")
  81. return None
  82. else:
  83. all_results[url] = results
  87. return all_results
  91. if __name__ == '__main__':
  92. # Example usage
  93. migration_config = {
  94. "https://example.com/data1": {
  95. "cookies": {"session_id": "12345", "user_agent": "Mozilla/5.0"},
  96. "operation_type": "get_data"
  97. },
  98. "https://example.com/data2": {
  99. "cookies": {"auth_token": "abcdefg"},
  100. "operation_type": "post_data"
  101. },
  102. "https://example.com/data3": {
  103. "cookies": {"

Add your comment