1. import re
  2. from urllib.parse import urlparse
  4. def validate_url_list(url_list):
  5. """
  6. Validates a list of URLs, handling potential errors and providing fallback options.
  7. """
  8. validated_urls = []
  9. invalid_urls = []
  11. for url in url_list:
  12. try:
  13. # Basic URL format validation using regex
  14. if not re.match(r"^(http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~:/?#[\]@!\$&'\(\)\*\+,;=.]+$", url):
  15. raise ValueError("Invalid URL format")
  17. # Parse the URL to check for scheme and netloc
  18. result = urlparse(url)
  19. if not all([result.scheme, result.netloc]):
  20. raise ValueError("Missing scheme or netloc")
  22. #Further validation by attempting to connect.
  23. #This helps catch URLs that are technically valid but don't resolve.
  24. result = urlparse(url)
  25. urllib.request.urlopen(url, timeout=5) #add timeout
  26. validated_urls.append(url)
  27. except ValueError as e:
  28. print(f"Invalid URL: {url} - {e}")
  29. invalid_urls.append(url)
  30. except urllib.error.URLError as e:
  31. print(f"URL Error: {url} - {e}")
  32. invalid_urls.append(url)
  33. except Exception as e:
  34. print(f"Unexpected Error: {url} - {e}")
  35. invalid_urls.append(url)
  37. return validated_urls, invalid_urls
  39. if __name__ == '__main__':
  40. #Example Usage
  41. url_list = [
  42. "https://www.google.com",
  43. "http://example.com",
  44. "invalid-url",
  45. "ftp://example.com",
  46. "https://notarealdomain.xyz",
  47. "www.example.com", #missing scheme
  48. "https://www.google.com/search?q=python",
  49. "https://subdomain.example.co.uk"
  50. ]
  52. validated, invalid = validate_url_list(url_list)
  54. print("\nValidated URLs:")
  55. for url in validated:
  56. print(url)
  58. print("\nInvalid URLs:")
  59. for url in invalid:
  60. print(url)

Add your comment