1. import java.net.MalformedURLException;
  2. import java.net.URL;
  3. import java.util.concurrent.ConcurrentHashMap;
  4. import java.util.concurrent.ConcurrentLinkedQueue;
  5. import java.util.concurrent.ExecutorService;
  6. import java.util.concurrent.Executors;
  7. import java.util.concurrent.TimeUnit;
  9. public class RequestThrottler {
  11. private final int maxRequestsPerSecond;
  12. private final int timeoutMillis;
  13. private final ConcurrentHashMap<URL, Long> lastRequestTimes = new ConcurrentHashMap<>();
  14. private final ConcurrentLinkedQueue<URL> queue = new ConcurrentLinkedQueue<>();
  15. private final ExecutorService executor = Executors.newFixedThreadPool(5); // Adjust thread pool size as needed
  17. public RequestThrottler(int maxRequestsPerSecond, int timeoutMillis) {
  18. this.maxRequestsPerSecond = maxRequestsPerSecond;
  19. this.timeoutMillis = timeoutMillis;
  20. }
  22. public void enqueueRequest(URL url) {
  23. queue.offer(url);
  24. processQueue();
  25. }
  27. private void processQueue() {
  28. while (!queue.isEmpty()) {
  29. URL url = queue.poll();
  30. if (isAllowedToRequest(url)) {
  31. executor.submit(fetchHtmlPage, url);
  32. } else {
  33. // Request throttled, re-enqueue after waiting
  34. long waitTime = calculateWaitTime(url);
  35. try {
  36. TimeUnit.MILLISECONDS.sleep(waitTime);
  37. } catch (InterruptedException e) {
  38. Thread.currentThread().interrupt();
  39. }
  40. queue.offer(url);
  41. }
  42. }
  43. }
  45. private boolean isAllowedToRequest(URL url) {
  46. long lastRequestTime = lastRequestTimes.getOrDefault(url, 0L);
  47. long currentTime = System.currentTimeMillis();
  49. if (currentTime - lastRequestTime < timeoutMillis) {
  50. return false; // Request throttled
  51. }
  53. lastRequestTimes.put(url, currentTime);
  54. return true; // Request allowed
  55. }
  57. private long calculateWaitTime(URL url) {
  58. long lastRequestTime = lastRequestTimes.getOrDefault(url, 0L);
  59. long currentTime = System.currentTimeMillis();
  60. long timeSinceLastRequest = currentTime - lastRequestTime;
  62. return (1000 / maxRequestsPerSecond) * (maxRequestsPerSecond - (int) (timeSinceLastRequest / (1000.0/maxRequestsPerSecond)));
  63. }
  66. private Runnable fetchHtmlPage(URL url) {
  67. return () -> {
  68. try {
  69. // Simulate fetching the HTML page
  70. System.out.println("Fetching: " + url.toURI());
  71. TimeUnit.SECONDS.sleep(2); // Simulate network latency
  72. System.out.println("Fetched: " + url.toURI());
  73. } catch (InterruptedException e) {
  74. Thread.currentThread().interrupt();
  75. }
  76. };
  77. }
  80. public void shutdown() {
  81. executor.shutdown();
  82. try {
  83. executor.awaitTermination(5, TimeUnit.SECONDS);
  84. } catch (InterruptedException e) {
  85. Thread.currentThread().interrupt();
  86. }
  87. }
  89. public static void main(String[] args) throws MalformedURLException {
  90. RequestThrottler throttler = new RequestThrottler(2, 1000); // Allow 2 requests per second, 1 second timeout
  92. URL url1 = new URL("https://www.example.com");
  93. URL url2 = new URL("https://www.google.com");
  94. URL url3 = new URL("https://www.example.com"); // Same as url1
  96. throttler.enqueueRequest(url1);
  97. throttler.enqueueRequest(url2);
  98. throttler.enqueueRequest(url3);
  99. throttler.enqueueRequest(new URL("https://www.yahoo.com"));
  100. throttler.enqueueRequest(new URL("https://www.bing.com"));
  103. throttler.shutdown();
  104. }
  105. }

Add your comment