Design a Distributed Web Crawler (Googlebot)

import asyncio
import time
from urllib.parse import urlparse

class PoliteCrawler:
    def __init__(self, min_interval=1.0):
        self.min_interval = min_interval
        self.last_fetch = {}
        self.lock = asyncio.Lock()

    async def crawl(self, target_url: str):
        host = urlparse(target_url).netloc
        async with self.lock:
            now = time.time()
            last_time = self.last_fetch.get(host, 0)
            wait_time = max(0, self.min_interval - (now - last_time))
            self.last_fetch[host] = now + wait_time

        if wait_time > 0:
            await asyncio.sleep(wait_time)

        print(f"[PYTHON CRAWL 200 OK] -> {target_url} (Host: {host})")

import java.net.URI;
import java.time.Instant;
import java.util.concurrent.ConcurrentHashMap;

public class PoliteCrawler {
    private final ConcurrentHashMap lastFetch = new ConcurrentHashMap<>();
    private final long minIntervalMs;

    public PoliteCrawler(long minIntervalMs) {
        this.minIntervalMs = minIntervalMs;
    }

    public synchronized void fetch(String urlStr) throws Exception {
        URI uri = new URI(urlStr);
        String host = uri.getHost();
        Instant now = Instant.now();
        Instant last = lastFetch.getOrDefault(host, Instant.EPOCH);

        long elapsed = java.time.Duration.between(last, now).toMillis();
        if (elapsed < minIntervalMs) {
            Thread.sleep(minIntervalMs - elapsed);
        }

        lastFetch.put(host, Instant.now());
        System.out.println("[JAVA CRAWL 200 OK] -> " + urlStr);
    }
}

use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::Mutex;

pub struct PoliteFrontier {
    last_fetch: Arc>>,
    min_interval: Duration,
}

impl PoliteFrontier {
    pub fn new(min_interval: Duration) -> Self {
        Self {
            last_fetch: Arc::new(Mutex::new(HashMap::new())),
            min_interval,
        }
    }

    pub async fn fetch(&self, host: &str, url: &str) {
        let mut map = self.last_fetch.lock().await;
        let now = Instant::now();
        if let Some(&last) = map.get(host) {
            if now.duration_since(last) < self.min_interval {
                let delay = self.min_interval - now.duration_since(last);
                tokio::time::sleep(delay).await;
            }
        }
        map.insert(host.to_string(), Instant::now());
        println!("[RUST CRAWL 200 OK] -> {}", url);
    }
}

import { URL } from 'url';

export class PoliteFrontier {
  private lastFetch = new Map();
  constructor(private minIntervalMs = 1000) {}

  async fetch(targetUrl: string): Promise {
    const host = new URL(targetUrl).hostname;
    const now = Date.now();
    const last = this.lastFetch.get(host) || 0;
    const waitTime = Math.max(0, this.minIntervalMs - (now - last));

    this.lastFetch.set(host, now + waitTime);
    if (waitTime > 0) {
      await new Promise(r => setTimeout(r, waitTime));
    }

    console.log(`[TS CRAWL 200 OK] -> ${targetUrl}`);
  }
}