php多域名蜘蛛

<?php
class MultiDomainCrawler {
    private $domains;
    private $maxDepth;

    public function __construct($domains, $maxDepth = 3) {
        $this->domains = $domains;
        $this->maxDepth = $maxDepth;
    }

    public function crawl() {
        foreach ($this->domains as $domain) {
            $this->crawlDomain($domain, 0);
        }
    }

    private function crawlDomain($domain, $depth) {
        if ($depth > $this->maxDepth) {
            return;
        }

        $url = "http://$domain";
        $html = $this->fetch($url);

        if ($html) {
            echo "Crawled: $url\n";
            // 解析页面并提取链接
            $links = $this->extractLinks($html);
            foreach ($links as $link) {
                if (strpos($link, $domain) !== false) {
                    $this->crawlDomain($link, $depth + 1);
                }
            }
        }
    }

    private function fetch($url) {
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $url);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (compatible; MultiDomainCrawler/1.0)');
        $html = curl_exec($ch);
        curl_close($ch);
        return $html;
    }

    private function extractLinks($html) {
        preg_match_all('/<a\s+(?:[^>]*?\s+)?href="([^"]*)"/i', $html, $matches);
        return $matches[1];
    }
}

$domains = ['example1.com', 'example2.com'];
$crawler = new MultiDomainCrawler($domains);
$crawler->crawl();
?>