php小说采集网站源码

// 设置目标网站的编码格式
header('Content-Type: text/html; charset=utf-8');
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type: text/html; charset=utf-8'));
$html = curl_exec($ch);
curl_close($ch);
$html = mb_convert_encoding($html, 'UTF-8', 'auto');

// 使用多线程或异步请求提高采集速度
use GuzzleHttp\Client;

$client = new Client(['timeout' => 5]);
$promises = [];
foreach ($urls as $url) {
    $promises[] = $client->getAsync($url);
}
$responses = GuzzleHttp\Promise\unwrap($promises);
foreach ($responses as $response) {
    $html = (string) $response->getBody();
    // 处理HTML内容
}

// 使用代理IP和User-Agent轮换
$proxy = 'http://your_proxy_ip:port';
$user_agents = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0',
    // 更多User-Agent
];

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_PROXY, $proxy);
curl_setopt($ch, CURLOPT_HTTPHEADER, array('User-Agent: ' . $user_agents[array_rand($user_agents)]));
$html = curl_exec($ch);
curl_close($ch);