​AI+代理IP手把手教你爬取某度

你是一个Python编程专家，要完成一个百度搜索页面爬取的Python脚本，具体任务如下：

1. 解析网页：https://www.baidu.com/s?rtt=1&bsst=1&cl=2&tn=news&ie=utf-8&word={keyword}&x_bfe_rqs=03E80&x_bfe_tjscore=0.100000&tngroupname=organic_news&newVideo=12&goods_entry_switch=1&rsv_dl=news_b_pn&pn={pagenumber}
其中{keyword}的值由用户输入决定；{pagenumber}的值从0开始，以10递增，到40结束。

2. 解析网页：定位其中所有class="result-op c-container xpath-log new-pmd"的div标签，定位div标签中class="news-title-font_1xS-F"的a标签，提取其href属性值作为网页下载URL，提取其aria-label属性值，作为网页文件名；定位div标签中class="c-font-normal c-color-text"的span标签，作为网页的内容摘要；

3. 网页下载URL、网页文件名、网页的内容摘要都写入到Excel文件中，Excel文件保存到与本程序相同的文件夹中，文件名为{keyword}的值加excel文件后缀。

4. 注意：每一步都要输出信息到屏幕上；每爬取一个网址后，随机暂停3-5秒；每下载一个网页后，随机暂停3-6秒；为这个程序设置请求标头：
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0
Accept: image/avif,image/webp,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5
Accept-Language: zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4,ru;q=0.2
Accept-Encoding: gzip, deflate, br, zstd
Connection: keep-alive
Sec-Fetch-Dest: image
Sec-Fetch-Mode: no-cors
Sec-Fetch-Site: same-origin

import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
import random
import os
from fake_useragent import UserAgent

def get_user_agent():
    return {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0',
        'Accept': 'image/avif,image/webp,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5',
        'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4,ru;q=0.2',
        'Accept-Encoding': 'gzip, deflate, br, zstd',
        'Connection': 'keep-alive',
        'Sec-Fetch-Dest': 'image',
        'Sec-Fetch-Mode': 'no-cors',
        'Sec-Fetch-Site': 'same-origin'
    }

def scrape_baidu_news(keyword):
    print(f"开始爬取关键词: {keyword}")
    
    # 准备存储数据的列表
    data = []
    
    # 从0开始，每次增加10，直到40
    for pn in range(0, 41, 10):
        print(f"正在处理第 {pn//10 + 1} 页...")
        
        # 构造URL
        url = f"https://www.baidu.com/s?rtt=1&bsst=1&cl=2&tn=news&ie=utf-8&word={keyword}&x_bfe_rqs=03E80&x_bfe_tjscore=0.100000&tngroupname=organic_news&newVideo=12&goods_entry_switch=1&rsv_dl=news_b_pn&pn={pn}"
        print(f"构造的URL: {url}")
        
        try:
            # 发送请求
            headers = get_user_agent()
            response = requests.get(url, headers=headers)
            response.raise_for_status()
            print(f"成功获取第 {pn//10 + 1} 页内容")
            
            # 解析HTML
            soup = BeautifulSoup(response.text, 'html.parser')
            
            # 查找所有结果项
            results = soup.find_all('div', class_='result-op c-container xpath-log new-pmd')
            print(f"在第 {pn//10 + 1} 页找到 {len(results)} 条结果")
            
            for result in results:
                # 提取标题和URL
                title_tag = result.find('a', class_='news-title-font_1xS-F')
                if title_tag:
                    url = title_tag.get('href', '')
                    title = title_tag.get('aria-label', '无标题')
                    
                    # 提取摘要
                    abstract_tag = result.find('span', class_='c-font-normal c-color-text')
                    abstract = abstract_tag.get_text(strip=True) if abstract_tag else '无摘要'
                    
                    # 添加到数据列表
                    data.append({
                        'URL': url,
                        '标题': title,
                        '摘要': abstract
                    })
                    
                    print(f"提取到结果: {title}")
                    
                    # 随机暂停3-6秒
                    sleep_time = random.uniform(3, 6)
                    print(f"等待 {sleep_time:.2f} 秒...")
                    time.sleep(sleep_time)
            
            # 随机暂停3-5秒
            sleep_time = random.uniform(3, 5)
            print(f"页面处理完成，等待 {sleep_time:.2f} 秒...")
            time.sleep(sleep_time)
            
        except Exception as e:
            print(f"处理第 {pn//10 + 1} 页时出错: {str(e)}")
            continue
    
    # 保存到Excel
    if data:
        df = pd.DataFrame(data)
        excel_filename = f"{keyword}.xlsx"
        df.to_excel(excel_filename, index=False)
        print(f"数据已保存到 {excel_filename}")
        
        # 获取文件完整路径
        file_path = os.path.abspath(excel_filename)
        print(f"文件保存路径: {file_path}")
    else:
        print("没有提取到任何数据，不创建Excel文件")

if __name__ == "__main__":
    keyword = input("请输入要搜索的关键词: ")
    scrape_baidu_news(keyword)

proxy = 'http://brd-customer-hl_a0a48734-zone-baidu:tt5c75qi712k@brd.superproxy.io:33335'
proxies = {'http': proxy,
         'https': proxy}