网络爬虫性能提升：requests.Session的会话持久化策略

python

import requests
from bs4 import BeautifulSoup

def fetch_data(url):
    # 创建Session对象
    session = requests.Session()
    # 设置请求头
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'
    }
    session.headers.update(headers)

    # 设置代理
    proxyHost = "www.16yun.cn"
    proxyPort = "5445"
    proxyUser = "16QMSOML"
    proxyPass = "280651"

    proxies = {
        "http": "http://" + proxyUser + ":" + proxyPass + "@" + proxyHost + ":" + proxyPort,
        "https": "https://" + proxyUser + ":" + proxyPass + "@" + proxyHost + ":" + proxyPort,
    }
    session.proxies.update(proxies)

    try:
        # 发送GET请求
        response = session.get(url, timeout=5)
        # 检查响应状态
        response.raise_for_status()
        # 解析HTML内容
        soup = BeautifulSoup(response.text, 'html.parser')
        # 提取标题
        titles = soup.find_all('h1')
        for title in titles:
            print(title.get_text())
    except requests.exceptions.RequestException as e:
        print(e)
    finally:
        # 关闭Session
        session.close()

# 目标URL
url = 'http://example.com'
fetch_data(url)