Selenium与Web Scraping：自动化获取电影名称和评分的实战指南

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time

# 设置代理IP相关信息 爬虫代理加强版
proxy_host = "proxy.16yun.cn"  # 代理服务器
proxy_port = "3100"  # 代理端口
proxy_username = "你的用户名"
proxy_password = "你的密码"

# 配置 ChromeOptions
chrome_options = Options()
chrome_options.add_argument("--proxy-server=http://%s:%s@%s:%s" % (proxy_username, proxy_password, proxy_host, proxy_port))

# 设置 User-Agent 和禁用浏览器检测功能
chrome_options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36")
chrome_options.add_argument("--disable-blink-features=AutomationControlled")
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument("--headless")  # 如果你希望无界面运行
chrome_options.add_argument("--no-sandbox")

# 设置 Cookies（这里假设你已经有一些 Cookie）
cookies = {
    "name": "your_cookie_name",
    "value": "your_cookie_value",
    "domain": "douban.com"
}

# 启动 WebDriver
service = Service(executable_path="你的ChromeDriver路径")
driver = webdriver.Chrome(service=service, options=chrome_options)

# 打开目标网页
driver.get("https://movie.douban.com")

# 添加 Cookie
driver.add_cookie(cookies)

# 刷新页面以使 Cookies 生效
driver.refresh()

try:
    # 等待页面元素加载
    wait = WebDriverWait(driver, 10)
    movie_name_elements = wait.until(EC.presence_of_all_elements_located((By.XPATH, "//div[@class='title']")))
    rating_elements = wait.until(EC.presence_of_all_elements_located((By.XPATH, "//span[@class='rating_num']")))

    # 提取电影名称和评分
    movie_names = [element.text for element in movie_name_elements]
    ratings = [element.text for element in rating_elements]

    # 打印结果
    for name, rating in zip(movie_names, ratings):
        print(f"电影名称: {name}, 评分: {rating}")

finally:
    # 关闭 WebDriver
    driver.quit()