传统 Scrapy 只能抓取静态 HTML,而现代网站大量使用 React、Vue、Angular 等前端框架,内容通过 JavaScript 动态渲染,直接请求只能拿到一个空壳。Scrapy 负责调度和解析,Playwright 负责驱动真实浏览器执行 JS,两者结合正好互补。
# 创建虚拟环境(推荐)
python -m venv venv
source venv/bin/activate # Linux/macOS
# venv\Scripts\activate # Windows
# 安装依赖
pip install scrapy scrapy-playwright
# 安装 Playwright 浏览器内核
playwright install chromium创建项目:
scrapy startproject dynamic_scraper
cd dynamic_scraper
scrapy genspider products wlt.lyrxbj.com修改 settings.py,关键配置如下:
# settings.py
BOT_NAME = "dynamic_scraper"
SPIDER_MODULES = ["dynamic_scraper.spiders"]
# 核心:用 Playwright 处理 HTTP/HTTPS 请求
DOWNLOAD_HANDLERS = {
"http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
"https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
}
# 启用异步 reactor(Playwright 基于 asyncio)
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
# 浏览器配置
PLAYWRIGHT_BROWSER_TYPE = "chromium"
PLAYWRIGHT_LAUNCH_OPTIONS = {
"headless": True, # 无头模式,生产环境推荐
"timeout": 60 * 1000, # 60秒超时
}
# 多上下文配置(可选):模拟不同设备/语言
PLAYWRIGHT_CONTEXTS = {
"desktop": {
"viewport": {"width": 1920, "height": 1080},
"locale": "zh-CN",
"timezone_id": "Asia/Shanghai",
},
"mobile": {
"viewport": {"width": 375, "height": 667},
"user_agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X)",
},
}
# 礼貌爬取
CONCURRENT_REQUESTS = 4
DOWNLOAD_DELAY = 1
ROBOTSTXT_OBEY = True# spiders/products.py
import scrapy
from scrapy_playwright.page import PageMethod
class ProductsSpider(scrapy.Spider):
name = "products"
allowed_domains = ["wlt.lyrxbj.com"]
start_urls = ["https://wlt.lyrxbj.com/products"]
def start_requests(self):
for url in self.start_urls:
yield scrapy.Request(
url,
callback=self.parse,
meta={
"playwright": True,
"playwright_include_page": True, # 在 response 中保留 page 对象
"playwright_page_methods": [
# 等待商品卡片加载完成
PageMethod("wait_for_selector", ".product-card", timeout=10000),
# 等待网络空闲(AJAX 请求完成)
PageMethod("wait_for_load_state", "networkidle"),
],
},
errback=self.errback_close_page,
)
async def parse(self, response):
page = response.meta["playwright_page"]
for card in response.css(".product-card"):
yield {
"name": card.css(".product-name::text").get("").strip(),
"price": card.css(".price::text").get("").strip(),
"url": response.urljoin(card.css("a::attr(href)").get("")),
}
# 翻页
next_page = response.css(".pagination .next::attr(href)").get()
if next_page:
yield response.follow(
next_page,
callback=self.parse,
meta={
"playwright": True,
"playwright_include_page": True,
"playwright_page_methods": [
PageMethod("wait_for_selector", ".product-card"),
],
},
errback=self.errback_close_page,
)
await page.close() # 必须手动关闭
async def errback_close_page(self, failure):
page = failure.request.meta.get("playwright_page")
if page:
await page.close()
self.logger.error(f"Request failed: {failure.request.url}")yield scrapy.Request(
url,
meta={
"playwright": True,
"playwright_include_page": True,
"playwright_page_methods": [
# 执行 JS 滚动,触发懒加载
PageMethod(
"evaluate",
"for (let i = 0; i < 5; i++) "
"setTimeout(() => window.scrollTo(0, document.body.scrollHeight), i * 1500);"
),
# 等待最后一个元素出现
PageMethod("wait_for_selector", ".item:last-child"),
],
},
)yield scrapy.Request(
url,
meta={
"playwright": True,
"playwright_include_page": True,
"playwright_page_methods": [
PageMethod("click", "button.load-more"),
PageMethod("wait_for_selector", ".new-content"),
],
},
)yield scrapy.Request(
"https://example.com/login",
meta={
"playwright": True,
"playwright_include_page": True,
"playwright_page_methods": [
PageMethod("wait_for_load_state", "networkidle"),
PageMethod("fill", "input[name='username']", "myuser"),
PageMethod("fill", "input[name='password']", "mypass"),
PageMethod("click", "button[type='submit']"),
PageMethod("wait_for_selector", ".welcome-msg"),
],
},
callback=self.after_login,
)async def parse(self, response):
page = response.meta["playwright_page"]
await page.screenshot(path="debug.png", full_page=True) # 截图确认渲染状态
# ... 解析逻辑
await page.close()优化方向 | 做法 |
|---|---|
降低并发 | CONCURRENT_REQUESTS 设为 2–8,避免浏览器实例过多 |
阻塞无用资源 | 在 PLAYWRIGHT_CONTEXTS 中通过 route 拦截图片/字体/广告 |
复用浏览器上下文 | 使用 playwright_context 指定预定义上下文,避免重复启动 |
精确等待 | 用 wait_for_selector 替代 wait_for_timeout,减少无效等待 |
及时关闭 page | 每个请求结束后务必 await page.close(),防止内存泄漏 |
parse 必须声明为 async,否则无法 await page.close()。playwright_include_page=True 时必须手动关闭 page,否则浏览器进程会堆积。async_playwright() 手动管理浏览器,scrapy-playwright 官方下载器已处理好生命周期,自行实现容易踩异步/事件循环的坑。PageCoroutine 是旧版 API,当前版本应使用 PageMethod。scrapy crawl products -o products.json这套组合的核心思路就是:Scrapy 管调度,Playwright 管渲染。对于绝大多数 JS 动态渲染的场景,meta 里设好 playwright=True + PageMethod 等待条件,剩下的解析逻辑和普通 Scrapy 爬虫完全一致。
原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。
如有侵权,请联系 cloudcommunity@tencent.com 删除。