使用Python抓取带有javascript格式的网站

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager

# 设置ChromeDriver服务
service = Service(ChromeDriverManager().install())

# 启动浏览器
driver = webdriver.Chrome(service=service)

# 打开目标网页
driver.get('https://example.com')

# 获取页面内容
page_content = driver.page_source

# 关闭浏览器
driver.quit()

print(page_content)

import asyncio
from pyppeteer import launch

async def main():
    browser = await launch()
    page = await browser.newPage()
    await page.goto('https://example.com')
    content = await page.content()
    await browser.close()
    return content

page_content = asyncio.get_event_loop().run_until_complete(main())
print(page_content)

pip install scrapy scrapy-splash

# settings.py
SPLASH_URL = 'http://localhost:8050'

DOWNLOADER_MIDDLEWARES = {
    'scrapy_splash.SplashCookiesMiddleware': 723,
    'scrapy_splash.SplashMiddleware': 725,
    'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,
}

SPIDER_MIDDLEWARES = {
    'scrapy_splash.SplashDeduplicateArgsMiddleware': 100,
}

DUPEFILTER_CLASS = 'scrapy_splash.SplashAwareDupeFilter'
HTTPCACHE_STORAGE = 'scrapy_splash.SplashAwareFSCacheStorage'

import scrapy
from scrapy_splash import SplashRequest

class ExampleSpider(scrapy.Spider):
    name = 'example'
    start_urls = ['https://example.com']

    def start_requests(self):
        for url in self.start_urls:
            yield SplashRequest(url, self.parse, args={'wait': 2})

    def parse(self, response):
        # 处理页面内容
        print(response.text)