数据可视化实战：如何采集并分析马蜂窝上的热门旅游信息？

import asyncio
from playwright.async_api import async_playwright
import json
import os

# 网络代理配置（参考亿牛云爬虫代理示例）
PROXY_SERVER = "http://proxy.16yun.cn:3100"
PROXY_USERNAME = "16YUN"
PROXY_PASSWORD = "16IP"

KEYWORD = "杭州"

os.makedirs("mafengwo_data", exist_ok=True)

async def run():
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)
        context = await browser.new_context(
            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/114.0.0.0 Safari/537.36",
            proxy={
                "server": PROXY_SERVER,
                "username": PROXY_USERNAME,
                "password": PROXY_PASSWORD
            }
        )

        page = await context.new_page()
        await page.goto("https://www.mafengwo.cn/")
        await page.context.add_cookies([{
            'name': 'mfw_uuid', 'value': 'example-uuid', 'domain': '.mafengwo.cn', 'path': '/'
        }])

        # 输入关键词并搜索
        await page.fill('input[id="search-input"]', KEYWORD)
        await page.click('div.search-btn')
        await page.wait_for_timeout(3000)

        # 提取搜索结果中的景点卡片
        scenic_cards = await page.locator('div[class*="list_mod"]').all()
        results = []

        for card in scenic_cards:
            title = await card.locator('h3').inner_text()
            link = await card.locator('a').get_attribute('href')
            results.append({
                "景点": title.strip(),
                "链接": f"https://www.mafengwo.cn{link}" if link.startswith('/') else link
            })

        # 保存每个景点信息为JSON文件
        for r in results:
            filename = os.path.join("mafengwo_data", f"{r['景点']}.json")
            with open(filename, "w", encoding="utf-8") as f:
                json.dump(r, f, ensure_ascii=False, indent=2)

        await browser.close()

asyncio.run(run())

import os
import json
import matplotlib.pyplot as plt
from collections import Counter
from wordcloud import WordCloud

files = os.listdir("mafengwo_data")
scenic_counts = Counter()
qa_keywords = Counter()

for file in files:
    with open(os.path.join("mafengwo_data", file), "r", encoding="utf-8") as f:
        data = json.load(f)
        scenic_counts[data["景点"]] += 1
        # 模拟关键词统计（实际可根据内容语义提取）
        qa_keywords.update(["交通", "门票", "住宿", "路线"])

# 热门景点Top10
plt.figure(figsize=(10, 5))
names, counts = zip(*scenic_counts.most_common(10))
plt.bar(names, counts, color='skyblue')
plt.title("热门景点排行")
plt.ylabel("次数")
plt.xticks(rotation=45)
plt.tight_layout()
plt.savefig("hot_scenic.png")
plt.close()

# 关键词词云图
wc = WordCloud(font_path="msyh.ttc", background_color="white", width=800, height=400)
wc.generate_from_frequencies(qa_keywords)
wc.to_file("qa_wordcloud.png")