JSON数据解析实战：从嵌套结构到结构化表格

import requests
import json
import pandas as pd
from graphviz import Digraph

# ---------------------------
# 代理IP设置（参考亿牛云爬虫代理）
# 代理信息：域名(proxy.16yun.cn)、端口(8080)、用户名(16YUN)、密码(16IP)
# ---------------------------
proxies = {
    "http": "http://16YUN:16IP@proxy.16yun.cn:8080",
    "https": "http://16YUN:16IP@proxy.16yun.cn:8080"
}

# ---------------------------
# 请求头设置：模拟浏览器请求，包含Cookie和User-Agent
# ---------------------------
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " \
                  "(KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36",
    "Cookie": "your_cookie_here"  # 请替换为真实的Cookie
}

# ---------------------------
# 目标网址：Google Scholar
# ---------------------------
url = "https://scholar.google.com/"

try:
    # 发起请求（代理、Cookie、User-Agent 均已设置）
    response = requests.get(url, headers=headers, proxies=proxies, timeout=10)
    response.raise_for_status()
    # 假设目标页面中嵌入了包含文献信息的JSON数据（实际中可能需要解析HTML提取）
    # 此处仅用模拟数据演示
    print("成功获取页面内容！")
except Exception as e:
    print(f"请求失败：{e}")

# ---------------------------
# 模拟嵌套JSON数据结构（实际爬取后需解析页面提取）
# ---------------------------
json_data = {
    "results": [
         {
             "title": "Deep Learning for AI",
             "authors": ["Ian Goodfellow", "Yoshua Bengio", "Aaron Courville"],
             "journal": "Journal of AI Research",
             "keywords": ["deep learning", "neural networks", "machine learning"]
         },
         {
             "title": "Machine Learning in Action",
             "authors": ["Peter Harrington"],
             "journal": "Machine Learning Journal",
             "keywords": ["machine learning", "algorithm", "data science"]
         }
    ]
}

# ---------------------------
# 解析JSON数据，转换为结构化表格
# ---------------------------
records = []
for item in json_data.get("results", []):
    records.append({
        "标题": item.get("title", ""),
        "作者": ", ".join(item.get("authors", [])),
        "期刊": item.get("journal", ""),
        "关键词": ", ".join(item.get("keywords", []))
    })

# 利用 pandas 构造数据表
df = pd.DataFrame(records)
print("\n结构化表格数据：")
print(df)

# ---------------------------
# 制作技术关系图谱：以文献、作者、关键词之间的关系为例
# ---------------------------
dot = Digraph(comment='技术关系图谱')
# 添加节点（文献、作者、关键词）
for item in json_data.get("results", []):
    # 添加文献节点
    doc_node = item.get("title", "文献")
    dot.node(doc_node, doc_node, shape='box', color='blue')
    # 添加作者节点并连接
    for author in item.get("authors", []):
        dot.node(author, author, shape='ellipse', color='green')
        dot.edge(doc_node, author, label="作者")
    # 添加关键词节点并连接
    for keyword in item.get("keywords", []):
        dot.node(keyword, keyword, shape='diamond', color='red')
        dot.edge(doc_node, keyword, label="关键词")

# 将图谱保存为PDF（需要安装Graphviz软件）
dot.render('技术关系图谱', view=False)
print("\n技术关系图谱已生成：文件名为 '技术关系图谱.pdf'")