小白如何正确使用爬虫代码

Python

import requests
import json

# 设置请求头，模拟浏览器
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'Cookie': '你的Cookie'
}

# 抖音视频API示例URL（需要根据实际分析得到）
url = 'https://www.douyin.com/aweme/v1/web/aweme/post/'

# 发送请求
response = requests.get(url, headers=headers)
if response.status_code == 200:
    data = response.json()
# 解析JSON数据，提取视频信息
# 例如：视频标题、点赞数、评论数等
for item in data['aweme_list']:
        title = item['desc']
        like_count = item['statistics']['digg_count']
print(f'标题: {title}, 点赞数: {like_count}')
else:
print('请求失败:', response.status_code)

Python

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
import time

# 设置Chrome选项
options = webdriver.ChromeOptions()
options.add_argument('--headless')  # 无头模式，不显示浏览器窗口
options.add_argument('--disable-gpu')
options.add_argument('user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36')

# 安装并启动ChromeDriver
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)

try:
    driver.get('https://www.douyin.com/')
# 等待页面加载
    time.sleep(5)  # 根据网络情况调整等待时间

# 通过Selenium选择器获取元素
# 注意：需要根据实际网页结构调整选择器
    video_list = driver.find_elements(By.CSS_SELECTOR, 'div.some-class')  # 替换为实际的CSS选择器
for video in video_list:
# 提取视频信息，例如标题
        title = video.find_element(By.CSS_SELECTOR, 'h2.title').text
print(f'标题: {title}')

finally:
    driver.quit()

Python

import csv

# 假设我们有一个数据列表，每个元素是一个字典
data_list = [
    {'title': '视频1', 'like_count': 100},
    {'title': '视频2', 'like_count': 200}
]

withopen('videos.csv', 'w', newline='', encoding='utf-8') as f:
    writer = csv.DictWriter(f, fieldnames=['title', 'like_count'])
    writer.writeheader()
    writer.writerows(data_list)

Python

# 基础工具链
import requests  # HTTP请求
from bs4 import BeautifulSoup  # HTML解析
from selenium import webdriver  # 动态页面渲染
import json  # 处理API返回数据
import re  # 正则表达式匹配

# 可选扩展库
from fake_useragent import UserAgent  # 随机UA
import pandas as pd  # 数据存储

Python

from selenium.webdriver.chrome.options import Options

# 配置无头浏览器
chrome_options = Options()
chrome_options.add_argument("--headless")  # 无界面模式
chrome_options.add_argument(f"user-agent={UserAgent().random}")  # 随机UA

driver = webdriver.Chrome(options=chrome_options)
driver.get("https://www.douyin.com/video/123456")

# 等待动态加载（显式/隐式等待）
driver.implicitly_wait(10)  

# 提取视频元数据
video_data = driver.execute_script("""
    return window.__INITIAL_STATE__.videoModule
""")  # 抖音的全局数据对象

Python

headers = {
"User-Agent": UserAgent().random,
"X-Requested-With": "XMLHttpRequest"
}

# 示例：快手视频API
api_url = "https://api.kuaishou.com/rest/r/feed"
params = {
"count": 20,
"page": 1,
"token": "xxx"# 需实际获取
}

response = requests.get(api_url, headers=headers, params=params)
video_list = response.json()["data"]["feeds"]

Python

defdownload_video(url, save_path):
    headers = {'Referer': 'https://www.target-domain.com/'}  # 关键：添加来源头
with requests.get(url, headers=headers, stream=True) as r:
        r.raise_for_status()
withopen(save_path, 'wb') as f:
for chunk in r.iter_content(chunk_size=8192): 
                f.write(chunk)

# 示例调用
download_video(video_list[0]["playAddr"], "video.mp4")

Python

# 请求间隔随机化
import random
import time

time.sleep(random.uniform(1.5, 3.0))  # 随机等待

Python

# CSV存储示例
df = pd.DataFrame({
"title": [v["title"] for v in video_list],
"views": [v["play_count"] for v in video_list],
"url": [v["video_url"] for v in video_list]
})
df.to_csv("videos.csv", index=False)

# MongoDB存储（推荐）
from pymongo import MongoClient
client = MongoClient('mongodb://localhost:27017/')
db = client['video_db']
db.videos.insert_many(video_list)

Python

title = title.encode('utf-8', 'ignore').decode('utf-8')