源代码和软件 | 采集 1w+ B 站视频评论（年底更新版）

# -*- coding: utf-8 -*-
# 作者:             inspurer(月小水长)
# 创建时间:          2020/10/30 23:48
# 运行环境           Python3.6+
# github            https://github.com/inspurer
# qq邮箱            2391527690@qq.com
# 微信公众号         月小水长(ID: inspurer)
# 文件备注信息       回复没有展开

import requests
import os
from time import sleep
import json
import time
import pandas as pd

import re

os.environ['NO_PROXY'] = 'bilibili.com'


def intToStrTime(a):
    b = time.localtime(a)  # 转为日期字符串
    c = time.strftime("%Y/%m/%d %H:%M:%S", b)  # 格式化字符串
    return c


headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3970.5 Safari/537.36',
    'Referer': 'https://www.bilibili.com/'
}


def validateTitle(title):
    re_str = r"[\/\\\:\*\?\"\<\>\|]"  # '/ \ : * ? " < > |'
    new_title = re.sub(re_str, "_", title)  # 替换为下划线
    return new_title


def main():
    save_folder = 'Comment'
    if not os.path.exists(save_folder):
        os.mkdir(save_folder)
    # bvs = ['BV1Zo4y1N7f1']
    input_bvs = input('请输入 bv 号，多个 bv 号以空格分隔：')
    bvs = input_bvs.split(' ')
    for bv in bvs:
        print(bv)
        response = requests.get(url='https://www.bilibili.com/video/{}'.format(bv), headers=headers)

        res = re.findall('<script>window.__INITIAL_STATE__=(.*)?;\(function\(\)', response.text, re.S)

        json_data = json.loads(res[0])
        if 'message' in json_data['error']:
            print(json_data['error']['message'])
            continue
        aid = json_data['aid']
        title = json_data['videoData']['title']

        title = validateTitle(title)

        comment_url = 'https://api.bilibili.com/x/v2/reply?callback=jQueryjsonp=jsonp&pn={}&type=1&oid={}&sort=2&_=1594459235799'

        response = requests.get(url=comment_url.format(1, aid), headers=headers)

        total_page = json.loads(response.text)['data']['page']['count'] // 20 + 1
        page = 1
        is_root, uname, comments, times, likes = [], [], [], [], []
        while True:
            data = json.loads(response.text)['data']['replies']
            print(data)
            if not data:
                data = json.loads(response.text)['data']
                if 'hots' in data.keys():
                    data = data['hots']
                else:
                    break
            for row in data:
                print('根评论', row['member']['uname'], row['content']['message'])
                is_root.append('是')
                times.append(intToStrTime(row['ctime']))
                uname.append(row['member']['uname'])
                comments.append(row['content']['message'])
                likes.append(row['like'])
                if row.get('replies'):
                    for crow in row['replies']:
                        is_root.append('否')
                        times.append(intToStrTime(crow['ctime']))
                        uname.append(crow['member']['uname'])
                        comments.append(crow['content']['message'])
                        likes.append(crow['like'])
                        print('---子评论', crow['member']['uname'], crow['content']['message'])

            page += 1
            if page > total_page:
                break
            sleep(1)
            response = requests.get(url=comment_url.format(page, aid), headers=headers)

            # 边爬取边保存
            df = pd.DataFrame(
                {'评论时间': times, '评论者': uname, '评论内容': [''.join(comment.split()) for comment in comments], '点赞数': likes})
            df.to_csv(f'{save_folder}/{title}.csv', encoding='utf-8-sig', index=False)

            print(f'\n\n已经保存 {df.shape[0]} 条评论到 {save_folder}/{title}.csv\n\n')

            sleep(1)

        # 每抓完 1 条视频的评论休眠 10s
        sleep(10)


if __name__ == '__main__':
    main()