源代码和工具 | 2023 bilibili 视频弹幕爬虫，单条视频最多可爬取 10000 条弹幕

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3970.5 Safari/537.36',
    'Referer': 'https://www.bilibili.com/'
}

def timeFormatter(param):
    minute = int(param) // 60
    second = float(param) - minute * 60
    return f'{str(minute).zfill(2)}:{str(second).zfill(5)}'

def validateTitle(title):
    re_str = r"[\/\\\:\*\?\"\<\>\|]"  # '/ \ : * ? " < > |'
    new_title = re.sub(re_str, "_", title)  # 替换为下划线
    return new_title

def getHTML(url):
    try:
        response = requests.get(url=url, headers=headers,
                                timeout=timeout)
        # 自适应编码
        response.encoding = response.apparent_encoding
        return response.text
        # 下句作用等同于上两句
        # return response.text.encode(response.encoding).decode('utf-8')
    except:
        print(f"reqeuset url : {url} error...")
        print(traceback.format_exc())
        return None

# -*- coding: utf-8 -*-
# 作者:             inspurer(月小水长)
# 创建时间:          2020/10/30 23:16
# 运行环境           Python3.6+
# github            https://github.com/inspurer
# qq邮箱            2391527690@qq.com
# 微信公众号         月小水长(ID: inspurer)
# 文件备注信息       b 站弹幕爬虫

import requests
import re
from bs4 import BeautifulSoup
import operator
import traceback
import os
import pandas as pd
from lxml import etree
from time import sleep

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3970.5 Safari/537.36',
    'Referer': 'https://www.bilibili.com/'
}

timeout = 5


def getHTML(url):
    try:
        response = requests.get(url=url, headers=headers,
                                timeout=timeout)
        # 自适应编码
        response.encoding = response.apparent_encoding
        return response.text
        # 下句作用等同于上两句
        # return response.text.encode(response.encoding).decode('utf-8')
    except:
        print(f"reqeuset url : {url} error...")
        print(traceback.format_exc())
        return None


def parsePage(page):
    try:
        print("parsing...")
        html_ = etree.HTML(page)
        meta_title = html_.xpath('//meta[@name="title"]/@content')[0]
        if meta_title == '视频去哪了呢？_哔哩哔哩_bilibili':
            print(f'视频 404 not found')
            return [], '视频 404 not found'
        syntax = [':', '=']
        flag = 0
        keys = re.findall(r'"cid":[\d]*', page)
        if not keys:
            keys = re.findall(r'cid=[\d]*', page)
            flag = 1
        comments, title = {}, None
        keys = [keys[1]]
        for index, item in enumerate(keys):
            key = item.split(syntax[flag])[1]
            print(f'{index + 1}/{len(keys)}: {key}')
            comment_url = f'https://comment.bilibili.com/{key}.xml'  # 弹幕地址
            comment_text = getHTML(comment_url)
            bs4 = BeautifulSoup(comment_text, "html.parser")
            if not title:
                title = BeautifulSoup(page, "html.parser").find('h1').get_text().strip()
            for comment in bs4.find_all('d'):
                time = float(comment.attrs['p'].split(',')[0])
                time = timeFormatter(time)
                comments[time] = comment.string
        sorted_comments = sorted(comments.items(), key=operator.itemgetter(0))  # 排序
        comments = dict(sorted_comments)
        print("parse finish")
        return comments, title
    except:
        print("parse error")
        print(traceback.format_exc())


def validateTitle(title):
    re_str = r"[\/\\\:\*\?\"\<\>\|]"  # '/ \ : * ? " < > |'
    new_title = re.sub(re_str, "_", title)  # 替换为下划线
    return new_title


def timeFormatter(param):
    minute = int(param) // 60
    second = float(param) - minute * 60
    return f'{str(minute).zfill(2)}:{str(second).zfill(5)}'


def main():
    bvs = ['BV1mL411z7Kf', 'BV1CC4y1a7ee', 'BV1hx411e7KP', 'BV16F411B7Ek']
    for bv in bvs:
        url = f"https://www.bilibili.com/video/{bv}"
        save_folder = "BarRage"
        if not os.path.exists(save_folder):
            os.mkdir(save_folder)
        comments, title = parsePage(getHTML(url))
        if len(comments) == 0:
            continue
        title = validateTitle(title)
        df = pd.DataFrame({'时刻': list(comments.keys()), '弹幕文本': list(comments.values())})
        df.drop_duplicates(subset=['时刻', '弹幕文本'], keep='first', inplace=True)
        df.to_csv(f"{save_folder}/{title}.csv", index=False, encoding='utf-8-sig')
        print(f'已经保存 {df.shape[0]} 条弹幕到 {save_folder}/{title}.csv\n\n')

        sleep(10)


if __name__ == '__main__':
    main()