不会 Python 没关系，手把手教你用 web scraper 抓取豆瓣电影 top 250 和 b 站排行榜

import requests
import re
import bs4
from selenium import webdriver
from matplotlib import pyplot as plt
from wordcloud import WordCloud
import jieba
import pandas as pd

def request_url(url):
    headers = {
        'Referer': 'https://movie.douban.com',
        'Host': 'movie.douban.com',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36'
    }
    response = requests.get(url, headers=headers)
    return response.text
def movie_info(html):
    data = []
    soup = bs4.BeautifulSoup(html, 'html.parser')
    items = soup.select('li > div.item')
    for item in items:
        desc_item = item.select('div.info > div.bd > p.quote > span')
        desc = ''
        if desc_item is not None and len(desc_item) > 0:
         desc = desc_item[0].text
        data.append({
         'url':item.select('div.info > div.hd > a')[0]['href'],
         'title':item.select('div.info > div.hd > a > span')[0].text,
            'rank':item.select('div.pic > em')[0].text,
            'score':item.select('div.info > div.bd > div.star > span.rating_num')[0].text,
            'desc':desc,
         })#{'url': 'https://movie.douban.com/subject/1292052/', 'title': '肖申克的救赎', 'rank': '1', 'score':
 '9.7', 'desc': '希望让人自由。'}
    return data

if __name__ == '__main__':
    urls = ['https://movie.douban.com/top250?start={0}&filter='.format(i * 25) for i in range(10)]
    movie_list = [request_url(url) for url in urls]
    movie_url_list = [movie_info(movie) for movie in movie_list]
    data = []
    for j in movie_url_list:
     for k in j:
      data.append(k)
    print(data)
    df = pd.DataFrame(data)
    df.to_csv("douban_movies.csv",encoding="utf_8_sig",index=False)

{"_id":"douban","startUrl":["https://movie.douban.com/top250?start=[0-250:25]&filter="],"selectors":[{"id":"row","type":"SelectorElement","parentSelectors":["_root"],"selector":".grid_view li","multiple":true,"delay":0},{"id":"电影名","type":"SelectorText","parentSelectors":["row"],"selector":"span.title","multiple":false,"regex":"","delay":0},{"id":"豆瓣链接","type":"SelectorLink","parentSelectors":["row"],"selector":".hd a","multiple":false,"delay":0},{"id":"电影排名","type":"SelectorText","parentSelectors":["row"],"selector":"em","multiple":false,"regex":"","delay":0},{"id":"电影简介","type":"SelectorText","parentSelectors":["row"],"selector":"span.inq","multiple":false,"regex":"","delay":0},{"id":"豆瓣评分","type":"SelectorText","parentSelectors":["row"],"selector":"span.rating_num","multiple":false,"regex":"","delay":0}]}

{"_id":"bilibili_rank","startUrl":["https://www.bilibili.com/v/popular/rank/all"],"selectors":[{"id":"row","type":"SelectorElement","parentSelectors":["_root"],"selector":"li.rank-item","multiple":true,"delay":0},{"id":"视频排名","type":"SelectorText","parentSelectors":["row"],"selector":"div.num","multiple":false,"regex":"","delay":0},{"id":"视频标题","type":"SelectorText","parentSelectors":["row"],"selector":"a.title","multiple":false,"regex":"","delay":0},{"id":"播放量","type":"SelectorText","parentSelectors":["row"],"selector":".detail > span:nth-of-type(1)","multiple":false,"regex":"","delay":0},{"id":"弹幕数","type":"SelectorText","parentSelectors":["row"],"selector":"span:nth-of-type(2)","multiple":false,"regex":"","delay":0},{"id":"up主","type":"SelectorText","parentSelectors":["row"],"selector":"a span","multiple":false,"regex":"","delay":0},{"id":"视频链接","type":"SelectorLink","parentSelectors":["row"],"selector":"a.title","multiple":false,"delay":0},{"id":"点赞数","type":"SelectorText","parentSelectors":["视频链接"],"selector":"span.like","multiple":false,"regex":"","delay":0},{"id":"投币数","type":"SelectorText","parentSelectors":["视频链接"],"selector":"span.coin","multiple":false,"regex":"","delay":0},{"id":"收藏数","type":"SelectorText","parentSelectors":["视频链接"],"selector":"span.collect","multiple":false,"regex":"","delay":0}]}