第一次爬虫尝试

import re
from datetime import datetime
from os import error
from urllib import request, error
try:
    page = 1
    video_url = []
    code = 0
    file_number = 1
    pages = 10
    cid = 275359
    for a in range(pages):
        while True:
            resp = request.urlopen("http://www.leleketang.com/let3/knowledge_list.php?cid={}&p={}".format(cid, page))
            html = resp.read().decode("utf-8")
            video_url = re.findall("http://.+.mp4", html)
            if len(video_url) != 0:
                break
            else:
                video_url = []
        print(video_url)
        print("共有{}个视频正在下载".format(len(video_url)))
        for img in video_url:
            file_name = "/Users/yejizhi/Desktop/video/{}.mp4".format(file_number)
            request.urlretrieve(img, file_name)
            file_number = file_number+1
        print("{}个视频已下载完成".format(len(video_url)))
        video_url = []
        page = page+1
        
except error.HTTPError as e:
    print("error:{}".format(e.code))

import json
from urllib import request
import time,random
import csv

all_athlete = []

for page in range(1, 29):
    url = "https://gw.m.163.com/olympic/2020/athleteList/chinaAllItem_{}.json".format(page)
    resp = request.urlopen(url)
    data = resp.read().decode("utf-8")
    data_obj = json.loads(data) #json转python
    athleteLst = data_obj["data"]["ul"]
    for athlete in athleteLst:
        d_url = "https://gw.m.163.com/olympic/2020/athlete/{}.json".format(athlete["athleteId"])
        d_resp = request.urlopen(d_url)
        d_data = json.loads(d_resp.read().decode("utf-8"))
        #print(d_data)
        athlete_info = d_data["data"]["athleteInfo"]
        node = {}
        node["name"] = athlete["athleteName"]
        info_lst = athlete_info.split("#")
        for info_item in info_lst:
            item_data = info_item.split("：")
            if item_data[0] == "姓名":
                node["sex"] = item_data[1]
            elif item_data[0] == "生日":
                node["birthday"] = item_data[1]
            elif item_data[0] == "项目":
                node["item"] = item_data[1]
        
        all_athlete.append(node)
        print("=", end="", flush=True)
        time.sleep(random.randint(1,3))

with open("/Users/yejizhi/Desktop/athlete.csv", "w", encoding = "utf-8", newline = "") as f:
    field_name = ["name","sex","birthday","item"]
    writer = csv.DictWriter(f, fieldnames = field_name)

    for athlete in all_athlete:
        writer.writerow(athlete)
print("爬虫结束"）