使用BeautifulSoup从多个页面进行多线程文件下载的网络抓取

import requests
from bs4 import BeautifulSoup
import threading

def download_file(url, filename):
    response = requests.get(url)
    with open(filename, 'wb') as file:
        file.write(response.content)

urls = ['url1', 'url2', 'url3']  # 替换为实际的URL列表
for url in urls:
    response = requests.get(url)
    soup = BeautifulSoup(response.content, 'html.parser')
    # 根据需要使用BeautifulSoup提取URL或其他信息

threads = []
for i, url in enumerate(urls):
    filename = f'file{i}.txt'  # 替换为实际的文件名
    thread = threading.Thread(target=download_file, args=(url, filename))
    thread.start()
    threads.append(thread)

# 等待所有线程完成
for thread in threads:
    thread.join()