抓取下一页的抓取循环

import requests
from bs4 import BeautifulSoup

visited_urls = set()
base_url = "http://example.com/page/"
next_page = base_url

while next_page not in visited_urls:
    visited_urls.add(next_page)
    response = requests.get(next_page)
    soup = BeautifulSoup(response.text, 'html.parser')
    
    # 处理当前页面的数据
    print(f"Processing {next_page}")
    
    # 查找下一页链接
    next_link = soup.find('a', {'class': 'next-page'})
    if next_link:
        next_page = next_link.get('href')
        if not next_page.startswith('http'):
            next_page = base_url + next_page
    else:
        break  # 如果没有找到下一页链接，则退出循环

print("抓取完成")