相关行业发展趋势写一个爬虫程序

import requests
from bs4 import BeautifulSoup
import time
import pandas as pd
from collections import Counter
import jieba  # 中文分词库
​
# 配置参数（需根据目标网站结构调整）
BASE_URL = "https://36kr.com/hot-list/catalog"  # 示例网站，实际需替换为合法目标
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
    "Referer": "https://36kr.com/"
}
MAX_PAGES = 3  # 控制爬取页数
DELAY = 3  # 请求间隔（秒）
​
def crawl_industry_news():
    news_data = []
    
    for page in range(1, MAX_PAGES + 1):
        url = f"{BASE_URL}/page/{page}"
        try:
            response = requests.get(url, headers=HEADERS, timeout=10)
            response.raise_for_status()
            soup = BeautifulSoup(response.text, 'html.parser')
            
            # 定位新闻条目（根据实际页面结构调整选择器）
            articles = soup.find_all('div', class_='article-item')
            for article in articles:
                title = article.find('a', class_='title').text.strip()
                summary = article.find('div', class_='summary').text.strip()
                publish_time = article.find('span', class_='time').text.strip()
                link = article.find('a', class_='title')['href']
                
                news_data.append({
                    "title": title,
                    "summary": summary,
                    "time": publish_time,
                    "link": link
                })
            
            print(f"第 {page} 页爬取完成")
            time.sleep(DELAY)  # 控制频率
            
        except Exception as e:
            print(f"爬取失败: {e}")
            break
    
    # 保存为CSV
    df = pd.DataFrame(news_data)
    df.to_csv("industry_news.csv", index=False, encoding='utf-8-sig')
    return df
​
def analyze_trends(df):
    # 合并所有文本内容
    all_text = ' '.join(df['title'] + ' ' + df['summary'])
    
    # 中文分词与停用词过滤
    words = jieba.lcut(all_text)
    stopwords = set(['的', '是', '在', '和', '了', '等', '与', '为'])  # 自定义停用词表
    filtered_words = [word for word in words if len(word) > 1 and word not in stopwords]
    
    # 统计高频词
    word_counts = Counter(filtered_words)
    top_words = word_counts.most_common(20)
    print("行业高频关键词Top20:")
    for word, count in top_words:
        print(f"{word}: {count}次")
​
if __name__ == '__main__':
    df = crawl_industry_news()
    analyze_trends(df)

# 抓取政策文件标题和发布日期
def crawl_government_policies():
    url = "http://www.gov.cn/zhengce/zhengceku/"
    response = requests.get(url, headers=HEADERS)
    soup = BeautifulSoup(response.text, 'html.parser')
    policies = []
    
    for item in soup.select('.news_box .list li'):
        title = item.find('a').text.strip()
        date = item.find('span').text.strip()
        policies.append({"title": title, "date": date})
    
    return pd.DataFrame(policies)

# 需使用Selenium模拟浏览器（动态加载页面）
from selenium import webdriver
​
def crawl_patents(keyword="人工智能"):
    driver = webdriver.Chrome()
    driver.get("http://pss-system.cnipa.gov.cn/")
    driver.find_element_by_id("searchKey").send_keys(keyword)
    driver.find_element_by_id("searchBtn").click()
    time.sleep(5)  # 等待加载
    
    patents = []
    # 解析专利名称、申请号、申请人等数据
    # （需根据实际页面结构编写解析逻辑）
    
    driver.quit()
    return patents

# 需处理反爬机制（如加密参数）
def crawl_job_trends(keyword="数据分析"):
    url = "https://www.lagou.com/jobs/list_数据分析"
    headers = {..., "Cookie": "需自行获取有效Cookie"}
    response = requests.get(url, headers=headers)
    # 解析职位数量、薪资范围、技能要求等