医疗爬虫实战：手把手教你抓取丁香园药品信息库

import requests
from lxml import etree
import pandas as pd

def fetch_drug_data(drug_url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
    }
    
    try:
        response = requests.get(drug_url, headers=headers, timeout=10)
        response.raise_for_status()
        html = etree.HTML(response.text)
        
        # 提取数据
        name = html.xpath('//h1[@class="drug-name"]/text()')[0].strip()
        ingredient = html.xpath('//div[@class="ingredient"]//text()')
        ingredient = ''.join([i.strip() for i in ingredient if i.strip()])
        
        # 其他字段提取类似...
        
        return {
            '药品名称': name,
            '成分': ingredient,
            # 其他字段...
        }
        
    except Exception as e:
        print(f"抓取失败: {e}")
        return None

# 示例调用
drug_url = "https://drugs.dxy.cn/drug/123456.htm"
data = fetch_drug_data(drug_url)
if data:
    df = pd.DataFrame([data])
    df.to_csv('drug_data.csv', index=False, encoding='utf_8_sig')

from proxypool import ProxyPool

pool = ProxyPool()  # 初始化代理池

def fetch_with_proxy(url):
    proxy = pool.get_proxy()  # 获取代理
    proxies = {
        'http': f'http://{proxy}',
        'https': f'https://{proxy}'
    }
    try:
        response = requests.get(url, proxies=proxies, timeout=10)
        if response.status_code == 200:
            return response.text
        else:
            pool.mark_invalid(proxy)  # 标记无效代理
            return fetch_with_proxy(url)  # 递归重试
    except:
        pool.mark_invalid(proxy)
        return fetch_with_proxy(url)

def get_random_headers():
    return {
        'User-Agent': random.choice([
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36...',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15...'
        ]),
        'Accept-Language': random.choice(['zh-CN,zh;q=0.9', 'en-US,en;q=0.8']),
        'Referer': 'https://drugs.dxy.cn/'
    }

import time
import random

def fetch_with_delay(url):
    delay = random.uniform(1, 3)  # 随机延迟1-3秒
    time.sleep(delay)
    return fetch_drug_data(url)  # 使用基础爬虫函数

def clean_data(raw_data):
    # 去除HTML标签
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(raw_data['适应症'], 'html.parser')
    clean_text = soup.get_text(strip=True)
    
    # 单位统一
    if 'mg' in raw_data['剂量']:
        raw_data['剂量_g'] = float(raw_data['剂量'].replace('mg', '')) / 1000
    
    # 填充缺失值
    raw_data['禁忌症'] = raw_data.get('禁忌症', '未提及')
    
    return raw_data

import requests
from lxml import etree
import pandas as pd
import random
import time
from proxypool import ProxyPool

class DrugSpider:
    def __init__(self):
        self.pool = ProxyPool()
        self.base_url = "https://drugs.dxy.cn/drug/{}.htm"
        
    def get_proxy(self):
        return self.pool.get_proxy()
        
    def fetch_page(self, drug_id):
        url = self.base_url.format(drug_id)
        proxy = self.get_proxy()
        proxies = {'http': f'http://{proxy}', 'https': f'https://{proxy}'}
        
        try:
            headers = self.get_random_headers()
            time.sleep(random.uniform(1, 3))
            response = requests.get(url, headers=headers, proxies=proxies, timeout=10)
            response.raise_for_status()
            return response.text
        except Exception as e:
            print(f"抓取失败: {e}")
            return None
            
    def parse_page(self, html):
        if not html:
            return None
            
        html = etree.HTML(html)
        data = {
            '名称': html.xpath('//h1[@class="drug-name"]/text()')[0].strip(),
            '成分': ''.join([i.strip() for i in html.xpath('//div[@class="ingredient"]//text()') if i.strip()]),
            # 其他字段解析...
        }
        return data
        
    def run(self, drug_ids):
        results = []
        for drug_id in drug_ids:
            html = self.fetch_page(drug_id)
            data = self.parse_page(html)
            if data:
                results.append(data)
                
        df = pd.DataFrame(results)
        df.to_csv('drug_data.csv', index=False, encoding='utf_8_sig')
        print(f"成功抓取{len(results)}条药品数据")

# 使用示例
spider = DrugSpider()
drug_ids = ['123456', '654321', '789012']  # 实际应从列表获取
spider.run(drug_ids)