代理ip池搭建

import requests
import redis
import threading

# 初始化Redis连接
redis_client = redis.StrictRedis(host='localhost', port=6379, db=0)

# 收集代理IP
def fetch_proxies():
    # 这里可以替换为从免费网站或付费服务获取代理IP的逻辑
    proxies = [
        'http://1.2.3.4:8080',
        'http://5.6.7.8:8080',
        # 更多代理IP...
    ]
    return proxies

# 验证代理IP
def check_proxy(proxy):
    try:
        response = requests.get('http://httpbin.org/ip', proxies={"http": proxy, "https": proxy}, timeout=5)
        if response.status_code == 200:
            return True
    except:
        pass
    return False

# 存储代理IP到Redis
def store_proxies(proxies):
    for proxy in proxies:
        if check_proxy(proxy):
            redis_client.sadd('proxies', proxy)

# 轮询调度代理IP
def get_proxy():
    return redis_client.srandmember('proxies').decode('utf-8')

# 定期更新代理IP池
def update_proxies_periodically():
    while True:
        proxies = fetch_proxies()
        store_proxies(proxies)
        threading.Event().wait(3600)  # 每小时更新一次

# 启动定期更新线程
threading.Thread(target=update_proxies_periodically).start()

# 使用示例
if __name__ == '__main__':
    proxy = get_proxy()
    print(f'Using proxy: {proxy}')
    response = requests.get('http://httpbin.org/ip', proxies={"http": proxy, "https": proxy})
    print(response.json())