Python爬虫如何应对贝壳网的IP封禁与人机验证？

import requests
from fake_useragent import UserAgent

# 代理IP示例（实际使用时建议从代理池API获取）
PROXIES = {
    "http": "http://123.123.123.123:8888",
    "https": "http://123.123.123.123:8888"
}

headers = {
    "User-Agent": UserAgent().random
}

url = "https://www.ke.com"

try:
    response = requests.get(url, headers=headers, proxies=PROXIES, timeout=5)
    print(response.status_code)
except Exception as e:
    print("请求失败:", e)

import time
import random

def crawl_with_delay(url):
    try:
        time.sleep(random.uniform(1, 3))  # 随机延迟1~3秒
        response = requests.get(url, headers=headers, proxies=PROXIES)
        return response.text
    except Exception as e:
        print("请求失败:", e)
        return None

from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
import time

# 配置Chrome无头模式（可选）
options = webdriver.ChromeOptions()
options.add_argument("--headless")  # 无界面模式
options.add_argument("--disable-blink-features=AutomationControlled")

driver = webdriver.Chrome(options=options)
driver.get("https://www.ke.com")

# 找到滑块元素并模拟拖动
slider = driver.find_element_by_css_selector(".slider")  # 替换为实际滑块元素
action = ActionChains(driver)
action.click_and_hold(slider).move_by_offset(300, 0).release().perform()

time.sleep(2)
driver.quit()

import pytesseract
from PIL import Image
import requests
from io import BytesIO

# 下载验证码图片
response = requests.get("https://www.ke.com/captcha.jpg")
img = Image.open(BytesIO(response.content))
text = pytesseract.image_to_string(img)
print("识别结果:", text)

import requests
from selenium import webdriver
import time
import random
from selenium.webdriver.chrome.options import Options

# 16yun代理信息
proxyHost = "www.16yun.cn"
proxyPort = "5445"
proxyUser = "16QMSOML"
proxyPass = "280651"

# 代理IP池（包含认证信息）
PROXIES = [
    f"http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}",
    # 可以添加更多代理
]

def get_with_proxy(url):
    proxy = random.choice(PROXIES)
    try:
        # 设置代理
        proxies = {
            "http": proxy,
            "https": proxy
        }
        response = requests.get(url, proxies=proxies, timeout=5)
        if "验证码" in response.text:
            return handle_captcha(url)  # 处理验证码
        return response.text
    except Exception as e:
        print("请求失败:", e)
        return None

def handle_captcha(url):
    # 配置Chrome代理
    chrome_options = Options()
    
    # 添加代理认证
    proxy_auth = f"{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}"
    chrome_options.add_argument(f'--proxy-server=http://{proxy_auth}')
    
    # 其他Chrome配置
    chrome_options.add_argument("--disable-blink-features=AutomationControlled")
    # chrome_options.add_argument("--headless")  # 无头模式
    
    driver = webdriver.Chrome(options=chrome_options)
    driver.get(url)
    time.sleep(2)
    
    # 这里添加滑块/验证码处理逻辑
    # 示例：打印页面标题
    print("当前页面标题:", driver.title)
    
    page_source = driver.page_source
    driver.quit()
    return page_source

if __name__ == "__main__":
    data = get_with_proxy("https://www.ke.com")
    if data:
        print(data[:500])  # 打印部分内容