利用Selenium和PhantomJS提升网页内容抓取与分析的效率

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
import time
import zipfile

# 代理信息
proxyHost = "www.16yun.cn"
proxyPort = "5445"
proxyUser = "16QMSOML"
proxyPass = "280651"

# 创建Chrome选项
chrome_options = Options()

# 创建代理认证插件
def create_proxy_auth_extension(proxy_host, proxy_port, proxy_username, proxy_password, scheme='http'):
    """创建带有代理认证的Chrome扩展"""
    
    # 插件配置
    manifest_json = """
    {
        "version": "1.0.0",
        "manifest_version": 2,
        "name": "Chrome Proxy",
        "permissions": [
            "proxy",
            "tabs",
            "unlimitedStorage",
            "storage",
            "<all_urls>",
            "webRequest",
            "webRequestBlocking"
        ],
        "background": {
            "scripts": ["background.js"]
        },
        "minimum_chrome_version":"22.0.0"
    }
    """
    
    background_js = """
    var config = {
        mode: "fixed_servers",
        rules: {
            singleProxy: {
                scheme: "%s",
                host: "%s",
                port: parseInt(%s)
            },
            bypassList: ["localhost"]
        }
    };
    
    chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});
    
    chrome.webRequest.onAuthRequired.addListener(
        function(details) {
            return {
                authCredentials: {
                    username: "%s",
                    password: "%s"
                }
            };
        },
        {urls: ["<all_urls>"]},
        ['blocking']
    );
    """ % (scheme, proxy_host, proxy_port, proxy_username, proxy_password)
    
    # 创建临时插件文件
    import tempfile
    import os
    from pathlib import Path
    
    plugin_dir = tempfile.mkdtemp()
    plugin_file = Path(plugin_dir) / "proxy_auth_plugin.zip"
    
    with zipfile.ZipFile(plugin_file, 'w') as zp:
        zp.writestr("manifest.json", manifest_json)
        zp.writestr("background.js", background_js)
    
    return plugin_file

# 创建代理认证插件
proxy_plugin = create_proxy_auth_extension(
    proxy_host=proxyHost,
    proxy_port=proxyPort,
    proxy_username=proxyUser,
    proxy_password=proxyPass
)

# 添加插件到Chrome选项
chrome_options.add_extension(proxy_plugin)

# 核心：启用无头模式
chrome_options.add_argument("--headless=new")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument("--window-size=1920,1080")

# 可选：禁用图片加载
prefs = {"profile.managed_default_content_settings.images": 2}
chrome_options.add_experimental_option("prefs", prefs)

# 初始化驱动
driver = webdriver.Chrome(options=chrome_options)

try:
    # 测试代理是否生效
    driver.get("https://httpbin.org/ip")
    page_source = driver.page_source
    print(page_source)
    print("代理设置成功！")
    
    # 您的实际爬虫代码
    # driver.get("https://example.com")
    # ... 其他操作
    
finally:
    driver.quit()
    # 清理临时文件
    import os
    import shutil
    plugin_dir = os.path.dirname(proxy_plugin)
    if os.path.exists(plugin_dir):
        shutil.rmtree(plugin_dir)

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

driver.get("https://example.com/dynamic-content")

try:
    # 显式等待：最多等10秒，直到指定元素出现在DOM中
    # 一旦元素出现，立即继续执行，无需等待完10秒
    target_element = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.ID, "dynamic-content-list"))
    )
    
    # 对找到的元素进行操作
    items = target_element.find_elements(By.TAG_NAME, "li")
    for item in items:
        print(item.text)
        
except Exception as e:
    print("元素未在指定时间内加载出来:", e)

# 假设页面有一个JS变量：var userData = {name: "John", id: 123};
user_data = driver.execute_script("return window.userData;")
print(user_data) # 直接得到Python字典：{'name': 'John', 'id': 123}

# 或者，用JS进行复杂的DOM查询或操作
all_links_text = driver.execute_script("""
    var links = Array.from(document.querySelectorAll('a'));
    return links.map(link => link.textContent);
""")
print(all_links_text)

from selenium_stealth import stealth
# ... 驱动初始化后
stealth(driver,
        languages=["en-US", "en"],
        vendor="Google Inc.",
        platform="Win32",
        webgl_vendor="Intel Inc.",
        renderer="Intel Iris OpenGL Engine",
        fix_hairline=True,
        )