Kotlin与HttpClient编写视频爬虫

import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.HttpClients
import org.jsoup.Jsoup
import java.io.File
import java.io.FileOutputStream
​
fun main() {
    // 目标视频页面URL（需要替换为实际视频页面）
    val pageUrl = "https://example.com/video-page"
    
    // 创建HTTP客户端
    val httpClient = HttpClients.createDefault()
    
    try {
        // 1. 获取视频页面内容
        val pageResponse = httpClient.execute(HttpGet(pageUrl))
        val html = pageResponse.entity.content.bufferedReader().use { it.readText() }
​
        // 2. 解析HTML获取视频链接
        val doc = Jsoup.parse(html)
        
        // 查找视频标签（根据目标网站结构调整选择器）
        val videoElement = doc.selectFirst("video source[src]")
        val videoUrl = videoElement?.attr("src") ?: throw Exception("Video URL not found")
        
        // 处理相对路径
        val absoluteVideoUrl = if (videoUrl.startsWith("http")) videoUrl 
            else "${pageUrl.substringBefore("/", pageUrl)}/$videoUrl"
​
        println("Found video URL: $absoluteVideoUrl")
​
        // 3. 下载视频文件
        val videoResponse = httpClient.execute(HttpGet(absoluteVideoUrl))
        val contentType = videoResponse.entity.contentType.value
        val fileExtension = when {
            contentType.contains("mp4") -> "mp4"
            contentType.contains("webm") -> "webm"
            else -> "dat"
        }
​
        val outputFile = File("downloaded_video.$fileExtension")
        videoResponse.entity.content.use { input ->
            FileOutputStream(outputFile).use { output ->
                input.copyTo(output)
            }
        }
​
        println("Video downloaded to: ${outputFile.absolutePath}")
​
    } finally {
        httpClient.close()
    }
}

dependencies {
    implementation("org.apache.httpcomponents:httpclient:4.5.13")
    implementation("org.jsoup:jsoup:1.15.3")
}

// 可能需要改用支持JavaScript渲染的工具：
implementation("org.seleniumhq.selenium:selenium-java:4.4.0")