Kotlin编写Android爬虫教程

dependencies {
    implementation("org.jsoup:jsoup:1.15.3") // HTML解析库
    implementation("com.squareup.okhttp3:okhttp:4.10.0") // 网络请求
}

<uses-permission android:name="android.permission.INTERNET" />

import android.os.Bundle
import androidx.appcompat.app.AppCompatActivity
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import okhttp3.OkHttpClient
import okhttp3.Request
import org.jsoup.Jsoup
​
class MainActivity : AppCompatActivity() {
​
    override fun onCreate(savedInstanceState: Bundle?) {
        super.onCreate(savedInstanceState)
        setContentView(R.layout.activity_main)
​
        // 启动爬虫
        crawlData("https://books.toscrape.com/")
    }
​
    private fun crawlData(url: String) {
        CoroutineScope(Dispatchers.IO).launch {
            try {
                // 1. 获取网页HTML
                val html = fetchHtml(url)
​
                // 2. 解析数据
                val bookTitles = parseHtml(html)
​
                // 3. 主线程更新UI
                withContext(Dispatchers.Main) {
                    // 此处替换为你的UI更新代码
                    println("爬取结果: ${bookTitles.joinToString()}")
                }
            } catch (e: Exception) {
                e.printStackTrace()
            }
        }
    }
​
    private fun fetchHtml(url: String): String {
        val client = OkHttpClient()
        val request = Request.Builder().url(url).build()
        return client.newCall(request).execute().use { response ->
            if (!response.isSuccessful) throw Exception("请求失败: ${response.code}")
            response.body?.string() ?: throw Exception("空响应")
        }
    }
​
    private fun parseHtml(html: String): List<String> {
        val doc = Jsoup.parse(html)
        return doc.select("article.product_pod h3 a").map { it.attr("title") }
    }
}

// 如果需要处理动态加载内容（如JavaScript渲染）：
implementation("org.seleniumhq.selenium:selenium-android-driver:4.3.0")
​
// 处理分页
fun crawlMultiPage(baseUrl: String, pages: Int) {
    (1..pages).forEach { page ->
        crawlData("$baseUrl/page-$page.html")
    }
}
​
// 设置请求头模拟浏览器
val request = Request.Builder()
    .url(url)
    .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36...")
    .build()

delay(3000) // 每次请求间隔3秒

是否启动爬虫发送HTTP请求请求成功?Jsoup解析HTML错误处理CSS选择器提取数据处理/存储数据更新UI