Fuel 爬虫：Scala 中的图片数据采集与分析

import akka.actor.ActorSystem
import akka.http.scaladsl.Http
import akka.http.scaladsl.model._
import akka.http.scaladsl.model.headers.{BasicHttpCredentials, HttpCredentials}
import akka.stream.scaladsl.FileIO
import akka.stream.ActorMaterializer
import org.jsoup.Jsoup
import cats.effect.IO
import cats.effect.unsafe.implicits.global

import java.io.File
import scala.concurrent.Future
import scala.concurrent.duration._
import scala.collection.JavaConverters._

// 代理服务器配置
val proxyHost = "www.16yun.cn"
val proxyPort = 5445
val proxyUser = "16QMSOML"
val proxyPass = "280651"

// 1. HTTP 客户端模块：用于发送请求并获取网页内容
object HttpClient {
  implicit val system = ActorSystem()
  implicit val materializer = ActorMaterializer()
  import system.dispatcher

  // 配置代理服务器
  val proxy = Http().outgoingConnectionHttps(proxyHost, proxyPort)
  val credentials: HttpCredentials = BasicHttpCredentials(proxyUser, proxyPass)

  def fetch(url: String): Future[String] = {
    val request = HttpRequest(uri = url).addCredentials(credentials)
    Http().singleRequest(request, proxy)
      .flatMap(response => response.entity.toStrict(5000.millis))
      .map(entity => entity.data.utf8String)
  }
}

// 2. HTML 解析模块：用于提取图片链接
object Parser {
  def extractImageUrls(html: String, baseUrl: String): List[String] = {
    val doc = Jsoup.parse(html, baseUrl)
    val elements = doc.select("img[src]")
    elements.asScala.map(_.attr("abs:src")).toList
  }
}

// 3. 图片下载模块：用于下载图片到本地
object ImageDownloader {
  def downloadImage(url: String, destination: String): Future[Unit] = {
    val request = HttpRequest(uri = url)
    val responseFuture = HttpClient.fetch(url) // 使用 HttpClient 发起请求
    responseFuture.flatMap { _ =>
      val destinationFile = new File(destination)
      Http().singleRequest(request, HttpClient.proxy)
        .flatMap(response => response.entity.dataBytes.runWith(FileIO.toPath(destinationFile.toPath)))
    }.map(_ => ())
  }
}

// 4. 并发下载模块：使用 Cats Effect 处理并发下载任务
object ConcurrentDownloader {
  def downloadImages(urls: List[String], destinationDir: String): IO[Unit] = {
    val tasks = urls.map { url =>
      val fileName = url.split("/").last
      val destination = s"$destinationDir/$fileName"
      IO.fromFuture(IO(ImageDownloader.downloadImage(url, destination)))
    }
    IO.parSequenceN(4)(tasks).void // 限制并发数为4
  }
}

// 5. 图片分析模块：用于分析下载的图片数据
object ImageAnalyzer {
  // 获取所有图片的大小
  def getImageSizes(directory: String): List[Long] = {
    val dir = new File(directory)
    dir.listFiles().filter(_.isFile).map(_.length()).toList
  }

  // 计算图片的平均大小
  def averageImageSize(sizes: List[Long]): Double = {
    sizes.sum.toDouble / sizes.length
  }

  // 统计图片格式
  def getImageFormats(directory: String): Map[String, Int] = {
    val dir = new File(directory)
    dir.listFiles().filter(_.isFile)
      .map(_.getName.split("\\.").last)
      .groupBy(identity)
      .mapValues(_.length)
  }
}

// 6. 主程序：整合所有模块功能
object FuelCrawler extends App {
  // 目标网站和存储目录
  val baseUrl = "https://example.com" // 替换为你要抓取的网站
  val destinationDir = "images" // 图片存储目录

  // 创建存储目录
  new File(destinationDir).mkdirs()

  // 发送 HTTP 请求并获取网页内容
  val htmlFuture = HttpClient.fetch(baseUrl)

  // 提取图片链接
  val imageUrlsFuture = htmlFuture.map(html => Parser.extractImageUrls(html, baseUrl))

  // 下载图片并进行分析
  imageUrlsFuture.flatMap { urls =>
    println(s"找到 ${urls.length} 张图片，开始下载...")
    ConcurrentDownloader.downloadImages(urls, destinationDir).unsafeToFuture()
  }.onComplete { _ =>
    println("图片下载完成！")

    // 分析图片数据
    val sizes = ImageAnalyzer.getImageSizes(destinationDir)
    val avgSize = ImageAnalyzer.averageImageSize(sizes)
    val formats = ImageAnalyzer.getImageFormats(destinationDir)

    println(s"平均图片大小: $avgSize 字节")
    println("图片格式统计:")
    formats.foreach { case (format, count) =>
      println(s"$format: $count 张")
    }

    // 关闭 Akka 系统
    ActorSystem().terminate()
  }
}