用TypeScript和library needle来创建视频爬虫程序

import needle from 'needle';
import cheerio from 'cheerio';
import path from 'path';
import fs from 'fs';

// 下载视频函数
async function downloadVideo(videoUrl: string, filename: string): Promise<void> {
  console.log(`开始下载视频: ${videoUrl}`);
  const response = await needle('get', videoUrl, { response_type: 'buffer' });
  const videoData = response.body;
  
  fs.writeFileSync(path.join(__dirname, 'downloads', filename), videoData);
  console.log(`视频已下载并保存为: ${filename}`);
}

// 抓取网页并提取视频链接
async function crawlAndDownloadVideos(url: string): Promise<void> {
  try {
    // 请求网页内容
    const response = await needle('get', url);
    const html = response.body;
    
    // 使用 cheerio 解析 HTML 内容
    const $ = cheerio.load(html);
    
    // 提取所有的视频链接，这里假设视频链接存放在 <video> 标签的 src 属性中
    const videoUrls: string[] = [];
    
    $('video').each((i, element) => {
      const videoUrl = $(element).attr('src');
      if (videoUrl) {
        // 处理相对路径的视频 URL
        if (!videoUrl.startsWith('http')) {
          const absoluteUrl = new URL(videoUrl, url).href;
          videoUrls.push(absoluteUrl);
        } else {
          videoUrls.push(videoUrl);
        }
      }
    });

    // 创建下载文件夹（如果没有的话）
    const downloadDir = path.join(__dirname, 'downloads');
    if (!fs.existsSync(downloadDir)) {
      fs.mkdirSync(downloadDir);
    }

    // 下载所有视频
    for (let i = 0; i < videoUrls.length; i++) {
      const videoUrl = videoUrls[i];
      const filename = `video_${i + 1}.mp4`; // 设置视频文件名
      await downloadVideo(videoUrl, filename);
    }
  } catch (error) {
    console.error('抓取过程中发生错误:', error);
  }
}

// 主函数
const url = 'https://example.com';  // 替换为你想抓取的网页
crawlAndDownloadVideos(url);