scrapy笔记六 scrapy运行架构的实例配合解析

import scrapy
class MeizituItem(scrapy.Item):
    url = scrapy.Field()
    name = scrapy.Field()
    tags = scrapy.Field()
    image_urls = scrapy.Field()
    images = scrapy.Field()

def parse_item(self, response):
        #l=用ItemLoader载入MeizituItem()
        l = ItemLoader(item=MeizituItem(), response=response)
        #名字
        l.add_xpath('name', '//h2/a/text()')
        #标签
        l.add_xpath('tags', "//div[@id='maincontent']/div[@class='postmeta  clearfix']/div[@class='metaRight']/p")
        #图片连接
        l.add_xpath('image_urls', "//div[@id='picture']/p/img/@src", Identity())
        #url
        l.add_value('url', response.url)
 
        return l.load_item()

<span class="n">ITEM_PIPELINES</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'scrapy.pipeline.<span style="color: #ff0000;">images</span>.<span style="color: #ff0000;">Images</span>Pipeline'</span><span class="p">:</span> <span class="mi">1</span><span class="p">}</span>

<span class="n">ITEM_PIPELINES</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'scrapy.pipeline.<span style="color: #ff0000;">files</span>.<span style="color: #ff0000;">Files</span>Pipeline'</span><span class="p">:</span> <span class="mi">1</span><span class="p">}</span>

BOT_NAME = 'meizitu'
 
SPIDER_MODULES = ['meizitu.spiders']
NEWSPIDER_MODULE = 'meizitu.spiders'
#载入ImageDownLoadPipeline类
#为了启用一个Item Pipeline组件，你必须将它的类添加到 ITEM_PIPELINES 配置
#分配给每个类的整型值，确定了他们运行的顺序，item按数字从低到高的顺序，通过pipeline，
TEM_PIPELINES = {'meizitu.pipelines.ImageDownloadPipeline': 1}
#图片储存,第一个meizitu是项目名称,imagedown部分则是类的名字.
IMAGES_STORE = '.'

# -*- coding: utf-8 -*-
 
#图片下载部分（自动增量）
import requests
from meizitu import settings
import os
 
#导入重写图片下载类
class ImageDownloadPipeline(object):
    def process_item(self, item, spider):
        #每个item pipeline组件都需要调用该方法，这个方法必须返回一个 Item (或任何继承类)对象，
        # 或是抛出 DropItem 异常，被丢弃的item将不会被之后的pipeline组件所处理
        if 'image_urls' in item:#如果‘图片地址’在项目中
            images = []#定义图片空集
            
            dir_path = '%s/%s' % (settings.IMAGES_STORE, spider.name)
 
            #建立目录名字和项目名称一致
            if not os.path.exists(dir_path):
                os.makedirs(dir_path)
    
            #根据item字典进行查询   
            for image_url in item['image_urls']:
                us = image_url.split('/')[3:]
                image_file_name = '_'.join(us)
                file_path = '%s/%s' % (dir_path, image_file_name)
                images.append(file_path)
 
            #如果这个文件存在则跳过
                if os.path.exists(file_path):
                    continue
            #进行图片文件写入,wb模式打开文件,然后requests.get获取图片流,
                with open(file_path, 'wb') as handle:
                    response = requests.get(image_url, stream=True)
                    for block in response.iter_content(1024):
                        #获取的流如果有不存在的,则使用break结束,如果没有一次结束则进行写入
                        if not block:
                            break
 
                        handle.write(block)
 
            item['images'] = images
           
        return item

# -*- coding: utf-8 -*-
import scrapy
from scrapy.selector import Selector
from scrapy.contrib.loader import ItemLoader, Identity
from meizitu.items import MeizituItem
 
 
class MeiziSpider(scrapy.Spider):
    name = "meizi"
    allowed_domains = ["meizitu.com"]
    start_urls = (
        'http://www.meizitu.com/',
    )
 
    def parse(self, response):
        #sel是页面源代码，载入scrapy.selector
        sel = Selector(response)
        #每个连接，用@href属性
        for link in sel.xpath('//h2/a/@href').extract():
            #请求=Request(连接，parese_item)
            request = scrapy.Request(link, callback=self.parse_item)
            yield request#返回请求
        #获取页码集合
        pages = sel.xpath('//*[@id="wp_page_numbers"]/ul/li/a/@href').extract()
        print('pages: %s' % pages)#打印页码
        if len(pages) > 2:#如果页码集合>2
            page_link = pages[-2]#图片连接=读取页码集合的倒数第二个页码
            page_link = page_link.replace('/a/', '')#图片连接=page_link（a替换成空）
            request = scrapy.Request('http://www.meizitu.com/a/%s' % page_link, callback=self.parse)
            yield request#返回请求
 
    def parse_item(self, response):
        #l=用ItemLoader载入MeizituItem()
        l = ItemLoader(item=MeizituItem(), response=response)
        #名字
        l.add_xpath('name', '//h2/a/text()')
        #标签
        l.add_xpath('tags', "//div[@id='maincontent']/div[@class='postmeta  clearfix']/div[@class='metaRight']/p")
        #图片连接
        l.add_xpath('image_urls', "//div[@id='picture']/p/img/@src", Identity())
        #url
        l.add_value('url', response.url)
 
        return l.load_item()