三、scrapy后续 LinkExtractorsrules Logging发送POST请求内置设置参考手册

scrapy genspider -t crawl tencent tencent.com

 1 class CrawlSpider(Spider):
 2     rules = ()
 3     def __init__(self, *a, **kw):
 4         super(CrawlSpider, self).__init__(*a, **kw)
 5         self._compile_rules()
 6 
 7     #首先调用parse()来处理start_urls中返回的response对象
 8     #parse()则将这些response对象传递给了_parse_response()函数处理，并设置回调函数为parse_start_url()
 9     #设置了跟进标志位True
10     #parse将返回item和跟进了的Request对象    
11     def parse(self, response):
12         return self._parse_response(response, self.parse_start_url, cb_kwargs={}, follow=True)
13 
14     #处理start_url中返回的response，需要重写
15     def parse_start_url(self, response):
16         return []
17 
18     def process_results(self, response, results):
19         return results
20 
21     #从response中抽取符合任一用户定义'规则'的链接，并构造成Resquest对象返回
22     def _requests_to_follow(self, response):
23         if not isinstance(response, HtmlResponse):
24             return
25         seen = set()
26         #抽取之内的所有链接，只要通过任意一个'规则'，即表示合法
27         for n, rule in enumerate(self._rules):
28             links = [l for l in rule.link_extractor.extract_links(response) if l not in seen]
29             #使用用户指定的process_links处理每个连接
30             if links and rule.process_links:
31                 links = rule.process_links(links)
32             #将链接加入seen集合，为每个链接生成Request对象，并设置回调函数为_repsonse_downloaded()
33             for link in links:
34                 seen.add(link)
35                 #构造Request对象，并将Rule规则中定义的回调函数作为这个Request对象的回调函数
36                 r = Request(url=link.url, callback=self._response_downloaded)
37                 r.meta.update(rule=n, link_text=link.text)
38                 #对每个Request调用process_request()函数。该函数默认为indentify，即不做任何处理，直接返回该Request.
39                 yield rule.process_request(r)
40 
41     #处理通过rule提取出的连接，并返回item以及request
42     def _response_downloaded(self, response):
43         rule = self._rules[response.meta['rule']]
44         return self._parse_response(response, rule.callback, rule.cb_kwargs, rule.follow)
45 
46     #解析response对象，会用callback解析处理他，并返回request或Item对象
47     def _parse_response(self, response, callback, cb_kwargs, follow=True):
48         #首先判断是否设置了回调函数。（该回调函数可能是rule中的解析函数，也可能是 parse_start_url函数）
49         #如果设置了回调函数（parse_start_url()），那么首先用parse_start_url()处理response对象，
50         #然后再交给process_results处理。返回cb_res的一个列表
51         if callback:
52             #如果是parse调用的，则会解析成Request对象
53             #如果是rule callback，则会解析成Item
54             cb_res = callback(response, **cb_kwargs) or ()
55             cb_res = self.process_results(response, cb_res)
56             for requests_or_item in iterate_spider_output(cb_res):
57                 yield requests_or_item
58 
59         #如果需要跟进，那么使用定义的Rule规则提取并返回这些Request对象
60         if follow and self._follow_links:
61             #返回每个Request对象
62             for request_or_item in self._requests_to_follow(response):
63                 yield request_or_item
64 
65     def _compile_rules(self):
66         def get_method(method):
67             if callable(method):
68                 return method
69             elif isinstance(method, basestring):
70                 return getattr(self, method, None)
71 
72         self._rules = [copy.copy(r) for r in self.rules]
73         for rule in self._rules:
74             rule.callback = get_method(rule.callback)
75             rule.process_links = get_method(rule.process_links)
76             rule.process_request = get_method(rule.process_request)
77 
78     def set_crawler(self, crawler):
79         super(CrawlSpider, self).set_crawler(crawler)
80         self._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)

class scrapy.linkextractors.LinkExtractor

 1 class scrapy.linkextractors.LinkExtractor(
 2     allow = (),
 3     deny = (),
 4     allow_domains = (),
 5     deny_domains = (),
 6     deny_extensions = None,
 7     restrict_xpaths = (),
 8     tags = ('a','area'),
 9     attrs = ('href'),
10     canonicalize = True,
11     unique = True,
12     process_value = None
13 )

1 allow：满足括号中“正则表达式”的值会被提取，如果为空，则全部匹配。
2 
3 deny：与这个正则表达式(或正则表达式列表)不匹配的URL一定不提取。
4 
5 allow_domains：会被提取的链接的domains。
6 
7 deny_domains：一定不会被提取链接的domains。
8 
9 restrict_xpaths：使用xpath表达式，和allow共同作用过滤链接。

1 class scrapy.spiders.Rule(
2         link_extractor, 
3         callback = None, 
4         cb_kwargs = None, 
5         follow = None, 
6         process_links = None, 
7         process_request = None
8 )

LOG_FILE = "TencentSpider.log"
LOG_LEVEL = "INFO"

 1 # 部分代码
 2 class Request(object_ref):
 3 
 4     def __init__(self, url, callback=None, method='GET', headers=None, body=None, 
 5                  cookies=None, meta=None, encoding='utf-8', priority=0,
 6                  dont_filter=False, errback=None):
 7 
 8         self._encoding = encoding  # this one has to be set first
 9         self.method = str(method).upper()
10         self._set_url(url)
11         self._set_body(body)
12         assert isinstance(priority, int), "Request priority not an integer: %r" % priority
13         self.priority = priority
14 
15         assert callback or not errback, "Cannot use errback without a callback"
16         self.callback = callback
17         self.errback = errback
18 
19         self.cookies = cookies or {}
20         self.headers = Headers(headers or {}, encoding=encoding)
21         self.dont_filter = dont_filter
22 
23         self._meta = dict(meta) if meta else None
24 
25     @property
26     def meta(self):
27         if self._meta is None:
28             self._meta = {}
29         return self._meta

url: 就是需要请求，并进行下一步处理的url

callback: 指定该请求返回的Response，由那个函数来处理。

method: 请求一般不需要指定，默认GET方法，可设置为"GET", "POST", "PUT"等，且保证字符串大写

headers: 请求时，包含的头文件。一般不需要。内容一般如下：
        # 自己写过爬虫的肯定知道
        Host: media.readthedocs.org
        User-Agent: Mozilla/5.0 (Windows NT 6.2; WOW64; rv:33.0) Gecko/20100101 Firefox/33.0
        Accept: text/css,*/*;q=0.1
        Accept-Language: zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3
        Accept-Encoding: gzip, deflate
        Referer: http://scrapy-chs.readthedocs.org/zh_CN/0.24/
        Cookie: _ga=GA1.2.1612165614.1415584110;
        Connection: keep-alive
        If-Modified-Since: Mon, 25 Aug 2014 21:59:35 GMT
        Cache-Control: max-age=0

meta: 比较常用，在不同的请求之间传递数据使用的。字典dict型

        request_with_cookies = Request(
            url="http://www.example.com",
            cookies={'currency': 'USD', 'country': 'UY'},
            meta={'dont_merge_cookies': True}
        )

encoding: 使用默认的 'utf-8' 就行。

dont_filter: 表明该请求不由调度器过滤。这是当你想使用多次执行相同的请求,忽略重复的过滤器。默认为False。

errback: 指定错误处理函数

 1 # 部分代码
 2 class Response(object_ref):
 3     def __init__(self, url, status=200, headers=None, body='', flags=None, request=None):
 4         self.headers = Headers(headers or {})
 5         self.status = int(status)
 6         self._set_body(body)
 7         self._set_url(url)
 8         self.request = request
 9         self.flags = [] if flags is None else list(flags)
10 
11     @property
12     def meta(self):
13         try:
14             return self.request.meta
15         except AttributeError:
16             raise AttributeError("Response.meta not available, this response " \
17                 "is not tied to any request")

status: 响应码
_set_body(body)： 响应体
_set_url(url)：响应url
self.request = request

 1 class mySpider(scrapy.Spider):
 2     # start_urls = ["http://www.example.com/"]
 3 
 4     def start_requests(self):
 5         url = 'http://www.renren.com/PLogin.do'
 6 
 7         # FormRequest 是Scrapy发送POST请求的方法
 8         yield scrapy.FormRequest(
 9             url = url,
10             formdata = {"email" : "mr_mao_hacker@163.com", "password" : "axxxxxxxe"},
11             callback = self.parse_page
12         )
13     def parse_page(self, response):
14         # do something

DOWNLOADER_MIDDLEWARES = {
    'mySpider.middlewares.MyDownloaderMiddleware': 543,
}

class scrapy.contrib.downloadermiddleware.DownloaderMiddleware

 1 # middlewares.py
 2 
 3 #!/usr/bin/env python
 4 # -*- coding:utf-8 -*-
 5 
 6 import random
 7 import base64
 8 
 9 from settings import USER_AGENTS
10 from settings import PROXIES
11 
12 # 随机的User-Agent
13 class RandomUserAgent(object):
14     def process_request(self, request, spider):
15         useragent = random.choice(USER_AGENTS)
16 
17         request.headers.setdefault("User-Agent", useragent)
18 
19 class RandomProxy(object):
20     def process_request(self, request, spider):
21         proxy = random.choice(PROXIES)
22 
23         if proxy['user_passwd'] is None:
24             # 没有代理账户验证的代理使用方式
25             request.meta['proxy'] = "http://" + proxy['ip_port']
26         else:
27             # 对账户密码进行base64编码转换
28             base64_userpasswd = base64.b64encode(proxy['user_passwd'])
29             # 对应到代理服务器的信令格式里
30             request.headers['Proxy-Authorization'] = 'Basic ' + base64_userpasswd
31             request.meta['proxy'] = "http://" + proxy['ip_port']

 1 # -*- coding: utf-8 -*-
 2 
 3 # Define here the models for your scraped items
 4 #
 5 # See documentation in:
 6 # https://doc.scrapy.org/en/latest/topics/items.html
 7 
 8 import scrapy
 9 
10 '''Item 定义结构化数据字段，用来保存爬取到的数据，有点像Python中的dict，但是提供了一些额外的保护减少错误。
11 
12 可以通过创建一个 scrapy.Item 类， 并且定义类型为 scrapy.Field的类属性来定义一个Item（可以理解成类似于ORM的映射关系）。'''
13 class MyspiderItem(scrapy.Item):
14     # define the fields for your item here like:
15     #职位名
16     name = scrapy.Field()
17     #详细链接
18     detailLink = scrapy.Field()
19     #职位信息
20     positionInfo = scrapy.Field()
21     #人数
22     peopleNumber = scrapy.Field()
23     #工作地点
24     workLocation = scrapy.Field()
25     #发布时间
26     publishTime = scrapy.Field()

 1 # -*- coding: utf-8 -*-
 2 import scrapy
 3 from myspider.items import MyspiderItem
 4 from scrapy.linkextractors import LinkExtractor
 5 from scrapy.spiders import CrawlSpider, Rule
 6 
 7 
 8 class TencentSpider(CrawlSpider):
 9     name = 'tencent'
10     allowed_domains = ['tencent.com']
11     start_urls = ['http://hr.tencent.com/position.php?&start=0#a']
12     rules = (
13         Rule(LinkExtractor(allow=r'position\.php\?&start=\d+'), callback='parse_item', follow=True),
14     )
15 
16     def parse_item(self, response):
17         #i = {}
18         #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()
19         #i['name'] = response.xpath('//div[@id="name"]').extract()
20         #i['description'] = response.xpath('//div[@id="description"]').extract()
21         #return i
22         for each in response.xpath('//*[@class="even"]'):
23             name = each.xpath('./td[1]/a/text()').extract()[0]
24             detailLink = each.xpath('./td[1]/a/@href').extract()[0]
25             positionInfo = each.xpath('./td[2]/text()').extract()[0]
26 
27             peopleNumber = each.xpath('./td[3]/text()').extract()[0]
28             workLocation = each.xpath('./td[4]/text()').extract()[0]
29             publishTime = each.xpath('./td[5]/text()').extract()[0]
30             # print name, detailLink, catalog,recruitNumber,workLocation,publishTime
31 
32             item = MyspiderItem()
33             item['name'] = name
34             item['detailLink'] = detailLink
35             item['positionInfo'] = positionInfo
36             item['peopleNumber'] = peopleNumber
37             item['workLocation'] = workLocation
38             item['publishTime'] = publishTime
39 
40             yield item

 1 import json
 2  
 3 class BaiSispiderPipeline():
 4     def __init__(self):
 5         self.filename = open("tencent.json", "w")
 6 
 7     def process_item(self, item, spider):
 8         text = json.dumps(dict(item), ensure_ascii = False) + ",\n"
 9         self.filename.write(text.encode('utf8'))
10         return item
11 
12     def close_spider(self, spider):
13         self.filename.close()

 1 # -*- coding: utf-8 -*-
 2 import scrapy
 3 from scrapy.linkextractors import LinkExtractor
 4 from scrapy.spiders import CrawlSpider,Rule
 5 from myspider.items import MyspiderItem
 6 
 7 class BaisiSpider(CrawlSpider):
 8     name = 'xiaohua'
 9     allowed_domains = ['www.521609.com']
10     page_list = LinkExtractor(allow=('list\\d+\.html'))
11     start_urls = ['http://www.521609.com/daxuexiaohua/list35.html']
12     rules = (
13         Rule(page_list,callback='parseImg',follow=True),
14     )
15     def parseImg(self, response):
16 
17         # 将我们得到的数据封装到一个 `MyspiderItem` 对象
18         item = MyspiderItem()
19 
20         #提取数据
21         img_list = response.xpath('//div[@class="index_img list_center"]/ul/li')
22         for img in img_list:
23             img_name = img.xpath('./a/img/@alt')[0].extract()
24             img_url = img.xpath('./a/img/@src')[0].extract()
25             item['img_name'] = img_name
26             item['img_url'] = img_url
27 
28             # 将获取的数据交给pipelines
29             yield item
30 
31