源碼解析

class CrawlSpider(Spider):
  rules = ()
  def __init__(self, *a, **kw):
    super(CrawlSpider, self).__init__(*a, **kw)
    self._compile_rules()

  # 首先調(diào)用parse()來(lái)處理start_urls中返回的response對(duì)象
  # parse()則將這些response對(duì)象傳遞給了_parse_response()函數(shù)處理，并設(shè)置回調(diào)函數(shù)為parse_start_url()
  # 設(shè)置了跟進(jìn)標(biāo)志位True
  # parse將返回item和跟進(jìn)了的Request對(duì)象  
  def parse(self, response):
    return self._parse_response(response, self.parse_start_url, cb_kwargs={}, follow=True)

  # 處理start_url中返回的response，需要重寫(xiě)
  def parse_start_url(self, response):
    return []

  def process_results(self, response, results):
    return results

  # 從response中抽取符合任一用戶定義'規(guī)則'的鏈接，并構(gòu)造成Resquest對(duì)象返回
  def _requests_to_follow(self, response):
    if not isinstance(response, HtmlResponse):
      return
    seen = set()
    # 抽取之內(nèi)的所有鏈接，只要通過(guò)任意一個(gè)'規(guī)則'，即表示合法
    for n, rule in enumerate(self._rules):
      links = [l for l in rule.link_extractor.extract_links(response) if l not in seen]
      # 使用用戶指定的process_links處理每個(gè)連接
      if links and rule.process_links:
        links = rule.process_links(links)
      # 將鏈接加入seen集合，為每個(gè)鏈接生成Request對(duì)象，并設(shè)置回調(diào)函數(shù)為_(kāi)repsonse_downloaded()
      for link in links:
        seen.add(link)
        # 構(gòu)造Request對(duì)象，并將Rule規(guī)則中定義的回調(diào)函數(shù)作為這個(gè)Request對(duì)象的回調(diào)函數(shù)
        r = Request(url=link.url, callback=self._response_downloaded)
        r.meta.update(rule=n, link_text=link.text)
        # 對(duì)每個(gè)Request調(diào)用process_request()函數(shù)。該函數(shù)默認(rèn)為indentify，即不做任何處理，直接返回該Request.
        yield rule.process_request(r)

  # 處理通過(guò)rule提取出的連接，并返回item以及request
  def _response_downloaded(self, response):
    rule = self._rules[response.meta['rule']]
    return self._parse_response(response, rule.callback, rule.cb_kwargs, rule.follow)

  # 解析response對(duì)象，會(huì)用callback解析處理他，并返回request或Item對(duì)象
  def _parse_response(self, response, callback, cb_kwargs, follow=True):
    # 首先判斷是否設(shè)置了回調(diào)函數(shù)。（該回調(diào)函數(shù)可能是rule中的解析函數(shù)，也可能是 parse_start_url函數(shù)）
    # 如果設(shè)置了回調(diào)函數(shù)（parse_start_url()），那么首先用parse_start_url()處理response對(duì)象，
    # 然后再交給process_results處理。返回cb_res的一個(gè)列表
    if callback:
      #如果是parse調(diào)用的，則會(huì)解析成Request對(duì)象
      #如果是rule callback，則會(huì)解析成Item
      cb_res = callback(response, **cb_kwargs) or ()
      cb_res = self.process_results(response, cb_res)
      for requests_or_item in iterate_spider_output(cb_res):
        yield requests_or_item

    # 如果需要跟進(jìn)，那么使用定義的Rule規(guī)則提取并返回這些Request對(duì)象
    if follow and self._follow_links:
      #返回每個(gè)Request對(duì)象
      for request_or_item in self._requests_to_follow(response):
        yield request_or_item

  def _compile_rules(self):
    def get_method(method):
      if callable(method):
        return method
      elif isinstance(method, basestring):
        return getattr(self, method, None)

    self._rules = [copy.copy(r) for r in self.rules]
    for rule in self._rules:
      rule.callback = get_method(rule.callback)
      rule.process_links = get_method(rule.process_links)
      rule.process_request = get_method(rule.process_request)

  def set_crawler(self, crawler):
    super(CrawlSpider, self).set_crawler(crawler)
    self._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)

class scrapy.linkextractors.LinkExtractor(
  allow = (),
  deny = (),
  allow_domains = (),
  deny_domains = (),
  deny_extensions = None,
  restrict_xpaths = (),
  tags = ('a','area'),
  attrs = ('href'),
  canonicalize = True,
  unique = True,
  process_value = None
)

class scrapy.spiders.Rule(
    link_extractor, 
    callback = None, 
    cb_kwargs = None, 
    follow = None, 
    process_links = None, 
    process_request = None
)

LOG_FILE = "TencentSpider.log"
LOG_LEVEL = "INFO"

模型類
import scrapy
class CrawlyouyuanItem(scrapy.Item):
  # 用戶名
  username = scrapy.Field()
  # 年齡
  age = scrapy.Field()
  # 頭像圖片的鏈接
  header_url = scrapy.Field()
  # 相冊(cè)圖片的鏈接
  images_url = scrapy.Field()
  # 內(nèi)心獨(dú)白
  content = scrapy.Field()
  # 籍貫
  place_from = scrapy.Field()
  # 學(xué)歷
  education = scrapy.Field()
  # 興趣愛(ài)好
  hobby = scrapy.Field()
  # 個(gè)人主頁(yè)
  source_url = scrapy.Field()
  # 數(shù)據(jù)來(lái)源網(wǎng)站
  sourec = scrapy.Field()
  # utc 時(shí)間
  time = scrapy.Field()
  # 爬蟲(chóng)名
  spidername = scrapy.Field()

爬蟲(chóng)文件
# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from CrawlYouYuan.items import CrawlyouyuanItem
import re
class YouyuanSpider(CrawlSpider):
  name = 'youyuan'
  allowed_domains = ['youyuan.com']
  start_urls = ['http://www.youyuan.com/find/beijing/mm18-25/advance-0-0-0-0-0-0-0/p1/']
  # 自動(dòng)生成的文件不需要改東西，只需要添加rules文件里面Rule角色就可以
  # 每一頁(yè)匹配規(guī)則
  page_links = LinkExtractor(allow=(r"youyuan.com/find/beijing/mm18-25/advance-0-0-0-0-0-0-0/p\d+/"))
  # 每個(gè)人個(gè)人主頁(yè)匹配規(guī)則
  profile_links = LinkExtractor(allow=(r"youyuan.com/\d+-profile/"))
  rules = (
    # 沒(méi)有回調(diào)函數(shù)，說(shuō)明follow是True
    Rule(page_links),
    # 有回調(diào)函數(shù)，說(shuō)明follow是False
    Rule(profile_links, callback='parse_item', follow=True),
  )

  def parse_item(self, response):
    item = CrawlyouyuanItem()

    item['username'] = self.get_username(response)
    # 年齡
    item['age'] = self.get_age(response)
    # 頭像圖片的鏈接
    item['header_url'] = self.get_header_url(response)
    # 相冊(cè)圖片的鏈接
    item['images_url'] = self.get_images_url(response)
    # 內(nèi)心獨(dú)白
    item['content'] = self.get_content(response)
    # 籍貫
    item['place_from'] = self.get_place_from(response)
    # 學(xué)歷
    item['education'] = self.get_education(response)
    # 興趣愛(ài)好
    item['hobby'] = self.get_hobby(response)
    # 個(gè)人主頁(yè)
    item['source_url'] = response.url
    # 數(shù)據(jù)來(lái)源網(wǎng)站
    item['sourec'] = "youyuan"

    yield item

  def get_username(self, response):
    username = response.xpath("http://dl[@class='personal_cen']//div[@class='main']/strong/text()").extract()
    if len(username):
      username = username[0]
    else:
      username = "NULL"
    return username.strip()

  def get_age(self, response):
    age = response.xpath("http://dl[@class='personal_cen']//dd/p/text()").extract()
    if len(age):
      age = re.findall(u"\d+歲", age[0])[0]
    else:
      age = "NULL"
    return age.strip()

  def get_header_url(self, response):
    header_url = response.xpath("http://dl[@class='personal_cen']/dt/img/@src").extract()
    if len(header_url):
      header_url = header_url[0]
    else:
      header_url = "NULL"
    return header_url.strip()

  def get_images_url(self, response):
    images_url = response.xpath("http://div[@class='ph_show']/ul/li/a/img/@src").extract()
    if len(images_url):
      images_url = ", ".join(images_url)
    else:
      images_url = "NULL"
    return images_url

  def get_content(self, response):
    content = response.xpath("http://div[@class='pre_data']/ul/li/p/text()").extract()
    if len(content):
      content = content[0]
    else:
      content = "NULL"
    return content.strip()

  def get_place_from(self, response):
    place_from = response.xpath("http://div[@class='pre_data']/ul/li[2]//ol[1]/li[1]/span/text()").extract()
    if len(place_from):
      place_from = place_from[0]
    else:
      place_from = "NULL"
    return place_from.strip()

  def get_education(self, response):
    education = response.xpath("http://div[@class='pre_data']/ul/li[3]//ol[2]/li[2]/span/text()").extract()
    if len(education):
      education = education[0]
    else:
      education = "NULL"
    return education.strip()

  def get_hobby(self, response):
    hobby = response.xpath("http://dl[@class='personal_cen']//ol/li/text()").extract()
    if len(hobby):
      hobby = ",".join(hobby).replace(" ", "")
    else:
      hobby = "NULL"
    return hobby.strip()

管道文件
import json
import codecs
class CrawlyouyuanPipeline(object):

  def __init__(self):
    self.filename = codecs.open('content.json', 'w', encoding='utf-8')

  def process_item(self, item, spider):
    html = json.dumps(dict(item), ensure_ascii=False)
    self.filename.write(html + '\n')
    return item

  def spider_closed(self, spider):
    self.filename.close()

BOT_NAME = 'CrawlYouYuan'
SPIDER_MODULES = ['CrawlYouYuan.spiders']
NEWSPIDER_MODULE = 'CrawlYouYuan.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:56.0)'
# Obey robots.txt rules
ROBOTSTXT_OBEY = True
ITEM_PIPELINES = {
  'CrawlYouYuan.pipelines.CrawlyouyuanPipeline': 300,
}

from scrapy import cmdline
cmdline.execute('scrapy crawl youyuan'.split())