scrapy startproject quotes

scrapy genspider -t quotes quotes.toscrape.com

import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor

class Quotes(CrawlSpider):
 # 爬蟲名稱
  name = "get_quotes"
  allow_domain = ['quotes.toscrape.com']
  start_urls = ['http://quotes.toscrape.com/']

# 設定規(guī)則
  rules = (
    # 對于quotes內容頁URL，調用parse_quotes處理，
    # 并以此規(guī)則跟進獲取的鏈接
    Rule(LinkExtractor(allow=r'/page/\d+'), callback='parse_quotes', follow=True),
    # 對于author內容頁URL，調用parse_author處理，提取數(shù)據(jù)
    Rule(LinkExtractor(allow=r'/author/\w+'), callback='parse_author')
  )

# 提取內容頁數(shù)據(jù)方法
  def parse_quotes(self, response):
    for quote in response.css(".quote"):
      yield {'content': quote.css('.text::text').extract_first(),
          'author': quote.css('.author::text').extract_first(),
          'tags': quote.css('.tag::text').extract()
          }
 # 獲取作者數(shù)據(jù)方法

  def parse_author(self, response):
    name = response.css('.author-title::text').extract_first()
    author_born_date = response.css('.author-born-date::text').extract_first()
    author_bron_location = response.css('.author-born-location::text').extract_first()
    author_description = response.css('.author-description::text').extract_first()

    return ({'name': name,
         'author_bron_date': author_born_date,
         'author_bron_location': author_bron_location,
         'author_description': author_description
         })

scrapy crawl quotes