sudo pip install virtualenv #安裝虛擬環(huán)境工具
virtualenv ENV #創(chuàng)建一個(gè)虛擬環(huán)境目錄
source ./ENV/bin/active #激活虛擬環(huán)境
pip install Scrapy
#驗(yàn)證是否安裝成功
pip list

#輸出如下
cffi (0.8.6)
cryptography (0.6.1)
cssselect (0.9.1)
lxml (3.4.1)
pip (1.5.6)
pycparser (2.10)
pyOpenSSL (0.14)
queuelib (1.2.2)
Scrapy (0.24.4)
setuptools (3.6)
six (1.8.0)
Twisted (14.0.2)
w3lib (1.10.0)
wsgiref (0.1.2)
zope.interface (4.1.1)

$ scrapy startproject tutorial

.
├── scrapy.cfg
└── tutorial
 ├── __init__.py
 ├── items.py
 ├── pipelines.py
 ├── settings.py
 └── spiders
  └── __init__.py

from scrapy.item import Item, Field


class DmozItem(Item):
 # define the fields for your item here like:
 name = Field()
 description = Field()
 url = Field()

import scrapy

class DmozSpider(scrapy.Spider):
 name = "dmoz"
 allowed_domains = ["dmoz.org"]
 start_urls = [
  "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
  "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/"
 ]

 def parse(self, response):
  filename = response.url.split("/")[-2]
  with open(filename, 'wb') as f:
   f.write(response.body)

├── scrapy.cfg
└── tutorial
 ├── __init__.py
 ├── items.py
 ├── pipelines.py
 ├── settings.py
 └── spiders
  ├── __init__.py
  └── dmoz_spider.py

$ scrapy crawl dmoz

2014-12-15 09:30:59+0800 [scrapy] INFO: Scrapy 0.24.4 started (bot: tutorial)
2014-12-15 09:30:59+0800 [scrapy] INFO: Optional features available: ssl, http11
2014-12-15 09:30:59+0800 [scrapy] INFO: Overridden settings: {'NEWSPIDER_MODULE': 'tutorial.spiders', 'SPIDER_MODULES': ['tutorial.spiders'], 'BOT_NAME': 'tutorial'}
2014-12-15 09:30:59+0800 [scrapy] INFO: Enabled extensions: LogStats, TelnetConsole, CloseSpider, WebService, CoreStats, SpiderState
2014-12-15 09:30:59+0800 [scrapy] INFO: Enabled downloader middlewares: HttpAuthMiddleware, DownloadTimeoutMiddleware, UserAgentMiddleware, RetryMiddleware, DefaultHeadersMiddleware, MetaRefreshMiddleware, HttpCompressionMiddleware, RedirectMiddleware, CookiesMiddleware, ChunkedTransferMiddleware, DownloaderStats
2014-12-15 09:30:59+0800 [scrapy] INFO: Enabled spider middlewares: HttpErrorMiddleware, OffsiteMiddleware, RefererMiddleware, UrlLengthMiddleware, DepthMiddleware
2014-12-15 09:30:59+0800 [scrapy] INFO: Enabled item pipelines:
2014-12-15 09:30:59+0800 [dmoz] INFO: Spider opened
2014-12-15 09:30:59+0800 [dmoz] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
2014-12-15 09:30:59+0800 [scrapy] DEBUG: Telnet console listening on 127.0.0.1:6023
2014-12-15 09:30:59+0800 [scrapy] DEBUG: Web service listening on 127.0.0.1:6080
2014-12-15 09:31:00+0800 [dmoz] DEBUG: Crawled (200) <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/> (referer: None)
2014-12-15 09:31:00+0800 [dmoz] DEBUG: Crawled (200) <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> (referer: None)
2014-12-15 09:31:00+0800 [dmoz] INFO: Closing spider (finished)
2014-12-15 09:31:00+0800 [dmoz] INFO: Dumping Scrapy stats:
 {'downloader/request_bytes': 516,
  'downloader/request_count': 2,
  'downloader/request_method_count/GET': 2,
  'downloader/response_bytes': 16338,
  'downloader/response_count': 2,
  'downloader/response_status_count/200': 2,
  'finish_reason': 'finished',
  'finish_time': datetime.datetime(2014, 12, 15, 1, 31, 0, 666214),
  'log_count/DEBUG': 4,
  'log_count/INFO': 7,
  'response_received_count': 2,
  'scheduler/dequeued': 2,
  'scheduler/dequeued/memory': 2,
  'scheduler/enqueued': 2,
  'scheduler/enqueued/memory': 2,
  'start_time': datetime.datetime(2014, 12, 15, 1, 30, 59, 533207)}
2014-12-15 09:31:00+0800 [dmoz] INFO: Spider closed (finished)

<ul class="directory-url" style="margin-left:0;">

 <li><a  class="listinglink">Core Python Programming</a> 
- By Wesley J. Chun; Prentice Hall PTR, 2001, ISBN 0130260363. For experienced developers to improve extant skills; professional level examples. Starts by introducing syntax, objects, error handling, functions, classes, built-ins. [Prentice Hall]
<div class="flag"><a href="/public/flag?cat=Computers%2FProgramming%2FLanguages%2FPython%2FBooks&url=http%3A%2F%2Fwww.pearsonhighered.com%2Feducator%2Facademic%2Fproduct%2F0%2C%2C0130260363%2C00%252Ben-USS_01DBC.html"><img src="/img/flag.png" alt="[!]" title="report an issue with this listing"></a></div>
</li>
...省略部分...
</ul>

#通過如下命令選擇每個(gè)在網(wǎng)站中的 <li> 元素:
sel.xpath('//ul/li')

#網(wǎng)站描述:
sel.xpath('//ul/li/text()').extract()

#網(wǎng)站標(biāo)題:
sel.xpath('//ul/li/a/text()').extract()

#網(wǎng)站鏈接:
sel.xpath('//ul/li/a/@href').extract()

for sel in response.xpath('//ul/li')
 title = sel.xpath('a/text()').extract()
 link = sel.xpath('a/@href').extract()
 desc = sel.xpath('text()').extract()
 print title, link, desc

import scrapy

class DmozSpider(scrapy.Spider):
 name = "dmoz"
 allowed_domains = ["dmoz.org"]
 start_urls = [
  "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
  "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/"
 ]

 def parse(self, response):
  for sel in response.xpath('//ul/li'):
   title = sel.xpath('a/text()').extract()
   link = sel.xpath('a/@href').extract()
   desc = sel.xpath('text()').extract()
   print title, link, desc

>>> item = DmozItem()
>>> item['title'] = 'Example title'
>>> item['title']
'Example title'

from scrapy.spider import Spider
from scrapy.selector import Selector
from tutorial.items import DmozItem


class DmozSpider(Spider):
 name = "dmoz"
 allowed_domains = ["dmoz.org"]
 start_urls = [
  "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
  "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/",
 ]

 def parse(self, response):
  sel = Selector(response)
  sites = sel.xpath('//ul[@class="directory-url"]/li')
  items = []

  for site in sites:
   item = DmozItem()
   item['name'] = site.xpath('a/text()').extract()
   item['url'] = site.xpath('a/@href').extract()
   item['description'] = site.xpath('text()').re('-\s[^\n]*\\r')
   items.append(item)
  return items

from scrapy.exceptions import DropItem

class TutorialPipeline(object):

 # put all words in lowercase
 words_to_filter = ['politics', 'religion']

 def process_item(self, item, spider):
  for word in self.words_to_filter:
   if word in unicode(item['description']).lower():
    raise DropItem("Contains forbidden word: %s" % word)
  else:
   return item

ITEM_PIPELINES = {'tutorial.pipelines.FilterWordsPipeline': 1}

class Spider(object_ref):
 """Base class for scrapy spiders. All spiders must inherit from this
 class.
 """
 
 name = None
 
 def __init__(self, name=None, **kwargs):
  if name is not None:
   self.name = name
  elif not getattr(self, 'name', None):
   raise ValueError("%s must have a name" % type(self).__name__)
  self.__dict__.update(kwargs)
  if not hasattr(self, 'start_urls'):
   self.start_urls = []
 
 def start_requests(self):
  for url in self.start_urls:
   yield self.make_requests_from_url(url)
 
 def make_requests_from_url(self, url):
  return Request(url, dont_filter=True)
 
 def parse(self, response):
  raise NotImplementedError
 
 
BaseSpider = create_deprecated_class('BaseSpider', Spider)

import scrapyfrom myproject.items import MyItemclass MySpider(scrapy.Spider):
 name = 'example.com'
 allowed_domains = ['example.com']
 start_urls = [
  'http://www.example.com/1.html',
  'http://www.example.com/2.html',
  'http://www.example.com/3.html',
 ]
 
 def parse(self, response):
  sel = scrapy.Selector(response)
  for h3 in response.xpath('//h3').extract():
   yield MyItem(title=h3)
 
  for url in response.xpath('//a/@href').extract():
   yield scrapy.Request(url, callback=self.parse)

class MininovaSpider(CrawlSpider):
 name = 'mininova'
 allowed_domains = ['mininova.org']
 start_urls = ['http://www.mininova.org/yesterday']
 rules = [Rule(LinkExtractor(allow=['/tor/\d+']), 'parse_torrent')]
 
 def parse_torrent(self, response):
  torrent = TorrentItem()
  torrent['url'] = response.url
  torrent['name'] = response.xpath("http://h1/text()").extract()
  torrent['description'] = response.xpath("http://div[@id='description']").extract()
  torrent['size'] = response.xpath("http://div[@id='specifications']/p[2]/text()[2]").extract()
  return torrent

 rules = (
  # 提取匹配 'category.php' (但不匹配 'subsection.php') 的鏈接并跟進(jìn)鏈接(沒有callback意味著follow默認(rèn)為True)
  Rule(LinkExtractor(allow=('category\.php', ), deny=('subsection\.php', ))),
 
  # 提取匹配 'item.php' 的鏈接并使用spider的parse_item方法進(jìn)行分析
  Rule(LinkExtractor(allow=('item\.php', )), callback='parse_item'),
 )