import pymongo

# 首先需要注意，mongodb數(shù)據(jù)庫(kù)存儲(chǔ)的類型是以鍵值對(duì)類型進(jìn)行存儲(chǔ)，所以在存儲(chǔ)以前一定要進(jìn)行數(shù)據(jù)篩選

def save_mongo(傳入的數(shù)據(jù))：

　　# 創(chuàng)建連接 因?yàn)槭褂玫臑楸緳C(jī)數(shù)據(jù)庫(kù)，所以IP寫localhost即可，端口號(hào)為27017

　　client = pymongo.MongoClient('localhost',27017)

　　# 連接數(shù)據(jù)庫(kù)(這里注意一點(diǎn)，mongo數(shù)據(jù)庫(kù)有一個(gè)優(yōu)點(diǎn)，就是當(dāng)自己連接的數(shù)據(jù)庫(kù)和表都沒(méi)有的時(shí)候，會(huì)自動(dòng)創(chuàng)建，所以不用擔(dān)心寫錯(cuò)或者沒(méi)有表)

　　db = client['自己創(chuàng)建數(shù)據(jù)庫(kù)名']

　　# 連接表

　　collection = db['自己的表名(mongo中叫做集合)']

　　# 插入到數(shù)據(jù)庫(kù)中(這里使用dict進(jìn)行強(qiáng)制轉(zhuǎn)換，是為了保證數(shù)據(jù)為字典格式)

　　collection.insert(dict(傳入的數(shù)據(jù)))

# coding=utf-8

import re
import requests
from lxml import etree
import pymongo
import sys

reload(sys)
sys.setdefaultencoding('utf-8')

def getpages(url, total):
 nowpage = int(re.search('(\d+)', url, re.S).group(1))
 urls = []

 for i in range(nowpage, total + 1):
  link = re.sub('(\d+)', '%s' % i, url, re.S)
  urls.append(link)

 return urls

def spider(url):
 html = requests.get(url)

 selector = etree.HTML(html.text)

 book_name = selector.xpath('//*[@id="container"]/ul/li//div/div[2]/a/text()')
 book_author = selector.xpath('//*[@id="container"]/ul/li//div/div[2]/div/a/text()')

 saveinfo(book_name, book_author)

def saveinfo(book_name, book_author):
 connection = pymongo.MongoClient()
 BookDB = connection.BookDB
 BookTable = BookDB.books

 length = len(book_name)

 for i in range(0, length):
  books = {}
  books['name'] = str(book_name[i]).replace('\n','')
  books['author'] = str(book_author[i]).replace('\n','')
  BookTable.insert_one(books)

if __name__ == '__main__':
 url = 'http://readfree.me/shuffle/?page=1'
 urls = getpages(url,3)

 for each in urls:
  spider(each)