快捷導(dǎo)航

Python抓取京東圖書(shū)評(píng)論數(shù)據(jù)

更新時(shí)間：2014年08月31日 15:33:57 投稿：hebedich

最近接了個(gè)項(xiàng)目，需要抓取京東圖書(shū)的評(píng)論，把代碼放出來(lái)給大家分享下，希望能有所幫助

京東圖書(shū)評(píng)論有非常豐富的信息，這里面就包含了購(gòu)買(mǎi)日期、書(shū)名、作者、好評(píng)、中評(píng)、差評(píng)等等。以購(gòu)買(mǎi)日期為例，使用Python + Mysql的搭配進(jìn)行實(shí)現(xiàn)，程序不大，才100行。相關(guān)的解釋我都在程序里加注了：

from selenium import webdriver
from bs4 import BeautifulSoup
import re
import win32com.client
import threading,time
import MySQLdb

def mydebug():
driver.quit()
exit(0)

def catchDate(s):
    """頁(yè)面數(shù)據(jù)提取"""
    soup = BeautifulSoup(s)
    z = []
    global nowtimes

    m = soup.findAll("div",class_="date-buy")
    for obj in m:
        try:
            tmp = obj.find('br').contents
        except Exception, e:
            continue
        if(tmp != ""):
            z.append(tmp)
            nowtimes += 1
    return z

def getTimes(n,t):
"""獲取當(dāng)前進(jìn)度"""
return "當(dāng)前進(jìn)度為：" + str(int(100*n/t)) + "%"

#———————————————————————————————————| 程序開(kāi)始 |—————————————————————————————————
#確定圖書(shū)大類(lèi)
cate = {"3273":"歷史","3279":"心理學(xué)","3276":"政治軍事","3275":"國(guó)學(xué)古籍","3274":"哲學(xué)宗教","3277":"法律","3280":"文化","3281":"社會(huì)科學(xué)"}

#斷點(diǎn)續(xù)抓
num1 = input("bookid:")
num2 = input("pagenumber:")

#生成圖書(shū)大類(lèi)鏈接，共需17355*20 = 347100次
totaltimes = 347100.0
nowtimes = 0

#開(kāi)啟webdirver的PhantomJS對(duì)象
#driver = webdriver.PhantomJS()
driver = webdriver.Ie('C:\Python27\Scripts\IEDriverServer')
#driver = webdriver.Chrome('C:\Python27\Scripts\chromedriver')

#讀出Mysql中的評(píng)論頁(yè)面，進(jìn)行抓取
# 連接數(shù)據(jù)庫(kù)　
try:
    conn = MySQLdb.connect(host='localhost',user='root',passwd='',db='jd')
except Exception, e:
    print e
    sys.exit()

# 獲取cursor對(duì)象
cursor = conn.cursor()
sql = "SELECT * FROM booknew ORDER BY pagenumber DESC"
cursor.execute(sql)
alldata = cursor.fetchall()

flag = 0
flag2 = 0

# 如果有數(shù)據(jù)返回就循環(huán)輸出,http://club.jd.com/review/10178500-1-154.html
if alldata:
    for rec in alldata:
        #rec[0]--bookid,rec[1]--cateid,rec[2]--pagenumber
        if(rec[0] != str(num1) and flag == 0):
            continue
        else:
            flag = 1
        for p in range(num2,rec[2]):
            if(flag2 == 0):
                num2 = 0
                flag2 = 1
            p += 1
            link = "             #抓網(wǎng)頁(yè)
            driver.get(link)
            html = driver.page_source
            #抓評(píng)論
            buydate = catchDate(html)
            #寫(xiě)入數(shù)據(jù)庫(kù)
            for z in buydate:
                sql = "INSERT INTO ljj (id, cateid, bookid, date) VALUES (NULL, '" + rec[0] + "','" + rec[1] + "','" + z[0] + "');"
                try:
                    cursor.execute(sql)
                except Exception, e:
                    print e
            conn.commit()
        print getTimes(nowtimes,totaltimes)