import requests
from lxml import etree
import json
import time
class Spider(object):
    def __init__(self):
        #seif.ure='https://movie.douban.com/subject/23885074/reviews?start=0'
        self.headers={
            'User-Agent':'Mozilla/5.0(Windows NT6.1;Win64;x64)AppleWebKit/537.36(KHTML,like Gecko)Chrome/75.0.3700.100Safari/537.36'
        }
    def get_data(self,url):
        response = requests.get(url,headers=self.headers).content.decode('utf-8')
        page=etree.HTML(response)#xpath 對(duì)象
        #獲取所有數(shù)據(jù)節(jié)點(diǎn)
        node_list = page.xpath('//div[@class="review-list  "]/div')
        for node in node_list:
            #作者
            author = node.xpath('.//header[@class="main-hd"]//a[2]/text()')[0]
            #評(píng)論
            text = node.xpath('string(.//div[@class="main-bd"]//div[@class="short-content"])')

            items={
                'author':author,
                'text':text.strip()
            }
            #持久化存儲(chǔ)
            with open('yewen.json','a',encoding='utf-8') as f:
                f.write(json.dumps(items,ensure_ascii=False)+'\n')
    def run(self):
        for i in range(1,47):
            url='https://movie.douban.com/subject/26885074/reviews?start{}'.format(i*20)
            print('正在爬取第{}頁'.format(i))
            self.get_data(url)
            time.sleep(3)
if __name__=='__main__':
    s=Spider()
    s.run()

二、制作詞云

該程序?qū)son中的數(shù)據(jù)進(jìn)行處理，提取重要信息，并用wordcloud庫制作詞云圖片，同樣保存到與該python文件同一級(jí)目錄下
注意需要下載這幾個(gè)庫:jieba、wordcloud、json

import jieba
from wordcloud import WordCloud
import json
f= open("yewen.json", "r", encoding="utf-8")
data_list= f.readlines()
str =''
for data in data_list:
    text= json.loads(data)['text']
    str +=text
#替換無關(guān)緊要的詞語
result_str = str.replace('展開', '').replace('這篇','').replace('影評(píng)','').replace('電影','').replace('這部', '').replace('可能', '').replace('劇情','')
cut_text = jieba.lcut(result_str)
result = " ".join(cut_text)
wc = WordCloud(font_path='simhei.ttf',
               background_color="white",
               max_words=600,
               width=1000,
               height=1000,
               min_font_size=20,
               max_font_size=100,)
    #mast=plt.imreda('snake.jpg')#背景圖片
wc.generate(result)#轉(zhuǎn)化為詞云的操作
wc.to_file("text.jpg")#保存
f.close()

在這里插入圖片描述