import scrapy
from selenium import webdriver
from wangyiPro.items import WangyiproItem

class WangyiSpider(scrapy.Spider):
    name = 'wangyi'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['https://news.163.com/']
    model_url = []

   
    hro = webdriver.Chrome()

    def parse(self, response):
    # 從所有版塊中篩選出所需要下載的4個版塊
        li_list = response.xpath('/html/body/div[1]/div[1]/div[2]/div[2]/div[2]/div[2]/div/ul/li')
        indexs = [3,4,6,7]
        for index in indexs:
            model_li = li_list[index]
            model_list = model_li.xpath('./a/@href').extract_first()
            self.model_url.append(model_list)
        for url in self.model_url:
            yield scrapy.Request(url=url,callback=self.parse_model)

    # 數(shù)據(jù)解析：新聞標題+新聞詳情頁的url（動態(tài)加載數(shù)據(jù)）
    def parse_model(self,response):
        # 直接對response解析無法獲取該數(shù)據(jù)（動態(tài)加載數(shù)據(jù)）
        # 無法拿到動態(tài)加載數(shù)據(jù)返回的響應(yīng)數(shù)據(jù)就不足
        # 使用中間件修改不滿足需求的響應(yīng)對象中的響應(yīng)數(shù)據(jù)，將其改為包含動態(tài)加載數(shù)據(jù)的響應(yīng)數(shù)據(jù)
        div_list = response.xpath('/html/body/div/div[3]/div[4]/div[1]/div[1]/div/ul/li/div/div')
        for div in div_list:
            title = div.xpath('./div/div[1]/h3/a/text()').extract_first()
            new_url_detail = div.xpath('./div/div[1]/h3/a/@href').extract_first()

            if new_url_detail:
                item = WangyiproItem()
                item['title'] = title


                yield scrapy.Request(url=new_url_detail,callback=self.parse_detail,meta={'item':item})


    def parse_detail(self,response):
        # 返回的列表，需要將其轉(zhuǎn)換為字符串
        content = response.xpath('/html/body/div[3]/div[1]/div[3]/div[2]/p/text()').extract()
        content = ''.join(content)
        item = response.meta['item']
        item['content'] = content

        yield item

    # 重寫爬蟲類父類方法，該方法在爬蟲程序執(zhí)行結(jié)束后立即執(zhí)行
    def closed(self,spider):
        # 打開selenium需要關(guān)閉
        self.hro.quit()

from time import sleep
from scrapy.http import HtmlResponse#scrapy封裝好的響應(yīng)類

class WangyiproDownloaderMiddleware:


    def process_request(self, request, spider):

        return None

    # 攔截所有的響應(yīng)對象
    # 整個工程發(fā)起的請求：1+4+n 對應(yīng)相同的響應(yīng)
    def process_response(self, request, response, spider):
        # 從所有攔截對象中找到4個需修改的響應(yīng)對象
        if request.url in spider.model_url:
            hro = spider.hro
            hro.get(request.url)
            sleep(2)
            # 網(wǎng)頁下拉到底，獲取更多動態(tài)加載數(shù)據(jù)
            hro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
            sleep(1)
            page_text = hro.page_source
            # response.text = page_text
            # 返回一個新的響應(yīng)對象
            return HtmlResponse(url=request.url,body=page_text,encoding='utf-8',request=request)

        # 如果不是需要修改的響應(yīng)數(shù)據(jù)，不做修改
        else:
            return response

    def process_exception(self, request, exception, spider):

        pass