scrapy startproject LaGou

scrapy genspider lagou www.lagou.com

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html

import scrapy


class LagouItem(scrapy.Item):
 # define the fields for your item here like:
 job_url = scrapy.Field()
 job_name = scrapy.Field()
 salary = scrapy.Field()
 city = scrapy.Field()
 area = scrapy.Field()
 experience = scrapy.Field()
 education = scrapy.Field()
 labels = scrapy.Field()
 publish_date = scrapy.Field()
 company = scrapy.Field()
 company_feature = scrapy.Field()
 company_public = scrapy.Field()
 company_size= scrapy.Field()

# -*- coding: utf-8 -*-
import scrapy
from LaGou.items import LagouItem
import json
from pprint import pprint
import time


class LagouSpider(scrapy.Spider):
 name = 'lagou'
 allowed_domains = ['www.lagou.com']
 start_urls = ['https://www.lagou.com/jobs/list_python?']

 def __init__(self):
  # 設(shè)置頭信息, 若不設(shè)置的話, 在請(qǐng)求第二頁(yè)時(shí)即被拉勾網(wǎng)認(rèn)為是爬蟲而不能爬取數(shù)據(jù)
  self.headers = {
   "Accept": "application/json, text/javascript, */*; q=0.01",
   "Connection": "keep-alive",
   "Host": "www.lagou.com",
   "Referer": 'https://www.lagou.com/jobs/list_Python?',
   "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
   "referer": "https://www.lagou.com/jobs/list_python?"
  }
  self.sid = ''
  self.job_url_temp = 'https://www.lagou.com/jobs/{}.html?show={}'
  # 清空文件
  with open('jobs.json', 'w') as f:
   f.truncate()

 def parse(self, response):
  """
  解析起始頁(yè)
  """
  # response為GET請(qǐng)求的起始頁(yè), 自動(dòng)獲取cookie
  # 提交POST帶上前面返回的cookies, 訪問(wèn)數(shù)據(jù)結(jié)果第一頁(yè)
  yield scrapy.FormRequest(
   'https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false',
   callback=self.parse_list,
   formdata={"first": "false",
      "pn": "1",
      "kd": "python",
      },
   headers=self.headers
  )
 def parse_list(self, response):
  """
  解析結(jié)果列表頁(yè)的json數(shù)據(jù)
  """
  # 獲取返回的json,轉(zhuǎn)為字典
  res_dict = json.loads(response.text)
  # 判斷返回是否成功
  if not res_dict.get('success'):
   print(res_dict.get('msg', '返回異常'))
  else:
   # 獲取當(dāng)前頁(yè)數(shù)
   page_num = res_dict['content']['pageNo']
   print('正在爬取第{}頁(yè)'.format(page_num))
   # 獲取sid
   if not self.sid:
    self.sid = res_dict['content']['showId']
   # 獲取響應(yīng)中的職位url字典
   part_url_dict = res_dict['content']['hrInfoMap']
   # 遍歷職位字典
   for key in part_url_dict:
    # 初始化保存職位的item
    item = LagouItem()
    # 拼接完整職位url
    item['job_url'] = self.job_url_temp.format(key, self.sid)
    # 請(qǐng)求職位詳情頁(yè)
    yield scrapy.Request(
     item['job_url'],
     callback=self.parse_detail,
     headers=self.headers,
     meta={'item': item}
    )

   # 獲取下一頁(yè)
   if page_num < 30:
    # time.sleep(2)
    yield scrapy.FormRequest(
     'https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false',
     callback=self.parse_list,
     formdata={"first": "false",
        "pn": str(page_num+1),
        "kd": "python",
        "sid": self.sid
        },
     headers=self.headers
    )

 def parse_detail(self, response):
  """
  解析職位詳情頁(yè)
  """
  # 接收item
  item = response.meta['item']
  # 解析數(shù)據(jù)
  # 獲取職位頭div
  job_div = response.xpath('//div[@class="position-content-l"]')
  if job_div:
   item['job_name'] = job_div.xpath('./div/h1/text()').extract_first()
   item['salary'] = job_div.xpath('./dd/h3/span[1]/text()').extract_first().strip()
   item['city'] = job_div.xpath('./dd/h3/span[2]/text()').extract_first().strip('/').strip()
   item['area'] = response.xpath('//div[@class="work_addr"]/a[2]/text()').extract_first()
   item['experience'] = job_div.xpath('./dd/h3/span[3]/text()').extract_first().strip('/').strip()
   item['education'] = job_div.xpath('./dd/h3/span[4]/text()').extract_first().strip('/').strip()
   item['labels'] = response.xpath('//ul[@class="position-label clearfix"]/li/text()').extract()
   item['publish_date'] = response.xpath('//p[@class="publish_time"]/text()').extract_first()
   item['publish_date'] = item['publish_date'].split('&')[0]
   # 獲取公司dl
   company_div = response.xpath('//dl[@class="job_company"]')
   item['company'] = company_div.xpath('./dt/a/img/@alt').extract_first()
   item['company_feature'] = company_div.xpath('./dd//li[1]/h4[@class="c_feature_name"]/text()').extract_first()
   item['company_feature'] = item['company_feature'].split(',')
   item['company_public'] = company_div.xpath('./dd//li[2]/h4[@class="c_feature_name"]/text()').extract_first()
   item['company_size'] = company_div.xpath('./dd//li[4]/h4[@class="c_feature_name"]/text()').extract_first()
   yield item

from fake_useragent import UserAgent
import random

class RandomUserAgentDM:
 """
 隨機(jī)獲取userAgent
 """
 def __init__(self):
  self.user_agent = UserAgent()

 def process_request(self, request, spider):
  request.headers['User-Agent'] = self.user_agent.random

import json

class LagouPipeline:
 def process_item(self, item, spider):
  with open('jobs.json', 'a', encoding='utf-8') as f:
   item_json = json.dumps(dict(item), ensure_ascii=False, indent=2)
   f.write(item_json)
   f.write('\n')

# 設(shè)置日志顯示
LOG_LEVEL = 'WARNING'

# 設(shè)置ROBOTSTXT協(xié)議, 若為true則不能爬取數(shù)據(jù)
ROBOTSTXT_OBEY = False

# 設(shè)置下載器延遲, 反爬蟲的一種策略
DOWNLOAD_DELAY = 0.25

# 開啟DOWNLOADER_MIDDLEWARES
DOWNLOADER_MIDDLEWARES = {
 # 'LaGou.middlewares.LagouDownloaderMiddleware': 543,
 'LaGou.middlewares.RandomUserAgentDM' :100,
}

# 開啟ITEM_PIPELINES
ITEM_PIPELINES = {
 'LaGou.pipelines.LagouPipeline': 300,
}