def start_requests(self):
 for i in range(1,101):
 page=i*2-1 #這里是構(gòu)造請求url的page,表示奇數(shù)
 url=self.start_url+str(page)
 yield scrapy.Request(url,meta={'search_page':page+1},callback=self.parse_url) #這里使用meta想回調(diào)函數(shù)傳入數(shù)據(jù)，回調(diào)函數(shù)使用response.meta['search-page']接受數(shù)據(jù)

def parse_url(self,response):
 if response.status==200: #判斷是否請求成功
 # print response.url
 pids = set() #這個集合用于過濾和保存得到的id,用于作為后面的ajax請求的url構(gòu)成
 try:
 all_goods = response.xpath("http://div[@id='J_goodsList']/ul/li") #首先得到所有衣服的整個框架，然后從中抽取每一個框架
 for goods in all_goods: #從中解析每一個
 # scrapy.shell.inspect_response(response,self) #這是一個調(diào)試的方法，這里會直接打開調(diào)試模式
 items = JdSpiderItem() #定義要抓取的數(shù)據(jù)
 img_url_src = goods.xpath("div/div[1]/a/img/@src").extract() # 如果不存在就是一個空數(shù)組[]，因此不能在這里取[0]
 img_url_delay = goods.xpath(
  "div/div[1]/a/img/@data-lazy-img").extract() # 這個是沒有加載出來的圖片，這里不能寫上數(shù)組取第一個[0]
 price = goods.xpath("div/div[3]/strong/i/text()").extract() #價格
 cloths_name = goods.xpath("div/div[4]/a/em/text()").extract()
 shop_id = goods.xpath("div/div[7]/@ data-shopid").extract()
 cloths_url = goods.xpath("div/div[1]/a/@href").extract()
 person_number = goods.xpath("div/div[5]/strong/a/text()").extract()
 pid = goods.xpath("@data-pid").extract()
 # product_id=goods.xpath("@data-sku").extract()
 if pid:
  pids.add(pid[0])
 if img_url_src: # 如果img_url_src存在
  print img_url_src[0]
  items['img_url'] = img_url_src[0]
 if img_url_delay: # 如果到了沒有加載完成的圖片，就取這個url
  print img_url_delay[0]
  items['img_url'] = img_url_delay[0] # 這里如果數(shù)組不是空的，就能寫了
 if price:
  items['price'] = price[0]
 if cloths_name:
  items['cloths_name'] = cloths_name[0]
 if shop_id:
  items['shop_id'] = shop_id[0]
  shop_url = "https://mall.jd.com/index-" + str(shop_id[0]) + ".html"
  items['shop_url'] = shop_url
 if cloths_url:
  items['cloths_url'] = cloths_url[0]
 if person_number:
  items['person_number'] = person_number[0]
 # if product_id:
 # print "************************************csdjkvjfskvnk***********************"
 # print self.comments_url.format(str(product_id[0]),str(self.count))
 # yield scrapy.Request(url=self.comments_url.format(str(product_id[0]),str(self.count)),callback=self.comments)
 #yield scrapy.Request寫在這里就是每解析一個鍵褲子就會調(diào)用回調(diào)函數(shù)一次
 yield items
 except Exception:
 print "********************************************ERROR**********************************************************************"
 yield scrapy.Request(url=self.search_url.format(str(response.meta['search_page']),",".join(pids)),callback=self.next_half_parse) #再次請求，這里是請求ajax加載的數(shù)據(jù)，必須放在這里，因為只有等到得到所有的pid才能構(gòu)成這個請求，回調(diào)函數(shù)用于下面的解析

#分析異步加載的網(wǎng)頁
 def next_half_parse(self,response):
 if response.status==200:
 print response.url
 items=JdSpiderItem()
 #scrapy.shell.inspect_response(response,self) #y用來調(diào)試的
 try:
 lis=response.xpath("http://li[@class='gl-item']")
 for li in lis:
  cloths_url=li.xpath("div/div[1]/a/@href").extract()
  img_url_1=li.xpath("div/div[1]/a/img/@src").extract()
  img_url_2=li.xpath("div/div[1]/a/img/@data-lazy-img").extract()
  cloths_name=li.xpath("div/div[4]/a/em/text()").extract()
  price=li.xpath("div/div[3]/strong/i/text()").extract()
  shop_id=li.xpath("div/div[7]/@data-shopid").extract()
  person_number=li.xpath("div/div[5]/strong/a/text()").extract()
  if cloths_url:
  print cloths_url[0]
  items['cloths_url']=cloths_url[0]
  if img_url_1:
  print img_url_1[0]
  items['img_url']=img_url_1
  if img_url_2:
  print img_url_2[0]
  items['img_url']=img_url_2[0]
  if cloths_name:
  items['cloths_name']=cloths_name[0]
  if price:
  items['price']=price[0]
  if shop_id:
  items['shop_id']=shop_id[0]
  items['shop_url']="https://mall.jd.com/index-" + str(shop_id[0]) + ".html"
  if person_number:
  items['person_number']=person_number[0]
  yield items #又一次的生成，這里是完整的數(shù)據(jù)，因此可以yield items
 except Exception:
 print "**************************************************"