# coding: utf-8
from selenium.webdriver.support.ui import WebDriverWait
from selenium import webdriver
from bs4 import BeautifulSoup
from threading import Thread,Lock
import os
import csv
# 下面是利用 selenium 抓取html頁面的代碼
# 初始化函數
def initSpider():
  driver = webdriver.PhantomJS(executable_path=r"你phantomjs可執(zhí)行文件的絕對路徑")
  driver.get("http://fund.eastmoney.com/f10/jjjz_519961.html") # 要抓取的網頁地址
  # 找到"下一頁"按鈕,就可以得到它前面的一個label,就是總頁數
  getPage_text = driver.find_element_by_id("pagebar").find_element_by_xpath(
    "div[@class='pagebtns']/label[text()='下一頁']/preceding-sibling::label[1]").get_attribute("innerHTML")
  # 得到總共有多少頁
  total_page = int("".join(filter(str.isdigit, getPage_text)))
  # 返回
  return (driver,total_page)
# 獲取html內容
def getData(myrange,driver,lock):
  for x in myrange:
    # 鎖住
    lock.acquire()
    tonum = driver.find_element_by_id("pagebar").find_element_by_xpath(
      "div[@class='pagebtns']/input[@class='pnum']") # 得到 頁碼文本框
    jumpbtn = driver.find_element_by_id("pagebar").find_element_by_xpath(
      "div[@class='pagebtns']/input[@class='pgo']") # 跳轉到按鈕
    tonum.clear() # 第x頁 輸入框
    tonum.send_keys(str(x)) # 去第x頁
    jumpbtn.click() # 點擊按鈕
    # 抓取
    WebDriverWait(driver, 20).until(lambda driver: driver.find_element_by_id("pagebar").find_element_by_xpath("div[@class='pagebtns']/label[@value={0} and @class='cur']".format(x)) != None)
    # 保存到項目中
    with open("../htmls/details/{0}.txt".format(x), 'wb') as f:
      f.write(driver.find_element_by_id("jztable").get_attribute("innerHTML").encode('utf-8'))
      f.close()
    # 解鎖
    lock.release()
# 開始抓取函數
def beginSpider():
  # 初始化爬蟲
  (driver, total_page) = initSpider()
  # 創(chuàng)建鎖
  lock = Lock()
  r = range(1, int(total_page)+1)
  step = 10
  range_list = [r[x:x + step] for x in range(0, len(r), step)]  #把頁碼分段
  thread_list = []
  for r in range_list:
    t = Thread(target=getData, args=(r,driver,lock))
    thread_list.append(t)
    t.start()
  for t in thread_list:
    t.join() # 這一步是需要的,等待線程全部執(zhí)行完成
  print("抓取完成")
# #################上面代碼就完成了 抓取遠程網站html內容并保存到項目中的 過程

beginSpider()

CREATE TABLE `fund_detail` (
 `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
 `fcode` varchar(10) NOT NULL DEFAULT '' COMMENT '基金代碼',
 `fdate` datetime DEFAULT NULL COMMENT '基金日期',
 `NAV` decimal(10,4) DEFAULT NULL COMMENT '單位凈值',
 `ACCNAV` decimal(10,4) DEFAULT NULL COMMENT '累計凈值',
 `DGR` varchar(20) DEFAULT NULL COMMENT '日增長率',
 `pstate` varchar(20) DEFAULT NULL COMMENT '申購狀態(tài)',
 `rstate` varchar(20) DEFAULT NULL COMMENT '贖回狀態(tài)',
 PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='基金詳細數據表';