function HttpPost(str,url,path) {//將json發(fā)送到服務(wù)器，str為json內(nèi)容，url為歷史消息頁面地址，path是接收程序的路徑和文件名
     console.log("開始執(zhí)行轉(zhuǎn)發(fā)操作");
     try{
     var http = require('http');
     var data = {
         str: encodeURIComponent(str),
         url: encodeURIComponent(url)
     };
     data = require('querystring').stringify(data);
     var options = {
         method: "POST",
         host: "xxx",//注意沒有http://，這是服務(wù)器的域名。
         port: xxx,
         path: path,//接收程序的路徑和文件名
         headers: {
             'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
             "Content-Length": data.length
         }
     };
     var req = http.request(options, function (res) {
         res.setEncoding('utf8');
         res.on('data', function (chunk) {
             console.log('BODY: ' + chunk);
         });
     });
     req.on('error', function (e) {
         console.log('problem with request: ' + e.message);
     });
     
     req.write(data);
     req.end();
     }catch(e){
         console.log("錯誤信息："+e);
     }
     console.log("轉(zhuǎn)發(fā)操作結(jié)束");
 }

public void getMsgJson(String str ,String url) throws UnsupportedEncodingException {
    // TODO Auto-generated method stub
    String biz = "";
    Map<String,String> queryStrs = HttpUrlParser.parseUrl(url);
    if(queryStrs != null){
      biz = queryStrs.get("__biz");
      biz = biz + "==";
    }
    /**
     * 從數(shù)據(jù)庫中查詢biz是否已經(jīng)存在，如果不存在則插入，
     * 這代表著我們新添加了一個采集目標公眾號。
     */
    List<WeiXin> results = weiXinMapper.selectByBiz(biz);
    if(results == null || results.size() == 0){
      WeiXin weiXin = new WeiXin();
      weiXin.setBiz(biz);
      weiXin.setCollect(System.currentTimeMillis());
      weiXinMapper.insert(weiXin);
    }
    //System.out.println(str);
    //解析str變量
    List<Object> lists = JsonPath.read(str, "['list']");
    for(Object list : lists){
      Object json = list;
      int type = JsonPath.read(json, "['comm_msg_info']['type']");
      if(type == 49){//type=49表示是圖文消息
        String content_url = JsonPath.read(json, "$.app_msg_ext_info.content_url");
        content_url = content_url.replace("\\", "").replaceAll("amp;", "");//獲得圖文消息的鏈接地址
        int is_multi = JsonPath.read(json, "$.app_msg_ext_info.is_multi");//是否是多圖文消息
        Integer datetime = JsonPath.read(json, "$.comm_msg_info.datetime");//圖文消息發(fā)送時間
        /**
         * 在這里將圖文消息鏈接地址插入到采集隊列庫tmplist中
         * （隊列庫將在后文介紹，主要目的是建立一個批量采集隊列，
         * 另一個程序?qū)⒏鶕?jù)隊列安排下一個采集的公眾號或者文章內(nèi)容）
         */
        try{
          if(content_url != null && !"".equals(content_url)){
            TmpList tmpList = new TmpList();
            tmpList.setContentUrl(content_url);
            tmpListMapper.insertSelective(tmpList);
          }
        }catch(Exception e){
          System.out.println("隊列已存在,不插入！");
        }
        
        /**
         * 在這里根據(jù)$content_url從數(shù)據(jù)庫post中判斷一下是否重復(fù)
         */
        List<Post> postList = postMapper.selectByContentUrl(content_url);
        boolean contentUrlExist = false;
        if(postList != null && postList.size() != 0){
          contentUrlExist = true;
        }
      
        
        if(!contentUrlExist){//'數(shù)據(jù)庫post中不存在相同的$content_url'
          Integer fileid = JsonPath.read(json, "$.app_msg_ext_info.fileid");//一個微信給的id
          String title = JsonPath.read(json, "$.app_msg_ext_info.title");//文章標題
          String title_encode = URLEncoder.encode(title, "utf-8");
          String digest = JsonPath.read(json, "$.app_msg_ext_info.digest");//文章摘要
          String source_url = JsonPath.read(json, "$.app_msg_ext_info.source_url");//閱讀原文的鏈接
          source_url = source_url.replace("\\", "");
          String cover = JsonPath.read(json, "$.app_msg_ext_info.cover");//封面圖片
          cover = cover.replace("\\", "");
          /**
           * 存入數(shù)據(jù)庫
           */
//          System.out.println("頭條標題："+title);
//          System.out.println("微信ID："+fileid);
//          System.out.println("文章摘要:"+digest);
//          System.out.println("閱讀原文鏈接:"+source_url);
//          System.out.println("封面圖片地址:"+cover);          
          
          Post post = new Post();
          post.setBiz(biz);
          post.setTitle(title);
          post.setTitleEncode(title_encode);
          post.setFieldId(fileid);
          post.setDigest(digest);
          post.setSourceUrl(source_url);
          post.setCover(cover);
          post.setIsTop(1);//標記一下是頭條內(nèi)容
          post.setIsMulti(is_multi);
          post.setDatetime(datetime);
          post.setContentUrl(content_url);
          
          postMapper.insert(post);
        }
      
        if(is_multi == 1){//如果是多圖文消息
          List<Object> multiLists = JsonPath.read(json, "['app_msg_ext_info']['multi_app_msg_item_list']");
          for(Object multiList : multiLists){
            Object multiJson = multiList;          
            content_url = JsonPath.read(multiJson, "['content_url']").toString().replace("\\", "").replaceAll("amp;", "");//圖文消息鏈接地址
            /**
             * 這里再次根據(jù)$content_url判斷一下數(shù)據(jù)庫中是否重復(fù)以免出錯
             */
            contentUrlExist = false;
            List<Post> posts = postMapper.selectByContentUrl(content_url);
            if(posts != null && posts.size() != 0){
              contentUrlExist = true;
            }
            if(!contentUrlExist){//'數(shù)據(jù)庫中不存在相同的$content_url'
              /**
               * 在這里將圖文消息鏈接地址插入到采集隊列庫中
               * （隊列庫將在后文介紹，主要目的是建立一個批量采集隊列，
               * 另一個程序?qū)⒏鶕?jù)隊列安排下一個采集的公眾號或者文章內(nèi)容）
               */
              if(content_url != null && !"".equals(content_url)){
                TmpList tmpListT = new TmpList();
                tmpListT.setContentUrl(content_url);
                tmpListMapper.insertSelective(tmpListT);
              }
              
              String title = JsonPath.read(multiJson, "$.title");
              String title_encode = URLEncoder.encode(title, "utf-8");
              Integer fileid = JsonPath.read(multiJson, "$.fileid");
              String digest = JsonPath.read(multiJson, "$.digest");
              String source_url = JsonPath.read(multiJson, "$.source_url");
              source_url = source_url.replace("\\", "");
              String cover = JsonPath.read(multiJson, "$.cover");
              cover = cover.replace("\\", "");            
//              System.out.println("標題:"+title);
//              System.out.println("微信ID:"+fileid);
//              System.out.println("文章摘要:"+digest);
//              System.out.println("閱讀原文鏈接:"+source_url);
//              System.out.println("封面圖片地址:"+cover);              
              Post post = new Post();
              post.setBiz(biz);
              post.setTitle(title);
              post.setTitleEncode(title_encode);
              post.setFieldId(fileid);
              post.setDigest(digest);
              post.setSourceUrl(source_url);
              post.setCover(cover);
              post.setIsTop(0);//標記一下不是頭條內(nèi)容
              post.setIsMulti(is_multi);
              post.setDatetime(datetime);
              post.setContentUrl(content_url);
              
              postMapper.insert(post);
              
            }
          }
        }      
      }    
    }
  }

public String getWxPost() {
    // TODO Auto-generated method stub
    /**
     * 當前頁面為公眾號文章頁面時，讀取這個程序
     * 首先刪除采集隊列表中l(wèi)oad=1的行
     * 然后從隊列表中按照“order by id asc”選擇多行(注意這一行和上面的程序不一樣)
     */
    tmpListMapper.deleteByLoad(1);
    List<TmpList> queues = tmpListMapper.selectMany(5);
    String url = "";
    if(queues != null && queues.size() != 0 && queues.size() > 1){
      TmpList queue = queues.get(0);
      url = queue.getContentUrl();
      queue.setIsload(1);
      int result = tmpListMapper.updateByPrimaryKey(queue);
      System.out.println("update result:"+result);
    }else{
      System.out.println("getpost queues is null?"+queues==null?null:queues.size());
      WeiXin weiXin = weiXinMapper.selectOne();
      String biz = weiXin.getBiz();
      if((Math.random()>0.5?1:0) == 1){
        url = "http://mp.weixin.qq.com/mp/getmasssendmsg?__biz=" + biz + 
            "#wechat_webview_type=1&wechat_redirect";//拼接公眾號歷史消息url地址（第一種頁面形式）
      }else{
        url = "https://mp.weixin.qq.com/mp/profile_ext?action=home&__biz=" + biz + 
            "#wechat_redirect";//拼接公眾號歷史消息url地址（第二種頁面形式）
      }
      url = "https://mp.weixin.qq.com/mp/profile_ext?action=home&__biz=" + biz + 
          "#wechat_redirect";//拼接公眾號歷史消息url地址（第二種頁面形式）
      //更新剛才提到的公眾號表中的采集時間time字段為當前時間戳。
      weiXin.setCollect(System.currentTimeMillis());
      int result = weiXinMapper.updateByPrimaryKey(weiXin);
      System.out.println("getPost weiXin updateResult:"+result);
    }
    int randomTime = new Random().nextInt(3) + 3;
    String jsCode = "<script>setTimeout(function(){window.location.href='"+url+"';},"+randomTime*1000+");</script>";
    return jsCode;
    
  }

public void getMsgExt(String str,String url) {
    // TODO Auto-generated method stub
    String biz = "";
    String sn = "";
    Map<String,String> queryStrs = HttpUrlParser.parseUrl(url);
    if(queryStrs != null){
      biz = queryStrs.get("__biz");
      biz = biz + "==";
      sn = queryStrs.get("sn");
      sn = "%" + sn + "%";
    }
    /**
     * $sql = "select * from `文章表` where `biz`='".$biz."'
     * and `content_url` like '%".$sn."%'" limit 0,1;
     * 根據(jù)biz和sn找到對應(yīng)的文章
     */
    Post post = postMapper.selectByBizAndSn(biz, sn);
    
    if(post == null){
      System.out.println("biz:"+biz);
      System.out.println("sn:"+sn);
      tmpListMapper.deleteByLoad(1);
      return;
    }
    
//    System.out.println("json數(shù)據(jù):"+str);
    Integer read_num;
    Integer like_num;
    try{
      read_num = JsonPath.read(str, "['appmsgstat']['read_num']");//閱讀量
      like_num = JsonPath.read(str, "['appmsgstat']['like_num']");//點贊量
    }catch(Exception e){
      read_num = 123;//閱讀量
      like_num = 321;//點贊量
      System.out.println("read_num:"+read_num);
      System.out.println("like_num:"+like_num);
      System.out.println(e.getMessage());
    }    
    
    /**
     * 在這里同樣根據(jù)sn在采集隊列表中刪除對應(yīng)的文章，代表這篇文章可以移出采集隊列了
     * $sql = "delete from `隊列表` where `content_url` like '%".$sn."%'" 
     */
    tmpListMapper.deleteBySn(sn);
    
    //然后將閱讀量和點贊量更新到文章表中。
    post.setReadnum(read_num);
    post.setLikenum(like_num);
    postMapper.updateByPrimaryKey(post);
    
  }

public String getWxHis() {
    String url = "";
    // TODO Auto-generated method stub
    /**
     * 當前頁面為公眾號歷史消息時，讀取這個程序
     * 在采集隊列表中有一個load字段，當值等于1時代表正在被讀取
     * 首先刪除采集隊列表中l(wèi)oad=1的行
     * 然后從隊列表中任意select一行
     */
    tmpListMapper.deleteByLoad(1);
    TmpList queue = tmpListMapper.selectRandomOne();
    System.out.println("queue is null?"+queue);
    if(queue == null){//隊列表為空
      /**
       * 隊列表如果空了，就從存儲公眾號biz的表中取得一個biz，
       * 這里我在公眾號表中設(shè)置了一個采集時間的time字段，按照正序排列之后，
       * 就得到時間戳最小的一個公眾號記錄，并取得它的biz
       */
      WeiXin weiXin = weiXinMapper.selectOne();
      
      String biz = weiXin.getBiz();
      url = "https://mp.weixin.qq.com/mp/profile_ext?action=home&__biz=" + biz + 
          "#wechat_redirect";//拼接公眾號歷史消息url地址（第二種頁面形式）
      //更新剛才提到的公眾號表中的采集時間time字段為當前時間戳。
      weiXin.setCollect(System.currentTimeMillis());
      int result = weiXinMapper.updateByPrimaryKey(weiXin);
      System.out.println("getHis weiXin updateResult:"+result);
    }else{
      //取得當前這一行的content_url字段
      url = queue.getContentUrl();
      //將load字段update為1
      tmpListMapper.updateByContentUrl(url);
    }
    //將下一個將要跳轉(zhuǎn)的$url變成js腳本，由anyproxy注入到微信頁面中。
    //echo "<script>setTimeout(function(){window.location.href='".$url."';},2000);</script>";
    int randomTime = new Random().nextInt(3) + 3;
    String jsCode = "<script>setTimeout(function(){window.location.href='"+url+"';},"+randomTime*1000+");</script>";
    return jsCode;
  }

public class SpiderModel implements PageProcessor{
  
  private static PostMapper postMapper;
  
  private static List<Post> posts;
  
  // 抓取網(wǎng)站的相關(guān)配置，包括編碼、抓取間隔、重試次數(shù)等
  private Site site = Site.me().setRetryTimes(3).setSleepTime(100);
  
  public Site getSite() {
    // TODO Auto-generated method stub
    return this.site;
  }
  
  public void process(Page page) {
    // TODO Auto-generated method stub
    Post post = posts.remove(0);
    String content = page.getHtml().xpath("http://div[@id='js_content']").get();
    //存在和諧文章 此處做判定如果有直接刪除記錄或設(shè)置表示位表示文章被和諧
    if(content == null){
      System.out.println("文章已和諧！");
      //postMapper.deleteByPrimaryKey(post.getId());
      return;
    }
    String contentSnap = content.replaceAll("data-src", "src").replaceAll("preview.html", "player.html");//快照
    String contentTxt = HtmlToWord.stripHtml(content);//純文本內(nèi)容
    
    Selectable metaContent = page.getHtml().xpath("http://div[@id='meta_content']");
    String pubTime = null;
    String wxname = null;
    String author = null;
    if(metaContent != null){
      pubTime = metaContent.xpath("http://em[@id='post-date']").get();
      if(pubTime != null){
        pubTime = HtmlToWord.stripHtml(pubTime);//文章發(fā)布時間
      }
      wxname = metaContent.xpath("http://a[@id='post-user']").get();
      if(wxname != null){
        wxname = HtmlToWord.stripHtml(wxname);//公眾號名稱
      }
      author = metaContent.xpath("http://em[@class='rich_media_meta rich_media_meta_text' and @id!='post-date']").get();
      if(author != null){
        author = HtmlToWord.stripHtml(author);//文章作者
      }
    }
    
//    System.out.println("發(fā)布時間:"+pubTime);
//    System.out.println("公眾號名稱:"+wxname);
//    System.out.println("文章作者:"+author);
    
    String title = post.getTitle().replaceAll("&nbsp;", "");//文章標題
    String digest = post.getDigest();//文章摘要
    int likeNum = post.getLikenum();//文章點贊數(shù)
    int readNum = post.getReadnum();//文章閱讀數(shù)
    String contentUrl = post.getContentUrl();//文章鏈接
    
    WechatInfoBean wechatBean = new WechatInfoBean();
    wechatBean.setTitle(title);
    wechatBean.setContent(contentTxt);//純文本內(nèi)容
    wechatBean.setSourceCode(contentSnap);//快照
    wechatBean.setLikeCount(likeNum);
    wechatBean.setViewCount(readNum);
    wechatBean.setAbstractText(digest);//摘要
    wechatBean.setUrl(contentUrl);
    wechatBean.setPublishTime(pubTime);
    wechatBean.setSiteName(wxname);//站點名稱 公眾號名稱
    wechatBean.setAuthor(author);
    wechatBean.setMediaType("微信公眾號");//來源媒體類型
    
    WechatStorage.saveWechatInfo(wechatBean);
    
    //標示文章已經(jīng)被爬取
    post.setIsSpider(1);
    postMapper.updateByPrimaryKey(post);
    
  }  
  
  public static void startSpider(List<Post> inposts,PostMapper myPostMapper,String... urls){
    
    long startTime, endTime;
    startTime = System.currentTimeMillis();
    postMapper = myPostMapper;
    posts = inposts;
    
    HttpClientDownloader httpClientDownloader = new HttpClientDownloader();    
    SpiderModel spiderModel = new SpiderModel();
    Spider mySpider = Spider.create(spiderModel).addUrl(urls);
    mySpider.setDownloader(httpClientDownloader);
    try {
      SpiderMonitor.instance().register(mySpider);
      mySpider.thread(1).run();
    } catch (JMException e) {
      e.printStackTrace();
    }
    
    endTime = System.currentTimeMillis();
    System.out.println("爬取時間" + ((endTime - startTime) / 1000) + "秒--");
    
  }
  
}