import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.List;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;

/**
 * <p>@PostJsonParamsTest.java</p> 
 * @version 1.0
 * @author zxk
 * @Date 2018-3-3
 */
public class PostJsonParamsTest {

  // 超時時間
  private static final int RUN_TIME =10000;

  // 爬取初始頁數(shù)
  private String page;

  public static void main(String[] args) throws Exception {
    PostJsonParamsTest crawl = new PostJsonParamsTest();

    // 請求的url地址
    String url ="http://www.gzcredit.gov.cn/Service/CreditService.asmx/searchOrgWithPage";
    // 設(shè)置起始訪問頁碼
    crawl.setPage("1");
    String isStop = "";

    // 設(shè)置請求
    HttpRequestBase request = null;
    request = new HttpPost(url);

    try {
      // 設(shè)置config
      RequestConfig requestConfig = RequestConfig.custom()
            .setSocketTimeout(RUN_TIME)
            .setConnectTimeout(RUN_TIME)
            .setConnectionRequestTimeout(RUN_TIME)
            .build();
      request.setConfig(requestConfig);

      // json 格式的 post 參數(shù)
      String postParams ="{\"condition\":{\"qymc\":\"%%%%\",\"cydw\":\"\"},\"pageNo\":"+crawl.getPage()+",\"pageSize\":100,count:2709846}";
      System.out.println(postParams);
      HttpEntity httpEntity = new StringEntity(postParams);
      ((HttpPost) request).setEntity(httpEntity);

      // 添加請求頭，可以繞過驗證碼
      request.addHeader("Accept","application/json, text/javascript, */*");
      request.addHeader("Accept-Encoding","gzip, deflate");
      request.addHeader("Accept-Language", "zh-CN,zh;q=0.8");
      request.addHeader("Connection", "keep-alive");
      request.addHeader("Host", "www.gzcredit.gov.cn");
      request.addHeader("Content-Type", "application/json; charset=UTF-8");

      URIBuilder builder = new URIBuilder(url);       
      URI uri = builder.build();
      uri = new URI(URLDecoder.decode(uri.toString(), "UTF-8"));
      request.setURI(uri);

      while(!isStop.equals("停止")||isStop.equals("重跑")){
        isStop = crawl.crawlList(request);
        if(isStop.equals("爬取")){
          crawl.setPage(String.valueOf(Integer.parseInt(crawl.getPage())+1));
        }

        // if("2713".equals(crawl.getPage())) break;
        if("2".equals(crawl.getPage())){
          break;
        }
      }
    } catch (NumberFormatException e) {
      e.printStackTrace();
      throw new NumberFormatException("數(shù)字格式錯誤");
    } catch (UnsupportedEncodingException e) {
      e.printStackTrace();
      throw new UnsupportedEncodingException("不支持的編碼集");
    }
  }
  /**
   * 爬取搜索列表
   * @param page
   * @return
   */
  private String crawlList(HttpRequestBase request){
    int statusCode = 0;

    // 下面兩種方式都可以用來創(chuàng)建客戶端連接,相當于打開了一個瀏覽器
    CloseableHttpClient httpClient = HttpClients.createDefault(); 
    // HttpClient httpClient = HttpClientBuilder.create().build();

    HttpEntity httpEntity = null;
    HttpResponse response = null;
    try {      
      try {        
        response = httpClient.execute(request);
      } catch (Exception e){
        e.printStackTrace();
        EntityUtils.consumeQuietly(httpEntity);
        return "重跑";
      } 

      //打印狀態(tài)
      statusCode =response.getStatusLine().getStatusCode();
      if(statusCode!=200){
        EntityUtils.consumeQuietly(httpEntity);
        return "重跑";
      }
      //實體
      httpEntity = response.getEntity();
      String searchListStr = EntityUtils.toString(httpEntity,"GBK").replaceAll("\\\\米", "米");
      String allData = (String) JSONObject.parseObject(searchListStr).get("d");
      // 字符串值中間含雙引號的替換處理
      String s = allData.replaceAll("\\{\"","{'")
          .replaceAll("\":\"", "':'")
          .replaceAll("\",\"", "','")
          .replaceAll("\":", "':")
          .replaceAll(",\"", ",'")
          .replaceAll("\"\\}", "'}")
          .replaceAll("\"", "")
          .replaceAll("'", "\"")
          .replaceAll("<br />", "")        
          .replaceAll("\t", "")
          .replaceAll("\\\\", "?");
      JSONObject jsonData = JSONObject.parseObject(s);
      JSONArray jsonContent = jsonData.getJSONArray("orgList");

      searchListStr = null;
      allData = null; 
      s = null;

      if (jsonContent==null || jsonContent.size()<1) {
        return "重跑";
      }
      System.out.println(jsonContent.toJSONString());
      return "爬取";
    } catch (Exception e) {
      e.printStackTrace();
      return "重跑";
    } finally{
      EntityUtils.consumeQuietly(httpEntity);
    }
  }

  private String getPage() {
    return page;
  }

  private void setPage(String page) {
    this.page = page;
  }

}

  /**
   * post請求 ，請求數(shù)據(jù)放到body里
   * @param url  請求地址
   * @param bodyData 參數(shù)
   * @author wangyj
   * @date 2019年4月20日
   */
  public static String doPostBodyData(String url, String bodyData) throws Exception{
    String result = "";
    CloseableHttpClient httpClient = null;
    CloseableHttpResponse response = null;
    try {
      HttpPost httpPost = getHttpPost(url, null); // 請求地址
      httpPost.setEntity(new StringEntity(bodyData, Encoding));
      httpClient = getHttpClient();
      // 得到返回的response
      response = httpClient.execute(httpPost);
      HttpEntity entity = response.getEntity();
      result = getResult(entity, Encoding);
    } catch (Exception e) {
      throw e;
    } finally {
      // 關(guān)閉httpClient
      if (null != httpClient) {
        httpClient.close();
      }
      // 關(guān)閉response
      if (null != response) {
        EntityUtils.consume(response.getEntity()); // 會自動釋放連接
        response.close();
      }
    }
    return result;
  }