# 數(shù)據(jù)庫地址
HOST = '127.0.0.1'
# MySql端口
MYSQL_PORT = 3306
# MySQl用戶名、密碼
MYSQL_USERNAME = '***'
MYSQL_PASSWORD = '***'
# 數(shù)據(jù)庫名
SQL_NAME = 'test'

# 代理等級(jí)
MAX_SCORE = 30
MIN_SCORE = 0
INITIAL_SCORE = 10

# 代理池?cái)?shù)量界限
POOL_UPPER_THRESHOLD = 1000

VALID_STATUS_CODES = [200, 302]

# 測(cè)試API，建議抓哪個(gè)網(wǎng)站測(cè)哪個(gè)
TEST_URL = 'http://www.baidu.com'

# 最大批測(cè)試量
BATCH_TEST_SIZE = 30

import asyncio
import aiohttp
import time
import sys
from aiohttp import ClientError
from db import MySqlClient
from setting import *


class Tester(object):
 def __init__(self):
  self.mysql = MySqlClient()
 
 # 測(cè)試單個(gè)代理
 async def test_single_ip(self, ip):
  conn = aiohttp.TCPConnector(verify_ssl=False)
  async with aiohttp.ClientSession(connector=conn) as session:
   try:
    if isinstance(ip, bytes):
     ip = ip.decode('utf-8')
    real_ip = 'http://' + ip
    print('正在測(cè)試', ip)
    async with session.get(TEST_URL, proxy=real_ip, timeout=15, allow_redirects=False) as response:
     if response.status in VALID_STATUS_CODES:
      self.mysql.max(ip)
      print('代理可用', ip)
     else:
      self.mysql.decrease(ip)
      print('請(qǐng)求響應(yīng)碼不合法 ', response.status, 'IP', ip)
   except (ClientError, aiohttp.client_exceptions.ClientConnectorError, asyncio.TimeoutError, AttributeError):
    self.mysql.decrease(ip)
    print('代理請(qǐng)求失敗', ip)
 
 # 主函數(shù)
 def run(self):
  print('測(cè)試器開始運(yùn)行')
  try:
   count = self.mysql.count()
   print('當(dāng)前剩余', count, '個(gè)代理')
   for i in range(0, count, BATCH_TEST_SIZE):
    start = i
    stop = min(i + BATCH_TEST_SIZE, count)
    print('正在測(cè)試第', start + 1, '-', stop, '個(gè)代理')
    test_ip_group = self.mysql.batch(start, stop)
    loop = asyncio.get_event_loop()
    tasks = [self.test_single_ip(ip_tuple[0]) for ip_tuple in test_ip_group]
    loop.run_until_complete(asyncio.wait(tasks))
    sys.stdout.flush()
    time.sleep(5)
  except Exception as e:
   print('測(cè)試器發(fā)生錯(cuò)誤', e.args)


if __name__ == "__main__":
 test = Tester()
 test.run()