# Path: lib/_db.py
# def get_db(db_name, client='mongo'):
#     client = client_db_map.get(client)
#     return getattr(client, db_name, None)


import _env
import re
import concurrent.futures
import requests
from pprint import pprint
from single_process import single_process
from lib._db import get_db
from thread_pool_spider import ThreadPoolCrawler
from ua import random_search_engine_ua
from web_util import (
    get, logged, change_ip, get_proxy_dict, chunks, get_requests_proxy_ip
)
                    executor.submit(
                        self.get, url, proxies=get_proxy_dict(ip, int(port)), timeout=self.timeout
                    ): (url, ip, port)
                    for (url, ip, port) in url_list
                }
                for future in concurrent.futures.as_completed(future_to_url):
                    url, ip, port = future_to_url[future]
                    try:
                        response = future.result()
                        if response.status_code != 200:
                            self.logger.info(
                                'status_code %d ,delete proxy %s:%s',
                                response.status_code, ip, port
                            )
                            self.col.delete_one({'ip': ip, 'port': port})
                    except Exception as e:  # 之前使用的自己的get导致异常没raise
                        self.logger.info('delete proxy %s:%s', ip, port)
                        self.col.delete_one({'ip': ip, 'port': port})
                    else:
                        self.handle_response(url, response)

        self.logger.info('after check %d proixes', self.col.count())

    def handle_response(self, url, response):
        """handle_response 验证代理的合法性。通过发送简单请求检测是否超时"""
        if response:
            self.logger.info('url: %s %s', url, response.status_code)


class CheckKuaidailiCralwer(CheckXiciCralwer):

# Path: lib/_db.py
# def get_db(db_name, client='mongo'):
#     client = client_db_map.get(client)
#     return getattr(client, db_name, None)


# Path: crawler/proxy_check.py
import _env
import re
import concurrent.futures
import requests
from pprint import pprint
from single_process import single_process
from lib._db import get_db
from thread_pool_spider import ThreadPoolCrawler
from ua import random_search_engine_ua
from web_util import (
    get, logged, change_ip, get_proxy_dict, chunks, get_requests_proxy_ip
)

                    executor.submit(
                        self.get, url, proxies=get_proxy_dict(ip, int(port)), timeout=self.timeout
                    ): (url, ip, port)
                    for (url, ip, port) in url_list
                }
                for future in concurrent.futures.as_completed(future_to_url):
                    url, ip, port = future_to_url[future]
                    try:
                        response = future.result()
                        if response.status_code != 200:
                            self.logger.info(
                                'status_code %d ,delete proxy %s:%s',
                                response.status_code, ip, port
                            )
                            self.col.delete_one({'ip': ip, 'port': port})
                    except Exception as e:  # 之前使用的自己的get导致异常没raise
                        self.logger.info('delete proxy %s:%s', ip, port)
                        self.col.delete_one({'ip': ip, 'port': port})
                    else:
                        self.handle_response(url, response)

        self.logger.info('after check %d proixes', self.col.count())

    def handle_response(self, url, response):
        """handle_response 验证代理的合法性。通过发送简单请求检测是否超时"""
        if response:
            self.logger.info('url: %s %s', url, response.status_code)


class CheckKuaidailiCralwer(CheckXiciCralwer):

    db = get_db('htmldb')