# Path: lib/_db.py
# def get_db(db_name, client='mongo'):
#     client = client_db_map.get(client)
#     return getattr(client, db_name, None)


import _env
import re
import concurrent.futures
from io import open
from collections import namedtuple
from pprint import pprint
from random import choice
from requests.exceptions import ProxyError, ConnectTimeout
from lib._db import get_db
from html_parser import Bs4HtmlParser
from thread_pool_spider import ThreadPoolCrawler
from web_util import (
    get, logged, change_ip, get_proxy_dict, chunks, get_requests_proxy_ip
)
from single_process import single_process


class XiciHtmlParser(Bs4HtmlParser):
    # url = 'http://www.xicidaili.com/'

    fields = """country, ip, port, address, anonymous, type,
    speed, connect_time, live_time, verify_time"""
    fields_list = [i.strip() for i in fields.split(',')]
    IpInfoTag = namedtuple('IpInfo', fields)
    IpInfoText = namedtuple('IpInfoText', fields)
    pat = re.compile(r'[\n\r\t]')

    def extract_tag_default(self, tag):
        return tag.get_text()

    def extract_tag_country(self, tag):
        return tag.find('img').get('alt')

    def extract_tag_connect_time(self, tag):
        return tag.find('div').get('title')

    extract_tag_speed = extract_tag_connect_time

    def get_tags_text_from_tags(self, ip_info_tags):
        special_fileds = ['country', 'speed', 'connect_time']
        for (index, tag) in enumerate(ip_info_tags):
            tag_name = self.fields_list[index]
            if tag_name in special_fileds:
                func_name = 'extract_tag_' + tag_name
            else:

# Path: lib/_db.py
# def get_db(db_name, client='mongo'):
#     client = client_db_map.get(client)
#     return getattr(client, db_name, None)


# Path: crawler/proxy.py
import _env
import re
import concurrent.futures
from io import open
from collections import namedtuple
from pprint import pprint
from random import choice
from requests.exceptions import ProxyError, ConnectTimeout
from lib._db import get_db
from html_parser import Bs4HtmlParser
from thread_pool_spider import ThreadPoolCrawler
from web_util import (
    get, logged, change_ip, get_proxy_dict, chunks, get_requests_proxy_ip
)
from single_process import single_process



class XiciHtmlParser(Bs4HtmlParser):
    # url = 'http://www.xicidaili.com/'

    fields = """country, ip, port, address, anonymous, type,
    speed, connect_time, live_time, verify_time"""
    fields_list = [i.strip() for i in fields.split(',')]
    IpInfoTag = namedtuple('IpInfo', fields)
    IpInfoText = namedtuple('IpInfoText', fields)
    pat = re.compile(r'[\n\r\t]')

    def extract_tag_default(self, tag):
        return tag.get_text()

    def extract_tag_country(self, tag):
        return tag.find('img').get('alt')

    def extract_tag_connect_time(self, tag):
        return tag.find('div').get('title')

    extract_tag_speed = extract_tag_connect_time

    def get_tags_text_from_tags(self, ip_info_tags):
        special_fileds = ['country', 'speed', 'connect_time']
        for (index, tag) in enumerate(ip_info_tags):
            tag_name = self.fields_list[index]
            if tag_name in special_fileds:
                func_name = 'extract_tag_' + tag_name
            else:

                func_name = 'extract_tag_' + 'default'