# Path: lib/_db.py
# def get_db(db_name, client='mongo'):
#     client = client_db_map.get(client)
#     return getattr(client, db_name, None)


import _env
import re
import concurrent.futures
from io import open
from collections import namedtuple
from pprint import pprint
from random import choice
from requests.exceptions import ProxyError, ConnectTimeout
from lib._db import get_db
from html_parser import Bs4HtmlParser
from thread_pool_spider import ThreadPoolCrawler
from web_util import (
    get, logged, change_ip, get_proxy_dict, chunks, get_requests_proxy_ip
)
from single_process import single_process
            else:
                func_name = 'extract_tag_' + 'default'
            try:
                value = getattr(self, func_name)(tag).strip()
            except Exception as e:
                print(e)
                value = ''
            yield self.pat.sub('', value)

    def parse(self):
        """parse

        :yields: many OrderedDict
        OrderedDict([('country', 'Cn'), ('ip', u'115.46.80.120'), ('port', u'8123'), ('address', u'\u5e7f\u897f\u5357\u5b81'), ('anonymous', u'\u9ad8\u533f'), ('type', u'HTTP'), ('speed', u'3.687\u79d2'), ('connect_time', u'0.737\u79d2'), ('live_time', u'1\u5206\u949f'), ('verify_time', u'16-07-26 10:54')])
        """
        table_tag = self.bs.find('table', id='ip_list')
        tr_tags = table_tag.find_all('tr')

        for tr_tag in tr_tags:
            td_tags = tr_tag.find_all('td')
            if td_tags:
                ip_info_tags = self.IpInfoTag(*td_tags)
                text_list = list(self.get_tags_text_from_tags(ip_info_tags))
                ip_info_texts = self.IpInfoText(*text_list)
                yield ip_info_texts._asdict()


@logged
class XiciCrawler(ThreadPoolCrawler):


# Path: lib/_db.py
# def get_db(db_name, client='mongo'):
#     client = client_db_map.get(client)
#     return getattr(client, db_name, None)


# Path: crawler/proxy.py
import _env
import re
import concurrent.futures
from io import open
from collections import namedtuple
from pprint import pprint
from random import choice
from requests.exceptions import ProxyError, ConnectTimeout
from lib._db import get_db
from html_parser import Bs4HtmlParser
from thread_pool_spider import ThreadPoolCrawler
from web_util import (
    get, logged, change_ip, get_proxy_dict, chunks, get_requests_proxy_ip
)
from single_process import single_process

            else:
                func_name = 'extract_tag_' + 'default'
            try:
                value = getattr(self, func_name)(tag).strip()
            except Exception as e:
                print(e)
                value = ''
            yield self.pat.sub('', value)

    def parse(self):
        """parse

        :yields: many OrderedDict
        OrderedDict([('country', 'Cn'), ('ip', u'115.46.80.120'), ('port', u'8123'), ('address', u'\u5e7f\u897f\u5357\u5b81'), ('anonymous', u'\u9ad8\u533f'), ('type', u'HTTP'), ('speed', u'3.687\u79d2'), ('connect_time', u'0.737\u79d2'), ('live_time', u'1\u5206\u949f'), ('verify_time', u'16-07-26 10:54')])
        """
        table_tag = self.bs.find('table', id='ip_list')
        tr_tags = table_tag.find_all('tr')

        for tr_tag in tr_tags:
            td_tags = tr_tag.find_all('td')
            if td_tags:
                ip_info_tags = self.IpInfoTag(*td_tags)
                text_list = list(self.get_tags_text_from_tags(ip_info_tags))
                ip_info_texts = self.IpInfoText(*text_list)
                yield ip_info_texts._asdict()


@logged
class XiciCrawler(ThreadPoolCrawler):


    db = get_db('htmldb')