# Path: lib/_db.py
# def get_db(db_name, client='mongo'):
#     client = client_db_map.get(client)
#     return getattr(client, db_name, None)


import _env
import re
import concurrent.futures
from io import open
from collections import namedtuple
from pprint import pprint
from random import choice
from requests.exceptions import ProxyError, ConnectTimeout
from lib._db import get_db
from html_parser import Bs4HtmlParser
from thread_pool_spider import ThreadPoolCrawler
from web_util import (
    get, logged, change_ip, get_proxy_dict, chunks, get_requests_proxy_ip
)
from single_process import single_process
            return
        if response.status_code == 200:
            html = response.text
            html_parser = XiciHtmlParser(url, html)
            ip_info_dict_yield = html_parser.parse()
            self.bulk_update_to_mongo(ip_info_dict_yield)
        elif response.status_code == 503:
            change_ip()
            self.urls.append(url)    # retry


class KuaidailiHtmlParser(Bs4HtmlParser):
    fields = """ip, port, anonymous, type, address, speed, verify_time"""
    IpInfo = namedtuple('IpInfo', fields)

    def parse(self):
        table_tag = self.bs.find(
            'table', class_="table table-bordered table-striped"
        )
        tr_tags = table_tag.find_all('tr')
        for tr_tag in tr_tags[1:]:    # skip header
            td_tags = tr_tag.find_all('td')
            td_text_list = [tag.get_text().strip() for tag in td_tags]
            yield self.IpInfo(*td_text_list)._asdict()


@logged
class KuaidailiCrawler(ThreadPoolCrawler):

    """http://www.kuaidaili.com/"""

# Path: lib/_db.py
# def get_db(db_name, client='mongo'):
#     client = client_db_map.get(client)
#     return getattr(client, db_name, None)


# Path: crawler/proxy.py
import _env
import re
import concurrent.futures
from io import open
from collections import namedtuple
from pprint import pprint
from random import choice
from requests.exceptions import ProxyError, ConnectTimeout
from lib._db import get_db
from html_parser import Bs4HtmlParser
from thread_pool_spider import ThreadPoolCrawler
from web_util import (
    get, logged, change_ip, get_proxy_dict, chunks, get_requests_proxy_ip
)
from single_process import single_process

            return
        if response.status_code == 200:
            html = response.text
            html_parser = XiciHtmlParser(url, html)
            ip_info_dict_yield = html_parser.parse()
            self.bulk_update_to_mongo(ip_info_dict_yield)
        elif response.status_code == 503:
            change_ip()
            self.urls.append(url)    # retry


class KuaidailiHtmlParser(Bs4HtmlParser):
    fields = """ip, port, anonymous, type, address, speed, verify_time"""
    IpInfo = namedtuple('IpInfo', fields)

    def parse(self):
        table_tag = self.bs.find(
            'table', class_="table table-bordered table-striped"
        )
        tr_tags = table_tag.find_all('tr')
        for tr_tag in tr_tags[1:]:    # skip header
            td_tags = tr_tag.find_all('td')
            td_text_list = [tag.get_text().strip() for tag in td_tags]
            yield self.IpInfo(*td_text_list)._asdict()


@logged
class KuaidailiCrawler(ThreadPoolCrawler):

    """http://www.kuaidaili.com/"""

    db = get_db('htmldb')