# Path: lib/_db.py
# def get_db(db_name, client='mongo'):
#     client = client_db_map.get(client)
#     return getattr(client, db_name, None)


import _env
import re
import concurrent.futures
import requests
from pprint import pprint
from single_process import single_process
from lib._db import get_db
from thread_pool_spider import ThreadPoolCrawler
from ua import random_search_engine_ua
from web_util import (
    get, logged, change_ip, get_proxy_dict, chunks, get_requests_proxy_ip
)
#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
通过对mongodb中的代理进行并发测试有效性，无效就及时删除。
"""




@logged
class CheckXiciCralwer(ThreadPoolCrawler):
    """CheckXiciCralwer 用来测试代理的有效性，及时剔除没用的代理"""


# Path: lib/_db.py
# def get_db(db_name, client='mongo'):
#     client = client_db_map.get(client)
#     return getattr(client, db_name, None)


# Path: crawler/proxy_check.py
import _env
import re
import concurrent.futures
import requests
from pprint import pprint
from single_process import single_process
from lib._db import get_db
from thread_pool_spider import ThreadPoolCrawler
from ua import random_search_engine_ua
from web_util import (
    get, logged, change_ip, get_proxy_dict, chunks, get_requests_proxy_ip
)

#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
通过对mongodb中的代理进行并发测试有效性，无效就及时删除。
"""




@logged
class CheckXiciCralwer(ThreadPoolCrawler):
    """CheckXiciCralwer 用来测试代理的有效性，及时剔除没用的代理"""


    db = get_db('htmldb')