# Path: lib/_db.py
# def get_db(db_name, client='mongo'):
#     client = client_db_map.get(client)
#     return getattr(client, db_name, None)


import _env
from pprint import pformat
from extract import extract_all
from lib._db import get_db
from thread_pool_spider import ThreadPoolCrawler
#!/usr/bin/env python
# -*- coding:utf-8 -*-



class LagouSitemap(ThreadPoolCrawler):
    db = get_db('htmldb')
    col = getattr(db, 'lagou_url')    # collection

    def get(self, url, *args, **kwargs):
        headers = {
            'User-Agent': 'mozilla/5.0 (compatible; baiduspider/2.0; +http://www.baidu.com/search/spider.html)',
        }
        return super(LagouSitemap, self).get(
            url, headers=headers, *args, **kwargs
        )

    def handle_response(self, url, response):
        """处理http响应，对于200响应码直接处理html页面，
        否则按照需求处理不同响应码"""
        self.logger.info('url:%s', url)
        if response.status_code == 200:
            self.handle_html(url, response.text)


# Path: lib/_db.py
# def get_db(db_name, client='mongo'):
#     client = client_db_map.get(client)
#     return getattr(client, db_name, None)


# Path: crawler/lagou/sitemap.py
import _env
from pprint import pformat
from extract import extract_all
from lib._db import get_db
from thread_pool_spider import ThreadPoolCrawler

#!/usr/bin/env python
# -*- coding:utf-8 -*-



class LagouSitemap(ThreadPoolCrawler):
    db = get_db('htmldb')
    col = getattr(db, 'lagou_url')    # collection

    def get(self, url, *args, **kwargs):
        headers = {
            'User-Agent': 'mozilla/5.0 (compatible; baiduspider/2.0; +http://www.baidu.com/search/spider.html)',
        }
        return super(LagouSitemap, self).get(
            url, headers=headers, *args, **kwargs
        )

    def handle_response(self, url, response):
        """处理http响应，对于200响应码直接处理html页面，
        否则按照需求处理不同响应码"""
        self.logger.info('url:%s', url)
        if response.status_code == 200:
            self.handle_html(url, response.text)


    def init_urls(self):