# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
#     client = client_db_map.get(client)
#     db = getattr(client, db_name, None)
#     return getattr(db, collection_name, None)


import _env
    import sys
from extract import extract
from tornado import gen
from async_spider import AsySpider
from lib._db import get_collection
from jb51_parse import parse_jb51
from config.config import CONFIG

class Jb51Spider(AsySpider):

    def __init__(self, urls, concurrency=10, results=None, **kwargs):
        super(Jb51Spider, self).__init__(urls, concurrency, results, **kwargs)
        self.db = get_collection(DB, 'article_pyhome', 'motor')    # change coll

    @gen.coroutine
    def update_doc(self, url, data_dict):
        yield self.db.update(
            {'source_url': url},
            {
                '$set': data_dict
            },
            True
        )

    @gen.coroutine
    def handle_html(self, url, html):
        print(url)
        data = parse_jb51(html)
        if is_python_article(data):
            print('is python article')
            data['source_url'] = url
            data['read_count'] = 0
            data['source'] = 'www.jb51.net'
            yield self.update_doc(url, data)


if __name__ == '__main__':

# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
#     client = client_db_map.get(client)
#     db = getattr(client, db_name, None)
#     return getattr(db, collection_name, None)


# Path: crawler/jb51/jb51.py
import _env
    import sys
from extract import extract
from tornado import gen
from async_spider import AsySpider
from lib._db import get_collection
from jb51_parse import parse_jb51
from config.config import CONFIG


class Jb51Spider(AsySpider):

    def __init__(self, urls, concurrency=10, results=None, **kwargs):
        super(Jb51Spider, self).__init__(urls, concurrency, results, **kwargs)
        self.db = get_collection(DB, 'article_pyhome', 'motor')    # change coll

    @gen.coroutine
    def update_doc(self, url, data_dict):
        yield self.db.update(
            {'source_url': url},
            {
                '$set': data_dict
            },
            True
        )

    @gen.coroutine
    def handle_html(self, url, html):
        print(url)
        data = parse_jb51(html)
        if is_python_article(data):
            print('is python article')
            data['source_url'] = url
            data['read_count'] = 0
            data['source'] = 'www.jb51.net'
            yield self.update_doc(url, data)


if __name__ == '__main__':

    try: