# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
#     client = client_db_map.get(client)
#     db = getattr(client, db_name, None)
#     return getattr(db, collection_name, None)


import _env
    import sys
from extract import extract
from tornado import gen
from async_spider import AsySpider
from lib._db import get_collection
from jb51_parse import parse_jb51
from config.config import CONFIG
#!/usr/bin/env python
# -*- coding: utf-8 -*-

DB = CONFIG.MONGO.DATABASE


def is_python_article(data_dict):
    title = data_dict.get('title').lower()
    content = data_dict.get('content').lower()
    if ('python' in title) or (content.count('python') > 2):
        return True
    return False


class Jb51Spider(AsySpider):

    def __init__(self, urls, concurrency=10, results=None, **kwargs):
        super(Jb51Spider, self).__init__(urls, concurrency, results, **kwargs)

# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
#     client = client_db_map.get(client)
#     db = getattr(client, db_name, None)
#     return getattr(db, collection_name, None)


# Path: crawler/jb51/jb51.py
import _env
    import sys
from extract import extract
from tornado import gen
from async_spider import AsySpider
from lib._db import get_collection
from jb51_parse import parse_jb51
from config.config import CONFIG

#!/usr/bin/env python
# -*- coding: utf-8 -*-

DB = CONFIG.MONGO.DATABASE


def is_python_article(data_dict):
    title = data_dict.get('title').lower()
    content = data_dict.get('content').lower()
    if ('python' in title) or (content.count('python') > 2):
        return True
    return False


class Jb51Spider(AsySpider):

    def __init__(self, urls, concurrency=10, results=None, **kwargs):
        super(Jb51Spider, self).__init__(urls, concurrency, results, **kwargs)

        self.db = get_collection(DB, 'article_pyhome', 'motor')    # change coll