# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
#     client = client_db_map.get(client)
#     db = getattr(client, db_name, None)
#     return getattr(db, collection_name, None)


import _env
import re
from async_spider import AsySpider
from bs4 import BeautifulSoup
from lib._db import get_collection
from tornado import gen
from config.config import CONFIG
from html2text import html2text
#!/usr/bin/env python
# -*- coding:utf-8 -*-

DB = CONFIG.MONGO.DATABASE


class CategorySpider(AsySpider):
    PAT = re.compile('http://www.ahlinux.com/python/(\d+).html')

    def handle_html(self, url, html):
        base_url = 'http://www.ahlinux.com/python/%s.html'
        id_list = re.findall(CategorySpider.PAT, html)
        url_list = [base_url % id for id in id_list]
        self.results.extend(url_list)


class ArticleSpider(AsySpider):

    def __init__(self, urls, concurrency=10, results=None, **kwargs):
        super(ArticleSpider, self).__init__(urls,
                                            concurrency, results, **kwargs)

# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
#     client = client_db_map.get(client)
#     db = getattr(client, db_name, None)
#     return getattr(db, collection_name, None)


# Path: crawler/ahlinux/ahlinux.py
import _env
import re
from async_spider import AsySpider
from bs4 import BeautifulSoup
from lib._db import get_collection
from tornado import gen
from config.config import CONFIG
from html2text import html2text

#!/usr/bin/env python
# -*- coding:utf-8 -*-

DB = CONFIG.MONGO.DATABASE


class CategorySpider(AsySpider):
    PAT = re.compile('http://www.ahlinux.com/python/(\d+).html')

    def handle_html(self, url, html):
        base_url = 'http://www.ahlinux.com/python/%s.html'
        id_list = re.findall(CategorySpider.PAT, html)
        url_list = [base_url % id for id in id_list]
        self.results.extend(url_list)


class ArticleSpider(AsySpider):

    def __init__(self, urls, concurrency=10, results=None, **kwargs):
        super(ArticleSpider, self).__init__(urls,
                                            concurrency, results, **kwargs)

        self.db = get_collection(DB, 'ahlinux_pyhome', 'motor')