# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
#     client = client_db_map.get(client)
#     db = getattr(client, db_name, None)
#     return getattr(db, collection_name, None)


import _env
import re
from async_spider import AsySpider
from bs4 import BeautifulSoup
from lib._db import get_collection
from tornado import gen
from config.config import CONFIG
from html2text import html2text
        content = str(content_tag)
        #content = html2text(str(content_tag))    # use html
        data = {
            'title': title,
            'brief': brief,
            'content': content,
        }
        yield self.update_doc(url, data)


def test_CategorySpider():
    urls = ['http://www.ahlinux.com/python/']
    base_url = 'http://www.ahlinux.com/python/list%d.html'
    for page in range(2, 57):
        urls.append(base_url % page)
    s = CategorySpider(urls)
    s.run()
    for i in s.results:
        print(i)
    assert False


def test_ArticleSpider():
    urls = ['http://www.ahlinux.com/python/15440.html']
    s = ArticleSpider(urls)
    s.run()


def main():
    urls = ['http://www.ahlinux.com/python/']

# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
#     client = client_db_map.get(client)
#     db = getattr(client, db_name, None)
#     return getattr(db, collection_name, None)


# Path: crawler/ahlinux/ahlinux.py
import _env
import re
from async_spider import AsySpider
from bs4 import BeautifulSoup
from lib._db import get_collection
from tornado import gen
from config.config import CONFIG
from html2text import html2text

        content = str(content_tag)
        #content = html2text(str(content_tag))    # use html
        data = {
            'title': title,
            'brief': brief,
            'content': content,
        }
        yield self.update_doc(url, data)


def test_CategorySpider():
    urls = ['http://www.ahlinux.com/python/']
    base_url = 'http://www.ahlinux.com/python/list%d.html'
    for page in range(2, 57):
        urls.append(base_url % page)
    s = CategorySpider(urls)
    s.run()
    for i in s.results:
        print(i)
    assert False


def test_ArticleSpider():
    urls = ['http://www.ahlinux.com/python/15440.html']
    s = ArticleSpider(urls)
    s.run()


def main():
    urls = ['http://www.ahlinux.com/python/']

    base_url = 'http://www.ahlinux.com/python/list%d.html'