# Path: lib/debug_tools.py
# def print_li(li):
#     if isinstance(li, (list, tuple)):
#         for i in li:
#             print_li(i)
#     elif isinstance(li, (str, unicode)):
#         print(li)
#     elif isinstance(li, dict):
#         for k, v in li.items():
#             print(k, v)
#     else:
#         print(li)
# 
# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
#     client = client_db_map.get(client)
#     db = getattr(client, db_name, None)
#     return getattr(db, collection_name, None)


import _env
import os
import re
import html2text
import requests
from extract import extract, extract_all
from tornado import gen
from async_spider import AsySpider
from lib.debug_tools import print_li
from lib._db import get_collection
from pprint import pprint
from config.config import CONFIG
    tag_urls = [i + '?start=0' for i in tag_urls]
    return tag_urls


class TagSpider(AsySpider):
    """拿到每个分类的所有页面"""
    def handle_html(self, url, html):
        html = html.decode('utf-8')
        start_list = extract_all('<a href="?start=', '"', html)
        if start_list:
            max_query = max((int(i) for i in start_list))
            for index in range(0, max_query, 30):
                base_url = url.rsplit('=', 1)[0] + '=' + str(index)
                self.results.append(base_url)


class TagpageSpider(AsySpider):
    """处理每个tag的单独页面，这个页面可以拿到一个文章列表"""
    def handle_html(self, url, html):
        html = html.decode('utf-8')
        url_list = extract_all('<a href="', '"',
                           extract('<div class="code_list">', '</ul>', html))
        article_list = [i for i in url_list if 'author' not in i]
        base_url = 'http://www.sharejs.com'
        article_list = [base_url+i for i in article_list]
        article_list.pop(0)
        self.results.extend(article_list)


class ArticleSpider(AsySpider):

# Path: lib/debug_tools.py
# def print_li(li):
#     if isinstance(li, (list, tuple)):
#         for i in li:
#             print_li(i)
#     elif isinstance(li, (str, unicode)):
#         print(li)
#     elif isinstance(li, dict):
#         for k, v in li.items():
#             print(k, v)
#     else:
#         print(li)
# 
# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
#     client = client_db_map.get(client)
#     db = getattr(client, db_name, None)
#     return getattr(db, collection_name, None)


# Path: crawler/sharejs/sharejs.py
import _env
import os
import re
import html2text
import requests
from extract import extract, extract_all
from tornado import gen
from async_spider import AsySpider
from lib.debug_tools import print_li
from lib._db import get_collection
from pprint import pprint
from config.config import CONFIG

    tag_urls = [i + '?start=0' for i in tag_urls]
    return tag_urls


class TagSpider(AsySpider):
    """拿到每个分类的所有页面"""
    def handle_html(self, url, html):
        html = html.decode('utf-8')
        start_list = extract_all('<a href="?start=', '"', html)
        if start_list:
            max_query = max((int(i) for i in start_list))
            for index in range(0, max_query, 30):
                base_url = url.rsplit('=', 1)[0] + '=' + str(index)
                self.results.append(base_url)


class TagpageSpider(AsySpider):
    """处理每个tag的单独页面，这个页面可以拿到一个文章列表"""
    def handle_html(self, url, html):
        html = html.decode('utf-8')
        url_list = extract_all('<a href="', '"',
                           extract('<div class="code_list">', '</ul>', html))
        article_list = [i for i in url_list if 'author' not in i]
        base_url = 'http://www.sharejs.com'
        article_list = [base_url+i for i in article_list]
        article_list.pop(0)
        self.results.extend(article_list)


class ArticleSpider(AsySpider):

    coll = get_collection(DB, 'code_pyhome', 'motor')