# Path: lib/debug_tools.py
# def print_li(li):
#     if isinstance(li, (list, tuple)):
#         for i in li:
#             print_li(i)
#     elif isinstance(li, (str, unicode)):
#         print(li)
#     elif isinstance(li, dict):
#         for k, v in li.items():
#             print(k, v)
#     else:
#         print(li)
# 
# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
#     client = client_db_map.get(client)
#     db = getattr(client, db_name, None)
#     return getattr(db, collection_name, None)


import _env
import os
import re
import html2text
import requests
from extract import extract, extract_all
from tornado import gen
from async_spider import AsySpider
from lib.debug_tools import print_li
from lib._db import get_collection
from pprint import pprint
from config.config import CONFIG
#!/usr/bin/env python
# -*- coding:utf-8 -*-

DB = CONFIG.MONGO.DATABASE


def get_all_tag_urls(url='http://www.sharejs.com/codes/'):
    html = requests.get(url).content.decode('utf-8')

# Path: lib/debug_tools.py
# def print_li(li):
#     if isinstance(li, (list, tuple)):
#         for i in li:
#             print_li(i)
#     elif isinstance(li, (str, unicode)):
#         print(li)
#     elif isinstance(li, dict):
#         for k, v in li.items():
#             print(k, v)
#     else:
#         print(li)
# 
# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
#     client = client_db_map.get(client)
#     db = getattr(client, db_name, None)
#     return getattr(db, collection_name, None)


# Path: crawler/sharejs/sharejs.py
import _env
import os
import re
import html2text
import requests
from extract import extract, extract_all
from tornado import gen
from async_spider import AsySpider
from lib.debug_tools import print_li
from lib._db import get_collection
from pprint import pprint
from config.config import CONFIG

#!/usr/bin/env python
# -*- coding:utf-8 -*-

DB = CONFIG.MONGO.DATABASE


def get_all_tag_urls(url='http://www.sharejs.com/codes/'):
    html = requests.get(url).content.decode('utf-8')

    tag_urls = extract_all('<a href="', '"',