# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
#     client = client_db_map.get(client)
#     db = getattr(client, db_name, None)
#     return getattr(db, collection_name, None)
# 
# Path: lib/debug_tools.py
# def print_li(li):
#     if isinstance(li, (list, tuple)):
#         for i in li:
#             print_li(i)
#     elif isinstance(li, (str, unicode)):
#         print(li)
#     elif isinstance(li, dict):
#         for k, v in li.items():
#             print(k, v)
#     else:
#         print(li)


import _env
import json
import io
import os
import re
import requests
from lib._db import get_collection
from lib.debug_tools import print_li
from pprint import pprint
from html2text import html2text
from extract import extract as et
from extract import extract_all as et_all
from markdown2 import markdown, markdown_path
#!/usr/bin/env python
# -*- coding: utf-8 -*-



def get_all_files(path):
    res = set()
    list_dirs = os.walk(path)
    for root, dirs, files in list_dirs:
        for f in files:
            res.add(os.path.join(root, f))
    return res


def remove_words(s):
    for key in ['<!--NEWSZW_HZH_END-->', '<!--NEWSZW_HZH_BEGIN-->']:
                #'http://www.jb51.net', 'www.jb51.net', 'jb51.net']:

# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
#     client = client_db_map.get(client)
#     db = getattr(client, db_name, None)
#     return getattr(db, collection_name, None)
# 
# Path: lib/debug_tools.py
# def print_li(li):
#     if isinstance(li, (list, tuple)):
#         for i in li:
#             print_li(i)
#     elif isinstance(li, (str, unicode)):
#         print(li)
#     elif isinstance(li, dict):
#         for k, v in li.items():
#             print(k, v)
#     else:
#         print(li)


# Path: crawler/jb51/jb51_parse.py
import _env
import json
import io
import os
import re
import requests
from lib._db import get_collection
from lib.debug_tools import print_li
from pprint import pprint
from html2text import html2text
from extract import extract as et
from extract import extract_all as et_all
from markdown2 import markdown, markdown_path

#!/usr/bin/env python
# -*- coding: utf-8 -*-



def get_all_files(path):
    res = set()
    list_dirs = os.walk(path)
    for root, dirs, files in list_dirs:
        for f in files:
            res.add(os.path.join(root, f))
    return res


def remove_words(s):
    for key in ['<!--NEWSZW_HZH_END-->', '<!--NEWSZW_HZH_BEGIN-->']:
                #'http://www.jb51.net', 'www.jb51.net', 'jb51.net']:

        s = s.replace(key, '')