# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
#     client = client_db_map.get(client)
#     db = getattr(client, db_name, None)
#     return getattr(db, collection_name, None)
# 
# Path: lib/debug_tools.py
# def print_li(li):
#     if isinstance(li, (list, tuple)):
#         for i in li:
#             print_li(i)
#     elif isinstance(li, (str, unicode)):
#         print(li)
#     elif isinstance(li, dict):
#         for k, v in li.items():
#             print(k, v)
#     else:
#         print(li)


import _env
import json
import io
import os
import re
import requests
from lib._db import get_collection
from lib.debug_tools import print_li
from pprint import pprint
from html2text import html2text
from extract import extract as et
from extract import extract_all as et_all
from markdown2 import markdown, markdown_path
            file_id = os.path.basename(each).rsplit('.', 1)[0]
            data['source'] = 'http://www.jb51.net/article/%s.htm' % file_id
            data['source_id'] = file_id
            filename = os.path.join(output_path, file_id + '.txt')
            print(filename)

            if data.get('brief'):
                print(len(data.get('brief')))
            with io.open(filename, 'w+', encoding='utf-8') as outfile:
                data = json.dumps(data, ensure_ascii=False, encoding='utf-8',
                                  indent=4)

                outfile.write(unicode(data))


def all_to_html(input_path, output_path):
    all_files = get_all_files(input_path)
    for each in all_files:
        if each.endswith('txt'):
            with io.open(each, 'r', encoding='utf-8') as f:
                d = json.load(f)
                md = d.get('content')
                filename = os.path.join(output_path,
                            os.path.basename(each).rsplit('.', 1)[0] + '.html')
                print(filename)
                with io.open(filename, 'w+', encoding='utf-8') as f:
                    f.write(markdown2html(md))


def save_to_mongo(db_name, col_name, doc_path):

# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
#     client = client_db_map.get(client)
#     db = getattr(client, db_name, None)
#     return getattr(db, collection_name, None)
# 
# Path: lib/debug_tools.py
# def print_li(li):
#     if isinstance(li, (list, tuple)):
#         for i in li:
#             print_li(i)
#     elif isinstance(li, (str, unicode)):
#         print(li)
#     elif isinstance(li, dict):
#         for k, v in li.items():
#             print(k, v)
#     else:
#         print(li)


# Path: crawler/jb51/jb51_parse.py
import _env
import json
import io
import os
import re
import requests
from lib._db import get_collection
from lib.debug_tools import print_li
from pprint import pprint
from html2text import html2text
from extract import extract as et
from extract import extract_all as et_all
from markdown2 import markdown, markdown_path

            file_id = os.path.basename(each).rsplit('.', 1)[0]
            data['source'] = 'http://www.jb51.net/article/%s.htm' % file_id
            data['source_id'] = file_id
            filename = os.path.join(output_path, file_id + '.txt')
            print(filename)

            if data.get('brief'):
                print(len(data.get('brief')))
            with io.open(filename, 'w+', encoding='utf-8') as outfile:
                data = json.dumps(data, ensure_ascii=False, encoding='utf-8',
                                  indent=4)

                outfile.write(unicode(data))


def all_to_html(input_path, output_path):
    all_files = get_all_files(input_path)
    for each in all_files:
        if each.endswith('txt'):
            with io.open(each, 'r', encoding='utf-8') as f:
                d = json.load(f)
                md = d.get('content')
                filename = os.path.join(output_path,
                            os.path.basename(each).rsplit('.', 1)[0] + '.html')
                print(filename)
                with io.open(filename, 'w+', encoding='utf-8') as f:
                    f.write(markdown2html(md))


def save_to_mongo(db_name, col_name, doc_path):

    articles = get_collection(db_name, col_name)  # collection articles