# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
#     client = client_db_map.get(client)
#     db = getattr(client, db_name, None)
#     return getattr(db, collection_name, None)


import _env
import json
import re
import time
    import sys
from html2text import html2text
from extract import extract
from lib._db import get_collection
from judge_upload import exist_or_insert
from config.config import CONFIG
    tags_id_map = {}
    posts_tags = []
    tag_id = 1000
    index = 0

    for doc in coll.find().batch_size(1000):
        #print(doc.get('title'))
        index += 1
        if index > limit:
            break

        posts.append(replace_post(doc))
        post_id = int(doc['source_url'].rsplit('/', 1)[1].split('.')[0])
        tag_list = doc.get('tag_list')
        tag = tag_list[0] if tag_list else ''
        tag = remove_china_char(tag)
        if tag:
            save_tag = tag.replace(' ', '-').lower()
            save_tag = find_first_tag(save_tag)
            if len(save_tag) > 10:
                posts_tags.append(
                    {"tag_id": 1, "post_id": post_id}
                )
                continue

            if save_tag not in tags_id_map:
                tag_id += 1
                TAGS.append({
                    "id": tag_id,
                    "name": save_tag,

# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
#     client = client_db_map.get(client)
#     db = getattr(client, db_name, None)
#     return getattr(db, collection_name, None)


# Path: crawler/xianguo/program_to_ghost.py
import _env
import json
import re
import time
    import sys
from html2text import html2text
from extract import extract
from lib._db import get_collection
from judge_upload import exist_or_insert
from config.config import CONFIG

    tags_id_map = {}
    posts_tags = []
    tag_id = 1000
    index = 0

    for doc in coll.find().batch_size(1000):
        #print(doc.get('title'))
        index += 1
        if index > limit:
            break

        posts.append(replace_post(doc))
        post_id = int(doc['source_url'].rsplit('/', 1)[1].split('.')[0])
        tag_list = doc.get('tag_list')
        tag = tag_list[0] if tag_list else ''
        tag = remove_china_char(tag)
        if tag:
            save_tag = tag.replace(' ', '-').lower()
            save_tag = find_first_tag(save_tag)
            if len(save_tag) > 10:
                posts_tags.append(
                    {"tag_id": 1, "post_id": post_id}
                )
                continue

            if save_tag not in tags_id_map:
                tag_id += 1
                TAGS.append({
                    "id": tag_id,
                    "name": save_tag,

                    "slug": save_tag,