<commit_before>import os
import yaml
import pymongo
from os import listdir
from os.path import isfile, join

from tears import basedir

client = pymongo.MongoClient("localhost", 27017)
db = client.tears
post_collection = db.posts
category_collection = db.categories
tag_collection = db.tags
about_collection = db.about
link_collection = db.link

basedir += '/source/'


def get_all_file(target='posts'):
    return [basedir + target + '/' + i for i in listdir(basedir + target)]


def generate_url(date, full_file_path):
    file_name = full_file_path.split('/')[-1].split('.')[0]
    return '/' + str(date.year) + '/' + str(date.month) + '/' + str(date.day) + '/' + file_name


def check_file():
    """check file then drop collection and insert into database"""
    posts = get_all_file('posts')
    link = get_all_file('link')
    about = get_all_file('about')
    for post in posts + link + about:
        with open(post, 'r') as stream:
            _stream = stream.read()
            if len(_stream.split('---', 1)) < 1:
                print('error on --- split between ')
                return False
            _dict = yaml.load(_stream.split('---', 1)[0])
            if '---' in _dict.get('title'):
                print('--- is not available in title')
                return False
            content = _stream.split('---', 1)[1]
            if not content.replace(' ', '').replace(' ', ''):
                print('content in {name} is blank'.format(name=stream.name))
                return False

    return True


# generate posts
def generate_posts():
    # TODO: better check the files and backup the old file then insert
    client.drop_database('tears')

    category_dict = {}
    tag_dict = {}

    posts = get_all_file('posts')
    if not posts:
        print('no post until now')
    else:
        for post in posts:
            with open(post, 'r') as stream:<commit_msg>Ignore link and about md file<commit_after>import os
import yaml
import pymongo
from os import listdir
from os.path import isfile, join

from tears import basedir

client = pymongo.MongoClient("localhost", 27017)
db = client.tears
post_collection = db.posts
category_collection = db.categories
tag_collection = db.tags
about_collection = db.about
link_collection = db.link

basedir += '/source/'


def get_all_file(target='posts'):
    return [basedir + target + '/' + i for i in listdir(basedir + target)]


def generate_url(date, full_file_path):
    file_name = full_file_path.split('/')[-1].split('.')[0]
    return '/' + str(date.year) + '/' + str(date.month) + '/' + str(date.day) + '/' + file_name


def check_file():
    """check file then drop collection and insert into database"""
    posts = get_all_file('posts')
    for post in posts:
        with open(post, 'r') as stream:
            _stream = stream.read()
            if len(_stream.split('---', 1)) < 1:
                print('error on --- split between ')
                return False
            _dict = yaml.load(_stream.split('---', 1)[0])
            if '---' in _dict.get('title'):
                print('--- is not available in title')
                return False
            content = _stream.split('---', 1)[1]
            if not content.replace(' ', '').replace(' ', ''):
                print('content in {name} is blank'.format(name=stream.name))
                return False

    return True


# generate posts
def generate_posts():
    # TODO: better check the files and backup the old file then insert
    client.drop_database('tears')

    category_dict = {}
    tag_dict = {}

    posts = get_all_file('posts')
    if not posts:
        print('no post until now')
    else:
        for post in posts:
            with open(post, 'r') as stream:
{'fraction_non_alphanumeric': 0.13730222579780102, 'fraction_numerical': 0.006972378653794583, 'mean_word_length': 2.4601113172541744, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '601986', 'n_tokens_mistral': 1138, 'n_tokens_neox': 1091, 'n_words': 328}