<commit_before>#!/usr/bin/env python3
#
# aberdeen
#
"""
A static file CMS generator
"""
__author__ = "<NAME>"
__version__ = "0.1.0"
__license__ = "Apache 2.0"
__contact__ = '<EMAIL>'
__homepage__ = 'https://github.com/akubera/aberdeen'

import sys

from configparser import ConfigParser
config = ConfigParser()
config['mongo'] = {
    'host': 'localhost',
    'port': '27017',
    'database': 'aberdeen_blog',
    'collection': 'blog_posts'
}

config.read(['aberdeen.cfg'])

def call_git(args, verbose=False):
    """
    Helper function for calling a 'git' command.
    @param args: list of arguments to the git shell script.
    @return string stdout of git
    """
    from subprocess import check_output
    if isinstance(args,str):
        args = args.split()
    if verbose:
        print ("[call_git] << git ", ' '.join(args))
    res = check_output(['git'] + args)
    if verbose:
        print ("[call_git] >> {}".format(res))
    return res.decode()


def determine_time(timestr):
    """Deterimes a time based on a string."""
    from time import strptime
    formats = ["%b %d, %Y"]
    res = None
    for frm in formats:
        try:
            res = strptime(timestr, frm)
        except ValueError:
            continue
    if not res:
        raise "Time value could not be determined from time string '%s'" % \
            (timestr)
    return res


def generate_from_markdown(src):
    """
    Generates a blogpost from a markdown script, src.
    @param src str: The plaintext markdown string.
    @return dict: Dictionary with header objects and 'html_content'
    """
    from termcolor2 import c

    REDERR = c('ERROR: ').red

    from markdown import Markdown
    md = Markdown(extensions=['markdown.extensions.meta'])
    html = md.convert(src)
    # copy the metadata
    res = {}
    for key, val in md.Meta.items():
        if len(val) == 1:
            # prefix lists with '@'
            if key[0] == '@':
                res[key[1:]] = val
            # special case for 'tag'
            elif key in ['tags', 'tag']:
                res[key] = val
            else:
                res[key] = val[0]
        else:
            res[key] = val

    res['html_content'] = html
    if 'date' not in res:
        raise Exception("    "+REDERR+"No 'date' component found in post.")
    return res

def upload_posts_to_database(posts):
    """
    Uploads the list of items to the database as specified by the configuration
    file.
    @param posts: list of dicts containing post objects
    """
    from pymongo import MongoClient
    # from pprint import pprint
    cfg = config['mongo']
    if 'username' in cfg and 'password' in cfg:
        host = '%s:%s@%s' % map(cfg.get, ['username', 'password', 'host'])
    else:
        host = cfg.get('host', '127.0.0.1')
    mongo = MongoClient(host, int(cfg.get('port')), tz_aware=True)
    db_name = cfg.get('database')
    coll_name = cfg['collection']
    db = mongo[db_name]
    db[coll_name].find()
    # pprint([i for i in post_ids])
    db.drop_collection(coll_name)
    collection = db[coll_name]
    collection.insert(posts)
    # pprint(post_ids)


def main(args):
    """The 'main' function called when executing aberdeen"""
    if len(args) == 0:
        print ("Aberdeen!!")
    else:
        pass

if __name__ == "__main__":
    main(sys.argv[1:])
<commit_msg>Version bump to 0.1.1<commit_after>#!/usr/bin/env python3
#
# aberdeen
#
"""
A static file CMS generator
"""
__author__ = "<NAME>"
__version__ = "0.1.1"
__license__ = "Apache 2.0"
__contact__ = '<EMAIL>'
__homepage__ = 'https://github.com/akubera/aberdeen'

import sys

from configparser import ConfigParser
config = ConfigParser()
config['mongo'] = {
    'host': 'localhost',
    'port': '27017',
    'database': 'aberdeen_blog',
    'collection': 'blog_posts'
}

config.read(['aberdeen.cfg'])

def call_git(args, verbose=False):
    """
    Helper function for calling a 'git' command.
    @param args: list of arguments to the git shell script.
    @return string stdout of git
    """
    from subprocess import check_output
    if isinstance(args,str):
        args = args.split()
    if verbose:
        print ("[call_git] << git ", ' '.join(args))
    res = check_output(['git'] + args)
    if verbose:
        print ("[call_git] >> {}".format(res))
    return res.decode()


def determine_time(timestr):
    """Deterimes a time based on a string."""
    from time import strptime
    formats = ["%b %d, %Y"]
    res = None
    for frm in formats:
        try:
            res = strptime(timestr, frm)
        except ValueError:
            continue
    if not res:
        raise "Time value could not be determined from time string '%s'" % \
            (timestr)
    return res


def generate_from_markdown(src):
    """
    Generates a blogpost from a markdown script, src.
    @param src str: The plaintext markdown string.
    @return dict: Dictionary with header objects and 'html_content'
    """
    from termcolor2 import c

    REDERR = c('ERROR: ').red

    from markdown import Markdown
    md = Markdown(extensions=['markdown.extensions.meta'])
    html = md.convert(src)
    # copy the metadata
    res = {}
    for key, val in md.Meta.items():
        if len(val) == 1:
            # prefix lists with '@'
            if key[0] == '@':
                res[key[1:]] = val
            # special case for 'tag'
            elif key in ['tags', 'tag']:
                res[key] = val
            else:
                res[key] = val[0]
        else:
            res[key] = val

    res['html_content'] = html
    if 'date' not in res:
        raise Exception("    "+REDERR+"No 'date' component found in post.")
    return res

def upload_posts_to_database(posts):
    """
    Uploads the list of items to the database as specified by the configuration
    file.
    @param posts: list of dicts containing post objects
    """
    from pymongo import MongoClient
    # from pprint import pprint
    cfg = config['mongo']
    if 'username' in cfg and 'password' in cfg:
        host = '%s:%s@%s' % map(cfg.get, ['username', 'password', 'host'])
    else:
        host = cfg.get('host', '127.0.0.1')
    mongo = MongoClient(host, int(cfg.get('port')), tz_aware=True)
    db_name = cfg.get('database')
    coll_name = cfg['collection']
    db = mongo[db_name]
    db[coll_name].find()
    # pprint([i for i in post_ids])
    db.drop_collection(coll_name)
    collection = db[coll_name]
    collection.insert(posts)
    # pprint(post_ids)


def main(args):
    """The 'main' function called when executing aberdeen"""
    if len(args) == 0:
        print ("Aberdeen!!")
    else:
        pass

if __name__ == "__main__":
    main(sys.argv[1:])

{'fraction_non_alphanumeric': 0.13331345064121683, 'fraction_numerical': 0.00760512973456606, 'mean_word_length': 2.518887722980063, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1416181', 'n_tokens_mistral': 2183, 'n_tokens_neox': 2119, 'n_words': 641}