<commit_before>import os
import sys
import jinja2
import yaml
import click
import markdown
import shutil
import datetime


CONTENT_TYPES = {
    'markdown': 'md',
    'html': 'html',
    'gallery': 'gallery',
}


def generate_navbar(structure):
    navbar = []
    for section in structure['sections']:
        navbar_section = []
        section_data = structure['sections'][section]
        section_url = '/' + section + '/'
        section_title = section_data['title']
        if 'pages' in section_data:
            for page in section_data['pages']:
                url = '/' + section + '/' + page
                title = structure['sections'][section]['pages'][page]['title']
                #navbar.append(structure['sections'][section]['pages'][page])
                if title:
                    navbar_section.append((url, page, title))
        if section_title:
            navbar.append((section_url, section, section_title, navbar_section))
    print navbar
    return navbar


def ensure_dir(f):
    d = os.path.dirname(f)
    if not os.path.exists(d):
        os.makedirs(d)


def get_file_contents(filename):
    data = None
    try:
        with open(filename) as pf:
            data = pf.read()
    except IOError:
        #print 'File not found: ' + filename
        pass
    return data


def load_page_source(source_uses_subdirs, section_dir, page, page_data):
    if page:
        if source_uses_subdirs:
            filename = os.path.join(section_dir, page)
        else:
            filename = section_dir + '_' + page
    else:
        # Section without pages, only one source file
        filename = section_dir

    try:
        content_type = page_data['type']
    except KeyError:
        content_type = 'unknown'
    try:
        filename = page_data['file']
    except KeyError:
        # No filename was defined explicitely (normal behaviour)
        pass
    if content_type == 'unknown':
        # Try to find the file
        for ct in CONTENT_TYPES:
            try_filename = filename + '.' + ct
            data = get_file_contents(try_filename)
            if data:
                print ' r  ' + try_filename
                break
    else:
        filename += '.' + CONTENT_TYPES[content_type]
        print ' r  ' + filename
        data = get_file_contents(filename)
    if data and content_type == 'markdown':
        data = markdown.markdown(data, output_format='html5')
    return data


## Main program
@click.group()
def cli():
    """
    Paragoo website generator
    """
    pass


@cli.command()
@click.option('-s', '--site', prompt='Site path')
def check_config(site):
    """
    Check site config (site.yaml) for correctness
    """
    click.secho('Needs implementing', fg='red')


<EMAIL>('run_disruptions')
@cli.command()
@click.option('-s', '--site', help='Site path', prompt='Site path')
@click.option('-t', '--template', help='Template path', prompt='Template path')
@click.option('-o', '--output_dir', help='Output path for generated content', prompt='Output path for generated content')
@click.option('--clean/--noclean', help='Clean the output_dir first or not', default=False)
def generate_site(site, template, output_dir, clean):
    """
    Generate the website specified in the config
    """
    try:
        f = open(os.path.join(site, 'site.yaml'))

        print(' r  Reading structure from ' + os.path.join(site, 'site.yaml'))

        structure = yaml.safe_load(f)
        f.close()
    except IOError as e:
        print e
        sys.exit(1)

    print structure

    # Templates can live anywhere, define them on the command line
    template_dir = template
    #loader = jinja2.FileSystemLoader(template_dir)
    loader = jinja2.FileSystemLoader(
            [template_dir,
             os.path.join(os.path.dirname(__file__), 'templates/includes'),
             os.path.join(os.path.dirname(__file__), 'templates')])
    environment = jinja2.Environment(loader=loader, trim_blocks=True, lstrip_blocks=True)

    template = environment.get_template('base.html')

    if clean:
        print ' d  Cleaning up output_dir ' + output_dir
        if os.path.exists(output_dir):
            current_time = datetime.datetime.now()
            dt_format = '%Y-%m-%dT%H:%M:%S%z'
            timestamp = current_time.strftime(dt_format)
            #dst = os.path.join(os.path.dirname(output_dir), timestamp)
            dst = output_dir + '_' + timestamp
            shutil.move(output_dir, dst)
    else:
        print '[!] Not cleaning up, overwrite existing, keeping others'

    ensure_dir(output_dir)<commit_msg>Support for ordered loading of a yaml file<commit_after>import os
import sys
import jinja2
import yaml
import click
import markdown
import shutil
import datetime
from collections import OrderedDict


def ordered_load(stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict):
    class OrderedLoader(Loader):
        pass
    def construct_mapping(loader, node):
        loader.flatten_mapping(node)
        return object_pairs_hook(loader.construct_pairs(node))
    OrderedLoader.add_constructor(
        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
        construct_mapping)
    return yaml.load(stream, OrderedLoader)


CONTENT_TYPES = {
    'markdown': 'md',
    'html': 'html',
    'gallery': 'gallery',
}


def generate_navbar(structure):
    navbar = []
    for section in structure['sections']:
        navbar_section = []
        section_data = structure['sections'][section]
        section_url = '/' + section + '/'
        section_title = section_data['title']
        if 'pages' in section_data:
            for page in section_data['pages']:
                url = '/' + section + '/' + page
                title = structure['sections'][section]['pages'][page]['title']
                #navbar.append(structure['sections'][section]['pages'][page])
                if title:
                    navbar_section.append((url, page, title))
        if section_title:
            navbar.append((section_url, section, section_title, navbar_section))
    print navbar
    return navbar


def ensure_dir(f):
    d = os.path.dirname(f)
    if not os.path.exists(d):
        os.makedirs(d)


def get_file_contents(filename):
    data = None
    try:
        with open(filename) as pf:
            data = pf.read()
    except IOError:
        #print 'File not found: ' + filename
        pass
    return data


def load_page_source(source_uses_subdirs, section_dir, page, page_data):
    if page:
        if source_uses_subdirs:
            filename = os.path.join(section_dir, page)
        else:
            filename = section_dir + '_' + page
    else:
        # Section without pages, only one source file
        filename = section_dir

    try:
        content_type = page_data['type']
    except KeyError:
        content_type = 'unknown'
    try:
        filename = page_data['file']
    except KeyError:
        # No filename was defined explicitely (normal behaviour)
        pass
    if content_type == 'unknown':
        # Try to find the file
        for ct in CONTENT_TYPES:
            try_filename = filename + '.' + ct
            data = get_file_contents(try_filename)
            if data:
                print ' r  ' + try_filename
                break
    else:
        filename += '.' + CONTENT_TYPES[content_type]
        print ' r  ' + filename
        data = get_file_contents(filename)
    if data and content_type == 'markdown':
        data = markdown.markdown(data, output_format='html5')
    return data


## Main program
@click.group()
def cli():
    """
    Paragoo website generator
    """
    pass


@cli.command()
@click.option('-s', '--site', prompt='Site path')
def check_config(site):
    """
    Check site config (site.yaml) for correctness
    """
    click.secho('Needs implementing', fg='red')


<EMAIL>('run_disruptions')
@cli.command()
@click.option('-s', '--site', help='Site path', prompt='Site path')
@click.option('-t', '--template', help='Template path', prompt='Template path')
@click.option('-o', '--output_dir', help='Output path for generated content', prompt='Output path for generated content')
@click.option('--clean/--noclean', help='Clean the output_dir first or not', default=False)
def generate_site(site, template, output_dir, clean):
    """
    Generate the website specified in the config
    """
    try:
        f = open(os.path.join(site, 'site.yaml'))

        print(' r  Reading structure from ' + os.path.join(site, 'site.yaml'))

        structure = ordered_load(f, yaml.SafeLoader)
        f.close()
    except IOError as e:
        print e
        sys.exit(1)

    print structure

    # Templates can live anywhere, define them on the command line
    template_dir = template
    #loader = jinja2.FileSystemLoader(template_dir)
    loader = jinja2.FileSystemLoader(
            [template_dir,
             os.path.join(os.path.dirname(__file__), 'templates/includes'),
             os.path.join(os.path.dirname(__file__), 'templates')])
    environment = jinja2.Environment(loader=loader, trim_blocks=True, lstrip_blocks=True)

    template = environment.get_template('base.html')

    if clean:
        print ' d  Cleaning up output_dir ' + output_dir
        if os.path.exists(output_dir):
            current_time = datetime.datetime.now()
            dt_format = '%Y-%m-%dT%H:%M:%S%z'
            timestamp = current_time.strftime(dt_format)
            #dst = os.path.join(os.path.dirname(output_dir), timestamp)
            dst = output_dir + '_' + timestamp
            shutil.move(output_dir, dst)
    else:
        print '[!] Not cleaning up, overwrite existing, keeping others'

    ensure_dir(output_dir)
{'fraction_non_alphanumeric': 0.1194014447884417, 'fraction_numerical': 0.001238390092879257, 'mean_word_length': 2.573377581120944, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1136563', 'n_tokens_mistral': 2909, 'n_tokens_neox': 2830, 'n_words': 704}