<commit_before>import os
import logging
import shutil
import fnmatch

from jinja2 import Environment, FileSystemLoader


logger = logging.getLogger('jinger')





def generate_html(sitedir, conf):
    """
    Compile Jinja2 templates in `sourcedir` to html files saved in
    `targetdir` preserving the directory structure.
    """
    sourcedir, targetdir = conf['sourcedir'], conf['targetdir']

    def templates_filter(filename):
        """
        Function for filtering the templates as per the 
        `skip_patterns` specified in the config
        """
        return filename.startswith('.html') and \
            not reduce(lambda x, y: x or fnmatch.fnmatch(filename, y),
                       conf['skip_templates'],
                       False)

    env = Environment(loader=FileSystemLoader(conf['sourcedir']))    
    templates = env.list_templates(filter_func=templates_filter)
    for t in templates:
        filepath = os.path.join(targetdir,
                                *tuple(t.split(os.path.sep)))
        dirname = os.path.dirname(filepath)
        if not os.path.exists(dirname):
            os.makedirs(dirname)
        with open(filepath, 'w') as f:
            f.writelines(env.get_template(t).generate())
            logger.info("Generated %s" % filepath)


def generate_webassets(sitedir, conf):
    """
    Generate or rather copy the contents of the `webassets` 
    directory to the `public` directory
    """
    ignore = shutil.ignore_patterns(('*~', '.gitignore'))
    for p in os.listdir(os.path.join(sitedir, 'webassets')):
        srcpath = os.path.join(sitedir, 'webassets', p)
        trgtpath = os.path.join(sitedir, conf['targetdir'], p)

        if os.path.exists(trgtpath):
            if os.path.isfile(trgtpath):
                os.unlink(trgtpath)
            else:
                shutil.rmtree(trgtpath)

        shutil.copytree(srcpath, trgtpath, ignore=ignore)

<commit_msg>Fix silly mistake in generate_html function<commit_after>import os
import logging
import shutil
import fnmatch

from jinja2 import Environment, FileSystemLoader


logger = logging.getLogger('jinger')





def generate_html(sitedir, conf):
    """
    Compile Jinja2 templates in `sourcedir` to html files saved in
    `targetdir` preserving the directory structure.
    """
    sourcedir, targetdir = conf['sourcedir'], conf['targetdir']

    def templates_filter(filename):
        """
        Function for filtering the templates as per the 
        `skip_patterns` specified in the config
        """
        return filename.endswith('.html') and \
            not reduce(lambda x, y: x or fnmatch.fnmatch(filename, y),
                       conf['skip_templates'],
                       False)

    env = Environment(loader=FileSystemLoader(conf['sourcedir']))    
    templates = env.list_templates(filter_func=templates_filter)
    for t in templates:
        filepath = os.path.join(targetdir,
                                *tuple(t.split(os.path.sep)))
        dirname = os.path.dirname(filepath)
        if not os.path.exists(dirname):
            os.makedirs(dirname)
        with open(filepath, 'w') as f:
            f.writelines(env.get_template(t).generate())
            logger.info("Generated %s" % filepath)


def generate_webassets(sitedir, conf):
    """
    Generate or rather copy the contents of the `webassets` 
    directory to the `public` directory
    """
    ignore = shutil.ignore_patterns(('*~', '.gitignore'))
    for p in os.listdir(os.path.join(sitedir, 'webassets')):
        srcpath = os.path.join(sitedir, 'webassets', p)
        trgtpath = os.path.join(sitedir, conf['targetdir'], p)

        if os.path.exists(trgtpath):
            if os.path.isfile(trgtpath):
                os.unlink(trgtpath)
            else:
                shutil.rmtree(trgtpath)

        shutil.copytree(srcpath, trgtpath, ignore=ignore)


{'fraction_non_alphanumeric': 0.11059670781893004, 'fraction_numerical': 0.00102880658436214, 'mean_word_length': 2.4847670250896057, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4781034', 'n_tokens_mistral': 1174, 'n_tokens_neox': 1121, 'n_words': 273}