<commit_before>from mako.template import Template, exceptions
from mako.lookup import TemplateLookup
from binaryornot.check import is_binary
from time import gmtime, strftime
from creer.utilities import uncapitalize, camel_case_to_underscore, camel_case_to_hyphenate, list_dirs, copy_dict, sort_dict_keys, upcase_first, lowercase_first, is_primitive_type
import creer.merge as merge
import creer.githash as githash
import os
import json

templates_folder = "_creer"
template_header = ("Generated by Creer at " + strftime("%I:%M%p on %B %d, %Y UTC", gmtime()) + ", git hash: '" + githash.get() + "'").replace("\n", "").replace("\n", "") # yuk

def build_all(prototype, inputs, output, do_merge=False, tagless=False):
    generated_files = []
    game = prototype['game']
    game_name = game['name']
    game_objects = prototype['game_objects']
    ai = prototype['ai']

    if not inputs:
        inputs = default_input()

    if not output:
        output = "../" if do_merge else "./output"

    for input_directory in inputs:
        full_path = os.path.join(input_directory, templates_folder)
        for root, dirnames, filenames in os.walk(full_path):
            for filename in filenames:
                extensionless, extension = os.path.splitext(filename)

                if extension == '.noCreer': # noCreer files are not to be templated
                    continue

                filepath = os.path.join(root, filename)

                dirs = list_dirs(filepath)
                output_path = ""
                for i, d in enumerate(dirs):<commit_msg>Add checking for junk files to not template<commit_after>from mako.template import Template, exceptions
from mako.lookup import TemplateLookup
from binaryornot.check import is_binary
from time import gmtime, strftime
from creer.utilities import uncapitalize, camel_case_to_underscore, camel_case_to_hyphenate, list_dirs, copy_dict, sort_dict_keys, upcase_first, lowercase_first, is_primitive_type
import creer.merge as merge
import creer.githash as githash
import os
import json

templates_folder = "_creer"
template_header = ("Generated by Creer at " + strftime("%I:%M%p on %B %d, %Y UTC", gmtime()) + ", git hash: '" + githash.get() + "'").replace("\n", "").replace("\n", "") # yuk

# might be better to grab the gitignore at some point in the future for these
always_ignore_filenames = [
    'temp',
    '.DS_Store',
    'desktop.ini',
    'Thumbs.db'
]

def build_all(prototype, inputs, output, do_merge=False, tagless=False):
    generated_files = []
    game = prototype['game']
    game_name = game['name']
    game_objects = prototype['game_objects']
    ai = prototype['ai']

    if not inputs:
        inputs = default_input()

    if not output:
        output = "../" if do_merge else "./output"

    for input_directory in inputs:
        full_path = os.path.join(input_directory, templates_folder)
        for root, dirnames, filenames in os.walk(full_path):
            for filename in filenames:
                extensionless, extension = os.path.splitext(filename)

                if extension.lower() == '.nocreer': # noCreer files are not to be templated
                    continue

                if filename in always_ignore_filenames:
                    continue

                filepath = os.path.join(root, filename)

                dirs = list_dirs(filepath)
                output_path = ""
                for i, d in enumerate(dirs):
{'fraction_non_alphanumeric': 0.10293690026170399, 'fraction_numerical': 0.0, 'mean_word_length': 2.9953542392566783, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1021074', 'n_tokens_mistral': 1052, 'n_tokens_neox': 1005, 'n_words': 295}