<commit_before>/*  __  __    _    _  _______    ____ _   _    _    _   _  ____ _____ ____  _
   |  \/  |  / \  | |/ / ____|  / ___| | | |  / \  | \ | |/ ___| ____/ ___|| |
   | |\/| | / _ \ | ' /|  _|   | |   | |_| | / _ \ |  \| | |  _|  _| \___ \| |
   | |  | |/ ___ \| . \| |___  | |___|  _  |/ ___ \| |\  | |_| | |___ ___) |_|
   |_|  |_/_/   \_\_|\_\_____|  \____|_| |_/_/   \_\_| \_|\____|_____|____/(_)

   There is a 100% chance that this project can use improvements.
   Pull requests are ALWAYS welcome, even if just amounts to a conversation.  */

var fs = require('fs')
var dir = require('node-dir')
var nunjucks = require('nunjucks')

module.exports = function (state, callback, error) {
    var parse = state.data.parse

    // First parse the data for any template tags
    state.data = JSON.parse(nunjucks.renderString(JSON.stringify(state.data), state.data))

    // If parse is "*" then parse all files
    if(parse == '*') {
        dir.readFiles(state.template.tmp,
            function(err, content, fileToParse, next) {
                if(err) {
                    error(err)
                }
                var renderedContent = nunjucks.renderString(content, state.data)
                fs.writeFileSync(fileToParse, renderedContent)
                next()
            },
            function(err, files){
                if(err) {
                    error(err)
                }
                callback(state)
            }
        )

        // Go no further
        return
    }

    // If parse is undefined do nothing and exit function.
    if(typeof parse == 'undefined') {
        callback(state)
        return
    }

    // Loop through files specified for parsing
    for(i = 0; i < parse.length; i++) {<commit_msg>Parsing all files if template data file is not present.<commit_after>/*  __  __    _    _  _______    ____ _   _    _    _   _  ____ _____ ____  _
   |  \/  |  / \  | |/ / ____|  / ___| | | |  / \  | \ | |/ ___| ____/ ___|| |
   | |\/| | / _ \ | ' /|  _|   | |   | |_| | / _ \ |  \| | |  _|  _| \___ \| |
   | |  | |/ ___ \| . \| |___  | |___|  _  |/ ___ \| |\  | |_| | |___ ___) |_|
   |_|  |_/_/   \_\_|\_\_____|  \____|_| |_/_/   \_\_| \_|\____|_____|____/(_)

   There is a 100% chance that this project can use improvements.
   Pull requests are ALWAYS welcome, even if just amounts to a conversation.  */

var fs = require('fs')
var dir = require('node-dir')
var nunjucks = require('nunjucks')

module.exports = function (state, callback, error) {
    var parse = state.data.parse

    // First parse the data for any template tags
    state.data = JSON.parse(nunjucks.renderString(JSON.stringify(state.data), state.data))

    // If parse is "*" then parse all files
    if(parse == '*' || typeof state.template.data == 'undefined') {
        dir.readFiles(state.template.tmp,
            function(err, content, fileToParse, next) {
                if(err) {
                    error(err)
                }
                var renderedContent = nunjucks.renderString(content, state.data)
                fs.writeFileSync(fileToParse, renderedContent)
                next()
            },
            function(err, files){
                if(err) {
                    error(err)
                }
                callback(state)
            }
        )

        // Go no further
        return
    }

    // If parse is undefined do nothing and exit function.
    if(typeof parse == 'undefined') {
        callback(state)
        return
    }

    // Loop through files specified for parsing
    for(i = 0; i < parse.length; i++) {
{'fraction_non_alphanumeric': 0.14444753687726133, 'fraction_numerical': 0.002226551628165878, 'mean_word_length': 1.5525568181818181, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6219698', 'n_tokens_mistral': 1196, 'n_tokens_neox': 1127, 'n_words': 228}