<commit_before>#!/usr/bin/env node

var PEG = require('pegjs');
var argv = require('optimist').argv;
var fs = require('fs');
var exec = require('child_process').exec;
var walk = require('walk');
var path = require('path');
var colors = require('colors');
var mkdir = require('mkdirp');
var beautify = require('js-beautify').js_beautify;

// Command-line use of bailey.js
function main () {
    var source = argv._[0];
    var target = argv._[1];

    if (!source || !target) {
        console.error('Usage: bailey sourcedir/ targetdir/ [--node] [--remove-comments] [--bare]')
        process.exit(1);
    }

    var options = {
        node: !!argv.node,
        removeComments: !!argv['remove-comments'],
        bare: !!argv.bare,
    }

    parseFiles(source, target, options, function(sourcePath, targetPath) {
        if (argv.verbose) {
            console.log(sourcePath, "->", targetPath);
        }
    }, function(err) {
        console.error(err.toString().red);
        process.exit(1);
    });
}

// Whenever we hit an indented block, make sure all preceding
// empty lines are made to have this indentation level
function normalizeBlocks (input) {

    var numberOfLinesToIndent = 0,
        thisLineContainsStuff = false,
        thisLinesIndentation = '',
        out = '';

    for (var i = 0; i < input.length; i++) {

        var chr = input[i];

        if (chr == '\r') {
            continue;
        }

        if (chr === '\n') {
            if (!thisLineContainsStuff) {
                numberOfLinesToIndent++;
            }
            else {
                out += '\n';
            }
            thisLineContainsStuff = false;
            thisLinesIndentation = '';
            continue;
        }

        if (!thisLineContainsStuff && (chr === ' ' || chr === '\t')) {
            thisLinesIndentation += chr;
            continue;
        }

        if (chr !== ' ' || chr !== '\t') {
            
            if (!thisLineContainsStuff) {
                for (var j = 0; j < numberOfLinesToIndent; j++) {
                    out += thisLinesIndentation + '\n';
                }
                out += thisLinesIndentation + chr;
            }
            else {
                out += chr;
            }
            numberOfLinesToIndent = 0;
            thisLineContainsStuff = true;

        }

    }

    return out;

}

function parse (parser, input, options) {

    input = normalizeBlocks(input);

    options = options || {};
    options.removeComments = options.removeComments !== undefined ? options.removeComments : false;
    options.node = options.node !== undefined ? options.node : false;
    options.bare = options.bare !== undefined ? options.bare : false;
    options.parse = parse;

    try {
        var ast = parser.parse(input);
    }
    catch (e) {
        throw new ParserError(e, input, options)
    }

    return beautify(ast.toJS(options));

}

function parseFiles (source, target, options, onFile, onError) {

    var parser = require('./src/parser.js');
    var walker = walk.walk(source, {
        followLinks: false
    });

    // Make sure the source and target are properly formatted
    if (source[source.length-1] != '/') {
        source += '/'
    }

    if (target[target.length-1] != '/') {
        target += '/'
    }

    // From here on we need a ./ from the start to be removed
    source = source.replace(/^\.\//, '');
    target = target.replace(/^\.\//, '');

    walker.on("file", function(root, fileStats, next) {
        
        if (fileStats.name[0] == ".") {
            return next();<commit_msg>move / appending over file walking in bailey.js<commit_after>#!/usr/bin/env node

var PEG = require('pegjs');
var argv = require('optimist').argv;
var fs = require('fs');
var exec = require('child_process').exec;
var walk = require('walk');
var path = require('path');
var colors = require('colors');
var mkdir = require('mkdirp');
var beautify = require('js-beautify').js_beautify;

// Command-line use of bailey.js
function main () {
    var source = argv._[0];
    var target = argv._[1];

    if (!source || !target) {
        console.error('Usage: bailey sourcedir/ targetdir/ [--node] [--remove-comments] [--bare]')
        process.exit(1);
    }

    var options = {
        node: !!argv.node,
        removeComments: !!argv['remove-comments'],
        bare: !!argv.bare,
    }

    parseFiles(source, target, options, function(sourcePath, targetPath) {
        if (argv.verbose) {
            console.log(sourcePath, "->", targetPath);
        }
    }, function(err) {
        console.error(err.toString().red);
        process.exit(1);
    });
}

// Whenever we hit an indented block, make sure all preceding
// empty lines are made to have this indentation level
function normalizeBlocks (input) {

    var numberOfLinesToIndent = 0,
        thisLineContainsStuff = false,
        thisLinesIndentation = '',
        out = '';

    for (var i = 0; i < input.length; i++) {

        var chr = input[i];

        if (chr == '\r') {
            continue;
        }

        if (chr === '\n') {
            if (!thisLineContainsStuff) {
                numberOfLinesToIndent++;
            }
            else {
                out += '\n';
            }
            thisLineContainsStuff = false;
            thisLinesIndentation = '';
            continue;
        }

        if (!thisLineContainsStuff && (chr === ' ' || chr === '\t')) {
            thisLinesIndentation += chr;
            continue;
        }

        if (chr !== ' ' || chr !== '\t') {
            
            if (!thisLineContainsStuff) {
                for (var j = 0; j < numberOfLinesToIndent; j++) {
                    out += thisLinesIndentation + '\n';
                }
                out += thisLinesIndentation + chr;
            }
            else {
                out += chr;
            }
            numberOfLinesToIndent = 0;
            thisLineContainsStuff = true;

        }

    }

    return out;

}

function parse (parser, input, options) {

    input = normalizeBlocks(input);

    options = options || {};
    options.removeComments = options.removeComments !== undefined ? options.removeComments : false;
    options.node = options.node !== undefined ? options.node : false;
    options.bare = options.bare !== undefined ? options.bare : false;
    options.parse = parse;

    try {
        var ast = parser.parse(input);
    }
    catch (e) {
        throw new ParserError(e, input, options)
    }

    return beautify(ast.toJS(options));

}

function parseFiles (source, target, options, onFile, onError) {

    // Make sure the source and target are properly formatted
    if (source[source.length-1] != '/') {
        source += '/'
    }

    if (target[target.length-1] != '/') {
        target += '/'
    }

    var parser = require('./src/parser.js');
    var walker = walk.walk(source, {
        followLinks: false
    });

    // From here on we need a ./ from the start to be removed
    source = source.replace(/^\.\//, '');
    target = target.replace(/^\.\//, '');

    walker.on("file", function(root, fileStats, next) {
        
        if (fileStats.name[0] == ".") {
            return next();
{'fraction_non_alphanumeric': 0.14805555555555555, 'fraction_numerical': 0.0030555555555555557, 'mean_word_length': 1.9106709781729991, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5464379', 'n_tokens_mistral': 2162, 'n_tokens_neox': 2059, 'n_words': 519}