<commit_before>var dedent_token = "\u21d0";
var indent_token = "\u21d2";

function preprocessIndentation(text) {
    // var lines = text.split("\n").filter(function(s) {
    //     return !s.split("").every(function(c) {
    //         return c == " " && c == "\t"
    //     });
    // });
    var lines = text.split("\n");
    var processedLines = insertTokens(lines, []);
    return processedLines.join("\n");
}

function insertTokens(lines, stack) {
    if (lines.length === 0) {
        var out = [];
        for (var i=0; i<stack.length; i++) {
            out.push(dedent_token);
        }
        return out;
    }

    var line = lines[0];
    var rest = lines.slice(1);

    //new 4/24
    if (line.length === 0) {
        return [line].concat(insertTokens(rest, stack))
    }

    var indentation = calculateIndentation(line);
    var top = stack.length ? stack[stack.length-1] : 0;
<commit_msg>Preprocessing adds a single \n at end<commit_after>var dedent_token = "\u21d0";
var indent_token = "\u21d2";

function preprocessIndentation(text) {
    // var lines = text.split("\n").filter(function(s) {
    //     return !s.split("").every(function(c) {
    //         return c == " " && c == "\t"
    //     });
    // });
    var lines = text.split("\n");
    var processedLines = insertTokens(lines, []);
    return processedLines.join("\n")+"\n";
}

function insertTokens(lines, stack) {
    if (lines.length === 0) {
        var out = [];
        for (var i=0; i<stack.length; i++) {
            out.push(dedent_token);
        }
        return out;
    }

    var line = lines[0];
    var rest = lines.slice(1);

    //new 4/24
    if (line.length === 0) {
        return [line].concat(insertTokens(rest, stack))
    }

    var indentation = calculateIndentation(line);
    var top = stack.length ? stack[stack.length-1] : 0;

{'fraction_non_alphanumeric': 0.17870722433460076, 'fraction_numerical': 0.017381857686040194, 'mean_word_length': 2.2717584369449377, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6299480', 'n_tokens_mistral': 631, 'n_tokens_neox': 616, 'n_words': 132}