<commit_before>var fs = require("fs");

if(process.argv.length < 3) {
    console.error("Missing filename");
    process.exit(1);
}

var DEBUG=false;

var label_re=/^>.+/;
var import_re=/^<.+>$/;

var num_expr_re=/^[\s\d\-\+\*\/\(\)]+$/;

var re_flags = ["g", "i", "m"];

Array.prototype.contains = function(a) {
    return this.indexOf(a) !== -1;
};

function re_filter(a) {
    return re_flags.contains(a);
}

function action_filter(a) {
    return !re_flags.contains(a);
}

var refs = {};

function Block(line, parent_block) {
    line = line || "/";

    if(DEBUG) {
        this.line = line;
    }

    this.parent_block = parent_block || null;

    this.children = [];

    // Parse the code
    this.code = {
        flags: []
    };

    if(label_re.test(line)) {
        this.code.ref = line.replace(/^>/, "");
        refs[this.code.ref] = this;
    } else if(import_re.test(line)) {
        this.code.import_ref = line.replace(/^</, "").replace(/>$/, "");
    } else {
        var separator = line[0];
        var parts = line.split(separator);

        parts.splice(0, 1);

        if(parts.length > 1) {
            this.code.flags = parts[parts.length - 1].split("");
            
            if(parts.length > 2) {
                this.code.replacement = parts[1];
            }
        }

        if(this.code.flags.contains("b")) {
            if(!/^\^/.test(parts[0])) {
                parts[0] = "^.*" + parts[0];
            }

            if(!/\$$/.test(parts[0])) {
                parts[0] += ".*$";
            }
        }

        this.code.match = new RegExp(parts[0], this.code.flags.filter(re_filter));

        this.code.flags = this.code.flags.filter(action_filter);
    }
}

Block.prototype.add_child = function(line) {
    this.children.push(new Block(line, this));
};

Block.prototype.execute = function(data, by_ref) {
    var do_children = false;
    var new_data = data;

    if(DEBUG) {
        console.log("LINE:", this.line);
    }

    if(this.code.hasOwnProperty("ref")) {
        if(by_ref) {
            do_children = true;
        }
    } else if(this.code.hasOwnProperty("import_ref")) {
        new_data = refs[this.code.import_ref].execute(data, true);
    } else {
        if(DEBUG) {
            this.print(false, "Exec: ");
        }

        var match = this.code.match.exec(data);

        if(match) {
            do_children = true;

            if(this.code.hasOwnProperty("replacement")) {
                var replacement = match[0].replace(this.code.match, this.code.replacement);

                if(this.code.flags.contains("n")) {
                    if(!num_expr_re.test(replacement)) {
                        console.error("Invalid expression:", replacement);
                        process.exit(1);
                    }

                    replacement = eval(replacement);
                }

                new_data = new_data.substring(0, match.index) + replacement + new_data.substring(match.index + match[0].length);

                match = replacement;
            } else {
                match = match[0];
            }

            if(this.code.flags.contains("w")) {
                match = new_data;
            }

            if(DEBUG) {
                console.log("Operating on:", match);
            }

            // Now the actions
            if(this.code.flags.contains("p")) {
                console.log(match);
            }
        }
    }

    if(do_children) {
        this.children.forEach(function(child) {
            new_data = child.execute(new_data);
        });
    }

    if(this.code.flags.contains("t")) {
        new_data = data;
    }

    return new_data;
};

Block.prototype.print = function(recursive, indent) {
    indent = indent || "";

    var line = indent;

    if(this.code.hasOwnProperty("import_ref")) {
        line += "IMPORT " + this.code.import_ref;
    } else if(this.code.hasOwnProperty("ref")) {
        line += "REF " + this.code.ref;
    } else {
        line += this.code.match.source;
        
        if(this.code.hasOwnProperty("replacement")) {
            line += " -> " + this.code.replacement;
        }

        if(this.code.flags.length > 0) {
            line += " (" + this.code.flags.join(", ") + ")";
        }
    }

    console.log(line);
<commit_msg>Convert parser over to callback style code<commit_after>var fs = require("fs");

if(process.argv.length < 3) {
    console.error("Missing filename");
    process.exit(1);
}

var DEBUG=false;

var label_re=/^>.+/;
var import_re=/^<.+>$/;

var num_expr_re=/^[\s\d\-\+\*\/\(\)]+$/;

var re_flags = ["g", "i", "m"];

Array.prototype.contains = function(a) {
    return this.indexOf(a) !== -1;
};

function re_filter(a) {
    return re_flags.contains(a);
}

function action_filter(a) {
    return !re_flags.contains(a);
}

var refs = {};

function Block(line, parent_block) {
    line = line || "/";

    if(DEBUG) {
        this.line = line;
    }

    this.parent_block = parent_block || null;

    this.children = [];

    // Parse the code
    this.code = {
        flags: []
    };

    if(label_re.test(line)) {
        this.code.ref = line.replace(/^>/, "");
        refs[this.code.ref] = this;
    } else if(import_re.test(line)) {
        this.code.import_ref = line.replace(/^</, "").replace(/>$/, "");
    } else {
        var separator = line[0];
        var parts = line.split(separator);

        parts.splice(0, 1);

        if(parts.length > 1) {
            this.code.flags = parts[parts.length - 1].split("");
            
            if(parts.length > 2) {
                this.code.replacement = parts[1];
            }
        }

        if(this.code.flags.contains("b")) {
            if(!/^\^/.test(parts[0])) {
                parts[0] = "^.*" + parts[0];
            }

            if(!/\$$/.test(parts[0])) {
                parts[0] += ".*$";
            }
        }

        this.code.match = new RegExp(parts[0], this.code.flags.filter(re_filter));

        this.code.flags = this.code.flags.filter(action_filter);
    }
}

Block.prototype.add_child = function(line) {
    this.children.push(new Block(line, this));
};

Block.prototype.perform_match = function(data, callback) {
    var match = this.code.match.exec(data);

    if(this.code.hasOwnProperty("replacement")) {
        var replacement = match[0].replace(this.code.match, this.code.replacement);

        if(this.code.flags.contains("n")) {
            if(!num_expr_re.test(replacement)) {
                console.error("Invalid expression:", replacement);
                process.exit(1);
            }

            replacement = eval(replacement);
        }

        data = data.substring(0, match.index) + replacement + data.substring(match.index + match[0].length);

        match = replacement;
    } else {
        match = match[0];
    }

    if(this.code.flags.contains("w")) {
        match = data;
    }

    // Now the actions
    if(this.code.flags.contains("p")) {
        console.log(match);
    }

    this.execute_children(data, callback);
};

Block.prototype.execute_children = function(data, callback) {
    for(var i=0; i<this.children.length; i++) {
        (function(child) {
            var old_callback = callback;

            callback = function(d) {
                child.execute(d, false, old_callback);
            };
        }(this.children[this.children.length - i - 1]));
    }

    callback(data);
};

Block.prototype.execute = function(data, by_ref, callback) {
    var do_children = false;

    var old_callback = callback;

    callback = callback || function(){
        if(DEBUG) {
            console.log("EOL");
        }
    };

    if(DEBUG) {
        console.log("LINE:", this.line);
    }

    // Handle temporary data
    if(this.code.flags.contains("t")) {
        callback = function() {
            old_callback(data);
        };
    }

    if(this.code.hasOwnProperty("ref")) {
        if(by_ref) {
            if(DEBUG) {
                console.log("Call: ", this.code.ref);
            }

            this.execute_children(data, callback);
        } else {
            if(DEBUG) {
                console.log("Skip");
            }

            callback(data);
        }
    } else if(this.code.hasOwnProperty("import_ref")) {
        if(DEBUG) {
            console.log("Incl:", this.code.import_ref);
        }

        refs[this.code.import_ref].execute(data, true, callback);
    } else {
        if(DEBUG) {
            this.print(false, "Exec: ");
        }

        if(this.code.match.test(data)) {
            if(this.code.flags.contains("o")) {
                process.stdin.on("data", function(data) {
                    process.stdin.removeAllListeners("data");

                    this.perform_match(data, callback);
                });
            } else {
                this.perform_match(data, callback);
            }
        } else {
            callback(data);
        }
    }
};

Block.prototype.print = function(recursive, indent) {
    indent = indent || "";

    var line = indent;

    if(this.code.hasOwnProperty("import_ref")) {
        line += "IMPORT " + this.code.import_ref;
    } else if(this.code.hasOwnProperty("ref")) {
        line += "REF " + this.code.ref;
    } else {
        line += this.code.match.source;
        
        if(this.code.hasOwnProperty("replacement")) {
            line += " -> " + this.code.replacement;
        }

        if(this.code.flags.length > 0) {
            line += " (" + this.code.flags.join(", ") + ")";
        }
    }

    console.log(line);

{'fraction_non_alphanumeric': 0.16750471401634193, 'fraction_numerical': 0.004818772260632726, 'mean_word_length': 1.773678094131319, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1582402', 'n_tokens_mistral': 3081, 'n_tokens_neox': 2971, 'n_words': 454}