<commit_before>/*!
 * XRegExp v1.6.0-dev
 * Copyright 2007-2012 <NAME> <http://xregexp.com/>
 * Available under the MIT License
 * Augmented, extensible, cross-browser regular expressions
 */

// Avoid running twice; that could break references to native globals
;if (typeof XRegExp === "undefined") {
(function (root, undefined) {
    "use strict";

    //---------------------------------
    //  Constructor
    //---------------------------------

    // Accepts a pattern and flags; returns a new, extended `RegExp` object. Differs from a native
    // regular expression in that additional syntax and flags are supported and cross-browser
    // syntax inconsistencies are ameliorated. `XRegExp(/regex/)` clones an existing regex
    function XRegExp (pattern, flags) {
        if (XRegExp.isRegExp(pattern)) {
            if (flags !== undefined)
                throw new TypeError("can't supply flags when constructing one RegExp from another");
            return copy(pattern);
        }
        // Tokens become part of the regex construction process, so protect against infinite
        // recursion when an XRegExp is constructed within a token handler or trigger
        if (isInsideConstructor)
            throw new Error("can't call the XRegExp constructor within token definition functions");

        var output = [],
            currScope = XRegExp.OUTSIDE_CLASS,
            context = { // `this` object for custom tokens
                hasNamedCapture: false,
                captureNames: [],
                hasFlag: function (flag) {return flags.indexOf(flag) > -1;},
                setFlag: function (flag) {flags += flag;}
            },
            pos = 0,
            tokenResult, match, chr;

        pattern = pattern === undefined ? "" : pattern + "";
        flags = flags === undefined ? "" : flags + "";

        while (pos < pattern.length) {
            // Check for custom tokens at the current position
            tokenResult = runTokens(pattern, pos, currScope, context);

            if (tokenResult) {
                output.push(tokenResult.output);
                pos += (tokenResult.match[0].length || 1);
            } else {
                // Check for native multicharacter metasequences (excluding character classes) at
                // the current position
                if (match = nativ.exec.call(nativeTokens[currScope], pattern.slice(pos))) {
                    output.push(match[0]);
                    pos += match[0].length;
                } else {
                    chr = pattern.charAt(pos);
                    if (chr === "[")
                        currScope = XRegExp.INSIDE_CLASS;
                    else if (chr === "]")
                        currScope = XRegExp.OUTSIDE_CLASS;
                    // Advance position one character
                    output.push(chr);
                    pos++;
                }
            }
        }

        return augment(RegExp(output.join(""), nativ.replace.call(flags, flagClip, "")), context);
    }


    //---------------------------------
    //  Public properties
    //---------------------------------

    XRegExp.version = "1.6.0-dev";

    // Token scope bitflags
    XRegExp.INSIDE_CLASS = 1;
    XRegExp.OUTSIDE_CLASS = 2;


    //---------------------------------
    //  Private variables
    //---------------------------------

    // Used during XRegExp construction
    var isInsideConstructor = false;

    // Store native globals to use and restore ("native" is an ES3 reserved keyword)
    var nativ = {
        exec: RegExp.prototype.exec,
        test: RegExp.prototype.test,
        match: String.prototype.match,
        replace: String.prototype.replace,
        split: String.prototype.split,
        // Hold these so they can be given back if they were added before XRegExp ran
        apply: RegExp.prototype.apply,
        call: RegExp.prototype.call
    };

    // Storage for fixed/extended native methods
    var fixed = {};

    // Storage for addon tokens
    var tokens = [];

    // Optional features (can be installed and uninstalled)
    var features = {
        natives: false,
        methods: false,
        extensibility: false
    };

    // Any backreference in replacement strings
    var replacementToken = /\$(?:(\d\d?|[$&`'])|{([$\w]+)})/g;

    // Nonnative and duplicate flags
    var flagClip = /[^gimy]+|([\s\S])(?=[\s\S]*\1)/g;

    // Any greedy or lazy regex quantifier
    var quantifier = /^(?:[?*+]|{\d+(?:,\d*)?})\??/;

    // All native RegExp multicharacter metasequences (including deprecated octals, excluding
    // character classes)
    var nativeTokens = {};
    nativeTokens[XRegExp.INSIDE_CLASS] = /^(?:\\(?:[0-3][0-7]{0,2}|[4-7][0-7]?|x[\dA-Fa-f]{2}|u[\dA-Fa-f]{4}|c[A-Za-z]|[\s\S]))/;
    nativeTokens[XRegExp.OUTSIDE_CLASS] = /^(?:\\(?:0(?:[0-3][0-7]{0,2}|[4-7][0-7]?)?|[1-9]\d*|x[\dA-Fa-f]{2}|u[\dA-Fa-f]{4}|c[A-Za-z]|[\s\S])|\(\?[:=!]|[?*+]\?|{\d+(?:,\d*)?}\??)/;

    // Check for correct `exec` handling of nonparticipating capturing groups
    var compliantExecNpcg = nativ.exec.call(/()??/, "")[1] === undefined;

    // Check for correct handling of `lastIndex` after zero-length matches
    var compliantLastIndexIncrement = function () {
        var x = /^/g;
        nativ.test.call(x, "");
        return !x.lastIndex;
    }();

    // Check for "y" flag support
    var hasNativeY = RegExp.prototype.sticky !== undefined;


    //---------------------------------
    //  Public methods
    //---------------------------------

    function activeAddToken (regex, handler, scope, trigger) {
        tokens.push({
            pattern: copy(regex, "g" + (hasNativeY ? "y" : "")),
            handler: handler,
            scope: scope || XRegExp.OUTSIDE_CLASS,
            trigger: trigger || null<commit_msg>create private token scope vars<commit_after>/*!
 * XRegExp v1.6.0-dev
 * Copyright 2007-2012 <NAME> <http://xregexp.com/>
 * Available under the MIT License
 * Augmented, extensible, cross-browser regular expressions
 */

// Avoid running twice; that could break references to native globals
;if (typeof XRegExp === "undefined") {
(function (root, undefined) {
    "use strict";

    //---------------------------------
    //  Constructor
    //---------------------------------

    // Accepts a pattern and flags; returns a new, extended `RegExp` object. Differs from a native
    // regular expression in that additional syntax and flags are supported and cross-browser
    // syntax inconsistencies are ameliorated. `XRegExp(/regex/)` clones an existing regex
    function XRegExp (pattern, flags) {
        if (XRegExp.isRegExp(pattern)) {
            if (flags !== undefined)
                throw new TypeError("can't supply flags when constructing one RegExp from another");
            return copy(pattern);
        }
        // Tokens become part of the regex construction process, so protect against infinite
        // recursion when an XRegExp is constructed within a token handler or trigger
        if (isInsideConstructor)
            throw new Error("can't call the XRegExp constructor within token definition functions");
        var output = [],
            scope = defaultScope,
            pos = 0,
            context = { // `this` object for custom tokens
                hasNamedCapture: false,
                captureNames: [],
                hasFlag: function (flag) {return flags.indexOf(flag) > -1;},
                setFlag: function (flag) {flags += flag;}
            },
            tokenResult, match, chr;
        pattern = pattern === undefined ? "" : pattern + "";
        flags = flags === undefined ? "" : flags + "";
        while (pos < pattern.length) {
            // Check for custom tokens at the current position
            tokenResult = runTokens(pattern, pos, scope, context);
            if (tokenResult) {
                output.push(tokenResult.output);
                pos += (tokenResult.match[0].length || 1);
            } else {
                // Check for native multicharacter metasequences (excluding character classes) at
                // the current position
                if (match = nativ.exec.call(nativeTokens[scope], pattern.slice(pos))) {
                    output.push(match[0]);
                    pos += match[0].length;
                } else {
                    chr = pattern.charAt(pos);
                    if (chr === "[")
                        scope = classScope;
                    else if (chr === "]")
                        scope = defaultScope;
                    // Advance position one character
                    output.push(chr);
                    pos++;
                }
            }
        }
        return augment(RegExp(output.join(""), nativ.replace.call(flags, flagClip, "")), context);
    }


    //---------------------------------
    //  Public properties
    //---------------------------------

    XRegExp.version = "1.6.0-dev";

    // Token scope bitflags
    // Create private copies to protect core operations
    var classScope = XRegExp.INSIDE_CLASS = 1;
    var defaultScope = XRegExp.OUTSIDE_CLASS = 2;


    //---------------------------------
    //  Private variables
    //---------------------------------

    // Used during XRegExp construction
    var isInsideConstructor = false;

    // Store native globals to use and restore ("native" is an ES3 reserved keyword)
    var nativ = {
        exec: RegExp.prototype.exec,
        test: RegExp.prototype.test,
        match: String.prototype.match,
        replace: String.prototype.replace,
        split: String.prototype.split,
        // Hold these so they can be given back if they were added before XRegExp ran
        apply: RegExp.prototype.apply,
        call: RegExp.prototype.call
    };

    // Storage for fixed/extended native methods
    var fixed = {};

    // Storage for addon tokens
    var tokens = [];

    // Optional features (can be installed and uninstalled)
    var features = {
        natives: false,
        methods: false,
        extensibility: false
    };

    // Any backreference in replacement strings
    var replacementToken = /\$(?:(\d\d?|[$&`'])|{([$\w]+)})/g;

    // Nonnative and duplicate flags
    var flagClip = /[^gimy]+|([\s\S])(?=[\s\S]*\1)/g;

    // Any greedy or lazy regex quantifier
    var quantifier = /^(?:[?*+]|{\d+(?:,\d*)?})\??/;

    // All native RegExp multicharacter metasequences (including deprecated octals, excluding
    // character classes)
    var nativeTokens = {};
    nativeTokens[classScope] = /^(?:\\(?:[0-3][0-7]{0,2}|[4-7][0-7]?|x[\dA-Fa-f]{2}|u[\dA-Fa-f]{4}|c[A-Za-z]|[\s\S]))/;
    nativeTokens[defaultScope] = /^(?:\\(?:0(?:[0-3][0-7]{0,2}|[4-7][0-7]?)?|[1-9]\d*|x[\dA-Fa-f]{2}|u[\dA-Fa-f]{4}|c[A-Za-z]|[\s\S])|\(\?[:=!]|[?*+]\?|{\d+(?:,\d*)?}\??)/;

    // Check for correct `exec` handling of nonparticipating capturing groups
    var compliantExecNpcg = nativ.exec.call(/()??/, "")[1] === undefined;

    // Check for correct handling of `lastIndex` after zero-length matches
    var compliantLastIndexIncrement = function () {
        var x = /^/g;
        nativ.test.call(x, "");
        return !x.lastIndex;
    }();

    // Check for "y" flag support
    var hasNativeY = RegExp.prototype.sticky !== undefined;


    //---------------------------------
    //  Public methods
    //---------------------------------

    function activeAddToken (regex, handler, scope, trigger) {
        tokens.push({
            pattern: copy(regex, "g" + (hasNativeY ? "y" : "")),
            handler: handler,
            scope: scope || defaultScope,
            trigger: trigger || null
{'fraction_non_alphanumeric': 0.17924770799417358, 'fraction_numerical': 0.008910975923228514, 'mean_word_length': 1.7645665561345334, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3253640', 'n_tokens_mistral': 3309, 'n_tokens_neox': 3130, 'n_words': 874}