<commit_before>(function (root) {
    "use strict";

    var
        SEPERATOR_REGEX     = /^\-\>/,
        TYPE_REGEX          = /^[A-z]+/,
        VALUE_REGEX         = /^#{(.+?)}/,
        WHITESPACE_REGEX    = /^[\ \t]+/,

        consume = function (string) {
            var result = {
                type:   undefined,
                value:  undefined,
                left:   undefined
            },
            intermediateResult;

            //Consume leading whitespace
            string = consumeWhitespace(string).left;

            intermediateResult = consumeType(string);
            string             = intermediateResult.left;
            result.type        = intermediateResult.result;

            string = consumeWhitespace(string).left;

            intermediateResult = consumeValue(string);
            string             = intermediateResult.left;
            result.value       = intermediateResult.result;

            string = consumeWhitespace(string).left;
            intermediateResult = consumeSeparator(string);
            string             = intermediateResult.left;


            result.left = consumeWhitespace(string).left;


            return result;
        },

        consumeType = function (string) {
            var found = string.match(TYPE_REGEX);

            if (found) {
                return {
                    result: found[0],
                    consumed: found[0].length,
                    left: string.substring(found[0].length)
                };
            }

            return {
                result: undefined,
                consumed: 0,
                left: string
            };
        },

        consumeValue = function (string) {
            var found = string.match(VALUE_REGEX),
                value = undefined;

            if (found) {
                if (found[1] === "true" || found[1] === "false") {
                    value = found[1] === "true";
                }
                else if (!isNaN(parseInt(found[1], 10))) {
                    value = parseInt(found[1], 10);
                }
                else {
                    value = found[1];
                }

                return {
                    result: value,
                    consumed: found[0].length,
                    left: string.substring(found[0].length)
                };
            }

            return {
                result: undefined,
                consumed: 0,
                left: string
            };
        },

        consumeSeparator = function (string) {
            var found = string.match(SEPERATOR_REGEX);

            if (found) {
                return {
                    consumed: found[0].length,
                    left: string.substring(found[0].length)
                };
            }

            return {
                consumed: 0,
                left: string
            };
        },

        consumeWhitespace = function (string) {
            var found = string.match(WHITESPACE_REGEX);

            if (found) {
                return {
                    consumed: found[0].length,
                    left: string.substring(found[0].length)
                };
            }

            return {
                consumed: 0,
                left: string
            };
        },

        ptrn = function ptrn (hash) {
            var struct = {},
                k, i, types, match, current, consumeResult;

            for (k in hash) {
                if (hash.hasOwnProperty(k)) {
                    match   = k;
                    current = struct;

                    do {
                        consumeResult = consume(match);

                        if (consumeResult.type && !current[consumeResult.type]) {
                            current[consumeResult.type] = {};
                        }
                        current = current[consumeResult.type];

                        if (consumeResult.value) {
                            if (!current["__values"]) {
                                current["__values"] = {};
                            }
                            current = (current["__values"][consumeResult.value] = {});
                        }
                        match = consumeResult.left;
                    } while(match.length != 0);

                    current["__fn"] = hash[k];
                }
            }

            return function () {
                var argc = arguments.length,
                    i,
                    arg,
                    type,
                    current = struct;

                for (i = 0; i < argc; i += 1) {
                    arg     = arguments[i];
                    type    = typeof arg;

                    if (current[type]) {
                        current = current[type];
                        if (current["__values"] && current["__values"]["" + arg]) {
                            current = current["__values"]["" + arg];
                        }
                    }
                    else {
                        return;
                    }
                }

                if (current["__fn"]) {
                    return current["__fn"].apply(this, arguments);

                }

                return;
            };
        };

    root.ptrn = ptrn;
} (typeof window !== "undefined" ? window : module.exports));<commit_msg>Use supplied types for lookups in the hash structure instead of incorrectly converting them to strings<commit_after>(function (root) {
    "use strict";

    var
        SEPERATOR_REGEX     = /^\-\>/,
        TYPE_REGEX          = /^[A-z]+/,
        VALUE_REGEX         = /^#{(.+?)}/,
        WHITESPACE_REGEX    = /^[\ \t]+/,

        consume = function (string) {
            var result = {
                type:   undefined,
                value:  undefined,
                left:   undefined
            },
            intermediateResult;

            //Consume leading whitespace
            string = consumeWhitespace(string).left;

            intermediateResult = consumeType(string);
            string             = intermediateResult.left;
            result.type        = intermediateResult.result;

            string = consumeWhitespace(string).left;

            intermediateResult = consumeValue(string);
            string             = intermediateResult.left;
            result.value       = intermediateResult.result;

            string = consumeWhitespace(string).left;
            intermediateResult = consumeSeparator(string);
            string             = intermediateResult.left;


            result.left = consumeWhitespace(string).left;


            return result;
        },

        consumeType = function (string) {
            var found = string.match(TYPE_REGEX);

            if (found) {
                return {
                    result: found[0],
                    consumed: found[0].length,
                    left: string.substring(found[0].length)
                };
            }

            return {
                result: undefined,
                consumed: 0,
                left: string
            };
        },

        consumeValue = function (string) {
            var found = string.match(VALUE_REGEX),
                value = undefined;

            if (found) {
                if (found[1] === "true" || found[1] === "false") {
                    value = found[1] === "true";
                }
                else if (!isNaN(parseInt(found[1], 10))) {
                    value = parseInt(found[1], 10);
                }
                else {
                    value = found[1];
                }

                return {
                    result: value,
                    consumed: found[0].length,
                    left: string.substring(found[0].length)
                };
            }

            return {
                result: undefined,
                consumed: 0,
                left: string
            };
        },

        consumeSeparator = function (string) {
            var found = string.match(SEPERATOR_REGEX);

            if (found) {
                return {
                    consumed: found[0].length,
                    left: string.substring(found[0].length)
                };
            }

            return {
                consumed: 0,
                left: string
            };
        },

        consumeWhitespace = function (string) {
            var found = string.match(WHITESPACE_REGEX);

            if (found) {
                return {
                    consumed: found[0].length,
                    left: string.substring(found[0].length)
                };
            }

            return {
                consumed: 0,
                left: string
            };
        },

        ptrn = function ptrn (hash) {
            var struct = {},
                k, i, types, match, current, consumeResult;

            for (k in hash) {
                if (hash.hasOwnProperty(k)) {
                    match   = k;
                    current = struct;

                    do {
                        consumeResult = consume(match);

                        if (consumeResult.type && !current[consumeResult.type]) {
                            current[consumeResult.type] = {};
                        }
                        current = current[consumeResult.type];

                        if (consumeResult.value) {
                            if (!current["__values"]) {
                                current["__values"] = {};
                            }
                            current = (current["__values"][consumeResult.value] = {});
                        }
                        match = consumeResult.left;
                    } while(match.length != 0);

                    current["__fn"] = hash[k];
                }
            }

            return function () {
                var argc = arguments.length,
                    i,
                    arg,
                    type,
                    current = struct;

                for (i = 0; i < argc; i += 1) {
                    arg     = arguments[i];
                    type    = typeof arg;

                    if (current[type]) {
                        current = current[type];
                        if (current["__values"] && current["__values"][arg]) {
                            current = current["__values"][arg];
                        }
                    }
                    else {
                        return;
                    }
                }

                if (current["__fn"]) {
                    return current["__fn"].apply(this, arguments);

                }

                return;
            };
        };

    root.ptrn = ptrn;
} (typeof window !== "undefined" ? window : module.exports));
{'fraction_non_alphanumeric': 0.1006358860934476, 'fraction_numerical': 0.004792185052068933, 'mean_word_length': 0.92070796460177, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1939742', 'n_tokens_mistral': 2633, 'n_tokens_neox': 2416, 'n_words': 500}