<commit_before>(function(window,document,undefined){
    var Pizzle = (function(window){
        var Express = {};
        Express.filter = {
            "ID":function(id){
                return function(elem){
                    return elem.getAttribute("id") === id;
                }
            },

            "CLASS":function(className){
                var pattern;
                return (pattern = new RegExp( "(^|" + Express.expr['whitespace'] + ")" + className + "(" + Express.expr['whitespace'] + "|$)" ))
                    && function(elem){
                        return pattern.test( typeof elem.className === "string" && elem.className || typeof elem.getAttribute !== "undefined" && elem.getAttribute("class") || "" );
                    }
            },

            "TAG":function(nodeNameSelector){
                var nodeName = nodeNameSelector.toLowerCase();
                return nodeNameSelector === "*" ? function(){return true;} :function(elem){
                    return elem.nodeName && elem.nodeName.toLowerCase() === nodeName;
                }
            },

            "ATTR":function(name, operator, check){

            },

            "CHILD":function(type, what, argument, first, last){

            }
        };

        function init(){
            Express.expr = getExpr();
            Express.matchExpr = getMatchExpr();
        }

        function getExpr(){
            var whitespace = "[\\x20\\t\\r\\n\\f]";//空白字符正则字符串
            var operators = "([*^$|!~]?=)";//可用的属性操作符
            var rcomma = "^" + whitespace + "*," + whitespace + "*";//并联选择器的正则
            var rcombinators = "^" + whitespace + "*([\\x20\\t\\r\\n\\f>+~])" + whitespace + "*";//关系选择器正则
            var characterEncoding = "(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+";//字符编码正则字符串
            var identifier = characterEncoding.replace( "w", "w#" );
            var attributes = "\\[" + whitespace + "*(" + characterEncoding + ")" + whitespace +
                "*(?:" + operators + whitespace + "*(?:(['\"])((?:\\\\.|[^\\\\])*?)\\3|(" + identifier + ")|)|)" + whitespace + "*\\]";//属性选择器正则
            var pseudos = ":(" + characterEncoding + ")(?:\\(((['\"])((?:\\\\.|[^\\\\])*?)\\3|((?:\\\\.|[^\\\\()[\\]]|" + attributes.replace( 3, 8 ) + ")*)|.*)\\)|)";//伪类正则字符串
            var rtrim = "^" + whitespace + "+|((?:^|[^\\\\])(?:\\\\.)*)" + whitespace + "+$"; // 去掉两端空白和字符串中的反斜杠（如果连续两个去掉一个）
            //var rquickExpr = /^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/;//仅仅单个id或tag、class选择器正则（用来快速解析并获取元素）
            var rquickExpr = /^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/;
            var expando = "pizzle" + 1 * new Date();

            return {
                rcomma:new RegExp(rcomma),
                rcombinators:new RegExp(rcombinators),
                characterEncoding:characterEncoding,
                attributes:attributes,
                pseudos:pseudos,
                rtrim:new RegExp(rtrim,"g"),
                rquickExpr:new RegExp(rquickExpr),
                expando:expando
            };
        }

        function getMatchExpr(){
            var idExpr = "^#(" + Express.expr['characterEncoding'] + ")";
            var classExpr = "^\\.(" + Express.expr['characterEncoding'] + ")";
            var tagExpr =  "^(" + Express.expr['characterEncoding'].replace( "w", "w*" ) + ")";
            var attrExpr = "^" + Express.expr['attributes'];
            var pseudosExpr = "^" + Express.expr['pseudos'];
            var childEpr = "^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\(" + Express.expr['whitespace'] +
                "*(even|odd|(([+-]|)(\\d*)n|)" + Express.expr['whitespace'] + "*(?:([+-]|)" + Express.expr['whitespace'] +
                "*(\\d+)|))" + Express.expr['whitespace'] + "*\\)|)";

            return {
                "ID":new RegExp(idExpr),
                "CLASS":new RegExp(classExpr),
                "TAG":new RegExp(tagExpr),
                "ATTR":new RegExp(attrExpr),
                "PSEUDO":new RegExp(pseudosExpr),
                "CHILD":new RegExp(childEpr, "i" )
            };
        }

        function Pizzle(selector){
            if(!selector)
                return;

            init();
            return Pizzle.participle(selector);


            //return Pizzle.tokenize(selector);
        }

        /**
         * participle selector
         * @param selector
         */
        Pizzle.participle = function(selector){
            var match,matched,tokens = [],groups = [];

            //Handle HTML strings
            if(typeof selector === "string"){
                if ( selector.charAt(0) === "<" && selector.charAt( selector.length - 1 ) === ">" && selector.length >= 3 ) {
                    // Assume that strings that start and end with <> are HTML and skip the regex check
                    var m = selector.substring(1,selector.length - 1);
                    match = [ null, null,m, null ];
                } else {
                    match = Express.expr.rquickExpr.exec(selector);
                }
            }

            matched = match[1] || match[2] || match[3] ||'';

            if(matched){
                var type = (matched == match[1]) ? "ID" : ((matched == match[2]) ? "TAG" : (matched == match[3]) ? "CLASS" : "");
                tokens.push({
                    value: matched,
                    type: type,
                    matches: selector
                });
               return groups.push(tokens);
            }else{
               return Pizzle.tokenize(selector);
            }
        };

        /**
         * 解析多个选择器
         * @param selector
         * @returns {Array}
         */
        Pizzle.tokenize = function(selector){
            var matched,match,tokens,type,str = selector,groups = [];
            while(str){
                //,
                if(!matched || (match = Express.expr.rcomma.exec(str))){
                    if(match){
                        str = str.slice(match[0].length) || str;
                    }
                    groups.push(tokens = []);<commit_msg>Update pizzle.js<commit_after>(function(window,document,undefined){
    var Pizzle = (function(window){
        var Express = {};
        Express.filter = {
            "ID":function(id){
                return function(elem){
                    return elem.getAttribute("id") === id;
                }
            },

            "CLASS":function(className){
                var pattern;
                return (pattern = new RegExp( "(^|" + Express.expr['whitespace'] + ")" + className + "(" + Express.expr['whitespace'] + "|$)" ))
                    && function(elem){
                        return pattern.test( typeof elem.className === "string" && elem.className || typeof elem.getAttribute !== "undefined" && elem.getAttribute("class") || "" );
                    }
            },

            "TAG":function(nodeNameSelector){
                var nodeName = nodeNameSelector.toLowerCase();
                return nodeNameSelector === "*" ? function(){return true;} :function(elem){
                    return elem.nodeName && elem.nodeName.toLowerCase() === nodeName;
                }
            },

            "ATTR":function(name, operator, check){

            },

            "CHILD":function(type, what, argument, first, last){

            }
        };

        function init(){
            Express.expr = getExpr();
            Express.matchExpr = getMatchExpr();
        }

        function getExpr(){
            var whitespace = "[\\x20\\t\\r\\n\\f]";//空白字符正则字符串
            var operators = "([*^$|!~]?=)";//可用的属性操作符
            var rcomma = "^" + whitespace + "*," + whitespace + "*";//并联选择器的正则
            var rcombinators = "^" + whitespace + "*([\\x20\\t\\r\\n\\f>+~])" + whitespace + "*";//关系选择器正则
            var characterEncoding = "(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+";//字符编码正则字符串
            var identifier = characterEncoding.replace( "w", "w#" );
            var attributes = "\\[" + whitespace + "*(" + characterEncoding + ")" + whitespace +
                "*(?:" + operators + whitespace + "*(?:(['\"])((?:\\\\.|[^\\\\])*?)\\3|(" + identifier + ")|)|)" + whitespace + "*\\]";//属性选择器正则
            var pseudos = ":(" + characterEncoding + ")(?:\\(((['\"])((?:\\\\.|[^\\\\])*?)\\3|((?:\\\\.|[^\\\\()[\\]]|" + attributes.replace( 3, 8 ) + ")*)|.*)\\)|)";//伪类正则字符串
            var rtrim = "^" + whitespace + "+|((?:^|[^\\\\])(?:\\\\.)*)" + whitespace + "+$"; // 去掉两端空白和字符串中的反斜杠（如果连续两个去掉一个）
            //var rquickExpr = /^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/;//仅仅单个id或tag、class选择器正则（用来快速解析并获取元素）
            var rquickExpr = /^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/;
            var expando = "pizzle" + 1 * new Date();

            return {
                rcomma:new RegExp(rcomma),
                rcombinators:new RegExp(rcombinators),
                characterEncoding:characterEncoding,
                attributes:attributes,
                pseudos:pseudos,
                rtrim:new RegExp(rtrim,"g"),
                rquickExpr:new RegExp(rquickExpr),
                expando:expando
            };
        }

        function getMatchExpr(){
            var idExpr = "^#(" + Express.expr['characterEncoding'] + ")";
            var classExpr = "^\\.(" + Express.expr['characterEncoding'] + ")";
            var tagExpr =  "^(" + Express.expr['characterEncoding'].replace( "w", "w*" ) + ")";
            var attrExpr = "^" + Express.expr['attributes'];
            var pseudosExpr = "^" + Express.expr['pseudos'];
            var childEpr = "^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\(" + Express.expr['whitespace'] +
                "*(even|odd|(([+-]|)(\\d*)n|)" + Express.expr['whitespace'] + "*(?:([+-]|)" + Express.expr['whitespace'] +
                "*(\\d+)|))" + Express.expr['whitespace'] + "*\\)|)";

            return {
                "ID":new RegExp(idExpr),
                "CLASS":new RegExp(classExpr),
                "TAG":new RegExp(tagExpr),
                "ATTR":new RegExp(attrExpr),
                "PSEUDO":new RegExp(pseudosExpr),
                "CHILD":new RegExp(childEpr, "i" )
            };
        }

        function Pizzle(selector){
            if(!selector)
                return;

            init();
            return Pizzle.participle(selector);
        }

        /**
         * participle selector
         * @param selector
         */
        Pizzle.participle = function(selector){
            var match,matched,tokens = [],groups = [];

            //Handle HTML strings
            if(typeof selector === "string"){
                if ( selector.charAt(0) === "<" && selector.charAt( selector.length - 1 ) === ">" && selector.length >= 3 ) {
                    // Assume that strings that start and end with <> are HTML and skip the regex check
                    var m = selector.substring(1,selector.length - 1);
                    match = [ null, null,m, null ];
                } else {
                    match = Express.expr.rquickExpr.exec(selector);
                }
            }

            if(match){
                matched = match[1] || match[2] || match[3] ||'';
                var type = (matched == match[1]) ? "ID" : ((matched == match[2]) ? "TAG" : (matched == match[3]) ? "CLASS" : "");
                tokens.push({
                    value: matched,
                    type: type,
                    matches: selector
                });
               return groups.push(tokens);
            }else{
               return Pizzle.tokenize(selector);
            }
        };

        /**
         * 解析多个选择器
         * @param selector
         * @returns {Array}
         */
        Pizzle.tokenize = function(selector){
            var matched,match,tokens,type,str = selector,groups = [];
            while(str){
                //,
                if(!matched || (match = Express.expr.rcomma.exec(str))){
                    if(match){
                        str = str.slice(match[0].length) || str;
                    }
                    groups.push(tokens = []);
{'fraction_non_alphanumeric': 0.1941029900332226, 'fraction_numerical': 0.003986710963455149, 'mean_word_length': 1.544052398056201, 'pattern_counts': {'":': 24, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1254117', 'n_tokens_mistral': 3689, 'n_tokens_neox': 3574, 'n_words': 565}