<commit_before>var Utils = GLOBAL.Utils = {

    getCount: function(a) {
        var b = 0;
        for(var c in a) {
            b++;
        }
        return b;
    },

    getBoundsParamsForWorld: function() {

        return {
            bounds: {
                sw: {
                    lat: function() {return -90;},
                    lng: function() {return -180;}
                },
                ne: {
                    lat: function() {return 90;},
                    lng: function() {return 180;}
                }
            }
        }

    },

    extractNormalizeFunction: function(nemesis) {

        var funcOriginal = nemesis.dashboard.requests.normalizeParamCount.toString();
        funcOriginal = funcOriginal.replace(/goog\.now/g, 'Date.now');
        funcOriginal = funcOriginal.replace(/goog\.object\.getCount/g, 'Utils.getCount');
        funcOriginal = funcOriginal.replace(/nemesis\.dashboard\.BoundsParams\.getBoundsParamsForWorld/g, 'Utils.getBoundsParamsForWorld');

        return funcOriginal;

    },

    createNormalizeFunction: function(str) {

        return new Function('obj', 'return (' + str + ')(obj)');
    },

    extractMungeFromStock: function(nemesis) {

        var munges = Utils.extractMungeFromStockIITC(nemesis);
        var new_munges = {};

        for (var key in munges) {
            new_munges[key.replace(/\./g, '_')] = munges[key];
        }

        return new_munges;

    },

    extractMungeFromStockIITC: function(nemesis) {

        var foundMunges = {};

        // these are easy - directly available in variables
        // NOTE: the .toString() is there so missing variables throw an exception, rather than storing 'undefined'
        foundMunges['dashboard.getArtifactInfo'] = nemesis.dashboard.requests.MethodName.GET_ARTIFACT_INFO.toString();
        foundMunges['dashboard.getGameScore'] = nemesis.dashboard.requests.MethodName.GET_GAME_SCORE.toString();
        foundMunges['dashboard.getPaginatedPlexts'] = nemesis.dashboard.requests.MethodName.GET_PAGINATED_PLEXTS.toString();
        foundMunges['dashboard.getThinnedEntities'] = nemesis.dashboard.requests.MethodName.GET_THINNED_ENTITIES.toString();
        foundMunges['dashboard.getPortalDetails'] = nemesis.dashboard.requests.MethodName.GET_PORTAL_DETAILS.toString();
        foundMunges['dashboard.redeemReward'] = nemesis.dashboard.requests.MethodName.REDEEM_REWARD.toString();
        foundMunges['dashboard.sendInviteEmail'] = nemesis.dashboard.requests.MethodName.SEND_INVITE_EMAIL.toString();
        foundMunges['dashboard.sendPlext'] = nemesis.dashboard.requests.MethodName.SEND_PLEXT.toString();

        // the rest are trickier - we need to parse the functions of the stock site. these break very often
        // on site updates

        // regular expression - to match either x.abcdef123456wxyz or x["123456abcdefwxyz"] format for property access
        var mungeRegExpProp = '(?:\\.([a-z][a-z0-9]{15})|\\["([0-9][a-z0-9]{15})"\\])';
        // and one to match members of object literal initialisation - {abcdef123456wxyz: or {"123456abcdefwxyz":
        var mungeRegExpLit = '(?:([a-z][a-z0-9]{15})|"([0-9][a-z0-9]{15})"):';

        // common parameters - method, version, version_parameter - currently found in the 
        // nemesis.dashboard.network.XhrController.prototype.doSendRequest_ function
        // look for something like
        //  var e = a.getData();
        //  e["3sld77nsm0tjmkvi"] = c;
        //  e.xz7q6r3aja5ttvoo = "b121024077de2a0dc6b34119e4440785c9ea5e64";
        var reg = new RegExp('getData\\(\\);.*\\n.*'+mungeRegExpProp+' =.*\n.*'+mungeRegExpProp+' *= *"([a-z0-9]{40})','m');
        var result = reg.exec(nemesis.dashboard.network.XhrController.prototype.doSendRequest_.toString());
        // there's two ways of matching the munge expression, so try both
        foundMunges.method = result[1] || result[2];
        foundMunges.version = result[3] || result[4];
        foundMunges.version_parameter = result[5];

        // GET_THINNED_ENTITIES parameters
        var reg = new RegExp('GET_THINNED_ENTITIES, nemesis.dashboard.network.XhrController.Priority.[A-Z]+, {'+mungeRegExpLit+'[a-z]');
        var result = reg.exec(nemesis.dashboard.network.DataFetcher.prototype.getGameEntities.toString());
        foundMunges.quadKeys = result[1] || result[2];

        // GET_PAGINATED_PLEXTS
        var reg = new RegExp('GET_PAGINATED_PLEXTS, [a-z] = [a-z] \\|\\| nemesis.dashboard.BoundsParams.getBoundsParamsForWorld\\(\\), [a-z] = [a-z] \\|\\| -1, [a-z] = [a-z] \\|\\| -1, [a-z] = {'+mungeRegExpLit+'[a-z], '+mungeRegExpLit+'Math.round\\([a-z].bounds.sw.lat\\(\\) \\* 1E6\\), '+mungeRegExpLit+'Math.round\\([a-z].bounds.sw.lng\\(\\) \\* 1E6\\), '+mungeRegExpLit+'Math.round\\([a-z].bounds.ne.lat\\(\\) \\* 1E6\\), '+mungeRegExpLit+'Math.round\\([a-z].bounds.ne.lng\\(\\) \\* 1E6\\), '+mungeRegExpLit+'[a-z], '+mungeRegExpLit+'[a-z]};\n *[a-z]'+mungeRegExpProp+' = [a-z];\n *[a-z] > -1 && \\([a-z]'+mungeRegExpProp+' = true\\);', 'm');
        var result = reg.exec(nemesis.dashboard.network.PlextStore.prototype.getPlexts.toString());

        foundMunges.desiredNumItems = result[1] || result[2];

        foundMunges.minLatE6 = result[3] || result[4];
        foundMunges.minLngE6 = result[5] || result[6];
        foundMunges.maxLatE6 = result[7] || result[8];
        foundMunges.maxLngE6 = result[9] || result[10];
        foundMunges.minTimestampMs = result[11] || result[12];
        foundMunges.maxTimestampMs = result[13] || result[14];
        foundMunges.chatTab = result[15] || result[16];  //guessed parameter name - only seen munged
        foundMunges.ascendingTimestampOrder = result[17] || result[18];
<commit_msg>update to latest munge detecter<commit_after>var Utils = GLOBAL.Utils = {

    getCount: function(a) {
        var b = 0;
        for(var c in a) {
            b++;
        }
        return b;
    },

    getBoundsParamsForWorld: function() {

        return {
            bounds: {
                sw: {
                    lat: function() {return -90;},
                    lng: function() {return -180;}
                },
                ne: {
                    lat: function() {return 90;},
                    lng: function() {return 180;}
                }
            }
        }

    },

    extractNormalizeFunction: function(nemesis) {

        var funcOriginal = nemesis.dashboard.requests.normalizeParamCount.toString();
        funcOriginal = funcOriginal.replace(/goog\.now/g, 'Date.now');
        funcOriginal = funcOriginal.replace(/goog\.object\.getCount/g, 'Utils.getCount');
        funcOriginal = funcOriginal.replace(/nemesis\.dashboard\.BoundsParams\.getBoundsParamsForWorld/g, 'Utils.getBoundsParamsForWorld');

        return funcOriginal;

    },

    createNormalizeFunction: function(str) {

        return new Function('obj', 'return (' + str + ')(obj)');
    },

    extractMungeFromStock: function(nemesis) {

        var munges = Utils.extractMungeFromStockIITC(nemesis);
        var new_munges = {};

        for (var key in munges) {
            new_munges[key.replace(/\./g, '_')] = munges[key];
        }

        return new_munges;

    },

    extractMungeFromStockIITC: function(nemesis) {

        var foundMunges = {};

        // these are easy - directly available in variables
        // NOTE: the .toString() is there so missing variables throw an exception, rather than storing 'undefined'
        foundMunges['dashboard.getArtifactInfo'] = nemesis.dashboard.requests.MethodName.GET_ARTIFACT_INFO.toString();
        foundMunges['dashboard.getGameScore'] = nemesis.dashboard.requests.MethodName.GET_GAME_SCORE.toString();
        foundMunges['dashboard.getPaginatedPlexts'] = nemesis.dashboard.requests.MethodName.GET_PAGINATED_PLEXTS.toString();
        foundMunges['dashboard.getThinnedEntities'] = nemesis.dashboard.requests.MethodName.GET_THINNED_ENTITIES.toString();
        foundMunges['dashboard.getPortalDetails'] = nemesis.dashboard.requests.MethodName.GET_PORTAL_DETAILS.toString();
        foundMunges['dashboard.redeemReward'] = nemesis.dashboard.requests.MethodName.REDEEM_REWARD.toString();
        foundMunges['dashboard.sendInviteEmail'] = nemesis.dashboard.requests.MethodName.SEND_INVITE_EMAIL.toString();
        foundMunges['dashboard.sendPlext'] = nemesis.dashboard.requests.MethodName.SEND_PLEXT.toString();

        // the rest are trickier - we need to parse the functions of the stock site. these break very often
        // on site updates

        // regular expression - to match either x.abcdef123456wxyz or x["123456abcdefwxyz"] format for property access
        var mungeRegExpProp = '(?:\\.([a-z][a-z0-9]{15})|\\["([0-9][a-z0-9]{15})"\\])';
        // and one to match members of object literal initialisation - {abcdef123456wxyz: or {"123456abcdefwxyz":
        var mungeRegExpLit = '(?:([a-z][a-z0-9]{15})|"([0-9][a-z0-9]{15})"):';

        // common parameters - method, version, version_parameter - currently found in the 
        // nemesis.dashboard.network.XhrController.prototype.doSendRequest_ function
        // look for something like
        //  var e = a.getData();
        //  e["3sld77nsm0tjmkvi"] = c;
        //  e.xz7q6r3aja5ttvoo = "b121024077de2a0dc6b34119e4440785c9ea5e64";
        var reg = new RegExp('getData\\(\\);.*\\n.*'+mungeRegExpProp+' =.*\n.*'+mungeRegExpProp+' *= *"([a-z0-9]{40})','m');
        var result = reg.exec(nemesis.dashboard.network.XhrController.prototype.doSendRequest_.toString());
        // there's two ways of matching the munge expression, so try both
        foundMunges.method = result[1] || result[2];
        foundMunges.version = result[3] || result[4];
        foundMunges.version_parameter = result[5];

        // GET_THINNED_ENTITIES parameters
        var reg = new RegExp('GET_THINNED_ENTITIES, nemesis.dashboard.network.XhrController.Priority.[A-Z]+, {'+mungeRegExpLit+'[a-z]');
        var result = reg.exec(nemesis.dashboard.network.DataFetcher.prototype.getGameEntities.toString());
        foundMunges.quadKeys = result[1] || result[2];

        // GET_PAGINATED_PLEXTS
        var reg = new RegExp('[a-z] = {'+mungeRegExpLit+'[a-z], '
                            +mungeRegExpLit+'Math.round\\(1E6 \\* [a-z].bounds.sw.lat\\(\\)\\), '
                            +mungeRegExpLit+'Math.round\\(1E6 \\* [a-z].bounds.sw.lng\\(\\)\\), '
                            +mungeRegExpLit+'Math.round\\(1E6 \\* [a-z].bounds.ne.lat\\(\\)\\), '
                            +mungeRegExpLit+'Math.round\\(1E6 \\* [a-z].bounds.ne.lng\\(\\)\\), '
                            +mungeRegExpLit+'[a-z], '+mungeRegExpLit+'[a-z]};\n'
                            +' *[a-z]'+mungeRegExpProp+' = [a-z];\n'
                            +' *-1 < [a-z] && \\([a-z]'+mungeRegExpProp+' = !0\\);', 'm');

        var result = reg.exec(nemesis.dashboard.network.PlextStore.prototype.getPlexts.toString());

        foundMunges.desiredNumItems = result[1] || result[2];

        foundMunges.minLatE6 = result[3] || result[4];
        foundMunges.minLngE6 = result[5] || result[6];
        foundMunges.maxLatE6 = result[7] || result[8];
        foundMunges.maxLngE6 = result[9] || result[10];
        foundMunges.minTimestampMs = result[11] || result[12];
        foundMunges.maxTimestampMs = result[13] || result[14];
        foundMunges.chatTab = result[15] || result[16];  //guessed parameter name - only seen munged
        foundMunges.ascendingTimestampOrder = result[17] || result[18];

{'fraction_non_alphanumeric': 0.1572010751755831, 'fraction_numerical': 0.02488511228648227, 'mean_word_length': 2.65, 'pattern_counts': {'":': 2, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2708663', 'n_tokens_mistral': 3980, 'n_tokens_neox': 3790, 'n_words': 600}