<commit_before>/**
 * Provides utility functions.
 * All functions should have external dependencies (DB, etc.) self-contained.
 * A bare Node installation should be able to require() this file without errors.
 **/
var config = require('../config');
var constants = require('dotaconstants');
var request = require('request');
var BigNumber = require('big-number');
var urllib = require('url');
/**
 * Tokenizes an input string.
 *
 * @param {String} Input
 *
 * @return {Array}
 */
function tokenize(input)
{
    return input.replace(/[^a-zA-Z- ]+/g, '').replace('/ {2,}/', ' ').toLowerCase().split(' ');
}
/**
 * Creates a job object for enqueueing that contains details such as the Steam API endpoint to hit
 **/
function generateJob(type, payload)
{
    var api_url = "http://api.steampowered.com";
    var api_key;
    var opts = {
        "api_details": function ()
        {
            return {
                url: api_url + "/IDOTA2Match_570/GetMatchDetails/V001/?key=" + api_key + "&match_id=" + payload.match_id,
                title: [type, payload.match_id].join(),
                type: "api",
                payload: payload
            };
        },
        "api_history": function ()
        {
            return {
                url: api_url + "/IDOTA2Match_570/GetMatchHistory/V001/?key=" + api_key + (payload.account_id ? "&account_id=" + payload.account_id : "") + (payload.matches_requested ? "&matches_requested=" + payload.matches_requested : "") + (payload.hero_id ? "&hero_id=" + payload.hero_id : "") + (payload.leagueid ? "&league_id=" + payload.leagueid : "") + (payload.start_at_match_id ? "&start_at_match_id=" + payload.start_at_match_id : ""),
                title: [type, payload.account_id].join(),
                type: "api",
                payload: payload
            };
        },
        "api_summaries": function ()
        {
            return {
                url: api_url + "/ISteamUser/GetPlayerSummaries/v0002/?key=" + api_key + "&steamids=" + payload.players.map(function (p)
                {
                    return convert32to64(p.account_id).toString();
                }).join(),
                title: [type, payload.summaries_id].join(),
                type: "api",
                payload: payload
            };
        },
        "api_sequence": function ()
        {
            return {
                url: api_url + "/IDOTA2Match_570/GetMatchHistoryBySequenceNum/V001/?key=" + api_key + "&start_at_match_seq_num=" + payload.start_at_match_seq_num,
                title: [type, payload.seq_num].join(),
                type: "api"
            };
        },
        "api_heroes": function ()
        {
            return {
                url: api_url + "/IEconDOTA2_570/GetHeroes/v0001/?key=" + api_key + "&language=" + payload.language,
                title: [type, payload.language].join(),
                type: "api",
                payload: payload
            };
        },
        "api_leagues": function ()
        {
            return {
                url: api_url + "/IDOTA2Match_570/GetLeagueListing/v0001/?key=" + api_key,
                title: [type].join(),
                type: "api",
                payload: payload
            };
        },
        "api_skill": function ()
        {
            return {
                url: api_url + "/IDOTA2Match_570/GetMatchHistory/v0001/?key=" + api_key + "&start_at_match_id=" + payload.start_at_match_id + "&skill=" + payload.skill + "&hero_id=" + payload.hero_id + "&min_players=10",
                title: [type, payload.skill].join(),
                type: "api",
                payload: payload
            };
        },
        "api_live": function ()
        {
            return {
                url: api_url + "/IDOTA2Match_570/GetLiveLeagueGames/v0001/?key=" + api_key,
                title: [type].join(),
                type: "api",
                payload: payload
            };
        },
        "api_notable": function ()
        {
            return {
                url: api_url + "/IDOTA2Fantasy_570/GetProPlayerList/v1/?key=" + api_key,
                title: [type].join(),
                type: "api",
                payload: payload
            };
        },
        "api_teams": function ()
        {
            return {
                url: api_url + "/IDOTA2Teams_570/GetTeamInfo/v1/?key=" + api_key + "&team_id=" + payload.team_id,
                title: [type].join(),
                type: "api",
                payload: payload,
            };
        },
        "parse": function ()
        {
            return {
                title: [type, payload.match_id].join(),
                type: type,
                url: payload.url,
                payload: payload
            };
        },
        "request": function ()
        {
            return {
                url: api_url + "/IDOTA2Match_570/GetMatchDetails/V001/?key=" + api_key + "&match_id=" + payload.match_id,
                title: [type, payload.match_id].join(),
                type: type,
                request: true,
                payload: payload
            };
        },
        "fullhistory": function ()
        {
            return {
                title: [type, payload.account_id].join(),
                type: type,
                payload: payload
            };
        },
        "mmr": function ()
        {
            return {
                title: [type, payload.match_id, payload.account_id].join(),
                type: type,
                payload: payload
            };
        },
        "cache": function ()
        {
            return {
                title: [type, payload.match_id, payload.account_id].join(),
                type: type,
                payload: payload
            };
        },
        "rank": function ()
        {
            return {
                title: [type, payload.account_id, payload.hero_id].join(),
                type: type,
                payload: payload
            };
        }
    };
    return opts[type]();
}
/**
 * A wrapper around HTTP requests that handles:
 * proxying
 * retries/retry delay
 * Injecting API key for Steam API
 * Errors from Steam API
 **/
function getData(url, cb)
{
    var u;
    var delay = Number(config.DEFAULT_DELAY);
    var proxyAffinityRange;
    if (url.constructor === Array)
    {
        //select a random element if array
        u = url[Math.floor(Math.random() * url.length)];
    }
    else if (typeof url === "object")
    {
        //options object
        u = url.url;
        delay = url.delay || delay;
        proxyAffinityRange = url.proxyAffinityRange || proxyAffinityRange;
    }
    else
    {
        u = url;
    }
    var parse = urllib.parse(u, true);
    var proxy;
    var steam_api = false;
    if (parse.host === "api.steampowered.com")
    {
        steam_api = true;
        //choose an api key to use
        var api_keys = config.STEAM_API_KEY.split(",");
        parse.query.key = api_keys[Math.floor(Math.random() * api_keys.length)];
        parse.search = null;
        /*
        //choose a proxy to request through
        var proxies = config.PROXY_URLS.split(",");
        //add no proxy option
        proxies.push(null);
        proxy = proxies[Math.floor(Math.random() * proxies.length)];
        console.error(proxies, proxy);
        */
        //choose a steam api host
        var api_hosts = config.STEAM_API_HOST.split(",");
        api_hosts = proxyAffinityRange ? api_hosts.slice(0, proxyAffinityRange) : api_hosts;
        parse.host = api_hosts[Math.floor(Math.random() * api_hosts.length)];
    }
    var target = urllib.format(parse);
    console.error("%s - getData: %s", new Date(), target);
    return setTimeout(function ()
    {
        request(
        {
            proxy: proxy,
            url: target,
            json: true,
            timeout: 30000
        }, function (err, res, body)
        {
            if (body && body.error)
            {
                //body contained specific error (probably from retriever)
                //non-retryable
                return cb(body);
            }
            if (err || res.statusCode !== 200 || !body || (steam_api && !body.result && !body.response && !body.player_infos && !body.teams))
            {
                //invalid response
                if (url.noRetry)
                {
                    return cb(err || "invalid response");
                }
                else
                {
                    console.error("[INVALID] status: %s, retrying: %s", res.statusCode, target);
                    return getData(url, cb);
                }
            }
            else if (body.result)
            {
                //steam api usually returns data with body.result, getplayersummaries has body.response
                if (body.result.status === 15 || body.result.error === "Practice matches are not available via GetMatchDetails" || body.result.error === "No Match ID specified" || body.result.error === "Match ID not found")
                {
                    //user does not have stats enabled or attempting to get private match/invalid id, don't retry
                    //non-retryable
                    return cb(body);
                }
                else if (body.result.error || body.result.status === 2)
                {
                    //valid response, but invalid data, retry
                    if (url.noRetry)
                    {
                        return cb(err || "invalid data");
                    }
                    else
                    {
                        console.error("invalid data, retrying: %s, %s", target, JSON.stringify(body));
                        return getData(url, cb);
                    }
                }
            }<commit_msg>Update utility.js<commit_after>/**
 * Provides utility functions.
 * All functions should have external dependencies (DB, etc.) self-contained.
 * A bare Node installation should be able to require() this file without errors.
 **/
var config = require('../config');
var constants = require('dotaconstants');
var request = require('request');
var BigNumber = require('big-number');
var urllib = require('url');
/**
 * Tokenizes an input string.
 *
 * @param {String} Input
 *
 * @return {Array}
 */
function tokenize(input)
{
    return input.replace(/[^a-zA-Z- ]+/g, '').replace('/ {2,}/', ' ').toLowerCase().split(' ');
}
/**
 * Creates a job object for enqueueing that contains details such as the Steam API endpoint to hit
 **/
function generateJob(type, payload)
{
    var api_url = "http://api.steampowered.com";
    var api_key;
    var opts = {
        "api_details": function ()
        {
            return {
                url: api_url + "/IDOTA2Match_570/GetMatchDetails/V001/?key=" + api_key + "&match_id=" + payload.match_id,
                title: [type, payload.match_id].join(),
                type: "api",
                payload: payload
            };
        },
        "api_history": function ()
        {
            return {
                url: api_url + "/IDOTA2Match_570/GetMatchHistory/V001/?key=" + api_key + (payload.account_id ? "&account_id=" + payload.account_id : "") + (payload.matches_requested ? "&matches_requested=" + payload.matches_requested : "") + (payload.hero_id ? "&hero_id=" + payload.hero_id : "") + (payload.leagueid ? "&league_id=" + payload.leagueid : "") + (payload.start_at_match_id ? "&start_at_match_id=" + payload.start_at_match_id : ""),
                title: [type, payload.account_id].join(),
                type: "api",
                payload: payload
            };
        },
        "api_summaries": function ()
        {
            return {
                url: api_url + "/ISteamUser/GetPlayerSummaries/v0002/?key=" + api_key + "&steamids=" + payload.players.map(function (p)
                {
                    return convert32to64(p.account_id).toString();
                }).join(),
                title: [type, payload.summaries_id].join(),
                type: "api",
                payload: payload
            };
        },
        "api_sequence": function ()
        {
            return {
                url: api_url + "/IDOTA2Match_570/GetMatchHistoryBySequenceNum/V001/?key=" + api_key + "&start_at_match_seq_num=" + payload.start_at_match_seq_num,
                title: [type, payload.seq_num].join(),
                type: "api"
            };
        },
        "api_heroes": function ()
        {
            return {
                url: api_url + "/IEconDOTA2_570/GetHeroes/v0001/?key=" + api_key + "&language=" + payload.language,
                title: [type, payload.language].join(),
                type: "api",
                payload: payload
            };
        },
        "api_leagues": function ()
        {
            return {
                url: api_url + "/IDOTA2Match_570/GetLeagueListing/v0001/?key=" + api_key,
                title: [type].join(),
                type: "api",
                payload: payload
            };
        },
        "api_skill": function ()
        {
            return {
                url: api_url + "/IDOTA2Match_570/GetMatchHistory/v0001/?key=" + api_key + "&start_at_match_id=" + payload.start_at_match_id + "&skill=" + payload.skill + "&hero_id=" + payload.hero_id + "&min_players=10",
                title: [type, payload.skill].join(),
                type: "api",
                payload: payload
            };
        },
        "api_live": function ()
        {
            return {
                url: api_url + "/IDOTA2Match_570/GetLiveLeagueGames/v0001/?key=" + api_key,
                title: [type].join(),
                type: "api",
                payload: payload
            };
        },
        "api_notable": function ()
        {
            return {
                url: api_url + "/IDOTA2Fantasy_570/GetProPlayerList/v1/?key=" + api_key,
                title: [type].join(),
                type: "api",
                payload: payload
            };
        },
        "api_teams": function ()
        {
            return {
                url: api_url + "/IDOTA2Teams_570/GetTeamInfo/v1/?key=" + api_key + "&team_id=" + payload.team_id,
                title: [type].join(),
                type: "api",
                payload: payload,
            };
        },
        "parse": function ()
        {
            return {
                title: [type, payload.match_id].join(),
                type: type,
                url: payload.url,
                payload: payload
            };
        },
        "request": function ()
        {
            return {
                url: api_url + "/IDOTA2Match_570/GetMatchDetails/V001/?key=" + api_key + "&match_id=" + payload.match_id,
                title: [type, payload.match_id].join(),
                type: type,
                request: true,
                payload: payload
            };
        },
        "fullhistory": function ()
        {
            return {
                title: [type, payload.account_id].join(),
                type: type,
                payload: payload
            };
        },
        "mmr": function ()
        {
            return {
                title: [type, payload.match_id, payload.account_id].join(),
                type: type,
                payload: payload
            };
        },
        "cache": function ()
        {
            return {
                title: [type, payload.match_id, payload.account_id].join(),
                type: type,
                payload: payload
            };
        },
        "rank": function ()
        {
            return {
                title: [type, payload.account_id, payload.hero_id].join(),
                type: type,
                payload: payload
            };
        }
    };
    return opts[type]();
}
/**
 * A wrapper around HTTP requests that handles:
 * proxying
 * retries/retry delay
 * Injecting API key for Steam API
 * Errors from Steam API
 **/
function getData(url, cb)
{
    var u;
    var delay = Number(config.DEFAULT_DELAY);
    var proxyAffinityRange;
    if (url.constructor === Array)
    {
        //select a random element if array
        u = url[Math.floor(Math.random() * url.length)];
    }
    else if (typeof url === "object")
    {
        //options object
        u = url.url;
        delay = url.delay || delay;
        proxyAffinityRange = url.proxyAffinityRange || proxyAffinityRange;
    }
    else
    {
        u = url;
    }
    var parse = urllib.parse(u, true);
    var proxy;
    var steam_api = false;
    if (parse.host === "api.steampowered.com")
    {
        steam_api = true;
        //choose an api key to use
        var api_keys = config.STEAM_API_KEY.split(",");
        parse.query.key = api_keys[Math.floor(Math.random() * api_keys.length)];
        parse.search = null;
        /*
        //choose a proxy to request through
        var proxies = config.PROXY_URLS.split(",");
        //add no proxy option
        proxies.push(null);
        proxy = proxies[Math.floor(Math.random() * proxies.length)];
        console.error(proxies, proxy);
        */
        //choose a steam api host
        var api_hosts = config.STEAM_API_HOST.split(",");
        api_hosts = proxyAffinityRange ? api_hosts.slice(0, proxyAffinityRange) : api_hosts;
        parse.host = api_hosts[Math.floor(Math.random() * api_hosts.length)];
    }
    var target = urllib.format(parse);
    console.error("%s - getData: %s", new Date(), target);
    return setTimeout(function ()
    {
        request(
        {
            proxy: proxy,
            url: target,
            json: true,
            timeout: 30000
        }, function (err, res, body)
        {
            if (body && body.error)
            {
                //body contained specific error (probably from retriever)
                //non-retryable
                return cb(body);
            }
            if (err || res.statusCode !== 200 || !body || (steam_api && !body.result && !body.response && !body.player_infos && !body.teams))
            {
                //invalid response
                if (url.noRetry)
                {
                    return cb(err || "invalid response");
                }
                else
                {
                    console.error("[INVALID] status: %s, retrying: %s", res.statusCode, target);
                    var backoff = res.statusCode === 429 ? 1000 : 0;
                    setTimeout(function()
                    {
                        getData(url, cb);
                    }, backoff);
                }
            }
            else if (body.result)
            {
                //steam api usually returns data with body.result, getplayersummaries has body.response
                if (body.result.status === 15 || body.result.error === "Practice matches are not available via GetMatchDetails" || body.result.error === "No Match ID specified" || body.result.error === "Match ID not found")
                {
                    //user does not have stats enabled or attempting to get private match/invalid id, don't retry
                    //non-retryable
                    return cb(body);
                }
                else if (body.result.error || body.result.status === 2)
                {
                    //valid response, but invalid data, retry
                    if (url.noRetry)
                    {
                        return cb(err || "invalid data");
                    }
                    else
                    {
                        console.error("invalid data, retrying: %s, %s", target, JSON.stringify(body));
                        return getData(url, cb);
                    }
                }
            }
{'fraction_non_alphanumeric': 0.12715506345113503, 'fraction_numerical': 0.00980838262803984, 'mean_word_length': 1.4495356037151703, 'pattern_counts': {'":': 32, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6765768', 'n_tokens_mistral': 5769, 'n_tokens_neox': 5382, 'n_words': 1206}