<commit_before>var express = require('express');
var players = express.Router();
var async = require('async');
var constants = require('../constants.js');
var queries = require("../queries");
var getPlayerMatches = queries.getPlayerMatches;
var utility = require('../utility');
var generatePositionData = utility.generatePositionData;
var reduceMatch = utility.reduceMatch;
var config = require('../config');
var preprocessQuery = utility.preprocessQuery;
var filter = require('../filter');
var querystring = require('querystring');
var moment = require('moment');
var cacher = require('../cacher');
var readCache = cacher.readCache;
var writeCache = cacher.writeCache;
var subkeys = {
    "kills": 1,
    "deaths": 1,
    "assists": 1,
    "kda": 1,
    "gold_per_min": 1,
    "xp_per_min": 1,
    "last_hits": 1,
    "denies": 1,
    "lane_efficiency": 1,
    "duration": 1,
    "first_blood_time": 1,
    "level": 1,
    "hero_damage": 1,
    "tower_damage": 1,
    "hero_healing": 1,
    "stuns": 1,
    "tower_kills": 1,
    "neutral_kills": 1,
    "courier_kills": 1,
    "purchase_tpscroll": 1,
    "purchase_ward_observer": 1,
    "purchase_ward_sentry": 1,
    "purchase_gem": 1,
    "purchase_rapier": 1,
    "pings": 1,
    "throw": 1,
    "comeback": 1,
    "stomp": 1,
    "loss": 1,
    "actions_per_min": 1
};
//optimize by only projecting certain columns based on tab  set query.project based on info
var basic = ['player_matches.match_id', 'hero_id', 'start_time', 'duration', 'kills', 'deaths', 'assists', 'player_slot', 'account_id', 'game_mode', 'lobby_type', 'match_skill.skill', 'parse_status', 'radiant_win', 'leaver_status', 'version', 'cluster'];
var advanced = ['last_hits', 'denies', 'gold_per_min', 'xp_per_min', 'gold_t', 'first_blood_time', 'level', 'hero_damage', 'tower_damage', 'hero_healing', 'stuns', 'killed', 'pings', 'radiant_gold_adv', 'actions'];
var others = ['pgroup', 'kill_streaks', 'multi_kills', 'obs', 'sen', 'purchase_log', 'item_uses', 'hero_hits', 'ability_uses', 'chat'];
var filter = ['purchase', 'lane_pos'];
var everything = basic.concat(advanced).concat(others).concat(filter);
var projections = {
    index: basic,
    matches: basic,
    heroes: basic.concat('pgroup'),
    peers: basic.concat('pgroup'),
    activity: basic,
    histograms: basic.concat(advanced).concat(['purchase']),
    records: basic.concat(advanced).concat(['purchase']),
    counts: basic.concat(advanced),
    trends: basic.concat(advanced).concat(['purchase']),
    sprees: basic.concat(['kill_streaks', 'multi_kills']),
    wardmap: basic.concat(['obs', 'sen']),
    items: basic.concat(['purchase', 'purchase_log', 'item_uses']),
    skills: basic.concat(['hero_hits', 'ability_uses']),
    wordcloud: basic.concat('chat'),
    rating: basic
};
var playerPages = constants.player_pages;
module.exports = function(db, redis)
{
    players.get('/sql/:account_id/:info?/:subkey?', function(req, res, next)
    {
        var info = playerPages[req.params.info] ? req.params.info : "index";
        var account_id = req.params.account_id;
        var query = req.query;
        if (Number(account_id) === constants.anonymous_account_id)
        {
            return next("cannot generate profile for anonymous account_id");
        }
        //TODO
        //last played
        //significance check
        //result for matches table
        //make team/against filters work
        //disable filters, purchases, lane, patch, region
        //ag -i "aggData" views for more dependencies that need to be replaced
        //make histograms/records/trends work
        //get player db object
        async.parallel(
        {
            players_with: function(cb)
            {
                //db.raw("select js2.value as account_id, count(*) as games from player_matches join matches on matches.match_id = player_matches.match_id, json_each(pgroup) AS js, json_each_text(value) js2 where account_id = ? and js2.key = 'account_id' group by js2.value order by games desc", [account_id]).asCallback(cb);
                db.raw("select personaname, res.account_id, with_games, with_win from (select player_matches.account_id, count(*) as with_games, sum(case when player_win then 1 else 0 end) as with_win from player_matches join (select player_matches.match_id, (player_slot < 64 = radiant_win) as player_win, (player_slot < 64) as player_radiant from player_matches join matches on matches.match_id = player_matches.match_id where player_matches.account_id = ?) pm on pm.match_id = player_matches.match_id where (player_slot < 64) = player_radiant group by player_matches.account_id order by with_games desc limit 100) res left join players on players.account_id = res.account_id", [account_id]).asCallback(cb);
            },
            players_against: function(cb)
            {
                db.raw("select personaname, res.account_id, against_games, against_win from (select player_matches.account_id, count(*) as against_games, sum(case when player_win then 1 else 0 end) as against_win from player_matches join (select player_matches.match_id, (player_slot < 64 = radiant_win) as player_win, (player_slot < 64) as player_radiant from player_matches join matches on matches.match_id = player_matches.match_id where player_matches.account_id = ?) pm on pm.match_id = player_matches.match_id where (player_slot < 64) != player_radiant group by player_matches.account_id order by against_games desc limit 100) res left join players on players.account_id = res.account_id", [account_id]).asCallback(cb);
            },
            heroes: function(cb)
            {
                db.raw("select hero_id, count(*) as games, sum(case when radiant_win = player_slot < 64 then 1 else 0 end) as win from player_matches join matches on matches.match_id = player_matches.match_id where account_id = ? group by hero_id order by games desc", [account_id]).asCallback(cb);
            },
            heroes_with: function(cb)
            {
                db.raw("select hero_id, count(*) as with_games, sum(case when player_win then 1 else 0 end) as with_win from player_matches join (select player_matches.match_id, (player_slot < 64 = radiant_win) as player_win, (player_slot < 64) as player_radiant from player_matches join matches on matches.match_id = player_matches.match_id where account_id = ?) pm on pm.match_id = player_matches.match_id where (player_slot < 64) = player_radiant group by hero_id order by with_games desc", [account_id]).asCallback(cb);
            },
            heroes_against: function(cb)
            {
                db.raw("select hero_id, count(*) as against_games, sum(case when player_win then 1 else 0 end) as against_win from player_matches join (select player_matches.match_id, (player_slot < 64 = radiant_win) as player_win, (player_slot < 64) as player_radiant from player_matches join matches on matches.match_id = player_matches.match_id where account_id = ?) pm on pm.match_id = player_matches.match_id where (player_slot < 64) != player_radiant group by hero_id order by against_games desc", [account_id]).asCallback(cb);
            },
            matches: function(cb)
            {
                db.select(basic).from('player_matches').join('matches', 'matches.match_id', 'player_matches.match_id').leftJoin('match_skill', 'player_matches.match_id', "match_skill.match_id").where(
                {
                    account_id: account_id
                }).orderBy('match_id', 'desc').limit(20).asCallback(cb);
            },
            counts: function(cb)
            {
                db.raw("select sum(case when radiant_win = player_slot < 64 then 1 else 0 end) as win, sum(case when radiant_win = player_slot < 64 then 0 else 1 end) as lose, sum(case when leaver_status > 1 then 1 else 0 end) as abandon_count, count(*) as match_count, sum(case when version > 0 then 1 else 0 end) as parsed_match_count from player_matches join matches on player_matches.match_id = matches.match_id where account_id = ?", [account_id]).asCallback(cb);
            },
            sets: function(cb)
            {
                queries.getSets(redis, cb);
            },
            ratings: function(cb)
            {
                queries.getPlayerRatings(db, account_id, cb);
            }
        }, function(err, result)
        {
            if (err)
            {
                return next(err);
            }
            //console.log(result)
            var player = {
                account_id: account_id,
                ratings: result.ratings,
                data: result.matches,
                counts: result.counts.rows[0],
                heroes_list: result.heroes.rows,
                heroes_with: result.heroes_with.rows,
                heroes_against: result.heroes_against.rows,
                players_with: result.players_with.rows,
                players_against: result.players_against.rows
            };
            if (req.query.json)
            {
                return res.json(player);
            }
            res.render("player/player_" + info,
            {
                q: req.query,
                querystring: Object.keys(req.query).length ? "?" + querystring.stringify(req.query) : "",
                route: info,
                tabs: playerPages,
                player: player,
                trackedPlayers: result.sets.trackedPlayers,
                histograms: subkeys,
                subkey: req.params.subkey || "kills",
                times:
                {
                    "duration": 1,
                    "first_blood_time": 1
                },
                teammate_ids: player.players_with,
                title: (player.personaname || player.account_id) + " - YASP"
            });
        });
    });
    players.get('/:account_id/:info?/:subkey?', function(req, res, next)
    {
        console.time("player " + req.params.account_id);
        var info = playerPages[req.params.info] ? req.params.info : "index";
        var account_id = req.params.account_id;
        var query = req.query;
        var compare_data;
        if (Number(account_id) === constants.anonymous_account_id)
        {
            return next("cannot generate profile for anonymous account_id");
        }
        //copy the query in case we need the original for compare passing
        var qCopy = JSON.parse(JSON.stringify(query));
        async.series(
        {
            "player": function(cb)
            {
                fillPlayerData(account_id,
                {
                    info: info,
                    queryObj:
                    {
                        select: req.query
                    }
                }, cb);
            },
            "sets": function(cb)
            {
                queries.getSets(redis, cb);
            },
            "ratings": function(cb)
            {
                queries.getPlayerRatings(db, account_id, cb);
            }
        }, function(err, result)
        {
            if (err)
            {
                return next(err);
            }
            var player = result.player;
            var aggData = player.aggData;
            async.parallel(
            {
                //the array of teammates under the filter condition
                teammate_list: function(cb)
                {
                    generateTeammateArrayFromHash(aggData.teammates, player, cb);
                },
                /*
                all_teammate_list: function(cb) {
                    generateTeammateArrayFromHash(player.all_teammates, player, cb);
                }
                */
            }, function(err, lists)
            {
                if (err)
                {
                    return next(err);
                }
                player.teammate_list = lists.teammate_list;
                var teammate_ids = JSON.parse(JSON.stringify(lists.all_teammate_list || lists.teammate_list));
                //remove self account id from query
                delete req.query.account_id;
                //add custom tagged elements to teammate_ids so they show up in field, but ensure there are no duplicates.
                var ids = {};
                teammate_ids.forEach(function(t)
                {
                    ids[t.account_id] = 1;
                });
                for (var key in req.query)
                {
                    if (key.indexOf("account_id") !== -1 && req.query[key].constructor === Array)
                    {
                        req.query[key].forEach(function(id)
                        {
                            //iterate through array
                            //check for duplicates
                            //append to teammate_ids
                            if (!ids[id])
                            {
                                teammate_ids.unshift(
                                {
                                    account_id: Number(id),
                                    personaname: id
                                });
                            }
                            ids[id] = 1;
                        });
                    }
                }
                render();
                /*
                if (info === "compare") {
                    doCompare(qCopy, req.params.account_id.toString(), function(err, results) {
                        if (err) {
                            return next(err);
                        }
                        compare_data = results;
                        render();
                    });
                }
                else {
                    render();
                }
                */
                function render()
                {
                    console.timeEnd("player " + req.params.account_id);
                    if (req.query.json)
                    {
                        return res.json(result.player);
                    }
                    var ratings = result.ratings || [];
                    player.soloRating = ratings[0] ? ratings[ratings.length - 1].solo_competitive_rank : null;
                    player.partyRating = ratings[0] ? ratings[ratings.length - 1].competitive_rank : null;
                    player.ratings = ratings;
                    player.counts = {
                        match_count: Object.keys(player.aggData.match_ids).length,
                        parsed_match_count: Object.keys(player.aggData.parsed_match_ids).length,
                        abandon_count: player.abandons,
                        win: player.aggData.win,
                        lose: player.aggData.lose
                    };
                    res.render("player/player_" + info,
                    {
                        q: req.query,
                        querystring: Object.keys(req.query).length ? "?" + querystring.stringify(req.query) : "",
                        route: info,
                        tabs: playerPages,
                        player: player,
                        trackedPlayers: result.sets.trackedPlayers,
                        //bots: result.sets.bots,
                        //ratingPlayers: result.sets.ratingPlayers,
                        histograms: subkeys,
                        subkey: req.params.subkey || "kills",
                        times:
                        {
                            "duration": 1,
                            "first_blood_time": 1
                        },
                        teammate_ids: teammate_ids,
                        compare_data: compare_data,
                        compare: info === "compare",
                        title: (player.personaname || player.account_id) + " - YASP"
                    });
                }
            });
        });
    });
    //return router
    return players;
    /*
        function doCompare(query, account_id, cb) {
            var account_ids = ["all", account_id];
            //var compareIds = query.compare_account_id;
            //compareIds = compareIds ? [].concat(compareIds) : [];
            //account_ids = account_ids.concat(compareIds).slice(0, 6);
            async.map(account_ids, function(account_id, cb) {
                //pass a copy of the original query, saved from before
                fillPlayerData(account_id, {
                    query: {
                        select: JSON.parse(JSON.stringify(query))
                    }
                }, function(err, player) {
                    console.log("computing averages %s", player.account_id);
                    //create array of results.aggData for each account_id
                    for (var key in histograms) {
                        //mean
                        //if (player.aggData[key].sum && player.aggData[key].n) {
                        //    player.aggData[key].avg = player.aggData[key].sum / player.aggData[key].n;
                        //}
                        //median
                        var arr = [];
                        for (var value in player.aggData[key].counts) {
                            for (var i = 0; i < player.aggData[key].counts[value]; i++) {
                                arr.push(Number(value));
                            }
                        }
                        arr.sort(function(a, b) {
                            return a - b;
                        });
                        player.aggData[key].avg = arr[Math.floor(arr.length / 2)];
                    }
                    cb(err, player);
                });
            }, function(err, results) {
                if (err) {
                    return cb(err);
                }
                console.time("computing percentiles");
                //compute percentile for each stat
                //for each stat average in each player's aggdata, iterate through all's stat counts and determine whether this average is gt/lt key, then add count to appropriate bucket. percentile is gt/(gt+lt)
                results.forEach(function(r, i) {
                    for (var key in histograms) {
                        var avg = results[i].aggData[key].avg;
                        var allCounts = results[0].aggData[key].counts;
                        var gt = 0;
                        var total = 0;
                        for (var value in allCounts) {
                            var valueCount = allCounts[value];
                            if (avg >= Number(value)) {
                                gt += valueCount;
                            }
                            total += valueCount;
                        }
                        results[i].aggData[key].percentile = total ? (gt / total) : 0;
                    }
                });
                console.timeEnd("computing percentiles");
                return cb(err, results);
            });
        }
    */
    function generateTeammateArrayFromHash(input, player, cb)
    {
        if (!input)
        {
            return cb();
        }
        console.time('teammate list');
        var teammates_arr = [];
        var teammates = input;
        for (var id in teammates)
        {
            var tm = teammates[id];
            id = Number(id);
            //don't include if anonymous, self or if few games together
            if (id !== Number(player.account_id) && id !== constants.anonymous_account_id && (tm.games >= 5))
            {
                teammates_arr.push(tm);
            }
        }
        teammates_arr.sort(function(a, b)
        {
            return b.games - a.games;
        });
        //limit to 200 max players
        teammates_arr = teammates_arr.slice(0, 200);
        async.each(teammates_arr, function(t, cb)
        {
            db.first().from('players').where(
            {
                account_id: t.account_id
            }).asCallback(function(err, row)
            {
                if (err || !row)
                {
                    return cb(err);
                }
                t.personaname = row.personaname;
                t.last_login = row.last_login;
                t.avatar = row.avatar;
                cb(err);
            });
        }, function(err)
        {
            console.timeEnd('teammate list');
            cb(err, teammates_arr);
        });
    }

    function getPlayer(account_id, cb)
    {
        if (!isNaN(account_id))
        {
            db.first().from('players').where(
            {
                account_id: Number(account_id)
            }).asCallback(cb);
        }
        else
        {
            cb();
        }
    }

    function validateCache(account_id, cache, cb)
    {
        if (!cache)
        {
            return cb();
        }
        if (!isNaN(account_id))
        {
            console.time("validate");
            db('player_matches').count().where(
            {
                account_id: Number(account_id)
            }).asCallback(function(err, count)
            {
                if (err)
                {
                    return cb(err);
                }
                count = Number(count[0].count);
                console.timeEnd("validate");
                var cacheValid = cache && cache.data && cache.data.length && cache.data.length === count;
                return cb(err, cacheValid);
            });
        }
        else
        {
            //non-integer account_id (all/professional)
            //skip validation (always valid)
            cb(null, true);
        }
    }

    function fillPlayerData(account_id, options, cb)
    {
        //options.info, the tab the player is on
        //options.queryObj, the query object to use
        var orig_account_id = account_id;
        account_id = Number(account_id);
        //select player_matches with this account_id
        options.queryObj.select.account_id = account_id;
        options.queryObj = preprocessQuery(options.queryObj, constants);
        var filter_exists = Object.keys(options.queryObj.js_select).length;
        //try to find player in db
        getPlayer(orig_account_id, function(err, player)
        {
            if (err)
            {
                return cb(err);
            }
            player = player ||
            {
                account_id: account_id,
                personaname: account_id<commit_msg>use numberified account_id for getplayer<commit_after>var express = require('express');
var players = express.Router();
var async = require('async');
var constants = require('../constants.js');
var queries = require("../queries");
var getPlayerMatches = queries.getPlayerMatches;
var utility = require('../utility');
var generatePositionData = utility.generatePositionData;
var reduceMatch = utility.reduceMatch;
var config = require('../config');
var preprocessQuery = utility.preprocessQuery;
var filter = require('../filter');
var querystring = require('querystring');
var moment = require('moment');
var cacher = require('../cacher');
var readCache = cacher.readCache;
var writeCache = cacher.writeCache;
var subkeys = {
    "kills": 1,
    "deaths": 1,
    "assists": 1,
    "kda": 1,
    "gold_per_min": 1,
    "xp_per_min": 1,
    "last_hits": 1,
    "denies": 1,
    "lane_efficiency": 1,
    "duration": 1,
    "first_blood_time": 1,
    "level": 1,
    "hero_damage": 1,
    "tower_damage": 1,
    "hero_healing": 1,
    "stuns": 1,
    "tower_kills": 1,
    "neutral_kills": 1,
    "courier_kills": 1,
    "purchase_tpscroll": 1,
    "purchase_ward_observer": 1,
    "purchase_ward_sentry": 1,
    "purchase_gem": 1,
    "purchase_rapier": 1,
    "pings": 1,
    "throw": 1,
    "comeback": 1,
    "stomp": 1,
    "loss": 1,
    "actions_per_min": 1
};
//optimize by only projecting certain columns based on tab  set query.project based on info
var basic = ['player_matches.match_id', 'hero_id', 'start_time', 'duration', 'kills', 'deaths', 'assists', 'player_slot', 'account_id', 'game_mode', 'lobby_type', 'match_skill.skill', 'parse_status', 'radiant_win', 'leaver_status', 'version', 'cluster'];
var advanced = ['last_hits', 'denies', 'gold_per_min', 'xp_per_min', 'gold_t', 'first_blood_time', 'level', 'hero_damage', 'tower_damage', 'hero_healing', 'stuns', 'killed', 'pings', 'radiant_gold_adv', 'actions'];
var others = ['pgroup', 'kill_streaks', 'multi_kills', 'obs', 'sen', 'purchase_log', 'item_uses', 'hero_hits', 'ability_uses', 'chat'];
var filter = ['purchase', 'lane_pos'];
var everything = basic.concat(advanced).concat(others).concat(filter);
var projections = {
    index: basic,
    matches: basic,
    heroes: basic.concat('pgroup'),
    peers: basic.concat('pgroup'),
    activity: basic,
    histograms: basic.concat(advanced).concat(['purchase']),
    records: basic.concat(advanced).concat(['purchase']),
    counts: basic.concat(advanced),
    trends: basic.concat(advanced).concat(['purchase']),
    sprees: basic.concat(['kill_streaks', 'multi_kills']),
    wardmap: basic.concat(['obs', 'sen']),
    items: basic.concat(['purchase', 'purchase_log', 'item_uses']),
    skills: basic.concat(['hero_hits', 'ability_uses']),
    wordcloud: basic.concat('chat'),
    rating: basic
};
var playerPages = constants.player_pages;
module.exports = function(db, redis)
{
    players.get('/sql/:account_id/:info?/:subkey?', function(req, res, next)
    {
        var info = playerPages[req.params.info] ? req.params.info : "index";
        var account_id = req.params.account_id;
        var query = req.query;
        if (Number(account_id) === constants.anonymous_account_id)
        {
            return next("cannot generate profile for anonymous account_id");
        }
        //TODO
        //last played
        //significance check
        //result for matches table
        //make team/against filters work
        //disable filters, purchases, lane, patch, region
        //ag -i "aggData" views for more dependencies that need to be replaced
        //make histograms/records/trends work
        //get player db object
        async.parallel(
        {
            players_with: function(cb)
            {
                //db.raw("select js2.value as account_id, count(*) as games from player_matches join matches on matches.match_id = player_matches.match_id, json_each(pgroup) AS js, json_each_text(value) js2 where account_id = ? and js2.key = 'account_id' group by js2.value order by games desc", [account_id]).asCallback(cb);
                db.raw("select personaname, res.account_id, with_games, with_win from (select player_matches.account_id, count(*) as with_games, sum(case when player_win then 1 else 0 end) as with_win from player_matches join (select player_matches.match_id, (player_slot < 64 = radiant_win) as player_win, (player_slot < 64) as player_radiant from player_matches join matches on matches.match_id = player_matches.match_id where player_matches.account_id = ?) pm on pm.match_id = player_matches.match_id where (player_slot < 64) = player_radiant group by player_matches.account_id order by with_games desc limit 100) res left join players on players.account_id = res.account_id", [account_id]).asCallback(cb);
            },
            players_against: function(cb)
            {
                db.raw("select personaname, res.account_id, against_games, against_win from (select player_matches.account_id, count(*) as against_games, sum(case when player_win then 1 else 0 end) as against_win from player_matches join (select player_matches.match_id, (player_slot < 64 = radiant_win) as player_win, (player_slot < 64) as player_radiant from player_matches join matches on matches.match_id = player_matches.match_id where player_matches.account_id = ?) pm on pm.match_id = player_matches.match_id where (player_slot < 64) != player_radiant group by player_matches.account_id order by against_games desc limit 100) res left join players on players.account_id = res.account_id", [account_id]).asCallback(cb);
            },
            heroes: function(cb)
            {
                db.raw("select hero_id, count(*) as games, sum(case when radiant_win = player_slot < 64 then 1 else 0 end) as win from player_matches join matches on matches.match_id = player_matches.match_id where account_id = ? group by hero_id order by games desc", [account_id]).asCallback(cb);
            },
            heroes_with: function(cb)
            {
                db.raw("select hero_id, count(*) as with_games, sum(case when player_win then 1 else 0 end) as with_win from player_matches join (select player_matches.match_id, (player_slot < 64 = radiant_win) as player_win, (player_slot < 64) as player_radiant from player_matches join matches on matches.match_id = player_matches.match_id where account_id = ?) pm on pm.match_id = player_matches.match_id where (player_slot < 64) = player_radiant group by hero_id order by with_games desc", [account_id]).asCallback(cb);
            },
            heroes_against: function(cb)
            {
                db.raw("select hero_id, count(*) as against_games, sum(case when player_win then 1 else 0 end) as against_win from player_matches join (select player_matches.match_id, (player_slot < 64 = radiant_win) as player_win, (player_slot < 64) as player_radiant from player_matches join matches on matches.match_id = player_matches.match_id where account_id = ?) pm on pm.match_id = player_matches.match_id where (player_slot < 64) != player_radiant group by hero_id order by against_games desc", [account_id]).asCallback(cb);
            },
            matches: function(cb)
            {
                db.select(basic).from('player_matches').join('matches', 'matches.match_id', 'player_matches.match_id').leftJoin('match_skill', 'player_matches.match_id', "match_skill.match_id").where(
                {
                    account_id: account_id
                }).orderBy('match_id', 'desc').limit(20).asCallback(cb);
            },
            counts: function(cb)
            {
                db.raw("select sum(case when radiant_win = player_slot < 64 then 1 else 0 end) as win, sum(case when radiant_win = player_slot < 64 then 0 else 1 end) as lose, sum(case when leaver_status > 1 then 1 else 0 end) as abandon_count, count(*) as match_count, sum(case when version > 0 then 1 else 0 end) as parsed_match_count from player_matches join matches on player_matches.match_id = matches.match_id where account_id = ?", [account_id]).asCallback(cb);
            },
            sets: function(cb)
            {
                queries.getSets(redis, cb);
            },
            ratings: function(cb)
            {
                queries.getPlayerRatings(db, account_id, cb);
            }
        }, function(err, result)
        {
            if (err)
            {
                return next(err);
            }
            //console.log(result)
            var player = {
                account_id: account_id,
                ratings: result.ratings,
                data: result.matches,
                counts: result.counts.rows[0],
                heroes_list: result.heroes.rows,
                heroes_with: result.heroes_with.rows,
                heroes_against: result.heroes_against.rows,
                players_with: result.players_with.rows,
                players_against: result.players_against.rows
            };
            if (req.query.json)
            {
                return res.json(player);
            }
            res.render("player/player_" + info,
            {
                q: req.query,
                querystring: Object.keys(req.query).length ? "?" + querystring.stringify(req.query) : "",
                route: info,
                tabs: playerPages,
                player: player,
                trackedPlayers: result.sets.trackedPlayers,
                histograms: subkeys,
                subkey: req.params.subkey || "kills",
                times:
                {
                    "duration": 1,
                    "first_blood_time": 1
                },
                teammate_ids: player.players_with,
                title: (player.personaname || player.account_id) + " - YASP"
            });
        });
    });
    players.get('/:account_id/:info?/:subkey?', function(req, res, next)
    {
        console.time("player " + req.params.account_id);
        var info = playerPages[req.params.info] ? req.params.info : "index";
        var account_id = req.params.account_id;
        var query = req.query;
        var compare_data;
        if (Number(account_id) === constants.anonymous_account_id)
        {
            return next("cannot generate profile for anonymous account_id");
        }
        //copy the query in case we need the original for compare passing
        var qCopy = JSON.parse(JSON.stringify(query));
        async.series(
        {
            "player": function(cb)
            {
                fillPlayerData(account_id,
                {
                    info: info,
                    queryObj:
                    {
                        select: req.query
                    }
                }, cb);
            },
            "sets": function(cb)
            {
                queries.getSets(redis, cb);
            },
            "ratings": function(cb)
            {
                queries.getPlayerRatings(db, account_id, cb);
            }
        }, function(err, result)
        {
            if (err)
            {
                return next(err);
            }
            var player = result.player;
            var aggData = player.aggData;
            async.parallel(
            {
                //the array of teammates under the filter condition
                teammate_list: function(cb)
                {
                    generateTeammateArrayFromHash(aggData.teammates, player, cb);
                },
                /*
                all_teammate_list: function(cb) {
                    generateTeammateArrayFromHash(player.all_teammates, player, cb);
                }
                */
            }, function(err, lists)
            {
                if (err)
                {
                    return next(err);
                }
                player.teammate_list = lists.teammate_list;
                var teammate_ids = JSON.parse(JSON.stringify(lists.all_teammate_list || lists.teammate_list));
                //remove self account id from query
                delete req.query.account_id;
                //add custom tagged elements to teammate_ids so they show up in field, but ensure there are no duplicates.
                var ids = {};
                teammate_ids.forEach(function(t)
                {
                    ids[t.account_id] = 1;
                });
                for (var key in req.query)
                {
                    if (key.indexOf("account_id") !== -1 && req.query[key].constructor === Array)
                    {
                        req.query[key].forEach(function(id)
                        {
                            //iterate through array
                            //check for duplicates
                            //append to teammate_ids
                            if (!ids[id])
                            {
                                teammate_ids.unshift(
                                {
                                    account_id: Number(id),
                                    personaname: id
                                });
                            }
                            ids[id] = 1;
                        });
                    }
                }
                render();
                /*
                if (info === "compare") {
                    doCompare(qCopy, req.params.account_id.toString(), function(err, results) {
                        if (err) {
                            return next(err);
                        }
                        compare_data = results;
                        render();
                    });
                }
                else {
                    render();
                }
                */
                function render()
                {
                    console.timeEnd("player " + req.params.account_id);
                    if (req.query.json)
                    {
                        return res.json(result.player);
                    }
                    var ratings = result.ratings || [];
                    player.soloRating = ratings[0] ? ratings[ratings.length - 1].solo_competitive_rank : null;
                    player.partyRating = ratings[0] ? ratings[ratings.length - 1].competitive_rank : null;
                    player.ratings = ratings;
                    player.counts = {
                        match_count: Object.keys(player.aggData.match_ids).length,
                        parsed_match_count: Object.keys(player.aggData.parsed_match_ids).length,
                        abandon_count: player.abandons,
                        win: player.aggData.win,
                        lose: player.aggData.lose
                    };
                    res.render("player/player_" + info,
                    {
                        q: req.query,
                        querystring: Object.keys(req.query).length ? "?" + querystring.stringify(req.query) : "",
                        route: info,
                        tabs: playerPages,
                        player: player,
                        trackedPlayers: result.sets.trackedPlayers,
                        //bots: result.sets.bots,
                        //ratingPlayers: result.sets.ratingPlayers,
                        histograms: subkeys,
                        subkey: req.params.subkey || "kills",
                        times:
                        {
                            "duration": 1,
                            "first_blood_time": 1
                        },
                        teammate_ids: teammate_ids,
                        compare_data: compare_data,
                        compare: info === "compare",
                        title: (player.personaname || player.account_id) + " - YASP"
                    });
                }
            });
        });
    });
    //return router
    return players;
    /*
        function doCompare(query, account_id, cb) {
            var account_ids = ["all", account_id];
            //var compareIds = query.compare_account_id;
            //compareIds = compareIds ? [].concat(compareIds) : [];
            //account_ids = account_ids.concat(compareIds).slice(0, 6);
            async.map(account_ids, function(account_id, cb) {
                //pass a copy of the original query, saved from before
                fillPlayerData(account_id, {
                    query: {
                        select: JSON.parse(JSON.stringify(query))
                    }
                }, function(err, player) {
                    console.log("computing averages %s", player.account_id);
                    //create array of results.aggData for each account_id
                    for (var key in histograms) {
                        //mean
                        //if (player.aggData[key].sum && player.aggData[key].n) {
                        //    player.aggData[key].avg = player.aggData[key].sum / player.aggData[key].n;
                        //}
                        //median
                        var arr = [];
                        for (var value in player.aggData[key].counts) {
                            for (var i = 0; i < player.aggData[key].counts[value]; i++) {
                                arr.push(Number(value));
                            }
                        }
                        arr.sort(function(a, b) {
                            return a - b;
                        });
                        player.aggData[key].avg = arr[Math.floor(arr.length / 2)];
                    }
                    cb(err, player);
                });
            }, function(err, results) {
                if (err) {
                    return cb(err);
                }
                console.time("computing percentiles");
                //compute percentile for each stat
                //for each stat average in each player's aggdata, iterate through all's stat counts and determine whether this average is gt/lt key, then add count to appropriate bucket. percentile is gt/(gt+lt)
                results.forEach(function(r, i) {
                    for (var key in histograms) {
                        var avg = results[i].aggData[key].avg;
                        var allCounts = results[0].aggData[key].counts;
                        var gt = 0;
                        var total = 0;
                        for (var value in allCounts) {
                            var valueCount = allCounts[value];
                            if (avg >= Number(value)) {
                                gt += valueCount;
                            }
                            total += valueCount;
                        }
                        results[i].aggData[key].percentile = total ? (gt / total) : 0;
                    }
                });
                console.timeEnd("computing percentiles");
                return cb(err, results);
            });
        }
    */
    function generateTeammateArrayFromHash(input, player, cb)
    {
        if (!input)
        {
            return cb();
        }
        console.time('teammate list');
        var teammates_arr = [];
        var teammates = input;
        for (var id in teammates)
        {
            var tm = teammates[id];
            id = Number(id);
            //don't include if anonymous, self or if few games together
            if (id !== Number(player.account_id) && id !== constants.anonymous_account_id && (tm.games >= 5))
            {
                teammates_arr.push(tm);
            }
        }
        teammates_arr.sort(function(a, b)
        {
            return b.games - a.games;
        });
        //limit to 200 max players
        teammates_arr = teammates_arr.slice(0, 200);
        async.each(teammates_arr, function(t, cb)
        {
            db.first().from('players').where(
            {
                account_id: t.account_id
            }).asCallback(function(err, row)
            {
                if (err || !row)
                {
                    return cb(err);
                }
                t.personaname = row.personaname;
                t.last_login = row.last_login;
                t.avatar = row.avatar;
                cb(err);
            });
        }, function(err)
        {
            console.timeEnd('teammate list');
            cb(err, teammates_arr);
        });
    }

    function getPlayer(account_id, cb)
    {
        if (!isNaN(account_id))
        {
            db.first().from('players').where(
            {
                account_id: Number(account_id)
            }).asCallback(cb);
        }
        else
        {
            cb();
        }
    }

    function validateCache(account_id, cache, cb)
    {
        if (!cache)
        {
            return cb();
        }
        if (!isNaN(account_id))
        {
            console.time("validate");
            db('player_matches').count().where(
            {
                account_id: Number(account_id)
            }).asCallback(function(err, count)
            {
                if (err)
                {
                    return cb(err);
                }
                count = Number(count[0].count);
                console.timeEnd("validate");
                var cacheValid = cache && cache.data && cache.data.length && cache.data.length === count;
                return cb(err, cacheValid);
            });
        }
        else
        {
            //non-integer account_id (all/professional)
            //skip validation (always valid)
            cb(null, true);
        }
    }

    function fillPlayerData(account_id, options, cb)
    {
        //options.info, the tab the player is on
        //options.queryObj, the query object to use
        var orig_account_id = account_id;
        account_id = Number(account_id);
        //select player_matches with this account_id
        options.queryObj.select.account_id = account_id;
        options.queryObj = preprocessQuery(options.queryObj, constants);
        var filter_exists = Object.keys(options.queryObj.js_select).length;
        //try to find player in db
        getPlayer(account_id, function(err, player)
        {
            if (err)
            {
                return cb(err);
            }
            player = player ||
            {
                account_id: account_id,
                personaname: account_id
{'fraction_non_alphanumeric': 0.10697196010911855, 'fraction_numerical': 0.005411207012208756, 'mean_word_length': 1.524868740473099, 'pattern_counts': {'":': 74, '<': 35, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7012242', 'n_tokens_mistral': 12447, 'n_tokens_neox': 11680, 'n_words': 2757}