<commit_before>var utility = require('./utility');
var isSignificant = utility.isSignificant;
var reduceMatch = utility.reduceMatch;
var constants = require('./constants.json');
var aggregator = require('./aggregator');
var async = require('async');
var db = require('./db');
var r = require('./redis');
var redis = r.client;
var zlib = require('zlib');
var computeMatchData = require('./compute').computeMatchData;
module.exports = function updatePlayerCaches(match, options, cb) {
    //insert the match into db, then based on the existing document determine whether to do aggregations
    db.matches.findAndModify({
        match_id: match.match_id
    }, {
        $set: match
    }, {
        //TODO if we use this function for updating skill in player caches we don't want to upsert or overwrite the cache.data match
        upsert: true,
        //explicitly declare we want the pre-modification document
        "new": false
    }, function(err, doc) {
        if (err) {
            return cb(err);
        }
        async.each(match.players, function(p, cb) {
                redis.get("player:" + p.account_id, function(err, result) {
                    //if player cache doesn't exist, skip
                    //if insignificant, skip
                    var cache = result && !err ? JSON.parse(zlib.inflateSync(new Buffer(result, 'base64'))) : null;
                    if (cache) {
                        //m.players[0] should be this player
                        //m.all_players should be all players
                        //duplicate this data into a copy to avoid corrupting original match object
                        var match_copy = JSON.parse(JSON.stringify(match));
                        match_copy.all_players = match.players.slice(0);
                        match_copy.players = [p];
                        //some data fields require computeMatchData in order to aggregate correctly
                        computeMatchData(match_copy);
                        //check for doc.players containing this match_id (do aggregation for new inserts where player was formerly anonymous)
                        var playerInMatch = doc.players.some(function(player) {
                            player.account_id === p.account_id;
                        });
                        var reInsert = doc && options.type === "api" && playerInMatch;
                        //determine if we're reparsing this match		
                        var reParse = doc && doc.parsed_data && options.type === "parsed";
                        if (!reInsert && !reParse && cache.aggData) {
                            //do aggregations on fields based on type		
                            cache.aggData = aggregator([match_copy], options.type, cache.aggData);
                        }
                        //TODO it may be more performant to just push the match now and then deduplicate at view time (taking the last entry of each match_id to get state changes)
                        //add match to array
                        var ids = {};
                        //deduplicate matches by id
                        cache.data.forEach(function(m) {
                            ids[m.match_id] = m;
                        });
                        //update this match with latest state (parse_status/skill may have changed)
                        ids[match_copy.match_id] = reduceMatch(match_copy);
                        cache.data = [];
                        for (var key in ids) {
                            cache.data.push(ids[key]);
                        }
                        redis.setex("player:" + p.account_id, 60 * 60 * 24 * 7, zlib.deflateSync(JSON.stringify(cache)).toString('base64'));
                    }
                    /*
                    //temporarily disable inserting new players into db
                    db.players.update({
                        account_id: p.account_id
                    }, {
                        $set: {
                            account_id: p.account_id,
                        }
                    }, {
                        upsert: true
                    }, function(err) {
                        if (err) {
                            return cb(err);
                        }
                        cb(err);
                    });
                    */
                    cb(err);
                });
            },
            //done with all 10 players
            cb);
    });
};<commit_msg>added further checks to updateplayercaches<commit_after>var utility = require('./utility');
var isSignificant = utility.isSignificant;
var reduceMatch = utility.reduceMatch;
var constants = require('./constants.json');
var aggregator = require('./aggregator');
var async = require('async');
var db = require('./db');
var r = require('./redis');
var redis = r.client;
var zlib = require('zlib');
var computeMatchData = require('./compute').computeMatchData;
module.exports = function updatePlayerCaches(match, options, cb) {
    //insert the match into db, then based on the existing document determine whether to do aggregations
    db.matches.findAndModify({
        match_id: match.match_id
    }, {
        $set: match
    }, {
        //TODO if we use this function for updating skill in player caches we don't want to upsert or overwrite the cache.data match
        upsert: true,
        //explicitly declare we want the pre-modification document
        "new": false
    }, function(err, doc) {
        if (err) {
            return cb(err);
        }
        async.each(match.players, function(p, cb) {
                redis.get("player:" + p.account_id, function(err, result) {
                    //if player cache doesn't exist, skip
                    //if insignificant, skip
                    var cache = result && !err ? JSON.parse(zlib.inflateSync(new Buffer(result, 'base64'))) : null;
                    if (cache) {
                        //m.players[0] should be this player
                        //m.all_players should be all players
                        //duplicate this data into a copy to avoid corrupting original match object
                        var match_copy = JSON.parse(JSON.stringify(match));
                        match_copy.all_players = match.players.slice(0);
                        match_copy.players = [p];
                        //some data fields require computeMatchData in order to aggregate correctly
                        computeMatchData(match_copy);
                        //check for doc.players containing this match_id (do aggregation for new inserts where player was formerly anonymous)
                        var playerInMatch = doc && doc.players && doc.players.some(function(player) {
                            player.account_id === p.account_id;
                        });
                        var reInsert = doc && options.type === "api" && playerInMatch;
                        //determine if we're reparsing this match		
                        var reParse = doc && doc.parsed_data && options.type === "parsed";
                        if (!reInsert && !reParse && cache.aggData) {
                            //do aggregations on fields based on type		
                            cache.aggData = aggregator([match_copy], options.type, cache.aggData);
                        }
                        //TODO it may be more performant to just push the match now and then deduplicate at view time (taking the last entry of each match_id to get state changes)
                        //add match to array
                        var ids = {};
                        //deduplicate matches by id
                        cache.data.forEach(function(m) {
                            ids[m.match_id] = m;
                        });
                        //update this match with latest state (parse_status/skill may have changed)
                        ids[match_copy.match_id] = reduceMatch(match_copy);
                        cache.data = [];
                        for (var key in ids) {
                            cache.data.push(ids[key]);
                        }
                        redis.setex("player:" + p.account_id, 60 * 60 * 24 * 7, zlib.deflateSync(JSON.stringify(cache)).toString('base64'));
                    }
                    /*
                    //temporarily disable inserting new players into db
                    db.players.update({
                        account_id: p.account_id
                    }, {
                        $set: {
                            account_id: p.account_id,
                        }
                    }, {
                        upsert: true
                    }, function(err) {
                        if (err) {
                            return cb(err);
                        }
                        cb(err);
                    });
                    */
                    cb(err);
                });
            },
            //done with all 10 players
            cb);
    });
};
{'fraction_non_alphanumeric': 0.09321566464423607, 'fraction_numerical': 0.003309431880860452, 'mean_word_length': 1.2676338169084542, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1165814', 'n_tokens_mistral': 2209, 'n_tokens_neox': 2058, 'n_words': 634}