<commit_before>var async = require('async');
var utility = require('./utility');
var convert64to32 = utility.convert64to32;
var queueReq = utility.queueReq;
var computePlayerMatchData = require('./compute').computePlayerMatchData;
var zlib = require('zlib');
var aggregator = require('./aggregator');
var reduceMatch = utility.reduceMatch;
var config = require('./config');
var constants = require('./constants');
var columnInfo = null;

function getSets(redis, cb) {
    async.parallel({
        /*
        "bots": function(cb) {
            redis.get("bots", function(err, bots) {
                bots = JSON.parse(bots || "[]");
                //sort list of bots descending, but full bots go to end (concentrates load)

                // bots.sort(function(a, b) {
                //     var threshold = 50;
                //     if (a.friends > threshold) {
                //         return 1;
                //     }
                //     if (b.friends > threshold) {
                //         return -1;
                //     }
                //     return (b.friends - a.friends);
                // });

                //sort ascending (distributes load)
                bots.sort(function(a, b) {
                    return a.friends - b.friends;
                });
                cb(err, bots);
            });
        },
        "ratingPlayers": function(cb) {
            redis.get("ratingPlayers", function(err, rps) {
                cb(err, JSON.parse(rps || "{}"));
            });
        },
        */
        "trackedPlayers": function(cb) {
            redis.get("trackedPlayers", function(err, tps) {
                cb(err, JSON.parse(tps || "{}"));
            });
        },
        "userPlayers": function(cb) {
            redis.get("userPlayers", function(err, ups) {
                cb(err, JSON.parse(ups || "{}"));
            });
        },
        "donators": function(cb) {
            redis.get("donators", function(err, ds) {
                cb(err, JSON.parse(ds || "{}"));
            });
        }
    }, function(err, results) {
        cb(err, results);
    });
}

function getColumnInfo(db, cb) {
    if (columnInfo) {
        return cb();
    }
    else {
        async.parallel({
            "matches": function(cb) {
                db('matches').columnInfo().asCallback(cb);
            },
            "player_matches": function(cb) {
                db('player_matches').columnInfo().asCallback(cb);
            },
            "players": function(cb) {
                db('players').columnInfo().asCallback(cb);
            },
            "player_ratings": function(cb) {
                db('player_ratings').columnInfo().asCallback(cb);
            }
        }, function(err, results) {
            columnInfo = results;
            cb(err);
        });
    }
}

function insertMatch(db, redis, queue, match, options, cb) {
    var players = match.players ? JSON.parse(JSON.stringify(match.players)) : undefined;
    delete match.players;
    //options specify api, parse, or skill
    //we want to insert into matches, then insert into player_matches for each entry in players
    async.series([
    function(cb) {
            getColumnInfo(db, cb);
    },
    insertMatchTable,
    insertPlayerMatchesTable,
    //ensurePlayers,
    updatePlayerCaches,
    clearMatchCache
    ], decideParse);

    function insertMatchTable(cb) {
        var row = match;
        for (var key in row) {
            if (!(key in columnInfo.matches)) {
                delete row[key];
                //console.error(key);
            }
        }
        //TODO use psql upsert when available
        //upsert on api, update only otherwise
        if (options.type === "api") {
            db('matches').insert(row).where({<commit_msg>disable cache updating for now<commit_after>var async = require('async');
var utility = require('./utility');
var convert64to32 = utility.convert64to32;
var queueReq = utility.queueReq;
var computePlayerMatchData = require('./compute').computePlayerMatchData;
var zlib = require('zlib');
var aggregator = require('./aggregator');
var reduceMatch = utility.reduceMatch;
var config = require('./config');
var constants = require('./constants');
var columnInfo = null;

function getSets(redis, cb) {
    async.parallel({
        /*
        "bots": function(cb) {
            redis.get("bots", function(err, bots) {
                bots = JSON.parse(bots || "[]");
                //sort list of bots descending, but full bots go to end (concentrates load)

                // bots.sort(function(a, b) {
                //     var threshold = 50;
                //     if (a.friends > threshold) {
                //         return 1;
                //     }
                //     if (b.friends > threshold) {
                //         return -1;
                //     }
                //     return (b.friends - a.friends);
                // });

                //sort ascending (distributes load)
                bots.sort(function(a, b) {
                    return a.friends - b.friends;
                });
                cb(err, bots);
            });
        },
        "ratingPlayers": function(cb) {
            redis.get("ratingPlayers", function(err, rps) {
                cb(err, JSON.parse(rps || "{}"));
            });
        },
        */
        "trackedPlayers": function(cb) {
            redis.get("trackedPlayers", function(err, tps) {
                cb(err, JSON.parse(tps || "{}"));
            });
        },
        "userPlayers": function(cb) {
            redis.get("userPlayers", function(err, ups) {
                cb(err, JSON.parse(ups || "{}"));
            });
        },
        "donators": function(cb) {
            redis.get("donators", function(err, ds) {
                cb(err, JSON.parse(ds || "{}"));
            });
        }
    }, function(err, results) {
        cb(err, results);
    });
}

function getColumnInfo(db, cb) {
    if (columnInfo) {
        return cb();
    }
    else {
        async.parallel({
            "matches": function(cb) {
                db('matches').columnInfo().asCallback(cb);
            },
            "player_matches": function(cb) {
                db('player_matches').columnInfo().asCallback(cb);
            },
            "players": function(cb) {
                db('players').columnInfo().asCallback(cb);
            },
            "player_ratings": function(cb) {
                db('player_ratings').columnInfo().asCallback(cb);
            }
        }, function(err, results) {
            columnInfo = results;
            cb(err);
        });
    }
}

function insertMatch(db, redis, queue, match, options, cb) {
    var players = match.players ? JSON.parse(JSON.stringify(match.players)) : undefined;
    delete match.players;
    //options specify api, parse, or skill
    //we want to insert into matches, then insert into player_matches for each entry in players
    async.series([
    function(cb) {
            getColumnInfo(db, cb);
    },
    insertMatchTable,
    insertPlayerMatchesTable,
    //ensurePlayers,
    //updatePlayerCaches,
    //clearMatchCache
    ], decideParse);

    function insertMatchTable(cb) {
        var row = match;
        for (var key in row) {
            if (!(key in columnInfo.matches)) {
                delete row[key];
                //console.error(key);
            }
        }
        //TODO use psql upsert when available
        //upsert on api, update only otherwise
        if (options.type === "api") {
            db('matches').insert(row).where({
{'fraction_non_alphanumeric': 0.14272523413797653, 'fraction_numerical': 0.0031658092599920855, 'mean_word_length': 1.5921367521367522, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4953991', 'n_tokens_mistral': 2169, 'n_tokens_neox': 2060, 'n_words': 447}