<commit_before>var async = require("async"),
    request = require('request'),
    utility = require('./utility'),
    matches = utility.matches,
    players = utility.players,
    cheerio = require('cheerio'),
    winston = require('winston'),
    reds = require('reds');
var jobs = utility.jobs;
var api_url = "https://api.steampowered.com/IDOTA2Match_570"
var summaries_url = "http://api.steampowered.com/ISteamUser"
var remote = "http://dotabuff.com"
var trackedPlayers = {}
var transports = []
if (process.env.NODE_ENV === "production") {
    transports.push(new(winston.transports.File)({
        filename: 'backend.log',
        level: 'info'
    }))
}
else {
    transports.push(new(winston.transports.Console)());
}
var logger = new(winston.Logger)({
    transports: transports
});

updateConstants(function(err) {});
async.series([
    function(cb) {
        utility.clearActiveJobs('api', function(err) {
            cb(err)
        })
    },
    function(cb) {
        //scrape full match history for requested players
        players.find({
                full_history: 1
            }, function(err, docs) {
                async.mapSeries(docs, function(player, cb2) {
                    var account_id = player.account_id
                    var player_url = remote + "/players/" + account_id + "/matches"
                    getMatchPage(player_url, function(err) {
                        //done scraping player
                        players.update({
                            account_id: account_id
                        }, {
                            $set: {
                                full_history: 0
                            }
                        })
                        cb2(null)
                    })
                }, function(err) {
                    //done scraping all players
                    cb(null)
                })
            })
            //check most recent 100 matches for tracked players
        players.find({
                track: 1
            }, function(err, docs) {
                docs.forEach(function(player) {
                    queueReq("api", player)
                })
            })
            //parse unparsed matches
        matches.find({
            parse_status: 0
        }, function(err, docs) {
            docs.forEach(function(match) {
                queueReq("parse", match)
            })
        })
        cb(null)
    },
    function(cb) {
        //determine sequence number to begin scan at
        if (process.env.SAVE_ALL_MATCHES) {
            matches.findOne({}, {
                sort: {
                    match_seq_num: -1
                }
            }, function(err, doc) {
                getMatches(doc ? doc.match_seq_num + 1 : 0)<commit_msg>remove adding last 100 matches<commit_after>var async = require("async"),
    request = require('request'),
    utility = require('./utility'),
    matches = utility.matches,
    players = utility.players,
    cheerio = require('cheerio'),
    winston = require('winston'),
    reds = require('reds');
var jobs = utility.jobs;
var api_url = "https://api.steampowered.com/IDOTA2Match_570"
var summaries_url = "http://api.steampowered.com/ISteamUser"
var remote = "http://dotabuff.com"
var trackedPlayers = {}
var transports = []
if (process.env.NODE_ENV === "production") {
    transports.push(new(winston.transports.File)({
        filename: 'backend.log',
        level: 'info'
    }))
}
else {
    transports.push(new(winston.transports.Console)());
}
var logger = new(winston.Logger)({
    transports: transports
});

updateConstants(function(err) {});
async.series([
    function(cb) {
        utility.clearActiveJobs('api', function(err) {
            cb(err)
        })
    },
    function(cb) {
        //scrape full match history for requested players
        players.find({
                full_history: 1
            }, function(err, docs) {
                async.mapSeries(docs, function(player, cb2) {
                    var account_id = player.account_id
                    var player_url = remote + "/players/" + account_id + "/matches"
                    getMatchPage(player_url, function(err) {
                        //done scraping player
                        players.update({
                            account_id: account_id
                        }, {
                            $set: {
                                full_history: 0
                            }
                        })
                        cb2(null)
                    })
                }, function(err) {
                    //done scraping all players
                    cb(null)
                })
            })
        //parse unparsed matches
        matches.find({
            parse_status: 0
        }, function(err, docs) {
            docs.forEach(function(match) {
                queueReq("parse", match)
            })
        })
        cb(null)
    },
    function(cb) {
        //determine sequence number to begin scan at
        if (process.env.SAVE_ALL_MATCHES) {
            matches.findOne({}, {
                sort: {
                    match_seq_num: -1
                }
            }, function(err, doc) {
                getMatches(doc ? doc.match_seq_num + 1 : 0)
{'fraction_non_alphanumeric': 0.11735662742119256, 'fraction_numerical': 0.005886821116597038, 'mean_word_length': 1.418273645546373, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3563179', 'n_tokens_mistral': 1483, 'n_tokens_neox': 1392, 'n_words': 277}