<commit_before>process.env.MONGO_URL = "mongodb://localhost/test";
process.env.REDIS_URL = "redis://localhost:6379/1";
process.env.SESSION_SECRET = "testsecretvalue";
process.env.KUE_USER = "user";
process.env.KUE_PASS = "<PASSWORD>";
process.env.PORT = 5000;
process.env.ROOT_URL = "http://localhost:5000";
process.env.NODE_ENV = "test";
process.env.STEAM_API_KEY = "fakekey";
var async = require('async');
var db = require('../db');
var r = require('../redis');
var redis = r.client;
var kue = r.kue;
var jobs = r.jobs;
var testdata = require('./test.json');
var nock = require('nock');
var moment = require('moment');
var assert = require('assert');
var processApi = require('../processApi');
var processFullHistory = require('../processFullHistory');
var processParse = require('../processParse');
var processMmr = require('../processMmr');
var fs = require('fs');
var request = require('request');
var unparsed = require('../tasks/unparsed');
var updateNames = require('../tasks/updateNames');
var constants = require('../tasks/constants');
var operations = require('../operations');
var queueReq = operations.queueReq;
var supertest = require('supertest');
var wait = 60000;
//set up nock
//fake retriever response
nock("http://" + process.env.RETRIEVER_HOST).filteringPath(function(path) {
    var split = path.split("?");
    split = split[1].split("&");
    console.log(split[split.length - 1])
    return split[split.length - 1];
}).get('key=shared_secret_with_retriever').reply(200, {
    "accounts": {
        "76561198186929683": {
            "steamID": "76561198186929683",
            "replays": 6,
            "profiles": 0,
            "friends": 0
        }
    },
    "accountToIdx": {
        "26949": "76561198186929683"
    }
}).get('account_id=88367253').reply(200, {
    "accountId": 88367253,
    "wins": 889,
    "xp": 52,
    "level": 153,
    "lowPriorityUntilDate": 0,
    "preventVoiceUntilDate": 0,
    "teaching": 6,
    "leadership": 4,
    "friendly": 10,
    "forgiving": 5,
    "lowPriorityGamesRemaining": 0,
    "competitiveRank": 3228,
    "calibrationGamesRemaining": 0,
    "soloCompetitiveRank": 3958,
    "soloCalibrationGamesRemaining": 0,
    "recruitmentLevel": 0
}).get('match_id=1151783218').reply(200, {
    match: {
        cluster: 1,
        replaySalt: 1
    }
}).get('match_id=2').reply(200, {
    match: {
        cluster: 1,
        replaySalt: 1
    }
});
//fake api response
nock('http://api.steampowered.com').filteringPath(function(path) {
        var split = path.split("?");
        var split2 = split[0].split(".com");
        return split2[0];
    })
    //throw some errors to test handling
    .get('/IDOTA2Match_570/GetMatchDetails/V001/').reply(500, {}).get('/IDOTA2Match_570/GetMatchDetails/V001/').times(10).reply(200, testdata.details_api).get('/ISteamUser/GetPlayerSummaries/v0002/').reply(200, testdata.summaries_api).get('/IDOTA2Match_570/GetMatchHistory/V001/').reply(200, {
        result: {
            error: "error"
        }
    }).get('/IDOTA2Match_570/GetMatchHistory/V001/').reply(200, testdata.history_api).get('/IDOTA2Match_570/GetMatchHistory/V001/').times(2).reply(200, testdata.history_api2).get('/IEconDOTA2_570/GetHeroes/v0001/').reply(200, testdata.heroes_api);
console.log('starting web');
var app = require('../web');
console.log('starting parser');
var parser = require('../parser');
console.log('starting parseManager');
var parseManager = require('../parseManager');
//stuff we don't run in test
//scanner
//worker
var replay_dir = process.env.REPLAY_DIR;
if (!fs.existsSync(replay_dir)) {
    fs.mkdir(replay_dir);
}
before(function(done) {
    this.timeout(wait);
    var DatabaseCleaner = require('database-cleaner');
    var databaseCleaner = new DatabaseCleaner('mongodb');
    var connect = require('mongodb').connect;
    nock.enableNetConnect();
    async.series([
            function(cb) {
            console.log("wiping mongodb");
            connect(process.env.MONGO_URL, function(err, db) {
                assert(!err);
                databaseCleaner.clean(db, function(err) {
                    cb(err);
                });
            });
            },
            function(cb) {
            console.log("wiping redis");
            redis.flushall(function(err) {
                cb(err);
            });
            },
            function(cb) {
            console.log("loading matches");
            async.mapSeries(testdata.matches, function(m, cb) {
                db.matches.insert(m, function(err) {
                    cb(err);
                });
            }, function(err) {
                cb(err);
            });
            },
            function(cb) {
            console.log("loading players");
            //set visited date on first player
            testdata.players[0].last_visited = new Date();
            testdata.players[0].join_date = new Date("2012-08-31T15:59:02.161+0100");
            testdata.players[1].last_visited = new Date("2012-08-31T15:59:02.161+0100");
            async.mapSeries(testdata.players, function(p, cb) {
                db.players.insert(p, function(err) {
                    cb(err);
                });
            }, function(err) {
                cb(err);
            });
            },
            function(cb) {
            console.log('downloading replays');

            function dl(filename, cb) {
                //keep only the match id segment as the filename
                var arr = filename.split(".");
                arr[0] = arr[0].split("_")[0];
                var path = replay_dir + arr.join(".");
                //currently disabled caching of replays, get a fresh copy with each test
                if (fs.existsSync(path) && false) {
                    cb();
                }
                else {
                    request('http://cdn.rawgit.com/yasp-dota/testfiles/master/' + filename).pipe(fs.createWriteStream(path)).on('error', function(err) {
                        console.log(err);
                        //detect errors and retry?
                        return dl(filename, cb);
                    }).on('finish', function(err) {
                        cb(err);
                    });
                }
            }
            var files = ['1193091757.dem', '1181392470_1v1.dem', '1189263979_ardm.dem', 'invalid.dem'];
            async.each(files, dl, function(err) {
                cb(err);
            });
    }
        ], function(err) {
        done(err);
    });
});
describe("services", function() {
    it("mongodb connected", function(done) {
        assert(db);<commit_msg>changing retry logic<commit_after>process.env.MONGO_URL = "mongodb://localhost/test";
process.env.REDIS_URL = "redis://localhost:6379/1";
process.env.SESSION_SECRET = "testsecretvalue";
process.env.KUE_USER = "user";
process.env.KUE_PASS = "<PASSWORD>";
process.env.PORT = 5000;
process.env.ROOT_URL = "http://localhost:5000";
process.env.NODE_ENV = "test";
process.env.STEAM_API_KEY = "fakekey";
var async = require('async');
var db = require('../db');
var r = require('../redis');
var redis = r.client;
var kue = r.kue;
var jobs = r.jobs;
var testdata = require('./test.json');
var nock = require('nock');
var moment = require('moment');
var assert = require('assert');
var processApi = require('../processApi');
var processFullHistory = require('../processFullHistory');
var processParse = require('../processParse');
var processMmr = require('../processMmr');
var fs = require('fs');
var request = require('request');
var unparsed = require('../tasks/unparsed');
var updateNames = require('../tasks/updateNames');
var constants = require('../tasks/constants');
var operations = require('../operations');
var queueReq = operations.queueReq;
var supertest = require('supertest');
var wait = 60000;
//set up nock
//fake retriever response
nock("http://" + process.env.RETRIEVER_HOST).filteringPath(function(path) {
    var split = path.split("?");
    split = split[1].split("&");
    console.log(split[split.length - 1])
    return split[split.length - 1];
}).get('key=shared_secret_with_retriever').reply(200, {
    "accounts": {
        "76561198186929683": {
            "steamID": "76561198186929683",
            "replays": 6,
            "profiles": 0,
            "friends": 0
        }
    },
    "accountToIdx": {
        "26949": "76561198186929683"
    }
}).get('account_id=88367253').reply(200, {
    "accountId": 88367253,
    "wins": 889,
    "xp": 52,
    "level": 153,
    "lowPriorityUntilDate": 0,
    "preventVoiceUntilDate": 0,
    "teaching": 6,
    "leadership": 4,
    "friendly": 10,
    "forgiving": 5,
    "lowPriorityGamesRemaining": 0,
    "competitiveRank": 3228,
    "calibrationGamesRemaining": 0,
    "soloCompetitiveRank": 3958,
    "soloCalibrationGamesRemaining": 0,
    "recruitmentLevel": 0
}).get('match_id=1151783218').reply(200, {
    match: {
        cluster: 1,
        replaySalt: 1
    }
}).get('match_id=2').reply(200, {
    match: {
        cluster: 1,
        replaySalt: 1
    }
});
//fake api response
nock('http://api.steampowered.com').filteringPath(function(path) {
        var split = path.split("?");
        var split2 = split[0].split(".com");
        return split2[0];
    })
    //throw some errors to test handling
    .get('/IDOTA2Match_570/GetMatchDetails/V001/').reply(500, {}).get('/IDOTA2Match_570/GetMatchDetails/V001/').times(10).reply(200, testdata.details_api).get('/ISteamUser/GetPlayerSummaries/v0002/').reply(200, testdata.summaries_api).get('/IDOTA2Match_570/GetMatchHistory/V001/').reply(200, {
        result: {
            error: "error"
        }
    }).get('/IDOTA2Match_570/GetMatchHistory/V001/').reply(200, testdata.history_api).get('/IDOTA2Match_570/GetMatchHistory/V001/').times(2).reply(200, testdata.history_api2).get('/IEconDOTA2_570/GetHeroes/v0001/').reply(200, testdata.heroes_api);
console.log('starting web');
var app = require('../web');
console.log('starting parser');
var parser = require('../parser');
console.log('starting parseManager');
var parseManager = require('../parseManager');
//stuff we don't run in test
//scanner
//worker
var replay_dir = process.env.REPLAY_DIR;
if (!fs.existsSync(replay_dir)) {
    fs.mkdir(replay_dir);
}
before(function(done) {
    this.timeout(wait);
    var DatabaseCleaner = require('database-cleaner');
    var databaseCleaner = new DatabaseCleaner('mongodb');
    var connect = require('mongodb').connect;
    nock.enableNetConnect();
    async.series([
            function(cb) {
            console.log("wiping mongodb");
            connect(process.env.MONGO_URL, function(err, db) {
                assert(!err);
                databaseCleaner.clean(db, function(err) {
                    cb(err);
                });
            });
            },
            function(cb) {
            console.log("wiping redis");
            redis.flushall(function(err) {
                cb(err);
            });
            },
            function(cb) {
            console.log("loading matches");
            async.mapSeries(testdata.matches, function(m, cb) {
                db.matches.insert(m, function(err) {
                    cb(err);
                });
            }, function(err) {
                cb(err);
            });
            },
            function(cb) {
            console.log("loading players");
            //set visited date on first player
            testdata.players[0].last_visited = new Date();
            testdata.players[0].join_date = new Date("2012-08-31T15:59:02.161+0100");
            testdata.players[1].last_visited = new Date("2012-08-31T15:59:02.161+0100");
            async.mapSeries(testdata.players, function(p, cb) {
                db.players.insert(p, function(err) {
                    cb(err);
                });
            }, function(err) {
                cb(err);
            });
            },
            function(cb) {
            console.log('downloading replays');

            function dl(filename, cb) {
                //keep only the match id segment as the filename
                var arr = filename.split(".");
                arr[0] = arr[0].split("_")[0];
                var path = replay_dir + arr.join(".");
                //currently disabled caching of replays, get a fresh copy with each test
                if (fs.existsSync(path) && false) {
                    cb();
                }
                else {
                    request('http://cdn.rawgit.com/yasp-dota/testfiles/master/' + filename).pipe(fs.createWriteStream(path)).on('finish', function(err) {
                        if (err) {
                            console.log(err);
                            return dl(filename, cb);
                        }
                        else {
                            cb(err);
                        }
                    });
                }
            }
            var files = ['1193091757.dem', '1181392470_1v1.dem', '1189263979_ardm.dem', 'invalid.dem'];
            async.each(files, dl, function(err) {
                cb(err);
            });
        }], function(err) {
        done(err);
    });
});
describe("services", function() {
    it("mongodb connected", function(done) {
        assert(db);
{'fraction_non_alphanumeric': 0.16083018867924528, 'fraction_numerical': 0.04618867924528302, 'mean_word_length': 2.2233033325225007, 'pattern_counts': {'":': 48, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5952064', 'n_tokens_mistral': 4520, 'n_tokens_neox': 4012, 'n_words': 720}