<commit_before>/**
 * Main test script to run tests
 **/
var config = require('../config');
var constants = require('dotaconstants');
var redis = require('../store/redis');
var queue = require('../store/queue');
var queries = require('../store/queries');
config.PORT = ""; //use service defaults
config.POSTGRES_URL = "postgres://postgres:postgres@localhost/yasp_test";
config.REDIS_URL = "redis://localhost:6379/1";
config.SESSION_SECRET = "testsecretvalue";
config.NODE_ENV = "test";
config.ENABLE_MATCH_CACHE = 1;
config.FRONTEND_PORT = 5001;
config.PARSER_PORT = 5201;
config.ENABLE_CASSANDRA_MATCH_STORE_WRITE = "";
config.ENABLE_CASSANDRA_MATCH_STORE_READ = "";
var async = require('async');
var nock = require('nock');
var moment = require('moment');
var assert = require('assert');
var init_db = "postgres://postgres:postgres@localhost/postgres";
var pQueue = queue.getQueue('parse');
var supertest = require('supertest');
var replay_dir = "./test/testfiles/";
var pg = require('pg');
var fs = require('fs');
var wait = 90000;
//var cassandra = require('../store/cassandra');
var buildMatch = require('../store/buildMatch');
// these are loaded later, as the database needs to be created when these are required
var db;
var app;
var details_api = require('./details_api.json');
//nock.disableNetConnect();
//nock.enableNetConnect();
//fake api response
nock('http://api.steampowered.com')
    //500 error
    .get('/IDOTA2Match_570/GetMatchDetails/V001/').query(true).reply(500,
    {})
    //fake match details
    .get('/IDOTA2Match_570/GetMatchDetails/V001/').query(true).times(10).reply(200, details_api)
    //fake player summaries
    .get('/ISteamUser/GetPlayerSummaries/v0002/').query(true).reply(200, require('./summaries_api.json'))
    //non-retryable error
    .get('/IDOTA2Match_570/GetMatchHistory/V001/').query(true).reply(200,
    {
        result:
        {
            error: "error"
        }
    })
    //fake full history
    .get('/IDOTA2Match_570/GetMatchHistory/V001/').query(true).reply(200, require('./history_api.json'));
//fake heroes list
//.get('/IEconDOTA2_570/GetHeroes/v0001/').query(true).reply(200, require('./heroes_api.json')
//fake leagues
//.get('/IDOTA2Match_570/GetLeagueListing/v0001/').query(true).reply(200, require('./leagues_api.json'));
//fake mmr response
nock("http://" + config.RETRIEVER_HOST).get('/?account_id=88367253').reply(200, require('./retriever_player.json'));
before(function(done)
{
    this.timeout(wait);
    async.series([
        function(cb)
        {
            console.log('removing old test database');
            pg.connect(init_db, function(err, client)
            {
                if (err)
                {
                    return cb(err);
                }
                console.log('cleaning test database', config.POSTGRES_URL);
                client.query('DROP DATABASE IF EXISTS yasp_test', function(err, result)
                {
                    cb(err);
                });
            });
        },
        function(cb)
        {
            console.log('creating test database');
            pg.connect(init_db, function(err, client)
            {
                if (err)
                {
                    return cb(err);
                }
                console.log('creation of test database', config.POSTGRES_URL);
                client.query('CREATE DATABASE yasp_test', function(err, result)
                {
                    cb(err);
                });
            });
        },
        function(cb)
        {
            console.log('connecting to test database and creating tables');
            pg.connect(config.POSTGRES_URL, function(err, client)
            {
                if (err)
                {
                    return cb(err);
                }
                // create tables
                var query = fs.readFileSync("./sql/create_tables.sql", "utf8");
                client.query(query, function(err, result)
                {
                    console.log('set up %s', config.POSTGRES_URL);
                    cb(err);
                });
            });
        },
        function(cb)
        {
            console.log("wiping redis");
            redis.flushdb(cb);
        },
        function(cb)
        {
            db = require('../store/db');
            app = require('../svc/web');
            require('../svc/parser');
            console.log("loading matches");
            async.mapSeries([details_api.result], function(m, cb)
            {
                queries.insertMatch(db, redis, m,
                {
                    type: "api",
                    skipParse: true,
                }, cb);
            }, cb);
        },
        function(cb)
        {
            console.log("loading players");
            async.mapSeries(require('./summaries_api').response.players, function(p, cb)
            {
                queries.insertPlayer(db, p, cb);
            }, cb);
        }], done);
});
describe("parser", function()
{
    this.timeout(wait);
    var tests = {
        '1781962623_source2.dem': details_api.result
    };
    for (var key in tests)
    {
        it('parse replay', function(done)
        {
            nock("http://" + config.RETRIEVER_HOST).get('/').query(true).reply(200,
            {
                match:
                {
                    cluster: 1,
                    replay_salt: key.split(".")[0].split("_")[1]
                }
            });
            //fake replay download
            nock("http://replay1.valve.net").get('/570/' + key).replyWithFile(200, replay_dir + key);
            var match = {
                match_id: tests[key].match_id,
                start_time: tests[key].start_time,
                duration: tests[key].duration,
                radiant_win: tests[key].radiant_win,
            };
            queue.addToQueue(pQueue, match,
            {}, function(err, job)
            {
                assert(job && !err);
                var poll = setInterval(function()
                {
                    pQueue.getJob(job.jobId).then(function(job)
                    {
                        job.getState().then(function(state)<commit_msg>add more props to mock<commit_after>/**
 * Main test script to run tests
 **/
var config = require('../config');
var constants = require('dotaconstants');
var redis = require('../store/redis');
var queue = require('../store/queue');
var queries = require('../store/queries');
config.PORT = ""; //use service defaults
config.POSTGRES_URL = "postgres://postgres:postgres@localhost/yasp_test";
config.REDIS_URL = "redis://localhost:6379/1";
config.SESSION_SECRET = "testsecretvalue";
config.NODE_ENV = "test";
config.ENABLE_MATCH_CACHE = 1;
config.FRONTEND_PORT = 5001;
config.PARSER_PORT = 5201;
config.ENABLE_CASSANDRA_MATCH_STORE_WRITE = "";
config.ENABLE_CASSANDRA_MATCH_STORE_READ = "";
var async = require('async');
var nock = require('nock');
var moment = require('moment');
var assert = require('assert');
var init_db = "postgres://postgres:postgres@localhost/postgres";
var pQueue = queue.getQueue('parse');
var supertest = require('supertest');
var replay_dir = "./test/testfiles/";
var pg = require('pg');
var fs = require('fs');
var wait = 90000;
//var cassandra = require('../store/cassandra');
var buildMatch = require('../store/buildMatch');
// these are loaded later, as the database needs to be created when these are required
var db;
var app;
var details_api = require('./details_api.json');
//nock.disableNetConnect();
//nock.enableNetConnect();
//fake api response
nock('http://api.steampowered.com')
    //500 error
    .get('/IDOTA2Match_570/GetMatchDetails/V001/').query(true).reply(500,
    {})
    //fake match details
    .get('/IDOTA2Match_570/GetMatchDetails/V001/').query(true).times(10).reply(200, details_api)
    //fake player summaries
    .get('/ISteamUser/GetPlayerSummaries/v0002/').query(true).reply(200, require('./summaries_api.json'))
    //non-retryable error
    .get('/IDOTA2Match_570/GetMatchHistory/V001/').query(true).reply(200,
    {
        result:
        {
            error: "error"
        }
    })
    //fake full history
    .get('/IDOTA2Match_570/GetMatchHistory/V001/').query(true).reply(200, require('./history_api.json'));
//fake heroes list
//.get('/IEconDOTA2_570/GetHeroes/v0001/').query(true).reply(200, require('./heroes_api.json')
//fake leagues
//.get('/IDOTA2Match_570/GetLeagueListing/v0001/').query(true).reply(200, require('./leagues_api.json'));
//fake mmr response
nock("http://" + config.RETRIEVER_HOST).get('/?account_id=88367253').reply(200, require('./retriever_player.json'));
before(function(done)
{
    this.timeout(wait);
    async.series([
        function(cb)
        {
            console.log('removing old test database');
            pg.connect(init_db, function(err, client)
            {
                if (err)
                {
                    return cb(err);
                }
                console.log('cleaning test database', config.POSTGRES_URL);
                client.query('DROP DATABASE IF EXISTS yasp_test', function(err, result)
                {
                    cb(err);
                });
            });
        },
        function(cb)
        {
            console.log('creating test database');
            pg.connect(init_db, function(err, client)
            {
                if (err)
                {
                    return cb(err);
                }
                console.log('creation of test database', config.POSTGRES_URL);
                client.query('CREATE DATABASE yasp_test', function(err, result)
                {
                    cb(err);
                });
            });
        },
        function(cb)
        {
            console.log('connecting to test database and creating tables');
            pg.connect(config.POSTGRES_URL, function(err, client)
            {
                if (err)
                {
                    return cb(err);
                }
                // create tables
                var query = fs.readFileSync("./sql/create_tables.sql", "utf8");
                client.query(query, function(err, result)
                {
                    console.log('set up %s', config.POSTGRES_URL);
                    cb(err);
                });
            });
        },
        function(cb)
        {
            console.log("wiping redis");
            redis.flushdb(cb);
        },
        function(cb)
        {
            db = require('../store/db');
            app = require('../svc/web');
            require('../svc/parser');
            console.log("loading matches");
            async.mapSeries([details_api.result], function(m, cb)
            {
                queries.insertMatch(db, redis, m,
                {
                    type: "api",
                    skipParse: true,
                }, cb);
            }, cb);
        },
        function(cb)
        {
            console.log("loading players");
            async.mapSeries(require('./summaries_api').response.players, function(p, cb)
            {
                queries.insertPlayer(db, p, cb);
            }, cb);
        }], done);
});
describe("parser", function()
{
    this.timeout(wait);
    var tests = {
        '1781962623_source2.dem': details_api.result
    };
    for (var key in tests)
    {
        it('parse replay', function(done)
        {
            nock("http://" + config.RETRIEVER_HOST).get('/').query(true).reply(200,
            {
                match:
                {
                    match_id: 1781962623,
                    cluster: 1,
                    replay_salt: key.split(".")[0].split("_")[1],
                    series_id: 0,
                    series_type: 0,
                    players: [],
                }
            });
            //fake replay download
            nock("http://replay1.valve.net").get('/570/' + key).replyWithFile(200, replay_dir + key);
            var match = {
                match_id: tests[key].match_id,
                start_time: tests[key].start_time,
                duration: tests[key].duration,
                radiant_win: tests[key].radiant_win,
            };
            queue.addToQueue(pQueue, match,
            {}, function(err, job)
            {
                assert(job && !err);
                var poll = setInterval(function()
                {
                    pQueue.getJob(job.jobId).then(function(job)
                    {
                        job.getState().then(function(state)
{'fraction_non_alphanumeric': 0.14663902400127582, 'fraction_numerical': 0.021529383621720757, 'mean_word_length': 1.7595159515951595, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4459278', 'n_tokens_mistral': 3874, 'n_tokens_neox': 3568, 'n_words': 632}