<commit_before>var config = require('./config');
var Steam = require("steam");
var Dota2 = require("dota2");
var utility = require("./utility");
var async = require('async');
var convert64To32 = utility.convert64to32;
var express = require('express');
var app = express();
var users = config.STEAM_USER.split(",");
var passes = config.STEAM_PASS.split(",");
var steamObj = {};
var accountToIdx = {};
var replayRequests = 0;
var launch = new Date();
var launched = false;
var a = [];
var port = config.PORT || config.RETRIEVER_PORT;
//create array of numbers from 0 to n
var count = 0;
while (a.length < users.length) a.push(a.length + 0);
async.each(a, function(i, cb) {
    
    // Start listening if we have over 80% of accounts ready
    if (count/users.length > 0.8) {
        launchServer();
        cb();
    }
    
    var dotaReady = false;
    var relationshipReady = false;
    var client = new Steam.SteamClient();
    client.steamUser = new Steam.SteamUser(client);
    client.steamFriends = new Steam.SteamFriends(client);
    client.Dota2 = new Dota2.Dota2Client(client, false, false);
    var user = users[i];
    var pass = passes[i];
    var logOnDetails = {
        "account_name": user,
        "password": <PASSWORD>
    };
    client.connect();
    client.on('connected', function() {
        console.log("[STEAM] Trying to log on with %s,%s", user, pass);
        client.steamUser.logOn(logOnDetails);
        client.once('error', function onSteamError(e) {
            //reset
            console.log(e);
            console.log("reconnecting");
            client.connect();
        });
    });
    client.on("logOnResponse", function(logonResp) {
        if (logonResp.eresult !== Steam.EResult.OK) {
            //try logging on again
            return client.steamUser.logOn(logOnDetails);
        }
        console.log("[STEAM] Logged on %s", client.steamID);
        client.steamFriends.setPersonaName("[YASP] " + client.steamID);
        client.replays = 0;
        client.profiles = 0;
        client.Dota2.once("ready", function() {
            //console.log("Dota 2 ready");
            steamObj[client.steamID] = client;
            dotaReady = true;
            allDone();
        });
        client.Dota2.launch();
        client.steamFriends.on("relationships", function() {
            //console.log(Steam.EFriendRelationship);
            console.log("searching for pending friend requests...");
            //friends is a object with key steam id and value relationship
            //console.log(Steam.friends);
            for (var prop in client.steamFriends.friends) {
                //iterate through friends and accept requests/populate hash
                var steamID = prop;
                var relationship = client.steamFriends.friends[prop];
                //friends that came in while offline
                if (relationship === Steam.EFriendRelationship.RequestRecipient) {
                    client.steamFriends.addFriend(steamID);
                    console.log(steamID + " was added as a friend");
                }
                accountToIdx[convert64To32(steamID)] = client.steamID;
            }
            console.log("finished searching");
        });
        client.steamFriends.once("relationships", function() {
            //console.log("relationships obtained");
            relationshipReady = true;
            allDone();
        });
        client.steamFriends.on("friend", function(steamID, relationship) {
            //immediately accept incoming friend requests
            if (relationship === Steam.EFriendRelationship.RequestRecipient) {
                console.log("friend request received");
                client.steamFriends.addFriend(steamID);
                console.log("friend request accepted");
                accountToIdx[convert64To32(steamID)] = client.steamID;
            }
            if (relationship === Steam.EFriendRelationship.None) {
                delete accountToIdx[convert64To32(steamID)];
            }
        });
        client.once('loggedOff', function() {
            console.log("relogging");
            client.steamUser.logOn(logOnDetails);
        });
    });

    function allDone() {
        if (dotaReady && relationshipReady) {
            count += 1;
            console.log("acct %s ready, %s/%s", i, count, users.length);    
        }
        
        if (!launched) {
            cb();
        }
    }
}, function() {
    launchServer();
});

function launchServer() {
    //start listening
    if (launched) return;
    
    launched = true;
    var server = app.listen(port, function() {
        var host = server.address().address;
        console.log('[RETRIEVER] listening at http://%s:%s', host, port);
    });
    app.use(function(req, res, next) {
        if (config.RETRIEVER_SECRET && config.RETRIEVER_SECRET !== req.query.key) {
            //reject request if doesnt have key
            return next("invalid key");
        } else{
            next(null);
        }
    });
    app.get('/', function(req, res, next) {
        //console.log(process.memoryUsage());
        var keys = Object.keys(steamObj);
        var r = keys[Math.floor((Math.random() * keys.length))];
        if (req.query.mmstats) {
            getMMStats(r, function(err, data) {
                res.locals.data = data;
                return next(err);
            });
        }    
        else if (req.query.match_id) {
            getGCReplayUrl(r, req.query.match_id, function(err, data) {
                res.locals.data = data;
                return next(err);
            });
        }
        else if (req.query.account_id) {
            var idx = accountToIdx[req.query.account_id] || r;
            getPlayerProfile(idx, req.query.account_id, function(err, data) {
                res.locals.data = data;
                return next(err);
            });
        }
        else {
            res.locals.data = genStats();
            return next();
        }
    });
    app.use(function(req, res) {
        res.json(res.locals.data);
    });
    app.use(function(err, req, res, next) {
        return res.status(500).json({
            error: err
        });
    });
}

function genStats() {
    var stats = {};
    for (var key in steamObj) {
        stats[key] = {
            steamID: key,
            replays: steamObj[key].replays,
            profiles: steamObj[key].profiles,
            friends: Object.keys(steamObj[key].steamFriends.friends).length
        };
    }
    var data = {
        replayRequests: replayRequests,
        uptime: (new Date() - launch) / 1000,
        accounts: stats,
        accountToIdx: accountToIdx<commit_msg>start server then add accounts<commit_after>var config = require('./config');
var Steam = require("steam");
var Dota2 = require("dota2");
var utility = require("./utility");
var async = require('async');
var convert64To32 = utility.convert64to32;
var express = require('express');
var app = express();
var users = config.STEAM_USER.split(",");
var passes = config.STEAM_PASS.split(",");
var steamObj = {};
var accountToIdx = {};
var replayRequests = 0;
var launch = new Date();
var launched = false;
var a = [];
var port = config.PORT || config.RETRIEVER_PORT;
//create array of numbers from 0 to n
var count = 0;
while (a.length < users.length) a.push(a.length + 0);

 
app.use(function(req, res, next) {
    if (config.RETRIEVER_SECRET && config.RETRIEVER_SECRET !== req.query.key) {
        //reject request if doesnt have key
        return next("invalid key");
    } else{
        next(null);
    }
});
app.get('/', function(req, res, next) {
    //console.log(process.memoryUsage());
    var keys = Object.keys(steamObj);
    
    if (keys.length == 0) return next("No accounts ready");
    
    var r = keys[Math.floor((Math.random() * keys.length))];
    if (req.query.mmstats) {
        getMMStats(r, function(err, data) {
            res.locals.data = data;
            return next(err);
        });
    }    
    else if (req.query.match_id) {
        getGCReplayUrl(r, req.query.match_id, function(err, data) {
            res.locals.data = data;
            return next(err);
        });
    }
    else if (req.query.account_id) {
        var idx = accountToIdx[req.query.account_id] || r;
        getPlayerProfile(idx, req.query.account_id, function(err, data) {
            res.locals.data = data;
            return next(err);
        });
    }
    else {
        res.locals.data = genStats();
        return next();
    }
});
app.use(function(req, res) {
    res.json(res.locals.data);
});
app.use(function(err, req, res, next) {
    return res.status(500).json({
        error: err
    });
});

var server = app.listen(port, function() {
    var host = server.address().address;
    console.log('[RETRIEVER] listening at http://%s:%s', host, port);
});
    
async.each(a, function(i, cb) {
    
    var dotaReady = false;
    var relationshipReady = false;
    var client = new Steam.SteamClient();
    client.steamUser = new Steam.SteamUser(client);
    client.steamFriends = new Steam.SteamFriends(client);
    client.Dota2 = new Dota2.Dota2Client(client, false, false);
    var user = users[i];
    var pass = <PASSWORD>[i];
    var logOnDetails = {
        "account_name": user,
        "password": <PASSWORD>
    };
    client.connect();
    client.on('connected', function() {
        console.log("[STEAM] Trying to log on with %s,%s", user, pass);
        client.steamUser.logOn(logOnDetails);
        client.once('error', function onSteamError(e) {
            //reset
            console.log(e);
            console.log("reconnecting");
            client.connect();
        });
    });
    client.on("logOnResponse", function(logonResp) {
        if (logonResp.eresult !== Steam.EResult.OK) {
            //try logging on again
            return client.steamUser.logOn(logOnDetails);
        }
        console.log("[STEAM] Logged on %s", client.steamID);
        client.steamFriends.setPersonaName("[YASP] " + client.steamID);
        client.replays = 0;
        client.profiles = 0;
        client.Dota2.once("ready", function() {
            //console.log("Dota 2 ready");
            steamObj[client.steamID] = client;
            dotaReady = true;
            allDone();
        });
        client.Dota2.launch();
        client.steamFriends.on("relationships", function() {
            //console.log(Steam.EFriendRelationship);
            console.log("searching for pending friend requests...");
            //friends is a object with key steam id and value relationship
            //console.log(Steam.friends);
            for (var prop in client.steamFriends.friends) {
                //iterate through friends and accept requests/populate hash
                var steamID = prop;
                var relationship = client.steamFriends.friends[prop];
                //friends that came in while offline
                if (relationship === Steam.EFriendRelationship.RequestRecipient) {
                    client.steamFriends.addFriend(steamID);
                    console.log(steamID + " was added as a friend");
                }
                accountToIdx[convert64To32(steamID)] = client.steamID;
            }
            console.log("finished searching");
        });
        client.steamFriends.once("relationships", function() {
            //console.log("relationships obtained");
            relationshipReady = true;
            allDone();
        });
        client.steamFriends.on("friend", function(steamID, relationship) {
            //immediately accept incoming friend requests
            if (relationship === Steam.EFriendRelationship.RequestRecipient) {
                console.log("friend request received");
                client.steamFriends.addFriend(steamID);
                console.log("friend request accepted");
                accountToIdx[convert64To32(steamID)] = client.steamID;
            }
            if (relationship === Steam.EFriendRelationship.None) {
                delete accountToIdx[convert64To32(steamID)];
            }
        });
        client.once('loggedOff', function() {
            console.log("relogging");
            client.steamUser.logOn(logOnDetails);
        });
    });

    function allDone() {
        if (dotaReady && relationshipReady) {
            count += 1;
            console.log("acct %s ready, %s/%s", i, count, users.length);
            cb();
        }
    }
});

function genStats() {
    var stats = {};
    for (var key in steamObj) {
        stats[key] = {
            steamID: key,
            replays: steamObj[key].replays,
            profiles: steamObj[key].profiles,
            friends: Object.keys(steamObj[key].steamFriends.friends).length
        };
    }
    var data = {
        replayRequests: replayRequests,
        uptime: (new Date() - launch) / 1000,
        accounts: stats,
        accountToIdx: accountToIdx,
        ready: Object.keys(steamObj).length === users.length
{'fraction_non_alphanumeric': 0.13395398938216513, 'fraction_numerical': 0.006847734092482881, 'mean_word_length': 2.06123410268488, 'pattern_counts': {'":': 4, '<': 8, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2700786', 'n_tokens_mistral': 3899, 'n_tokens_neox': 3768, 'n_words': 819}