<commit_before>/**
 * Worker interfacing with the Steam GC.
 * Provides HTTP endpoints for other workers.
 * */
const config = require('../config');
const Steam = require('steam');
const Dota2 = require('dota2');
const async = require('async');
const express = require('express');
const cp = require('child_process');

const advancedAuth = config.ENABLE_RETRIEVER_ADVANCED_AUTH ? {
  /* eslint-disable global-require */
  redis: require('../store/redis'),
  crypto: require('crypto'),
  /* eslint-enable global-require */
  pendingTwoFactorAuth: {},
  pendingSteamGuardAuth: {},
} : null;

const app = express();
const steamObj = {};
const launch = new Date();
const minUpTimeSeconds = 630;
const maxUpTimeSeconds = 3600;
const timeoutMs = 10000;
const timeoutThreshold = 50;
const accountsToUse = 3;
const port = config.PORT || config.RETRIEVER_PORT;
const matchRequestDelay = 500;
const matchRequestLimit = 250;

let matchRequestDelayIncr = 0;
let lastRequestTime;
let matchRequests = 0;
let matchSuccesses = 0;
let profileRequests = 0;
let profileSuccesses = 0;
let timeouts = 0;
let allReady = false;
let users = config.STEAM_USER.split(',');
let passes = config.STEAM_PASS.split(',');

function selfDestruct() {
  process.exit(0);
}

function getUptime() {
  return (new Date() - launch) / 1000;
}

function genStats() {
  const data = {
    matchRequests,
    uptime: getUptime(),
    numReadyAccounts: Object.keys(steamObj).length,
    totalAccounts: users.length,
  };
  return data;
}

function shaHash(buffer) {
  const h = advancedAuth.crypto.createHash('sha1');
  h.update(buffer);
  return h.digest();
}

function getSentryHashKey(user) {
  return Buffer.from(`retriever:sentry:${user}`);
}

function getMMStats(idx, cb) {
  steamObj[idx].Dota2.requestMatchmakingStats();
  steamObj[idx].Dota2.once('matchmakingStatsData', (waitTimes, searchingPlayers, disabledGroups) => {
    if (disabledGroups) {
      cb(null, disabledGroups.legacy_searching_players_by_group_source2);
    } else {
      cb('error mmstats');
    }
  });
}

function getPlayerProfile(idx, accountId, cb) {
  accountId = Number(accountId);
  const Dota2 = steamObj[idx].Dota2;
  console.log('requesting player profile %s', accountId);
  profileRequests += 1;
  Dota2.requestProfileCard(accountId, (err, profileData) => {
    /*
    enum EStatID {
    k_eStat_SoloRank = 1;
    k_eStat_PartyRank = 2;
    k_eStat_Wins = 3;
    k_eStat_Commends = 4;
    k_eStat_GamesPlayed = 5;
    k_eStat_FirstMatchDate = 6;
    }
    */
    if (err) {
      return cb(err);
    }
    const response = {};
    profileSuccesses += 1;
    profileData.slots.forEach((s) => {
      if (s.stat && s.stat.stat_id === 1) {
        response.solo_competitive_rank = s.stat.stat_score;
      }
      if (s.stat && s.stat.stat_id === 2) {
        response.competitive_rank = s.stat.stat_score;
      }
    });
    return cb(err, response);
  });
}

function getGcMatchData(idx, matchId, cb) {
  const Dota2 = steamObj[idx].Dota2;
  matchRequests += 1;
  const timeout = setTimeout(() => {
    timeouts += 1;
    matchRequestDelayIncr += 100;
  }, timeoutMs);
  return Dota2.requestMatchDetails(Number(matchId), (err, matchData) => {
    matchSuccesses += 1;
    // Reset delay on success
    matchRequestDelayIncr = 0;
    console.log('received match %s', matchId);
    clearTimeout(timeout);
    cb(err, matchData);
  });
}

function init() {
  async.each(Array.from(new Array(users.length), (v, i) => i), (i, cb) => {
    const client = new Steam.SteamClient();
    const user = users[i];
    const pass = passes[i];
    const logOnDetails = {
      account_name: user,
      password: <PASSWORD>,
    };

    client.steamUser = new Steam.SteamUser(client);
    client.steamFriends = new Steam.SteamFriends(client);
    client.logOnDetails = logOnDetails;
    client.Dota2 = new Dota2.Dota2Client(client, false);
    client.Dota2.on('ready', () => {<commit_msg>update retriever config<commit_after>/**
 * Worker interfacing with the Steam GC.
 * Provides HTTP endpoints for other workers.
 * */
const config = require('../config');
const Steam = require('steam');
const Dota2 = require('dota2');
const async = require('async');
const express = require('express');
const cp = require('child_process');

const advancedAuth = config.ENABLE_RETRIEVER_ADVANCED_AUTH ? {
  /* eslint-disable global-require */
  redis: require('../store/redis'),
  crypto: require('crypto'),
  /* eslint-enable global-require */
  pendingTwoFactorAuth: {},
  pendingSteamGuardAuth: {},
} : null;

const app = express();
const steamObj = {};
const launch = new Date();
const minUpTimeSeconds = 660;
const maxUpTimeSeconds = 3600;
const timeoutMs = 10000;
const timeoutThreshold = 50;
const accountsToUse = 3;
const port = config.PORT || config.RETRIEVER_PORT;
const matchRequestDelay = 700;
const matchRequestDelayStep = 100;
const matchRequestLimit = 250;

let matchRequestDelayIncr = 0;
let lastRequestTime;
let matchRequests = 0;
let matchSuccesses = 0;
let profileRequests = 0;
let profileSuccesses = 0;
let timeouts = 0;
let allReady = false;
let users = config.STEAM_USER.split(',');
let passes = config.STEAM_PASS.split(',');

function selfDestruct() {
  process.exit(0);
}

function getUptime() {
  return (new Date() - launch) / 1000;
}

function genStats() {
  const data = {
    matchRequests,
    uptime: getUptime(),
    numReadyAccounts: Object.keys(steamObj).length,
    totalAccounts: users.length,
  };
  return data;
}

function shaHash(buffer) {
  const h = advancedAuth.crypto.createHash('sha1');
  h.update(buffer);
  return h.digest();
}

function getSentryHashKey(user) {
  return Buffer.from(`retriever:sentry:${user}`);
}

function getMMStats(idx, cb) {
  steamObj[idx].Dota2.requestMatchmakingStats();
  steamObj[idx].Dota2.once('matchmakingStatsData', (waitTimes, searchingPlayers, disabledGroups) => {
    if (disabledGroups) {
      cb(null, disabledGroups.legacy_searching_players_by_group_source2);
    } else {
      cb('error mmstats');
    }
  });
}

function getPlayerProfile(idx, accountId, cb) {
  accountId = Number(accountId);
  const Dota2 = steamObj[idx].Dota2;
  console.log('requesting player profile %s', accountId);
  profileRequests += 1;
  Dota2.requestProfileCard(accountId, (err, profileData) => {
    /*
    enum EStatID {
    k_eStat_SoloRank = 1;
    k_eStat_PartyRank = 2;
    k_eStat_Wins = 3;
    k_eStat_Commends = 4;
    k_eStat_GamesPlayed = 5;
    k_eStat_FirstMatchDate = 6;
    }
    */
    if (err) {
      return cb(err);
    }
    const response = {};
    profileSuccesses += 1;
    profileData.slots.forEach((s) => {
      if (s.stat && s.stat.stat_id === 1) {
        response.solo_competitive_rank = s.stat.stat_score;
      }
      if (s.stat && s.stat.stat_id === 2) {
        response.competitive_rank = s.stat.stat_score;
      }
    });
    return cb(err, response);
  });
}

function getGcMatchData(idx, matchId, cb) {
  const Dota2 = steamObj[idx].Dota2;
  matchRequests += 1;
  const timeout = setTimeout(() => {
    timeouts += 1;
    matchRequestDelayIncr += matchRequestDelayStep;
  }, timeoutMs);
  return Dota2.requestMatchDetails(Number(matchId), (err, matchData) => {
    matchSuccesses += 1;
    // Reset delay on success
    matchRequestDelayIncr = 0;
    console.log('received match %s', matchId);
    clearTimeout(timeout);
    cb(err, matchData);
  });
}

function init() {
  async.each(Array.from(new Array(users.length), (v, i) => i), (i, cb) => {
    const client = new Steam.SteamClient();
    const user = users[i];
    const pass = <PASSWORD>];
    const logOnDetails = {
      account_name: user,
      password: <PASSWORD>,
    };

    client.steamUser = new Steam.SteamUser(client);
    client.steamFriends = new Steam.SteamFriends(client);
    client.logOnDetails = logOnDetails;
    client.Dota2 = new Dota2.Dota2Client(client, false);
    client.Dota2.on('ready', () => {
{'fraction_non_alphanumeric': 0.13931418448690372, 'fraction_numerical': 0.016449449576110338, 'mean_word_length': 3.799028536733455, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5853862', 'n_tokens_mistral': 2756, 'n_tokens_neox': 2657, 'n_words': 583}