<commit_before>var request = require('request');
var userAgents = require('user-agents');
var cheerio = require('cheerio');
var exec = require('child_process').exec;

var getRequestOptions = function (url, desktop) {
  /**
   * We need to set a desktop user agent, if we get a random one and we set it to mobile,
   * Filmaffinity response could be different and all dom logic would be broken
   */
  const userAgent = (new userAgents({deviceCategory: desktop ? 'desktop' : 'mobile'})).data.userAgent;

  return {
    url: url,
    proxy: global.parameters.proxy,
    timeout: 5000,
    headers: {
      'User-Agent': userAgent
    }
  };
};

var getProxyRequestOptions = function (url, desktop, callback) {
  if (Math.random() >= 0.5) {
    global.log.info('Using Tor');
    callback(getRequestOptions(url, desktop));
    return;
  }

  global.log.info('Requesting rotating proxy');

  const requestOption = {
    url: 'your-own-proxy',
    timeout: 2000,
  };

  request.get(requestOption, function (error, response, body) {
    if (error || response.statusCode != 200) {
      global.log.error('Error getting proxy');
      global.log.error('Requesting proxy again');

      getProxyRequestOptions(url, desktop, callback);
    } else {
      let response = JSON.parse(body);

      let requestOptions = getRequestOptions(url);
      requestOptions.proxy = response.protocol + '://' + response.ip + ':' + response.port;

      global.log.info('Returning proxy');
      callback(requestOptions);
    }
  });
}

var handleFailedRequest = function (requestFuncName, error, response, cb) {
  global.log.error('Some error happened with request in ' + requestFuncName);
  global.log.error(error);

  if (response && response.statusCode) {
    global.log.error(response.statusCode + "\n");
  }

  global.log.info('Refreshing Tor IP...');

  try {
    var command = 'echo \'AUTHENTICATE ""\\nsignal NEWNYM\\nQUIT\' | nc localhost 9051';
    exec(command, function (error, out) {<commit_msg>Pass desktop variable when using proxies<commit_after>var request = require('request');
var userAgents = require('user-agents');
var cheerio = require('cheerio');
var exec = require('child_process').exec;

var getRequestOptions = function (url, desktop) {
  /**
   * We need to set a desktop user agent, if we get a random one and we set it to mobile,
   * Filmaffinity response could be different and all dom logic would be broken
   */
  const userAgent = (new userAgents({deviceCategory: desktop ? 'desktop' : 'mobile'})).data.userAgent;

  return {
    url: url,
    proxy: global.parameters.proxy,
    timeout: 5000,
    headers: {
      'User-Agent': userAgent
    }
  };
};

var getProxyRequestOptions = function (url, desktop, callback) {
  if (Math.random() >= 0.5) {
    global.log.info('Using Tor');
    callback(getRequestOptions(url, desktop));
    return;
  }

  global.log.info('Requesting rotating proxy');

  const requestOption = {
    url: 'your-own-proxy',
    timeout: 2000,
  };

  request.get(requestOption, function (error, response, body) {
    if (error || response.statusCode != 200) {
      global.log.error('Error getting proxy');
      global.log.error('Requesting proxy again');

      getProxyRequestOptions(url, desktop, callback);
    } else {
      let response = JSON.parse(body);

      let requestOptions = getRequestOptions(url, desktop);
      requestOptions.proxy = response.protocol + '://' + response.ip + ':' + response.port;

      global.log.info('Returning proxy');
      callback(requestOptions);
    }
  });
}

var handleFailedRequest = function (requestFuncName, error, response, cb) {
  global.log.error('Some error happened with request in ' + requestFuncName);
  global.log.error(error);

  if (response && response.statusCode) {
    global.log.error(response.statusCode + "\n");
  }

  global.log.info('Refreshing Tor IP...');

  try {
    var command = 'echo \'AUTHENTICATE ""\\nsignal NEWNYM\\nQUIT\' | nc localhost 9051';
    exec(command, function (error, out) {
{'fraction_non_alphanumeric': 0.1386113886113886, 'fraction_numerical': 0.008491508491508492, 'mean_word_length': 3.5254237288135593, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2918046', 'n_tokens_mistral': 1250, 'n_tokens_neox': 1186, 'n_words': 322}