<commit_before>let request = require('request-promise')
const log = require('./logger')

let singleton = null

module.exports = function (options = {}) {
  if (singleton) return singleton

  const defaultOptions = {
    debug: false,
    json: true,
    cheerio: false,
    headers: {
      // a lot of web service do not want to be called by robots and then check the user agent to
      // be sure they are called by a browser. This user agent works most of the time.
      'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:36.0) ' +
                    'Gecko/20100101 Firefox/36.0'
    },
    followAllRedirects: true
  }

  options = Object.assign(defaultOptions, options)

  const requestOptions = {}
  if (options.debug) require('request-debug')(request)

  requestOptions.json = options.json
  requestOptions.jar = options.jar
  requestOptions.headers = options.headers
  requestOptions.followAllRedirects = options.followAllRedirects

  if (options.cheerio) requestOptions.transform = (body) => require('cheerio').load(body)<commit_msg>Added strictSSL option<commit_after>let request = require('request-promise')
const log = require('./logger')

let singleton = null

module.exports = function (options = {}) {
  if (singleton) return singleton

  const defaultOptions = {
    debug: false,
    json: true,
    cheerio: false,
    strictSSL: false,
    headers: {
      // a lot of web service do not want to be called by robots and then check the user agent to
      // be sure they are called by a browser. This user agent works most of the time.
      'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:36.0) ' +
                    'Gecko/20100101 Firefox/36.0'
    },
    followAllRedirects: true
  }

  options = Object.assign(defaultOptions, options)

  if (options.debug) require('request-debug')(request)

  const requestOptions = {}

  requestOptions.json = options.json
  requestOptions.jar = options.jar
  requestOptions.headers = options.headers
  requestOptions.followAllRedirects = options.followAllRedirects
  requestOptions.strictSSL = options.strictSSL

  if (options.cheerio) requestOptions.transform = (body) => require('cheerio').load(body)
{'fraction_non_alphanumeric': 0.10663615560640732, 'fraction_numerical': 0.020137299771167048, 'mean_word_length': 3.416161616161616, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2525714', 'n_tokens_mistral': 680, 'n_tokens_neox': 633, 'n_words': 191}