<commit_before>// Copyright (c) 2015, <NAME> (twitter.com/evanxsummers)
// ISC license, see http://github.com/evanx/redexutil/LICENSE

import fs from 'fs';
import request from 'request';
import lodash from 'lodash';

import Loggers from './Loggers';

const logger = Loggers.create(module.filename, 'info');

module.exports = {
   requestJson(url) {
      return module.exports.request({url: url, json: true});
   },
   request(options) {
      logger.debug('request', options);
      if (lodash.isObject(options)) {
         if (options.lastModified) {
            options.headers = {'If-Modified-Since': options.lastModified};
         }
      }
      return new Promise((resolve, reject) => {
         request(options, (err, response, content) => {
            logger.debug('response', options.url, err || response.statusCode);
            if (err) {
               reject(err);
            } else if (response.statusCode !== 200) {
               reject({statusCode: response.statusCode});
            } else {
               resolve(content);
            }
         });
      });
   },
   response(options) {
      logger.debug('request', options);
      return new Promise((resolve, reject) => {
         request(options, (err, response, content) => {
            logger.debug('response', options.url, err || response.statusCode);
            if (err) {
               reject(err);
            } else {
               resolve([response, content]);
            }
         });
      });
   },
   head(options) {
      if (typeof options === 'string') {
         options = {url: options};
      }
      assert(options.url, 'url');
      options.method = 'HEAD';
      logger.debug('head', options);
      return new Promise((resolve, reject) => {
         request(options, (err, response) => {
            logger.debug('response', options.url, err || response.statusCode);
            if (err) {
               reject(err);
            } else {
               resolve(response);
            }
         });
      });
   }
};<commit_msg>request timeout<commit_after>// Copyright (c) 2015, <NAME> (twitter.com/evanxsummers)
// ISC license, see http://github.com/evanx/redexutil/LICENSE

import fs from 'fs';
import _request from 'request';
import lodash from 'lodash';

import Loggers from './Loggers';

const logger = Loggers.create(module.filename);

export function requestJson(options) {
   if (typeof options === 'string') {
      options = {url: options};
   }
   logger.debug('requestJson', options);
   assert(options.url, 'url');
   options.json = true;
   return request(options);
}

export function request(options) {
   logger.debug('request', options);
   if (lodash.isObject(options)) {
      if (options.lastModified) {
         options.headers = {'If-Modified-Since': options.lastModified};
      }
   }
   return new Promise((resolve, reject) => {
      _request(options, (err, response, content) => {
         logger.debug('response', options.url, err || response.statusCode);
         if (err) {
            reject(err);
         } else if (response.statusCode !== 200) {
            reject({statusCode: response.statusCode});
         } else {
            resolve(content);
         }
      });
   });
}

export function response(options) {
   logger.debug('request', options);
   return new Promise((resolve, reject) => {
      _request(options, (err, response, content) => {
         logger.debug('response', options.url, err || response.statusCode);
         if (err) {
            reject(err);
         } else {
            resolve([response, content]);
         }
      });
   });
}

export function head(options) {
   if (typeof options === 'string') {
      options = {url: options};
   }
   assert(options.url, 'url');
   options.method = 'HEAD';
   logger.debug('head', options);
   return new Promise((resolve, reject) => {
      request(options, (err, response) => {
         logger.debug('response', options.url, err || response.statusCode);
         if (err) {
            reject(err);
         } else {
            resolve(response);
         }
      });
   });
}
{'fraction_non_alphanumeric': 0.152587890625, 'fraction_numerical': 0.00341796875, 'mean_word_length': 2.2057902973395933, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2788383', 'n_tokens_mistral': 1162, 'n_tokens_neox': 1143, 'n_words': 259}