<commit_before>'use strict';

const superagent = require('superagent');

module.exports = function(options) {
  var agent = superagent.agent(options || {});

  // console.log('using engine: http');
  return function(context) {
    return agent
      .get(context.url)
      .set(context.headers)
      .then((res) => {
        context.status = res.status;
        context.set(res.headers);
        context.body = 'application/json' == context.type ? res.body : res.text;

        // update the URL if there were redirects
        context.url = res.redirects.length ? res.redirects.pop() : context.url;

        return context;
      })
      .catch((e) => {
        // with new superagent i dont think we need this anymore since error will be automatically http error
        // if (e && !e.status) {
        //   // might not be here anyway!
        //   console.error(e.stack);
        //   context.status = 500;
        //   return context;
        // } else {
          context.status = e.status;
          return context;
        // }
      });
  };
};<commit_msg>FIXED non-http error will throw error<commit_after>'use strict';

const superagent = require('superagent');

module.exports = function(options) {
  var agent = superagent.agent(options || {});

  // console.log('using engine: http');
  return function(context) {
    return agent
      .get(context.url)
      .set(context.headers)
      .then((res) => {
        context.status = res.status;
        context.set(res.headers);
        context.body = 'application/json' == context.type ? res.body : res.text;

        // update the URL if there were redirects
        context.url = res.redirects.length ? res.redirects.pop() : context.url;

        return context;
      })
      .catch((e) => {
        if (e && !e.status) {
          throw e;
          // console.error(e.stack);
          // context.status = 500;
          // return context;
        } else {
          context.status = e.status;
          return context;
        }
      });
  };
};
{'fraction_non_alphanumeric': 0.14207920792079207, 'fraction_numerical': 0.0029702970297029703, 'mean_word_length': 1.9503649635036497, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5955010', 'n_tokens_mistral': 592, 'n_tokens_neox': 579, 'n_words': 139}