<commit_before>'use strict';

const RSVP = require('rsvp');
const request = RSVP.denodeify(require('request'));
const jsdom = require('jsdom');
const jQuery = require('jquery');

// ignore external resources
jsdom.defaultDocumentFeatures = {
  FetchExternalResources: false,
  ProcessExternalResources: false
};

function visit(opts) {
  return request(opts)
    .then(function(response) {
      let doc = jsdom.jsdom(response.body);
      let window = doc.defaultView;
      let jq = jQuery(window);

      return {
        response: response,
        jQuery: jq
      };
    });
}
<commit_msg>Only parse response body if content type is HTML<commit_after>'use strict';

const RSVP = require('rsvp');
const request = RSVP.denodeify(require('request'));
const jsdom = require('jsdom');
const jQuery = require('jquery');

// ignore external resources
jsdom.defaultDocumentFeatures = {
  FetchExternalResources: false,
  ProcessExternalResources: false
};

function visit(opts) {
  return request(opts)
    .then(function(response) {
      let result = {
        response
      };

      let contentType = response.headers['content-type'];
      if (contentType && contentType.match(/^text\/html/)) {
        let doc = jsdom.jsdom(response.body);
        let window = doc.defaultView;
        let jq = jQuery(window);
        result.jQuery = jq;
      }

      return result;
    });
}

{'fraction_non_alphanumeric': 0.12716763005780346, 'fraction_numerical': 0.0, 'mean_word_length': 3.301242236024845, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3468584', 'n_tokens_mistral': 405, 'n_tokens_neox': 395, 'n_words': 98}