<commit_before>'use strict';

var request = require('request');
var htmlparser = require('htmlparser2');

export function parse(url, cb) {
  var data = '';
  var inBody = false;

  var parser = new htmlparser.Parser({
    onopentagname: name => {
      if(name === 'body') inBody = true;
      else if(name === 'script' || name === 'style') inBody = false;
    },
    ontext: text => {
      if(inBody) data+=text;
    },
    onclosetag: name => {
      if(name === 'body') inBody = false;
      else if(name === 'script' || name === 'style') inBody = true;
    },
    onend: () => {
      cb(data);
    }
  }, {decodeEntities: true});

  request(url, (err,response,body) => {

    parser.write(body);
    parser.end();

  });
}<commit_msg>Handling errors<commit_after>'use strict';

var request = require('request');
var htmlparser = require('htmlparser2');

export function parse(url, cb) {
  var data = '';
  var inBody = false;

  var parser = new htmlparser.Parser({
    onopentagname: name => {
      if(name === 'body') inBody = true;
      else if(name === 'script' || name === 'style') inBody = false;
    },
    ontext: text => {
      if(inBody) data+=text;
    },
    onclosetag: name => {
      if(name === 'body') inBody = false;
      else if(name === 'script' || name === 'style') inBody = true;
    },
    onend: () => {
      cb(null, JSON.stringify(data));
    }
  }, {decodeEntities: true});

  request(url, (err,response,body) => {
    if(err) {
      cb(err)
      throw err;
    }

    parser.write(body);
    parser.end();

  });
}
{'fraction_non_alphanumeric': 0.1909967845659164, 'fraction_numerical': 0.0012861736334405145, 'mean_word_length': 2.758454106280193, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3197288', 'n_tokens_mistral': 530, 'n_tokens_neox': 522, 'n_words': 124}